diff --git a/.github/actions/pg-query-ext/action.yml b/.github/actions/pg-query-ext/action.yml new file mode 100644 index 0000000000..fd757eb010 --- /dev/null +++ b/.github/actions/pg-query-ext/action.yml @@ -0,0 +1,41 @@ +name: "pg-query-ext from source" +description: "Builds src/extension/pg-query-ext from the checked-out source, cached by its source hash, and enables it" + +runs: + using: "composite" + steps: + - name: "Resolve PHP version" + id: php + shell: bash + run: echo "version=$(php -r 'echo PHP_VERSION;')" >> "$GITHUB_OUTPUT" + + - name: "Cache pg_query.so" + id: cache + uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6 + with: + path: src/extension/pg-query-ext/ext/modules/pg_query.so + key: pg-query-ext-${{ runner.os }}-${{ runner.arch }}-php-${{ steps.php.outputs.version }}-${{ hashFiles('src/extension/pg-query-ext/**') }} + + - name: "Install build dependencies" + if: ${{ steps.cache.outputs.cache-hit != 'true' }} + shell: bash + run: sudo apt-get update && sudo apt-get install -y build-essential autoconf automake libtool protobuf-compiler libprotobuf-c-dev + + - name: "Build pg-query-ext" + if: ${{ steps.cache.outputs.cache-hit != 'true' }} + shell: bash + working-directory: src/extension/pg-query-ext + # a shallow checkout has no tags for config.m4 to describe; this build only runs the test suite + run: | + if ! git describe --tags --match '[0-9]*' >/dev/null 2>&1; then + export PG_QUERY_EXT_VERSION="0.0.0-0-g$(git rev-parse --short HEAD)" + fi + make build + + - name: "Enable pg-query-ext" + shell: bash + working-directory: src/extension/pg-query-ext + run: | + sudo cp ext/modules/pg_query.so "$(php -r 'echo ini_get("extension_dir");')/" + echo "extension=pg_query.so" | sudo tee -a "$(php -r 'echo php_ini_loaded_file();')" + php -m | grep -x pg_query diff --git a/.github/workflows/baseline.yml b/.github/workflows/baseline.yml index c165593a27..7613bd28c4 100644 --- a/.github/workflows/baseline.yml +++ b/.github/workflows/baseline.yml @@ -46,7 +46,9 @@ jobs: extensions: ':psr, bcmath, dom, hash, json, mbstring, xml, xmlwriter, xmlreader, zlib' cache-key-suffix: "-locked-phar" apt-packages: "build-essential autoconf automake libtool protobuf-compiler libprotobuf-c-dev" - pie-extensions: "flow-php/pg-query-ext:1.x-dev" + + - name: "Build pg-query-ext from source" + uses: "./.github/actions/pg-query-ext" - name: "Build PHAR file" run: "just phar" @@ -116,11 +118,13 @@ jobs: extensions: ':psr, bcmath, dom, hash, json, mbstring, xml, xmlwriter, xmlreader, zlib' cache-key-suffix: "-website-phar" apt-packages: "build-essential autoconf automake libtool protobuf-compiler libprotobuf-c-dev" - pie-extensions: "flow-php/pg-query-ext:1.x-dev" extra-working-directory: "web/landing" extra-cache-key-suffix: "-website" extra-composer-file: "web/landing/composer.lock" + - name: "Build pg-query-ext from source" + uses: "./.github/actions/pg-query-ext" + - name: "Generate documentation" run: "just docs" diff --git a/.github/workflows/job-arrow-extension.yml b/.github/workflows/job-arrow-extension.yml index d5296a92ba..f73ad4045f 100644 --- a/.github/workflows/job-arrow-extension.yml +++ b/.github/workflows/job-arrow-extension.yml @@ -34,7 +34,7 @@ jobs: install-dependencies: 'false' - name: Install Rust toolchain - uses: dtolnay/rust-toolchain@6bed0761d98439e5a578e2877258200ad565ba87 # stable + uses: dtolnay/rust-toolchain@89b12181fb390509a0842a86cc55eeb8eb928c1d # stable with: toolchain: stable @@ -86,6 +86,12 @@ jobs: working-directory: src/extension/arrow-ext run: make build + - name: Install Composer Dependencies + uses: ./.github/actions/composer-install + with: + php-version: ${{ matrix.php }} + dependencies: locked + - name: Run PHPT tests working-directory: src/extension/arrow-ext run: make test @@ -98,12 +104,6 @@ jobs: echo "extension=arrow.so" | sudo tee -a "$(php -r 'echo php_ini_loaded_file();')" php -m | grep arrow - - name: Install Composer Dependencies - uses: ./.github/actions/composer-install - with: - php-version: ${{ matrix.php }} - dependencies: locked - - name: Run Parquet Integration Tests with Arrow run: just test --testsuite=lib-parquet-integration ${{ (matrix.php == '8.3' && runner.os == 'Linux') && '--coverage-clover=./var/phpunit/coverage/clover/coverage.xml' || '' }} @@ -141,7 +141,7 @@ jobs: tools: 'composer:v2, phpize, php-config' - name: Install Rust toolchain - uses: dtolnay/rust-toolchain@6bed0761d98439e5a578e2877258200ad565ba87 # stable + uses: dtolnay/rust-toolchain@89b12181fb390509a0842a86cc55eeb8eb928c1d # stable with: toolchain: stable @@ -202,7 +202,13 @@ jobs: - name: Verify extension version run: | - expected="$(git describe --tags --match '[0-9]*' | sed -E 's/-([0-9]+)-(g[0-9a-f]+)$/+\1.\2/')" + describe="$(git describe --tags --match '[0-9]*')" + if git describe --exact-match --tags --match '[0-9]*' HEAD >/dev/null 2>&1; then + expected="$describe" + else + # commits past a tag are a pre-release of the next minor: 0.44.1-65-g4525c4adc -> 0.45.0-dev+65.g4525c4adc + expected="$(echo "$describe" | awk -F- '{split($1, v, "."); printf "%d.%d.0-dev+%s.%s", v[1], v[2] + 1, $2, $3}')" + fi actual="$(php -r 'echo phpversion("arrow");')" echo "expected: $expected, actual: $actual" test "$actual" = "$expected" diff --git a/.github/workflows/job-flow-php-extension.yml b/.github/workflows/job-flow-php-extension.yml index 21023c8d85..c47cd6204e 100644 --- a/.github/workflows/job-flow-php-extension.yml +++ b/.github/workflows/job-flow-php-extension.yml @@ -34,7 +34,7 @@ jobs: install-dependencies: 'false' - name: Install Rust toolchain - uses: dtolnay/rust-toolchain@6bed0761d98439e5a578e2877258200ad565ba87 # stable + uses: dtolnay/rust-toolchain@89b12181fb390509a0842a86cc55eeb8eb928c1d # stable with: toolchain: stable @@ -45,7 +45,8 @@ jobs: ~/.cargo/registry ~/.cargo/git src/extension/flow-php-ext/target - key: ${{ runner.os }}-php${{ matrix.php }}-cargo-${{ hashFiles('src/extension/flow-php-ext/Cargo.lock') }} + src/extension/arrow-ext/target + key: ${{ runner.os }}-php${{ matrix.php }}-cargo-${{ hashFiles('src/extension/flow-php-ext/Cargo.lock', 'src/extension/arrow-ext/Cargo.lock') }} restore-keys: | ${{ runner.os }}-php${{ matrix.php }}-cargo- @@ -76,10 +77,29 @@ jobs: echo "LIBCLANG_PATH=$(dirname "$LIBCLANG_SO")" >> "$GITHUB_ENV" echo "Detected libclang at: $LIBCLANG_SO" + - name: Verify the nix development version literals + run: | + if git describe --exact-match --tags --match '[0-9]*' HEAD >/dev/null 2>&1; then + exit 0 + fi + tag="$(git describe --tags --abbrev=0 --match '[0-9]*')" + expected="$(echo "$tag" | awk -F. '{printf "%d.%d.0-dev", $1, $2 + 1}')" + for f in .nix/pkgs/php-flow-php-ext/package.nix .nix/pkgs/php-arrow-ext/package.nix; do + grep -q "version ? \"$expected\"" "$f" || { echo "$f: version literal must be $expected"; exit 1; } + done + + - name: Build arrow extension + working-directory: src/extension/arrow-ext + run: make build + - name: Build extension working-directory: src/extension/flow-php-ext run: make build + - name: Test flow-batch-frame crate + working-directory: src/extension/flow-php-ext/crates/flow-batch-frame + run: cargo test --locked + - name: Install Composer Dependencies uses: ./.github/actions/composer-install with: @@ -88,7 +108,7 @@ jobs: - name: Run PHPT tests working-directory: src/extension/flow-php-ext - run: make test + run: make test ARROW_SO=../arrow-ext/ext/modules/arrow.so - name: Install extension and verify working-directory: src/extension/flow-php-ext @@ -104,6 +124,12 @@ jobs: - name: Run ETL Integration Tests with Flow PHP Extension run: just test --testsuite=etl-integration + - name: Run CSV Adapter Unit Tests with Flow PHP Extension + run: just test --testsuite=adapter-csv-unit + + - name: Run CSV Adapter Integration Tests with Flow PHP Extension + run: just test --testsuite=adapter-csv-integration + - name: Upload to Codecov if: ${{ !cancelled() && matrix.php == '8.3' && runner.os == 'Linux' }} uses: codecov/codecov-action@303a32d7a59b442fa8d48b6a1cc6825c09c847a5 # v7.1.1 @@ -138,7 +164,7 @@ jobs: tools: 'composer:v2, phpize, php-config' - name: Install Rust toolchain - uses: dtolnay/rust-toolchain@6bed0761d98439e5a578e2877258200ad565ba87 # stable + uses: dtolnay/rust-toolchain@89b12181fb390509a0842a86cc55eeb8eb928c1d # stable with: toolchain: stable @@ -203,7 +229,13 @@ jobs: - name: Verify extension version run: | - expected="$(git describe --tags --match '[0-9]*' | sed -E 's/-([0-9]+)-(g[0-9a-f]+)$/+\1.\2/')" + describe="$(git describe --tags --match '[0-9]*')" + if git describe --exact-match --tags --match '[0-9]*' HEAD >/dev/null 2>&1; then + expected="$describe" + else + # commits past a tag are a pre-release of the next minor: 0.44.1-65-g4525c4adc -> 0.45.0-dev+65.g4525c4adc + expected="$(echo "$describe" | awk -F- '{split($1, v, "."); printf "%d.%d.0-dev+%s.%s", v[1], v[2] + 1, $2, $3}')" + fi actual="$(php -r 'echo phpversion("flow_php");')" echo "expected: $expected, actual: $actual" test "$actual" = "$expected" diff --git a/.github/workflows/job-phpunit-telemetry-tests.yml b/.github/workflows/job-phpunit-telemetry-tests.yml index 7276369bf1..a98e3da236 100644 --- a/.github/workflows/job-phpunit-telemetry-tests.yml +++ b/.github/workflows/job-phpunit-telemetry-tests.yml @@ -54,7 +54,9 @@ jobs: extensions: ':psr, bcmath, dom, hash, json, mbstring, xml, xmlwriter, xmlreader, zlib, curl, pgsql, grpc, protobuf' ini-values: 'memory_limit=-1' apt-packages: "build-essential autoconf automake libtool protobuf-compiler libprotobuf-c-dev" - pie-extensions: "flow-php/pg-query-ext:1.x-dev" + + - name: "Build pg-query-ext from source" + uses: "./.github/actions/pg-query-ext" - name: "Install PHPUnit ${{ matrix.phpunit-version }}" if: ${{ matrix.phpunit-version != '11' }} diff --git a/.github/workflows/job-tests.yml b/.github/workflows/job-tests.yml index 418f578b17..646c98a24a 100644 --- a/.github/workflows/job-tests.yml +++ b/.github/workflows/job-tests.yml @@ -113,7 +113,9 @@ jobs: extensions: ':psr, apcu, bcmath, dom, hash, json, mbstring, xml, xmlwriter, xmlreader, zlib, curl, pgsql, grpc, protobuf' ini-values: 'memory_limit=-1, post_max_size=32M, upload_max_filesize=32M, apc.enable_cli=1' apt-packages: "build-essential autoconf automake libtool protobuf-compiler libprotobuf-c-dev" - pie-extensions: "flow-php/pg-query-ext:1.x-dev" + + - name: "Build pg-query-ext from source" + uses: "./.github/actions/pg-query-ext" - name: Set up Node.js uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0 diff --git a/.github/workflows/pr-check.yml b/.github/workflows/pr-check.yml index 9042481b39..4f38ee48ac 100644 --- a/.github/workflows/pr-check.yml +++ b/.github/workflows/pr-check.yml @@ -18,7 +18,7 @@ jobs: permissions: pull-requests: write steps: - - uses: codelytv/pr-size-labeler@095a41fca88b8764fd9e008ad269bcdb82bb38b9 # v1 + - uses: codelytv/pr-size-labeler@095a41fca88b8764fd9e008ad269bcdb82bb38b9 # v1.10.4 with: GITHUB_TOKEN: "${{ secrets.GITHUB_TOKEN }}" xs_label: 'size: XS' diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index a7057e7595..f6e5b1ce95 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -35,7 +35,9 @@ jobs: extensions: ':psr, bcmath, dom, hash, json, mbstring, xml, xmlwriter, xmlreader, zlib' cache-key-suffix: "-locked-phar" apt-packages: "build-essential autoconf automake libtool protobuf-compiler libprotobuf-c-dev" - pie-extensions: "flow-php/pg-query-ext:1.x-dev" + + - name: "Build pg-query-ext from source" + uses: "./.github/actions/pg-query-ext" - name: "Build PHAR file" run: "just phar" diff --git a/.github/workflows/test-suite.yml b/.github/workflows/test-suite.yml index 6264d3f797..56167f8d6a 100644 --- a/.github/workflows/test-suite.yml +++ b/.github/workflows/test-suite.yml @@ -11,6 +11,7 @@ on: - 'composer.lock' - 'composer.json' - 'phpunit.xml.dist' + - '.nix/pkgs/**' push: branches: [ 1.x ] paths: @@ -22,6 +23,7 @@ on: - 'composer.lock' - 'composer.json' - 'phpunit.xml.dist' + - '.nix/pkgs/**' schedule: - cron: '0 4 * * *' workflow_dispatch: diff --git a/.github/workflows/test-website.yml b/.github/workflows/test-website.yml index ab4055a26f..10611881e9 100644 --- a/.github/workflows/test-website.yml +++ b/.github/workflows/test-website.yml @@ -38,7 +38,9 @@ jobs: extensions: ':psr, bcmath, dom, hash, json, mbstring, xml, xmlwriter, xmlreader, zlib' cache-key-suffix: "-locked-phar" apt-packages: "build-essential autoconf automake libtool protobuf-compiler libprotobuf-c-dev" - pie-extensions: "flow-php/pg-query-ext:1.x-dev" + + - name: "Build pg-query-ext from source" + uses: "./.github/actions/pg-query-ext" - name: "Build PHAR file" run: "just phar" diff --git a/.nix/blackfire/.gitignore b/.nix/blackfire/.gitignore new file mode 100644 index 0000000000..21b52734b0 --- /dev/null +++ b/.nix/blackfire/.gitignore @@ -0,0 +1 @@ +blackfire.env diff --git a/.nix/blackfire/blackfire.env.dist b/.nix/blackfire/blackfire.env.dist new file mode 100644 index 0000000000..19d9e5ec4d --- /dev/null +++ b/.nix/blackfire/blackfire.env.dist @@ -0,0 +1,2 @@ +# Copy to blackfire.env (gitignored) and fill in; nix-shell exports it for `blackfire run`. +BLACKFIRE_ENV= diff --git a/.nix/pkgs/php-arrow-ext/package.nix b/.nix/pkgs/php-arrow-ext/package.nix index 1a36e7224c..32833f97a1 100644 --- a/.nix/pkgs/php-arrow-ext/package.nix +++ b/.nix/pkgs/php-arrow-ext/package.nix @@ -5,7 +5,7 @@ rustPlatform, clang, llvmPackages, - arrow-ext-version ? "dev", + arrow-ext-version ? "0.45.0-dev", }: let @@ -58,7 +58,7 @@ let doCheck = false; meta = with lib; { - description = "Apache Arrow PHP extension powered by Rust"; + description = "Apache Arrow and Parquet bindings for flow-php/parquet (Rust)"; license = licenses.mit; }; }; diff --git a/.nix/pkgs/php-flow-php-ext/package.nix b/.nix/pkgs/php-flow-php-ext/package.nix index 50e21c59d3..127977c554 100644 --- a/.nix/pkgs/php-flow-php-ext/package.nix +++ b/.nix/pkgs/php-flow-php-ext/package.nix @@ -5,7 +5,7 @@ rustPlatform, clang, llvmPackages, - flow-php-ext-version ? "dev", + flow-php-ext-version ? "0.45.0-dev", }: let @@ -58,7 +58,7 @@ let doCheck = false; meta = with lib; { - description = "Flow PHP native extension (Rust) - Floe frame-body encoder/decoder for DataFrame Rows"; + description = "Flow PHP native extension (Rust) - native column backend, CSV/JSON readers, Parquet ETL through arrow-ext"; license = licenses.mit; }; }; diff --git a/benchmarks/src/Datasets/SellersDataset.php b/benchmarks/src/Datasets/SellersDataset.php index dbbae23813..02e46047a2 100644 --- a/benchmarks/src/Datasets/SellersDataset.php +++ b/benchmarks/src/Datasets/SellersDataset.php @@ -31,9 +31,8 @@ public function parquet(): string $sellerIds = []; foreach (data_frame()->read(from_parquet($ordersPath)->withColumns(['seller_id']))->get() as $batch) { - foreach ($batch->all() as $row) { - $sellerId = $row->get('seller_id'); - + // @mago-ignore analysis:mixed-assignment + foreach ($batch->column('seller_id')->values() as $sellerId) { if (is_scalar($sellerId) || $sellerId instanceof Stringable) { $sellerIds[(string) $sellerId] = true; } diff --git a/benchmarks/src/Datasets/TextDataset.php b/benchmarks/src/Datasets/TextDataset.php index bdaa454e0c..a6ad0fbb97 100644 --- a/benchmarks/src/Datasets/TextDataset.php +++ b/benchmarks/src/Datasets/TextDataset.php @@ -31,9 +31,12 @@ public function path(): string $lines = []; foreach (data_frame()->read(from_parquet($parquetPath)->withColumns(['customer', 'notes']))->get() as $batch) { - foreach ($batch->all() as $row) { - $customer = $row->get('customer'); - $notes = $row->get('notes'); + $allNotes = $batch->column('notes')->values(); + + // @mago-ignore analysis:mixed-assignment + foreach ($batch->column('customer')->values() as $i => $customer) { + // @mago-ignore analysis:mixed-assignment + $notes = $allNotes[$i]; $customerText = is_scalar($customer) || $customer instanceof Stringable ? (string) $customer : ''; $notesText = is_array($notes) diff --git a/benchmarks/src/Format/Floe/FloeReadScenario.php b/benchmarks/src/Format/Floe/FloeReadScenario.php index ed4e408c6f..557bc4d9a3 100644 --- a/benchmarks/src/Format/Floe/FloeReadScenario.php +++ b/benchmarks/src/Format/Floe/FloeReadScenario.php @@ -5,10 +5,7 @@ namespace Flow\Benchmarks\Format\Floe; use Flow\Benchmarks\Datasets\Datasets; -use Flow\ETL\Row\PhpRowHydrator; -use Flow\Floe\FloeEngine; -use function Flow\ETL\DSL\config_builder; use function Flow\ETL\DSL\data_frame; use function Flow\Floe\DSL\from_floe; @@ -16,17 +13,10 @@ { public function __construct( private int $rows, - private FloeEngine $engine, ) {} public function run(): void { - $config = config_builder(); - - if ($this->engine === FloeEngine::php) { - $config->hydrator(new PhpRowHydrator()); - } - - data_frame($config)->read(from_floe(Datasets::orders($this->rows)->floe(), engine: $this->engine))->run(); + data_frame()->read(from_floe(Datasets::orders($this->rows)->floe()))->run(); } } diff --git a/benchmarks/src/Format/Floe/FloeWriteScenario.php b/benchmarks/src/Format/Floe/FloeWriteScenario.php index 3fb0c64e35..c835ad9e44 100644 --- a/benchmarks/src/Format/Floe/FloeWriteScenario.php +++ b/benchmarks/src/Format/Floe/FloeWriteScenario.php @@ -6,10 +6,7 @@ use Flow\Benchmarks\Datasets\Datasets; use Flow\Benchmarks\Datasets\Paths; -use Flow\ETL\Row\PhpRowHydrator; -use Flow\Floe\FloeEngine; -use function Flow\ETL\DSL\config_builder; use function Flow\ETL\DSL\data_frame; use function Flow\Floe\DSL\from_floe; use function Flow\Floe\DSL\to_floe; @@ -19,20 +16,13 @@ { public function __construct( private int $rows, - private FloeEngine $engine, ) {} public function run(): void { - $config = config_builder(); - - if ($this->engine === FloeEngine::php) { - $config->hydrator(new PhpRowHydrator()); - } - - data_frame($config) - ->read(from_floe(Datasets::orders($this->rows)->floe(), engine: $this->engine)) - ->write(to_floe(Paths::var() . '/format_write_floe_' . uniqid('', true) . '.floe', engine: $this->engine)) + data_frame() + ->read(from_floe(Datasets::orders($this->rows)->floe())) + ->write(to_floe(Paths::var() . '/format_write_floe_' . uniqid('', true) . '.floe')) ->run(); } } diff --git a/benchmarks/src/Pipeline/InMemoryOrders.php b/benchmarks/src/Pipeline/InMemoryOrders.php index 408be928d8..e35bf5bcc2 100644 --- a/benchmarks/src/Pipeline/InMemoryOrders.php +++ b/benchmarks/src/Pipeline/InMemoryOrders.php @@ -35,10 +35,12 @@ public static function of(int $rows): array $data = []; foreach (data_frame()->read(from_parquet(Datasets::orders($rows)->parquet()))->get() as $batch) { - foreach ($batch->all() as $row) { - // Row::toArray() is declared array; names() is list, so - // recombining them is what makes the string keys ArrayMemory requires provable. - $data[] = array_combine($row->names(), array_values($row->toArray())); + $names = $batch->schema()->references()->names(); + + // Rows::toArray() is declared array; names() is array, so + // recombining them is what makes the string keys ArrayMemory requires provable. + foreach ($batch->toArray(withKeys: false) as $values) { + $data[] = array_combine($names, $values); } } diff --git a/benchmarks/src/Pipeline/SourceExtractor.php b/benchmarks/src/Pipeline/SourceExtractor.php index 7dd863d43c..15bfccd716 100644 --- a/benchmarks/src/Pipeline/SourceExtractor.php +++ b/benchmarks/src/Pipeline/SourceExtractor.php @@ -16,6 +16,7 @@ use function Flow\ETL\DSL\from_array; use function Flow\ETL\DSL\from_memory; use function Flow\Floe\DSL\from_floe; +use function in_array; final readonly class SourceExtractor { @@ -40,7 +41,7 @@ public function extractor(): Extractor Source::parquet => from_parquet($orders->parquet()), }; - return $this->mode === SchemaMode::declared + return $this->mode === SchemaMode::declared && !in_array($this->source, [Source::floe, Source::parquet], true) ? $extractor->withSchema(OrdersSchema::of($this->source)) : $extractor; } diff --git a/benchmarks/src/Schema/RowsSchemaScenario.php b/benchmarks/src/Schema/RowsSchemaScenario.php index 653c323748..52104ebcc3 100644 --- a/benchmarks/src/Schema/RowsSchemaScenario.php +++ b/benchmarks/src/Schema/RowsSchemaScenario.php @@ -7,8 +7,7 @@ use Flow\ETL\Rows; use function array_key_exists; -use function Flow\ETL\DSL\row; -use function Flow\ETL\DSL\rows; +use function Flow\ETL\DSL\array_to_rows; use function Flow\ETL\DSL\schema; use function Flow\ETL\DSL\str_schema; @@ -60,9 +59,9 @@ public function rows(): Rows $list = []; for ($index = 0; $index < $this->batch; $index++) { - $list[] = row(['a' => 'x' . $index]); + $list[] = ['a' => 'x' . $index]; } - return self::$batches[$this->batch] = rows(schema(str_schema('a')), ...$list); + return self::$batches[$this->batch] = array_to_rows($list, schema(str_schema('a'))); } } diff --git a/benchmarks/src/Service/Doctrine/DoctrineConnection.php b/benchmarks/src/Service/Doctrine/DoctrineConnection.php index c9b90adfcb..be523853f7 100644 --- a/benchmarks/src/Service/Doctrine/DoctrineConnection.php +++ b/benchmarks/src/Service/Doctrine/DoctrineConnection.php @@ -47,9 +47,10 @@ public static function createTable(Connection $connection, string $table, bool $ $definition = to_dbal_schema_table((new FakeRandomOrdersExtractor())->schema(), $table); if ($keyed) { - $definition->addPrimaryKeyConstraint( - PrimaryKeyConstraint::editor()->setUnquotedColumnNames('order_id')->create(), - ); + $definition = $definition + ->edit() + ->addPrimaryKeyConstraint(PrimaryKeyConstraint::editor()->setUnquotedColumnNames('order_id')->create()) + ->create(); } $connection->createSchemaManager()->createTable($definition); diff --git a/benchmarks/suites/Format/Floe/FloeReadBench.php b/benchmarks/suites/Format/Floe/FloeReadBench.php index e877411b3b..2fc3753998 100644 --- a/benchmarks/suites/Format/Floe/FloeReadBench.php +++ b/benchmarks/suites/Format/Floe/FloeReadBench.php @@ -6,12 +6,9 @@ use Flow\Benchmarks\BenchmarkRows; use Flow\Benchmarks\Datasets\Datasets; -use Flow\Floe\FloeEngine; use Generator; use PhpBench\Attributes as Bench; -use function extension_loaded; - #[Bench\BeforeMethods('warm')] final class FloeReadBench { @@ -20,11 +17,11 @@ public function warm(array $params): void Datasets::orders((int) $params['rows'])->floe(); } - #[Bench\ParamProviders(['rows', 'engines'])] + #[Bench\ParamProviders(['rows'])] #[Bench\Groups(['format', 'format-floe'])] public function bench_floe_read(array $params): void { - (new FloeReadScenario((int) $params['rows'], FloeEngine::from((string) $params['engine'])))->run(); + (new FloeReadScenario((int) $params['rows']))->run(); } public function rows(): Generator @@ -33,13 +30,4 @@ public function rows(): Generator yield number_format($rows) => ['rows' => $rows]; } - - public function engines(): Generator - { - yield 'php' => ['engine' => FloeEngine::php->value]; - - if (extension_loaded('flow_php')) { - yield 'native' => ['engine' => FloeEngine::native->value]; - } - } } diff --git a/benchmarks/suites/Format/Floe/FloeWriteBench.php b/benchmarks/suites/Format/Floe/FloeWriteBench.php index 940b2ab2bc..aaada5f55b 100644 --- a/benchmarks/suites/Format/Floe/FloeWriteBench.php +++ b/benchmarks/suites/Format/Floe/FloeWriteBench.php @@ -6,12 +6,9 @@ use Flow\Benchmarks\BenchmarkRows; use Flow\Benchmarks\Datasets\Datasets; -use Flow\Floe\FloeEngine; use Generator; use PhpBench\Attributes as Bench; -use function extension_loaded; - #[Bench\BeforeMethods('warm')] final class FloeWriteBench { @@ -20,11 +17,11 @@ public function warm(array $params): void Datasets::orders((int) $params['rows'])->floe(); } - #[Bench\ParamProviders(['rows', 'engines'])] + #[Bench\ParamProviders(['rows'])] #[Bench\Groups(['format', 'format-floe'])] public function bench_floe_write(array $params): void { - (new FloeWriteScenario((int) $params['rows'], FloeEngine::from((string) $params['engine'])))->run(); + (new FloeWriteScenario((int) $params['rows']))->run(); } public function rows(): Generator @@ -33,13 +30,4 @@ public function rows(): Generator yield number_format($rows) => ['rows' => $rows]; } - - public function engines(): Generator - { - yield 'php' => ['engine' => FloeEngine::php->value]; - - if (extension_loaded('flow_php')) { - yield 'native' => ['engine' => FloeEngine::native->value]; - } - } } diff --git a/benchmarks/suites/Format/Parquet/ParquetReadBench.php b/benchmarks/suites/Format/Parquet/ParquetReadBench.php index 86c7f670ff..366d7fa6ee 100644 --- a/benchmarks/suites/Format/Parquet/ParquetReadBench.php +++ b/benchmarks/suites/Format/Parquet/ParquetReadBench.php @@ -6,8 +6,8 @@ use Flow\Benchmarks\BenchmarkRows; use Flow\Benchmarks\Datasets\Datasets; -use Flow\Parquet\Engine\ArrowParquetEngine; use Flow\Parquet\Engine\PhpParquetEngine; +use Flow\Parquet\Engine\RustParquetEngine; use Generator; use PhpBench\Attributes as Bench; @@ -25,7 +25,7 @@ public function warm(array $params): void #[Bench\Groups(['format', 'format-parquet'])] public function bench_parquet_read(array $params): void { - $engine = $params['engine'] === ArrowParquetEngine::class ? new ArrowParquetEngine() : new PhpParquetEngine(); + $engine = $params['engine'] === RustParquetEngine::class ? new RustParquetEngine() : new PhpParquetEngine(); (new ParquetReadScenario((int) $params['rows'], $engine))->run(); } @@ -42,7 +42,7 @@ public function engines(): Generator yield 'php' => ['engine' => PhpParquetEngine::class]; if (extension_loaded('arrow')) { - yield 'arrow' => ['engine' => ArrowParquetEngine::class]; + yield 'arrow' => ['engine' => RustParquetEngine::class]; } } } diff --git a/benchmarks/suites/Format/Parquet/ParquetWriteBench.php b/benchmarks/suites/Format/Parquet/ParquetWriteBench.php index 6ffe41410d..0abe5acc71 100644 --- a/benchmarks/suites/Format/Parquet/ParquetWriteBench.php +++ b/benchmarks/suites/Format/Parquet/ParquetWriteBench.php @@ -6,8 +6,8 @@ use Flow\Benchmarks\BenchmarkRows; use Flow\Benchmarks\Datasets\Datasets; -use Flow\Parquet\Engine\ArrowParquetEngine; use Flow\Parquet\Engine\PhpParquetEngine; +use Flow\Parquet\Engine\RustParquetEngine; use Generator; use PhpBench\Attributes as Bench; @@ -25,7 +25,7 @@ public function warm(array $params): void #[Bench\Groups(['format', 'format-parquet'])] public function bench_parquet_write(array $params): void { - $engine = $params['engine'] === ArrowParquetEngine::class ? new ArrowParquetEngine() : new PhpParquetEngine(); + $engine = $params['engine'] === RustParquetEngine::class ? new RustParquetEngine() : new PhpParquetEngine(); (new ParquetWriteScenario((int) $params['rows'], $engine))->run(); } @@ -42,7 +42,7 @@ public function engines(): Generator yield 'php' => ['engine' => PhpParquetEngine::class]; if (extension_loaded('arrow')) { - yield 'arrow' => ['engine' => ArrowParquetEngine::class]; + yield 'arrow' => ['engine' => RustParquetEngine::class]; } } } diff --git a/benchmarks/tests/Flow/Benchmarks/Tests/Unit/Pipeline/PipelineCostScenariosTest.php b/benchmarks/tests/Flow/Benchmarks/Tests/Unit/Pipeline/PipelineCostScenariosTest.php index 5a8ec7da22..438a7b1333 100644 --- a/benchmarks/tests/Flow/Benchmarks/Tests/Unit/Pipeline/PipelineCostScenariosTest.php +++ b/benchmarks/tests/Flow/Benchmarks/Tests/Unit/Pipeline/PipelineCostScenariosTest.php @@ -92,7 +92,7 @@ public function test_the_fetch_scenario_runs(): void $rows = (new FetchScenario(Source::csv, self::ROWS))->run(); static::assertCount(self::ROWS, $rows); - static::assertSame(['order_id', 'customer'], $rows->first()->names()); + static::assertSame(['order_id', 'customer'], array_keys($rows->values(0))); } #[DataProvider('limits')] diff --git a/benchmarks/tests/Flow/Benchmarks/Tests/Unit/Pipeline/PipelineScenarioTest.php b/benchmarks/tests/Flow/Benchmarks/Tests/Unit/Pipeline/PipelineScenarioTest.php index 74547de60a..def597cec1 100644 --- a/benchmarks/tests/Flow/Benchmarks/Tests/Unit/Pipeline/PipelineScenarioTest.php +++ b/benchmarks/tests/Flow/Benchmarks/Tests/Unit/Pipeline/PipelineScenarioTest.php @@ -40,7 +40,10 @@ public function test_it_writes_the_five_projected_columns_for_every_row(Source $ $written->remove(); static::assertCount(self::ROWS, $rows); - static::assertSame(['order_id', 'seller_id', 'created_at', 'customer', 'contact'], $rows->first()->names()); + static::assertSame( + ['order_id', 'seller_id', 'created_at', 'customer', 'contact'], + array_keys($rows->values(0)), + ); } #[DataProvider('sources')] @@ -50,12 +53,12 @@ public function test_the_inferred_and_declared_arms_produce_the_same_columns(Sou $declaredFile = new WrittenFile(Paths::var() . '/pipeline_' . $source->value . '_*.floe'); (new PipelineScenario($source, SchemaMode::declared, self::ROWS))->run(); - $declared = data_frame()->read(from_floe($declaredFile->path()))->fetch()->first()->names(); + $declared = array_keys(data_frame()->read(from_floe($declaredFile->path()))->fetch()->values(0)); $declaredFile->remove(); $inferredFile = new WrittenFile(Paths::var() . '/pipeline_' . $source->value . '_*.floe'); (new PipelineScenario($source, SchemaMode::inferred, self::ROWS))->run(); - $inferred = data_frame()->read(from_floe($inferredFile->path()))->fetch()->first()->names(); + $inferred = array_keys(data_frame()->read(from_floe($inferredFile->path()))->fetch()->values(0)); $inferredFile->remove(); static::assertSame($declared, $inferred); diff --git a/benchmarks/tests/Flow/Benchmarks/Tests/Unit/Pipeline/SourceExtractorTest.php b/benchmarks/tests/Flow/Benchmarks/Tests/Unit/Pipeline/SourceExtractorTest.php index a482e9b097..b2354f6ca3 100644 --- a/benchmarks/tests/Flow/Benchmarks/Tests/Unit/Pipeline/SourceExtractorTest.php +++ b/benchmarks/tests/Flow/Benchmarks/Tests/Unit/Pipeline/SourceExtractorTest.php @@ -24,6 +24,15 @@ public static function sources(): Generator } } + public static function sources_taking_a_schema(): Generator + { + foreach (Source::cases() as $source) { + if ($source !== Source::floe && $source !== Source::parquet) { + yield $source->value => [$source]; + } + } + } + #[DataProvider('sources')] public function test_declared_mode_attaches_the_frozen_schema(Source $source): void { @@ -40,7 +49,7 @@ public function test_declared_mode_attaches_the_frozen_schema(Source $source): v * comparing schemas passes whether or not withSchema() was applied. Comparing the extractors * themselves does distinguish them - withSchema() returns a clone carrying the schema. */ - #[DataProvider('sources')] + #[DataProvider('sources_taking_a_schema')] public function test_inferred_mode_attaches_no_schema(Source $source): void { (new SourceFixture($source, self::ROWS))->warm(); diff --git a/benchmarks/tests/Flow/Benchmarks/Tests/Unit/Pipeline/StageScenarioTest.php b/benchmarks/tests/Flow/Benchmarks/Tests/Unit/Pipeline/StageScenarioTest.php index 043c827ffe..689fbbabdd 100644 --- a/benchmarks/tests/Flow/Benchmarks/Tests/Unit/Pipeline/StageScenarioTest.php +++ b/benchmarks/tests/Flow/Benchmarks/Tests/Unit/Pipeline/StageScenarioTest.php @@ -50,6 +50,6 @@ public function test_the_read_select_write_stage_writes_the_five_projected_colum $written->remove(); static::assertCount(self::ROWS, $rows); - static::assertSame(['order_id', 'seller_id', 'created_at', 'customer', 'email'], $rows->first()->names()); + static::assertSame(['order_id', 'seller_id', 'created_at', 'customer', 'email'], array_keys($rows->values(0))); } } diff --git a/benchmarks/tests/Flow/Benchmarks/Tests/Unit/Window/WindowPartitioningTest.php b/benchmarks/tests/Flow/Benchmarks/Tests/Unit/Window/WindowPartitioningTest.php index 1c168140d8..eaed8a293f 100644 --- a/benchmarks/tests/Flow/Benchmarks/Tests/Unit/Window/WindowPartitioningTest.php +++ b/benchmarks/tests/Flow/Benchmarks/Tests/Unit/Window/WindowPartitioningTest.php @@ -25,11 +25,12 @@ public function test_both_arms_derive_the_key_so_only_the_partition_column_varie foreach (PartitionCardinality::cases() as $cardinality) { static::assertContains( 'day', - (new WindowPartitioning($cardinality)) - ->derive(data_frame()->read(from_floe(Datasets::orders(self::ROWS)->floe()))) - ->fetch() - ->first() - ->names(), + array_keys( + (new WindowPartitioning($cardinality)) + ->derive(data_frame()->read(from_floe(Datasets::orders(self::ROWS)->floe()))) + ->fetch() + ->values(0), + ), $cardinality->value, ); } diff --git a/composer.json b/composer.json index 492a42067e..5879bd35cb 100644 --- a/composer.json +++ b/composer.json @@ -26,7 +26,7 @@ "cmsig/seal": "^0.12", "coduo/php-humanizer": "^5.0", "cuyz/valinor": "^2.4", - "doctrine/dbal": "^4.4", + "doctrine/dbal": "^4.5", "google/apiclient": "^2.13", "google/protobuf": "^4.0 || ^5.0", "halaxa/json-machine": "^1.2", diff --git a/composer.lock b/composer.lock index f0f55c9063..cb7b675cc3 100644 --- a/composer.lock +++ b/composer.lock @@ -4,7 +4,7 @@ "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies", "This file is @generated automatically" ], - "content-hash": "2a326b7adbd5bb2d7624c82921172673", + "content-hash": "9cfb96472f0a2797d94d07502009edb1", "packages": [ { "name": "async-aws/core", @@ -427,16 +427,16 @@ }, { "name": "doctrine/dbal", - "version": "4.4.4", + "version": "4.5.0", "source": { "type": "git", "url": "https://github.com/doctrine/dbal.git", - "reference": "fb9e0ffe15e1590e24dc61c0c0a23f9a33ee42ce" + "reference": "c435cd7ea33e17d2a89288caf994cf3d4549b879" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/doctrine/dbal/zipball/fb9e0ffe15e1590e24dc61c0c0a23f9a33ee42ce", - "reference": "fb9e0ffe15e1590e24dc61c0c0a23f9a33ee42ce", + "url": "https://api.github.com/repos/doctrine/dbal/zipball/c435cd7ea33e17d2a89288caf994cf3d4549b879", + "reference": "c435cd7ea33e17d2a89288caf994cf3d4549b879", "shasum": "" }, "require": { @@ -452,9 +452,9 @@ "phpstan/phpstan": "2.1.30", "phpstan/phpstan-phpunit": "2.0.7", "phpstan/phpstan-strict-rules": "^2", - "phpunit/phpunit": "11.5.50", - "slevomat/coding-standard": "8.27.1", - "squizlabs/php_codesniffer": "4.0.1", + "phpunit/phpunit": "11.5.56", + "slevomat/coding-standard": "8.31.1", + "squizlabs/php_codesniffer": "4.0.4", "symfony/cache": "^6.3.8|^7.0|^8.0", "symfony/console": "^5.4|^6.3|^7.0|^8.0" }, @@ -513,7 +513,7 @@ ], "support": { "issues": "https://github.com/doctrine/dbal/issues", - "source": "https://github.com/doctrine/dbal/tree/4.4.4" + "source": "https://github.com/doctrine/dbal/tree/4.5.0" }, "funding": [ { @@ -529,7 +529,7 @@ "type": "tidelift" } ], - "time": "2026-07-21T14:34:40+00:00" + "time": "2026-09-24T16:11:26+00:00" }, { "name": "doctrine/deprecations", @@ -581,26 +581,26 @@ }, { "name": "firebase/php-jwt", - "version": "v7.1.0", + "version": "v7.2.1", "source": { "type": "git", "url": "https://github.com/googleapis/php-jwt.git", - "reference": "b374a5d1a4f1f67fadc2165cdb284645945e2fc0" + "reference": "0501f5729b9e2448516674aff07c7d1727869b8e" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/googleapis/php-jwt/zipball/b374a5d1a4f1f67fadc2165cdb284645945e2fc0", - "reference": "b374a5d1a4f1f67fadc2165cdb284645945e2fc0", + "url": "https://api.github.com/repos/googleapis/php-jwt/zipball/0501f5729b9e2448516674aff07c7d1727869b8e", + "reference": "0501f5729b9e2448516674aff07c7d1727869b8e", "shasum": "" }, "require": { "php": "^8.0" }, "require-dev": { - "guzzlehttp/guzzle": "^7.4", + "guzzlehttp/guzzle": "^7.4||^8.0", "phpfastcache/phpfastcache": "^9.2", "phpseclib/phpseclib": "~3.0", - "phpspec/prophecy-phpunit": "^2.0", + "phpspec/prophecy-phpunit": "^2.2", "phpunit/phpunit": "^9.5", "psr/cache": "^2.0||^3.0", "psr/http-client": "^1.0", @@ -612,6 +612,14 @@ "phpseclib/phpseclib": "Support PS256 (RSASSA-PSS) signatures" }, "type": "library", + "extra": { + "component": { + "id": "jwt", + "path": "Jwt", + "entry": "README.md", + "target": "googleapis/php-jwt.git" + } + }, "autoload": { "psr-4": { "Firebase\\JWT\\": "src" @@ -641,30 +649,30 @@ ], "support": { "issues": "https://github.com/googleapis/php-jwt/issues", - "source": "https://github.com/googleapis/php-jwt/tree/v7.1.0" + "source": "https://github.com/googleapis/php-jwt/tree/v7.2.1" }, - "time": "2026-06-11T17:54:14+00:00" + "time": "2026-09-28T17:51:46+00:00" }, { "name": "google/apiclient", - "version": "v2.19.4", + "version": "v2.20.1", "source": { "type": "git", "url": "https://github.com/googleapis/google-api-php-client.git", - "reference": "f745b310113d556c19d5e54ccc48b5821d1e2622" + "reference": "31b577e7012c4e68d5289d63aaf7e6d471ec1ff1" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/googleapis/google-api-php-client/zipball/f745b310113d556c19d5e54ccc48b5821d1e2622", - "reference": "f745b310113d556c19d5e54ccc48b5821d1e2622", + "url": "https://api.github.com/repos/googleapis/google-api-php-client/zipball/31b577e7012c4e68d5289d63aaf7e6d471ec1ff1", + "reference": "31b577e7012c4e68d5289d63aaf7e6d471ec1ff1", "shasum": "" }, "require": { "firebase/php-jwt": "^6.0||^7.0", "google/apiclient-services": "~0.350", - "google/auth": "^1.37", - "guzzlehttp/guzzle": "^7.4.5", - "guzzlehttp/psr7": "^2.6", + "google/auth": "^1.53", + "guzzlehttp/guzzle": "^7.8.2||^8.0", + "guzzlehttp/psr7": "^2.6.3||^3.0", "monolog/monolog": "^2.9||^3.0", "php": "^8.1" }, @@ -712,22 +720,22 @@ ], "support": { "issues": "https://github.com/googleapis/google-api-php-client/issues", - "source": "https://github.com/googleapis/google-api-php-client/tree/v2.19.4" + "source": "https://github.com/googleapis/google-api-php-client/tree/v2.20.1" }, - "time": "2026-06-29T08:12:37+00:00" + "time": "2026-09-25T18:54:19+00:00" }, { "name": "google/apiclient-services", - "version": "v0.459.0", + "version": "v0.461.0", "source": { "type": "git", "url": "https://github.com/googleapis/google-api-php-client-services.git", - "reference": "3b45be50c8420dea56b6f2326ff745175c3cafb8" + "reference": "653eec9b7767d34f0cbe2a702cf2c2b825a4857d" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/googleapis/google-api-php-client-services/zipball/3b45be50c8420dea56b6f2326ff745175c3cafb8", - "reference": "3b45be50c8420dea56b6f2326ff745175c3cafb8", + "url": "https://api.github.com/repos/googleapis/google-api-php-client-services/zipball/653eec9b7767d34f0cbe2a702cf2c2b825a4857d", + "reference": "653eec9b7767d34f0cbe2a702cf2c2b825a4857d", "shasum": "" }, "require": { @@ -756,22 +764,22 @@ ], "support": { "issues": "https://github.com/googleapis/google-api-php-client-services/issues", - "source": "https://github.com/googleapis/google-api-php-client-services/tree/v0.459.0" + "source": "https://github.com/googleapis/google-api-php-client-services/tree/v0.461.0" }, - "time": "2026-09-14T01:22:23+00:00" + "time": "2026-09-25T22:00:27+00:00" }, { "name": "google/auth", - "version": "v1.53.0", + "version": "v1.55.1", "source": { "type": "git", "url": "https://github.com/googleapis/google-auth-library-php.git", - "reference": "d677d0b0c4bd52ab222a85df8e74e0d491c0cc5a" + "reference": "ef1b92cb42470b09b3caaf23bb9b4bb9a2dc787e" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/googleapis/google-auth-library-php/zipball/d677d0b0c4bd52ab222a85df8e74e0d491c0cc5a", - "reference": "d677d0b0c4bd52ab222a85df8e74e0d491c0cc5a", + "url": "https://api.github.com/repos/googleapis/google-auth-library-php/zipball/ef1b92cb42470b09b3caaf23bb9b4bb9a2dc787e", + "reference": "ef1b92cb42470b09b3caaf23bb9b4bb9a2dc787e", "shasum": "" }, "require": { @@ -789,17 +797,25 @@ "kelvinmo/simplejwt": "^1.1.0", "phpseclib/phpseclib": "^3.0.35", "phpspec/prophecy-phpunit": "^2.1", - "phpunit/phpunit": "^9.6", + "phpunit/phpunit": "^9.6.34", "sebastian/comparator": ">=1.2.3", "squizlabs/php_codesniffer": "^4.0", - "symfony/filesystem": "^6.3||^7.3", - "symfony/process": "^6.0||^7.0", + "symfony/filesystem": "^6.4.39||^7.3", + "symfony/process": "^6.4.41||^7.0", "webmozart/assert": "^1.11||^2.0" }, "suggest": { "phpseclib/phpseclib": "May be used in place of OpenSSL for signing strings or for token management. Please require version ^2." }, "type": "library", + "extra": { + "component": { + "id": "auth", + "path": "Auth", + "entry": "README.md", + "target": "googleapis/google-auth-library-php.git" + } + }, "autoload": { "psr-4": { "Google\\Auth\\": "src" @@ -810,7 +826,7 @@ "Apache-2.0" ], "description": "Google Auth Library for PHP", - "homepage": "https://github.com/google/google-auth-library-php", + "homepage": "https://github.com/googleapis/google-auth-library-php", "keywords": [ "Authentication", "google", @@ -819,9 +835,9 @@ "support": { "docs": "https://cloud.google.com/php/docs/reference/auth/latest", "issues": "https://github.com/googleapis/google-auth-library-php/issues", - "source": "https://github.com/googleapis/google-auth-library-php/tree/v1.53.0" + "source": "https://github.com/googleapis/google-auth-library-php/tree/v1.55.1" }, - "time": "2026-07-22T22:36:10+00:00" + "time": "2026-09-28T17:51:46+00:00" }, { "name": "google/protobuf", @@ -1262,16 +1278,16 @@ }, { "name": "monolog/monolog", - "version": "3.12.0", + "version": "3.12.1", "source": { "type": "git", "url": "https://github.com/Seldaek/monolog.git", - "reference": "72c534fc0ab181ef52d92a68382318631e301608" + "reference": "d7059e4c691d05ea1222b5124bed0f961be46e15" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/Seldaek/monolog/zipball/72c534fc0ab181ef52d92a68382318631e301608", - "reference": "72c534fc0ab181ef52d92a68382318631e301608", + "url": "https://api.github.com/repos/Seldaek/monolog/zipball/d7059e4c691d05ea1222b5124bed0f961be46e15", + "reference": "d7059e4c691d05ea1222b5124bed0f961be46e15", "shasum": "" }, "require": { @@ -1351,7 +1367,7 @@ ], "support": { "issues": "https://github.com/Seldaek/monolog/issues", - "source": "https://github.com/Seldaek/monolog/tree/3.12.0" + "source": "https://github.com/Seldaek/monolog/tree/3.12.1" }, "funding": [ { @@ -1363,7 +1379,7 @@ "type": "tidelift" } ], - "time": "2026-09-09T08:34:20+00:00" + "time": "2026-09-29T19:21:12+00:00" }, { "name": "openspout/openspout", @@ -2350,16 +2366,16 @@ }, { "name": "symfony/console", - "version": "v7.4.18", + "version": "v7.4.20", "source": { "type": "git", "url": "https://github.com/symfony/console.git", - "reference": "23d6f88a29f6d0eac45bd77d70307adf83ba7ab0" + "reference": "d7bca2fe05241dc7732d0c8b6d551b879f5d0a7d" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/symfony/console/zipball/23d6f88a29f6d0eac45bd77d70307adf83ba7ab0", - "reference": "23d6f88a29f6d0eac45bd77d70307adf83ba7ab0", + "url": "https://api.github.com/repos/symfony/console/zipball/d7bca2fe05241dc7732d0c8b6d551b879f5d0a7d", + "reference": "d7bca2fe05241dc7732d0c8b6d551b879f5d0a7d", "shasum": "" }, "require": { @@ -2424,7 +2440,7 @@ "terminal" ], "support": { - "source": "https://github.com/symfony/console/tree/v7.4.18" + "source": "https://github.com/symfony/console/tree/v7.4.20" }, "funding": [ { @@ -2444,20 +2460,20 @@ "type": "tidelift" } ], - "time": "2026-08-25T14:18:37+00:00" + "time": "2026-09-18T15:55:00+00:00" }, { "name": "symfony/dependency-injection", - "version": "v7.4.17", + "version": "v7.4.20", "source": { "type": "git", "url": "https://github.com/symfony/dependency-injection.git", - "reference": "f318ac9da5aba0be2cf43ecdd562c05a33bc11c0" + "reference": "a7faee783a87b55f27460fc571f8c639bf9d93a2" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/symfony/dependency-injection/zipball/f318ac9da5aba0be2cf43ecdd562c05a33bc11c0", - "reference": "f318ac9da5aba0be2cf43ecdd562c05a33bc11c0", + "url": "https://api.github.com/repos/symfony/dependency-injection/zipball/a7faee783a87b55f27460fc571f8c639bf9d93a2", + "reference": "a7faee783a87b55f27460fc571f8c639bf9d93a2", "shasum": "" }, "require": { @@ -2508,7 +2524,7 @@ "description": "Allows you to standardize and centralize the way objects are constructed in your application", "homepage": "https://symfony.com", "support": { - "source": "https://github.com/symfony/dependency-injection/tree/v7.4.17" + "source": "https://github.com/symfony/dependency-injection/tree/v7.4.20" }, "funding": [ { @@ -2528,7 +2544,7 @@ "type": "tidelift" } ], - "time": "2026-08-21T17:40:08+00:00" + "time": "2026-09-29T12:35:16+00:00" }, { "name": "symfony/deprecation-contracts", @@ -2603,16 +2619,16 @@ }, { "name": "symfony/error-handler", - "version": "v7.4.17", + "version": "v7.4.20", "source": { "type": "git", "url": "https://github.com/symfony/error-handler.git", - "reference": "8373921e231e190a88e2ad526951bbaa791576fa" + "reference": "82239fe00f7cdc9f17c4166fff4ebdc22bd26ec0" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/symfony/error-handler/zipball/8373921e231e190a88e2ad526951bbaa791576fa", - "reference": "8373921e231e190a88e2ad526951bbaa791576fa", + "url": "https://api.github.com/repos/symfony/error-handler/zipball/82239fe00f7cdc9f17c4166fff4ebdc22bd26ec0", + "reference": "82239fe00f7cdc9f17c4166fff4ebdc22bd26ec0", "shasum": "" }, "require": { @@ -2661,7 +2677,7 @@ "description": "Provides tools to manage errors and ease debugging PHP code", "homepage": "https://symfony.com", "support": { - "source": "https://github.com/symfony/error-handler/tree/v7.4.17" + "source": "https://github.com/symfony/error-handler/tree/v7.4.20" }, "funding": [ { @@ -2681,7 +2697,7 @@ "type": "tidelift" } ], - "time": "2026-08-21T17:40:08+00:00" + "time": "2026-09-22T18:56:40+00:00" }, { "name": "symfony/event-dispatcher", @@ -2920,16 +2936,16 @@ }, { "name": "symfony/http-client", - "version": "v7.4.18", + "version": "v7.4.20", "source": { "type": "git", "url": "https://github.com/symfony/http-client.git", - "reference": "68d81f78d127984ff2e741f0e0d9cb1078f8f3ea" + "reference": "750a420b8dd0af3d6e1c6c1412802abf2cdd2b6c" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/symfony/http-client/zipball/68d81f78d127984ff2e741f0e0d9cb1078f8f3ea", - "reference": "68d81f78d127984ff2e741f0e0d9cb1078f8f3ea", + "url": "https://api.github.com/repos/symfony/http-client/zipball/750a420b8dd0af3d6e1c6c1412802abf2cdd2b6c", + "reference": "750a420b8dd0af3d6e1c6c1412802abf2cdd2b6c", "shasum": "" }, "require": { @@ -2997,7 +3013,7 @@ "http" ], "support": { - "source": "https://github.com/symfony/http-client/tree/v7.4.18" + "source": "https://github.com/symfony/http-client/tree/v7.4.20" }, "funding": [ { @@ -3017,7 +3033,7 @@ "type": "tidelift" } ], - "time": "2026-08-30T13:49:59+00:00" + "time": "2026-09-24T11:02:13+00:00" }, { "name": "symfony/http-client-contracts", @@ -3103,16 +3119,16 @@ }, { "name": "symfony/http-foundation", - "version": "v7.4.18", + "version": "v7.4.20", "source": { "type": "git", "url": "https://github.com/symfony/http-foundation.git", - "reference": "d070b716a32fbe3bf04204db0f58ace73b86d133" + "reference": "9d0761c0da6bd5b801b401666834901a9a1ffc33" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/symfony/http-foundation/zipball/d070b716a32fbe3bf04204db0f58ace73b86d133", - "reference": "d070b716a32fbe3bf04204db0f58ace73b86d133", + "url": "https://api.github.com/repos/symfony/http-foundation/zipball/9d0761c0da6bd5b801b401666834901a9a1ffc33", + "reference": "9d0761c0da6bd5b801b401666834901a9a1ffc33", "shasum": "" }, "require": { @@ -3161,7 +3177,7 @@ "description": "Defines an object-oriented layer for the HTTP specification", "homepage": "https://symfony.com", "support": { - "source": "https://github.com/symfony/http-foundation/tree/v7.4.18" + "source": "https://github.com/symfony/http-foundation/tree/v7.4.20" }, "funding": [ { @@ -3181,20 +3197,20 @@ "type": "tidelift" } ], - "time": "2026-08-30T20:10:52+00:00" + "time": "2026-09-24T07:12:44+00:00" }, { "name": "symfony/http-kernel", - "version": "v7.4.18", + "version": "v7.4.20", "source": { "type": "git", "url": "https://github.com/symfony/http-kernel.git", - "reference": "275d2d2d24530f2a0eaf17704a3a93860a036351" + "reference": "392784b5d7b959b8d872542a2530c50c540a788f" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/symfony/http-kernel/zipball/275d2d2d24530f2a0eaf17704a3a93860a036351", - "reference": "275d2d2d24530f2a0eaf17704a3a93860a036351", + "url": "https://api.github.com/repos/symfony/http-kernel/zipball/392784b5d7b959b8d872542a2530c50c540a788f", + "reference": "392784b5d7b959b8d872542a2530c50c540a788f", "shasum": "" }, "require": { @@ -3280,7 +3296,7 @@ "description": "Provides a structured process for converting a Request into a Response", "homepage": "https://symfony.com", "support": { - "source": "https://github.com/symfony/http-kernel/tree/v7.4.18" + "source": "https://github.com/symfony/http-kernel/tree/v7.4.20" }, "funding": [ { @@ -3300,7 +3316,7 @@ "type": "tidelift" } ], - "time": "2026-08-30T21:24:29+00:00" + "time": "2026-09-29T14:05:16+00:00" }, { "name": "symfony/polyfill-ctype", @@ -3387,16 +3403,16 @@ }, { "name": "symfony/polyfill-intl-grapheme", - "version": "v1.41.0", + "version": "v1.43.0", "source": { "type": "git", "url": "https://github.com/symfony/polyfill-intl-grapheme.git", - "reference": "bb899c1db0aa8127dc3afe8cda4a67eb24915f8d" + "reference": "5cac8f4db97208823aa353f59a01faad2da52c75" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/symfony/polyfill-intl-grapheme/zipball/bb899c1db0aa8127dc3afe8cda4a67eb24915f8d", - "reference": "bb899c1db0aa8127dc3afe8cda4a67eb24915f8d", + "url": "https://api.github.com/repos/symfony/polyfill-intl-grapheme/zipball/5cac8f4db97208823aa353f59a01faad2da52c75", + "reference": "5cac8f4db97208823aa353f59a01faad2da52c75", "shasum": "" }, "require": { @@ -3445,7 +3461,7 @@ "shim" ], "support": { - "source": "https://github.com/symfony/polyfill-intl-grapheme/tree/v1.41.0" + "source": "https://github.com/symfony/polyfill-intl-grapheme/tree/v1.43.0" }, "funding": [ { @@ -3465,20 +3481,20 @@ "type": "tidelift" } ], - "time": "2026-07-28T08:25:59+00:00" + "time": "2026-09-25T15:46:47+00:00" }, { "name": "symfony/polyfill-intl-normalizer", - "version": "v1.42.0", + "version": "v1.43.0", "source": { "type": "git", "url": "https://github.com/symfony/polyfill-intl-normalizer.git", - "reference": "aa20edea75bd9c48cfecc8360922e5a6e5c44502" + "reference": "ebd57ccc7edb93d632e184efbe7693d2517bab3b" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/symfony/polyfill-intl-normalizer/zipball/aa20edea75bd9c48cfecc8360922e5a6e5c44502", - "reference": "aa20edea75bd9c48cfecc8360922e5a6e5c44502", + "url": "https://api.github.com/repos/symfony/polyfill-intl-normalizer/zipball/ebd57ccc7edb93d632e184efbe7693d2517bab3b", + "reference": "ebd57ccc7edb93d632e184efbe7693d2517bab3b", "shasum": "" }, "require": { @@ -3530,7 +3546,7 @@ "shim" ], "support": { - "source": "https://github.com/symfony/polyfill-intl-normalizer/tree/v1.42.0" + "source": "https://github.com/symfony/polyfill-intl-normalizer/tree/v1.43.0" }, "funding": [ { @@ -3550,20 +3566,20 @@ "type": "tidelift" } ], - "time": "2026-08-07T06:33:24+00:00" + "time": "2026-09-23T22:02:09+00:00" }, { "name": "symfony/polyfill-mbstring", - "version": "v1.38.2", + "version": "v1.43.0", "source": { "type": "git", "url": "https://github.com/symfony/polyfill-mbstring.git", - "reference": "d3d318bad5e7a1bfbd026009c8bfb8d8f99ae6b6" + "reference": "4d5d331bf731e0aedbc6b2f5e61135b7b7996a72" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/symfony/polyfill-mbstring/zipball/d3d318bad5e7a1bfbd026009c8bfb8d8f99ae6b6", - "reference": "d3d318bad5e7a1bfbd026009c8bfb8d8f99ae6b6", + "url": "https://api.github.com/repos/symfony/polyfill-mbstring/zipball/4d5d331bf731e0aedbc6b2f5e61135b7b7996a72", + "reference": "4d5d331bf731e0aedbc6b2f5e61135b7b7996a72", "shasum": "" }, "require": { @@ -3615,7 +3631,7 @@ "shim" ], "support": { - "source": "https://github.com/symfony/polyfill-mbstring/tree/v1.38.2" + "source": "https://github.com/symfony/polyfill-mbstring/tree/v1.43.0" }, "funding": [ { @@ -3635,20 +3651,20 @@ "type": "tidelift" } ], - "time": "2026-05-27T06:59:30+00:00" + "time": "2026-09-24T15:19:04+00:00" }, { "name": "symfony/polyfill-php80", - "version": "v1.37.0", + "version": "v1.43.0", "source": { "type": "git", "url": "https://github.com/symfony/polyfill-php80.git", - "reference": "dfb55726c3a76ea3b6459fcfda1ec2d80a682411" + "reference": "9c6a5d6b01ca51d486c813e9a9c0ed55f208bb74" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/symfony/polyfill-php80/zipball/dfb55726c3a76ea3b6459fcfda1ec2d80a682411", - "reference": "dfb55726c3a76ea3b6459fcfda1ec2d80a682411", + "url": "https://api.github.com/repos/symfony/polyfill-php80/zipball/9c6a5d6b01ca51d486c813e9a9c0ed55f208bb74", + "reference": "9c6a5d6b01ca51d486c813e9a9c0ed55f208bb74", "shasum": "" }, "require": { @@ -3699,7 +3715,7 @@ "shim" ], "support": { - "source": "https://github.com/symfony/polyfill-php80/tree/v1.37.0" + "source": "https://github.com/symfony/polyfill-php80/tree/v1.43.0" }, "funding": [ { @@ -3719,20 +3735,20 @@ "type": "tidelift" } ], - "time": "2026-04-10T16:19:22+00:00" + "time": "2026-09-25T15:50:05+00:00" }, { "name": "symfony/polyfill-php82", - "version": "v1.38.1", + "version": "v1.43.0", "source": { "type": "git", "url": "https://github.com/symfony/polyfill-php82.git", - "reference": "002dc0cfe5fd4ed6033d48f27d4f19a486c4b04b" + "reference": "843bbe8b2a7934307b749751eede0f111f5d1cfc" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/symfony/polyfill-php82/zipball/002dc0cfe5fd4ed6033d48f27d4f19a486c4b04b", - "reference": "002dc0cfe5fd4ed6033d48f27d4f19a486c4b04b", + "url": "https://api.github.com/repos/symfony/polyfill-php82/zipball/843bbe8b2a7934307b749751eede0f111f5d1cfc", + "reference": "843bbe8b2a7934307b749751eede0f111f5d1cfc", "shasum": "" }, "require": { @@ -3779,7 +3795,7 @@ "shim" ], "support": { - "source": "https://github.com/symfony/polyfill-php82/tree/v1.38.1" + "source": "https://github.com/symfony/polyfill-php82/tree/v1.43.0" }, "funding": [ { @@ -3799,20 +3815,20 @@ "type": "tidelift" } ], - "time": "2026-05-26T12:45:58+00:00" + "time": "2026-09-25T15:50:05+00:00" }, { "name": "symfony/polyfill-php83", - "version": "v1.41.0", + "version": "v1.43.0", "source": { "type": "git", "url": "https://github.com/symfony/polyfill-php83.git", - "reference": "5ea99087fb99c273a9b9236ed4c31e78b16103c6" + "reference": "80ccff923a8d61f73ebe9da0aa94e25232f004ce" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/symfony/polyfill-php83/zipball/5ea99087fb99c273a9b9236ed4c31e78b16103c6", - "reference": "5ea99087fb99c273a9b9236ed4c31e78b16103c6", + "url": "https://api.github.com/repos/symfony/polyfill-php83/zipball/80ccff923a8d61f73ebe9da0aa94e25232f004ce", + "reference": "80ccff923a8d61f73ebe9da0aa94e25232f004ce", "shasum": "" }, "require": { @@ -3859,7 +3875,7 @@ "shim" ], "support": { - "source": "https://github.com/symfony/polyfill-php83/tree/v1.41.0" + "source": "https://github.com/symfony/polyfill-php83/tree/v1.43.0" }, "funding": [ { @@ -3879,20 +3895,20 @@ "type": "tidelift" } ], - "time": "2026-07-01T12:47:55+00:00" + "time": "2026-09-25T15:50:05+00:00" }, { "name": "symfony/polyfill-php85", - "version": "v1.41.0", + "version": "v1.43.0", "source": { "type": "git", "url": "https://github.com/symfony/polyfill-php85.git", - "reference": "255fab485aaa1006ed411040c42aecd7b5302d7a" + "reference": "86a42cbcc07adc331481a5ab310a4024bfc227f6" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/symfony/polyfill-php85/zipball/255fab485aaa1006ed411040c42aecd7b5302d7a", - "reference": "255fab485aaa1006ed411040c42aecd7b5302d7a", + "url": "https://api.github.com/repos/symfony/polyfill-php85/zipball/86a42cbcc07adc331481a5ab310a4024bfc227f6", + "reference": "86a42cbcc07adc331481a5ab310a4024bfc227f6", "shasum": "" }, "require": { @@ -3939,7 +3955,7 @@ "shim" ], "support": { - "source": "https://github.com/symfony/polyfill-php85/tree/v1.41.0" + "source": "https://github.com/symfony/polyfill-php85/tree/v1.43.0" }, "funding": [ { @@ -3959,20 +3975,20 @@ "type": "tidelift" } ], - "time": "2026-07-01T12:47:55+00:00" + "time": "2026-09-25T15:46:47+00:00" }, { "name": "symfony/polyfill-uuid", - "version": "v1.37.0", + "version": "v1.43.0", "source": { "type": "git", "url": "https://github.com/symfony/polyfill-uuid.git", - "reference": "26dfec253c4cf3e51b541b52ddf7e42cb0908e94" + "reference": "b23ff8245cc5c4479ea2be3d0132d2e6dd433993" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/symfony/polyfill-uuid/zipball/26dfec253c4cf3e51b541b52ddf7e42cb0908e94", - "reference": "26dfec253c4cf3e51b541b52ddf7e42cb0908e94", + "url": "https://api.github.com/repos/symfony/polyfill-uuid/zipball/b23ff8245cc5c4479ea2be3d0132d2e6dd433993", + "reference": "b23ff8245cc5c4479ea2be3d0132d2e6dd433993", "shasum": "" }, "require": { @@ -4022,7 +4038,7 @@ "uuid" ], "support": { - "source": "https://github.com/symfony/polyfill-uuid/tree/v1.37.0" + "source": "https://github.com/symfony/polyfill-uuid/tree/v1.43.0" }, "funding": [ { @@ -4042,7 +4058,7 @@ "type": "tidelift" } ], - "time": "2026-04-10T16:19:22+00:00" + "time": "2026-09-24T05:20:52+00:00" }, { "name": "symfony/service-contracts", @@ -4133,16 +4149,16 @@ }, { "name": "symfony/string", - "version": "v7.4.15", + "version": "v7.4.19", "source": { "type": "git", "url": "https://github.com/symfony/string.git", - "reference": "e394af32256bf9e7bf80849d95e589167c10097b" + "reference": "02731ce2bd26514bec9efc0baef2cd0b0cf67f46" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/symfony/string/zipball/e394af32256bf9e7bf80849d95e589167c10097b", - "reference": "e394af32256bf9e7bf80849d95e589167c10097b", + "url": "https://api.github.com/repos/symfony/string/zipball/02731ce2bd26514bec9efc0baef2cd0b0cf67f46", + "reference": "02731ce2bd26514bec9efc0baef2cd0b0cf67f46", "shasum": "" }, "require": { @@ -4200,7 +4216,7 @@ "utf8" ], "support": { - "source": "https://github.com/symfony/string/tree/v7.4.15" + "source": "https://github.com/symfony/string/tree/v7.4.19" }, "funding": [ { @@ -4220,7 +4236,7 @@ "type": "tidelift" } ], - "time": "2026-07-28T07:33:02+00:00" + "time": "2026-09-11T14:46:58+00:00" }, { "name": "symfony/translation", @@ -4406,16 +4422,16 @@ }, { "name": "symfony/uid", - "version": "v7.4.17", + "version": "v7.4.20", "source": { "type": "git", "url": "https://github.com/symfony/uid.git", - "reference": "69d732355a139c6f8881337d28515aa01f12b8be" + "reference": "a96bbdb02a04ff3b5592cee6741b03edc78b56aa" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/symfony/uid/zipball/69d732355a139c6f8881337d28515aa01f12b8be", - "reference": "69d732355a139c6f8881337d28515aa01f12b8be", + "url": "https://api.github.com/repos/symfony/uid/zipball/a96bbdb02a04ff3b5592cee6741b03edc78b56aa", + "reference": "a96bbdb02a04ff3b5592cee6741b03edc78b56aa", "shasum": "" }, "require": { @@ -4460,7 +4476,7 @@ "uuid" ], "support": { - "source": "https://github.com/symfony/uid/tree/v7.4.17" + "source": "https://github.com/symfony/uid/tree/v7.4.20" }, "funding": [ { @@ -4480,7 +4496,7 @@ "type": "tidelift" } ], - "time": "2026-08-11T07:38:58+00:00" + "time": "2026-09-23T16:18:28+00:00" }, { "name": "symfony/var-dumper", @@ -4868,16 +4884,16 @@ }, { "name": "composer/semver", - "version": "3.4.4", + "version": "3.5.0", "source": { "type": "git", "url": "https://github.com/composer/semver.git", - "reference": "198166618906cb2de69b95d7d47e5fa8aa1b2b95" + "reference": "f7a296f4c4cf8cb8bb83e35d6951a406bb11afa5" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/composer/semver/zipball/198166618906cb2de69b95d7d47e5fa8aa1b2b95", - "reference": "198166618906cb2de69b95d7d47e5fa8aa1b2b95", + "url": "https://api.github.com/repos/composer/semver/zipball/f7a296f4c4cf8cb8bb83e35d6951a406bb11afa5", + "reference": "f7a296f4c4cf8cb8bb83e35d6951a406bb11afa5", "shasum": "" }, "require": { @@ -4919,7 +4935,7 @@ "homepage": "http://robbast.nl" } ], - "description": "Semver library that offers utilities, version constraint parsing and validation.", + "description": "Version comparison library that offers utilities, version constraint parsing and validation.", "keywords": [ "semantic", "semver", @@ -4929,7 +4945,7 @@ "support": { "irc": "ircs://irc.libera.chat:6697/composer", "issues": "https://github.com/composer/semver/issues", - "source": "https://github.com/composer/semver/tree/3.4.4" + "source": "https://github.com/composer/semver/tree/3.5.0" }, "funding": [ { @@ -4941,7 +4957,7 @@ "type": "github" } ], - "time": "2025-08-20T19:15:30+00:00" + "time": "2026-09-24T14:38:51+00:00" }, { "name": "elastic/transport", @@ -5852,11 +5868,11 @@ }, { "name": "phpstan/phpstan", - "version": "2.2.14", + "version": "2.2.16", "dist": { "type": "zip", - "url": "https://api.github.com/repos/phpstan/phpstan/zipball/9c672e7a8e791dfc3d30e55f683e73fc0b63a3ac", - "reference": "9c672e7a8e791dfc3d30e55f683e73fc0b63a3ac", + "url": "https://api.github.com/repos/phpstan/phpstan/zipball/46a6d9060e5a7763adfcc21ebcb8b504ebdbcb92", + "reference": "46a6d9060e5a7763adfcc21ebcb8b504ebdbcb92", "shasum": "" }, "require": { @@ -5912,7 +5928,7 @@ "type": "github" } ], - "time": "2026-09-12T21:39:33+00:00" + "time": "2026-09-25T09:31:51+00:00" }, { "name": "ramsey/collection", @@ -6070,16 +6086,16 @@ }, { "name": "symfony/cache", - "version": "v7.4.18", + "version": "v7.4.20", "source": { "type": "git", "url": "https://github.com/symfony/cache.git", - "reference": "6c521e19a99e8ae57e22aa85b9271534513ee850" + "reference": "e7e604eab58639503495a46f5c194cc69a9c80a8" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/symfony/cache/zipball/6c521e19a99e8ae57e22aa85b9271534513ee850", - "reference": "6c521e19a99e8ae57e22aa85b9271534513ee850", + "url": "https://api.github.com/repos/symfony/cache/zipball/e7e604eab58639503495a46f5c194cc69a9c80a8", + "reference": "e7e604eab58639503495a46f5c194cc69a9c80a8", "shasum": "" }, "require": { @@ -6149,7 +6165,7 @@ "psr6" ], "support": { - "source": "https://github.com/symfony/cache/tree/v7.4.18" + "source": "https://github.com/symfony/cache/tree/v7.4.20" }, "funding": [ { @@ -6169,7 +6185,7 @@ "type": "tidelift" } ], - "time": "2026-08-30T20:10:52+00:00" + "time": "2026-09-29T12:35:16+00:00" }, { "name": "symfony/cache-contracts", @@ -6409,16 +6425,16 @@ }, { "name": "symfony/finder", - "version": "v7.4.17", + "version": "v7.4.20", "source": { "type": "git", "url": "https://github.com/symfony/finder.git", - "reference": "5ce28827081f6d1f0c32eaf3882750f19cb5bbe6" + "reference": "2dd31e5208db7a0fda7bfc998ad727b79c2e4d1e" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/symfony/finder/zipball/5ce28827081f6d1f0c32eaf3882750f19cb5bbe6", - "reference": "5ce28827081f6d1f0c32eaf3882750f19cb5bbe6", + "url": "https://api.github.com/repos/symfony/finder/zipball/2dd31e5208db7a0fda7bfc998ad727b79c2e4d1e", + "reference": "2dd31e5208db7a0fda7bfc998ad727b79c2e4d1e", "shasum": "" }, "require": { @@ -6453,7 +6469,7 @@ "description": "Finds files and directories via an intuitive fluent interface", "homepage": "https://symfony.com", "support": { - "source": "https://github.com/symfony/finder/tree/v7.4.17" + "source": "https://github.com/symfony/finder/tree/v7.4.20" }, "funding": [ { @@ -6473,20 +6489,20 @@ "type": "tidelift" } ], - "time": "2026-08-21T12:09:28+00:00" + "time": "2026-09-26T06:31:10+00:00" }, { "name": "symfony/framework-bundle", - "version": "v7.4.18", + "version": "v7.4.20", "source": { "type": "git", "url": "https://github.com/symfony/framework-bundle.git", - "reference": "45d6d66b6ea1ef7a3c4193dfe606370f0f80c9bf" + "reference": "99786f9338c24770ce4de6a95d049675aa75b129" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/symfony/framework-bundle/zipball/45d6d66b6ea1ef7a3c4193dfe606370f0f80c9bf", - "reference": "45d6d66b6ea1ef7a3c4193dfe606370f0f80c9bf", + "url": "https://api.github.com/repos/symfony/framework-bundle/zipball/99786f9338c24770ce4de6a95d049675aa75b129", + "reference": "99786f9338c24770ce4de6a95d049675aa75b129", "shasum": "" }, "require": { @@ -6540,7 +6556,6 @@ "symfony/workflow": "<7.4" }, "require-dev": { - "doctrine/persistence": "^1.3|^2|^3", "dragonmantank/cron-expression": "^3.1", "phpdocumentor/reflection-docblock": "^5.2|^6.0", "seld/jsonlint": "^1.10", @@ -6611,7 +6626,7 @@ "description": "Provides a tight integration between Symfony components and the Symfony full-stack framework", "homepage": "https://symfony.com", "support": { - "source": "https://github.com/symfony/framework-bundle/tree/v7.4.18" + "source": "https://github.com/symfony/framework-bundle/tree/v7.4.20" }, "funding": [ { @@ -6631,20 +6646,20 @@ "type": "tidelift" } ], - "time": "2026-08-30T20:10:52+00:00" + "time": "2026-09-28T19:09:50+00:00" }, { "name": "symfony/messenger", - "version": "v7.4.18", + "version": "v7.4.20", "source": { "type": "git", "url": "https://github.com/symfony/messenger.git", - "reference": "b109f9f4e54470ee6835751bc39318fe3d8275e6" + "reference": "4cf94f12c9e899a87e6b6bf800dffcb709585410" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/symfony/messenger/zipball/b109f9f4e54470ee6835751bc39318fe3d8275e6", - "reference": "b109f9f4e54470ee6835751bc39318fe3d8275e6", + "url": "https://api.github.com/repos/symfony/messenger/zipball/4cf94f12c9e899a87e6b6bf800dffcb709585410", + "reference": "4cf94f12c9e899a87e6b6bf800dffcb709585410", "shasum": "" }, "require": { @@ -6705,7 +6720,7 @@ "description": "Helps applications send and receive messages to/from other applications or via message queues", "homepage": "https://symfony.com", "support": { - "source": "https://github.com/symfony/messenger/tree/v7.4.18" + "source": "https://github.com/symfony/messenger/tree/v7.4.20" }, "funding": [ { @@ -6725,7 +6740,7 @@ "type": "tidelift" } ], - "time": "2026-08-30T00:47:26+00:00" + "time": "2026-09-29T06:38:53+00:00" }, { "name": "symfony/options-resolver", @@ -6876,16 +6891,16 @@ }, { "name": "symfony/polyfill-php84", - "version": "v1.38.1", + "version": "v1.43.0", "source": { "type": "git", "url": "https://github.com/symfony/polyfill-php84.git", - "reference": "f4e1dfaee5b74aba5964fe1fd4dfc7ba5e3085fa" + "reference": "22c6548499cbe5c5bb072fc0c42312dff062a015" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/symfony/polyfill-php84/zipball/f4e1dfaee5b74aba5964fe1fd4dfc7ba5e3085fa", - "reference": "f4e1dfaee5b74aba5964fe1fd4dfc7ba5e3085fa", + "url": "https://api.github.com/repos/symfony/polyfill-php84/zipball/22c6548499cbe5c5bb072fc0c42312dff062a015", + "reference": "22c6548499cbe5c5bb072fc0c42312dff062a015", "shasum": "" }, "require": { @@ -6932,7 +6947,7 @@ "shim" ], "support": { - "source": "https://github.com/symfony/polyfill-php84/tree/v1.38.1" + "source": "https://github.com/symfony/polyfill-php84/tree/v1.43.0" }, "funding": [ { @@ -6952,20 +6967,20 @@ "type": "tidelift" } ], - "time": "2026-05-26T12:51:13+00:00" + "time": "2026-09-23T22:01:09+00:00" }, { "name": "symfony/process", - "version": "v7.4.18", + "version": "v7.4.19", "source": { "type": "git", "url": "https://github.com/symfony/process.git", - "reference": "058d17fc284cce14efb2385783b55014a461b176" + "reference": "ed0ae095b86994d370d5791612e55984f15aa30e" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/symfony/process/zipball/058d17fc284cce14efb2385783b55014a461b176", - "reference": "058d17fc284cce14efb2385783b55014a461b176", + "url": "https://api.github.com/repos/symfony/process/zipball/ed0ae095b86994d370d5791612e55984f15aa30e", + "reference": "ed0ae095b86994d370d5791612e55984f15aa30e", "shasum": "" }, "require": { @@ -6997,7 +7012,7 @@ "description": "Executes commands in sub-processes", "homepage": "https://symfony.com", "support": { - "source": "https://github.com/symfony/process/tree/v7.4.18" + "source": "https://github.com/symfony/process/tree/v7.4.19" }, "funding": [ { @@ -7017,7 +7032,7 @@ "type": "tidelift" } ], - "time": "2026-08-21T17:40:08+00:00" + "time": "2026-09-02T12:38:34+00:00" }, { "name": "symfony/property-access", @@ -7102,16 +7117,16 @@ }, { "name": "symfony/property-info", - "version": "v7.4.18", + "version": "v7.4.20", "source": { "type": "git", "url": "https://github.com/symfony/property-info.git", - "reference": "52420d499b03ec20c0d38f0dc2f940bc9c43dc8e" + "reference": "d24c4c810b1285c5d8d77c6fd7b25e714479bd48" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/symfony/property-info/zipball/52420d499b03ec20c0d38f0dc2f940bc9c43dc8e", - "reference": "52420d499b03ec20c0d38f0dc2f940bc9c43dc8e", + "url": "https://api.github.com/repos/symfony/property-info/zipball/d24c4c810b1285c5d8d77c6fd7b25e714479bd48", + "reference": "d24c4c810b1285c5d8d77c6fd7b25e714479bd48", "shasum": "" }, "require": { @@ -7168,7 +7183,7 @@ "validator" ], "support": { - "source": "https://github.com/symfony/property-info/tree/v7.4.18" + "source": "https://github.com/symfony/property-info/tree/v7.4.20" }, "funding": [ { @@ -7188,20 +7203,20 @@ "type": "tidelift" } ], - "time": "2026-08-25T14:18:37+00:00" + "time": "2026-09-28T09:47:41+00:00" }, { "name": "symfony/routing", - "version": "v7.4.18", + "version": "v7.4.20", "source": { "type": "git", "url": "https://github.com/symfony/routing.git", - "reference": "ddd558991e98f693ae6bf5063cc1b0362c6bbec3" + "reference": "2bacb72be5eadeaf9874d5accb75bfb33d21c74b" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/symfony/routing/zipball/ddd558991e98f693ae6bf5063cc1b0362c6bbec3", - "reference": "ddd558991e98f693ae6bf5063cc1b0362c6bbec3", + "url": "https://api.github.com/repos/symfony/routing/zipball/2bacb72be5eadeaf9874d5accb75bfb33d21c74b", + "reference": "2bacb72be5eadeaf9874d5accb75bfb33d21c74b", "shasum": "" }, "require": { @@ -7253,7 +7268,7 @@ "url" ], "support": { - "source": "https://github.com/symfony/routing/tree/v7.4.18" + "source": "https://github.com/symfony/routing/tree/v7.4.20" }, "funding": [ { @@ -7273,20 +7288,20 @@ "type": "tidelift" } ], - "time": "2026-08-17T13:12:36+00:00" + "time": "2026-09-28T19:09:50+00:00" }, { "name": "symfony/security-bundle", - "version": "v7.4.18", + "version": "v7.4.20", "source": { "type": "git", "url": "https://github.com/symfony/security-bundle.git", - "reference": "caaa8b92f2657a6b99057f4da36ee89f3baab506" + "reference": "a2a8dc7ab9d5d90d5937963a4ef1b9c8e3e7258c" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/symfony/security-bundle/zipball/caaa8b92f2657a6b99057f4da36ee89f3baab506", - "reference": "caaa8b92f2657a6b99057f4da36ee89f3baab506", + "url": "https://api.github.com/repos/symfony/security-bundle/zipball/a2a8dc7ab9d5d90d5937963a4ef1b9c8e3e7258c", + "reference": "a2a8dc7ab9d5d90d5937963a4ef1b9c8e3e7258c", "shasum": "" }, "require": { @@ -7327,6 +7342,7 @@ "symfony/framework-bundle": "^6.4.13|^7.1.6|^8.0", "symfony/http-client": "^6.4|^7.0|^8.0", "symfony/ldap": "^6.4|^7.0|^8.0", + "symfony/lock": "^6.4|^7.0|^8.0", "symfony/process": "^6.4|^7.0|^8.0", "symfony/rate-limiter": "^6.4|^7.0|^8.0", "symfony/runtime": "^6.4.13|^7.1.6|^8.0", @@ -7365,7 +7381,7 @@ "description": "Provides a tight integration of the Security component into the Symfony full-stack framework", "homepage": "https://symfony.com", "support": { - "source": "https://github.com/symfony/security-bundle/tree/v7.4.18" + "source": "https://github.com/symfony/security-bundle/tree/v7.4.20" }, "funding": [ { @@ -7385,20 +7401,20 @@ "type": "tidelift" } ], - "time": "2026-08-30T17:23:54+00:00" + "time": "2026-09-24T11:02:13+00:00" }, { "name": "symfony/security-core", - "version": "v7.4.18", + "version": "v7.4.20", "source": { "type": "git", "url": "https://github.com/symfony/security-core.git", - "reference": "4fac226354298b8d66f680275b0daef1d3f580d2" + "reference": "4e5c01656aa79e6d463fde45117041ebccbaf829" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/symfony/security-core/zipball/4fac226354298b8d66f680275b0daef1d3f580d2", - "reference": "4fac226354298b8d66f680275b0daef1d3f580d2", + "url": "https://api.github.com/repos/symfony/security-core/zipball/4e5c01656aa79e6d463fde45117041ebccbaf829", + "reference": "4e5c01656aa79e6d463fde45117041ebccbaf829", "shasum": "" }, "require": { @@ -7457,7 +7473,7 @@ "description": "Symfony Security Component - Core Library", "homepage": "https://symfony.com", "support": { - "source": "https://github.com/symfony/security-core/tree/v7.4.18" + "source": "https://github.com/symfony/security-core/tree/v7.4.20" }, "funding": [ { @@ -7477,7 +7493,7 @@ "type": "tidelift" } ], - "time": "2026-08-30T00:47:26+00:00" + "time": "2026-09-28T16:00:27+00:00" }, { "name": "symfony/security-csrf", @@ -7555,16 +7571,16 @@ }, { "name": "symfony/security-http", - "version": "v7.4.18", + "version": "v7.4.20", "source": { "type": "git", "url": "https://github.com/symfony/security-http.git", - "reference": "ca575e23a2b5e068ce0cdf157767b63b438125e0" + "reference": "5fa566b8bfbb398cded6988f059c3d97506ed692" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/symfony/security-http/zipball/ca575e23a2b5e068ce0cdf157767b63b438125e0", - "reference": "ca575e23a2b5e068ce0cdf157767b63b438125e0", + "url": "https://api.github.com/repos/symfony/security-http/zipball/5fa566b8bfbb398cded6988f059c3d97506ed692", + "reference": "5fa566b8bfbb398cded6988f059c3d97506ed692", "shasum": "" }, "require": { @@ -7623,7 +7639,7 @@ "description": "Symfony Security Component - HTTP Integration", "homepage": "https://symfony.com", "support": { - "source": "https://github.com/symfony/security-http/tree/v7.4.18" + "source": "https://github.com/symfony/security-http/tree/v7.4.20" }, "funding": [ { @@ -7643,20 +7659,20 @@ "type": "tidelift" } ], - "time": "2026-08-30T17:23:54+00:00" + "time": "2026-09-28T18:33:58+00:00" }, { "name": "symfony/serializer", - "version": "v7.4.18", + "version": "v7.4.20", "source": { "type": "git", "url": "https://github.com/symfony/serializer.git", - "reference": "1f31d4e902c85b820af5dce27dbe17435524bed0" + "reference": "67c5b0b2ef39863835f43b9c34dcb3af80b892cf" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/symfony/serializer/zipball/1f31d4e902c85b820af5dce27dbe17435524bed0", - "reference": "1f31d4e902c85b820af5dce27dbe17435524bed0", + "url": "https://api.github.com/repos/symfony/serializer/zipball/67c5b0b2ef39863835f43b9c34dcb3af80b892cf", + "reference": "67c5b0b2ef39863835f43b9c34dcb3af80b892cf", "shasum": "" }, "require": { @@ -7727,7 +7743,7 @@ "description": "Handles serializing and deserializing data structures, including object graphs, into array structures or other formats like XML and JSON.", "homepage": "https://symfony.com", "support": { - "source": "https://github.com/symfony/serializer/tree/v7.4.18" + "source": "https://github.com/symfony/serializer/tree/v7.4.20" }, "funding": [ { @@ -7747,20 +7763,20 @@ "type": "tidelift" } ], - "time": "2026-08-30T00:47:26+00:00" + "time": "2026-09-28T09:39:38+00:00" }, { "name": "symfony/twig-bridge", - "version": "v7.4.17", + "version": "v7.4.19", "source": { "type": "git", "url": "https://github.com/symfony/twig-bridge.git", - "reference": "413485ee177ad5b321ad9a689a90fd3f2d3fefd7" + "reference": "10d369dd3845bb650a9450e643d6d9027e8a86af" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/symfony/twig-bridge/zipball/413485ee177ad5b321ad9a689a90fd3f2d3fefd7", - "reference": "413485ee177ad5b321ad9a689a90fd3f2d3fefd7", + "url": "https://api.github.com/repos/symfony/twig-bridge/zipball/10d369dd3845bb650a9450e643d6d9027e8a86af", + "reference": "10d369dd3845bb650a9450e643d6d9027e8a86af", "shasum": "" }, "require": { @@ -7842,7 +7858,7 @@ "description": "Provides integration for Twig with various Symfony components", "homepage": "https://symfony.com", "support": { - "source": "https://github.com/symfony/twig-bridge/tree/v7.4.17" + "source": "https://github.com/symfony/twig-bridge/tree/v7.4.19" }, "funding": [ { @@ -7862,20 +7878,20 @@ "type": "tidelift" } ], - "time": "2026-08-22T09:04:42+00:00" + "time": "2026-09-14T12:19:06+00:00" }, { "name": "symfony/twig-bundle", - "version": "v7.4.15", + "version": "v7.4.19", "source": { "type": "git", "url": "https://github.com/symfony/twig-bundle.git", - "reference": "e3d2bea0b594aa50c3482a278a5cd09d83d94517" + "reference": "231c0b37dada053458d47281261a3af0270a634d" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/symfony/twig-bundle/zipball/e3d2bea0b594aa50c3482a278a5cd09d83d94517", - "reference": "e3d2bea0b594aa50c3482a278a5cd09d83d94517", + "url": "https://api.github.com/repos/symfony/twig-bundle/zipball/231c0b37dada053458d47281261a3af0270a634d", + "reference": "231c0b37dada053458d47281261a3af0270a634d", "shasum": "" }, "require": { @@ -7932,7 +7948,7 @@ "description": "Provides a tight integration of Twig into the Symfony full-stack framework", "homepage": "https://symfony.com", "support": { - "source": "https://github.com/symfony/twig-bundle/tree/v7.4.15" + "source": "https://github.com/symfony/twig-bundle/tree/v7.4.19" }, "funding": [ { @@ -7952,20 +7968,20 @@ "type": "tidelift" } ], - "time": "2026-07-21T15:13:06+00:00" + "time": "2026-09-11T14:46:58+00:00" }, { "name": "symfony/type-info", - "version": "v7.4.17", + "version": "v7.4.20", "source": { "type": "git", "url": "https://github.com/symfony/type-info.git", - "reference": "62d0ad6995630a4f7ea8fd270aadb3e2545c8e31" + "reference": "bbaaf3366400f5c85f66800b955c77b46e8abad2" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/symfony/type-info/zipball/62d0ad6995630a4f7ea8fd270aadb3e2545c8e31", - "reference": "62d0ad6995630a4f7ea8fd270aadb3e2545c8e31", + "url": "https://api.github.com/repos/symfony/type-info/zipball/bbaaf3366400f5c85f66800b955c77b46e8abad2", + "reference": "bbaaf3366400f5c85f66800b955c77b46e8abad2", "shasum": "" }, "require": { @@ -8015,7 +8031,7 @@ "type" ], "support": { - "source": "https://github.com/symfony/type-info/tree/v7.4.17" + "source": "https://github.com/symfony/type-info/tree/v7.4.20" }, "funding": [ { @@ -8035,20 +8051,20 @@ "type": "tidelift" } ], - "time": "2026-08-21T15:52:48+00:00" + "time": "2026-09-28T12:19:37+00:00" }, { "name": "symfony/web-profiler-bundle", - "version": "v7.4.17", + "version": "v7.4.20", "source": { "type": "git", "url": "https://github.com/symfony/web-profiler-bundle.git", - "reference": "3abb070d460c61924c23edfda9e2e5af34e43a19" + "reference": "916c2a4edf51270a5173a57d8d3936c02a4543a1" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/symfony/web-profiler-bundle/zipball/3abb070d460c61924c23edfda9e2e5af34e43a19", - "reference": "3abb070d460c61924c23edfda9e2e5af34e43a19", + "url": "https://api.github.com/repos/symfony/web-profiler-bundle/zipball/916c2a4edf51270a5173a57d8d3936c02a4543a1", + "reference": "916c2a4edf51270a5173a57d8d3936c02a4543a1", "shasum": "" }, "require": { @@ -8105,7 +8121,7 @@ "dev" ], "support": { - "source": "https://github.com/symfony/web-profiler-bundle/tree/v7.4.17" + "source": "https://github.com/symfony/web-profiler-bundle/tree/v7.4.20" }, "funding": [ { @@ -8125,20 +8141,20 @@ "type": "tidelift" } ], - "time": "2026-08-22T09:04:42+00:00" + "time": "2026-09-28T19:09:50+00:00" }, { "name": "symfony/yaml", - "version": "v7.4.18", + "version": "v7.4.20", "source": { "type": "git", "url": "https://github.com/symfony/yaml.git", - "reference": "4cef939e55b8a21c5780418de0d98ab00d0736e1" + "reference": "0222bf4dfaa1e3ceb8fff579635d655f2950d5a7" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/symfony/yaml/zipball/4cef939e55b8a21c5780418de0d98ab00d0736e1", - "reference": "4cef939e55b8a21c5780418de0d98ab00d0736e1", + "url": "https://api.github.com/repos/symfony/yaml/zipball/0222bf4dfaa1e3ceb8fff579635d655f2950d5a7", + "reference": "0222bf4dfaa1e3ceb8fff579635d655f2950d5a7", "shasum": "" }, "require": { @@ -8181,7 +8197,7 @@ "description": "Loads and dumps YAML files", "homepage": "https://symfony.com", "support": { - "source": "https://github.com/symfony/yaml/tree/v7.4.18" + "source": "https://github.com/symfony/yaml/tree/v7.4.20" }, "funding": [ { @@ -8201,20 +8217,20 @@ "type": "tidelift" } ], - "time": "2026-08-30T00:47:26+00:00" + "time": "2026-09-26T06:31:10+00:00" }, { "name": "twig/twig", - "version": "v3.29.0", + "version": "v3.30.0", "source": { "type": "git", "url": "https://github.com/twigphp/Twig.git", - "reference": "45a3c6e9224c3377a39c7b150bb29d5d97d2c75d" + "reference": "8c737079b726af72ff8ef3c595be9f6a810ea1ef" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/twigphp/Twig/zipball/45a3c6e9224c3377a39c7b150bb29d5d97d2c75d", - "reference": "45a3c6e9224c3377a39c7b150bb29d5d97d2c75d", + "url": "https://api.github.com/repos/twigphp/Twig/zipball/8c737079b726af72ff8ef3c595be9f6a810ea1ef", + "reference": "8c737079b726af72ff8ef3c595be9f6a810ea1ef", "shasum": "" }, "require": { @@ -8269,7 +8285,7 @@ ], "support": { "issues": "https://github.com/twigphp/Twig/issues", - "source": "https://github.com/twigphp/Twig/tree/v3.29.0" + "source": "https://github.com/twigphp/Twig/tree/v3.30.0" }, "funding": [ { @@ -8281,7 +8297,7 @@ "type": "tidelift" } ], - "time": "2026-09-18T09:10:14+00:00" + "time": "2026-09-25T13:20:01+00:00" } ], "aliases": [], diff --git a/documentation/components/adapters/csv.md b/documentation/components/adapters/csv.md index e18e9aefb3..b5f988c396 100644 --- a/documentation/components/adapters/csv.md +++ b/documentation/components/adapters/csv.md @@ -33,20 +33,58 @@ $rows = data_frame() ->fetch(); ``` +### Header and missing cells + +``` +header row [' id ', '', 'name'] → columns ['id', 'e01', 'name'] (trimmed, a blank cell named e + its position) +record ['1', 'a'] → ['id' => 1, 'e01' => 'a', 'name' => null] (a cell the record lacks is null) +record ['1', '', 'x'] → 'e01' => null (a present '' is null; `withEmptyToNull(false)` keeps it '') +``` + ## Loader -```php +```php read(from_rows( - rows( - schema(int_schema('id'), str_schema('name')), - row(['id' => 1, 'name' => 'Norbert']), - row(['id' => 2, 'name' => 'Tomek']), - row(['id' => 3, 'name' => 'Dawid']), - ) + ->read(from_array( + [ + ['id' => 1, 'active' => true, 'price' => 0.1 + 0.2, 'at' => new DateTimeImmutable('2026-01-02 03:04:05 UTC'), 'days' => [new DateTimeImmutable('2026-01-02')], 'labels' => [0 => 'a', 1 => 'b'], 'size' => ['w' => 1, 'h' => 2], 'meta' => '{"tags": []}'], + ['id' => 2, 'active' => false, 'price' => 1.0, 'at' => null, 'days' => [], 'labels' => [], 'size' => ['w' => 3, 'h' => 4], 'meta' => '{}'], + ], + schema( + int_schema('id'), bool_schema('active'), float_schema('price'), datetime_schema('at', nullable: true), + list_schema('days', type_list(type_date())), map_schema('labels', type_map(type_integer(), type_string())), + structure_schema('size', type_structure(['w' => type_integer(), 'h' => type_integer()])), json_schema('meta'), + ), )) - ->load(to_csv($path, true, true)) + ->write(to_csv($path)) ->run(); ``` + +```csv +id,active,price,at,days,labels,size,meta +1,true,0.30000000000000004,2026-01-02T03:04:05+00:00,"[""2026-01-02""]","{""0"":""a"",""1"":""b""}","{""w"":1,""h"":2}","{""tags"": []}" +2,false,1.0,,[],{},"{""w"":3,""h"":4}",{} +``` + +| Column | Field | +|----------------------------|------------------------------------------------------------------------------------------------------------------------------------| +| `boolean` | `true` / `false` | +| `float` | the shortest text that reads back as the same float, always with a fraction or an exponent: `1.0`, `1.0e+25`; `NAN`, `INF`, `-INF` | +| `null`, `''` | an empty field - `from_csv()` reads both back as `null` unless `withEmptyToNull(false)` | +| `datetime`, `date` | `withDateTimeFormat()` (default `DATE_ATOM`) in the column zone, `withDateFormat()` (default `Y-m-d`) | +| `time` | microseconds | +| `list`, `map`, `structure` | JSON: a list is `[...]`, a map and a structure are always `{...}`; every element is written as a column of its type would be | +| `json` | the stored text | + +Floats are rendered under `serialize_precision = -1`. When the ini holds another value the writer sets it for the +render and restores it, so `ini_set()` must be allowed; otherwise it throws +`RuntimeException('Writing floats requires serialize_precision = -1 and ini_set() cannot change it')`. + +With the [`flow_php`](/documentation/components/extensions/flow-php-ext.md) extension loaded the same bytes are +rendered natively. diff --git a/documentation/components/adapters/doctrine.md b/documentation/components/adapters/doctrine.md index f2830d4353..e7651d6f8c 100644 --- a/documentation/components/adapters/doctrine.md +++ b/documentation/components/adapters/doctrine.md @@ -31,11 +31,16 @@ Adapter for [ETL](https://github.com/flow-php/etl) using bulk operations from [D ## Loader - DbalLoader ```php +use Doctrine\DBAL\Tools\DsnParser; + use function Flow\ETL\Adapter\Doctrine\to_dbal_table_insert; +use function Flow\ETL\DSL\{data_frame, from_array}; + +$params = (new DsnParser(['postgresql' => 'pdo_pgsql']))->parse(\getenv('PGSQL_DATABASE_URL')); data_frame() ->read(from_array([['id' => 1, 'name' => 'norbert']])) - ->write(to_dbal_table_insert(['url' => \getenv('PGSQL_DATABASE_URL')], 'your-table-name')) + ->write(to_dbal_table_insert($params, 'users')) ->run(); ``` @@ -48,7 +53,8 @@ All supported DbalLoader operations via DSL functions: You can also configure bulk operations with platform-specific options: ```php -use function Flow\ETL\Adapter\Doctrine\{to_dbal_table_insert, postgresql_insert_options}; +use function Flow\ETL\Adapter\Doctrine\{postgresql_insert_options, to_dbal_table_insert}; +use function Flow\ETL\DSL\{data_frame, from_array}; data_frame() ->read(from_array([['id' => 1, 'name' => 'norbert']])) @@ -60,36 +66,36 @@ data_frame() ->run(); ``` -### Type Detection and Optimization - -`DbalLoader` now provides advanced type detection capabilities to optimize database operations: +### Types -#### Automatic Type Detection from Flow Schema - -By default, `DbalLoader` automatically detects column types from the Flow Schema of your data: +By default, `DbalLoader` binds every column with the DBAL type mapped from its Flow type +(the default map is listed under [Schema Converter - Types Map](#schema-converter---types-map)): ```php use function Flow\ETL\Adapter\Doctrine\to_dbal_table_insert; +use function Flow\ETL\DSL\{data_frame, from_array}; data_frame() ->read(from_array([['id' => 1, 'name' => 'norbert']])) ->write(to_dbal_table_insert($connection, 'users')) ->run(); -// Types are automatically detected from the Flow Schema ``` #### Custom Types Map -For advanced scenarios, you can provide a custom type mapping to control how Flow types are converted to DBAL types: +A custom map replaces the default one, so spread `TypesMap::FLOW_TYPES` to keep the other types: ```php -use function Flow\ETL\Adapter\Doctrine\to_dbal_table_insert; +use Doctrine\DBAL\Types\TextType; use Flow\ETL\Adapter\Doctrine\TypesMap; use Flow\Types\Type\Native\StringType; -use Doctrine\DBAL\Types\TextType; + +use function Flow\ETL\Adapter\Doctrine\to_dbal_table_insert; +use function Flow\ETL\DSL\{data_frame, from_array}; $customTypesMap = new TypesMap([ - StringType::class => TextType::class, // Map Flow strings to DBAL text type + ...TypesMap::FLOW_TYPES, + StringType::class => TextType::class, ]); data_frame() @@ -99,26 +105,23 @@ data_frame() ->run(); ``` -#### Data Normalization +#### Bound values -`DbalLoader` automatically handles data normalization for database compatibility: - -- **XML Entries**: `XMLEntry` and `XMLElementEntry` objects are automatically converted to their string representation -- **Complex Types**: Lists, Maps, and Structures are serialized as JSON -- **Type Safety**: All data is normalized while preserving type information for optimal database performance +A column is bound as its values and converted by its DBAL type: `list`, `map` and `structure` become JSON through +`JsonType`. `xml`, `xml_element`, `html` and `html_element` are bound as their markup text: ```php -use function Flow\ETL\DSL\{data_frame, from_array, xml_entry}; use function Flow\ETL\Adapter\Doctrine\to_dbal_table_insert; +use function Flow\ETL\DSL\{data_frame, from_array, int_schema, schema, xml_schema}; data_frame() - ->read(from_array([ - ['id' => 1, 'data' => $domDocument], - ['id' => 2, 'data' => $domElement], - ])) + ->read(from_array( + [['id' => 1, 'data' => '1']], + schema(int_schema('id'), xml_schema('data')), + )) ->write(to_dbal_table_insert($connection, 'xml_table')) ->run(); -// XML entries are automatically converted to strings before database insertion +// data is bound as '1' ``` ## Transactional Loading @@ -127,11 +130,11 @@ data_frame() transaction, and if any sink throws, the open transaction is rolled back: ```php -use function Flow\ETL\DSL\{data_frame, from_array}; use function Flow\ETL\Adapter\Doctrine\{to_dbal_table_insert, to_dbal_transaction}; +use function Flow\ETL\DSL\{data_frame, from_array}; data_frame() - ->read(from_array($data)) + ->read(from_array([['id' => 1, 'name' => 'norbert']])) ->write(to_dbal_transaction( $connection, to_dbal_table_insert($connection, 'users'), @@ -143,7 +146,7 @@ data_frame() A plain loader child is a bare sink root; a `to_transformation(...)` child delivers inside the same transaction. Every child's loader must use the same connection as the transaction - pass one live `Connection` to `to_dbal_transaction()` and to every child. A loader built from array params (like -`to_dbal_table_insert(['url' => $url], 'users')`) opens its own connection and escapes the transaction. +`to_dbal_table_insert($params, 'users')`) opens its own connection and escapes the transaction. Sinks with blocking operations (`to_transformation()` / `to_branch(...)->withTransformation(...)` with `sortBy()`, `aggregate()`, `groupBy()->aggregate()`, `pivot()`, window functions, `collect()`, `join()` - see @@ -162,6 +165,8 @@ use Doctrine\DBAL\TransactionIsolationLevel; use Flow\ETL\Adapter\Doctrine\DbalTransaction; use Flow\ETL\Sink\Transactional; +use function Flow\ETL\Adapter\Doctrine\to_dbal_table_insert; + new Transactional( DbalTransaction::fromConnection($connection)->withIsolationLevel(TransactionIsolationLevel::SERIALIZABLE), to_dbal_table_insert($connection, 'users'), @@ -170,55 +175,56 @@ new Transactional( ## Extractor - DbalQuery -This simple but powerful extractor let you extract data from a single or multiple parametrized queries. - `from_dbal_*()` extractors derive their schema from the query unless `->withSchema()` declares it - how that works and fails: [Doctrine DBAL sources](../core/schema.md#doctrine-dbal-sources-describe-themselves-and-keep-withschema). ### Single Query -```php + +```php +use function Flow\ETL\Adapter\Doctrine\from_dbal_query; use function Flow\ETL\DSL\{data_frame, to_output}; data_frame() - ->read(DbalQueryExtractor::singleQuery($connection, "SELECT * FROM {$table} ORDER BY id")) + ->read(from_dbal_query($connection, 'SELECT * FROM users ORDER BY id')) ->write(to_output()) ->run(); ``` ### Single Parametrized Query -```php +```php +use function Flow\ETL\Adapter\Doctrine\from_dbal_query; use function Flow\ETL\DSL\{data_frame, to_output}; data_frame() - ->read(DbalQueryExtractor::singleQuery($connection, "SELECT * FROM {$table} WHERE id = :id", ['id' => 1])) + ->read(from_dbal_query($connection, 'SELECT * FROM users WHERE id = :id', ['id' => 1])) ->write(to_output()) ->run(); ``` -### Multiple Parametrized Query -```php ignore +### Multiple Parametrized Queries + +```php +use Flow\ETL\Adapter\Doctrine\ParametersSet; + +use function Flow\ETL\Adapter\Doctrine\from_dbal_queries; use function Flow\ETL\DSL\{data_frame, to_output}; data_frame() - ->read( - new DbalQueryExtractor( - $connection, - "SELECT * FROM {$table} ORDER BY id LIMIT :limit OFFSET :offset", - new ParametersSet( - ['limit' => 2, 'offset' => 0], - ['limit' => 2, 'offset' => 2], - ['limit' => 2, 'offset' => 4], - ['limit' => 2, 'offset' => 6], - ['limit' => 2, 'offset' => 8], - ) - ) - ) + ->read(from_dbal_queries( + $connection, + 'SELECT * FROM users ORDER BY id LIMIT :limit OFFSET :offset', + new ParametersSet( + ['limit' => 2, 'offset' => 0], + ['limit' => 2, 'offset' => 2], + ['limit' => 2, 'offset' => 4], + ), + )) ->write(to_output()) - ->run() + ->run(); ``` -In this case, query will be executed exactly five times, taking every time next entry of parameters from ParametersSet. +The query runs once per parameter set, three times here. ## Schema Converter @@ -230,6 +236,8 @@ like length, primary key, index, precision, etc ```php +use Flow\ETL\Adapter\Doctrine\DbalMetadata; + use function Flow\ETL\DSL\bool_schema; use function Flow\ETL\DSL\date_schema; use function Flow\ETL\DSL\float_schema; @@ -273,20 +281,21 @@ Will generate: use Doctrine\DBAL\Schema\Column; use Doctrine\DBAL\Schema\Index; use Doctrine\DBAL\Schema\Table; +use Doctrine\DBAL\Types\Type; new Table( 'test', [ new Column('int', Type::getType('integer'), ['notnull' => true]), - new Column('str', Type::getType('string'), ['notnull' => false]), + new Column('str', Type::getType('string'), ['notnull' => true]), new Column('str_with_length', Type::getType('string'), ['notnull' => false, 'length' => 255]), new Column('str_unique', Type::getType('string'), ['notnull' => false]), + new Column('date', Type::getType('date_immutable'), ['notnull' => false]), new Column('float', Type::getType('float'), ['notnull' => false, 'precision' => 10, 'scale' => 2]), new Column('bool', Type::getType('boolean'), ['notnull' => false, 'default' => true]), new Column('json', Type::getType('json'), ['notnull' => false, 'platformOptions' => ['jsonb' => true]]), new Column('list', Type::getType('json'), ['notnull' => true, 'columnDefinition' => 'integer[]']), new Column('map', Type::getType('json'), ['notnull' => true, 'comment' => 'test comment!']), - new Column('date', Type::getType('date_immutable'), ['notnull' => false]), ], [ new Index('pk_test', ['int', 'str'], true, true), @@ -301,7 +310,7 @@ new Table( When types map is not provided, the default one will be used: ```php ignore -public const FLOW_TYPES = [ +public const array FLOW_TYPES = [ StringType::class => \Doctrine\DBAL\Types\StringType::class, IntegerType::class => \Doctrine\DBAL\Types\IntegerType::class, FloatType::class => \Doctrine\DBAL\Types\FloatType::class, @@ -310,9 +319,13 @@ public const FLOW_TYPES = [ TimeType::class => \Doctrine\DBAL\Types\TimeImmutableType::class, DateTimeType::class => \Doctrine\DBAL\Types\DateTimeImmutableType::class, UuidType::class => \Doctrine\DBAL\Types\GuidType::class, + TimeZoneType::class => \Doctrine\DBAL\Types\StringType::class, JsonType::class => \Doctrine\DBAL\Types\JsonType::class, XMLType::class => \Doctrine\DBAL\Types\StringType::class, XMLElementType::class => \Doctrine\DBAL\Types\StringType::class, + HTMLType::class => \Doctrine\DBAL\Types\StringType::class, + HTMLElementType::class => \Doctrine\DBAL\Types\StringType::class, + EnumType::class => \Doctrine\DBAL\Types\StringType::class, ListType::class => \Doctrine\DBAL\Types\JsonType::class, MapType::class => \Doctrine\DBAL\Types\JsonType::class, StructureType::class => \Doctrine\DBAL\Types\JsonType::class, diff --git a/documentation/components/adapters/excel.md b/documentation/components/adapters/excel.md index 32696c41dd..6d34b653e4 100644 --- a/documentation/components/adapters/excel.md +++ b/documentation/components/adapters/excel.md @@ -71,6 +71,16 @@ $rows = data_frame() Row numbers count from 1 and include the header row, so `->withOffset(5)` on a sheet with a header skips rows 2-4 and starts at row 5. +### Header and missing cells + +``` +header row [' id ', '', 'name'] → columns ['id', 'e01', 'name'] (trimmed, a blank cell named e + its position) +record ['1', 'a'] → ['id' => '1', 'e01' => 'a', 'name' => null] (a cell the record lacks is null) +record ['1', '', 'x'] → 'e01' => null (a present '' is null; `withConvertEmptyToNull(false)` keeps it '') +``` + +A header cell that is not text (a date, a number) is named like a blank one. + ### Schema Without a declared schema, `from_excel()` samples the workbook before the first row is read, decides one schema, and @@ -168,9 +178,9 @@ data_frame() ->run(); ``` -### Dynamic Sheet Names from Entry +### Sheet Name from a Column -Route rows to different sheets based on entry values: +Write each row to the sheet named by its `category` value: ```php run(); ``` +### Cells + +| Column | Cell | +|----------------------------------------------|----------------------------------------------------------------------------------------| +| `string`, `integer`, `float`, `boolean` | the value | +| `datetime`, `date` | a date cell, displayed with `withDateTimeFormat()` / `withDateFormat()` | +| `time` | text, `withTimeFormat()` | +| `enum` | a backed enum's value, a unit enum's name | +| `uuid`, `json` | text | +| `xml`, `xml_element`, `html`, `html_element` | the markup, e.g. `` | +| `list`, `map`, `structure` | JSON text: a datetime element as `DATE_ATOM`, a time as microseconds, a float as `1.0` | + ### Cell Styling -Apply custom styles to individual cells based on their value and the column they belong to: +`$value` is the cell as it is written (see [Cells](#cells)), not the column value. ```php run(); ``` +### Header and missing cells + +``` +header row [' id ', '', 'name'] → columns ['id', 'e01', 'name'] (trimmed, a blank cell named e + its position) +record ['1', 'a'] → ['id' => '1', 'e01' => 'a', 'name' => null] (a cell the record lacks is null) +record ['1', '', 'x'] → 'e01' => null (a present '' is null; `withEmptyToNull(false)` keeps it '') +``` + +Blank rows before the header are skipped, also under `withHeader(false)`. + ## Needed parameters - `$auth_config` diff --git a/documentation/components/adapters/http.md b/documentation/components/adapters/http.md index 217e025340..e5f58ce4cc 100644 --- a/documentation/components/adapters/http.md +++ b/documentation/components/adapters/http.md @@ -170,6 +170,19 @@ data_frame() ->run(); ``` +Each response is one row and, by default, one batch. Group responses into batches with `withBatchSize()`; a pushed +limit stops sending requests once it is reached: + +```php +read(from_static_http_requests($psr18Client, $requests())->withBatchSize(50)) + ->limit(10) // 10 requests sent + ->write(to_output()) + ->run(); +``` + ## Extractor - PsrHttpClientDynamicExtractor > Reach for this only when no built-in paginator fits. It is the lowest-level extractor of the three - you hand-write @@ -218,9 +231,8 @@ data_frame() ## Typing the row with a schema All three extractors expose `withSchema(Schema)` (also the optional last argument of the `from_*` DSL functions). The -schema is passed straight to the DataFrame Hydrator's `cast()` - it describes the **row**, so to type the body you -declare `response_body` as a `structure`. As with every extractor, `cast()` keeps only the columns the schema mentions, -so include any envelope columns you want to keep. +schema describes the **row**, so to type the body you declare `response_body` as a `structure`. As with every +extractor, only the columns the schema mentions are kept, so include any envelope columns you want to keep. ```php Json library is not explicitly required, you need to make sure it is available in your composer.json file. -> If you are only using Loader, this dependency is optional. - -## Extractor - JSONMachine - JsonExtractor +## Extractor ```php read(from_json(__DIR__ . '/data.json')) - ->collect() + ->write(to_output()) + ->run(); + +data_frame() + ->read(from_json_lines(__DIR__ . '/data.jsonl')) ->write(to_output()) ->run(); ``` +Both return a `JsonExtractor`. `from_json()` reads one document: a top-level array, or the subtree selected with +`withPointer()`. + In JSON lines, a line holding only whitespace (space, tab, CR, LF, VT, FF) is skipped, as DuckDB does; this is not configurable. -## Loader - JsonLoader +With the [`flow_php`](/documentation/components/extensions/flow-php-ext.md) extension loaded, JSON lines and a document +holding a top-level array are read natively, as strict JSON: malformed input throws +`RuntimeException('Malformed JSON in "" at line N: ...')` (`at element N` in a document). A pointer, or a document +that is not an array, is read by the PHP reader. + +## Loader ```php read(from_array( - \array_map( - fn (int $i) : array => ['id' => $i, 'name' => 'name_' . $i], - \range(0, 10) - ) - )) - ->collect() + ->read(from_array([['id' => 1, 'name' => 'name_1'], ['id' => 2, 'name' => 'name_2']])) ->write(to_json(\sys_get_temp_dir() . '/file.json')) + ->write(to_json_lines(\sys_get_temp_dir() . '/file.jsonl')) ->run(); ``` -## Loader - JsonLoader - JSON lines +Both return a `JsonLoader`. `to_json()` writes one array document, `to_json_lines()` writes +[JSON lines](https://jsonlines.org/). -It is also possible to export the rows using the [json lines](https://jsonlines.org/) format +## What the loaders write ```php read(from_array( - \array_map( - fn (int $i) : array => ['id' => $i, 'name' => 'name_' . $i], - \range(0, 10) - ) + [ + ['id' => 1, 'active' => true, 'price' => 0.1 + 0.2, 'at' => new DateTimeImmutable('2026-01-02 03:04:05 UTC'), 'days' => [new DateTimeImmutable('2026-01-02')], 'labels' => [0 => 'a', 1 => 'b'], 'size' => ['w' => 1, 'h' => 2], 'meta' => '{"tags": []}'], + ['id' => 2, 'active' => false, 'price' => 1.0, 'at' => null, 'days' => [], 'labels' => [], 'size' => ['w' => 3, 'h' => 4], 'meta' => '{}'], + ], + schema( + int_schema('id'), bool_schema('active'), float_schema('price'), datetime_schema('at', nullable: true), + list_schema('days', type_list(type_date())), map_schema('labels', type_map(type_integer(), type_string())), + structure_schema('size', type_structure(['w' => type_integer(), 'h' => type_integer()])), json_schema('meta'), + ), )) - ->collect() - ->write(to_json_lines(\sys_get_temp_dir() . '/file.jsonl')) + ->write(to_json_lines($path)) ->run(); ``` +```json +{"id":1,"active":true,"price":0.30000000000000004,"at":"2026-01-02T03:04:05+00:00","days":["2026-01-02"],"labels":{"0":"a","1":"b"},"size":{"w":1,"h":2},"meta":{"tags":[]}} +{"id":2,"active":false,"price":1,"at":null,"days":[],"labels":{},"size":{"w":3,"h":4},"meta":{}} +``` + +The shape follows the column type, never the data: a `list` is always `[...]`, a `map`, a `structure` and a row are +always `{...}` (an empty map is `{}`, a map keyed `0, 1` is `{"0":...,"1":...}`), a `json` column keeps its objects and +lists. A `datetime` / `date` is written with `withDateTimeFormat()` / `withDateFormat()` at every depth, a `time` as +microseconds. A float has the shortest digits that read back as the same float (`1.0` is `1`, or `1.0` under +`JSON_PRESERVE_ZERO_FRACTION`); `NAN` and the infinities are refused: +`RuntimeException('Failed to encode JSON: Inf and NaN cannot be JSON encoded')`. + +Floats are rendered under `serialize_precision = -1`. When the ini holds another value the writer sets it for the +render and restores it, so `ini_set()` must be allowed; otherwise it throws +`RuntimeException('Writing floats requires serialize_precision = -1 and ini_set() cannot change it')`. + +With the [`flow_php`](/documentation/components/extensions/flow-php-ext.md) extension loaded and no flags beyond +`JSON_THROW_ON_ERROR`, `JSON_UNESCAPED_SLASHES`, `JSON_UNESCAPED_UNICODE` and `JSON_PRESERVE_ZERO_FRACTION`, the same +bytes are rendered natively. + ## JSON Schema conversion The adapter can convert [JSON Schema](https://json-schema.org) documents into Flow schemas and back. @@ -119,8 +148,10 @@ file. | `string` (+ `format: date` / `date-time` / `time` / `uuid` / `html` / `xml`) | `string` / `date` / `datetime` / `time` / `uuid` / `html` / `xml` | | `integer` / `number` / `boolean` / `null` | `integer` / `float` / `boolean` / null column | | `array` + `items` | `list` | +| `array` without `items` | `json` | | `object` + `properties` | `structure` (`required` controls optional members) | | `object` + `additionalProperties: ` | `map` | +| `object` without `properties` / `additionalProperties` | `json` | | `type: ["object", "array"]` | `json` | | empty schema `{}` / boolean schema `true` | `json` (accept anything, marked in metadata) | | `enum` / `const` | scalar type derived from the values, values preserved in metadata | diff --git a/documentation/components/adapters/logger.md b/documentation/components/adapters/logger.md index 193207b119..231ad46189 100644 --- a/documentation/components/adapters/logger.md +++ b/documentation/components/adapters/logger.md @@ -14,26 +14,26 @@ ETL Adapter that provides PSR Logger support for ETL. For detailed installation instructions, see the [installation page](/documentation/installation/packages/etl-adapter-logger.md). -## Loader - PsrLogger +## Loader - PsrLoggerLoader -Load each row into PsrLoggerInterface implementation. To get `TestLogger` mock class first run: - -``` -composer require fig/log-test -``` +Logs each row as one record: the row's values are the log context. `TestLogger` below comes from +`composer require --dev fig/log-test`. ```php load(rows( - schema(int_schema('id'), str_schema('name')), - row(['id' => 12345, 'name' => 'norbert']), -)); +$logger = new TestLogger(); + +data_frame() + ->read(from_array([['id' => 12345, 'name' => 'norbert']])) + ->write(new PsrLoggerLoader($logger, 'row log', LogLevel::ERROR)) + ->run(); -$this->assertTrue($logger->hasErrorRecords()); -$this->assertTrue($logger->hasError('row log')); -``` \ No newline at end of file +$logger->hasError(['message' => 'row log', 'context' => ['id' => 12345, 'name' => 'norbert']]); // true +``` diff --git a/documentation/components/adapters/parquet.md b/documentation/components/adapters/parquet.md index 3def9b0071..862480ec79 100644 --- a/documentation/components/adapters/parquet.md +++ b/documentation/components/adapters/parquet.md @@ -21,3 +21,55 @@ the robust and adaptable nature of the Flow PHP ecosystem. ## Installation For detailed installation instructions, see the [installation page](/documentation/installation/packages/etl-adapter-parquet.md). + +## Engines + +Without `engine:`, `AdaptiveParquetEngine` opens the file. The loaded extensions, arrow-ext and +[`flow_php`](/documentation/components/extensions/flow-php-ext.md), pick the lane: + +| arrow-ext | `flow_php` | Engine | Batches | +|-----------|------------|---------------------|---------------------------------------------------------------------| +| no | no | `PhpParquetEngine` | PHP columns | +| yes | no | `RustParquetEngine` | PHP columns built from arrow-ext's values | +| no | yes | `PhpParquetEngine` | native columns built from the PHP engine's values | +| yes | yes | `RustParquetEngine` | arrow-ext's batches straight into native columns, and back on write | + +`RustParquetEngine` needs little-endian byte order. Native columns come from the +[column backend](/documentation/components/core/column-backend.md). An explicit engine is always used; +`PhpParquetEngine` opts out of arrow-ext: + +```php +read(from_parquet(__DIR__ . '/orders.parquet', engine: new PhpParquetEngine())) + ->write(to_parquet(__DIR__ . '/copy.parquet', engine: new PhpParquetEngine())) + ->run(); +``` + +Flow `float` columns are written as Parquet `DOUBLE`. + +## Schema + +The file footer is the schema: `from_parquet(...)->withSchema()` throws. Project with `columns:`, change types after +reading: + +```php +read(from_parquet(__DIR__ . '/orders.parquet', columns: ['id', 'payload', 'created_at'])) + ->withEntry('payload', ref('payload')->cast(type_json())) + ->withEntry('created_at', to_timezone(ref('created_at'), 'Europe/Warsaw')) + ->write(to_output()) + ->run(); +``` diff --git a/documentation/components/adapters/postgresql.md b/documentation/components/adapters/postgresql.md index 639a1ef91f..fbdb221e69 100644 --- a/documentation/components/adapters/postgresql.md +++ b/documentation/components/adapters/postgresql.md @@ -292,6 +292,17 @@ df() ->run(); ``` +### Bound values + +`xml`, `xml_element`, `html` and `html_element` columns are bound as their markup text. An `xml` document is bound as +its root element only: the declaration, a DOCTYPE, and comments or processing instructions outside the root are +dropped. + +| Column | Value | Bound | +|---------------|----------------------------------------------------|------------------------| +| `xml` | `` | `` | +| `xml_element` | `` | `` | + ### Insert with Skip Conflicts (ON CONFLICT DO NOTHING) Skip rows that would cause a constraint violation: @@ -425,6 +436,8 @@ use Flow\ETL\Adapter\PostgreSql\PostgreSqlTransaction; use Flow\ETL\Sink\Transactional; use Flow\PostgreSql\QueryBuilder\Transaction\IsolationLevel; +use function Flow\ETL\Adapter\PostgreSql\to_pgsql_table; + new Transactional( (new PostgreSqlTransaction($client))->withIsolationLevel(IsolationLevel::SERIALIZABLE), to_pgsql_table($client, 'users'), @@ -433,13 +446,13 @@ new Transactional( ## Loader DSL Functions Reference -| Function | Description | -|------------------------------------------------|-----------------------------------------------------------| -| `to_pgsql_table($client, $table)` | Create a PostgreSQL loader for a table | -| `to_pgsql_transaction($client, ...$sinks)` | Write sinks, every delivery inside a transaction | -| `pgsql_insert_options(...)` | Configure insert behavior (conflicts, upsert) | -| `pgsql_update_options($primaryKeys)` | Configure update behavior (primary key columns) | -| `pgsql_delete_options($primaryKeys)` | Configure delete behavior (primary key columns) | +| Function | Description | +|--------------------------------------------|--------------------------------------------------| +| `to_pgsql_table($client, $table)` | Create a PostgreSQL loader for a table | +| `to_pgsql_transaction($client, ...$sinks)` | Write sinks, every delivery inside a transaction | +| `pgsql_insert_options(...)` | Configure insert behavior (conflicts, upsert) | +| `pgsql_update_options($primaryKeys)` | Configure update behavior (primary key columns) | +| `pgsql_delete_options($primaryKeys)` | Configure delete behavior (primary key columns) | ## Schema Conversion @@ -515,7 +528,7 @@ $schema = schema( | `PostgreSqlMetadata::indexUnique($name, $position)` | Include the column in a named `UNIQUE` constraint, optionally at an explicit position | | `PostgreSqlMetadata::index($name, $position)` | Include the column in a named index, optionally at an explicit position | | `PostgreSqlMetadata::identity($generation)` | Make the column an identity column | -| `PostgreSqlMetadata::generated($expr)` | Make the column a generated column | +| `PostgreSqlMetadata::generated($expr)` | Make the column a generated column | Columns sharing the same primary key, unique constraint, or index name are grouped together, so composite keys are expressed by attaching the same name to several definitions. @@ -554,6 +567,10 @@ Because each index is tracked under its own metadata key, a single column can be `merge()` to attach more than one: ```php +use Flow\ETL\Adapter\PostgreSql\PostgreSqlMetadata; + +use function Flow\ETL\DSL\{datetime_schema, int_schema, schema}; + $schema = schema( int_schema('id', metadata: PostgreSqlMetadata::primaryKey('pk_orders') ->merge(PostgreSqlMetadata::index('orders_created_at_id_idx', position: 2))), @@ -608,7 +625,7 @@ $table = to_pgsql_schema_table( ### Schema Conversion DSL Functions Reference -| Function | Description | -|-----------------------------------------------------------------------|--------------------------------------------------------| -| `to_pgsql_schema_table($schema, $tableName, $databaseSchema, $typesMap)` | Convert a Flow `Schema` into a PostgreSQL `Table` | -| `pgsql_table_to_flow_schema($table, $typesMap)` | Convert a PostgreSQL `Table` into a Flow `Schema` | +| Function | Description | +|--------------------------------------------------------------------------|---------------------------------------------------| +| `to_pgsql_schema_table($schema, $tableName, $databaseSchema, $typesMap)` | Convert a Flow `Schema` into a PostgreSQL `Table` | +| `pgsql_table_to_flow_schema($table, $typesMap)` | Convert a PostgreSQL `Table` into a Flow `Schema` | diff --git a/documentation/components/adapters/seal.md b/documentation/components/adapters/seal.md index 5ddf238d99..f6b59d6d40 100644 --- a/documentation/components/adapters/seal.md +++ b/documentation/components/adapters/seal.md @@ -24,17 +24,17 @@ In addition to this adapter, install the SEAL engine adapter for your search bac composer require flow-php/etl-adapter-seal cmsig/seal-elasticsearch-adapter ``` -| Backend | Package | -|---------------|----------------------------------| -| Algolia | `cmsig/seal-algolia-adapter` | +| Backend | Package | +|---------------|------------------------------------| +| Algolia | `cmsig/seal-algolia-adapter` | | Elasticsearch | `cmsig/seal-elasticsearch-adapter` | -| Loupe | `cmsig/seal-loupe-adapter` | -| Meilisearch | `cmsig/seal-meilisearch-adapter` | -| Memory | `cmsig/seal-memory-adapter` | -| OpenSearch | `cmsig/seal-opensearch-adapter` | -| RediSearch | `cmsig/seal-redisearch-adapter` | -| Solr | `cmsig/seal-solr-adapter` | -| Typesense | `cmsig/seal-typesense-adapter` | +| Loupe | `cmsig/seal-loupe-adapter` | +| Meilisearch | `cmsig/seal-meilisearch-adapter` | +| Memory | `cmsig/seal-memory-adapter` | +| OpenSearch | `cmsig/seal-opensearch-adapter` | +| RediSearch | `cmsig/seal-redisearch-adapter` | +| Solr | `cmsig/seal-solr-adapter` | +| Typesense | `cmsig/seal-typesense-adapter` | ## Description @@ -97,8 +97,8 @@ to_seal_upsert($engine, 'users')->withBulkSize(500); ### Deleting Documents -`to_seal_delete()` removes documents by their identifiers. By default, the identifier is taken from the `id` entry of -each row - use `withIdentifierEntry()` when your identifier entry has a different name. +`to_seal_delete()` removes documents by their identifiers, read from the `id` column. `withIdentifierEntry()` names +another column. ```php use function Flow\ETL\Adapter\Seal\to_seal_delete; @@ -116,18 +116,19 @@ to_seal_delete($engine, 'products')->withIdentifierEntry('sku'); ### Data Normalization -Before saving, rows are normalized into search documents: +Each row becomes one document: -| Flow Type | Document Value | -|---------------------------|-------------------------------------------------| -| `string`, `int`, `float`, `bool` | unchanged | -| `uuid` | string representation | -| `datetime` | string, `DateTimeInterface::ATOM` by default | -| `date` | string, `Y-m-d` by default | -| `enum` | case name | -| `xml`, `xml_element` | serialized XML string | -| `json` | decoded array | -| `list`, `map`, `structure` | array (nested values normalized recursively) | +| Flow Type | Document Value | +|----------------------------------------------|----------------------------------------------| +| `string`, `int`, `float`, `bool` | unchanged | +| `uuid` | string representation | +| `datetime` | string, `DateTimeInterface::ATOM` by default | +| `date` | string, `Y-m-d` by default | +| `enum` | case name | +| `time` | microseconds | +| `xml`, `xml_element`, `html`, `html_element` | markup text, e.g. `` | +| `json` | decoded array | +| `list`, `map`, `structure` | array (nested values normalized recursively) | ## Schema Conversion @@ -156,20 +157,20 @@ $sealSchema = to_seal_schema( ### Type Mapping -| Flow Type | SEAL Field | Default Flags | -|------------------------------------|-------------------|------------------------| -| `string`, `uuid`, `enum`, `xml`, `html` | `TextField` | searchable | -| `integer` | `IntegerField` | filterable, sortable | -| `float` | `FloatField` | filterable, sortable | -| `boolean` | `BooleanField` | filterable | -| `datetime`, `date` | `DateTimeField` | filterable, sortable | -| `json`, `map` | `JsonObjectField` | - | -| `list` | field of `T` with `multiple: true` | as `T` | -| `structure` | `ObjectField` (recursive) | - | +| Flow Type | SEAL Field | Default Flags | +|-----------------------------------------|------------------------------------|----------------------| +| `string`, `uuid`, `enum`, `xml`, `html` | `TextField` | searchable | +| `integer` | `IntegerField` | filterable, sortable | +| `float` | `FloatField` | filterable, sortable | +| `boolean` | `BooleanField` | filterable | +| `datetime`, `date` | `DateTimeField` | filterable, sortable | +| `json`, `map` | `JsonObjectField` | - | +| `list` | field of `T` with `multiple: true` | as `T` | +| `structure` | `ObjectField` (recursive) | - | ### Field Flags -Default flags can be overridden per definition through entry metadata: +Default flags can be overridden per definition through its metadata: | Metadata | Effect | |------------------------------|-----------------------------------------| diff --git a/documentation/components/adapters/text.md b/documentation/components/adapters/text.md index 4ca17f568a..1d75680647 100644 --- a/documentation/components/adapters/text.md +++ b/documentation/components/adapters/text.md @@ -29,32 +29,62 @@ For detailed installation instructions, see the [installation page](/documentati ```php read(from_text(path)) +$rows = data_frame() + ->read(from_text(__DIR__ . '/file.txt')) ->fetch(); ``` +Each line is one row with a single `string` column `text`. + ## Loader -> :warning: Heads up, TextLoader expects rows to have single entry in order to properly write them into file. +The batch must have at most one column. ```php process( - rows( - schema(str_schema('name')), - row(['name' => 'Norbert']), - row(['name' => 'Tomek']), - row(['name' => 'Dawid']), - ) - ) - ->load(to_text($path)) +use function Flow\ETL\Adapter\Text\to_text; +use function Flow\ETL\DSL\{data_frame, from_array}; + +data_frame() + ->read(from_array([['name' => 'Norbert'], ['name' => 'Tomek'], ['name' => 'Dawid']])) + ->write(to_text(__DIR__ . '/names.txt')) ->run(); ``` + +Every value of the single column is one line: + +```php +read(from_array([['price' => 0.1 + 0.2], ['price' => 1.0], ['price' => INF]], schema(float_schema('price')))) + ->write(to_text($path)) + ->run(); +``` + +``` +0.30000000000000004 +1.0 +INF +``` + +| Column | Line | +|----------------------------|------------------------------------------------------------------------------------------------------------------------------------| +| `boolean` | `true` / `false` | +| `float` | the shortest text that reads back as the same float, always with a fraction or an exponent: `1.0`, `1.0e+25`; `NAN`, `INF`, `-INF` | +| `null` | an empty line | +| `datetime`, `date` | `DATE_ATOM` in the column zone, `Y-m-d` | +| `time` | microseconds | +| `json` | the stored text | +| `list`, `map`, `structure` | refused: `Text data loader supports only scalar values, got array` | + +Floats are rendered under `serialize_precision = -1`. When the ini holds another value the writer sets it for the +render and restores it, so `ini_set()` must be allowed; otherwise it throws +`RuntimeException('Writing floats requires serialize_precision = -1 and ini_set() cannot change it')`. diff --git a/documentation/components/adapters/xml.md b/documentation/components/adapters/xml.md index 77e45da6dc..627c52def1 100644 --- a/documentation/components/adapters/xml.md +++ b/documentation/components/adapters/xml.md @@ -23,11 +23,9 @@ aligning perfectly with the robust and adaptable nature of the Flow PHP ecosyste For detailed installation instructions, see the [installation page](/documentation/installation/packages/etl-adapter-xml.md). -## Extractor - XMLExtractor +## Extractor -Memory safe XML extractor - -`xml/simple_items.xml` +`simple_items.xml` ```xml @@ -35,46 +33,119 @@ Memory safe XML extractor 1 2 3 - 4 - 5 - 6 ``` -```php +```php read(XML::from(__FLOW_DATA__ . '/simple_items.xml', 'root/items/item')) - ->write(To::output(false)) - ->run() -; -``` +use function Flow\ETL\Adapter\XML\from_xml; +use function Flow\ETL\DSL\{data_frame, ref, to_output}; -Above code will generate Rows with 5 entries like the one below: +data_frame() + ->read(from_xml(__DIR__ . '/simple_items.xml')->withXMLNodePath('root/items/item')) + ->withEntry('id', ref('node')->xpath('id')->domElementValue()) + ->write(to_output(truncate: false)) + ->run(); +``` ```shell -+----------------------------------------------+ -| row | -+----------------------------------------------+ -| 1 | -| 2 | -| 3 | -| 4 | -| 5 | -| 6 | -+----------------------------------------------+ ++-------------------------+----+ +| node | id | ++-------------------------+----+ +| 1 | 1 | +| 2 | 2 | +| 3 | 3 | ++-------------------------+----+ +3 rows +``` + +The file is streamed. Every node at `withXMLNodePath()` is one row with a single `xml` column `node`. Read it with +`xpath()` (returns a `list`), `domElementValue()`, `domElementAttributeValue()` and the other +`domElement*()` functions. + +## Loader + +```php +read(from_array( + [ + ['id' => 1, 'active' => true, 'price' => 0.1 + 0.2, 'at' => new DateTimeImmutable('2026-01-02 03:04:05 UTC'), 'days' => [new DateTimeImmutable('2026-01-02')], 'labels' => [0 => 'a', 1 => 'b'], 'size' => ['w' => 1, 'h' => 2], 'meta' => '{"tags": []}'], + ['id' => 2, 'active' => false, 'price' => 1.0, 'at' => null, 'days' => [], 'labels' => [], 'size' => ['w' => 3, 'h' => 4], 'meta' => '{}'], + ], + schema( + int_schema('id'), bool_schema('active'), float_schema('price'), datetime_schema('at', nullable: true), + list_schema('days', type_list(type_date())), map_schema('labels', type_map(type_integer(), type_string())), + structure_schema('size', type_structure(['w' => type_integer(), 'h' => type_integer()])), json_schema('meta'), + ), + )) + ->write(to_xml($path)) + ->run(); +``` + +```xml + + +1true0.300000000000000042026-01-02T03:04:05.000000+00:002026-01-020a1b12{"tags": []} +2false1.034{} + ``` -Each entry will be an XMLEntry type. -From there you can use built in expressions to extract data from XML. +| Column | Element | +|----------------------------|------------------------------------------------------------------------------------------------------------------------------------| +| `boolean` | `true` / `false` | +| `float` | the shortest text that reads back as the same float, always with a fraction or an exponent: `1.0`, `1.0e+25`; `NAN`, `INF`, `-INF` | +| `null` | an empty element | +| `datetime`, `date` | `withDateTimeFormat()` (default `Y-m-d\TH:i:s.uP`) in the column zone, `withDateFormat()` (default `Y-m-d`) | +| `time` | microseconds | +| `list`, `map`, `structure` | nested elements; every leaf is written as a column of its type would be | +| `json` | the stored text | -- `ref('row')->xpath('...');` -- `ref('row')->domNodeAttribute('...');` -- `ref('row')->domNodeValue('...');` +A column whose name starts with the attribute prefix (`_` by default) is an attribute of its row; a null there is +refused. -When working with collections XPath will return an ListEntry with XMLEntries inside. -From there you can for example unpack or expand them. +Floats are rendered under `serialize_precision = -1`. When the ini holds another value the writer sets it for the +render and restores it, so `ini_set()` must be allowed; otherwise it throws +`RuntimeException('Writing floats requires serialize_precision = -1 and ini_set() cannot change it')`. + +### Custom writers + +`XMLWriter` renders a column at a time: `elements()` and `attributes()` return one string per value, with the writer's +own escaping. A writer that only builds nodes delegates to `write()`: + +```php + substr($this->write(XMLNode::nested('x', new XMLAttribute($name, type_string()->cast($value)))), 2, -2), + $values, + ); + } + + public function elements(string $name, array $values): array + { + return array_map(fn (?string $value): string => $this->write(XMLNode::flatNode($name, $value ?? '')), $values); + } + + public function write(XMLNode $node): string + { + // ... + } +} +``` \ No newline at end of file diff --git a/documentation/components/bridges/symfony-http-foundation-bridge.md b/documentation/components/bridges/symfony-http-foundation-bridge.md index 94f3c34ba8..48f81b7c7f 100644 --- a/documentation/components/bridges/symfony-http-foundation-bridge.md +++ b/documentation/components/bridges/symfony-http-foundation-bridge.md @@ -77,9 +77,9 @@ final class ReportsController extends AbstractController ## Available Outputs - `Flow\Bridge\Symfony\HttpFoundation\Output\CSVOutput` - `http_csv_output()` - converts dataset to CSV format. -- `Flow\Bridge\Symfony\HttpFoundation\Output\JSONOutput` - `http_json_output()` -converts dataset to JSON format. -- `Flow\Bridge\Symfony\HttpFoundation\Output\ParquetOutput` - `http_parquet_output()` -converts dataset to Parquet format. -- `Flow\Bridge\Symfony\HttpFoundation\Output\XMLOutput` - `http_xml_output()` -converts dataset to XML format. +- `Flow\Bridge\Symfony\HttpFoundation\Output\JsonOutput` - `http_json_output()` - converts dataset to JSON format. +- `Flow\Bridge\Symfony\HttpFoundation\Output\ParquetOutput` - `http_parquet_output()` - converts dataset to Parquet format. +- `Flow\Bridge\Symfony\HttpFoundation\Output\XMLOutput` - `http_xml_output()` - converts dataset to XML format. ## Modify output on the fly @@ -87,10 +87,16 @@ Sometimes we need to modify the output on the fly. To do that, FlowStreamedResponse allows passing a Transformation that will be applied on the dataset. ```php +withColumn('time', \time()); + return $dataFrame->withEntry('time', lit(\time())); } }; ``` diff --git a/documentation/components/cli/docs.md b/documentation/components/cli/docs.md index 506580ec8c..dd6102054d 100644 --- a/documentation/components/cli/docs.md +++ b/documentation/components/cli/docs.md @@ -338,23 +338,25 @@ Arguments: Options: --input-file-format=INPUT-FILE-FORMAT Source file format. When not set file format is guessed from source file path extension --input-file-limit=INPUT-FILE-LIMIT Limit number of rows that are going to be used to infer file schema, when not set whole file is analyzed + --input-file-offset=INPUT-FILE-OFFSET Number of rows to skip before starting to read data --config=CONFIG Path to a local php file that MUST return instance of: Flow\ETL\Config --input-json-pointer=INPUT-JSON-POINTER JSON Pointer to a subtree from which schema should be extracted --input-json-pointer-entry-name When set, JSON Pointer will be used as an entry name in the schema + --input-excel-header[=INPUT-EXCEL-HEADER] When set, Excel header will be used as a schema + --input-excel-sheet-name=INPUT-EXCEL-SHEET-NAME When set, Excel sheet name will be selected for reading + --input-excel-offset=INPUT-EXCEL-OFFSET Offset to start reading from --input-csv-header[=INPUT-CSV-HEADER] When set, CSV header will be used as a schema --input-csv-empty-to-null[=INPUT-CSV-EMPTY-TO-NULL] When set, empty CSV values will be treated as NULL values --input-csv-separator=INPUT-CSV-SEPARATOR CSV separator character --input-csv-enclosure=INPUT-CSV-ENCLOSURE CSV enclosure character --input-csv-escape=INPUT-CSV-ESCAPE CSV escape character - --input-excel-header=INPUT-EXCEL-HEADER When set, Excel header will be used as a schema - --input-excel-sheet-name=INPUT-EXCEL-SHEETNAME When set, Excel sheet name will be selected for reading - --input-excel-offset=INPUT-EXCEL-OFFSET Offset to start reading from --input-xml-node-path=INPUT-XML-NODE-PATH XML node path to a subtree from which schema should be extracted, for example /root/element This is not xpath, just a node names separated by slash --input-xml-buffer-size=INPUT-XML-BUFFER-SIZE XML buffer size in bytes --input-parquet-columns=INPUT-PARQUET-COLUMNS Columns to read from parquet file (multiple values allowed) --input-parquet-offset=INPUT-PARQUET-OFFSET Offset to start reading from -h, --help Display help for the given command. When no command is given display help for the list command - -q, --quiet Do not output any message + --silent Do not output any message + -q, --quiet Only errors are displayed. All other output is suppressed -V, --version Display this application version --ansi|--no-ansi Force (or disable --no-ansi) ANSI output -n, --no-interaction Do not ask any interactive question @@ -381,7 +383,8 @@ Options: --statistics Display column chunks statistics details --page-headers Display page headers details -h, --help Display help for the given command. When no command is given display help for the list command - -q, --quiet Do not output any message + --silent Do not output any message + -q, --quiet Only errors are displayed. All other output is suppressed -V, --version Display this application version --ansi|--no-ansi Force (or disable --no-ansi) ANSI output -n, --no-interaction Do not ask any interactive question @@ -454,6 +457,7 @@ Options: --output-php Print schema as PHP code --output-table Print schema as ascii table --output-ascii Print schema as ascii list + --db-column[=DB-COLUMN] Filter schema by column name(s) (multiple values allowed) --config=CONFIG Path to a local php file that MUST return instance of: Flow\ETL\Config -c, --db-connection-file=DB-CONNECTION-FILE Path to file that returns and instance of \Doctrine\DBAL\Connection -h, --help Display help for the given command. When no command is given display help for the list command diff --git a/documentation/components/core/aggregations.md b/documentation/components/core/aggregations.md index 7087294a74..f956cfff11 100644 --- a/documentation/components/core/aggregations.md +++ b/documentation/components/core/aggregations.md @@ -17,17 +17,39 @@ Each group created by `groupBy` function can be aggregated using one of the aggr - [`COLLECT_UNIQUE()`](/src/core/etl/src/Flow/ETL/Function/CollectUnique.php) - [`FIRST()`](/src/core/etl/src/Flow/ETL/Function/First.php) - [`LAST()`](/src/core/etl/src/Flow/ETL/Function/Last.php) -- [`STRIRNG_AGG()`](/src/core/etl/src/Flow/ETL/Function/StringAggregate.php) +- [`STRING_AGG()`](/src/core/etl/src/Flow/ETL/Function/StringAggregate.php) -All aggregation functions are implementing [`AggregatingFunction`](/src/core/etl/src/Flow/ETL/Function/AggregatingFunction.php) interface. +All aggregation functions implement [`AggregatingFunction`](/src/core/etl/src/Flow/ETL/Function/AggregatingFunction.php): +`aggregate(Rows $rows, array $indices, FlowContext $context)` folds the rows of one group (`$indices` of the batch) +into the aggregator's state, and `merge()` folds in the partial state of another aggregator of the same class, which +group by uses after it spilled. + +## Float arithmetic + +`sum()` and `average()` add floats as IEEE 754 doubles with compensated summation. `exact: true` switches to decimal +arithmetic: + +```php +read(from_array([['a' => 0.1], ['a' => 0.2]]))->aggregate([sum(ref('a'))]); // 0.30000000000000004 +df()->read(from_array([['a' => 0.1], ['a' => 0.2]]))->aggregate([sum(ref('a'), exact: true)]); // 0.3 +``` ## Example ```php -read(from_csv(__DIR__ . '/power-plant-daily.csv', delimiter: ';')) + ->read(from_csv(__DIR__ . '/power-plant-daily.csv', separator: ';')) ->withEntry('production_kwh', ref('Produkcja(kWh)')) ->withEntry('consumption_kwh', ref('Zużycie(kWh)')) ->withEntry('date', ref('Zaktualizowany czas')->toDate('Y/m/d')->dateFormat('Y/m')) @@ -51,7 +73,7 @@ data_frame() ->withEntry('consumption_kwh_max', ref('consumption_kwh_max')->round(lit(2))) ->withEntry('production_kwh_sum', ref('production_kwh_sum')->round(lit(2))) ->withEntry('consumption_kwh_sum', ref('consumption_kwh_sum')->round(lit(2))) - ->withEntry('consumption', ref('consumption_kwh_sum')->divide(ref('production_kwh_sum'))) + ->withEntry('consumption', ref('consumption_kwh_sum')->divide(ref('production_kwh_sum'), 4, Rounding::HALF_UP)) ->withEntry('consumption', ref('consumption')->multiply(lit(100))->round(lit(2))) ->withEntry('consumption', concat(ref('consumption'), lit('%'))) ->write(to_output(truncate: false)) diff --git a/documentation/components/core/batch-processing.md b/documentation/components/core/batch-processing.md index 595d496e04..02b7371753 100644 --- a/documentation/components/core/batch-processing.md +++ b/documentation/components/core/batch-processing.md @@ -75,6 +75,7 @@ The `batchBy()` method ensures that all rows with the same column value stay in - When `minSize` is specified: batches accumulate until reaching minimum size, then yield on group boundary - When `minSize` is omitted: each unique group value gets its own batch - Batches may exceed `minSize` to keep large groups intact +- Group values compare by value (equal datetimes or uuids are one group); nulls form one group > **Use Case**: If you're loading orders with line items and using a DELETE+INSERT pattern, `batchBy('order_id')` ensures all line items for an order are in the same batch, preventing foreign key violations. @@ -89,7 +90,6 @@ $dataFrame = data_frame() ->read($extractor) ->filter($condition) ->collect() // Collect all filtered data into single batch - ->sortBy([col('name')]) // Now can sort the collected data ->write($loader) ->run(); ``` @@ -97,7 +97,7 @@ $dataFrame = data_frame() > **Memory Warning**: The `collect()` method loads all data into memory at once. This can cause memory exhaustion > with large datasets. Use only when: > - You're certain the entire dataset fits comfortably in available memory -> - You need operations that require all data (like sorting) +> - The next step needs the whole dataset in one batch > - You're working with small to medium datasets ## Memory Management Strategies @@ -116,5 +116,5 @@ $report = data_frame() ->write($loader) ->run(analyze: analyze()); -echo "Peak memory usage: " . $report->statistics()->memory->max()->inMb() . " bytes\n"; +echo "Peak memory usage: " . $report->statistics()->memory->max()->inMb() . " MB\n"; ``` \ No newline at end of file diff --git a/documentation/components/core/building-blocks.md b/documentation/components/core/building-blocks.md index 95ddb96b37..4a5431fe21 100644 --- a/documentation/components/core/building-blocks.md +++ b/documentation/components/core/building-blocks.md @@ -4,9 +4,8 @@ Columns of the [Data Frame](/documentation/components/core/core.md) are described by the [Schema](/src/core/etl/src/Flow/ETL/Schema.php) - it owns their names, order, types and nullability. -A [Row](/src/core/etl/src/Flow/ETL/Row.php) carries the values for one record, keyed by column name. -A group of Rows is called `Rows`, represented by the [Rows](/src/core/etl/src/Flow/ETL/Rows.php) -class, and every `Rows` carries the one `Schema` that describes it. +A batch of records is represented by the [Rows](/src/core/etl/src/Flow/ETL/Rows.php) class: one column per +schema definition, and every `Rows` carries the one `Schema` that describes it. Let's look at the following example: @@ -15,27 +14,6 @@ Let's look at the following example: declare(strict_types=1); -use function Flow\ETL\DSL\{bool_schema, int_schema, row, rows, schema, str_schema}; - -$rows = rows( - schema(int_schema('id'), str_schema('name'), bool_schema('active')), - row(['id' => 1, 'name' => 'user_01', 'active' => true]), - row(['id' => 2, 'name' => 'user_02', 'active' => false]), - row(['id' => 3, 'name' => 'user_03', 'active' => true]), - row(['id' => 4, 'name' => 'user_04', 'active' => false]), -); -``` - -Rows are the main data structure in Flow ETL, they're used to represent data in the data frame. -Extractors are yielding Rows and Loaders are saving Rows. - -The same can be achieved using the following code: - -```php -values($i)` (one row as an array), `$rows->toArray()` (every row) or `$rows->column('name')->value($i)`. + ## Column Types Every column is described by a [Definition](/src/core/etl/src/Flow/ETL/Schema/Definition.php), built with @@ -88,10 +69,10 @@ A column holds exactly one type. `null|T` spells a nullable column, and a nullab structure element; every other union is refused. Declare the widest common type with `str_schema()`, or `json_schema()` when the shape is genuinely dynamic. -The schema is declared, never guessed: `array_to_rows()` takes it as its second argument and a -[Hydrator](/src/core/etl/src/Flow/ETL/Row/Hydrator.php) turns the raw values into `Rows` against it, -casting each one to the type its column declares. A value the declared type refuses aborts the batch -with a `SchemaMismatchException` naming the column and its row. +The schema is declared, never guessed: `array_to_rows()` takes it as its second argument and the +[column backend](/documentation/components/core/column-backend.md) builds `Rows` against it, casting each value to the +type its column declares. A value the declared type refuses aborts the batch with a `SchemaMismatchException` naming +the column and its row. Schema *inference* belongs to the readers, which sample a source and derive a schema from it before any row flows - see [Schema](/documentation/components/core/schema.md). diff --git a/documentation/components/core/caching.md b/documentation/components/core/caching.md index 232398f1a3..5053b0a47f 100644 --- a/documentation/components/core/caching.md +++ b/documentation/components/core/caching.md @@ -7,14 +7,10 @@ The goal of cache is to serialize and save on disk (or in another location defined by Cache implementation) already transformed dataset. -Cache will run a pipeline, catching each Rows and saving them into cache -from where those rows can be later extracted. +Cache runs the pipeline and saves each batch (`Rows`) into the cache, from where it can be read later. -Internally, each cached batch of Rows is stored under its own cache key, and the pipeline's cache id -points at an index - a single-column (`key`) Rows value listing those chunk keys in insertion order. - -This is useful for operations that require full transformation of dataset before -moving forward, like, for example, sorting. +Each batch is stored under its own cache key, and the pipeline's cache id points at an index - a batch with `key` +and `rows` columns listing those chunk keys and their row counts in insertion order. Another interesting use case for caching would be to share the dataset between multiple data processing pipelines. So instead of going to datasource multiple times and then repeating all transformations, only one ETL would @@ -52,21 +48,37 @@ data_frame() By default, Flow is using Filesystem Cache, location of the cache storage can be adjusted through the `FLOW_LOCAL_FILESYSTEM_CACHE_DIR` environment variable. +To only move the cache directory, set it on the config - the default cache keeps the pipeline's serializer +and backend: + +```php +cacheDir(__DIR__ . '/var/cache'); +``` + To use different cache implementation please use `ConfigBuilder` ```php cache( - new PSRSimpleCache( - new Psr16Cache( - new ArrayAdapter() + ->cache( + new PSRSimpleCache( + new Psr16Cache(new ArrayAdapter()), + new FloeSerializer(new AdaptiveBackend()), ) - ) - ); + ); ``` The following implementations are available out of the box: @@ -74,6 +86,8 @@ The following implementations are available out of the box: * [InMemory](/src/core/etl/src/Flow/ETL/Cache/Implementation/InMemoryCache.php) * [LocalFilesystem](/src/core/etl/src/Flow/ETL/Cache/Implementation/FilesystemCache.php) * [PSRSimpleCache](/src/core/etl/src/Flow/ETL/Cache/Implementation/PSRSimpleCache.php) +* [ApcuCache](/src/core/etl/src/Flow/ETL/Cache/Implementation/ApcuCache.php) - `new ApcuCache($serializer, $namespace)`, + requires `ext-apcu` PSRSimpleCache makes possible to use any of the [psr/simple-cache-implementation](https://packagist.org/providers/psr/simple-cache-implementation) @@ -81,10 +95,12 @@ but it does not come with any out of the box. ## Serialization -Persisting caches (`FilesystemCache`, `PSRSimpleCache`) turn `Rows` into bytes through a +Persisting caches (`FilesystemCache`, `PSRSimpleCache`, `ApcuCache`) turn `Rows` into bytes through a [`Serializer`](/src/core/etl/src/Flow/Serializer/Serializer.php). The default is -[`FloeSerializer`](/src/core/etl/src/Flow/Floe/FloeSerializer.php), which encodes to the -[Floe](/documentation/components/core/floe.md) binary format. +[`FloeSerializer`](/src/core/etl/src/Flow/Floe/FloeSerializer.php), which writes each batch as one +[Floe](/documentation/components/core/floe.md) BATCH frame and reads it back into the +[column backend](/documentation/components/core/column-backend.md) it was built with - +`new FloeSerializer($backend, $batchSize = 1000)`. `Cache::get()` returns one fully materialized `Rows` per key. The caching processor writes exactly one batch per cache key, so each key holds a single batch bounded by `DataFrame::cache(cacheBatchSize:)` @@ -105,16 +121,18 @@ A `Serializer` writes to and reads from streams, so the default `FilesystemCache directly to and from the cache file. `PSRSimpleCache` materializes the payload string - its PSR-16 backend stores string values - bounded by `cacheBatchSize`. -You can swap the serializer per cache; the default `FilesystemCache` shares the same serializer as the -rest of the pipeline (so it uses the context hydrator): +The default `FilesystemCache` shares the serializer of the rest of the pipeline (so it decodes into the configured +backend). Build one with `filesystem_cache()` only for a custom serializer or filesystem - a cache you build takes +its serializer from you: ```php .schema` - the entry bytes are whatever the configured serializer +produced. \ No newline at end of file diff --git a/documentation/components/core/column-backend.md b/documentation/components/core/column-backend.md new file mode 100644 index 0000000000..770f0763c9 --- /dev/null +++ b/documentation/components/core/column-backend.md @@ -0,0 +1,69 @@ +# Column Backend + +[DOC_LINK:/documentation/components/core/core.md] + +[TOC] + +Every batch (`Rows`) stores one column per schema definition. The +[Backend](/src/core/etl/src/Flow/ETL/Column/Backend.php) decides where those columns live and builds them. A data +frame has exactly one backend, set on its config: + +```php +backend(new PhpBackend())) + ->read(from_array([['id' => 1], ['id' => 2]])) + ->withEntry('double', ref('id')->multiply(lit(2))) + ->fetch(); + +$rows->column('double')::class; // Flow\ETL\Column\Php\ScalarColumn +$rows->toArray(); // [['id' => 1, 'double' => 2], ['id' => 2, 'double' => 4]] +``` + +## Backends + +| Backend | Columns | Requires | +|--------------------------------------------------------------------------|-------------------------------|--------------------------------------------------------------------------------| +| [AdaptiveBackend](/src/core/etl/src/Flow/ETL/Column/AdaptiveBackend.php) | `RustBackend` or `PhpBackend` | nothing (the default) | +| [PhpBackend](/src/core/etl/src/Flow/ETL/Column/PhpBackend.php) | PHP arrays | nothing | +| `RustBackend` | Apache Arrow arrays | the [flow_php extension](/documentation/components/extensions/flow-php-ext.md) | + +When `extension_loaded('flow_php')` is true, `AdaptiveBackend` picks `RustBackend`. Else it picks `PhpBackend`. +Both produce the same values: `PhpBackend` is the reference implementation, the extension is an accelerator. Set +`PhpBackend` explicitly to opt out of the extension while it is loaded. + +## What the backend builds + +The configured backend builds every column a pipeline produces: + +- every batch an extractor yields: a batch built elsewhere is adopted (copied once) before the first step sees it, +- function results (`withEntry()`, `filter()`, `lit()`, ...), +- joins, sorts, group-by, window and repartition outputs, including their spilled reads, +- cache and Floe reads (the default `FloeSerializer` takes the same backend). + +Code that builds batches outside a pipeline passes the backend itself - `$context->backend()` inside an extractor, +transformer or function, or any `Backend` to the DSL: + +```php + 1], ['id' => 2]], schema(int_schema('id')), new PhpBackend()); +``` + +## Memory + +`RustBackend` keeps column buffers outside PHP's memory manager, where `memory_get_usage()` and `memory_limit` do not +see them. `Backend::allocatedBytes()` reports them, and Flow adds them to the process memory its spilling steps +(sort, join, group-by) compare with their memory limit. diff --git a/documentation/components/core/constraints.md b/documentation/components/core/constraints.md index ad3a821c4f..25d3a9e998 100644 --- a/documentation/components/core/constraints.md +++ b/documentation/components/core/constraints.md @@ -49,26 +49,31 @@ You can implement custom constraints by creating classes that implement the `Con ```php get('age')->value(); - return $age >= $this->minAge && $age <= $this->maxAge; + foreach ($rows->column('age')->values() as $i => $age) { + if ($age < $this->minAge || $age > $this->maxAge) { + return $i; + } + } + + return null; } - + public function toString(): string { return "Age must be between {$this->minAge} and {$this->maxAge}"; } - - public function violation(Row $row): string + + public function violation(Rows $rows, int $index): string { - return "Age {$row->get('age')->value()} is outside allowed range"; + return "Age {$rows->column('age')->value($index)} is outside allowed range"; } } ``` \ No newline at end of file diff --git a/documentation/components/core/core.md b/documentation/components/core/core.md index e6467b93cd..02254f9456 100644 --- a/documentation/components/core/core.md +++ b/documentation/components/core/core.md @@ -16,7 +16,8 @@ A Data Frame is the core component of Flow PHP's ETL framework. It represents a ## Key Features -- **Memory Efficient**: Processes data in chunks using generators, avoiding memory exhaustion +- **Memory Efficient**: Processes data in batches using generators, avoiding memory exhaustion +- **Columnar**: Every batch stores one column per schema definition, built by a configurable [column backend](/documentation/components/core/column-backend.md) - **Lazy Evaluation**: Operations are only executed when needed - **Immutable**: Each transformation returns a new DataFrame instance - **Type Safe**: Strict typing throughout with comprehensive schema support @@ -38,7 +39,7 @@ These methods build the processing pipeline without executing it immediately: These methods execute the entire pipeline and return results: -- **Data retrieval**: `get()`, `getEach()`, `fetch()`, `count()` +- **Data retrieval**: `get()`, `getEachAsArray()`, `fetch()`, `count()` - **Output operations**: `run()`, `forEach()`, `printRows()` - **Schema inspection**: `display()` @@ -52,28 +53,37 @@ trigger it should print - `Trigger::rows` by default, `Trigger::run` for the pla for the plan `count()` executes. ```php -echo data_frame() - ->read(from_csv('orders.csv')) +$dataFrame = data_frame() + ->read(from_csv('/data/orders.csv')) ->filter(ref('email')->isNotNull()) ->write(to_json('out.json')) - ->explain()->toString(); + ->select('id', 'email'); + +echo $dataFrame->explain()->toString(); ``` ```text Outputs -├─ #3 Result +├─ #4 Result │ │ Rows this plan hands out: to the trigger, or to the node reading it -│ └─ #2 Filter -│ │ Condition: IsNotNull -│ └─ #1 Read -│ Extractor: CSVExtractor -└─ #4 Write +│ └─ #3 Select +│ │ Columns: id, email +│ └─ #2 Filter +│ │ Condition: IsNotNull +│ └─ #1 Read +│ Extractor: CSVExtractor +│ Source: file:///data/orders.csv +└─ #5 Write │ Loader: JsonLoader └─ #2 Filter (shared) ``` ```php -echo $dataFrame->explain(Trigger::run)->toString(); +echo data_frame() + ->read(from_csv('/data/orders.csv')) + ->filter(ref('email')->isNotNull()) + ->write(to_json('out.json')) + ->explain(Trigger::run)->toString(); ``` ```text @@ -83,15 +93,15 @@ echo $dataFrame->explain(Trigger::run)->toString(); │ Condition: IsNotNull └─ #1 Read Extractor: CSVExtractor + Source: file:///data/orders.csv ``` `run()` takes no rows, so its plan carries only the sinks - the `Write` is the root and there is no `Result`. A `Result` comes back only when a verb follows the last `write()`: nothing would read that chain end, so one is put on it to pull the rows, and `run(analyze: ...)` still counts them. -`Outputs` lists everything the frame produces: `Result` is what a rows-returning trigger reads, `Write` is the sink, -and both read the same rows. A node several consumers read is printed once, and every other consumer points back at it by -number - `#2 Filter (shared)` is that same filter, not a second one. A node keeps its number in every format. +`Outputs` lists everything the frame produces: `Result` is what a rows-returning trigger reads, `Write` is the sink. +A node several consumers read is printed once, and every other consumer points back at it by number - `#2 Filter (shared)` is that same filter, not a second one. A node keeps its number in every format. A frame joined with `join()` / `crossJoin()` is part of the same tree: the join reads it the way it reads any other input, numbered with the rest of the plan, and it runs with this frame's configuration. A frame read with `from_data_frame()` is a `Read` of `DataFrameExtractor` and runs with its own. @@ -102,7 +112,7 @@ input, numbered with the rest of the plan, and it runs with this frame's configu |------------------------------|-------------------------------------------------------------------------------------| | `Stage::optimized` (default) | the plan the optimizer hands to the planner | | `Stage::unoptimized` | the plan as the frame built it | -| `Stage::physical` | the plan the executor runs: pipelines, their steps, and every setting each was given | +| `Stage::physical` | the plan the executor runs: pipelines, their steps and every setting each was given | | `Format::tree` (default) | the tree above: every node above the nodes it reads | | `Format::flow` | the same tree turned around: sources first, every node above the nodes that read it | | `Format::boxes` | a box per node, children side by side | @@ -115,11 +125,14 @@ echo $dataFrame->explain()->toString(format: Format::flow); ```text #1 Read │ Extractor: CSVExtractor +│ Source: file:///data/orders.csv └─ #2 Filter │ Condition: IsNotNull - ├─ #3 Result - │ Rows this plan hands out: to the trigger, or to the node reading it - └─ #4 Write + ├─ #3 Select + │ │ Columns: id, email + │ └─ #4 Result + │ Rows this plan hands out: to the trigger, or to the node reading it + └─ #5 Write Loader: JsonLoader ``` @@ -132,21 +145,28 @@ echo $dataFrame->explain()->toString(format: Format::boxes); │ Outputs ├──────────────┐ └─────────────┬─────────────┘ │ ┌─────────────┴─────────────┐┌─────────────┴─────────────┐ -│ #3 Result ││ #4 Write │ +│ #4 Result ││ #5 Write │ │ ──────────────────── ││ ──────────────────── │ │ Rows this plan hands out: ││ Loader: JsonLoader │ │ to the trigger, or to the ││ │ │ node reading it ││ │ └─────────────┬─────────────┘└─────────────┬─────────────┘ ┌─────────────┴─────────────┐┌─────────────┴─────────────┐ -│ #2 Filter ││ #2 Filter │ +│ #3 Select ││ #2 Filter │ │ ──────────────────── ││ (shared) │ -│ Condition: IsNotNull ││ │ +│ Columns: id, email ││ │ └─────────────┬─────────────┘└───────────────────────────┘ ┌─────────────┴─────────────┐ +│ #2 Filter │ +│ ──────────────────── │ +│ Condition: IsNotNull │ +└─────────────┬─────────────┘ +┌─────────────┴─────────────┐ │ #1 Read │ │ ──────────────────── │ │ Extractor: CSVExtractor │ +│ Source: │ +│ file:///data/orders.csv │ └───────────────────────────┘ ``` @@ -181,6 +201,7 @@ Physical plan │ On: id = id │ Prefix: joined_ │ Storage: FilesystemBuckets + │ Memory: 1000 MB │ Buckets: 64 │ Batch: 1000 └─ Right side: Pipeline #0 @@ -188,6 +209,9 @@ Physical plan Statistics: rows exact 1 · size unknown ``` +`Memory: 1000 MB` is the default limit under `memory_limit = -1`. With a set `memory_limit` it is 70% of it, or +`FLOW_MAX_MEMORY`. + A joined frame is planned apart, so its pipelines are numbered apart - `Right side:` says which plan they belong to. Every source lists what it declares about itself before a row is read - `exact n`, `≤ n` (a guaranteed bound), `~n ±e%` (an estimate) or `unknown`. A source that declares nothing lists no `Statistics:` line. @@ -203,7 +227,7 @@ DataFrames are created using the `data_frame()` DSL function and populated with ```php read(from_array([ @@ -221,7 +245,7 @@ $dataFrame = data_frame() ## Memory Management Best Practices -1. **Prefer Generator Methods**: Use `get()`, `getEach()`, `getEachAsArray()` over `fetch()` for large datasets +1. **Prefer Generator Methods**: Use `get()`, `getEachAsArray()` over `fetch()` for large datasets 2. **Avoid Memory-Intensive Operations**: Be cautious with `collect()`, `sortBy()`, `groupBy()`, and `join()` on large datasets 3. **Use Appropriate Batch Sizes**: Start with 1000-5000 rows and adjust based on your memory constraints 4. **Monitor Memory Usage**: Use `run(analyze: true)` to track memory consumption during development @@ -237,13 +261,13 @@ $dataFrame = data_frame() Before a frame runs, the optimizer rewrites its plan. `Optimizer::default()` runs these rules, in order: -| Rule | Rewrite | -|------------------------|----------------------------------------------------------------------------------| -| `CombineLimits` | two stacked `limit()` calls become one, with the smaller limit | -| `CombineSortAndLimit` | `sortBy()` followed by `limit()` keeps only the top rows instead of sorting all | -| `PushLimitIntoSource` | the extractor stops reading once the limit (plus any `offset()`) is reached | -| `PushFilterIntoSource` | a `filter()` on partition columns skips whole partition directories | -| `CountFromStatistics` | `count()` over a source that knows its rows exactly reads that number, no rows | +| Rule | Rewrite | +|------------------------|---------------------------------------------------------------------------------| +| `CombineLimits` | two stacked `limit()` calls become one, with the smaller limit | +| `CombineSortAndLimit` | `sortBy()` followed by `limit()` keeps only the top rows instead of sorting all | +| `PushLimitIntoSource` | the extractor stops reading once the limit (plus any `offset()`) is reached | +| `PushFilterIntoSource` | a `filter()` on partition columns skips whole partition directories | +| `CountFromStatistics` | `count()` over a source that knows its rows exactly reads that number, no rows | Rules live in `Flow\ETL\Optimizer\Rule` and are configured through `config_builder()->optimizer()`: @@ -273,7 +297,7 @@ data_frame(config_builder()->optimizer(Optimizer::default()->with(new MyRule())) For detailed information about specific DataFrame operations, see the following component documentation: ### Core Operations -- **[Building Blocks](/documentation/components/core/building-blocks.md)** - Understanding Rows, Entries, and basic data structures +- **[Building Blocks](/documentation/components/core/building-blocks.md)** - Rows, columns and the schema that describes them - **[Transformations](/documentation/components/core/transformations.md)** - Reusable DataFrame transformations and the Transformation interface - **[Select/Drop](/documentation/components/core/select-drop.md)** - Column selection and removal - **[Rename](/documentation/components/core/rename.md)** - Column renaming strategies @@ -291,10 +315,11 @@ For detailed information about specific DataFrame operations, see the following - **[Window Functions](/documentation/components/core/window-functions.md)** - Advanced analytical functions ### Memory & Performance +- **[Column Backend](/documentation/components/core/column-backend.md)** - Where batch columns live: PHP arrays or the flow_php extension - **[Batch Processing](/documentation/components/core/batch-processing.md)** - Controlling batch sizes and memory collection - **[Partitioning](/documentation/components/core/partitioning.md)** - Data partitioning for efficient processing - **[Caching](/documentation/components/core/caching.md)** - Performance optimization through caching -- **[Floe File Format](/documentation/components/core/floe.md)** - Flow's native self-describing binary row format +- **[Floe File Format](/documentation/components/core/floe.md)** - Flow's native self-describing columnar binary format - **[Source Statistics](/documentation/components/core/statistics.md)** - What a source declares about its rows and bytes before it is read - **[Data Retrieval](/documentation/components/core/data-retrieval.md)** - Methods for getting processed data diff --git a/documentation/components/core/data-manipulation.md b/documentation/components/core/data-manipulation.md index c2e4f9e50c..dae7a62fc9 100644 --- a/documentation/components/core/data-manipulation.md +++ b/documentation/components/core/data-manipulation.md @@ -37,7 +37,7 @@ data_frame() > the richest type that parses, so `'19.99'` read from a CSV infers as `?float`. Restrict the candidates with > `->types(...)`, or take strings only with `->allStrings()`. -## Adding Entries with withEntry() +## Adding Columns with withEntry() Add new columns or modify existing ones using expressions: @@ -61,7 +61,9 @@ data_frame() Create duplicate rows for testing or data expansion: -### duplicateRow() - Duplicate Specific Row +### duplicateRow() - Duplicate Matching Rows + +`duplicateRow()` takes a boolean condition; every row it is true for appears twice. ```php 2, 'name' => 'Product B'], ['id' => 3, 'name' => 'Product C'], ])) - ->duplicateRow(1) // Duplicate the second row (0-indexed) + ->duplicateRow(ref('id')->equals(lit(2))) ->write(to_output()) ->run(); @@ -81,29 +83,7 @@ data_frame() ## Removing Duplicates -Remove duplicate rows from your dataset: - -```php -read(from_array([ - ['id' => 1, 'name' => 'Product A', 'category' => 'Electronics'], - ['id' => 2, 'name' => 'Product B', 'category' => 'Books'], - ['id' => 1, 'name' => 'Product A', 'category' => 'Electronics'], // Duplicate - ['id' => 3, 'name' => 'Product C', 'category' => 'Electronics'], - ['id' => 2, 'name' => 'Product B', 'category' => 'Books'], // Duplicate - ])) - ->dropDuplicates() // Remove all duplicate rows - ->write(to_output()) - ->run(); - -// Result: Only unique rows remain -``` - -### Selective Duplicate Removal - -Remove duplicates based on specific columns: +`dropDuplicates()` takes at least one column and keeps the first row of each distinct combination of those columns: ```php read(from_array([ ['id' => 1, 'name' => 'Product A', 'version' => 1], - ['id' => 1, 'name' => 'Product A', 'version' => 2], // Same product, different version + ['id' => 1, 'name' => 'Product A', 'version' => 2], ['id' => 2, 'name' => 'Product B', 'version' => 1], - ['id' => 3, 'name' => 'Product C', 'version' => 1], ])) - ->dropDuplicates('id', 'name') // Remove duplicates based on id and name only + ->dropDuplicates('id', 'name') ->write(to_output()) ->run(); -// Result: Keep first occurrence of each id/name combination +// Result: [1, 'Product A', 1], [2, 'Product B', 1] ``` diff --git a/documentation/components/core/data-retrieval.md b/documentation/components/core/data-retrieval.md index 3192a17f29..4344f242ea 100644 --- a/documentation/components/core/data-retrieval.md +++ b/documentation/components/core/data-retrieval.md @@ -23,7 +23,7 @@ $dataFrame = data_frame()->read(from_array($largeDataset)); foreach ($dataFrame->get() as $rows) { echo "Processing batch of " . $rows->count() . " rows\n"; // Process each batch - foreach ($rows as $row) { + foreach ($rows->toArray() as $row) { // Process individual row } } @@ -36,17 +36,6 @@ which is what lets you call `get()` or `run()` on it more than once, exactly as array. (`schema()` never consumed the source in the first place - it answers from the plan.) Declare the schema up front with `from_array($largeDataset)->withSchema($schema)` to skip the spill entirely; a declared schema streams the source directly and can therefore be read only once. -### getEach() - Retrieve individual Rows - -```php -getEach() as $row) { - echo "ID: " . $row->get('id')->value() . "\n"; - echo "Name: " . $row->get('name')->value() . "\n"; -} -``` - ### getAsArray() - Retrieve as array batches ```php @@ -78,8 +67,8 @@ foreach ($dataFrame->getEachAsArray() as $rowArray) { // Fetch limited results (safe) $firstTen = $dataFrame->fetch(10); -foreach ($firstTen as $row) { - // Process row +foreach ($firstTen->toArray() as $row) { + echo "ID: " . $row['id'] . "\n"; } // Fetch all results (dangerous for large datasets!) diff --git a/documentation/components/core/error-handling.md b/documentation/components/core/error-handling.md index d4c405296d..b68c2b8b99 100644 --- a/documentation/components/core/error-handling.md +++ b/documentation/components/core/error-handling.md @@ -5,19 +5,19 @@ By default an exception thrown while reading, transforming or loading stops the pipeline. An [ErrorHandler](/src/core/etl/src/Flow/ETL/ErrorHandler.php) decides per failure instead: -| Method | Error carries | Action other than `propagate` (rethrow) | -|---|---|---| -| `onExtraction(ExtractionError): ExtractionAction` | `cause`, `extractor` | `endSource` - stop reading; rows read so far still flow and the run completes | -| `onTransformation(TransformationError): TransformationAction` | `cause`, `transformer`, `rows` | `skipBatch` - drop the batch, continue with the next one | -| `onLoading(LoadingError): LoadingAction` | `cause`, `loader`, `rows` | `skipLoader` - the failing loader misses this batch, the remaining steps still run | +| Method | Error carries | Action other than `propagate` (rethrow) | +|---------------------------------------------------------------|--------------------------------|------------------------------------------------------------------------------------| +| `onExtraction(ExtractionError): ExtractionAction` | `cause`, `extractor` | `endSource` - stop reading; rows read so far still flow and the run completes | +| `onTransformation(TransformationError): TransformationAction` | `cause`, `transformer`, `rows` | `skipBatch` - drop the batch, continue with the next one | +| `onLoading(LoadingError): LoadingAction` | `cause`, `loader`, `rows` | `skipLoader` - the failing loader misses this batch, the remaining steps still run | Built-in handlers: -| Handler | Extraction | Transformation | Loading | -|---|---|---|---| -| `throw_error_handler()` (default) | `propagate` | `propagate` | `propagate` | -| `ignore_error_handler()` | `endSource` | `skipBatch` | `skipLoader` | -| `skip_rows_handler()` | `endSource` | `skipBatch` | `propagate` | +| Handler | Extraction | Transformation | Loading | +|-----------------------------------|-------------|----------------|--------------| +| `throw_error_handler()` (default) | `propagate` | `propagate` | `propagate` | +| `ignore_error_handler()` | `endSource` | `skipBatch` | `skipLoader` | +| `skip_rows_handler()` | `endSource` | `skipBatch` | `propagate` | ```php upper()` over a null column. To tolerate the failure for +A scalar function that receives a value it cannot process throws `EvaluationException` (an +`InvalidArgumentException`) with the failing row of the batch, for example `ref('text')->upper()` over a null value: +`ToUpper function requires non-null value (row 1)`; the original exception is `getPrevious()`. To tolerate the failure for one function instead of the whole pipeline, wrap it in `optional()` - it evaluates the wrapped function and returns null when it throws: @@ -90,26 +91,31 @@ For fine-grained error handling during row processing operations: read($unreliableDataExtractor) - ->forEach(function(Row $row) use (&$successCount, &$errorCount) { - try { - validateAndProcess($row); - $successCount++; - } catch (InvalidArgumentException $e) { - logInvalidRow($row, $e->getMessage()); - $errorCount++; - } catch (Exception $e) { - logGeneralError($row, $e); - $errorCount++; + ->read(from_array([['id' => 1], ['id' => 2]])) + ->forEach(function(Rows $rows) use (&$successCount, &$errorCount) { + foreach ($rows->toArray() as $row) { + try { + validateAndProcess($row); + $successCount++; + } catch (InvalidArgumentException $e) { + logInvalidRow($row, $e->getMessage()); + $errorCount++; + } catch (Exception $e) { + logGeneralError($row, $e); + $errorCount++; + } } }); diff --git a/documentation/components/core/filter.md b/documentation/components/core/filter.md index 52df5d6767..0edb27daa4 100644 --- a/documentation/components/core/filter.md +++ b/documentation/components/core/filter.md @@ -15,36 +15,83 @@ data_frame() ['a' => 100, 'b' => 100], ['a' => 100, 'b' => 200] ])) - ->filter(ref('b')->divide(lit(2))->equals(lit('a'))) + ->filter(ref('b')->divide(lit(2))->equals(ref('a'))) ->write(to_output(false)) ->run(); ``` -## Complex Row-level Filtering +```text ++-----+-----+ +| a | b | ++-----+-----+ +| 100 | 200 | ++-----+-----+ +1 rows +``` + +## Custom Filter Functions -For advanced filtering that requires custom business logic, you can use callback functions: +Business logic the built-in functions do not cover goes into a `ScalarFunction` that returns a boolean column: ```php amount, $this->type]; + } + + public function withChildren(array $children): static + { + return new self($children[0], $children[1]); + } + + public function returns(): Type + { + return type_boolean(); + } + + public function eval(Rows $rows, FlowContext $context): Column + { + $amounts = (new Parameter($this->amount))->asNumbers($rows, $context); + $types = (new Parameter($this->type))->asStrings($rows, $context); + $results = []; + + foreach ($amounts as $i => $amount) { + $results[] = $amount > 1000 && $types[$i] === 'purchase'; + } + + return (new ResultColumn($context->backend()))->of($this, $results); + } +} data_frame() - ->read($transactionExtractor) - ->filter(function(Row $row): bool { - $amount = $row->get('amount')->value(); - $type = $row->get('type')->value(); - $date = $row->get('date')->value(); - - // Complex business logic - return $amount > 1000 - && $type === 'purchase' - && $date > new DateTime('-30 days'); - }) - ->write($highValueTransactionLoader) + ->read(from_array([ + ['amount' => 1500, 'type' => 'purchase'], + ['amount' => 1500, 'type' => 'refund'], + ['amount' => 200, 'type' => 'purchase'], + ])) + ->filter(new HighValuePurchase(ref('amount'), ref('type'))) + ->write(to_output(false)) ->run(); ``` -> **Performance Note**: Callback-based filtering cannot be optimized by the engine and should be used sparingly. When possible, prefer built-in scalar functions for better performance. +> **Performance Note**: prefer built-in scalar functions when they cover the logic. - [Until](/documentation/components/core/until.md) diff --git a/documentation/components/core/floe.md b/documentation/components/core/floe.md index 4db4d062c9..2ea723b4ca 100644 --- a/documentation/components/core/floe.md +++ b/documentation/components/core/floe.md @@ -4,9 +4,10 @@ [TOC] -Floe is Flow's native, self-describing binary file format for `Rows`. It stores the schema inside -the file, evolves seamlessly across appended sections, and reads back through the DataFrame API with -`from_floe()` / `to_floe()`. Files use the `.floe` extension. +Floe is Flow's native, self-describing, columnar binary file format. Each batch is written as one BATCH frame holding +every column in the Arrow buffer layout, the schema lives in the file's footer, and `from_floe()` reads the batches +back into the configured [column backend](/documentation/components/core/column-backend.md). Files use the `.floe` +extension. ## Writing @@ -38,10 +39,22 @@ data_frame() ->run(); ``` -When the [`flow_php` extension](/documentation/components/extensions/flow-php-ext.md) is loaded, the -strict read and the write fuse frame-split + value decode/encode + hydrate/dehydrate into one native -call per batch. It is transparent - the on-disk format is unchanged and the rows are byte-for-byte -identical to the pure-PHP engine. +Batches hold at most `withBatchSize()` rows, never spanning two BATCH frames. + +The footer is the schema: `from_floe(...)->withSchema()` throws. Change a column's type or zone after reading: + +```php +read(from_floe(__DIR__ . '/output.floe')) + ->withEntry('at', to_timezone(ref('at'), 'Europe/Warsaw')) + ->write(to_output()) + ->run(); +``` ## Save Modes @@ -57,8 +70,7 @@ machinery as other file loaders: ```php schema(flow_context(config())); +$schema = from_floe(__DIR__ . '/output.floe')->schema(); ``` ## Reading the First or Last Rows @@ -146,10 +158,11 @@ boundary section onward - the leading rows are never read: ```php read(path(__DIR__ . '/output.floe')); +$file = (new FloeReader(native_local_filesystem(), new AdaptiveBackend()))->read(path(__DIR__ . '/output.floe')); foreach ($file->head(300) as $rows) { // first 300 rows; stops reading once 300 are yielded @@ -160,7 +173,7 @@ foreach ($file->tail(300) as $rows) { } ``` -Both yield `Rows` in batches (default 1000, override with the second argument) and return every row when +Both yield `Rows` in batches of at most 1000 rows (override with the second argument) and return every row when the file holds fewer than the requested count. For the first N through the DataFrame API use `->limit(N)` (pushed into the extractor); `tail()` is a reader-level convenience because it needs the file's total from the footer. @@ -169,210 +182,151 @@ from the footer. `merge_floe()` combines several `.floe` files (same or append-compatible evolving schema) into one. The default byte-splices frame regions without re-encoding a single row - O(bytes); `compact: true` -re-encodes every row, coalescing same-schema runs into fewer sections: +re-reads and re-writes every batch, coalescing same-schema runs into fewer sections: ```php read(from_array([['id' => 1], ['id' => 'AB-1']])) - ->write(to_floe(__DIR__ . '/orders.floe')) - ->run(); -``` - -The first batch fixes the file's schema to `integer`, so the second one throws: +use Flow\ETL\Column\PhpBackend; +use Flow\Floe\FloeWriter; +use function Flow\ETL\DSL\{array_to_rows, int_schema, schema, str_schema}; +use function Flow\Filesystem\DSL\{native_local_filesystem, path}; +$writer = new FloeWriter(native_local_filesystem(), schema(int_schema('id')), new PhpBackend()); +$writer->create(path(__DIR__ . '/orders.floe')); +$writer->write(array_to_rows([['id' => 1]], schema(int_schema('id')), new PhpBackend())); +$writer->write(array_to_rows([['id' => 'AB-1']], schema(str_schema('id')), new PhpBackend())); ``` -Floe write session schema is fixed and this batch does not fit it: column "id" (row 0): -could not convert 'AB-1' (string) to integer. -``` - -Both checks throw `IncompatibleSchemaException` before any bytes reach the destination, so a rejected -batch leaves the file readable. Other messages take the same shape: ``` -column "amount" (row 0): could not convert null to string, column is not nullable -new column "email" +Floe write session schema is fixed and this batch does not fit it: Mismatched Definitions: + |-- expected: id, given: id ``` -`validate_data: false` skips the per-value check only. The column-set check still runs, so silent -column loss cannot be unlocked. This mirrors parquet-java's `ParquetWriter::withValidation()`. +The check runs before any bytes reach the destination, so a rejected batch leaves the file readable. -Two behaviours differ from other columnar formats on purpose: - -- **Absent is not null.** A row that omits a column writes an absent flag and reads back as null; - a row that carries an explicit `null` in a non-nullable column is rejected. Parquet turns a missing - optional field into a null and rejects a missing required one; Arrow cannot omit a column at all. -- **An int is not a float.** An integer value in a `float` column is rejected. pyarrow, parquet-java - and Avro all widen it silently. - -## Unsupported column types +## On-Disk Layout -Floe stores one type per column, so a column whose type is only known per value cannot be written. -These throw `FloeException` when the write session opens: +A `.floe` file is a 6-byte header, a stream of length-prefixed frames and a JSON footer located from the last +8 bytes. The schema lives only in the footer. Integers are little-endian. ``` -Floe does not support values of type "mixed" -Floe does not support map keys of type "uuid" +HEADER 6 bytes +FRAMES (write order) + BATCH (0x05) one per write() + ... +FOOTER (0x06) footer JSON + TRAILER (8 bytes) ``` -That covers `mixed`. -Use a declared element type, or a `json_schema()` column when the shape is genuinely dynamic. +### Header (6 bytes) -## On-Disk Layout +``` +'F' 'L' 'O' 'E' version codec + 0x02 0x00 = no compression +``` -A `.floe` file is a fixed 6-byte header, a stream of length-prefixed frames, and a JSON footer that a -reader can locate from the last 8 bytes without scanning the body. The schema lives **only in the -footer** - there is no inline schema frame. All multi-byte integers are **little-endian**. +### Frame envelope ``` -┌──────────────────────────────────────────────────────────┐ -│ HEADER 6 bytes │ -├──────────────────────────────────────────────────────────┤ -│ FRAMES (repeated, in write order) │ -│ PARTITIONS (0x03) emitted when combination changes │ -│ ROW (0x02) one frame per row │ -│ ROW (0x02) │ -│ ... │ -├──────────────────────────────────────────────────────────┤ -│ FOOTER FRAME (0x06) │ -│ footer JSON + TRAILER (8 bytes) │ -└──────────────────────────────────────────────────────────┘ +type 1 byte 0x05 BATCH 0x06 FOOTER +length 4 bytes (u32) +body bytes ``` -### Header (6 bytes) +Any other frame type is refused with `Floe found unknown frame type 0x%02X`. + +### BATCH body + +One batch in columnar form: a plaintext directory, then the buffer area. ``` - byte 0 1 2 3 4 5 - ┌────┬────┬────┬────┬────────┬────────┐ - │ 'F'│ 'L'│ 'O'│ 'E'│ version│ flags │ - └────┴────┴────┴────┴────────┴────────┘ - magic "FLOE" 0x02 codec id - (0x00 = no compression) +rowCount u32 +nodeCount u32 +bufferCount u32 +nodes nodeCount × {length u32, nullCount u32} pre-order, every column's type tree +extents bufferCount × {offset u32, length u32} into the buffer area +padding to 8 bytes +buffer area per non-empty buffer: i64 uncompressed length (-1 = stored raw) + bytes, padded to 8 ``` -### Frame envelope +- Nodes per type: scalar and null 1, list 1 + element, map 2 (map, entries) + key + value, structure 1 + children. +- Buffers per node: validity (`''` when the node holds no nulls), then offsets for list, map and string kinds, + then values/data; a null column has none. +- An empty buffer is extent `{0, 0}` with no prefix. +- A codec compresses each buffer on its own; the buffer is stored raw (`-1`) whenever the codec output is not + smaller. The directory is never compressed. -Every frame - `ROW`, `PARTITIONS`, `FOOTER` - shares the same envelope: a 1-byte type, a -4-byte little-endian body length, then the body. +`[['id' => 1, 'name' => 'ab'], ['id' => 2, 'name' => null]]` under `int_schema('id')`, +`str_schema('name', nullable: true)` - 72-byte directory, 80-byte buffer area: ``` - ┌────────┬───────────────┬───────────────────────┐ - │ type │ body length │ body │ - │ 1 byte │ 4 bytes (LE) │ bytes │ - └────────┴───────────────┴───────────────────────┘ - 0x02 ROW 0x03 PARTITIONS 0x06 FOOTER +02000000 02000000 05000000 rows 2, nodes 2, buffers 5 +02000000 00000000 02000000 01000000 id {2,0}, name {2,1} +00000000 00000000 00000000 18000000 18000000 09000000 id validity {0,0}, id values {0,24}, name validity {24,9} +28000000 14000000 40000000 0a000000 00000000 name offsets {40,20}, name data {64,10}, padding +ffffffffffffffff 0100000000000000 0200000000000000 id values +ffffffffffffffff 01 00000000000000 name validity +ffffffffffffffff 000000000200000002000000 00000000 name offsets +ffffffffffffffff 6162 000000000000 name data ``` -The file's schema is not a frame - it carries **exactly one schema**, fixed at session start (explicit, -or the first batch's union) and stored only in the footer (see `schema` below). A row narrower than the -schema rides it (see the ROW absent flag). A later batch that introduces a new column or an incompatible -type throws `IncompatibleSchemaException` - schema evolution lives only in `merge_floe()`. - -- **ROW (`0x02`)** - one row encoded **by column** against the file schema: for each schema - column, in order, a one-byte presence flag - - - `0x01` **present** - followed by the value encoded per the column's schema type, no per-value tag. - - `0x00` **null** - the value is null. - - `0x02` **null with metadata** - a null value carrying per-value metadata that diverges from the - column's schema metadata: a metadata block (4-byte little-endian JSON length + JSON), no value. - - `0x03` **absent** - the row has no such column. - - `0x04` **present with metadata** - a divergent-metadata block (4-byte length + JSON) followed by - the encoded value. - - The two metadata flags (`0x02`, `0x04`) appear only when a row's per-value metadata differs from the - column's schema metadata; otherwise every value uses `0x00`/`0x01`. Every value is encoded positionally - against its column type, so no value carries a type tag. A row whose columns exactly match the section, - with no divergent metadata, produces the same bytes as a plain positional encode. One frame per row. - - A **structure** value nests the same idea one level down: one flag byte per declared element, in the - structure type's declared field order, with no element names on the wire - - - - `0x01` **present** - followed by the element value encoded per its declared type. - - `0x00` **null** - the element key is present in the value, holding null. - - `0x03` **absent** - the value has no such key (an `optional` element that was omitted). - - Because the bytes carry no names, the declared field order binds each flag byte to its element. The - writer looks each element up **by name** in the value and emits in declared order, so a value whose - keys arrive in a different order still produces the declared-order bytes; the reader rebuilds keys - in declared order. -- **PARTITIONS (`0x03`)** - the partition key/value pairs for the section that follows: a 4-byte count - followed by repeated `[nameLen(4), name, valueLen(4), value]`. Written at the start of every section - whose combination differs from the previous one; the reader starts at the empty combination, so an - unpartitioned first section emits none and a later change back to unpartitioned emits a `count=0` - frame. -- **FOOTER (`0x06`)** - the footer JSON followed by the trailer (below). - ### Footer -The `FOOTER` frame body is a JSON object carrying everything needed to read the file **without -scanning rows**: - ```json { "version": 2, "writer": "1.x-dev", - "schema": { /* the file's single schema */ }, - "sections": [ { "offset": 6, "partitionsId": 0, "rowCount": 2 } ], - "partitions": [ { "country": "PL" } ], - "statistics": { "rows": 2, "byteSize": 118 }, - "metadata": { /* typed key/value, Schema\Metadata */ } + "schema": [ /* the file's single schema */ ], + "sections": [ { "offset": 6, "rowCount": 2 } ], + "statistics": { "rows": 2, "byteSize": 157 }, + "metadata": { /* Schema\Metadata */ } } ``` -- **`statistics`** holds the file's row count and the uncompressed bytes of its data frames (header and - footer excluded), so a reader can size the file before reading a row. The footer and its sections - ignore keys they do not know, so a field added later does not break an older reader. - -- **`sections`** map a byte `offset` → `partitionsId` + `rowCount`, so a reader can skip whole sections - (offset/limit pushdown) and know each section's partition combination up front. Sections bound - partition combinations and appends only; every section shares the file's one schema. -- **`partitions`** is the deduplicated, order-preserving table of partition combinations, indexed by - `partitionsId`; the unpartitioned combination is an empty object at its own id. A file can hold many - combinations (one per section). -- **`schema`** is the file's single schema and the sole place it is stored, so every read decodes - against it without scanning the body. On a read, a row narrower than it (an absent column) is padded - with null entries so every yielded row conforms. `merge_floe()` re-encodes drifted sources to their - union, so a merged file is still one schema. +- `sections` - byte `offset` and `rowCount` of each section, for offset/limit pushdown without a scan. +- `statistics` - row count and the bytes of the data frames (header and footer excluded). +- Unknown keys are ignored, so a field added later does not break an older reader. ### Trailer (last 8 bytes) diff --git a/documentation/components/core/group-by.md b/documentation/components/core/group-by.md index 3412d5681e..d6d3ddfa8f 100644 --- a/documentation/components/core/group-by.md +++ b/documentation/components/core/group-by.md @@ -2,11 +2,8 @@ [DOC_LINK:/documentation/components/core/core.md] -Grouping is a common operation in data analysis. The concept is no different from the one you know from SQL. - -To group data you need to use `DataFrame::groupBy` function. - -Example: +`DataFrame::groupBy()` returns a `GroupedDataFrame`; `aggregate()` turns it back into a `DataFrame` with one row per +group: ```php data_frame() @@ -16,41 +13,33 @@ data_frame() ['id' => 3, 'group' => 'A'], ['id' => 4, 'group' => 'B'], ['id' => 5, 'group' => 'A'], - ['id' => 6, 'group' => 'B'], - ['id' => 7, 'group' => 'A'], - ['id' => 8, 'group' => 'B'], - ['id' => 9, 'group' => 'A'], - ['id' => 10, 'group' => 'B'], ])) ->groupBy([ref('group')]) + ->aggregate(count(ref('id'))) ->write(to_output(truncate: false)) ->run(); ``` -However, the result of this operation is not very useful. It will just return a `DataFrame` with one column `group`: - ```console -+-------+ -| group | -+-------+ -| A | -| B | -+-------+ ++-------+----------+ +| group | id_count | ++-------+----------+ +| A | 3 | +| B | 2 | ++-------+----------+ 2 rows ``` -To make it more useful, you need to use one of the aggregation functions. - [Aggregations](/documentation/components/core/aggregations.md) ## Buckets Storage -`groupBy()` partitions rows by the hashed grouping key into buckets through a `BucketsStorage` - the -same abstraction used by [external sort](/documentation/components/core/sort.md) and -[join](/documentation/components/core/join.md). The default is `FilesystemBuckets`: buckets are -spilled to the local filesystem cache directory (as Floe files), so memory usage is bounded by the -largest bucket instead of the whole grouped dataset. Other implementations are `MemoryBuckets` -(buckets kept in memory) and `PSRCacheBuckets` (buckets in any PSR-16 cache). +While the process stays under the memory limit, `groupBy()` aggregates in one pass and writes nothing. Past the limit it +partitions the rest of the stream by the hashed grouping key into buckets of a `BucketsStorage` - the same abstraction +used by [external sort](/documentation/components/core/sort.md) and [join](/documentation/components/core/join.md) - +aggregates each bucket on its own and merges the groups aggregated before the limit into them. The default storage is +`FilesystemBuckets` (Floe files in the local cache directory); `MemoryBuckets` and `PSRCacheBuckets` (any PSR-16 +cache) are the alternatives. `groupBy()` takes an optional trailing `GroupByAlgorithmBuilder`, so one operation can override the configured algorithm: @@ -59,8 +48,8 @@ algorithm: ->groupBy([ref('country')], hash_group_by()->storage(new MemoryBuckets())) ``` -The algorithm is configured through `config_builder()->groupBy(hash_group_by())` - all its options -live on the `hash_group_by()` builder: +The algorithm is configured through `config_builder()->groupBy(hash_group_by())`. `memoryLimit()` defaults to the +same limit as the join and the sort (`FLOW_MAX_MEMORY`, otherwise 70% of `memory_limit`): ```php groupBy( hash_group_by() - ->bucketsCount(64) // number of buckets (default 64) - ->batchSize(1000) // rows per batch when reading buckets back (default 1000) - ->storage(new MemoryBuckets()) // keep buckets in memory instead of on disk + ->memoryLimit(Unit::fromMb(512)) // process memory past which buckets go to the storage + ->bucketsCount(64) // number of buckets (default 64) + ->batchSize(1000) // rows per bucket frame and per batch read back (default 1000) ) ) ->read(from_parquet('orders.parquet')) @@ -80,23 +69,17 @@ data_frame( ->run(); ``` -> While rows are partitioned, each spilled bucket is announced downstream as a single metadata row -> (`BucketShape`: `_bucket_id`, `_bucket_total_rows`). Those rows are an internal pipeline detail consumed -> by the aggregation processor - they never appear in the grouped output. -### Spill column pruning +### Column pruning -Before rows are spilled to buckets, they are pruned to only the columns the group by actually reads: -the grouping references plus every column referenced by the aggregators. A row that is missing one of -those columns is normalized to a `null` entry for it (Spark/DuckDB semantics) - aggregators silently -skip `null` values, and STRICT schema mode does not throw for aggregate columns missing under a -`groupBy()`. +Before aggregating, the batch is pruned to the columns the group by reads: the grouping references plus every column +referenced by the aggregators. A grouping or aggregated column the input does not have throws +`SchemaDefinitionNotFoundException`. -Custom aggregators participate through `AggregatingFunction::references()`: return the references the -aggregator reads, or `null` when they cannot be statically enumerated - returning `null` disables -column pruning for the whole pipeline (rows are spilled with all their columns). +Custom aggregators take part through `AggregatingFunction::references()`: return the references the aggregator reads, +or `null` when they cannot be listed - `null` turns pruning off for that group by. ### Output order -Grouped output follows hash-bucket order, not input order. Group order was never a contract - apply -[sortBy](/documentation/components/core/sort.md) after aggregation when a specific order is needed. \ No newline at end of file +Group order is not a contract - past the memory limit groups come out in bucket order. For a specific +order, apply [sortBy](/documentation/components/core/sort.md) after aggregation. \ No newline at end of file diff --git a/documentation/components/core/join.md b/documentation/components/core/join.md index 32cfd74263..2878496772 100644 --- a/documentation/components/core/join.md +++ b/documentation/components/core/join.md @@ -13,7 +13,7 @@ called, and runs with the left DataFrame's configuration and error handler. ### join () -Main join method that partitions the right DataFrame into buckets and probes them with left rows through a hash table. +Main join method: builds a hash table from the right DataFrame and probes it with left rows. ### crossJoin () - Cartesian Product @@ -21,7 +21,8 @@ Joins each row from the left side with each row on the right side, creating `cou ### joinEach () - Streaming Join -Right side is dynamically generated for each left row, useful for large right-side datasets that don't fit in memory. +The right side is built for each left batch by `DataFrameFactory::from(Rows $rows)`, useful for large right-side +datasets that don't fit in memory. ## Join Types @@ -38,36 +39,36 @@ Joins follow SQL semantics - every matching pair of rows produces one output row right rows is emitted multiple times. A `null` join key never matches anything, including another `null` - rows with `null` keys are dropped by inner joins and null-padded (or kept, for `left_anti`) by outer joins. -> Flow uses hash join implementation where hashes are stored in buckets to optimize memory usage and performance. -> Rows are bucketed by the values of the join columns and every candidate pair is verified against +> Rows are hashed by the values of the join columns and every candidate pair is verified against > the join expression, so non-equality expressions (like `compare_any()`) are supported as well. > Mixed expressions still hash by their equality conditions - in `compare_all(Equal, Any)` rows are -> bucketed by the `Equal` columns and the `Any` part is verified per candidate pair; only joins with +> hashed by the `Equal` columns and the `Any` part is verified per candidate pair; only joins with > no equality condition at all fall back to comparing every pair. ## Buckets Storage -`join()` always partitions the right DataFrame into buckets through a `BucketsStorage` - the same abstraction used by -external sort - and the configured implementation decides how the join executes: +While the process stays under the memory limit, `join()` holds the right DataFrame in memory and streams the left one +through its hash table, so left rows keep their order. Past the limit both sides are partitioned by join key into +buckets of a `BucketsStorage` - the same abstraction used by external sort - and joined bucket pair by bucket pair: -| Buckets storage | Behavior | -|-----------------------------------|-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| `FilesystemBuckets` - **Default** | Both sides are partitioned by join key into buckets spilled to disk (Floe files), then joined pair by pair - the hash table is built from the smaller bucket of each pair. Memory usage is bounded by that bucket, left row order is not preserved. | -| `MemoryBuckets` | Right DataFrame is held in memory, left rows are streamed through a hash table and keep their order. Memory usage is bounded by the right side. | -| `PSRCacheBuckets` | Buckets are spilled into any PSR-16 cache. Executes like `FilesystemBuckets` (left row order is not preserved). | +| Buckets storage | Past the memory limit | +|-----------------------------------|---------------------------------------------------------------------------------------------------------------------------------------------| +| `FilesystemBuckets` - **Default** | Both sides are spilled to disk (Floe files); the hash table is built from the smaller bucket of each pair. Left row order is not preserved. | +| `PSRCacheBuckets` | Like `FilesystemBuckets`, with buckets in any PSR-16 cache. | +| `MemoryBuckets` | Never partitions: the right DataFrame stays in memory whatever its size and left rows keep their order. | `join()` takes an optional trailing `JoinAlgorithmBuilder`, so one join can override the configured algorithm: ```php ignore -->join($right, on(['id' => 'id']), Join::left, hash_join()->storage(new MemoryBuckets())) +->join($right, join_on(['id' => 'id']), Join::left, hash_join()->storage(new MemoryBuckets())) ``` -`joinEach()` deliberately does not take one - it builds a DataFrame per row, so a per-call algorithm would be -rebuilt per row too. +`joinEach()` does not take one - it builds a DataFrame per left batch, so a per-call algorithm would be rebuilt per +batch too. The join algorithm is configured through `config_builder()->join(hash_join())` - all its options live on the -`hash_join()` builder; any storage implementing `ResidentBucketsStorage` (like `MemoryBuckets`) -enables the streaming, order-preserving execution: +`hash_join()` builder. `memoryLimit()` defaults to `FLOW_MAX_MEMORY`, otherwise 70% of PHP's `memory_limit` (1 GB +when it is `-1`), and is compared with the whole process, so a join and a sort in one pipeline stay under it together: ```php join( hash_join() - ->storage(new MemoryBuckets()) // right side fits in memory, keep left row order - ->bucketsCount(64) // number of disk buckets - ->batchSize(1000) // rows per batch when reading buckets back + ->memoryLimit(Unit::fromMb(512)) // process memory past which rows go to the storage + ->bucketsCount(64) // number of buckets + ->batchSize(1000) // rows per bucket frame and per batch read back ) ) ->read(from_parquet('orders.parquet')) ->join( data_frame()->read(from_parquet('sellers.parquet')), - join_on(['seller_id' => 'id'], join_prefix: 'seller_'), + join_on(['seller_id' => 'id'], join_prefix: 'joined_'), ) ->run(); ``` -> While rows are partitioned, each spilled bucket is announced downstream as a single metadata row -> (`BucketShape`: `_bucket_id`, `_bucket_total_rows`). Those rows are an internal pipeline detail consumed -> by the join processor - they never appear in the joined output. ## Example diff --git a/documentation/components/core/partitioning.md b/documentation/components/core/partitioning.md index 3ab9e81c47..7b004e4e4d 100644 --- a/documentation/components/core/partitioning.md +++ b/documentation/components/core/partitioning.md @@ -266,5 +266,7 @@ Output carries `path`, a `partitions` map, and one string column per partition ( ## Repartitioning -`repartition()` shuffles rows between batches in memory. It does not write directories - use the +`repartition()` shuffles the stream so every row sharing the given columns arrives in one batch. While the process +stays under the memory limit it groups in memory; past it the input is partitioned into buckets first, configured with +`config_builder()->repartition(hash_repartition()->memoryLimit(...))`. It does not write directories - use the loader's `partitionBy()` for that. diff --git a/documentation/components/core/pivot.md b/documentation/components/core/pivot.md index 1ff890945e..2827a62d2c 100644 --- a/documentation/components/core/pivot.md +++ b/documentation/components/core/pivot.md @@ -32,20 +32,21 @@ $salesData = data_frame() ['region' => 'South', 'product' => 'Phone', 'month' => 'Jan', 'sales' => 700], ['region' => 'South', 'product' => 'Phone', 'month' => 'Feb', 'sales' => 850], ])) - ->groupBy(['region', 'product']) + ->groupBy(['region']) ->pivot(col('product'), pivot_values('Laptop', 'Phone')) // creates the 'Laptop' and 'Phone' columns ->aggregate(sum(col('sales')->as('total_sales'))) - ->write(to_output()) + ->write(to_output(truncate: false)) ->run(); ``` -**Result Structure:** - -``` -| region | Laptop | Phone | -|--------|--------|-------| -| North | 2200 | 1700 | -| South | 2400 | 1550 | +```console ++--------+-------------+-------------+ +| region | Laptop | Phone | ++--------+-------------+-------------+ +| North | 2200.000000 | 1700.000000 | +| South | 2400.000000 | 1550.000000 | ++--------+-------------+-------------+ +2 rows ``` ## Monthly Sales Pivot @@ -69,17 +70,20 @@ $monthlySales = data_frame() ->groupBy(['region']) ->pivot(col('month'), discover_pivot_values()) // reads the month column once at build time ->aggregate(average(col('sales')->as('avg_sales'))) - ->write(to_output()) + ->write(to_output(truncate: false)) ->run(); ``` -**Result:** +Discovered values are sorted, strings by bytes: -``` -| region | Jan | Feb | Mar | -|--------|------|------|------| -| North | 5000 | 5500 | 6000 | -| South | 4500 | 4800 | 5200 | +```console ++--------+-------------+-------------+-------------+ +| region | Feb | Jan | Mar | ++--------+-------------+-------------+-------------+ +| North | 5500.000000 | 5000.000000 | 6000.000000 | +| South | 4800.000000 | 4500.000000 | 5200.000000 | ++--------+-------------+-------------+-------------+ +2 rows ``` @@ -102,7 +106,7 @@ data_frame() ->aggregate(sum(col('sales'))); ``` -The scan is `DISTINCT`, skips nulls, sorts, and is bounded - `discover_pivot_values(50)` refuses a column with +The scan is `DISTINCT`, skips nulls, sorts like [`sortBy()`](/documentation/components/core/sort.md), and is bounded - `discover_pivot_values(50)` refuses a column with more than 50 distinct values. Because it reads the frame before the real run, it needs a source it can read twice, and refuses one it cannot: diff --git a/documentation/components/core/rename.md b/documentation/components/core/rename.md index 3d4247470c..d98315feaf 100644 --- a/documentation/components/core/rename.md +++ b/documentation/components/core/rename.md @@ -4,11 +4,11 @@ [TOC] -DataFrame provides several methods for renaming entries (columns) in your data. These operations are lazy and don't execute until a trigger operation is called. +DataFrame provides several methods for renaming columns. These operations are lazy and don't execute until a trigger operation is called. ## Single Column Rename -To rename a single entry, use `DataFrame::rename()`: +To rename a single column, use `DataFrame::rename()`: ```php .csv (e.g., data_5f8a3b2c.csv) -// When reading data.csv, Flow reads all files matching the pattern: data*.csv +// Read them all with a glob: from_csv(__DIR__ . '/data*.csv') ``` **How it works:** - If destination file doesn't exist: writes normally - If destination file exists: generates a new file with randomized name in the same directory -- Flow treats file paths as directories that can contain multiple files with the same extension **File structure after multiple runs:** ``` @@ -90,7 +89,7 @@ output/ └── orders_2140bfc5fd.csv # Fourth run ``` -When you read from `orders.csv`, Flow automatically reads all `orders*.csv` files in the directory. +Reading `orders.csv` reads that one file; read `orders*.csv` to get every run. **Important:** Flow does not check for duplicates. If you run the same pipeline twice, data will be duplicated across multiple files. @@ -152,8 +151,9 @@ Save modes work with partitioned data: ['date' => '2024-01-01', 'value' => 100], ['date' => '2024-01-02', 'value' => 200], ])) - ->partitionBy('date') - ->write(to_parquet(__DIR__ . '/data')->saveMode(overwrite())) + ->write(to_parquet(__DIR__ . '/data/file.parquet') + ->partitionBy(partition_by(ref('date'))) + ->saveMode(overwrite())) ->run(); // Structure: diff --git a/documentation/components/core/schema.md b/documentation/components/core/schema.md index fff9bcf9c7..52b24b2e57 100644 --- a/documentation/components/core/schema.md +++ b/documentation/components/core/schema.md @@ -17,8 +17,8 @@ A schema consists of column definitions that specify: - **Metadata**: Key-value pairs for additional context Structure field order is part of a structure type's identity: `structure{a, b}` and `structure{b, a}` -are different types, so `Schema::isSame()` and `Definition::isSame()` distinguish them. Value-level -comparisons (`Row::isEqual()`, `Rows::unique()`) stay field-order-insensitive. +are different types, so `Schema::isSame()` and `Definition::isSame()` distinguish them. Comparing two such columns +(`ref('a')->equals(ref('b'))`) is refused at plan time; cast one of them to the other's type first. ## Arrays in a Schema @@ -72,8 +72,8 @@ header to check). Every inferred column is nullable, and where narrowing is not sample that does not fit the inferred type fails the read with an `InferredSchemaException` naming that row - infer from every row with `->inferSchema(infer_schema()->sampleSize(-1))`, or declare the schema. -**Sources that do not.** For the rest, every value still gets its type detected as rows are created and each batch -carries its own schema. +**Sources that do not.** The rest know their schema before the first row: Parquet and Floe read it from the file, +PostgreSQL and DBAL derive it from the query (below), and the text, XML and HTTP extractors yield fixed columns. **`DataFrame::schema()` describes the plan, not the data.** It walks the pipeline's steps once, threading each step's output schema into the next, starting from the source's own `schema()` - so it may do the I/O that source @@ -157,8 +157,8 @@ list_schema('tags', type_list(type_union(type_string(), type_integer()))); ## Declaring the Source Schema Most extractors accept a schema up front: fluent `withSchema(Schema)`, or the optional `$schema` argument of the -`from_*` DSL function. Declared types drive row creation instead of inference, and the row keeps only the columns the -schema mentions. +`from_*` DSL function. The declared types replace inference, and each batch keeps only the columns the schema +mentions. ```php withSchema(...)` skips the probe; it helps the last row, and the one before it when the query runs as written. @@ -360,25 +360,28 @@ cannot be null. Flow PHP provides three built-in validation strategies: - **[StrictValidator](/src/core/etl/src/Flow/ETL/Schema/Validator/StrictValidator.php)** - Rows must exactly match the - schema; extra entries cause validation failure -- **[SelectiveValidator](/src/core/etl/src/Flow/ETL/Schema/Validator/SelectiveValidator.php)** - Only validates entries - defined in schema; ignores extra entries + schema; extra columns cause validation failure +- **[SelectiveValidator](/src/core/etl/src/Flow/ETL/Schema/Validator/SelectiveValidator.php)** - Only validates columns + defined in schema; ignores extra columns - **[EvolvingValidator](/src/core/etl/src/Flow/ETL/Schema/Validator/EvolvingValidator.php)** - Allows missing and extra - entries as long as nullability permits it + columns as long as nullability permits it By default, DataFrame uses `StrictValidator`, but you can specify a different validator as the second parameter to `DataFrame::match()`. ## Basic Schema Matching -Use `DataFrame::match()` to validate data against a schema: +Use `DataFrame::match()` to validate data against a schema. `from_array()` infers every column as nullable, so the +expected schema declares them nullable too: ```php read(from_array([ ['id' => 1, 'name' => 'Product 1', 'active' => true], @@ -387,9 +390,9 @@ data_frame() ])) ->match( schema( - int_schema('id', $nullable = false), - str_schema('name', $nullable = false), - bool_schema('active', $nullable = false, Metadata::empty()->add('key', 'value')), + int_schema('id', nullable: true), + str_schema('name', nullable: true), + bool_schema('active', nullable: true, metadata: Metadata::empty()->add('key', 'value')), ) ) ->write(to_output(false, Output::rows_and_schema)) @@ -409,8 +412,8 @@ data_frame() ])) ->match( schema( - int_schema('id'), - str_schema('name') + int_schema('id', nullable: true), + str_schema('name', nullable: true) ), schema_selective_validator() // Only validate id and name, ignore other fields ) diff --git a/documentation/components/core/select-drop.md b/documentation/components/core/select-drop.md index 79257f1363..3b2f1898d2 100644 --- a/documentation/components/core/select-drop.md +++ b/documentation/components/core/select-drop.md @@ -6,13 +6,13 @@ ## Select -To quickly select only relevant entries use Rows `DataFrame::select` +Keep only the listed columns with `DataFrame::select()`: ```php read(from_array(...)) + ->read(from_array([['id' => 1, 'name' => 'Norbert', 'email' => 'norbert@example.com']])) ->select("id", "name") ->write(to_output()) ->run(); @@ -20,13 +20,13 @@ data_frame() ## Drop -To quickly drop irrelevant entries use Rows `DataFrame::drop` +Remove the listed columns with `DataFrame::drop()`: ```php read(from_array(...)) + ->read(from_array([['id' => 1, 'name' => 'Norbert', '_tags' => 'internal']])) ->drop("_tags") ->write(to_output()) ->run(); diff --git a/documentation/components/core/sort.md b/documentation/components/core/sort.md index c2934721b3..ef31f47cab 100644 --- a/documentation/components/core/sort.md +++ b/documentation/components/core/sort.md @@ -2,15 +2,49 @@ [DOC_LINK:/documentation/components/core/core.md] +## Order + +Nulls sort first ascending and last descending, NaN after every other number, strings by bytes; rows with equal keys +keep their input order. Both algorithms below produce the same order. + +```php +read(from_array([['code' => '9'], ['code' => null], ['code' => '10']])) + ->sortBy([ref('code')->asc()]) // null, '10', '9' + ->run(); +``` + +Each reference can move its nulls to the other end: + +```php +read(from_array([['code' => '9'], ['code' => null], ['code' => '10']])) + ->sortBy([ref('code')->asc(NullsOrder::LAST)]) // '10', '9', null + ->run(); + +data_frame() + ->read(from_array([['code' => '9'], ['code' => null], ['code' => '10']])) + ->sortBy([ref('code')->desc(NullsOrder::FIRST)]) // null, '9', '10' + ->run(); +``` + ## Sort Algorithm The sort algorithm is configured through `config_builder()->sort()` - all options live on the algorithm's own builder: -- `external_sort()` - **default**; buffers rows in memory, sorts them and spills each sorted run as a bucket through a - `BucketsStorage` (the same abstraction used by +- `external_sort()` - **default**; sorts in memory while the buffered rows stay under its memory limit. Past the limit + it spills each sorted run as a bucket through a `BucketsStorage` (the same abstraction used by [join](/documentation/components/core/join.md) and [group by](/documentation/components/core/group-by.md)), then k-way merges the runs back into one sorted stream. - Memory usage is bounded by one run. + The memory limit defaults to `FLOW_MAX_MEMORY` when set, otherwise 70% of PHP's `memory_limit` (1 GB when + `memory_limit` is `-1`), and is compared with the whole process - a sort and a join in one pipeline stay under it + together. - `memory_sort()` - buffers the whole dataset and sorts it in one pass; fastest, but everything must fit in RAM. It has no options. @@ -29,7 +63,7 @@ data_frame( | Buckets storage | Behavior | |-----------------------------------|----------------------------------------------------------------------------------------------| -| `FilesystemBuckets` - **Default** | Sorted runs are spilled to disk (Floe files). Memory usage is bounded by one run. | +| `FilesystemBuckets` - **Default** | Sorted runs are spilled to disk (Floe files). | | `MemoryBuckets` | Runs are kept in memory - prefer `memory_sort()` instead, it sorts once and skips the merge. | | `PSRCacheBuckets` | Runs are spilled into any PSR-16 cache. | @@ -40,9 +74,9 @@ data_frame( config_builder() ->sort( external_sort() - ->runSize(10_000) // rows buffered and sorted in memory before spilled as one run - ->bucketsCount(100) // how many runs are merged at once (merge fan-in) - ->batchSize(1000) // rows per spill/output batch + ->memoryLimit(Unit::fromMb(256)) // process memory past which sorted runs are spilled + ->bucketsCount(100) // how many runs are merged at once (merge fan-in) + ->batchSize(1000) // rows per output batch ) ) ->read(from_parquet('orders.parquet')) @@ -51,7 +85,10 @@ data_frame( ``` The storage implementation is swapped with `external_sort()->storage(BucketsStorage $storage)`, which covers -both phases - the runs spilled during bucketing and the runs written while merging. +both phases - the sorted runs and the runs written while merging. + +`sortBy()` followed by `limit(n)` keeps only the top `n` rows while it reads. Past the same memory limit it hands +the rows it kept and the rest of the stream to the external sort, whose merge stops after `n` rows. ## Overriding the algorithm for one sort @@ -75,13 +112,11 @@ to the spill storage, so `storage()` alone stays sufficient: ```php external_sort() - ->storage(new MemoryBuckets()) // spill runs - ->mergeStorage(new FilesystemBuckets($fs, path('/tmp/merge'))); // merged runs only + ->storage(new MemoryBuckets()) // spill runs + ->mergeStorage(new FilesystemBuckets($fs, path('/tmp/merge'), new AdaptiveBackend())); // merged runs only ``` -> While runs are spilled, each bucket is announced downstream as a single metadata row -> (`BucketShape`: `_bucket_id`, `_bucket_total_rows`). Those rows are an internal pipeline detail consumed -> by the merge processor - they never appear in the sorted output. +`FilesystemBuckets` reads runs back into the [column backend](/documentation/components/core/column-backend.md) it is given. ## Example diff --git a/documentation/components/core/telemetry.md b/documentation/components/core/telemetry.md index 9af0cf5d47..f6f0771ff3 100644 --- a/documentation/components/core/telemetry.md +++ b/documentation/components/core/telemetry.md @@ -124,7 +124,7 @@ $spanExporter = console_exporter(colors: true); $metricExporter = console_exporter(colors: true); // Logs are formatted with severity-based coloring -$logExporter = console_exporter(colors: true, maxBodyLength: 100); +$logExporter = console_exporter(colors: true, maxLogBodyLength: 100); ``` ### OTLP Export (Production) @@ -138,10 +138,7 @@ use function Flow\Telemetry\DSL\{ batching_log_processor, batching_metric_processor, batching_span_processor, logger_provider, memory_context_storage, meter_provider, resource, telemetry, tracer_provider }; -use function Flow\Bridge\Telemetry\OTLP\DSL\{ - otlp_curl_transport, otlp_json_serializer, - otlp_exporter, otlp_exporter, otlp_exporter -}; +use function Flow\Bridge\Telemetry\OTLP\DSL\{otlp_curl_transport, otlp_exporter, otlp_json_serializer}; $clock = new SystemClock(new DateTimeZone('UTC')); $contextStorage = memory_context_storage(); @@ -210,13 +207,14 @@ Every DataFrame execution creates a root span with the following attributes: When `trace_loading` is enabled, child spans are created for each loader with: -- Span name: Loader class name (e.g., `Flow\ETL\Loader\StreamLoader`) +- Span name: short loader class name (e.g., `StreamLoader`); the full name is in `flow.etl.loader.class` - Parent: DataFrame span - Status: OK on success, ERROR on failure When `trace_transformations` is enabled, child spans are created for each transformer with: -- Span name: Transformer class name (e.g., `Flow\ETL\Transformer\EntryNameTransformer`) +- Span name: short transformer class name (e.g., `ScalarFunctionTransformer`); the full name is in + `flow.etl.transformer.class`, the batch sizes in `flow.etl.transformation.input_rows` / `output_rows` - Parent: DataFrame span - Status: OK on success, ERROR on failure @@ -224,10 +222,10 @@ When `trace_transformations` is enabled, child spans are created for each transf When `collect_metrics` is enabled: -| Metric | Type | Unit | Description | -|----------------------------|------------|------------|------------------------------------| -| `flow.etl.rows.processed` | Counter | `{row}` | Cumulative count of processed rows | -| `flow.etl.rows.throughput` | Throughput | `{row}/s` | Rows processed per second | +| Metric | Type | Unit | Description | +|----------------------------|------------|---------|------------------------------------| +| `flow.etl.rows.processed` | Counter | `{row}` | Cumulative count of processed rows | +| `flow.etl.rows.throughput` | Throughput | `{row}` | Rows processed per second | ### Logs diff --git a/documentation/components/core/transformations.md b/documentation/components/core/transformations.md index 08b5e4afc6..0a3202c573 100644 --- a/documentation/components/core/transformations.md +++ b/documentation/components/core/transformations.md @@ -7,8 +7,8 @@ ## Introduction Transformations are a powerful abstraction in Flow PHP that allow you to modify DataFrames in a composable and reusable -way. Unlike Transformers which operate on individual Rows, Transformations work at the DataFrame level, providing access -to the full processing pipeline. +way. Unlike Transformers, which receive one batch (`Rows`) at a time, Transformations work at the DataFrame level, +providing access to the full processing pipeline. Every Transformation implements the `Transformation` interface with a single method: @@ -29,18 +29,18 @@ Transformations can be applied to DataFrames using two methods: Both methods accept `Transformation` objects directly or through convenient DSL functions. ```php -use function Flow\ETL\DSL\{df, from_array, select, drop}; +use function Flow\ETL\DSL\{df, from_array, select, drop, to_output}; // Using with() df() - ->read(from_array([/* ... */])) + ->read(from_array([['id' => 1, 'name' => 'Alice', 'temporary_column' => 'x']])) ->with(select('id', 'name')) ->write(to_output()) ->run(); // Using transform() - identical behavior df() - ->read(from_array([/* ... */])) + ->read(from_array([['id' => 1, 'name' => 'Alice', 'temporary_column' => 'x']])) ->transform(drop('temporary_column')) ->write(to_output()) ->run(); @@ -53,7 +53,7 @@ df() Select specific columns from the DataFrame, keeping only the columns you need. ```php -use function Flow\ETL\DSL\{df, from_array, select, ref}; +use function Flow\ETL\DSL\{df, from_array, select, ref, to_output}; // Select columns by name df() @@ -67,7 +67,7 @@ df() // Select using References for more control df() - ->read(from_array([/* ... */])) + ->read(from_array([['id' => 1, 'name' => 'Alice', 'city' => 'New York']])) ->with(select(ref('id'), ref('city'))) ->write(to_output()) ->run(); @@ -78,7 +78,7 @@ df() Remove unwanted columns from the DataFrame, keeping all other columns. ```php -use function Flow\ETL\DSL\{df, from_array, drop, ref}; +use function Flow\ETL\DSL\{df, from_array, drop, ref, to_output}; // Drop columns by name df() @@ -92,7 +92,7 @@ df() // Drop using References df() - ->read(from_array([/* ... */])) + ->read(from_array([['id' => 1, 'temp_column' => 'x', 'debug_info' => 'y']])) ->with(drop(ref('temp_column'), ref('debug_info'))) ->write(to_output()) ->run(); @@ -104,7 +104,9 @@ Control memory usage by setting the batch size for processing. Smaller batch siz processing large datasets. ```php -use function Flow\ETL\DSL\{df, from_csv, batch_size}; +use function Flow\ETL\Adapter\CSV\from_csv; +use function Flow\ETL\Adapter\Doctrine\to_dbal_table_insert; +use function Flow\ETL\DSL\{df, batch_size}; // Process large CSV file in batches of 100 rows df() @@ -119,7 +121,7 @@ df() Add an index column to each row, useful for tracking row position or creating unique identifiers. ```php -use function Flow\ETL\DSL\{df, from_array, add_row_index}; +use function Flow\ETL\DSL\{df, from_array, add_row_index, to_output}; use Flow\ETL\Transformation\AddRowIndex\StartFrom; // Add default index starting from 0 @@ -131,11 +133,11 @@ df() ->with(add_row_index()) ->write(to_output()) ->run(); -// Output: [['index' => 0, 'name' => 'Alice'], ['index' => 1, 'name' => 'Bob']] +// Output: [['name' => 'Alice', 'index' => 0], ['name' => 'Bob', 'index' => 1]] // Custom column name and start from 1 df() - ->read(from_array([/* ... */])) + ->read(from_array([['name' => 'Alice'], ['name' => 'Bob']])) ->with(add_row_index('row_number', StartFrom::ONE)) ->write(to_output()) ->run(); @@ -150,8 +152,9 @@ and every run uses that instance. Restrict the number of rows processed, useful for debugging or sampling data. ```php +use function Flow\ETL\Adapter\CSV\to_csv; use function Flow\ETL\Adapter\Doctrine\from_dbal_query; -use function Flow\ETL\DSL\{df, limit}; +use function Flow\ETL\DSL\{df, from_array, limit, to_output}; // Process only first 1000 rows df() @@ -162,7 +165,7 @@ df() // Remove limit (process all rows) df() - ->read(from_array([/* ... */])) + ->read(from_array([['id' => 1], ['id' => 2]])) ->with(limit(null)) ->write(to_output()) ->run(); @@ -173,7 +176,7 @@ df() Replace column values with a mask string, useful for hiding sensitive information. ```php -use function Flow\ETL\DSL\{df, from_array, mask_columns}; +use function Flow\ETL\DSL\{df, from_array, mask_columns, to_output}; // Mask sensitive columns with default mask df() @@ -188,7 +191,7 @@ df() // Use custom mask df() - ->read(from_array([/* ... */])) + ->read(from_array([['name' => 'Alice', 'credit_card' => '4111 1111 1111 1111']])) ->with(mask_columns(['credit_card'], '[REDACTED]')) ->write(to_output()) ->run(); @@ -198,8 +201,10 @@ df() Transformations can be chained together to create complex data processing pipelines: -```php -use function Flow\ETL\DSL\{df, from_csv, select, add_row_index, limit, batch_size}; +```php ignore +use function Flow\ETL\Adapter\CSV\from_csv; +use function Flow\ETL\Adapter\JSON\to_json; +use function Flow\ETL\DSL\{df, select, add_row_index, limit, batch_size}; df() ->read(from_csv('users.csv')) @@ -217,10 +222,11 @@ df() was written at, and planned and bound together with the rest of the frame. ```php -use function Flow\ETL\DSL\{df, from_array, lit, ref, select, to_branch, to_csv, to_transformation}; +use function Flow\ETL\Adapter\CSV\to_csv; +use function Flow\ETL\DSL\{df, from_array, lit, ref, select, to_branch, to_transformation}; df() - ->read(from_array([/* ... */])) + ->read(from_array([['id' => 1, 'name' => 'Alice', 'active' => true], ['id' => 2, 'name' => 'Bob', 'active' => false]])) ->write(to_transformation(select('id', 'name'), to_csv('names.csv'))) ->write(to_branch(ref('active')->equals(lit(true)), to_csv('active.csv'))) ->run(); @@ -252,7 +258,7 @@ $sortById = new class implements Transformation { }; df() - ->read(from_array([/* ... */])) + ->read(from_array([['id' => 2], ['id' => 1]])) ->write(to_transformation($sortById, to_output())) ->run(); ``` @@ -262,11 +268,11 @@ df() Correctness is the same everywhere; what differs between operations is how much they hold, and they hold it inside the sink. Three groups: -| Cost | Operations | -|----------------------------------------|-----------------------------------------------------------------------------------------------------------------------------------------------------------------| -| Grows with the whole stream | `sortBy()`, `aggregate()`, `groupBy()->aggregate()`, `pivot()`, window functions, `collect()`, `join()`, `repartition()` | -| Grows with the number of distinct keys | `dropDuplicates()`, `constrain()` with a `UniqueConstraint` | -| Constant | `select()`, `withEntry()`, `filter()`, `add_row_index()`, `limit()`, `until()`, `offset()`, `cache($id)`, `batch_size()`, `batchBy()` | +| Cost | Operations | +|----------------------------------------|---------------------------------------------------------------------------------------------------------------------------------------| +| Grows with the whole stream | `sortBy()`, `aggregate()`, `groupBy()->aggregate()`, `pivot()`, window functions, `collect()`, `join()`, `repartition()` | +| Grows with the number of distinct keys | `dropDuplicates()`, `constrain()` with a `UniqueConstraint` | +| Constant | `select()`, `withEntry()`, `filter()`, `add_row_index()`, `limit()`, `until()`, `offset()`, `cache($id)`, `batch_size()`, `batchBy()` | The first group buffers - in memory, or spilled to disk by the external sort - exactly as it does on an outer frame. `offset()` and `cache($id)` are in the constant group: `offset()` counts the rows it skips, and `cache($id)` writes each @@ -292,7 +298,7 @@ A sink runs under the frame's `->onError(...)` handler, and each failure is offe ```php df() - ->read(from_array([/* ... */])) + ->read(from_array([['id' => 2], ['id' => 1]])) ->onError(skip_rows_handler()) // a failing step inside the sink skips that batch ->write(to_transformation($sortById, to_csv('sorted.csv'))) ->run(); @@ -314,6 +320,8 @@ You can create custom transformations by implementing the `Transformation` inter ```php use Flow\ETL\{DataFrame, Transformation}; +use function Flow\ETL\DSL\{df, from_array, ref, to_output}; + final class UppercaseNames implements Transformation { public function transform(DataFrame $dataFrame): DataFrame @@ -327,7 +335,7 @@ final class UppercaseNames implements Transformation // Use custom transformation df() - ->read(from_array([/* ... */])) + ->read(from_array([['name' => 'alice'], ['name' => 'bob']])) ->with(new UppercaseNames()) ->write(to_output()) ->run(); diff --git a/documentation/components/core/window-functions.md b/documentation/components/core/window-functions.md index 616664e922..eeabdb7794 100644 --- a/documentation/components/core/window-functions.md +++ b/documentation/components/core/window-functions.md @@ -21,7 +21,7 @@ smaller partitions on which later window function will be applied. - [`AVERAGE`](/src/core/etl/src/Flow/ETL/Function/Average.php) - [`COUNT`](/src/core/etl/src/Flow/ETL/Function/Count.php) -All window functions are implementing [`WindowFunction`](/src/core/etl/src/Flow/ETL/Function/WindowFunction.php) +All window functions implement [`WindowFunction`](/src/core/etl/src/Flow/ETL/Function/WindowFunction.php) interface. ### Example @@ -47,22 +47,17 @@ data_frame() Output: ```console -+----+-------+------------+--------+------+ -| id | name | department | salary | rank | -+----+-------+------------+--------+------+ -| 5 | Jane | Finances | 14000 | 1 | -| 3 | Tomas | Finances | 11000 | 2 | -| 4 | John | Finances | 9000 | 3 | -| 6 | Janet | Finances | 4000 | 4 | -+----+-------+------------+--------+------+ -4 rows +----+--------+------------+--------+------+ | id | name | department | salary | rank | +----+--------+------------+--------+------+ +| 5 | Jane | Finances | 14000 | 1 | +| 3 | Tomas | Finances | 11000 | 2 | +| 4 | John | Finances | 9000 | 3 | +| 6 | Janet | Finances | 4000 | 4 | | 1 | Greg | IT | 6000 | 1 | | 2 | Michal | IT | 5000 | 2 | +----+--------+------------+--------+------+ -2 rows +6 rows ``` ## Frames diff --git a/documentation/components/extensions/arrow-ext.md b/documentation/components/extensions/arrow-ext.md index 73bedf0c0f..f79d40880f 100644 --- a/documentation/components/extensions/arrow-ext.md +++ b/documentation/components/extensions/arrow-ext.md @@ -34,13 +34,11 @@ The Arrow Rust crates offer additional I/O capabilities that are candidates for ## Features -- Read and write Apache Parquet files through PHP streaming interfaces -- Flat types: INT32, INT64, FLOAT, DOUBLE, BOOLEAN, STRING, BINARY, DATE, TIMESTAMP, TIME, DECIMAL +- Read and write Apache Parquet files over `flow-php/filesystem` streams, value for value as `PhpParquetEngine` - Nested types: LIST, STRUCT, MAP (arbitrarily nested) - Compression codecs: UNCOMPRESSED, SNAPPY, GZIP, ZSTD, LZ4_RAW, BROTLI -- Column projection for selective reads -- Configurable row group size, compression level, and writer version -- Columnar batch I/O for maximum throughput +- Column projection, offset and limit, read from one footer read +- Batches handed to other extensions through the [Arrow C Data Interface](https://arrow.apache.org/docs/format/CDataInterface.html) ## Requirements @@ -69,233 +67,71 @@ php -d extension=./ext/modules/arrow.so your_script.php ## Usage -### Implementing the Streaming Interfaces - -The extension operates on two PHP interfaces for I/O. You must provide implementations for your storage backend. - -**Source (reading):** +The extension registers the [parquet library](/documentation/components/libs/parquet.md)'s +`Flow\Parquet\{ParquetEngine, ParquetFileReader, ParquetFileWriter}` and implements them with +`Flow\Parquet\Engine\{RustParquetEngine, RustParquetFileReader, RustParquetFileWriter}`. `AdaptiveParquetEngine` - the +default of `Reader` and `Writer` - picks `RustParquetEngine` while the extension is loaded, `PhpParquetEngine` +otherwise. ```php data = file_get_contents($path); - } +use Flow\Parquet\{Reader, Writer}; - public function read(int $length, int $offset): string - { - return substr($this->data, $offset, $length); - } - - public function size(): ?int - { - return strlen($this->data); - } -} +$reader = new Reader(); // Flow\Parquet\Engine\AdaptiveParquetEngine +$writer = new Writer(); ``` -**Destination (writing):** +### Classes ```php fh = fopen($path, 'wb'); - } +$filesystem = native_local_filesystem(); - public function append(string $data): self - { - fwrite($this->fh, $data); - return $this; - } +$file = new RustParquetFileReader($filesystem->readFrom(path('data.parquet'))); // Flow\Parquet\ParquetFileReader, one footer read +$file->rowsNumber(); +$file->schema(); // Flow\Parquet\ParquetFile\Schema +$file->metadata(); // Flow\Parquet\ParquetFile\Metadata +$file->thrift(); // Flow\Parquet\ThriftModel\FileMetaData - public function __destruct() - { - fclose($this->fh); - } +foreach ($file->readColumns(['id', 'name'], 1_000, null, null) as $chunk) { // RustColumnsReader, an Iterator that streams once + // ['id' => [1, 2, ...], 'name' => ['a', 'b', ...]] } -``` - -### Reading Parquet Files - -```php -schema(); -$metadata = $reader->metadata(); -// Read row groups (with optional column projection) -while ($batch = $reader->readRowGroup(['id', 'name'])) { - // $batch is ['column_name' => [values...], ...] - foreach ($batch['id'] as $i => $id) { - echo "$id: {$batch['name'][$i]}\n"; - } -} - -$reader->close(); -``` - -### Writing Parquet Files - -```php - 'id', 'type' => 'INT64', 'optional' => false], - ['name' => 'name', 'type' => 'STRING', 'optional' => true], -]; - -$writer = new Writer(new FileDestination('output.parquet'), $schema, 'SNAPPY'); -$writer->writeBatch([ - 'id' => [1, 2, 3], - 'name' => ['Alice', 'Bob', null], -]); -$writer->close(); +$writer = new RustParquetFileWriter( // Flow\Parquet\ParquetFileWriter + $filesystem->writeTo(path('out.parquet')), + SchemaConverter::toExtension($schema), // Flow\Parquet\ParquetFile\Schema + Compressions::SNAPPY, // LZO is refused + OptionsConverter::toExtension(Options::default()), + 1_000, // rows per written batch +); +$writer->writeBatch([['id' => 1, 'name' => 'a']]); // an array or any Traversable of rows +$writer->writeColumns(['id' => [2, 3], 'name' => ['b', null]]); +$writer->close(); // footer, then the stream closed + +$batches = new RustBatchReader($file, ['id', 'name'], 1_000, null, null); // batch size, offset, limit +$batches->schema(); // Flow\Arrow\RustArrowSchema +$batches->next(); // Flow\Arrow\RustParquetBatch, null after the last ``` -### Schema Definition - -The schema is an array of column definitions. Each column has a `name`, `type`, and optional `optional` flag. - -| Type | PHP Read Value | Notes | -|-------------|-----------------------|-------------------------------------------------------------------------------------------------------------------| -| `BOOLEAN` | `bool` | | -| `INT32` | `int` | | -| `INT64` | `int` | | -| `FLOAT` | `float` | | -| `DOUBLE` | `float` | | -| `STRING` | `string` | | -| `BINARY` | `string` (raw bytes) | | -| `DATE` | `DateTimeImmutable` | `int` lane: days since epoch | -| `TIMESTAMP` | `DateTimeImmutable` | Keys `unit` (`MILLIS\|MICROS\|NANOS`, default `MICROS`) and `utc` (default `true`); `int` lane in the column unit | -| `TIME` | `DateInterval` | Key `unit` (`MILLIS\|MICROS\|NANOS`, default `MICROS`) | -| `DECIMAL` | `float` | Keys `precision`, `scale` | -| `UUID` | `string` | | -| `LIST` | `array` | Requires `children` key with 1 element | -| `STRUCT` | `array` (associative) | Requires `children` key with N elements | -| `MAP` | `array` (associative) | Requires `children` key with 2 elements (key + value) | - -**Nested schema example:** +Refusals are the `Flow\Parquet\Exception\*` the PHP engine throws; an exception the stream throws surfaces as itself. -```php - 'id', 'type' => 'INT64', 'optional' => false], - ['name' => 'tags', 'type' => 'LIST', 'optional' => true, 'children' => [ - ['name' => 'element', 'type' => 'STRING', 'optional' => true], - ]], - ['name' => 'address', 'type' => 'STRUCT', 'optional' => true, 'children' => [ - ['name' => 'street', 'type' => 'STRING', 'optional' => true], - ['name' => 'city', 'type' => 'STRING', 'optional' => true], - ]], - ['name' => 'metadata', 'type' => 'MAP', 'optional' => true, 'children' => [ - ['name' => 'key', 'type' => 'STRING', 'optional' => false], - ['name' => 'value', 'type' => 'STRING', 'optional' => true], - ]], -]; -``` +### Arrow C Data Interface -### Writer Options - -Options are passed as the fourth argument to the `Writer` constructor: - -```php - 128 * 1024 * 1024, - 'WRITER_VERSION' => '2.0', -]); -``` - -| Option Key | Type | Description | -|---|---|---| -| `ROW_GROUP_SIZE_BYTES` | `int` | Maximum row group size in bytes | -| `COMPRESSION_LEVEL` | `int` | Compression level (codec-specific) | -| `WRITER_VERSION` | `string` | `"1.0"` or `"2.0"` | - -## API Reference - -### Interfaces - -**`Flow\Arrow\RandomAccessFile`** - -| Method | Parameters | Returns | Description | -|---|---|---|---| -| `read` | `int $length, int $offset` | `string` | Read `$length` bytes starting at `$offset` | -| `size` | - | `?int` | Return total size in bytes, or `null` if unknown | - -**`Flow\Arrow\OutputStream`** - -| Method | Parameters | Returns | Description | -|---|---|---|---| -| `append` | `string $data` | `self` | Append data to the output stream | - -### Classes - -**`Flow\Arrow\Parquet\Reader`** - -| Method | Parameters | Returns | Description | -|---|---|---|---| -| `__construct` | `RandomAccessFile $source, array $options = []` | - | Open a Parquet source for reading | -| `schema` | - | `array` | Return the file schema as nested arrays | -| `metadata` | - | `array` | Return file-level metadata (row count, row groups, key-value metadata) | -| `readRowGroup` | `?array $columns = null` | `?array` | Read next row group as columnar batch, or `null` when exhausted | -| `close` | - | `void` | Release resources | - -**`Flow\Arrow\Parquet\Writer`** - -| Method | Parameters | Returns | Description | -|---|---|---|---| -| `__construct` | `OutputStream $stream, array $schema, string $compression = 'SNAPPY', array $options = []` | - | Open a Parquet destination for writing | -| `writeBatch` | `array $batch` | `void` | Write a columnar batch (`['col' => [values...]]`) | -| `close` | - | `void` | Flush and finalize the Parquet file | - -**`Flow\Arrow\Parquet\Exception`** - -Extends `\RuntimeException`. Thrown on all Parquet read/write errors originating from the Rust layer. - -## Error Handling - -```php -readRowGroup()) { - // process batch - } - $reader->close(); -} catch (Exception $e) { - echo "Parquet error: " . $e->getMessage(); -} -``` +`RustParquetBatch` (`RustBatchReader::next()`) carries one struct array: `arrowSchemaAddress()` (`FFI_ArrowSchema*`) and +`arrowArrayAddress()` (`FFI_ArrowArray*`). Its consumer, another extension, reads the schema by reference and moves the +array out. `RustParquetFileWriter::writeArrowBatch()` takes the same pair from any extension's final internal class exposing both +methods, its children by writer column name. Userland cannot build either: the classes are final and their +constructors throw. ## Development @@ -322,8 +158,7 @@ make test - Built with [ext-php-rs](https://github.com/extphprs/ext-php-rs), which generates PHP bindings from Rust code - Uses Apache Arrow and Parquet Rust crates from the [Arrow ecosystem](https://github.com/apache/arrow-rs) - All compression codecs compiled into the extension - no external PHP compression extensions needed -- PHP streaming interfaces (`RandomAccessFile`, `OutputStream`) called from Rust via ext-php-rs callbacks -- Columnar batch format aligns with Parquet's native columnar storage +- Stream methods (`read`, `size`, `append`, `close`) called from Rust via ext-php-rs - PIE-compatible via `ext/config.m4` that delegates to `cargo build` ## See Also diff --git a/documentation/components/extensions/flow-php-ext.md b/documentation/components/extensions/flow-php-ext.md index fc4281f73c..8cdfaf6576 100644 --- a/documentation/components/extensions/flow-php-ext.md +++ b/documentation/components/extensions/flow-php-ext.md @@ -8,19 +8,10 @@ package: flow-php/flow-php-ext [TOC] -Flow stores durable datasets (`to_floe()`) and caches intermediate ones (`DataFrame::cache()`, -sort / join / group-by spill buckets) using the native **Floe** binary format (`.floe`) - the schema -is written once per file and rows carry raw values only, which makes both the payload and the -hydration dramatically cheaper than native PHP `serialize()`/`unserialize()`. - -This extension encodes and decodes Floe **frames** natively in Rust via -[ext-php-rs](https://github.com/extphprs/ext-php-rs). The pure-PHP implementation in `Flow\Floe` -(flow-php/etl) is the canonical behavior reference and works without the extension - loading it is -purely an optimization. File header and footer assembly always stay in PHP. - -> You never need to call this extension directly. `Flow\Floe\FloeReader`/`FloeWriter` - used by -> `from_floe()`/`to_floe()`, the cache and the sort / join / group-by buckets caches - route whole -> batches to it automatically when `extension_loaded('flow_php')` is true. +This extension is Flow's native column backend and reads CSV and JSON and writes CSV and JSON natively in Rust - Parquet +through the [arrow extension](/documentation/components/extensions/arrow-ext.md) - via +[ext-php-rs](https://github.com/extphprs/ext-php-rs). The pure-PHP implementations in `flow-php/etl` are the canonical +behaviour reference and work without the extension - loading it is purely an optimization. ## Loading the Extension @@ -38,65 +29,121 @@ php -d extension=./ext/modules/flow_php.so your_script.php ## Usage -The extension is used implicitly through the ETL cache: +The extension is used implicitly by the CSV reader: ```php read(from_array($bigDataset)) - ->cache('my-dataset') // serialized with the extension when loaded - ->write(to_stream(__DIR__ . '/output.csv')) + ->read(from_csv(__DIR__ . '/input.csv')) // read natively when the extension is loaded + ->write(to_stream(__DIR__ . '/output.txt')) ->run(); ``` -The extension registers two native classes; the PHP side (`FloeStreamWriter`/`FloeStreamReader`) owns -file framing, sectioning, partitions, footer and - on read - skip/limit/padding, and picks the native -implementation automatically: - -- **`Flow\Floe\RustFloeEncoderNative`** - the Floe ROW frame-body codec: - `encode(list, schemaBody)` returns the encoded frame bodies, - `decode(list, schemaBody)` returns `list`, and - `decodeRows(list, schemaBody, Schema)` decodes and casts straight into `Flow\ETL\Rows` in one pass, and - `encodeFrames(Rows, schemaBody, Schema)` turns `Rows` straight into one string of complete ROW frames - what the - reader and the writer use when the configured hydrator is the native one. The userland wrapper - `Flow\Floe\NativeFloeEncoder` carries the `Flow\Floe\FloeEncoder` interface, and `FloeEngine::adaptive` - the - engine every writer/reader defaults to - builds it over `Flow\Floe\PhpFloeEncoder` when the extension is loaded. -- **`Flow\ETL\Row\RustRowHydratorNative`** - the native `hydrate`/`cast`/`dehydrate` behind - `Flow\ETL\Row\NativeRowHydrator`, which `Flow\ETL\Row\AdaptiveRowHydrator` (the config default) - selects when the extension is loaded - used by adapter loaders and raw-scalar extractors such as - CSV, JSON or XML. `cast(batch, Schema)` casts raw scalars and builds entries in a single native - pass; values outside the proven native subset (and every exotic type such as enum, xml or time) - cast per value through the schema's PHP `Type::cast`, so results and exceptions match - `PhpRowHydrator` exactly. Schema-less `cast` (type inference) stays PHP. Hydrate and cast plans are - cached on the `Schema` object identity and rebuilt only when a different schema arrives. - -All extension failures throw `Flow\Floe\Exception\ExtensionException`; `FloeReader`/`FloeWriter` wrap -it as `Flow\Floe\Exception\FloeException`. There is no silent fallback to the PHP engine. - -## Structure columns across the PHP/Rust boundary - -A structure column crosses to Rust in the schema JSON as tag `structure_v2` with an ordered `fields` -list - one `{name, type, optional}` object per field, `name` always a JSON string (PHP casts integer -element names on `normalize()`). The Rust side reads the list in order; the per-field `optional` flag -is used only by the cast plan (`required = !optional`), never by the encoder or decoder, which emit -and read one flag byte per field in declared order. - -A version-skewed pair fails loudly instead of writing wrong bytes: - -- new `flow-php/etl` with a pre-`structure_v2` extension: - `flow_php does not support values of type "structure_v2" in this build` -- old `flow-php/etl` with a current extension: - `flow_php does not support values of type "structure" in this build` - -Either message means the extension and `flow-php/etl` disagree on the structure schema format: -reinstall one to match the other, or set the Floe engine to `FloeEngine::php` while upgrading. - -Datetime column zones do not fail, they degrade: - -- new `flow-php/etl` with an extension that predates column zones: the extension is ignored and the - PHP engine runs -- old `flow-php/etl` with a current extension: a zone-less `datetime` column keeps that library's - behaviour, each value in its stored zone +With the extension loaded, `Flow\ETL\Column\AdaptiveBackend` - the default of `config_builder()->backend()` - picks the +extension's `RustBackend` and every batch column is a `Flow\ETL\Column\RustColumn` over an Apache Arrow array. How to +choose a backend and what it builds: [Column Backend](/documentation/components/core/column-backend.md). + +```php +builder(int_schema('id')); +$builder->appendMany([1, '2', 3.0]); + +$column = $builder->finish(); // Flow\ETL\Column\RustColumn +$column->values(); // [1, 2, 3] +(new RustBackend())->allocatedBytes(); // bytes held outside PHP's memory manager +``` + +Every contract with a Rust and a PHP implementation follows one pattern: the extension registers the interface, the +`Rust*` class implements it, `Php*` is the PHP implementation and `Adaptive*` picks one by `extension_loaded()`. + +| contract | Rust | PHP | picked by | +|----------------------------------------------|-------------------------|------------------------|-----------------------------| +| `Flow\ETL\Column\Backend` | `RustBackend` | `PhpBackend` | `AdaptiveBackend` | +| `Flow\ETL\Adapter\CSV\CSVOpenSource` | `RustCSVOpenSource` | `PhpCSVOpenSource` | `AdaptiveCSVOpenSource` | +| `Flow\ETL\Adapter\CSV\CSVEncoder` | `RustCSVEncoder` | `PhpCSVEncoder` | `AdaptiveCSVEncoder` | +| `Flow\ETL\Adapter\JSON\JsonOpenSource` | `RustJsonOpenSource` | `PhpJsonOpenSource` | `AdaptiveJsonOpenSource` | +| `Flow\ETL\Adapter\JSON\JsonEncoder` | `RustJsonEncoder` | `PhpJsonEncoder` | `AdaptiveJsonEncoder` | +| `Flow\ETL\Adapter\Parquet\ParquetOpenSource` | `RustParquetOpenSource` | `PhpParquetOpenSource` | `AdaptiveParquetOpenSource` | +| `Flow\ETL\Adapter\Parquet\ParquetOpenSink` | `RustParquetOpenSink` | `PhpParquetOpenSink` | `AdaptiveParquetOpenSink` | + +The extension also registers `Flow\ETL\Column\{Column, ColumnBuilder}` (`RustColumn`, `RustColumnBuilder`) and +`Flow\ETL\RustIterator`, the iterator its sources return. + +## CSV and JSON writers + +With the extension loaded, `to_csv()`, `to_json()` and `to_json_lines()` render a batch from the arrow buffers, without +a PHP value per cell. The bytes are the ones the PHP writers produce. + +```php +read(from_csv(__DIR__ . '/input.csv')) + ->write(to_csv(__DIR__ . '/output.csv')->withDateTimeFormat('Y-m-d H:i:s.u P')) // rendered natively + ->write(to_json_lines(__DIR__ . '/output.jsonl')->withDateTimeFormat('D, d M Y')) // `D` and `M`: this column renders in PHP + ->run(); +``` + +A datetime or date format is rendered natively when its letters are among `Y y m n d j H G i s u v e T P p O Z U c` +(and `\` escapes); any other letter renders that column in PHP. `xml`, `xml_element`, `html` and `html_element` columns, +a `json` column written to JSON, and a `list` / `map` / `structure` holding one of those also render in PHP - only +those columns, the rest of the batch stays native. `to_json()` and `to_json_lines()` render natively under the flags +`JSON_THROW_ON_ERROR`, `JSON_UNESCAPED_SLASHES`, `JSON_UNESCAPED_UNICODE` and `JSON_PRESERVE_ZERO_FRACTION`; any other +flag (`JSON_PRETTY_PRINT`, ...) writes through PHP. + +`RustCSVEncoder` / `RustJsonEncoder` hold the PHP encoder they render the PHP columns with. + +## Parquet + +With both this extension and the [arrow extension](/documentation/components/extensions/arrow-ext.md) loaded, +`from_parquet()` and `to_parquet()` without an `engine:` read and write native columns: arrow-ext reads and writes the +file, and its batches cross into `RustColumn`s (and back) through the Arrow C Data Interface, without a copy and +without a PHP value per cell. + +| loaded | `from_parquet()` / `to_parquet()` | +|-----------------|-------------------------------------------------------------------------| +| flow_php, arrow | native columns through the Arrow C Data Interface | +| arrow | `RustParquetEngine` (arrow-ext), PHP values | +| flow_php | `PhpParquetEngine`; `RustBackend` adopts the values into native columns | +| neither | `PhpParquetEngine` | + +```php +read(from_parquet(__DIR__ . '/orders.parquet')->withOffset(50_000)) // only the row groups from row 50 000 are decoded + ->write(to_parquet(__DIR__ . '/copy.parquet')) + ->run(); + +df() + ->read(from_parquet(__DIR__ . '/orders.parquet', engine: new PhpParquetEngine())) // PhpParquetEngine opts out + ->write(to_parquet(__DIR__ . '/copy.parquet', engine: new PhpParquetEngine())) + ->run(); +``` + +A schema type that stores another arrow type than the file column reads as is refused before the first batch. The +imported batches are counted by `RustBackend::allocatedBytes()` for as long as they live; arrow-ext's own read +buffers (a row group's projected column chunks, decode buffers) are not, so a `MemoryBudget` does not see them. Read and +write refusals are arrow-ext's `Flow\Parquet\Exception\*`, as with `RustParquetEngine`. + +`RustParquetOpenSource` reads through arrow-ext's `Flow\Arrow\Parquet\RustBatchReader`, `RustParquetOpenSink` writes +through its `Flow\Parquet\Engine\RustParquetFileWriter`. + diff --git a/documentation/components/libs/array-dot.md b/documentation/components/libs/array-dot.md index 5872327967..05899e8705 100644 --- a/documentation/components/libs/array-dot.md +++ b/documentation/components/libs/array-dot.md @@ -229,6 +229,8 @@ use Flow\ArrayDot\Step\Key; use Flow\ArrayDot\Step\Multimatch; use Flow\ArrayDot\Step\Wildcard; +use function Flow\ArrayDot\array_dot_get; + $path = new Path([new Key('users'), new Wildcard(), new Multimatch([ new Path([new Key('id')]), new Path([new Key('first.name', nullsafe: true)]), diff --git a/documentation/components/libs/doctrine-dbal-bulk.md b/documentation/components/libs/doctrine-dbal-bulk.md index 0dd940243d..5b82b294e8 100644 --- a/documentation/components/libs/doctrine-dbal-bulk.md +++ b/documentation/components/libs/doctrine-dbal-bulk.md @@ -26,98 +26,84 @@ For detailed installation instructions, see the [installation page](/documentati Insert: ```php -$bulk = Bulk::create(); -$bulk->insert( +insert( $dbalConnection, - 'your-table-name', + 'users', new BulkData([ ['id' => 1, 'name' => 'Name One', 'description' => 'Description One'], ['id' => 2, 'name' => 'Name Two', 'description' => 'Description Two'], ['id' => 3, 'name' => 'Name Three', 'description' => 'Description Three'], - ]) + ]), ); - ``` -### Type Optimization with BulkData +### Column types -`BulkData` can accept optional type information to optimize database operations: +Without types, `Bulk` reads the column types of the table from the database, once per table and `Bulk` instance. Pass +them to `BulkData` to bind with exactly these types: ```php -use Doctrine\DBAL\Types\Type; -use Doctrine\DBAL\Types\Types; +insert( +use Doctrine\DBAL\Types\{Type, Types}; +use Flow\Doctrine\Bulk\{Bulk, BulkData}; + +Bulk::create()->insert( $dbalConnection, - 'your-table-name', + 'users', new BulkData( [ - ['id' => 1, 'name' => 'Name One', 'created_at' => new \DateTime()], - ['id' => 2, 'name' => 'Name Two', 'created_at' => new \DateTime()], + ['id' => 1, 'name' => 'Name One', 'created_at' => new \DateTimeImmutable()], + ['id' => 2, 'name' => 'Name Two', 'created_at' => new \DateTimeImmutable()], ], [ 'id' => Type::getType(Types::INTEGER), 'name' => Type::getType(Types::STRING), 'created_at' => Type::getType(Types::DATETIME_IMMUTABLE), - ] - ) + ], + ), ); ``` -**Type Detection Behavior:** +Update (PostgreSQL): -- **With Types**: When column types are provided, `BulkData` uses them directly for optimal performance -- **Without Types**: When no types are provided, `BulkData` automatically queries the database to understand the table structure and determine appropriate types - -This automatic type detection ensures data consistency but comes with a performance cost due to the additional database query. For optimal performance in high-throughput scenarios, explicitly providing column types is recommended. +```php +update( +Bulk::create()->update( $dbalConnection, - 'your-table-name', + 'users', new BulkData([ ['id' => 1, 'name' => 'Name One', 'description' => 'Description One'], ['id' => 2, 'name' => 'Name Two', 'description' => 'Description Two'], - ['id' => 3, 'name' => 'Name Three', 'description' => 'Description Three'], ]), - [ - 'primary_key_columns' => ['id'], - 'update_columns' => ['name'] - ] + new PostgreSQLUpdateOptions(primaryKeyColumns: ['id'], updateColumns: ['name']), ); - ``` +`Bulk::delete()` removes the rows matching every column of `BulkData`. + ## Supported Dialects -* PostgreSQL -* MySQL / MariaDB -* SQLite +- PostgreSQL +- MySQL / MariaDB +- SQLite ### Adding new Dialects -[Dialect](/src/lib/doctrine-dbal-bulk/src/Flow/Doctrine/Bulk/Dialect/Dialect.php) is basic abstraction of this library. -The main role of Dialect is to prepare SQL insert/update statement based -on [BulkData](/src/lib/doctrine-dbal-bulk/src/Flow/Doctrine/Bulk/BulkData.php) -and provided `options`. - -* `$insertOptions` -* `$updateOptions` - -Options are key => value maps without predefined structure that allows to manipulate building SQL statement. -Each dialect should define it own structure for options in order to support db engine features, including those -that are specific for given engine. - -[QueryFactory](/src/lib/doctrine-dbal-bulk/src/Flow/Doctrine/Bulk/QueryFactory.php) is abstraction for creating queries, there is currently only one -implementation, DbalPlatform. QueryFactory `insertOptions` and `updateOptions` is combination of all options provided -by supported Dialects where each entry must be optional. - -example: -`dialect_option?: string` +A [Dialect](/src/lib/doctrine-dbal-bulk/src/Flow/Doctrine/Bulk/Dialect/Dialect.php) builds the SQL of one insert, +update or delete from a [BulkData](/src/lib/doctrine-dbal-bulk/src/Flow/Doctrine/Bulk/BulkData.php) and the dialect's +own `InsertOptions` / `UpdateOptions` implementation (`PostgreSQLInsertOptions`, `MySQLInsertOptions`, +`SqliteInsertOptions`, ...). -[DbalPlatform](/src/lib/doctrine-dbal-bulk/src/Flow/Doctrine/Bulk/DbalPlatform.php) is a factory that detects which Dialect should be used for given -Doctrine DBAL Platform. +[DbalPlatform](/src/lib/doctrine-dbal-bulk/src/Flow/Doctrine/Bulk/DbalPlatform.php) picks the Dialect for a Doctrine +DBAL platform, and [DbalQueryFactory](/src/lib/doctrine-dbal-bulk/src/Flow/Doctrine/Bulk/QueryFactory/DbalQueryFactory.php), +the [QueryFactory](/src/lib/doctrine-dbal-bulk/src/Flow/Doctrine/Bulk/QueryFactory.php) implementation, asks it for +the query. diff --git a/documentation/components/libs/parquet.md b/documentation/components/libs/parquet.md index cd7fed2030..37b52280a7 100644 --- a/documentation/components/libs/parquet.md +++ b/documentation/components/libs/parquet.md @@ -14,19 +14,11 @@ For detailed installation instructions, see the [installation page](/documentati ## Engine System -The parquet library supports a pluggable engine system with two implementations: - -- **PHP Engine** (`PhpParquetEngine`) - Pure PHP implementation. Works everywhere with no extra dependencies beyond - optional compression extensions. -- **Arrow Engine** (`ArrowParquetEngine`) - Uses - the [arrow PHP extension](/documentation/components/extensions/arrow-ext.md) for native Rust-powered performance. All - compression codecs are built into the extension. -- **Adaptive Engine** (`AdaptiveParquetEngine`) - Default. Automatically selects Arrow if the `arrow` extension is - loaded, otherwise falls back to PHP. - -### Engine Selection - -By default, `new Reader()` and `new Writer()` use the adaptive engine: +- **PHP Engine** (`PhpParquetEngine`) - pure PHP, no extra dependencies beyond optional compression extensions. +- **Rust Engine** (`RustParquetEngine`) - registered by the + [arrow extension](/documentation/components/extensions/arrow-ext.md), every compression codec built in. +- **Adaptive Engine** (`AdaptiveParquetEngine`) - the default: `RustParquetEngine` when the arrow extension is loaded, + `PhpParquetEngine` otherwise. A big-endian `Reader` uses `PhpParquetEngine`. ```php `new Reader()` uses the adaptive engine. See [Engine System](#engine-system) for how to explicitly choose the Arrow or > PHP engine. -The Reader accepts two arguments: +The Reader accepts three arguments: - `$byteOrder` - by default set to `ByteOrder::LITTLE_ENDIAN` - `$options` - a set of options that can be used to configure the reader. +- `$engine` - by default `AdaptiveParquetEngine` All available options are described in [Option](/src/lib/parquet/src/Flow/Parquet/Option.php) enum. @@ -159,7 +127,8 @@ All available options are described in [Option](/src/lib/parquet/src/Flow/Parque ### Reader Options -- `INT_96_AS_DATETIME` - default: `true` - if set to `true` then `INT96` values will be converted to `DateTime` objects. +- `INT_96_AS_DATETIME` - default: `true` - if set to `true` then `INT96` values are read as `DateTimeImmutable`, + floored to microseconds. ### Reading a file @@ -174,13 +143,16 @@ $file = $reader->read('path/to/file.parquet'); $file = $reader->readStream(\fopen('path/to/file.parquet', 'rb')); ``` -At this point, nothing is read yet. We just created a file object. +`RustParquetEngine` reads the footer when the file is opened, `PhpParquetEngine` on the first call that needs it. An +invalid file throws `InvalidArgumentException` `Given file is not valid Parquet file...`. There are several things we can read from parquet file: - `ParquetFile::values(array $columns = [], ?int $limit = null, ?int $offset = null) : \Generator` +- `ParquetFile::columns(int $batchSize, array $columns = [], ?int $limit = null, ?int $offset = null) : \Generator` - `ParquetFile::metadata() : Metadata` - `ParquetFile::schema() : Schema` - shortcut for `ParquetFile::metadata()->schema()` +- `ParquetFile::close() : void` - closes the reader and its stream ### Reading the whole file: @@ -225,6 +197,20 @@ foreach ($file->values(["column_1", "column_2"], limit: 100, offset: 1000) as $r } ``` +### Reading columns in batches + +```php +use Flow\Parquet\Reader; + +$file = (new Reader())->read('path/to/file.parquet'); + +foreach ($file->columns(batchSize: 1000, columns: ["column_1", "column_2"]) as $chunk) { + // $chunk = ['column_1' => [...up to 1000 values], 'column_2' => [...]] +} +``` + +`limit` and `offset` work as in `values()`. + ## Writing Parquet Files Since parquet is a binary format, we need to provide a schema for the writer so it can know how @@ -267,7 +253,7 @@ $writer = new Writer(); ``` > [!TIP] -> `new Writer()` uses the adaptive engine. Use `Writer::arrow()` or `Writer::php()` to explicitly select an engine. +> `new Writer()` uses `AdaptiveParquetEngine`. Use `Writer::php()` to select the PHP engine. > See [Engine System](#engine-system). and write our data: @@ -309,6 +295,24 @@ We can also open a file for a resource: $writer->openForStream($stream, $schema); ``` +### Writing columns + +A batch that is already held by column is written without building rows: one list per top-level column, every list of +the same length. A schema column the array lacks is written as nulls, a key the schema lacks is ignored. + +```php +$writer->open($path, $schema); +$writer->writeColumns(['id' => [1, 2, 3], 'name' => ['a', 'b', null]]); +$writer->writeColumns(['id' => [4], 'tags' => [['x', 'y']]]); +$writer->close(); +``` + +The file is the one `writeBatch()` writes for the same rows, and a value is accepted or refused as `writeBatch()` +accepts or refuses it. Lists of different lengths are refused: +`InvalidArgumentException('writeColumns() takes lists of one length, got "id": 3, "name": 2')`. + +A custom `ParquetFileWriter` implements `writeColumns(array $columns): void`. + ### Writing a single row ```php @@ -327,15 +331,13 @@ $writer->close(); ### Writer Options -- `BYTE_ARRAY_TO_STRING` - default: `true` - if set to `true` then `BYTE_ARRAY` values will be converted to `string` - objects. - `DICTIONARY_PAGE_MIN_CARDINALITY_RATION` - default '0.4' - minimum ratio of unique values to total values for a column to have dictionary encoding. - `DICTIONARY_PAGE_SIZE` - default: `1Mb` - maximum size of dictionary page. - `GZIP_COMPRESSION_LEVEL` - default: `9` - compression level for GZIP compression (applied only when GZIP compression is enabled). -- `PAGE_SIZE_BYTES` - default: `8Kb` - maximum size of data page. -- `ROW_GROUP_SIZE_BYTES` - default: `8Mb` - maximum size of row group. +- `PAGE_SIZE_BYTES` - default: `128Kb` - maximum size of data page. +- `ROW_GROUP_SIZE_BYTES` - default: `32Mb` - maximum size of row group. - `ROW_GROUP_SIZE_CHECK_INTERVAL` default: `1000` - number of rows to write before checking if row group size limit is reached. - `VALIDATE_DATA` - default: `true` - if set to `true` then writer will validate data against schema. @@ -370,8 +372,8 @@ You will need to play a bit with those values to find the best one for your use ## Compressions > [!NOTE] -> When using the Arrow engine, all compression codecs are built into the native extension. The PHP compression -> extensions listed below are only required when using the PHP engine. +> With the Arrow engine every compression codec is built into the extension. The PHP compression extensions listed below +> are only required by the PHP engine. Parquet supports several compression algorithms. @@ -428,7 +430,7 @@ $options = Options::default()->set(Option::COLUMNS_COMPRESSIONS, [ ]); // Global compression serves as fallback for unspecified columns -$writer = new Writer(compressions: Compressions::GZIP, options: $options); +$writer = new Writer(compression: Compressions::GZIP, options: $options); ``` #### Nested Column Compression @@ -646,7 +648,7 @@ $options = Options::default()->set(Option::COLUMNS_ENCODINGS, [ 'description' => Encodings::PLAIN // High variance text ]); -$writer = new Writer(compressions: Compressions::SNAPPY, options: $options); +$writer = new Writer(compression: Compressions::SNAPPY, options: $options); ``` #### Nested Column Encoding (Flat Path Notation) @@ -763,9 +765,9 @@ $options = Options::default()->set(Option::COLUMNS_ENCODINGS, [ | Encoding | INT32/INT64 | BYTE_ARRAY | BOOLEAN | FLOAT/DOUBLE | FIXED_LEN_BYTE_ARRAY | |---------------------|-------------|------------|---------|--------------|----------------------| -| PLAIN | yes | yes | yes | yes | yes | -| RLE_DICTIONARY | yes | yes | yes | yes | no | -| DELTA_BINARY_PACKED | yes | no | no | no | no | +| PLAIN | yes | yes | yes | yes | yes | +| RLE_DICTIONARY | yes | yes | yes | yes | no | +| DELTA_BINARY_PACKED | yes | no | no | no | no | ### Performance Guidelines diff --git a/documentation/components/libs/snappy.md b/documentation/components/libs/snappy.md index 935dbc7405..94b97a6ceb 100644 --- a/documentation/components/libs/snappy.md +++ b/documentation/components/libs/snappy.md @@ -41,7 +41,7 @@ $faker = \Faker\Factory::create(); $texts = []; for ($i = 0; $i < 10_000; $i++) { - $textSize = \Flow\ETL\UniqueFactory::int(100, 5000); + $textSize = \random_int(100, 5000); $texts[] = $faker->text($textSize); } diff --git a/documentation/components/libs/types.md b/documentation/components/libs/types.md index 73cc0a2a49..848720df06 100644 --- a/documentation/components/libs/types.md +++ b/documentation/components/libs/types.md @@ -48,7 +48,7 @@ Examples: ```php get('some-input'); @@ -61,7 +61,7 @@ On top of that it will also narrow the `$string` variable to a string type, so y ```php get('some-input'); @@ -89,7 +89,7 @@ To cast a variable to a specific type, you just simply need to use the `cast` me ```php get('some-input'); @@ -164,8 +164,8 @@ List is a collection of elements of the same where keys start from 0 and are aut ```php type_string(), @@ -210,9 +210,9 @@ anywhere - including before a required one: ```php type_string(), @@ -233,10 +233,10 @@ All above types can be easily combined together, so for to get the list of users ```php ` → `nix hash convert --to sri`). 3. Regenerate the PHP stubs from the new grammar - sync `resources/proto/pg_query.proto` from diff --git a/documentation/contributing/guidelines.md b/documentation/contributing/guidelines.md index f68b2d14d1..ba18edd720 100644 --- a/documentation/contributing/guidelines.md +++ b/documentation/contributing/guidelines.md @@ -6,7 +6,7 @@ The project includes several levels of documentation, including: - Markdown files in the `documentation` directory - API documentation generated by PHPDoc, built in the `/web/landing/build/documentation/` directory -- Usage examples in the `examples` directory, divided into categories and topics +- Usage examples in the `web/landing/content/examples` directory, divided into topics # Project structure @@ -17,7 +17,6 @@ The project is structured as follows: ├── bin ├── build ├── documentation -├── examples ├── phpdoc ├── src │ ├── adapter @@ -39,8 +38,6 @@ The project is structured as follows: `just phar`. - `documentation` contains the documentation files of the entire project but also of each package separately. This folder is used to generate the documentation website by converting markdown into html. -- `examples` contains example code that demonstrates how to use the project. Those examples are also displayed on the - documentation website. - `phpdoc` contains the API documentation generated by PHPDoc. It can be built by running `just docs-api`. - `src` contains the source code of the project, which is divided into several packages: - `adapter` adapters are packages that provides implementations for different building blocks of the DataFrame like @@ -83,8 +80,8 @@ src/core Tests are usually divided into: -- `Unit` - `src/core/etl/tests/Flow/ETL/Tests/Integration` -- `Integraiton` - `src/core/etl/tests/Flow/ETL/Tests/Unit` +- `Unit` - `src/core/etl/tests/Flow/ETL/Tests/Unit` +- `Integration` - `src/core/etl/tests/Flow/ETL/Tests/Integration` In some cases on that level, there might be some test helpers, like: @@ -108,9 +105,8 @@ The above rules apply also on namespaces. So for example Adapter for `CSV` can't # PHP Versions -This project supports only the latest three PHP versions, for example: 8.2, 8.3, and 8.4. (assuming that 8.4 is the -latest version). -Development is done using the lowest supported version, which is currently PHP 8.2. +This project supports only the latest three PHP versions, currently 8.3, 8.4 and 8.5. +Development is done using the lowest supported version, which is currently PHP 8.3. The project is tested against all three versions, so you can use any of them for development. # Tools diff --git a/documentation/contributing/rust.md b/documentation/contributing/rust.md index 6b1283773c..3a9573479d 100644 --- a/documentation/contributing/rust.md +++ b/documentation/contributing/rust.md @@ -5,13 +5,14 @@ This document describes how to develop the two Rust PHP extensions in this monorepo, both written with the [ext-php-rs](https://github.com/extphprs/ext-php-rs) framework. -| Extension | Package | What it provides | -|---|---|---| -| `arrow-ext` | `flow-php/arrow-ext` | Parquet reader and writer powered by the [Apache Arrow](https://arrow.apache.org/) Rust ecosystem, exposed as `Flow\Arrow\Parquet\Reader` and `Flow\Arrow\Parquet\Writer` | -| `flow-php-ext` | `flow-php/flow-php-ext` | Native Floe binary frame encoding/decoding and row hydration/casting against a schema | +| Extension | Package | What it provides | +|----------------|-------------------------|------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| `arrow-ext` | `flow-php/arrow-ext` | Parquet reader and writer powered by the [Apache Arrow](https://arrow.apache.org/) Rust ecosystem, exposed as `Flow\Parquet\Engine\RustParquetEngine` | +| `flow-php-ext` | `flow-php/flow-php-ext` | The native column backend (`Flow\ETL\Column\RustBackend`, `RustColumn`, `RustColumnBuilder` over Apache Arrow arrays) and the Rust CSV / JSON / Parquet sources and encoders | -Both are optional. The pure-PHP implementations in `flow-php/etl` remain the canonical behaviour reference, and Flow -routes to the native code automatically when the extension is loaded. +Both are optional. The pure-PHP implementations in `flow-php/etl` and `flow-php/parquet` remain the behaviour +reference; every contract has a `Php*`, a `Rust*` and an `Adaptive*` class that picks the Rust one when the extension +is loaded. For usage documentation, see [Arrow Extension](/documentation/components/extensions/arrow-ext.md) and [Flow PHP Extension](/documentation/components/extensions/flow-php-ext.md). @@ -42,13 +43,20 @@ src/extension/arrow-ext/ ├── Makefile # Build orchestration ├── src/ # Rust source code │ ├── lib.rs # Extension entry point, module registration -│ ├── parquet/ # Parquet reader, writer, type conversion -│ └── stream/ # PHP stream adapters (Read/Write traits) +│ ├── interfaces.rs # Flow\Parquet\{ParquetEngine, ParquetFileReader, ParquetFileWriter}, registered at MINIT +│ ├── parquet/ # RustParquetEngine, readers, writer, canonical Arrow types, Arrow C Data batches +│ ├── thrift.rs # Footer bytes as the PHP ThriftModel objects +│ ├── render.rs # Parquet refusals as the Flow\Parquet exceptions +│ ├── php.rs # PHP engine helpers (class entries, hashtables, calls) +│ ├── values.rs # PHP values through the engine's C date API +│ ├── alloc.rs # Counting global allocator +│ └── exception.rs # Exception mapping ├── php/ # PHP stubs for static analysis (used when extension is not loaded) -│ └── Flow/Arrow/ +│ └── Flow/ +│ ├── Arrow/ +│ └── Parquet/Engine/ ├── tests/ -│ ├── phpt/ # PHPT test files -│ └── fixtures/ # Test parquet files +│ └── phpt/ # PHPT test files └── ext/ └── config.m4 # PIE compatibility @@ -58,16 +66,31 @@ src/extension/flow-php-ext/ ├── Makefile # Build orchestration ├── src/ # Rust source code │ ├── lib.rs # Extension entry point, module registration -│ ├── encode.rs # Floe frame body encoder -│ ├── format.rs # Floe binary format primitives -│ ├── hydrate.rs # Row hydration against a schema +│ ├── interfaces.rs # the package interfaces the Rust classes implement, registered at MINIT +│ ├── iterator.rs # RustIterator, the iterator the Rust sources return +│ ├── backend.rs # RustBackend +│ ├── column.rs # RustColumn +│ ├── builder.rs # RustColumnBuilder: native cast lanes, PHP lane for the rest +│ ├── kind_builder.rs # Arrow storage appended row by row +│ ├── physical.rs # Arrow rows as physical and logical zvals +│ ├── plan.rs # Per-type plan, cached by Type object and type JSON +│ ├── render.rs # flow-batch-frame refusals as the PHP messages │ ├── cast.rs # Value casting │ ├── json_check.rs # JSON validation shared by casting and CSV inference -│ ├── csv/ # CSV tokenizer, reader and schema-inference fold -│ ├── plan.rs # Per-column plan resolved once per schema -│ ├── ctx.rs # Shared module context +│ ├── batch_columns.rs # BatchColumns: one native builder per definition, shared by the CSV and JSON sources +│ ├── source.rs # Shared stream loop of the Rust open sources +│ ├── csv/ # CSV tokenizer, RustCSVOpenSource, RustCSVEncoder, schema-inference fold +│ ├── json/ # RustJsonOpenSource, RustJsonEncoder +│ ├── text/ # Column values as the writers render them (TextValues) +│ ├── parquet/ # RustParquetOpenSource, RustParquetOpenSink +│ ├── arrow_c.rs # Batches across the Arrow C Data Interface (arrow-ext) +│ ├── ctx.rs # Request-scoped context and engine helpers +│ ├── globals.rs # Module globals, RINIT/RSHUTDOWN +│ ├── alloc.rs # Counting global allocator (allocatedBytes()) │ ├── values.rs # Zval <-> PHP value helpers │ └── exception.rs # Exception mapping +├── crates/ +│ └── flow-batch-frame/ # Pure-Rust column buffers and BATCH frame bodies (no PHP types) ├── php/ # PHP stubs for static analysis │ └── Flow/ ├── tests/ @@ -108,13 +131,13 @@ nix-shell --run "just test --testsuite=etl-unit" # flow-php- ## Make Targets -| Target | Description | -|-----------|------------------------------------| -| `build` | Build the extension (cargo + copy) | -| `test` | Build, then run PHPT tests | +| Target | Description | +|-----------|--------------------------------------------------| +| `build` | Build the extension (cargo + copy) | +| `test` | Build, then run PHPT tests | | `install` | Copy the built module into PHP's `extension_dir` | -| `clean` | Remove build artifacts | -| `rebuild` | Full clean + build | +| `clean` | Remove build artifacts | +| `rebuild` | Full clean + build | The two PHPT runners differ in two ways: @@ -133,3 +156,41 @@ Re-enter `nix-shell` to rebuild the derivation, or build and test the extension ```bash nix-shell --arg with-rust true --run "cd src/extension/flow-php-ext && make build && make test" ``` + +This covers `crates/flow-batch-frame`, which the flow-php-ext build compiles from the same directory. + +## Extension ABI + +Each extension registers an `int` constant with the version of the contract its PHP package expects. The PHP package +holds the same number: + +| Extension | Constant (Rust `lib.rs`) | PHP side | +|-----------|--------------------------|-------------------------------------------| +| flow_php | `FLOW_PHP_ABI` | `Flow\ETL\FlowPhpExtension::ABI` | +| arrow | `FLOW_ARROW_ABI` | `Flow\Parquet\Engine\ArrowExtension::ABI` | + +Bump both, in one change, whenever a class or interface the extension registers changes. Every `Adaptive*` pick calls +`available()`: an extension that is loaded with another ABI, or with none, is refused with a `RuntimeException` +naming the extension and its version. phpt `099` (flow_php) and `054` (arrow) pin that the two numbers are equal. + +The PHP interfaces an extension also registers are guarded by `interface_exists(X::class, false)`, so an extension +that does not register one still gets the PHP declaration and fails at the ABI check, not with "Class not found". + +## The flow-batch-frame crate + +`src/extension/flow-php-ext/crates/flow-batch-frame` holds Flow's batch layout in pure Rust: one column's buffers in the +canonical form of `Column::encode()` and back, and the BATCH frame body around them. It returns every refusal as a data +`Error` variant; `flow_php` renders each into the PHP message. flow-php-ext depends on it by `path`: + +```bash +nix-shell --arg with-rust true --run "cd src/extension/flow-php-ext/crates/flow-batch-frame && cargo test" +``` + +arrow-ext does not depend on the crate. Its `canonical_type()` (`src/parquet/canonical.rs`) must produce the Arrow +types of flow-batch-frame's `kind::data_type()`, pinned by flow-php-ext phpt 091. + +## Releasing + +Cutting a minor tag `X.Y.0` includes bumping `flow-php-ext-version` and `arrow-ext-version` in +`.nix/pkgs/php-{flow-php,arrow}-ext/package.nix` to `X.(Y+1).0-dev` in the commit right after the tag. Untagged builds +report `X.(Y+1).0-dev+.g`; the CI literal check backstops a missed bump. diff --git a/documentation/contributing/wasm.md b/documentation/contributing/wasm.md index 47475e22ab..7a87725ab7 100644 --- a/documentation/contributing/wasm.md +++ b/documentation/contributing/wasm.md @@ -56,8 +56,7 @@ check), which is what triggers the rebuild. `src/extension/pg-query-ext/Makefile`, so the playground and the native extension can never disagree about which PostgreSQL grammar parses, and fails fast if that variable cannot be read. -The pin is a **branch** (`18-latest`), not a tag. Changing it re-clones automatically; upstream -moving the branch under a fixed version does not, and needs `rm -rf wasm/libpg_query-*`. +The pin is a release tag (`18.0.0`), so a build is reproducible. Changing it re-clones automatically. ## Included PHP Extensions @@ -274,7 +273,7 @@ What is now true: - `pack()`/`unpack()` 64-bit format codes (`q`, `Q`, `J`, `P`) work. `Flow\Floe` therefore works, and so does **`sortBy()`** and everything else that spills runs to disk through - `Flow\Floe\Encoding\Int64Encoder`. Before this, every Floe write in the browser died with + `Flow\ETL\Column\Layout\Int64Layout` (`pack('P*')`). Before this, every Floe write in the browser died with `ValueError: 64-bit format codes are not available for 32-bit versions of PHP`. - Large integers from Thrift/Parquet metadata no longer overflow to float. On the 32-bit build, reading a parquet file emitted hundreds of @@ -282,7 +281,7 @@ What is now true: `Flow\Parquet\Thrift\CompactProtocol` and `Flow\ETL\Bucketing\HashBucketing`; the 64-bit build emits none. - Hex constants like `0x80000000` are plain integers and need no `(int)` cast. -- `DateTimeEncoder` packs `getTimestamp()`, which now represents dates past 2038-01-19. +- `DateTimePhysical` stores `getTimestamp() * 1_000_000 + µs`, which now represents dates past 2038-01-19. The `(int)` casts in the parquet library's `fromThrift()` methods are **left in place**. They are no longer load-bearing for the playground, but the library still supports 32-bit PHP builds generally, @@ -334,7 +333,7 @@ Two consequences worth knowing before changing any of this: file of the *wrong* architecture, which the existing `[ ! -f ]` guard cannot detect. - **Switching target does not invalidate the dependency guards.** Each guard tests its own build output (`libxml2-2.11.4/` the source dir, `libzip-1.11.3/install/lib/libzip.a`, - `libpg_query-18-latest/libpg_query.a`), and none of them knows about the target. libzip + `libpg_query-18.0.0/libpg_query.a`), and none of them knows about the target. libzip additionally caches the resolved zlib path in `build/CMakeCache.txt`, which is why `build.sh` removes that directory before configuring. diff --git a/documentation/installation/docker.md b/documentation/installation/docker.md index 63286b992f..55c0432b43 100644 --- a/documentation/installation/docker.md +++ b/documentation/installation/docker.md @@ -69,16 +69,16 @@ $ docker run -v $(pwd):/flow-workspace --rm -it ghcr.io/flow-php/flow:latest run ## Bundled extensions -Alongside PHP 8.5 and the extensions Flow's adapters need - `bcmath`, `gmp`, `pdo_mysql`, +Alongside PHP 8.5 and the extensions Flow's adapters need - `bcmath`, `gmp`, `zip`, `pdo_mysql`, `pdo_pgsql`, `pdo_sqlite`, `pgsql`, and the `brotli`, `lz4`, `snappy`, `zstd` codecs - the image ships four extensions that Flow detects and uses automatically: -| Extension | Package | Effect when loaded | -|------------|-------------------------------------------------------------------------------|--------------------------------------------------------------------------------------------------------------------------------------------------------| -| `flow_php` | [flow-php/flow-php-ext](/documentation/components/extensions/flow-php-ext.md) | `AdaptiveRowHydrator` and the adaptive Floe engine run native, fusing every Floe read/write and every raw-scalar hydration into one native call per batch | -| `arrow` | [flow-php/arrow-ext](/documentation/components/extensions/arrow-ext.md) | `AdaptiveParquetEngine` selects `ArrowParquetEngine`, so Parquet reads and writes run native | -| `pg_query` | [flow-php/pg-query-ext](/documentation/components/extensions/pg-query-ext.md) | `Flow\PostgreSql\Parser` becomes usable at all - SQL parsing, normalization and AST manipulation | -| `protobuf` | `pecl/protobuf` | `Flow\PostgreSql\Parser` decodes the parse tree in C instead of pure PHP - measured ~69x faster end to end | +| Extension | Package | Effect when loaded | +|------------|-------------------------------------------------------------------------------|-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| `flow_php` | [flow-php/flow-php-ext](/documentation/components/extensions/flow-php-ext.md) | `AdaptiveBackend` picks `RustBackend` ([Column Backend](/documentation/components/core/column-backend.md)); CSV and JSON are read and written in Rust; with `arrow`, Parquet batches pass between the extensions without PHP values | +| `arrow` | [flow-php/arrow-ext](/documentation/components/extensions/arrow-ext.md) | `AdaptiveParquetEngine` picks arrow-ext's `RustParquetEngine`, so Parquet files are decoded and encoded in Rust | +| `pg_query` | [flow-php/pg-query-ext](/documentation/components/extensions/pg-query-ext.md) | `Flow\PostgreSql\Parser` becomes usable at all - SQL parsing, normalization and AST manipulation | +| `protobuf` | `pecl/protobuf` | `Flow\PostgreSql\Parser` decodes the parse tree in C instead of pure PHP - measured ~69x faster end to end | `pdo_pgsql` and `pgsql` link libpq 18 from the PGDG repository, matching the PostgreSQL 18 grammar `pg_query` is built against. PHP 8.5 additionally compiles in `lexbor`, `uri` and Zend OPcache unconditionally. @@ -90,24 +90,25 @@ that Flow detects and uses automatically: ### Opting out of the native path -Engine selection happens per read and per write, so a single step can be pinned to the PHP implementation: +The backend is chosen per data frame, the Parquet engine per read: ```php hydrator(new PhpRowHydrator())) +return data_frame(config_builder()->backend(new PhpBackend())) ->read(from_parquet(__DIR__ . '/input.parquet', engine: new PhpParquetEngine())) - ->write(to_floe(__DIR__ . '/output.floe', engine: FloeEngine::php)); + ->write(to_floe(__DIR__ . '/output.floe')); ``` +`PhpBackend` keeps the batches in PHP, but CSV and JSON files are still parsed by `flow_php` while it is loaded. + To take the whole container off one native path, mount an empty file over that extension's ini: ```shell @@ -115,6 +116,6 @@ $ docker run --rm -v /dev/null:/usr/local/etc/php/conf.d/docker-php-ext-flow_php -v $(pwd):/flow-workspace ghcr.io/flow-php/flow:latest run /flow-workspace/pipeline.php ``` -Once a native engine is selected it does **not** silently degrade. Extension failures surface as -`Flow\Floe\Exception\ExtensionException`, which `FloeReader` and `FloeWriter` wrap as -`Flow\Floe\Exception\FloeException`. +Once a native path is selected it does **not** silently degrade. `flow_php` failures surface as +`Flow\ETL\Exception\RuntimeException`; a Floe read wraps them as `Flow\Floe\Exception\FloeException`. arrow-ext +failures surface as `Flow\Parquet\Exception\*`. diff --git a/documentation/installation/packages/arrow-ext.md b/documentation/installation/packages/arrow-ext.md index 64fc3c63bd..bddb914f30 100644 --- a/documentation/installation/packages/arrow-ext.md +++ b/documentation/installation/packages/arrow-ext.md @@ -44,6 +44,14 @@ echo "extension=arrow" > $(php -r "echo PHP_CONFIG_FILE_SCAN_DIR;")/arrow.ini php -m | grep arrow ``` +## Version + +Install the extension of the Flow release you use. `flow-php/parquet` and `flow-php/etl-adapter-parquet` conflict +with `ext-arrow <0.45`. + +An extension built for another release is refused with a `RuntimeException` naming its version and ABI, raised by +`AdaptiveParquetEngine`. `new PhpParquetEngine()` still runs. + ## PIE [PIE](https://github.com/php/pie) is the modern PHP extension installer. diff --git a/documentation/installation/packages/etl-adapter-csv.md b/documentation/installation/packages/etl-adapter-csv.md index 2e19ec4344..89726156ea 100644 --- a/documentation/installation/packages/etl-adapter-csv.md +++ b/documentation/installation/packages/etl-adapter-csv.md @@ -16,3 +16,8 @@ seo_description: > ```bash composer require flow-php/etl-adapter-csv:~--FLOW_PHP_VERSION-- ``` + +## Optional Extension + +With the [flow_php extension](/documentation/installation/packages/flow-php-ext.md) loaded, CSV files are read and +written in Rust. The adapter conflicts with `ext-flow_php <0.45`. diff --git a/documentation/installation/packages/etl-adapter-json.md b/documentation/installation/packages/etl-adapter-json.md index 98560e1203..28b1910842 100644 --- a/documentation/installation/packages/etl-adapter-json.md +++ b/documentation/installation/packages/etl-adapter-json.md @@ -20,3 +20,8 @@ composer require flow-php/etl-adapter-json:~--FLOW_PHP_VERSION-- ## Core Dependencies - [halaxa/json-machine](https://packagist.org/packages/halaxa/json-machine) + +## Optional Extension + +With the [flow_php extension](/documentation/installation/packages/flow-php-ext.md) loaded, JSON files are read and +written in Rust. The adapter conflicts with `ext-flow_php <0.45`. diff --git a/documentation/installation/packages/etl-adapter-parquet.md b/documentation/installation/packages/etl-adapter-parquet.md index e66a3b1590..26b48304d8 100644 --- a/documentation/installation/packages/etl-adapter-parquet.md +++ b/documentation/installation/packages/etl-adapter-parquet.md @@ -20,3 +20,11 @@ composer require flow-php/etl-adapter-parquet:~--FLOW_PHP_VERSION-- ## Core Dependencies - [flow-php/parquet](https://packagist.org/packages/flow-php/parquet) + +## Optional Extensions + +- [arrow-ext](/documentation/installation/packages/arrow-ext.md) - Parquet files are decoded and encoded in Rust. +- [flow_php](/documentation/installation/packages/flow-php-ext.md) - with arrow-ext also loaded, batches pass between + the extensions without PHP values. + +The adapter conflicts with `ext-arrow <0.45` and `ext-flow_php <0.45`. diff --git a/documentation/installation/packages/etl.md b/documentation/installation/packages/etl.md index 90e08442b0..1d695d54b0 100644 --- a/documentation/installation/packages/etl.md +++ b/documentation/installation/packages/etl.md @@ -25,3 +25,12 @@ composer require flow-php/etl:~--FLOW_PHP_VERSION-- ## Suggested Extensions - `ext-bcmath` - for more precise calculations (bundled with PHP, enable via your system package manager) + +## Optional Extension + +The [flow_php extension](/documentation/components/extensions/flow-php-ext.md) stores columns as Arrow arrays, see +[Column Backend](/documentation/components/core/column-backend.md). `flow-php/etl` conflicts with `ext-flow_php <0.45`. + +```bash +pie install flow-php/flow-php-ext +``` diff --git a/documentation/installation/packages/flow-php-ext.md b/documentation/installation/packages/flow-php-ext.md index 7a618ebde8..b41f432168 100644 --- a/documentation/installation/packages/flow-php-ext.md +++ b/documentation/installation/packages/flow-php-ext.md @@ -44,6 +44,15 @@ echo "extension=flow_php" > $(php -r "echo PHP_CONFIG_FILE_SCAN_DIR;")/flow_php. php -m | grep flow_php ``` +## Version + +Install the extension of the Flow release you use. `flow-php/etl`, `flow-php/etl-adapter-csv`, `-json` and `-parquet` +conflict with `ext-flow_php <0.45`. + +An extension built for another release is refused with a `RuntimeException` naming its version and ABI, raised by +the first `Adaptive*` class that picks a lane. A pipeline that reaches none, e.g. +`config_builder()->backend(new PhpBackend())` with `from_array()`, still runs. + ## PIE [PIE](https://github.com/php/pie) is the modern PHP extension installer. diff --git a/documentation/installation/packages/parquet.md b/documentation/installation/packages/parquet.md index d40a8d7dcb..b8aeb30b54 100644 --- a/documentation/installation/packages/parquet.md +++ b/documentation/installation/packages/parquet.md @@ -37,4 +37,5 @@ For high-performance Parquet reading/writing via Apache Arrow, install the arrow pie install flow-php/arrow-ext ``` -See [Arrow Extension installation](/documentation/installation/packages/arrow-ext.md) for full details. +`flow-php/parquet` conflicts with `ext-arrow <0.45`. See +[Arrow Extension installation](/documentation/installation/packages/arrow-ext.md) for full details. diff --git a/documentation/introduction.md b/documentation/introduction.md index 0004128a0b..ab8b15453d 100644 --- a/documentation/introduction.md +++ b/documentation/introduction.md @@ -50,7 +50,7 @@ Every Flow pipeline has three stages: - **Load** - `->write(...)` streams the result into a sink; `->run()` executes the whole thing. -Data moves through the pipeline as **DataFrame → Rows → Row**. Every `Rows` +Data moves through the pipeline as **DataFrame → Rows** batches of columns. Every `Rows` batch carries the schema that types its columns. ## Where to go from here diff --git a/documentation/quick-start.md b/documentation/quick-start.md index 630033c720..803ab93952 100644 --- a/documentation/quick-start.md +++ b/documentation/quick-start.md @@ -14,8 +14,7 @@ Let's take a look at a simple example of how to use Flow ETL to read a CSV file, declare(strict_types=1); use function Flow\ETL\Adapter\CSV\{from_csv, to_csv}; -use function Flow\ETL\DSL\{data_frame, lit, ref, sum, to_output}; -use Flow\ETL\Filesystem\SaveMode; +use function Flow\ETL\DSL\{data_frame, lit, overwrite, ref, sum, to_output}; require __DIR__ . '/vendor/autoload.php'; @@ -50,15 +49,13 @@ It creates a new instance of the `Flow\ETL\Flow` class, which is the main class For comprehensive DataFrame documentation, see the [Data Frame Guide](/documentation/components/core/core.md). -[Data Frame Examples](/data_frame/#example) - -[Examples Documentation](/documentation/examples) +[Data Frame Examples](/getting_started/#example) ## Extraction - Reading The first step in creating a data processing pipeline is to read the data from a data source. Extractors are responsible for reading data from a data source and converting it into a format that can be processed by Flow ETL. -All extractors return \Generator and by design will return rows one by one - this is to ensure that memory consumption is constant and low. +All extractors return a `\Generator` of batches (`Rows`, one column per schema definition), so memory stays constant no matter how big the source is. ```php data_frame() @@ -74,7 +71,7 @@ data_frame() ->read(from_csv(__DIR__ . '/reports/*.csv')); ``` -[Extractors Examples](/data_reading/#example) +[Extractors Examples](/reading/#example) ## Transformation @@ -92,7 +89,7 @@ One of the most powerful features of Flow ETL is the ability to transform data u ->withEntry('revenue', ref('total_price')->minus(ref('discount'))) ``` -`withEntry()` function accepts two arguments, the first one is the name of the new entry (column), the second one is the value of the new column. +`withEntry()` function accepts two arguments, the first one is the name of the new column, the second one is the value of the new column. The value of the new column can be a literal value, a reference to an existing column or a function call. - `ref('created_at')` - creates a reference to the `created_at` column. @@ -125,8 +122,8 @@ truncating the output. ->write(to_csv(__DIR__ . '/daily_revenue.csv')->saveMode(overwrite())) ``` -Second write is writing the data to a CSV file, we're using the `mode()` function to set the save mode to `overwrite`. -There are three save modes available: +Second write is writing the data to a CSV file, we're using `saveMode(overwrite())` to set the save mode to `overwrite`. +There are four save modes available: - `SaveMode::Append` - If data sink already exists, data will be appended. This solution might cause data duplication since it does not check if the given rows already exist. - `SaveMode::ExceptionIfExists` - If data sink already exists an error will be thrown. @@ -134,10 +131,10 @@ There are three save modes available: - `SaveMode::Overwrite` - If data sink already exists, it will be removed and written again. > [!NOTE] -> Append mode is not really appending anything to existing files, instead it creates a folder in which it stores outputs under randomized file names. -> It can be later read using a glob-pattern, for example `from_csv('/path/to/folder/*.csv')`. +> Append mode does not append to an existing file. It writes a sibling file `name_.ext` next to it +> (`daily_revenue_ecd5b31525.csv`). Read all of them with a glob: `from_csv(__DIR__ . '/daily_revenue*.csv')`. -[Loaders Examples](/data_writing/#example) +[Loaders Examples](/writing/#example) ## Lazy Execution diff --git a/documentation/upgrading-0.16-0.35.md b/documentation/upgrading-0.16-0.35.md new file mode 100644 index 0000000000..1ffcf62b98 --- /dev/null +++ b/documentation/upgrading-0.16-0.35.md @@ -0,0 +1,414 @@ +# Upgrade Guide 0.16.x - 0.35.x + +[TOC] + +This document provides guidelines for upgrading between versions of Flow PHP. Please follow the instructions for your +specific version to ensure a smooth upgrade process. + +Upgrading from older versions: [0.8.x to 0.16.x](/documentation/upgrading-0.8-0.16.md) + +--- + +## Upgrading from 0.34.x to 0.35.x + +### 1) `flow-php/postgresql` - `DataType` renamed to `ColumnType` + +The `DataType` class used for schema/DDL definitions has been renamed to `ColumnType` to better communicate its purpose. +All related DSL functions have been renamed from `data_type_*` to `column_type_*`. + +| Removed | Replacement | +|------------------------------------------------|--------------------------------------------------| +| `Flow\PostgreSql\QueryBuilder\Schema\DataType` | `Flow\PostgreSql\QueryBuilder\Schema\ColumnType` | +| `Flow\PostgreSql\Parser\DataTypeParser` | `Flow\PostgreSql\Parser\ColumnTypeParser` | +| `data_type_integer()` | `column_type_integer()` | +| `data_type_smallint()` | `column_type_smallint()` | +| `data_type_bigint()` | `column_type_bigint()` | +| `data_type_boolean()` | `column_type_boolean()` | +| `data_type_text()` | `column_type_text()` | +| `data_type_varchar()` | `column_type_varchar()` | +| `data_type_char()` | `column_type_char()` | +| `data_type_numeric()` | `column_type_numeric()` | +| `data_type_decimal()` | `column_type_decimal()` | +| `data_type_real()` | `column_type_real()` | +| `data_type_double_precision()` | `column_type_double_precision()` | +| `data_type_date()` | `column_type_date()` | +| `data_type_time()` | `column_type_time()` | +| `data_type_timestamp()` | `column_type_timestamp()` | +| `data_type_timestamptz()` | `column_type_timestamptz()` | +| `data_type_interval()` | `column_type_interval()` | +| `data_type_uuid()` | `column_type_uuid()` | +| `data_type_json()` | `column_type_json()` | +| `data_type_jsonb()` | `column_type_jsonb()` | +| `data_type_bytea()` | `column_type_bytea()` | +| `data_type_inet()` | `column_type_inet()` | +| `data_type_cidr()` | `column_type_cidr()` | +| `data_type_macaddr()` | `column_type_macaddr()` | +| `data_type_serial()` | `column_type_serial()` | +| `data_type_smallserial()` | `column_type_smallserial()` | +| `data_type_bigserial()` | `column_type_bigserial()` | +| `data_type_array()` | `column_type_array()` | +| `data_type_custom()` | `column_type_custom()` | +| `data_type_from_string()` | `column_type_from_string()` | + +Before: + +```php +use Flow\PostgreSql\QueryBuilder\Schema\DataType; +use function Flow\PostgreSql\DSL\data_type_integer; +use function Flow\PostgreSql\DSL\data_type_varchar; + +column('age', data_type_integer()); +column('name', data_type_varchar(255)); +cast(ref('id'), DataType::bigint()); +``` + +After: + +```php +use Flow\PostgreSql\QueryBuilder\Schema\ColumnType; +use function Flow\PostgreSql\DSL\column_type_integer; +use function Flow\PostgreSql\DSL\column_type_varchar; + +column('age', column_type_integer()); +column('name', column_type_varchar(255)); +cast(ref('id'), ColumnType::bigint()); +``` + +### 2) `flow-php/postgresql` - `PostgreSqlType` renamed to `ValueType` + +The `PostgreSqlType` enum used for value binding/casting has been renamed to `ValueType` to better communicate its +purpose. All related DSL functions have been renamed from `pgsql_type_*` to `value_type_*`. + +| Removed | Replacement | +|-----------------------------------------------|------------------------------------------| +| `Flow\PostgreSql\Client\Types\PostgreSqlType` | `Flow\PostgreSql\Client\Types\ValueType` | +| `pgsql_type_text()` | `value_type_text()` | +| `pgsql_type_varchar()` | `value_type_varchar()` | +| `pgsql_type_integer()` | `value_type_integer()` | +| `pgsql_type_bigint()` | `value_type_bigint()` | +| `pgsql_type_smallint()` | `value_type_smallint()` | +| `pgsql_type_boolean()` | `value_type_boolean()` | +| `pgsql_type_float4()` | `value_type_float4()` | +| `pgsql_type_float8()` | `value_type_float8()` | +| `pgsql_type_numeric()` | `value_type_numeric()` | +| `pgsql_type_date()` | `value_type_date()` | +| `pgsql_type_timestamp()` | `value_type_timestamp()` | +| `pgsql_type_timestamptz()` | `value_type_timestamptz()` | +| `pgsql_type_json()` | `value_type_json()` | +| `pgsql_type_jsonb()` | `value_type_jsonb()` | +| `pgsql_type_uuid()` | `value_type_uuid()` | +| `pgsql_type_bytea()` | `value_type_bytea()` | +| `pgsql_type_inet()` | `value_type_inet()` | +| `pgsql_type_cidr()` | `value_type_cidr()` | +| All other `pgsql_type_*()` functions | Corresponding `value_type_*()` functions | + +Before: + +```php +use Flow\PostgreSql\Client\Types\PostgreSqlType; +use function Flow\PostgreSql\DSL\pgsql_type_uuid; +use function Flow\PostgreSql\DSL\pgsql_type_text_array; + +typed('550e8400-e29b-41d4-a716-446655440000', pgsql_type_uuid()); +typed(['tag1', 'tag2'], pgsql_type_text_array()); +typed(42, PostgreSqlType::INT4); +``` + +After: + +```php +use Flow\PostgreSql\Client\Types\ValueType; +use function Flow\PostgreSql\DSL\value_type_uuid; +use function Flow\PostgreSql\DSL\value_type_text_array; + +typed('550e8400-e29b-41d4-a716-446655440000', value_type_uuid()); +typed(['tag1', 'tag2'], value_type_text_array()); +typed(42, ValueType::INT4); +``` + +--- + +## Upgrading from 0.31.x to 0.32.x + +### 1) Removal of Meilisearch Adapter + +The Meilisearch adapter has been removed from Flow PHP. If you were using it, please migrate to Elasticsearch adapter. + +### 2) Removed deprecated DSL functions + +All type-related DSL functions have been moved from `Flow\ETL\DSL` to `Flow\Types\DSL`. Update your imports accordingly. + +| Removed Function | Replacement | +|---------------------------|-------------------------------------------| +| `chunks_from()` | `batches()` | +| `type_structure()` | `\Flow\Types\DSL\type_structure()` | +| `type_union()` | `\Flow\Types\DSL\type_union()` | +| `type_optional()` | `\Flow\Types\DSL\type_optional()` | +| `type_from_array()` | `\Flow\Types\DSL\type_from_array()` | +| `is_nullable()` | `\Flow\Types\DSL\type_is_nullable()` | +| `type_equals()` | `\Flow\Types\DSL\type_equals()` | +| `types()` | `\Flow\Types\DSL\types()` | +| `type_list()` | `\Flow\Types\DSL\type_list()` | +| `type_map()` | `\Flow\Types\DSL\type_map()` | +| `type_json()` | `\Flow\Types\DSL\type_json()` | +| `type_datetime()` | `\Flow\Types\DSL\type_datetime()` | +| `type_date()` | `\Flow\Types\DSL\type_date()` | +| `type_time()` | `\Flow\Types\DSL\type_time()` | +| `type_xml()` | `\Flow\Types\DSL\type_xml()` | +| `type_xml_element()` | `\Flow\Types\DSL\type_xml_element()` | +| `type_uuid()` | `\Flow\Types\DSL\type_uuid()` | +| `type_int()` | `\Flow\Types\DSL\type_integer()` | +| `type_integer()` | `\Flow\Types\DSL\type_integer()` | +| `type_string()` | `\Flow\Types\DSL\type_string()` | +| `type_float()` | `\Flow\Types\DSL\type_float()` | +| `type_boolean()` | `\Flow\Types\DSL\type_boolean()` | +| `type_instance_of()` | `\Flow\Types\DSL\type_instance_of()` | +| `type_resource()` | `\Flow\Types\DSL\type_resource()` | +| `type_array()` | `\Flow\Types\DSL\type_array()` | +| `type_callable()` | `\Flow\Types\DSL\type_callable()` | +| `type_null()` | `\Flow\Types\DSL\type_null()` | +| `type_enum()` | `\Flow\Types\DSL\type_enum()` | +| `struct_schema()` | `structure_schema()` | +| `get_type()` | `\Flow\Types\DSL\get_type()` | +| `print_schema()` | `schema_to_ascii()` | +| `type_is()` | `\Flow\Types\DSL\type_is()` | +| `type_is_any()` | `\Flow\Types\DSL\type_is_any()` | +| `dom_element_to_string()` | `\Flow\Types\DSL\dom_element_to_string()` | + +### 3) Removed deprecated DataFrame methods + +| Removed Method | Replacement | +|----------------------------------------|--------------------------------------------------------------| +| `DataFrame::validate()` | `DataFrame::match()` | +| `DataFrame::renameAll()` | `DataFrame::renameEach(rename_replace(...))` | +| `DataFrame::renameAllLowerCase()` | `DataFrame::renameEach(rename_style(StringStyles::LOWER))` | +| `DataFrame::renameAllUpperCase()` | `DataFrame::renameEach(rename_style(StringStyles::UPPER))` | +| `DataFrame::renameAllUpperCaseFirst()` | `DataFrame::renameEach(rename_style(StringStyles::UCFIRST))` | +| `DataFrame::renameAllUpperCaseWord()` | `DataFrame::renameEach(rename_style(StringStyles::UCWORDS))` | +| `DataFrame::renameAllStyle()` | `DataFrame::renameEach(rename_style(...))` | + +### 4) Removed deprecated Schema methods + +| Removed Method | Replacement | +|---------------------------|-------------------------------| +| `Schema::entries()` | `Schema::references()->all()` | +| `Schema::getDefinition()` | `Schema::get()` | +| `Schema::nullable()` | `Schema::makeNullable()` | + +### 5) Removed deprecated Definition methods + +| Removed Method | Replacement | +|--------------------------|------------------------------| +| `Definition::nullable()` | `Definition::makeNullable()` | + +This applies to all Definition implementations: `BooleanDefinition`, `DateDefinition`, `DateTimeDefinition`, +`EnumDefinition`, `FloatDefinition`, `HTMLDefinition`, `HTMLElementDefinition`, `IntegerDefinition`, `JsonDefinition`, +`ListDefinition`, `MapDefinition`, `StringDefinition`, `StructureDefinition`, `TimeDefinition`, `UuidDefinition`, +`XMLDefinition`, `XMLElementDefinition`. + +### 6) Removed deprecated FileExtractor and PathFiltering methods + +| Removed Method | Replacement | +|------------------------------|-----------------------------------| +| `FileExtractor::addFilter()` | `FileExtractor::withPathFilter()` | +| `PathFiltering::addFilter()` | `PathFiltering::withPathFilter()` | + +`withPathFilter()` is removed in 0.45 - see 8) of that version. + +### 7) Removed deprecated ScalarFunctionChain methods + +| Removed Method | Replacement | +|----------------------------------------------|---------------------------------------------------| +| `ScalarFunctionChain::domElementAttribute()` | `ScalarFunctionChain::domElementAttributeValue()` | + +### 8) Removed deprecated Config constants + +| Removed Constant | Replacement | +|-------------------------------|-----------------------------------| +| `Config::CACHE_DIR_ENV` | `CacheConfig::CACHE_DIR_ENV` | +| `Config::SORT_MAX_MEMORY_ENV` | `SortConfig::SORT_MAX_MEMORY_ENV` | + +### 9) Removed deprecated Transformers + +| Removed Transformer | Replacement | +|-----------------------------------------|--------------------------------------------------| +| `EntryNameStyleConverterTransformer` | Use `DataFrame::renameEach(rename_style(...))` | +| `RenameAllCaseTransformer` | Use `DataFrame::renameEach(rename_style(...))` | +| `RenameStrReplaceAllEntriesTransformer` | Use `DataFrame::renameEach(rename_replace(...))` | + +### 10) Removed deprecated classes + +| Removed Class | Replacement | +|-------------------------------------------------|--------------------------------| +| `Flow\ETL\Function\StyleConverter\StringStyles` | `Flow\ETL\String\StringStyles` | + +--- + +## Upgrading from 0.28.x to 0.29.x + +### 1) JsonType now uses Json value object instead of string + +The `JsonType` has been refactored to use a dedicated `Json` value object (similar to `Uuid`/`UuidType` pattern). This +allows static analysis tools to distinguish between regular strings and JSON strings. + +**Breaking Changes:** + +- `JsonType::assert()` now returns `Json` instance instead of `string` +- `JsonType::cast()` now returns `Json` instance instead of `string` +- `JsonType::isValid()` now checks for `Json` instance (plain strings are no longer valid) +- `Cast::cast('json', $value)` function now returns `Json` object instead of string +- `type_json()` return type annotation changed from `Type` to `Type` +- `JsonEntry::value()` now returns `?Json` instead of `?array` (consistent with `UuidEntry::value()` returning `?Uuid`) +- `JsonEntry::json()` method removed (use `value()` instead) + +**Migration:** + +If you were using `type_json()->cast($value)` and expected a string, use `->toString()`: + +Before: + +```php +$jsonString = type_json()->cast($array); // was string +``` + +After: + +```php +$json = type_json()->cast($array); // now Json object +$jsonString = $json->toString(); // get the string +$jsonArray = $json->toArray(); // get as array +``` + +If you were using `JsonEntry::value()` and expected an array: + +Before: + +```php +$entry = json_entry('data', ['key' => 'value']); +$array = $entry->value(); // was array +``` + +After: + +```php +$entry = json_entry('data', ['key' => 'value']); +$json = $entry->value(); // now Json object +$array = $json?->toArray(); // get as array +$string = $json?->toString(); // get as string +``` + +If you were using `JsonEntry::json()`: + +Before: + +```php +$json = $entry->json(); +``` + +After: + +```php +$json = $entry->value(); // json() method removed, use value() instead +``` + +**New Json value object features:** + +```php +use Flow\Types\Value\Json; + +// Create from string +$json = new Json('{"key": "value"}'); + +// Create from array +$json = Json::fromArray(['key' => 'value']); + +// Check if valid JSON +Json::isValid('{"key": "value"}'); // true + +// Convert to string/array +$json->toString(); // '{"key":"value"}' +$json->toArray(); // ['key' => 'value'] + +// Json implements Stringable +(string) $json; // '{"key":"value"}' + +// Json implements JsonSerializable +json_encode($json); // '{"key":"value"}' +``` + +**Note:** `JsonEntry::value()` now returns `?Json` for consistency with `UuidEntry::value()` returning `?Uuid`. Use +`->toArray()` or `->toString()` on the Json object to get the underlying data. + +**Row methods behavior:** + +```php +// Row::toArray() converts Json to array automatically (for convenient serialization) +$row->toArray(); // Returns ['data' => ['key' => 'value']] not ['data' => Json(...)] + +// Row::valueOf() returns the raw value (Json object for json entries) +$row->valueOf('data'); // Returns Json object (use ->toArray() if you need array) + +// Entry value() returns the typed value +$row->get('data')->value(); // Returns Json object (use ->toArray() if you need array) +``` + +--- + +## Upgrading from 0.26.x to 0.27.x + +### 1) Force `EntryFactory $entryFactory` to be required on `array_to_row` & `array_to_row(s)` + +Before: + +```php +to_entry('name', 'data'); +array_to_row([]); +array_to_rows([]); +``` + +After: + +```php +to_entry('name', 'data', flow_context(config())->entryFactory()); +array_to_row([], flow_context(config())->entryFactory()); +array_to_rows([], flow_context(config())->entryFactory()); +``` + +--- + +## Upgrading from 0.16.x to 0.17.x + +### 1) Removed $nullable property from all types + +Before: + +```php +type_string(nullable:true)->toString() // ?string +``` + +After: + +```php +type_optional(string())->toString() // ?string +``` + +### 2) Removed precision from `float_type()` + +Before `float_type()` use to have default precision 6. This means that any operations on float had to round values to +given precision. The problem with this approach is that all operations now need to receive a dedicated rounding option. + +Instead, end users should handle precision of float columns through `round()` scalar function. + +### 3) Moved all Types to `Flow\Types\Type` namespace + +Before + +```php +\Flow\ETL\DSL\type_string(); // now deprecated, alias for \Flow\Types\DSL\type_string(); +``` + +After + +```php +\Flow\Types\DSL\type_string(); +``` diff --git a/documentation/upgrading-0.3-0.8.md b/documentation/upgrading-0.3-0.8.md new file mode 100644 index 0000000000..2ca4797ef5 --- /dev/null +++ b/documentation/upgrading-0.3-0.8.md @@ -0,0 +1,368 @@ +# Upgrade Guide 0.3.x - 0.8.x + +[TOC] + +This document provides guidelines for upgrading between versions of Flow PHP. Please follow the instructions for your +specific version to ensure a smooth upgrade process. + +--- + +## Upgrading from 0.7.x to 0.8.x + +### 1) Joins + +To support joining bigger datasets, we had to move from initial NestedLoop join algorithm into Hash Join algorithm. + +- the only supported coin expression is `=` (equals) that can be grouped with `AND` and `OR` operators. +- `joinPrefix` is now always required, and by default is set to 'joined_' +- join will always result all columns from both datasets, columns used in join condition will be prefixed with + `joinPrefix`. + +Other than that, API stays the same. + +Above changes were introduced in all 3 types of joins: + +- `DataFrame::join()` +- `DataFrame::joinEach()` +- `DataFrame::crossJoin()` + +### 2) GroupBy + +From now on, `DataFrame::groupBy()` method will return `GroupedDataFrame` object, which is nothing more than a GroupBy +statement Builder. To get the results, you first need to define the aggregation functions or optionally pivot the data. + +--- + +## Upgrading from 0.6.x to 0.7.x + +### 1) DataFrame::appendSafe () method was removed + +`DataFrame::appendSafe()` aka `DataFrame::threadSafe()` method was removed as it was introducing additional complexity +and was not used in any of the adapters. + +--- + +## Upgrading from 0.5.x to 0.6.x + +### 1) Rows::merge () accepts single instance of Rows + +Before: + +```php +Rows::merge(Rows ...$rows) : Rows +``` + +After: + +```php +Rows::merge(Rows $rows) : Rows +``` + +--- + +## Upgrading from 0.4.x to 0.5.x + +### 1) Entry factory moved from extractors to `FlowContext` + +To improve code quality and reduce code coupling `EntryFactory` was removed from all constructors of extractors, in +favor of passing it into `FlowContext` & re-using same entry factory in a whole pipeline. + +### 2) Invalid schema has no fallback in `NativeEntryFactory` + +Before, passing `Schema` into `NativeEntryFactory::create()` had fallback when the given entry was not found in a passed +schema, now the schema has higher priority & fallback is no longer available, instead when the definition is missing in +a passed schema, `InvalidArgumentException` will be thrown. + +### 3) BufferLoader was removed + +BufferLoader was removed in favor of `DataFrame::collect(int $batchSize = null)` method which now accepts additional +argument `$batchSize` that will keep collecting Rows from Extractor until the given batch size is reached. Which does +exactly the same thing as BufferLoader did, but in a more generic way. + +### 4) Pipeline Closure + +Pipeline Closure was reduced to be only Loader Closure and it was moved to \Flow\ETL\Loader namespace. Additionally, +\Closure::close method no longer requires Rows to be passed as an argument. + +### 5) Parallelize + +DataFrame::parallelize () method is deprecated, and it will be removed, instead use DataFrame::batchSize (int $size) +method. + +### 6) Rows in batch - Extractors + +From now, file-based Extractors will always throw one Row at time, in order to merge them into bigger groups use +`DataFrame::batchSize(int $size)` just after extractor method. + +Before: + +```php +read(CSV::from(__DIR__ . '/1_mln_rows.csv', rows_in_batch: 100)) + ->write(To::output()) + ->count(); +``` + +After: + +```php +(new Flow()) + ->read(CSV::from(__DIR__ . '/1_mln_rows.csv',)) + ->batchSize(100) + ->write(To::output()) + ->count(); +``` + +Affected extractors: + +- CSV +- Parquet +- JSON +- Text +- XML +- Avro +- DoctrineDBAL - `rows_in_batch` wasn't removed, but now results are thrown row by row, instead of whole page. +- GoogleSheet + +### 7) `GoogleSheetExtractor` + +Argument `$rows_in_batch` was renamed to `$rows_per_page` which no longer determines the size of the batch, but the size +of the page that will be fetched from Google API. Rows are yielded one by one. + +### 8) `DataFrame::threadSafe()` method was replaced by `DataFrame::appendSafe()` + +`DataFrame::appendSafe()` is doing exactly the same thing as the old method, it's just more descriptive and +self-explanatory. It's no longer mandatory to set this flat to true when using SaveMode::APPEND, it's now set +automatically. + +### 9) Loaders - chunk size + +Loaders are no longer accepting chunk_size parameter, from now in order to control the number of rows saved at once use +`DataFrame::batchSize(int $size)` method. + +### 10) Removed DSL functions: `datetime_string()`, `json_string()` + +Those functions were removed in favor of accepting string values in related DSL functions: + +- `datetime_string()` => `datetime()`, +- `json_string()` => `json()` & `json_object()` + +### 11) Removed Asynchronous Processing + +More details can be found in [this issue](https://github.com/flow-php/flow/issues/793). + +- Removed etl-adapter-amphp +- Removed etl-adapter-reactphp +- Removed `LocalSocketPipeline` +- Removed `DataFrame::pipeline()` + +### 12) `CollectionEntry` removal + +After adding native & logical types into the Flow, we remove the `CollectionEntry` as obsolete. New types that cover it +better are: `ListType`, `MapType` & `StructureType` along with related new entry types. + +### 13) Removed `from*()` methods from scalar entries + +Removed `BooleanEntry::from()`, `FloatEntry::from()`, `IntegerEntry::from()`, `StringEntry::fromDateTime()` methods in +favor of using DSL functions. + +### 14) Removed deprecated `Sha1IdFactory` + +Class `Sha1IdFactory` was removed, use `HashIdFactory` class: + +```php +(new HashIdFactory('entry_name'))->withAlgorithm('sha1'); +``` + +### 15) Deprecate DSL Static classes + +DSL static classes were deprecated in favor of using functions defined in `src/core/etl/src/Flow/ETL/DSL/functions.php` +file. + +Deprecated classes: + +- `src/core/etl/src/Flow/ETL/DSL/From.php` +- `src/core/etl/src/Flow/ETL/DSL/Handler.php` +- `src/core/etl/src/Flow/ETL/DSL/To.php` +- `src/core/etl/src/Flow/ETL/DSL/Transform.php` +- `src/core/etl/src/Flow/ETL/DSL/Partitions.php` +- `src/adapter/etl-adapter-avro/src/Flow/ETL/DSL/Avro.php` +- `src/adapter/etl-adapter-chartjs/src/Flow/ETL/DSL/ChartJS.php` +- `src/adapter/etl-adapter-csv/src/Flow/ETL/DSL/CSV.php` +- `src/adapter/etl-adapter-doctrine/src/Flow/ETL/DSL/Dbal.php` +- `src/adapter/etl-adapter-elasticsearch/src/Flow/ETL/DSL/Elasticsearch.php` +- `src/adapter/etl-adapter-google-sheet/src/Flow/ETL/DSL/GoogleSheet.php` +- `src/adapter/etl-adapter-json/src/Flow/ETL/DSL/Json.php` +- `src/adapter/etl-adapter-meilisearch/src/Flow/ETL/DSL/Meilisearch.php` +- `src/adapter/etl-adapter-parquet/src/Flow/ETL/DSL/Parquet.php` +- `src/adapter/etl-adapter-text/src/Flow/ETL/DSL/Text.php` +- `src/adapter/etl-adapter-xml/src/Flow/ETL/DSL/XML.php` + +--- + +## Upgrading from 0.3.x to 0.4.x + +### 1) Transformers replaced with scalar functions + +Transformers are a really powerful tool that was used in Flow since the beginning, but that tool was too powerful for +the simple cases that were needed, and introduced additional complexity and maintenance issues when they were +handwritten. + +We reworked most of the internal transformers to new scalar functions and entry scalar functions (based on the built-in +functions), and we still internally use that powerful tool, but we don't expose it to end users, instead, we provide +easy-to-use, covering all user needs functions. + +All available functions can be found in [`ETL\Row\Function` folder](src/core/etl/src/Flow/ETL/Function) or in [ +`ETL\DSL\functions` file](src/core/etl/src/Flow/ETL/DSL/functions.php), and entry scalar functions are defined in +`EntryScalarFunction`. + +Before: + +```php +read(new MemoryExtractor()) + ->rows(Transform::string_concat(['name', 'last name'], ' ', 'name')) +``` + +After: + +```php +read(new MemoryExtractor()) + ->withEntry('name', concat(ref('name'), lit(' '), ref('last name'))) +``` + +### 2) `ref` function nullability + +`ref("entry_name")` is no longer returning null when the entry is not found. Instead, it throws an exception. The same +behavior can be achieved through using a newly introduced `optional` function: + +Before: + +```php +cast('string'); +``` + +After: + +```php +cast('string'); +// or +optional(ref('non_existing_column')->cast('string')); +``` + +### 3) Extractors output + +Affected extractors: + +* CSV +* JSON +* Avro +* DBAL +* GoogleSheet +* Parquet +* Text +* XML + +Extractors are no longer returning data under an array entry called `row`, thanks to this unpacking row become +redundant. + +Because of that all DSL functions are no longer expecting `$entry_row_name` parameter, if it was used anywhere, please +remove it. + +Before: + +```php +read(From::array([['id' => 1, 'array' => ['a' => 1, 'b' => 2, 'c' => 3]]])) + ->withEntry('row', ref('row')->unpack()) + ->renameAll('row.', '') + ->drop('row') + ->withEntry('array', ref('array')->arrayMerge(lit(['d' => 4]))) + ->write(To::memory($memory = new ArrayMemory())) + ->run(); +``` + +After: + +```php +read(From::array([['id' => 1, 'array' => ['a' => 1, 'b' => 2, 'c' => 3]]])) + ->withEntry('array', ref('array')->arrayMerge(lit(['d' => 4]))) + ->write(To::memory($memory = new ArrayMemory())) + ->run(); +``` + +### 4) ConfigBuilder::putInputIntoRows () output is now prefixed with _ (underscore) + +In order to avoid collisions with datasets columns, additional columns created after using putInputIntoRows () +would now be prefixed with `_` (underscore) symbol. + +Before: + +```php +putInputIntoRows())) + ->read(Json::from(__DIR__ . '/../Fixtures/timezones.json', 5)) + ->fetch(); + +foreach ($rows as $row) { + $this->assertSame( + [ + ... + '_input_file_uri', + ], + \array_keys($row->toArray()) + ); +} +``` + +After: + +```php +putInputIntoRows())) + ->read(Json::from(__DIR__ . '/../Fixtures/timezones.json', 5)) + ->fetch(); + +foreach ($rows as $row) { + $this->assertSame( + [ + ... + '_input_file_uri', + ], + \array_keys($row->toArray()) + ); +} +``` diff --git a/documentation/upgrading-0.35-0.40.md b/documentation/upgrading-0.35-0.40.md new file mode 100644 index 0000000000..5f7ce54a62 --- /dev/null +++ b/documentation/upgrading-0.35-0.40.md @@ -0,0 +1,739 @@ +# Upgrade Guide 0.35.x - 0.40.x + +[TOC] + +This document provides guidelines for upgrading between versions of Flow PHP. Please follow the instructions for your +specific version to ensure a smooth upgrade process. + +Upgrading from older versions: [0.16.x to 0.35.x](/documentation/upgrading-0.16-0.35.md) + +--- + +## Upgrading from 0.39.x to 0.40.x + +### 1) `flow-php/postgresql` - column and domain defaults are modeled as `ColumnDefault` + +| Before | After | +|---------------------------------------------------------------|-----------------------------------------------------------------------------| +| `Column::$default` / `Domain::$default` type `?string` | `?Flow\PostgreSql\Schema\ColumnDefault` | +| `new Column('c', $type, true, "'0'")` | `new Column('c', $type, true, ColumnDefault::fromExpression("'0'", $type))` | +| `$column->default` (string) | `$column->default?->literal` / `$column->default?->applicableSql()` | +| `ColumnShape['default']` / `DomainShape['default']` `?string` | `?array{literal: string, type: ?ColumnTypeShape, kind: string}` | + +`Column::create()` / `Domain::create()` still accept `bool|float|int|string|Expression|null`. Schema arrays serialized +by `Column::normalize()` / `Domain::normalize()` before 0.40 must be regenerated - `fromArray()` reads the nested +`default` shape only. + +### 2) `flow-php/telemetry` - Log severity filtering moved to a pipeline middleware + +| Before | After | +|-----------------------------------------------------------------|-----------------------------------------------------------------------------------------| +| `Flow\Telemetry\Logger\Processor\SeverityFilteringLogProcessor` | `Flow\Telemetry\Logger\Middleware\SeverityFilteringLogMiddleware` | +| `new SeverityFilteringLogProcessor($inner, $minSeverity)` | `new PipelineLogProcessor([new SeverityFilteringLogMiddleware($minSeverity)], $inner)` | +| `severity_filtering_log_processor($processor, $minSeverity)` | `pipeline_log_processor([severity_filtering_log_middleware($minSeverity)], $processor)` | + +The previously wrapped processor is now the pipeline's **sink** and must implement +`Flow\Telemetry\Logger\LogSink` (the built-in `batching`, `pass_through`, `memory`, `void` and `composite` +log processors already do). + +Before: + +```php +$processor = severity_filtering_log_processor( + batching_log_processor($exporter), + Severity::WARN, +); +``` + +After: + +```php +$processor = pipeline_log_processor( + [severity_filtering_log_middleware(Severity::WARN)], + batching_log_processor($exporter), +); +``` + +### 2) `flow-php/symfony-telemetry-bundle` - `severity_filtering` log processor type replaced by `pipeline` + +The `severity_filtering` processor type (with its `inner_processor`) is no longer a `logger_provider` +processor type; it is a middleware inside a `pipeline`. + +Before: + +```yaml +flow_telemetry: + logger_provider: + processor: + type: severity_filtering + minimum_severity: warn + inner_processor: + type: batching + exporter: otlp +``` + +After: + +```yaml +flow_telemetry: + logger_provider: + processor: + type: pipeline + middleware: + - { type: severity_filtering, minimum_severity: warn } + sink: + type: batching + exporter: otlp +``` + +### 3) `flow-php/symfony-telemetry-bundle` - named scope `attributes` split into `scope` and `signal` + +Applies to `tracers`, `meters`, and `loggers`. The `attributes` map is no longer a flat list of scope attributes; scope +attributes move under `attributes.scope`. + +Before: + +```yaml +flow_telemetry: + loggers: + audit: + attributes: + team: checkout +``` + +After: + +```yaml +flow_telemetry: + loggers: + audit: + attributes: + scope: + team: checkout +``` + +### 4) `flow-php/postgresql` - `DateTimeConverter` split into `TimestampConverter` and `TimestampTzConverter` + +| Before | After | +|------------------------------------------------------------|-------------------------------------------------------------------------------| +| `Flow\PostgreSql\Client\Types\Converter\DateTimeConverter` | `TimestampConverter` (`TIMESTAMP`) and `TimestampTzConverter` (`TIMESTAMPTZ`) | +| `typed($value, ValueType::TIMESTAMP)` keeps the offset | `typed($value, ValueType::TIMESTAMP)` normalizes the value to UTC | +| `timestamp` column read as `2024-01-15 10:30:00` | `timestamp` column read as `2024-01-15 10:30:00+00:00` | + +### 5) `flow-php/etl-adapter-postgresql` - `DateTimeEntry` maps to `timestamp` instead of `timestamptz` + +| Flow type | Before | After | +|---------------------------|---------------|-------------| +| `DateTimeEntry` (binding) | `TIMESTAMPTZ` | `TIMESTAMP` | +| `DateTimeType` (DDL) | `timestamptz` | `timestamp` | + +To keep the previous behavior, pass overrides to `EntryTypesMap`: + +```php +to_pgsql_table($client, 'users')->withTypesMap(new EntryTypesMap( + [DateTimeEntry::class => ValueType::TIMESTAMPTZ], + [DateTimeType::class => ColumnType::timestamptz()], +)); +``` + +### 6) `flow-php/symfony-postgresql-messenger` - `messenger_messages` time columns use `timestamp` instead of + +`timestamptz` + +| Column type for `created_at`, `available_at`, `delivered_at` | Before | After | +|--------------------------------------------------------------|---------------|-------------| +| `MessengerCatalogProvider` (DDL) | `timestamptz` | `timestamp` | +| `Connection` bindings | `TIMESTAMPTZ` | `TIMESTAMP` | + +Existing tables, realign the column type (UTC instants preserved): + +```sql +ALTER TABLE messenger_messages + ALTER COLUMN created_at TYPE TIMESTAMP USING created_at AT TIME ZONE 'UTC', + ALTER COLUMN available_at TYPE TIMESTAMP USING available_at AT TIME ZONE 'UTC', + ALTER COLUMN delivered_at TYPE TIMESTAMP USING delivered_at AT TIME ZONE 'UTC'; +``` + +### 7) `flow-php/telemetry` - `OTEL_RESOURCE_ATTRIBUTES` keys and values are percent-decoded, not backslash-escaped + +| Escaping a `,` or `=` in `OTEL_RESOURCE_ATTRIBUTES` | Before | After | +|-----------------------------------------------------|---------------------------|-----------------------------| +| literal comma in a value | `key=value\,with\,commas` | `key=value%2Cwith%2Ccommas` | +| literal `=` in a value | not supported | `key=a%3Db` | + +Re-encode any `OTEL_RESOURCE_ATTRIBUTES` that relied on backslash escaping; both keys and values are now +percent-decoded. + +--- + +## Upgrading from 0.37.x to 0.38.x + +### 1) `flow-php/types` - PHPStan extension extracted to `flow-php/phpstan-types-bridge` + +The `StructureTypeReturnTypeExtension` - which narrows the return type of `type_structure()` for PHPStan - has been +moved out of `flow-php/types` into a dedicated package, `flow-php/phpstan-types-bridge`. +`flow-php/types` no longer ships any PHPStan code. + +| Before | After | +|-------------------------------------------------------|--------------------------------------------------------------| +| `Flow\Types\PHPStan\StructureTypeReturnTypeExtension` | `Flow\Bridge\PHPStan\Types\StructureTypeReturnTypeExtension` | +| shipped inside `flow-php/types` | shipped inside `flow-php/phpstan-types-bridge` | + +If you used `type_structure()` together with PHPStan, install the new package: + +``` +composer require --dev flow-php/phpstan-types-bridge +``` + +With [phpstan/extension-installer](https://github.com/phpstan/extension-installer) the extension is registered +automatically. If you registered it manually, update your `phpstan.neon`: + +Before: + +```neon +services: + - + class: Flow\Types\PHPStan\StructureTypeReturnTypeExtension + tags: + - phpstan.broker.dynamicFunctionReturnTypeExtension +``` + +After: + +```neon +includes: + - vendor/flow-php/phpstan-types-bridge/extension.neon +``` + +--- + +## Upgrading from 0.36.x to 0.37.x + +### 1) `flow-php/telemetry` - Per-signal exporter contracts merged into `Exporter` + +| Before | After | +|------------------------------------------------------------------------------------|------------------------------------------------------------------| +| `Flow\Telemetry\Tracer\SpanExporter` (interface) | `Flow\Telemetry\Exporter\Exporter` | +| `Flow\Telemetry\Meter\MetricExporter` (interface) | `Flow\Telemetry\Exporter\Exporter` | +| `Flow\Telemetry\Logger\LogExporter` (interface) | `Flow\Telemetry\Exporter\Exporter` | +| `VoidSpanExporter`, `VoidMetricExporter`, `VoidLogExporter` | `Flow\Telemetry\Provider\Void\VoidExporter` | +| `MemorySpanExporter`, `MemoryMetricExporter`, `MemoryLogExporter` | `Flow\Telemetry\Provider\Memory\MemoryExporter` | +| `ConsoleSpanExporter`, `ConsoleMetricExporter`, `ConsoleLogExporter` | `Flow\Telemetry\Provider\Console\ConsoleExporter` | +| `void_span_exporter()` / `void_metric_exporter()` / `void_log_exporter()` | `void_exporter()` | +| `memory_span_exporter()` / `memory_metric_exporter()` / `memory_log_exporter()` | `memory_exporter()` | +| `console_span_exporter()` / `console_metric_exporter()` / `console_log_exporter()` | `console_exporter()` | +| `MemoryLogExporter::entries()` | `MemoryExporter::logs()` | +| `MemorySpanExporter::spans()` | `MemoryExporter::spans()` | +| `MemoryMetricExporter::metrics()` | `MemoryExporter::metrics()` | +| `Exporter::transports()` | removed | +| `(Span\|Metric\|Log)Exporter::export(array $items) : bool` | `Exporter::export(Flow\Telemetry\Signal\Signals $signal) : bool` | +| - | `Exporter::shutdown() : void` (added) | + +### 2) `flow-php/telemetry` - `Transport` contract relocated to OTLP bridge + +| Before | After | +|-----------------------------------------------------------|-----------------------------------------------------------| +| `Flow\Telemetry\Transport\Transport` | `Flow\Bridge\Telemetry\OTLP\Transport\Transport` | +| `Flow\Telemetry\Transport\TransportException` | `Flow\Bridge\Telemetry\OTLP\Transport\TransportException` | +| `Flow\Telemetry\Transport\VoidTransport` | removed | +| `Transport::sendSpans()` / `sendMetrics()` / `sendLogs()` | `Transport::send(Flow\Telemetry\Signal\Signals $signal)` | +| `Flow\Bridge\Telemetry\OTLP\Exception\Exception` | removed | +| `Flow\Bridge\Telemetry\OTLP\Exception\TransportException` | `Flow\Bridge\Telemetry\OTLP\Transport\TransportException` | + +### 3) `flow-php/telemetry` - Processor interfaces + +Applies to `SpanProcessor`, `MetricProcessor`, `LogProcessor`. + +| Before | After | +|------------------------------------|-----------------------------| +| `exporter() : SpanExporter` (etc.) | removed | +| - | `shutdown() : void` (added) | + +### 4) `flow-php/telemetry` - `Serializer` contract removed + +| Before | After | +|------------------------------------------------------------------------|---------------------------------------------------------------------------------------------------------------| +| `Flow\Telemetry\Serializer\Serializer` | removed | +| `Flow\Bridge\Telemetry\OTLP\Serializer\GrpcSerializer` (interface) | renamed to `Flow\Bridge\Telemetry\OTLP\Serializer\GrpcRequestFactory` (class) | +| `CurlTransport(string $endpoint, Serializer $serializer, ...)` | `CurlTransport(string $endpoint, JsonSerializer\|ProtobufSerializer $serializer = new JsonSerializer(), ...)` | +| `GrpcTransport(string $endpoint, ProtobufSerializer $serializer, ...)` | `GrpcTransport(string $endpoint, ...)` - serializer parameter removed | + +### 5) `flow-php/telemetry` - `ErrorHandler` contract added + +New namespace `Flow\Telemetry\ErrorHandler` with: `ErrorHandler` (interface), `ErrorLogHandler`, `NullErrorHandler`, +`StreamHandler`, `SyslogHandler`, `UdpSyslogHandler`, `CompositeErrorHandler`. + +| Before | After | +|---------------------------------------------------------------------|---------------------------------------------------------------------------------------------------------------| +| `otlp_exporter($transport)` | `otlp_exporter($transport, ErrorHandler $errorHandler = new ErrorLogHandler())` | +| `telemetry_handler($logger, $converter, $level, $bubble)` (Monolog) | `telemetry_handler($logger, $converter, $level, $bubble, ErrorHandler $errorHandler = new ErrorLogHandler())` | + +### 6) `flow-php/telemetry-otlp-bridge` - `HttpTransport` removed + +| Before | After | +|------------------------------------------------------|---------------------------------------| +| `Flow\Bridge\Telemetry\OTLP\Transport\HttpTransport` | removed | +| `otlp_http_transport()` | removed (use `otlp_curl_transport()`) | +| `psr/http-client` (runtime require) | removed (dev only) | +| `psr/http-factory` (runtime require) | removed (dev only) | + +### 7) `flow-php/telemetry-otlp-bridge` - Per-signal OTLP exporters merged + +| Before | After | +|-------------------------------------------------------------|----------------------------------------------------| +| `OTLPSpanExporter`, `OTLPMetricExporter`, `OTLPLogExporter` | `Flow\Bridge\Telemetry\OTLP\Exporter\OTLPExporter` | +| `otlp_span_exporter($transport)` | `otlp_exporter($transport)` | +| `otlp_metric_exporter($transport)` | `otlp_exporter($transport)` | +| `otlp_log_exporter($transport)` | `otlp_exporter($transport)` | + +### 8) `flow-php/telemetry-otlp-bridge` - Curl/gRPC timeouts switched to milliseconds + +| Before | After | +|---------------------------------------------------------------------------------------------------------------------|---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| `CurlTransportOptions::withTimeout(int $seconds)`, default `30` | `CurlTransportOptions::withTimeout(int $milliseconds)`, default `250` | +| `CurlTransportOptions::withConnectTimeout(int $seconds)`, default `10` | `CurlTransportOptions::withConnectTimeout(int $milliseconds)`, default `250` | +| `CurlTransportOptions::timeout()` | `CurlTransportOptions::timeoutMs()` | +| `CurlTransportOptions::connectTimeout()` | `CurlTransportOptions::connectTimeoutMs()` | +| - | `CurlTransportOptions::withShutdownTimeout(int $milliseconds)` / `shutdownTimeoutMs()`, default `5000` (added) | +| `otlp_curl_transport(string $endpoint, Serializer $serializer, CurlTransportOptions $options)` | `otlp_curl_transport(string $endpoint, JsonSerializer\|ProtobufSerializer $serializer = new JsonSerializer(), CurlTransportOptions $options = new CurlTransportOptions(), ?Transport $failover = null)` | +| `otlp_grpc_transport(string $endpoint, ProtobufSerializer $serializer, array $headers = [], bool $insecure = true)` | `otlp_grpc_transport(string $endpoint, array $headers = [], bool $insecure = true, int $timeoutMs = 250, int $shutdownTimeoutMs = 5000, ?Transport $failover = null)` | + +### 9) `flow-php/telemetry-otlp-bridge` - `StreamTransport` added + +New transport, no removal counterpart. + +| Before | After | +|--------|-----------------------------------------------------------------------------------------------------------| +| - | `Flow\Bridge\Telemetry\OTLP\Transport\StreamTransport` | +| - | `otlp_stream_transport(string $destination, int $filePermissions = 0644, bool $createDirectories = true)` | + +### 10) `flow-php/telemetry-otlp-bridge` - `open-telemetry/gen-otlp-protobuf` dependency dropped + +| Before | After | +|---------------------------------------------------|--------------------------------------------------------------------------------------------------------| +| `open-telemetry/gen-otlp-protobuf` (required dep) | removed; protobuf classes shipped inside the bridge under the same `Opentelemetry\Proto\...` namespace | + +### 11) `flow-php/symfony-telemetry-bundle` - Configuration schema rewrite + +No BC shim. Configurations from 0.36 must be rewritten. + +| Before | After | +|-------------------------------------------------------------------------------------------------|----------------------------------------------------------------------------------------| +| `exporters..type: otlp\|service\|console\|memory\|void` | sub-block keyed by implementation: `otlp:`, `service:`, `console:`, `memory:`, `void:` | +| `exporters..service_id: ` (under `type: service`) | `exporters..service: { id: }` | +| Inline `processor.exporter: { type: otlp, transport: {...} }` | `processor.exporter: ` referencing top-level `exporters:` map | +| `transport.type: http` | removed | +| `transport.timeout` (seconds, default `30`) | `transport.timeout_ms` (default `250`) | +| `transport.connect_timeout` (seconds, default `10`) | `transport.connect_timeout_ms` (default `250`) | +| `transport.http_client_service_id` / `request_factory_service_id` / `stream_factory_service_id` | removed | +| - | `transport.type: stream` (added) | +| - | `transport.shutdown_timeout_ms` default `5000` (added) | +| - | `transport.failover: { type: ..., ... }` on curl/grpc primaries (added) | +| - | top-level `error_handlers:` map (added) | +| - | `error_handler: ` references on providers, processors, otlp exporter (added) | +| - | top-level `framework_logger: ` (added) | + +Before: + +```yaml +flow_telemetry: + exporters: + otlp: + type: otlp + transport: + type: curl + endpoint: 'http://otel-collector:4318' + timeout: 30 + connect_timeout: 10 + custom: { type: service, service_id: 'app.x' } + debug: { type: console } + + tracer_provider: + processor: + type: batching + exporter: { type: otlp, transport: { type: curl, endpoint: 'http://otel-collector:4318' } } +``` + +After: + +```yaml +flow_telemetry: + error_handlers: + default: { type: error_log } + + exporters: + otlp: + otlp: + transport: + type: curl + endpoint: 'http://otel-collector:4318' + timeout_ms: 250 + connect_timeout_ms: 250 + custom: { service: { id: 'app.x' } } + debug: { console: ~ } + + tracer_provider: + processor: + type: batching + exporter: otlp +``` + +### 12) `flow-php/phpunit-telemetry-bridge` - Configuration parameters + +| Before (parameter / default) | After (parameter / default) | +|--------------------------------------------------|----------------------------------------------------------------------------------| +| `curl_timeout` / `30` (seconds) | `curl_timeout_ms` / `250` (ms) | +| `curl_connect_timeout` / `10` (s) | `curl_connect_timeout_ms` / `250` (ms) | +| `transport: curl\|grpc` | `transport: curl\|grpc\|stream` | +| - | `grpc_timeout_ms` / `250` (added) | +| - | `shutdown_timeout_ms` / `5000` (added) | +| - | `batch_size` / `512` (added) | +| - | `error_handler` / `error_log` and `error_handler_*` family (added) | +| - | `stream_file_permissions` / `0644`, `stream_create_directories` / `true` (added) | +| Default span/metric/log processors: pass-through | Default span/metric/log processors: batching (`batch_size: 512`) | + +`otel_collector_url` / `FLOW_PHPUNIT_OTEL_COLLECTOR_URL` remain deprecated aliases for `endpoint` with +`transport=curl`. + +--- + +## Upgrading from 0.35.x to 0.36.x + +### 1) `flow-php/postgresql` - `RawCondition` and `RawExpression` removed + +The `raw_cond()` and `raw_expr()` escape hatches have been removed. All query builder operations are now covered by +type-safe DSL functions. + +| Removed | Replacement | +|---------------------------------------|---------------------------------------------------------------------------| +| `raw_cond('NOT col')` | `not_(is_true(col('col')))` | +| `raw_cond('a = ANY(b)')` | `any_(col('a'), ComparisonOperator::EQ, col('b'))` | +| `raw_cond("x IN ('a', 'b')")` | `in_(col('x'), [literal('a'), literal('b')])` | +| `raw_cond("x NOT LIKE 'pg_%'")` | `not_like(col('x'), literal('pg_%'))` | +| `raw_expr('NOT col')` | `not_(col('col'))` | +| `raw_expr('a \|\| b')` | `concat(col('a'), col('b'))` or `binary_expr(col('a'), '\|\|', col('b'))` | +| `raw_expr('CASE x WHEN ...')` | `case_when([when(...)], operand: col('x'))` | +| `raw_expr('array_agg(DISTINCT ...)')` | `agg('array_agg', [...], distinct: true)->withOrderBy(...)` | +| `RawCondition` class | Use specific condition classes | +| `RawExpression` class | Use specific expression classes | + +### 2) `flow-php/postgresql` - `Condition` now extends `Expression` + +Conditions are now expressions - they can be used in SELECT lists, CASE WHEN, ORDER BY, etc. + +```php +// Conditions can now be aliased and used as expressions: +select(eq(col('a'), col('b'))->as('is_equal')); + +// NOT works in both WHERE and SELECT: +not_(col('is_deleted'))->as('is_active'); + +// CASE WHEN accepts conditions directly: +case_when([when(eq(col('x'), literal(0)), literal('zero'))]); +``` + +### 3) `flow-php/postgresql` - DSL condition function renames + +Function names have been unified following standard SQL builder conventions (jOOQ, Diesel, SQLAlchemy). + +| Removed | Replacement | Reason | +|----------------------|---------------------------|-----------------------------------------------------------| +| `neq()` | `ne()` | Standard short form | +| `lte()` | `le()` | Standard short form | +| `gte()` | `ge()` | Standard short form | +| `is_in()` | `in_()` | Drop `is_` prefix, trailing underscore for PHP keyword | +| `is_distinct_from()` | `distinct_from()` | Drop `is_` prefix | +| `cond_and()` | `and_()` | Drop `cond_` prefix, trailing underscore for PHP keyword | +| `cond_or()` | `or_()` | Drop `cond_` prefix, trailing underscore for PHP keyword | +| `cond_not()` | `not_()` | Drop `cond_` prefix, trailing underscore for PHP keyword | +| `any_sub_select()` | `any_()` | Unified - accepts both `Expression` and `SelectFinalStep` | +| `all_sub_select()` | `all_()` | Unified - accepts both `Expression` and `SelectFinalStep` | +| `cond_true()` | `is_true(literal(true))` | Use `is_true()` with literal | +| `cond_false()` | `is_true(literal(false))` | Use `is_true()` with literal | +| `bool_cond()` | `is_true()` | Wraps expression as boolean condition | +| `any_array()` | `any_()` | Merged into unified `any_()` | +| `all_array()` | `all_()` | Merged into unified `all_()` | + +New functions added: + +| Function | Purpose | +|------------------------------------|------------------------------------------------| +| `is_true(Expression)` | Wrap expression as boolean condition for WHERE | +| `not_like(Expression, Expression)` | NOT LIKE condition | +| `concat(Expression, ...)` | String concatenation with `\|\|` operator | + +Before: + +```php +use function Flow\PostgreSql\DSL\{cond_and, cond_not, cond_true, neq, lte, gte, is_in, any_sub_select}; + +select(col('name')) + ->where(cond_and( + neq(col('status'), literal('deleted')), + lte(col('age'), literal(65)), + gte(col('age'), literal(18)), + is_in(col('role'), [literal('admin'), literal('user')]), + )); +``` + +After: + +```php +use function Flow\PostgreSql\DSL\{and_, ne, le, ge, in_}; + +select(col('name')) + ->where(and_( + ne(col('status'), literal('deleted')), + le(col('age'), literal(65)), + ge(col('age'), literal(18)), + in_(col('role'), [literal('admin'), literal('user')]), + )); +``` + +### 4) `flow-php/postgresql` - Schema builder methods accept `Expression`/`Condition` instead of strings + +Methods that previously accepted raw SQL strings now require typed `Expression` or `Condition` objects. + +| Method | Before (string) | After (typed) | +|----------------------------------------------|-------------------------------------------|---------------------------------------------------------| +| `ColumnDefinition::check()` | `->check('age > 0')` | `->check(gt(col('age'), literal(0)))` | +| `ColumnDefinition::defaultRaw()` | `->defaultRaw('CURRENT_TIMESTAMP')` | `->defaultRaw(current_timestamp())` | +| `ColumnDefinition::generatedAs()` | `->generatedAs("a \|\| b")` | `->generatedAs(concat(col('a'), col('b')))` | +| `CheckConstraint::create()` | `::create('age > 0')` | `::create(gt(col('age'), literal(0)))` | +| `ExcludeConstraint::element()` | `->element('col', '=')` | `->element(col('col'), '=')` | +| `ExcludeConstraint::where()` | `->where('active = true')` | `->where(eq(col('active'), literal(true)))` | +| `CreateDomainBuilder::check()` | `->check('VALUE > 0')` | `->check(gt(col('VALUE'), literal(0)))` | +| `CreateDomainBuilder::default()` | `->default("'text'")` | `->default(literal('text'))` | +| `AlterDomainBuilder::addConstraint()` | `->addConstraint('name', 'VALUE > 0')` | `->addConstraint('name', gt(col('VALUE'), literal(0)))` | +| `AlterDomainBuilder::setDefault()` | `->setDefault('100')` | `->setDefault(literal(100))` | +| `AlterTableBuilder::alterColumnSetDefault()` | `->alterColumnSetDefault('col', "'val'")` | `->alterColumnSetDefault('col', literal('val'))` | +| `CreateRuleBuilder::where()` | `->where("OLD.role = 'admin'")` | `->where(eq(col('role', 'OLD'), literal('admin')))` | + +### 5) `flow-php/postgresql` - DSL functions split into separate files + +The monolithic `functions.php` has been split into 5 focused files (same namespace, no import changes needed): + +| File | Purpose | +|-----------------|--------------------------------------------------------------------------------------------| +| `query.php` | Query building, expressions, tables, ordering, CTE, window, locking, transactions, cursors | +| `condition.php` | Comparisons, predicates, logic, JSON/array/regex operators | +| `schema.php` | DDL, constraints, indexes, maintenance, privileges, types, schema definitions | +| `client.php` | Connections, telemetry, mappers | +| `parser.php` | SQL parsing, formatting, analysis | + +### 6) `flow-php/filesystem` - `Protocol` and `Backend` removed, `Mount` rewritten + +The filesystem library has been redesigned around a single mount-protocol string. The `Protocol` and +`Backend` value types are gone; `Mount` now wraps just a protocol name. + +**`Protocol` class removed.** `Path::protocol()` now returns `string` instead of a `Protocol` object. Callsites that +unpacked `Protocol::$name` / `Protocol::scheme()` / `Protocol::is()` are mechanical updates: + +| Before | After | +|------------------------------------------|----------------------------------------------------------------------------------------------------| +| `$path->protocol()->name` | `$path->protocol()` | +| `$path->protocol()->scheme()` | `$path->protocol() . '://'` | +| `$path->protocol()->is('file')` | `$path->protocol() === 'file'` | +| `$fs->protocol()->validateScheme($path)` | `$fs->mount()->supports($path) \|\| throw new InvalidSchemeException(...)` | +| `new Protocol('file')` | `new Mount('file')` (if you need a Mount) or plain `'file'` (FilesystemTable::for accepts strings) | + +**`Backend` enum removed.** There's no closed set of backends anymore - any filesystem can mount under any protocol. The +Symfony bundle schema now uses a plain string `type:` field (see below). If you branched on `Backend` cases in +application code, replace with string comparisons against the factory `type()` or the mount protocol, whichever fits. + +**`Mount` rewritten.** The shape is now: + +```php +final readonly class Mount +{ + public string $protocol; + + public function __construct(string $protocol); // validates against PROTOCOL_REGEX + public function supports(Path|string $path) : bool; +} +``` + +**`Filesystem::protocol()` renamed to `Filesystem::mount()`.** The return type changed from +`Protocol` to `Mount`. Every `Filesystem` implementation must rename the method. + +**`Filesystem` ctors take `Mount` directly.** `NativeLocalFilesystem`, `MemoryFilesystem`, +`StdOutFilesystem`, `AsyncAWSS3Filesystem`, `AzureBlobFilesystem` now accept `Mount` as the first constructor argument +(local filesystems have a sensible default). DSL factory functions (`native_local_filesystem`, `memory_filesystem`, +`stdout_filesystem`, `aws_s3_filesystem`, +`azure_filesystem`) accept `string $protocol` as the **last** argument with a sensible default (`'file'`, `'memory'`, +`'stdout'`, `'aws-s3'`, `'azure-blob'`) and build the `Mount` internally - no caller change needed unless you +instantiate the filesystem class directly or mount two filesystems of the same backend under distinct protocols. + +**Auto-alias dropped.** Previously, mounting a single filesystem of a given backend would auto-register its canonical +scheme as an additional alias (e.g. mounting S3 as `warehouse` also made `aws-s3` +available). That behavior is gone - every mount is registered under exactly the protocol you pick. If you need two +protocols for the same filesystem, mount it twice explicitly. + +**`FilesystemTable::for(Path|Protocol)` → `for(Path|string)`.** Pass a `Path` or a plain protocol string. + +### 7) `flow-php/filesystem` - `NativeLocalFilesystem::list()` no longer sorts results + +`Glob::glob()` was replaced with lazy `Webmozart\Glob\Iterator\GlobIterator` to avoid materializing the entire matching +set up front (this gives a ~30× speedup on large trees when the caller only needs the first N entries). + +**Side effect:** `NativeLocalFilesystem::list()` no longer returns results in alphabetical order. Output now follows +filesystem traversal order. If your code depends on sort order, sort client-side after consuming the generator: + +```php +$statuses = iterator_to_array($fs->list(path('/some/dir/**/*.txt'))); +usort($statuses, static fn (FileStatus $a, FileStatus $b) => $a->path->uri() <=> $b->path->uri()); +``` + +### 8) `flow-php/symfony-filesystem-bundle` - YAML schema now uses `type:` + protocol-as-key + +The configuration schema changed significantly. The YAML key under `filesystems:` is now the **mount protocol** (any +valid URI scheme), and a separate `type:` field picks the factory. + +**Before:** + +```yaml +flow_filesystem: + fstabs: + default: + filesystems: + file: ~ + memory: ~ + aws-s3: + bucket: '%env(S3_BUCKET)%' +``` + +**After:** + +```yaml +flow_filesystem: + fstabs: + default: + filesystems: + file: + type: file + memory: + type: memory + aws-s3: # mount protocol - can be any valid URI scheme + type: aws_s3 # factory lookup key + bucket: '%env(S3_BUCKET)%' +``` + +Benefits of the new shape: + +- Mount the same backend twice under different protocols (e.g. `warehouse` + `archive` both `type: aws_s3` with + different buckets). +- Protocol names are no longer tied to factory names - pick whatever reads well in your application. + +Built-in `type` values: `file`, `memory`, `stdout`, `aws_s3`, `azure_blob`. + +### 9) `flow-php/symfony-filesystem-bundle` - `FilesystemFactory` interface and attribute changed + +```php +// Before +interface FilesystemFactory +{ + public function protocol() : Protocol; + public function create(string $mountName, array $config) : Filesystem; +} + +#[AsFilesystemFactory(protocol: 'my-fs')] + +// After +interface FilesystemFactory +{ + public function type() : string; + public function create(string $protocol, array $config) : Filesystem; +} + +#[AsFilesystemFactory(type: 'my_backend')] +``` + +The DI tag attribute is renamed from `protocol` to `type`. `FilesystemFactoryRegistry::get()` takes a +`string $type` instead of a `Backend`. + +### 10) `flow-php/symfony-filesystem-bundle` - `flow:filesystem:ls` CLI flags reshuffled + +| Before | After | +|------------------------------|--------------------------------------------------------------| +| `--long` (default: off) | 4-column output is now the default; use `--short` to drop it | +| `--no-limit` | Removed - default is unlimited now. Use `--limit=N` to cap. | +| (no `--page-size`) | New `--page-size=N` (default `10`) controls table page size | +| (no `--offset`) | New `--offset=N` skips the first N entries | +| `--format=json` → JSON array | `--format=json` now emits NDJSON (one JSON object per line) | + +Default behavior: list all entries, paginated in tables of 10 rows; interactive terminals prompt between pages (Enter +continues, "no" stops), piped output flows continuously. Size is formatted with binary units, Modified as ISO-8601 - +both read from the backend listing response, no per-file HEAD. + +### 11) `flow-php/symfony-filesystem-bundle` - `flow:filesystem:stat` rejects pattern paths + +`stat` now returns `Command::FAILURE` with a clear error when given a pattern path (`memory://*.txt`, +`**/*.parquet`, ...). Previously it returned metadata for the first match - confusing semantics. Use +`flow:filesystem:ls` for pattern inspection. + +### 12) `flow-php/filesystem-async-aws-bridge`, + +`flow-php/filesystem-azure-bridge` - DSL protocol is the last argument with a default + +The DSL factories expose the mount protocol as an optional last argument, defaulted to the conventional scheme. Common +cases work without passing it: + +```php +aws_s3_filesystem($bucket, $client); // mounts under 'aws-s3' +azure_filesystem($blobService); // mounts under 'azure-blob' + +// Pick a different protocol - e.g. mount the same bucket twice +aws_s3_filesystem($bucket, $client, protocol: 'warehouse'); +azure_filesystem($blobService, protocol: 'archive'); +``` + +### 13) `flow-php/filesystem` - `path_memory()` and `path_stdout()` DSL helpers removed + +Build `Path` directly instead: + +```php +// Before +$mem = path_memory(); +$out = path_stdout(['stream' => 'output']); + +// After +$mem = path('memory://' . bin2hex(random_bytes(16)) . '.memory'); +$out = path('stdout://' . bin2hex(random_bytes(16)) . '.stdout', ['stream' => 'output']); +``` + +### 14) `flow-php/etl` - `ConfigBuilder::cacheFilesystem()` and `externalSortFilesystem()` added + +Point cache and external-sort mechanisms at any mounted protocol; defaults remain `'file'`. The +`CacheConfig` and `SortConfig` value objects expose the chosen protocol as `->filesystemProtocol`. + +```php +$config = config_builder() + ->mount(aws_s3_filesystem($bucket, $client, protocol: 'sort-scratch')) + ->externalSortFilesystem('sort-scratch') + ->build(); +``` + +### 15) `flow-php/symfony-http-foundation-bridge` - `Output` interface collapsed to a single `loader(Path)` + +`Output::memoryLoader(string $id)` and `Output::stdoutLoader()` were replaced by +`Output::loader(Path $path)`. `FlowBufferedResponse` gained a `string $filesystem = 'memory'` +constructor argument (buffer protocol); `FlowStreamedResponse` gained +`string $stdoutFilesystemProtocol = 'stdout'`. Each response builds the path with its configured protocol and passes it +to the Output. + +```php +// Before +new FlowBufferedResponse($extractor, new CsvOutput(), $transformations); + +// After - same defaults, new constructor param available +new FlowBufferedResponse($extractor, new CsvOutput(), $transformations, filesystem: 'memory'); +``` + +### 16) `flow-php/filesystem` - `StdOutFilesystem` tracks open streams per php:// target + +Previously the "only one stdout stream" guard lived in `FilesystemStreams` (ETL core) and fired when two writing streams +used the `stdout://` protocol. The check now lives in `StdOutFilesystem` itself and is precise per underlying php:// +target (`stdout` / `stderr` / `output`): two streams with +`['stream' => 'stdout']` conflict; one stdout stream + one stderr stream do not. Error message changed from *"Only one +stdout filesystem stream can be open at the same time"* to *"Only one stream can be open at the same time for php: +//{target}"*. diff --git a/documentation/upgrading-0.8-0.16.md b/documentation/upgrading-0.8-0.16.md new file mode 100644 index 0000000000..ce8d4ff24c --- /dev/null +++ b/documentation/upgrading-0.8-0.16.md @@ -0,0 +1,155 @@ +# Upgrade Guide 0.8.x - 0.16.x + +[TOC] + +This document provides guidelines for upgrading between versions of Flow PHP. Please follow the instructions for your +specific version to ensure a smooth upgrade process. + +Upgrading from older versions: [0.3.x to 0.8.x](/documentation/upgrading-0.3-0.8.md) + +--- + +## Upgrading from 0.15.x to 0.16.x + +### 1) Deprecated `Flow\ETL\DataFrame::renameAll*` methods + +Methods: + +- `Flow\ETL\DataFrame::renameAll()`, +- `Flow\ETL\DataFrame::renameAllLowerCase()`, +- `Flow\ETL\DataFrame::renameAllUpperCase()`, +- `Flow\ETL\DataFrame::renameAllUpperCaseFirst()`, +- `Flow\ETL\DataFrame::renameAllUpperCaseWord()`, + +Were deprecated in favor of using new method: `DataFrame::renameEach()` with proper `RenameEntryStrategy` object. + +### 2) Deprecated `RenameAllCaseTransformer` & `RenameStrReplaceAllEntriesTransformer` + +Selected transformers were deprecated in favor of using `DataFrame::renameEach()` with related `RenameEntryStrategy`: + +- `RenameAllCaseTransformer` -> `RenameCaseTransformer`, +- `RenameStrReplaceAllEntriesTransformer` -> `RenameReplaceStrategy`, + +--- + +## Upgrading from 0.14.x to 0.15.x + +### 1) Removed `Flow\ETL\Row\Schema\Matcher` and implementations + +Schema Matcher was the initial attempt to implement a schema evolution next to schema validation that over time got +replaced with a different implementation of Schema Validator. + +### 2) Renamed `Flow\ETL\Row\Schema` namespace into `Flow\ETL\Schema`. + +This means all classes related to Schema now live under `Flow\ETL\Schema` namespace. + +--- + +## Upgrading from 0.11.x to 0.14.x + +### 1) Replaced `Flow\ETL\DataFrame::validate()` with `Flow\ETL\DataFrame::match()` + +The old method is now deprecated and will be removed in the next release. + +### 2) Replaced `Flow\ETL\Function\ScalarFunction\TypedScalarFunction` with + +`Flow\ETL\Function\ScalarFunction\ScalarResult`. + +The old interface was used to allow defining the return type of the ScalarFunctions. It was replaced with a ScalarResult +value object that is much more flexible than the interface, because it's allowing to return any type dynamically without +making the scalar function stateful. + +--- + +## Upgrading from 0.10.x to 0.11.x + +### 1) Removed StructureElement/struct_element/structure_element from StructureType Definition + +Before: + +```php +type_structure([ + struct_element('name', string()), + struct_element('age', integer()), +]); +``` + +After: + +```php +type_structure([ + 'name' => string(), + 'age' => integer(), +]); +``` + +### 2) Doctrine DBAL Adapter + +From now options for: + +- `to_dbal_table_insert()` +- `to_db_table_update()` + +are passed as objects (instance of UpdateOptions|InsertOptions interfaces) and they are platform specific, so please use +the proper class for the platform you are using. + +- PostgreSQL + - PostgreSQLInsertOptions + - PostgreSQLUpdateOptions +- MySQL + - MySQLInsertOptions + - MySQLUpdateOptions +- Sqlite + - SQLiteInsertOptions + - SQLiteUpdateOptions + +--- + +## Upgrading from 0.8.x to 0.10.x + +### 1) Providing multiple paths to a single extractor + +From now to read from multiple locations use `from_all(Extractor ...$extractors) : Exctractor` extractor. + +Before: + +```php +withSchema($schema); +``` diff --git a/documentation/upgrading.md b/documentation/upgrading.md index 3a48fcf309..135df3c1c5 100644 --- a/documentation/upgrading.md +++ b/documentation/upgrading.md @@ -5,6 +5,8 @@ This document provides guidelines for upgrading between versions of Flow PHP. Please follow the instructions for your specific version to ensure a smooth upgrade process. +Upgrading from older versions: [0.35.x to 0.40.x](/documentation/upgrading-0.35-0.40.md) + --- ## Upgrading from 0.44.x to 0.45.x @@ -55,9 +57,9 @@ Applies to `PaginationModifier`, `CountModifier` and `KeysetPaginationModifier` ### 6) `flow-php/etl-adapter-postgresql` - `from_pgsql_limit_offset()` requires the query's own `ORDER BY` -| Query | Before | After | -|--------------------------------------------------|-----------------------------|----------------------------| -| `SELECT * FROM (SELECT id FROM t ORDER BY id) s` | pages in no defined order | `InvalidArgumentException` | +| Query | Before | After | +|--------------------------------------------------|---------------------------|----------------------------| +| `SELECT * FROM (SELECT id FROM t ORDER BY id) s` | pages in no defined order | `InvalidArgumentException` | ### 7) `flow-php/etl` - `Pipeline\Optimizer` replaced by `Optimizer` + `Planner`, the `Pipeline` class removed @@ -228,15 +230,17 @@ after it shift by one. `Stage::physical` is new - see the core documentation. | `a,b` / `"x\"y",1` / `"p",2` | 1 row: `{"a":"x\\\"y","b":"1\n\"p\""}` | 2 rows: `{"a":"x\\\"y","b":1}`, `{"a":"p","b":2}` | | `a,b` / `x"y,1` / `"p",2` | 1 row: `{"a":"x\"y","b":"1\n\"p\""}` | 2 rows: `{"a":"x\"y","b":1}`, `{"a":"p","b":2}` | -### 23) `flow-php/etl-adapter-csv` - `CSVOpenSource` is an interface, `CSVLineReader` takes the separator and escape +### 23) `flow-php/etl-adapter-csv` - `CSVOpenSource` is an interface, `CSVSourceOpener` removed, `CSVLineReader` takes the separator and escape -| Before | After | -|----------------------------------------------------------------------------|------------------------------------------------------------------------------------------------| -| `new CSVLineReader($enclosure, $charactersReadInLine, $removeBOM)` | `new CSVLineReader($enclosure, $separator, $escape, $charactersReadInLine, $removeBOM)` | -| `new CSVOpenSource($stream, $dialect, $encoder, $lineReader)` | `new PhpCSVOpenSource($stream, $encoder, $lineReader)`; `CSVOpenSource` is its interface | -| `$open->stream`, `$open->dialect`, `$open->encoder`, `$open->lineReader` | removed | -| `CSVFileReader::samples()` yields `Generator`s | yields `CSVFileSample` (`IteratorAggregate`); `$unit->getIterator()` for the generator | -| `CSVFileReader::sample($source)` | `new CSVFileSample($opener, $source)` | +| Before | After | +|--------------------------------------------------------------------------------------------|---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| `new CSVLineReader($enclosure, $charactersReadInLine, $removeBOM)` | `new CSVLineReader($enclosure, $separator, $escape, $charactersReadInLine, $removeBOM)` | +| `final class CSVOpenSource`, `new CSVOpenSource($stream, $dialect, $encoder, $lineReader)` | `interface CSVOpenSource`, implemented by `PhpCSVOpenSource` (`new PhpCSVOpenSource($stream, $decoder, $lineReader)`, a `CSVDecoder`), `RustCSVOpenSource` (flow_php) and `AdaptiveCSVOpenSource` | +| `$open->stream`, `$open->dialect`, `$open->encoder`, `$open->lineReader` | removed | +| `(new CSVSourceOpener($filesystem, $options))->open($source)` | `new AdaptiveCSVOpenSource($filesystem, $source, $options)` | +| `new CSVFileReader($opener, $sources)` | `new CSVFileReader($filesystem, $options, $sources)` | +| `CSVFileReader::samples()` yields `Generator`s | yields `CSVFileSample` (`IteratorAggregate`); `$unit->getIterator()` for the generator | +| `CSVFileReader::sample($source)` | `new CSVFileSample($filesystem, $options, $source)` | ### 24) `flow-php/etl` - Floe footer carries a statistics block, existing `.floe` files must be rewritten @@ -358,10 +362,10 @@ final class MyExtractor implements Extractor ### 35) `flow-php/etl` - `AdaptiveFloeEncoder` removed, Floe engines build a `FloeEncoder` -| Before | After | -|------------------------------------------|--------------------------------------------------------------------------------------------------------------------------------------| -| `Flow\Floe\AdaptiveFloeEncoder` | removed - `FloeEngine::adaptive->encoder($schema)` returns `NativeFloeEncoder` when the extension supports it, else `PhpFloeEncoder` | -| `FloeEngine::encoder(): Encoder` | `FloeEngine::encoder(): Flow\Floe\FloeEncoder` (`Encoder` plus `decodeRows()` / `encodeFrames()`) | +| Before | After | +|------------------------------------------|-----------------------------------------------------------------------------------------------------------| +| `Flow\Floe\AdaptiveFloeEncoder` | removed | +| `FloeEngine::encoder(): Encoder` | `FloeEngine::encoder(): Flow\Floe\FloeEncoder` (`Encoder` plus `decodeRows()` / `encodeFrames()`) | ### 36) `flow-php/etl` - `UnionDefinition` and `union_schema()` removed @@ -417,7 +421,7 @@ The same applies to map values and structure elements. | `type_time()->cast(new DateTimeImmutable('10:00+05:00'))` = `05:00:00` | `10:00:00` - the value's wall clock | | Excel date cells in the host zone | UTC wall clock | | `new DateTimeDefinition('at', true)` | `new DateTimeDefinition('at', type_datetime(), true)` or `datetime_schema('at', nullable: true, zone: 'Europe/Warsaw')` | -| a `flow_php` extension older than this release | ignored - the PHP engine runs; a current extension with an older `flow-php/etl` keeps that library's behaviour | +| a `flow_php` extension older than this release | Composer refuses it (`conflict`) - see 67) | ### 41) `flow-php/parquet` - `Converter::isFor()` replaced by `static Converter::forColumn()`, `Int32DateTimeConverter` removed @@ -484,6 +488,939 @@ A value with a key its structure does not declare is invalid. Declare every elem `type_json()` when the keys are dynamic. flow-php 0.44.x cannot read a Floe file, cached schema or `schema_to_json()` document written by 0.45.x that has a structure column. Clear APCu and natively serialized caches after upgrading. +### 49) `flow-php/etl` - `Rows` is a columnar batch, `Row` is a view + +| Before | After | +|------------------------------------------------------------------------------------------------------------|------------------------------------------------------------------------------------------------------------------------------| +| `new Rows($schema, ...$rows)` | `Rows::of($schema, ...$rows)` / `rows($schema, ...$rows)` | +| `Rows::trusted($schema, $rows)` / `Rows::conformed($schema, $rows)` | `Rows::of($schema, ...$rows)` (views) / `Rows::fromColumns($schema, $columns, $count)` (columns) | +| `new Row(array $values)` | `array_to_row($values, $schema)` | +| - | `new Row(Rows $rows, int $index)` - a view over row `$index` of `$rows` | +| `Row::matchTo()` / `Row::conformTo()` / `Row::project()` | removed - `Rows::matchTo()` / `Rows::project()` | +| `Rows::empty(): bool` | `Rows::isEmpty(): bool` | +| `Row::get()` of a datetime / date / time / uuid / json / xml / html cell - the instance handed in | a new object per call - compare with `assertEquals`, not `assertSame` | +| a `DOMElement` / `HTMLElement` cell keeps its document (parent and siblings reachable) | the cell is the element's own markup - navigate from the document column (`xpath()`) | +| an optional and nullable structure element given `null` | reads back absent (the element's validity bit) | +| `$row == $other` - compares the rows' values | compares both views' whole batches and indices - compare `toArray()` | +| `serialize(Rows)` payload `{schema, rows}` | `{schema, frame}` - the Floe BATCH body; payloads of 0.44.x cannot be unserialized, a corrupt payload throws `FloeException` | +| `Rows::drop()` / `dropRight()` / `take()` / `takeRight()` with a negative size - `array_slice()` semantics | `InvalidArgumentException` | + +### 50) `flow-php/etl` - `row()`, `InferredBatch` and `DataFrame::getEach()` removed + +| Before | After | +|-----------------------------------|--------------------------------------------------------------------------------| +| `row($values)` | `array_to_row($values, $schema)` | +| `rows($schema, row($a), row($b))` | `array_to_rows([$a, $b], $schema)` | +| `Flow\ETL\Row\InferredBatch` | removed - `array_to_rows($data, $schema)`, or `from_array($data)` to infer one | +| `DataFrame::getEach()` | `getEachAsArray()` (values) / `get()` (batches with their schema) | + +`array_to_row()` / `array_to_rows()` cast values to the declared type and refuse keys the schema does not declare +(`SchemaMismatchException` naming the row and the column; before, those keys were silently dropped). + +### 51) `flow-php/etl` - date and time columns store days and microseconds + +| Before | After | +|-----------------------------------------------------------------------------------------|----------------------------------------------------------------------| +| a `date` cell - the value as given | midnight UTC (zone `UTC`) of its calendar day | +| a `time` cell - `d` kept | `d` folded into hours, `$days` is `false` | +| a `time` string with months / years (`'P1M'`) - accepted by the hydrate path | `SchemaMismatchException` "could not convert 'P1M' (string) to time" | +| a `datetime` / `time` beyond ±9 223 372 036 854.775807 s (about year ±292 000) - stored | refused | +| a `date` beyond the `i32` day range (about year ±5 880 000) - stored | refused | +| a PHP 8.4 `\Dom\Element` in an `xml_element` column - kept | read back as a `DOMElement` | + +### 52) `flow-php/etl` - batch columns refuse wildcard enums, `mixed` elements and foreign definitions + +| Before | After | +|--------------------------------------------------------------------|------------------------------------------------------| +| `enum_schema('x', UnitEnum::class)` / `BackedEnum::class` - stored | `ColumnMismatchException` "cannot be a batch column" | +| `list` / `map` / `structure{a: mixed}` - stored | `ColumnMismatchException` "cannot be a batch column" | +| a `Definition` class outside the 19 of `flow-php/etl` - stored | `ColumnMismatchException` "cannot be a batch column" | + +### 53) `flow-php/etl` - the column backend replaces the adaptive hydrator + +| Before | After | +|------------------------------------------------------------------------------------|-----------------------------------------------| +| `AdaptiveRowHydrator`, `NativeRowHydrator` | removed | +| `array_to_row(..., Hydrator $hydrator)` / `array_to_rows(..., Hydrator $hydrator)` | `Backend $backend` | +| `ConfigBuilder::hydrator(Hydrator)` | removed - `ConfigBuilder::backend(Backend)` | +| - | `Config::backend()`, `FlowContext::backend()` | + +### 54) `flow-php/etl` - `FloeEngine` and `NativeFloeEncoder` removed + +| Before | After | +|-----------------------------------------------------------------------------------------------------------------------------|---------| +| `from_floe(..., engine: ...)` / `to_floe(..., engine: ...)` | removed | +| the `engine` parameter of `FloeReader`, `FloeWriter`, `FloeStreamReader`, `FloeStreamWriter`, `FloeExtractor`, `FloeLoader` | removed | + +### 55) `flow-php/etl` - engine classes follow the batch + +| Before | After | +|------------------------------------------------------------------------------------------------|--------------------------------------------------------------| +| `Schema::zonedDefinitions()`, `ZoneAlignment`, `HydratedBatch`, `RowRenaming`, `DomValueCodec` | removed | +| `GroupBy::aggregatedRow(): Row` | `GroupBy::aggregatedValues(): array` | +| `RowMerger::merge(): Row` | `RowMerger::merge(): array` | +| `DerivedColumns::rows(Schema, Schema, list)` | `DerivedColumns::rows(Rows, Schema, Schema, string, Column)` | +| `UnpackedColumns::values(array, string, Schema, array)` | `UnpackedColumns::values(string, Schema, array)` | +| `InferredBatch::of(list)` | `InferredBatch::of(list)` | +| `InferredRows::of(..., Hydrator)` | `InferredRows::of(..., Backend)` | +| `FileConstants::fillRows(Rows, Schema)` | `FileConstants::fillRows(Rows, Schema, Backend $backend)` | + +### 56) `flow-php/etl-adapter-json` - untyped JSON Schema arrays and objects become `json` + +| Before | After | +|-------------------------------------------------------------------------------------------|---------------------------------| +| `{"type": "array"}` without `items` → `list` | `json` | +| `{"type": "object"}` without `properties` / `additionalProperties` → `map` | `json` | +| `schema_to_json_schema(schema_from_json_schema(...))` of an untyped `array` / `object` | `{"type": ["object", "array"]}` | + +### 57) `flow-php/etl` - a join name collision suggests a prefix + +| Before | After | +|-------------------------------------------------------------------------------------------------------------------|---------------------------------------------------------------------------------------------------------------| +| `SchemaDefinitionNotUniqueException`: `Entry definitions must be unique, duplicated entries: [id], all: [id, id]` | the same message with `. Please consider using join prefix option` appended - the text before it is unchanged | + +Every join shape (`joinCross()`, `join()`, `joinEach()`) reports the collision through `JoinSchema::of()`; the original +exception is the new one's `previous`. Only an exact match on the whole message is affected. + +### 58) `flow-php/types` - a date is midnight at UTC + +| Before | After | +|------------------------------------------------------------------------------------------------------------------|---------------------------------------------------------------------------------------------------| +| `type_date()->isValid()` / `DateDefinition::matches()` - midnight in any zone (`2026-01-01 00:00 Europe/Warsaw`) | only midnight at UTC offset `+00:00` | +| `row(['at' => new DateTimeImmutable('2026-01-01', new DateTimeZone('Europe/Warsaw'))])` - a `date` column | a `datetime` column - the instant is kept | +| `rows(schema(date_schema('at')), $rowWithWarsawMidnight)` / `matchTo()` - accepted | `SchemaMismatchException` "could not convert 2026-01-01T00:00:00.000000+01:00 (datetime) to date" | +| `type_date()->cast($warsawMidnight)` | unchanged - the value's own calendar day | + +A date column stores days and reads back midnight UTC (row 51), so a midnight in another zone was silently moved to a +different instant; it now stays a `datetime` unless it is cast to a date explicitly. + +### 59) `flow-php/types` - `Comparator` takes no arguments + +| Before | After | +|---------------------------------------------------|--------------------| +| `new Comparator(StructureComparison $structures)` | `new Comparator()` | + + +### 60) `flow-php/etl` - the row-shaped reader/writer API is removed + +| Before | After | +|---------------------------------------------------------------|-------------------------------------------------------------------------------------------------------| +| `Hydrator`, `PhpRowHydrator`, `TypedRowValues`, `Row\Encoder` | removed - readers build with `RowsBuilder`, encoders take `encode(Rows)` | +| `Row\RowsBuffer(Schema, int $size, ?Closure $batch)` | `Rows\RowsBuffer(Schema, Backend, int $size)` - `append()`, `appendFrom()`, `appendTake()`, `flush()` | +| `Config::hydrator()`, `FlowContext::hydrator()` | removed - `Config::backend()`, `FlowContext::backend()` | + +### 61) `flow-php/etl` - Floe takes a `Backend` + +No default: the caller passes the backend the rows are built in (`$context->backend()` inside a pipeline). + +| Before | After | +|---------------------------------------------------------------------------------------|--------------------------------------------------------------------------| +| `FloeReader(Filesystem, Codec = Noop, int $chunkSize = 65536, ?Hydrator, FloeEngine)` | `FloeReader(Filesystem, Backend, Codec = Noop, int $chunkSize = 65536)` | +| `FloeStreamReader(SourceStream, Codec, int $chunkSize, ?Hydrator, FloeEngine)` | `FloeStreamReader(SourceStream, Codec, int $chunkSize, Backend)` | +| `FloeWriter(Filesystem, Schema, Options = new Options(), ?Hydrator, FloeEngine)` | `FloeWriter(Filesystem, Schema, Backend, Options = new Options())` | +| `FloeStreamWriter(Schema, Options = new Options(), ?Hydrator, FloeEngine)` | `FloeStreamWriter(Schema, Backend, Options = new Options())` | +| `FloeMerger(Filesystem, ?Hydrator = null, Codec = Noop)` | `FloeMerger(Filesystem, Backend, Codec = Noop)` | +| `FloeSerializer(int $batchSize = 1000, ?Hydrator = null)` | `FloeSerializer(Backend, int $batchSize = 1000)` | +| `merge_floe(array $sources, $dest, bool $compact = false, ?Metadata, Filesystem)` | `merge_floe(array $sources, $dest, Backend, bool $compact = false, ...)` | + +### 62) adapters - encoders split into decoders and encoders + +| Before | After | +|--------------------------------------------------------------------------------------------------------------------------------------|----------------------------------------------------------------------------------------------------------------| +| `CSVEncoder::decode()`, `CSVEncoder::headers()` | `CSVDecoder::decode()`, `CSVDecoder::headers()` | +| `new CSVEncoder(withHeader:, emptyToNull:, ...)` | `new CSVDecoder(withHeader:, emptyToNull:, ...)` - `CSVEncoder` keeps only the write options | +| `TextEncoder::decode()` | removed - `TextExtractor` builds `['text' => $line]` rows itself | +| `XMLEncoder::decode()` | removed - `XMLReaderExtractor` builds `['node' => $xml]` rows itself | +| `ExcelEncoder(withHeader:, convertEmptyToNull:, timeFormat:)`, `decode()`, `headers()` | `ExcelDecoder(withHeader:, convertEmptyToNull:)` with `decode()`, `headers()`; `ExcelEncoder(timeFormat:)` | +| `OpenSheet::$encoder` (`ExcelEncoder`) | `OpenSheet::$decoder` (`ExcelDecoder`) | +| `GoogleSheetEncoder` | `GoogleSheetDecoder` - its throwing `encode()` is removed | +| `GoogleSheetReadOptions::encoder(): GoogleSheetEncoder` | `GoogleSheetReadOptions::decoder(): GoogleSheetDecoder` | +| `HttpEncoder` | `HttpDecoder` - `decode()` returns `list`, its throwing `encode()` is removed, `structuredBody()` stays | +| `JSONEncoder::decode()`, `DbalEncoder::decode()`, `PostgreSqlEncoder::decode()`, `SealEncoder::decode()`, `ParquetEncoder::decode()` | removed | +| `*Encoder::encode(list)` | `*Encoder::encode(Rows)`; no encoder implements `Row\Encoder` | +| `CSVLoader::write(Rows, array, FlowContext, array)` | `CSVLoader::write(Rows, array $headers, array $partitions)` | +| `JsonLoader::write(Rows, array, FlowContext)`, `JsonLinesLoader::write(...)`, `XMLLoader::write(...)` | `write(Rows, array $partitions)` | +| `XMLEncoder` refusing a structure value with more values than elements | removed - a structure cast drops undeclared keys, so the case cannot occur | +| `CSVOpenSource` implementations | must implement `headers(): list` - the header `records()` resolved | + +### 63) `flow-php/parquet` - `readColumns()` replaces `readValues()` + +| Before | After | +|-----------------------------------------------------------------------------------------------------------------|-----------------------------------------------------------------------------------------------------------------------------------------------| +| `ParquetEngine::readValues(SourceStream, Schema, array $columns = [], ?int $limit = null, ?int $offset = null)` | `ParquetEngine::readColumns(SourceStream, Schema, list $columns, int $batchSize, ?int $limit, ?int $offset)` - chunks keyed by column | +| - | `ParquetFile::columns(int $batchSize, array $columns = [], ?int $limit = null, ?int $offset = null)` | +| `ParquetFile::values(offset: -1)` without a limit - read from row 0 | `InvalidArgumentException` | + +### 64) `flow-php/etl-adapter-parquet`, `flow-php/etl` - self-describing formats take no schema + +| Before | After | +|-------------------------------------------------------------------------|------------------------------------------------------------| +| `from_parquet(...)->withSchema(...)`, `from_floe(...)->withSchema(...)` | `InvalidArgumentException` - the file footer is the schema | +| `InferredSchemaException::filesDiverge` suggesting `->withSchema(...)` | suggests only `->unionByName()` | +| `ValueConverter::decode()` | removed - values are cast by the batch builder | + +Project with `from_parquet(..., columns: [...])`. Change a type after reading, e.g. a JSON column another tool wrote as +`string`: `->withEntry('payload', ref('payload')->cast(type_json()))`. Re-zone a datetime after reading: +`->withEntry('at', to_timezone(ref('at'), 'Europe/Warsaw'))`. + +### 65) `flow-php/etl` - `RowPartitions` reads a batch + +| Before | After | +|-----------------------------------------------|---------------------------------------------| +| `RowPartitions::of(Row $row, Schema $schema)` | `RowPartitions::of(Rows $rows, int $index)` | + +### 66) `flow-php/etl` - Floe writes one BATCH frame per batch + +| Before | After | +|------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|---------------------------------------------------------------------------------------------------------------------------------| +| `FloeEncoder`, `PhpFloeEncoder`, `Format::rowFrames()`, `Format::metadataBytes()`, `Format::readMetadata()`, `Format::FRAME_ROW`, `Format::VALUE_*_BYTE`, `Flow\Floe\ValueEncoder`, `Flow\Floe\Encoding\*` | removed - `Rows::encodeFrame()` / `Flow\Floe\FrameEncoder` | +| `Flow\Floe\Decoding\*`, `Flow\Floe\ValueDecoder`, `Flow\Floe\SchemaDecoder`, `Flow\Floe\ColumnBlueprint` | removed - `Flow\Floe\FrameDecoder` | +| `FrameWriter::row(string $body)` | `FrameWriter::frame(int $type, string $body)` | +| `ValueEncoder::{xmlDocument,xmlElement,htmlElement}ToString()`, `ValueDecoder::{xmlDocument,xmlElement,htmlDocument,htmlElement}FromString()` | removed - `(new XmlDocumentPhysical())->toPhysical()` / `fromPhysical()` and the other `Flow\ETL\Column\Php\*Physical` | +| `Flow\Floe\Decoding\TimeZones` | `Flow\ETL\Column\Physical\TimeZones` | +| `Offsets::pack()` throws `InvalidArgumentException` past 2³¹−1 | throws `Flow\ETL\Exception\OffsetOverflow` (extends `InvalidArgumentException`) | +| `from_floe()->withBatchSize($n)`, `FloeReader::rows($n)` - batches of exactly `$n` rows | at most `$n` rows, never spanning two BATCH frames - re-batch with `->batchSize($n)` | +| a codec wraps every frame body | a codec wraps each buffer of a BATCH frame behind an i64 uncompressed length (`-1` = stored raw); the directory stays plaintext | +| a null in a NOT NULL column refused by the Floe writer (`SchemaMismatchException`) | refused by `Rows::fromColumns()` / `withColumns()` with the same exception and message, whatever the sink | +| an XML/HTML value that cannot be converted to or from its string while building a column throws `FloeException` | throws `Flow\ETL\Exception\InvalidArgumentException`, same message | +### 67) `flow-php/flow-php-ext`, `flow-php/etl`, `flow-php/etl-adapter-csv` - the extension is the column backend + +| Before | After | +|-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|--------------------------------------------------------------------------------------------------------------------------------| +| `Flow\Floe\RustFloeEncoderNative`, `Flow\ETL\Row\RustRowHydratorNative` | removed | +| - | `flow_php` registers `Flow\ETL\Column\{Backend, Column, ColumnBuilder, RustBackend, RustColumn, RustColumnBuilder}` | +| any `flow_php` installs beside `flow-php/etl` | Composer refuses `ext-flow_php` < 0.45 (`conflict`); forced past it: `Class "Flow\ETL\Column\RustBackend" not found` | +| untagged builds report `0.44.1+65.g…` | `0.45.0-dev+65.g…` (next minor, pre-release) | +| - | `Backend`: `builder()`, `constant()`, `decode()`, `adopt(Definition, Column): Column`, `allocatedBytes(): int` | +| `Consumption` / `analyze()` / telemetry memory = `memory_get_usage()` | plus the configured backend's `allocatedBytes()` (native columns are not bound by `memory_limit`) | +| `new Consumption(bool $realMemory = true)` | `new Consumption(Backend, bool $realMemory = true)` | +| `new TelemetryContext(Logger, Tracer, Meter, TelemetryOptions)` | `new TelemetryContext(Backend, Logger, Tracer, Meter, TelemetryOptions)` | +| `new KWayMerge(References, int $batchSize = 1000)` | `new KWayMerge(References, Backend, int $batchSize = 1000)` | +| `new Joiner(Expression, Join, int $batchSize = 1000)` | `new Joiner(Expression, Join, Backend, int $batchSize = 1000)` | +| `AggregatedGroups::flush(int $batchSize)` | `flush(int $batchSize, FlowContext)` | +| `BucketingConfigBuilder::build(Path)`, `SortAlgorithmBuilder`, `GroupByAlgorithmBuilder`, `JoinAlgorithmBuilder`, `RepartitionAlgorithmBuilder` and their implementations `build(Path)` | `build(Path, Backend)` - custom algorithm builders implement the new signature | +| `new FilesystemBuckets(Filesystem, Path, int $batchSize = 1000)` | `new FilesystemBuckets(Filesystem, Path, Backend, int $batchSize = 1000)` | +| sort merges, joins, group-by, `count()`, spilled bucket reads, partition filters, CSV reads | build in the configured backend (`config_builder()->backend()`, `AdaptiveBackend` by default) - `PhpBackend` keeps PHP columns | +| - | a native string/list column past 2 GiB of offsets throws `OffsetOverflow` at `concat()` / `appendMany()` / `encode()` | + +### 68) `flow-php/etl`, adapters - `RawRowValues` removed, CSV sources yield batches + +| Before | After | +|---------------------------------------------------------------------------------------------------------------------------|-------------------------------------------------------------------------------| +| `RawRowValues`; `SchemaSampler`, `SniffsColumnTypes`, `SchemaInferrer::infer()/sniff()`, `ColumnTypes::observe()` over it | plain `array` rows | +| `CSVOpenSource::records(): Generator` | `Iterator>` | +| - | `CSVOpenSource::batches(Schema, int, Backend): Iterator` | +| `CSVFileReader::batches(SourceFile, int): Generator>` returning the header | `batches(SourceFile, Schema, int, Backend): Generator`, no return value | + +### 69) `flow-php/etl` - `Row` removed, `Rows` is a columnar batch + +| Before | After | +|-------------------------------------------------------------------------------|-----------------------------------------------------------------------------------------------------------------------------------------| +| `foreach ($df->fetch() as $row) { $row->get('x'); }` | `foreach ($df->fetch()->toArray() as $row) { $row['x']; }` | +| `$rows->first()->get('x')`, `$rows[0]->get('x')` | `$rows->column('x')->value(0)`, `$rows->values(0)['x']` | +| `$rows->first()->toArray()` | `$rows->toArray()[0]` | +| `new Rows($schema)`, `rows($schema)` | `Rows::empty(Schema, Backend)`, `rows(Schema, Backend = new AdaptiveBackend())` | +| `new Rows($schema, ...$row)`, `rows($schema, ...$row)`, `$rows->add(...$row)` | `array_to_rows(array $data, Schema, Backend = new AdaptiveBackend())`, `$rows->concat(Backend, ...$others)`, `$rows->gather(list)` | +| `Rows` implements `ArrayAccess`, `IteratorAggregate` | `Countable` only | +| `Rows::all()`, `first()`, `last()`, `offsetGet()` | removed - `toArray()`, `values(int)`, `column(string)`, `slice(int, int)`, `gather(list)` | +| `Rows::unique(Comparator)`, `Row\Comparator`, `NativeComparator` | removed - `$df->dropDuplicates()` | +| `array_to_row(array $data, Schema, Hydrator, $partitions)` | `array_to_rows([$data], Schema, Backend = new AdaptiveBackend())` | +| `SerializedPayloadDecoder::decode(Row): array` | `decode(Rows, int $index): Rows` - no payload is a one-row batch without columns | +| `Formatter\ASCII\Body::rows(): array` | `count(): int`, `value(string $entry, int $index): mixed` | + +`Rows::values(int)` outside the batch throws `InvalidArgumentException`: `Row N does not exist in a batch of M rows`. + +### 70) `flow-php/etl` - `ScalarFunction` evaluates a batch + +| Before | After | +|-------------------------------------------------------------------------------------|-------------------------------------------------------------------------------------------------------------------| +| `ScalarFunction::eval(Row, FlowContext): mixed` | `eval(Rows, FlowContext): Column` - one value per row, built by `ResultColumn` | +| `ExpandResults::eval(Row): array`, `UnpackResults::eval(Row): array` | marker interfaces - `eval(Rows): Column` returns a list (expand) or structure (unpack) column | +| `Parameter::{as, asArray, asBoolean, asEnum, asFloat, asInstanceOf, asInt}(Row, …)` | `{asTypes, asArrays, asBooleans, asEnums, asFloats, asInstancesOf, asInts}(Rows, …)` - one list per batch | +| `Parameter::{asListOfObjects, asNumber, asObject, asString}(Row, …)` | `{asListsOfObjects, asNumbers, asObjects, asStrings}(Rows, …)` | +| `Parameter::eval(Row)`, `asValue(Row)` | `values(Rows, FlowContext)`, `column(Rows, FlowContext)` | +| `$function->eval($rows, $context)` on an unresolved tree | `InvalidLogicException` - `(new ReferenceResolver())->resolve($function, $rows->schema())->eval($rows, $context)` | + +Before: + +```php +public function eval(Row $row, FlowContext $context): ?string +{ + $value = (new Parameter($this->value))->asString($row, $context); + + return $value === null ? null : strtoupper($value); +} +``` + +After: + +```php +public function eval(Rows $rows, FlowContext $context): Column +{ + $results = []; + + foreach ((new Parameter($this->value))->asStrings($rows, $context) as $value) { + $results[] = $value === null ? null : strtoupper($value); + } + + return (new ResultColumn())->of($this, $results); +} +``` + +### 71) `flow-php/etl` - float arithmetic follows IEEE 754 + +| Before | After | +|------------------------------------------------------------------------------------------------------------|--------------------------------------------------------------------------------------------------------------------------| +| `ref('a')->multiply(lit(3))` with `a = 0.1` - `0.3` | `0.30000000000000004`; `ref('a')->multiply(lit(3), exact: true)` - `0.3` | +| `plus()`, `minus()`, `power()` - decimal arithmetic | IEEE 754; `exact: true` keeps decimal arithmetic | +| an integer result past `PHP_INT_MAX` - `Brick\Math\Exception\IntegerOverflowException` | `EvaluationException`: `Plus function integer overflow (row N)` | +| `power()` of an integer with a negative exponent | `EvaluationException`: `Power function of an integer requires a non-negative exponent (row N)` | +| `average(ref('amount'))` - decimal sum (`Calculator` per value), then rounded to `scale` | IEEE sum with Neumaier compensation, then the same rounding; `average(ref('amount'), exact: true)` keeps the decimal sum | +| `sum(ref('amount'))` (default `exact: false`) - plain IEEE addition, `0.1` ten times: `0.9999999999999999` | compensated IEEE addition: `1.0` | +| `RunningSum::add(float\|int $sum, $value, bool $exact): float\|int` | stateful: `add($value, bool $exact): void`, `value(): float\|int`, `merge(RunningSum, bool $exact)` | + +### 72) `flow-php/etl` - function errors carry the batch row + +| Before | After | +|----------------------------------------------------------------------|-------------------------------------------------------------------------------------------------------------------------------------| +| an exception thrown by a function reaches the caller as-is | `Flow\ETL\Exception\EvaluationException` (extends `InvalidArgumentException`), message `… (row N)`, the original as `getPrevious()` | +| `catch (SomeSpecificException $e)` around `withEntry()` / `filter()` | `catch (EvaluationException $e)` and read `$e->getPrevious()` | + +### 73) `flow-php/etl` - uuid, json, enum, xml, html and date/time values compare by value + +| Before | After | +|-------------------------------------------------------------------------------------------------------------------|----------------------------------------------------------------------------------| +| `equals()`, `same()`, `isIn()`, join `Identical`, `batchBy()` over these types - always false / one batch per row | value equality | +| json values | compare by their stored text - `{"a":1,"b":2}` and `{"b":2,"a":1}` are not equal | + +### 74) `flow-php/etl` - `array_keys_style_convert()` declares the converted nested shape + +| Before | After | +|---------------------------------------------------------------------------------------------------|--------------------------------------------------------------------------------------------------------------------------| +| `returns()` renames the top-level keys only; a nested conversion stored with `withEntry()` failed | `returns()` renames structure fields at every level (structures, lists, maps); key collisions are refused at every level | + +### 75) `flow-php/etl` - extension points take a batch + +| Before | After | +|--------------------------------------------------------------------|------------------------------------------------------------------------------------------------| +| `AggregatingFunction::aggregate(Row, FlowContext)` | `aggregate(Rows, list $indices, FlowContext)` + `merge(AggregatingFunction, FlowContext)` | +| `Join\Comparison::compare(Row, Row): bool` | `compare(Rows $left, Rows $right): list` - pairwise | +| `Constraint::isSatisfiedBy(Row, Schema)`, `violation(Row, Schema)` | `firstViolation(Rows): ?int`, `violation(Rows, int)` | +| `FrameAccumulator::accumulate(Row)` | `accumulate(Rows, int)` | +| `WindowContext::row()` | removed - `WindowContext(int, Rows, WindowFrame, FlowContext)` | +| `PeerComparator::arePeers(Row, Row, Schema)` | `arePeers(Rows, int, int)` | +| `Dataset\Statistics\Columns::add(Definition, mixed)` | `add(Definition, Column)` | + +### 76) `flow-php/etl` - one sort order: nulls smallest, strings by bytes + +| Before | After | +|---------------------------------------------------------------------------------------------------------------------|-----------------------------------------------------------------------------------------------------| +| `sortBy(ref('code')->asc())` over `'9', '10', '100'` - `'9', '10', '100'` (numeric when every value looked numeric) | `'10', '100', '9'` - strings always compare by bytes | +| `sortBy(ref('v')->asc())` over `3, null, 1` - unsorted (a null compared equal to every value) | `null, 1, 3` - nulls first ascending, last descending; descending is the exact reverse of ascending | +| NaN - no defined position | after every other value, equal to NaN; `-0.0` equals `0.0` | +| external sort (spilled runs) could order differently from the in-memory sort | both, `constraint_sorted_by()` and window peers use the same order | +| `Flow\ETL\Sort\ValuesSorter`, `Sort\Merge\RowsMinHeap`, `Sort\Merge\ComparableBucketRow` | removed - `Flow\ETL\Sort\RowOrder` | +| `new BucketCursor(Generator, References)`, `keys(): list`, `next()` | `new BucketCursor(Generator, RowOrder)`, `keys(): list`, `advance(int)` | +| - | `ref('x')->asc(NullsOrder::LAST)`, `ref('x')->desc(NullsOrder::FIRST)` move nulls to the other end | +| a custom `Reference` implementation | adds `nulls(): NullsOrder` | + +### 77) `flow-php/etl` - spilling steps stop at a memory limit, not a row count + +| Before | After | +|----------------------------------------------------------------------------------------------------------------------------------------------------------|-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| `external_sort()->runSize(int)` - rows per sorted run, default `10000` | `external_sort()->memoryLimit(Unit)`; data under it sorts in memory without spilling | +| - | `hash_join()->memoryLimit(Unit)`, `hash_group_by()->memoryLimit(Unit)`, `hash_repartition()->memoryLimit(Unit)` | +| - | every limit defaults to `Config\MemoryLimit::default()`: `FLOW_MAX_MEMORY`, otherwise 70% of `memory_limit`, 1 GiB when `memory_limit` is `-1` | +| - | a limit is compared with the whole process (`memory_get_usage()` + the backend's native memory), not with the step's own buffer - a sort and a join in one pipeline stay under it together | +| `hash_join()` with a disk storage partitioned both sides into buckets | the right side stays in memory under the limit and the left side streams in its own order; past it both sides are partitioned | +| `groupBy()` / window partitions always hash-partitioned their input into buckets | under the limit a group-by aggregates in one pass and a window groups its partitions in memory - no bucket is written; past it the rest of the stream is partitioned (group-by merges the groups aggregated before the limit) | +| group-by / repartition / join wrote one bucket frame per bucket per input batch | buckets stay in memory under the limit; past it every frame holds `batchSize` rows | +| `GroupBySteps::of()` / `RepartitionSteps::of()` - `[BucketingProcessor, GroupByAggregationProcessor]` / `[BucketingProcessor, RepartitionProcessor]` | `[GroupByAggregationProcessor]` / `[RepartitionProcessor]`; both take `HashBucketing` and `Unit $memoryLimit`; `GroupBy\BucketAggregation` removed | +| `new ExternalSortConfig($bucketing, ?BucketsStorage $merge = null, int $runSize)` | `new ExternalSortConfig($bucketing, Unit $memoryLimit, ?BucketsStorage $merge = null)` | +| `new HashJoinConfig($bucketing)`, `new HashGroupByConfig($bucketing)`, `new HashRepartitionConfig($bucketing)` | `new …Config($bucketing, Unit $memoryLimit)` | +| `new HashJoinProcessor($right, $executor, $on, $type, $left, $right, $random, $bucketsCount, $batchSize)` | `new HashJoinProcessor($right, $executor, $on, $type, $left, $right, $random, Unit $memoryLimit, $bucketsCount, $batchSize)` | +| `new MergeSortProcessor(References, Buckets, Buckets, RandomValueGenerator, int, int)` | `new MergeSortProcessor(ExternalSort)` | +| `SortSteps::of()` - `[BucketingProcessor, MergeSortProcessor]` for the external sort | `[MergeSortProcessor]`; `SortSteps::external(): ?ExternalSort` | +| `Flow\ETL\Bucketing\SortedRunBucketing` | removed - `Flow\ETL\Sort\ExternalSort` | +| `Flow\ETL\Processor\BucketingProcessor`, `Flow\ETL\Bucketing\BucketingStrategy`, `Flow\ETL\Bucketing\BucketShape`, `Bucket::schema()`, `Bucket::toRow()` | removed - sort, join, group-by and repartition partition inside their own step | +| `sortBy()->limit(n)` over the external sort became a top-n only when `n <= runSize` | always a top-n; past the memory limit it hands its kept rows and the rest of the stream to the external sort, whose merge stops after `n` rows | +| `new TopNRewrite(FlowContext)` | `new TopNRewrite()` | +| `new TopNProcessor(References, int, ?Schema)` | `new TopNProcessor(References, int, ?ExternalSort $fallback = null, ?Schema)` | +| `new Plan\Node\TopN(Node, References, int)` | `new TopN(Node, References, int, ?SortAlgorithmBuilder $algorithm = null)` | +| explain: `Storage: FilesystemBuckets` | adds `Memory: N MB` under the storage of a sort, join, group-by and repartition step | + +### 78) `flow-php/etl` - `sum(exact:)` is a bool; `batched_by()`, `string_agg()` and `pivot()` follow the one order + +| Before | After | +|---------------------------------------------------------------------------------------------------------------------------|------------------------------------------------------------------------------------------------------------------------------------------| +| `sum(ref('a'), exact: ref('is_exact'))` - exactness chosen per row | `sum(ref('a'), exact: true)` - one `bool` for the aggregate, as `average()` and `plus/minus/multiply/power` | +| `sum(…, exact: ScalarFunction)` disabled spill column pruning (`references()` was `null`) | `sum()` always enumerates its reference; a custom `AggregatingFunction` may still return `null` | +| `batched_by(from_…, 'at')` - equal datetimes / uuids split into one batch per row, a `null` value joined the next group | value equality, as `batchBy()`; nulls are a group of their own | +| `string_agg(ref('code'), ',', SortOrder::ASC)` over `'9', '10', '100'` - `9,10,100` (numeric-looking strings numerically) | `10,100,9` - strings by bytes, as `sortBy()` | +| `discover_pivot_values()` over `'9', '10', '100'` - columns `9, 10, 100` | `10, 100, 9`; integer pivot values stay numeric | +| `->stringMatch()` / `->stringMatchAll()` turned a PHP `\Error` into an exception `optional()` swallowed | an `\Error` propagates; `->jsonDecode()`, `cast()`, `->arrayPathExists()`, `->stringMatch*()` errors keep their cause as `getPrevious()` | + +### 79) `flow-php/etl-adapter-parquet` - with `flow_php` and arrow-ext loaded and no engine given, Parquet is read and written as native columns + +| Before | After | +|--------------------------------------------------------------------------------------------------------------------|-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| `from_parquet($path)` / `to_parquet($path)` - `AdaptiveParquetEngine` (arrow-ext when loaded, else the PHP engine) | with `flow_php` and arrow-ext loaded: arrow-ext's batches straight into native columns and back (Arrow C Data Interface); `flow_php` alone: the PHP engine, its values adopted into native columns; without `flow_php`: unchanged | +| `engine: new PhpParquetEngine()` / `new ArrowParquetEngine()` | `engine: new PhpParquetEngine()` opts out; `new AdaptiveParquetEngine()` reads and writes native columns when both extensions are loaded, `new RustParquetEngine()` reads them | +| a column type arrow-ext could not read failed inside arrow-ext | refused before the first batch: `Flow\Parquet\Exception\RuntimeException` `Parquet column "" () is not supported by the arrow Parquet reader; read the file with \Flow\Parquet\Reader::php()` | +| `flow-php/etl-adapter-parquet` installable with any `flow_php` or arrow-ext version | `conflict: ext-flow_php <0.45`, `ext-arrow <0.45` - load extensions of this release or none | + +### 80) `flow-php/etl-adapter-parquet` - Flow `float` columns are written as Parquet `DOUBLE` + +| Before | After | +|---------------------------------------------------------------------------------------------------------|--------------------------------------------------------------------------------------------| +| `float_schema('amount')` written as Parquet `FLOAT` (32-bit): `0.14` read back as `0.14000000059604645` | written as `DOUBLE` (64-bit), read back exactly; readers of the file see a `DOUBLE` column | + +### 81) `flow-php/etl-adapter-parquet` - with `flow_php` and arrow-ext, a string that is not valid UTF-8 is refused on write + +| Before | After | +|------------------------------------------------------------------------------------------------------------------------------------|---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| without arrow-ext, `PhpParquetEngine` wrote `"\xff\xfe"` into a `STRING` column other readers reject (DuckDB: `is not valid UTF8`) | `Flow\Parquet\Exception\RuntimeException` `Parquet column "name" row 0 holds a string that is not valid UTF-8; Parquet STRING columns require UTF-8` - the row is 0-based within the batch written | +| with arrow-ext: `Column 'name': expected string, got string` | the same refusal; invalid UTF-8 into `STRING` / `JSON` / `ENUM` is refused by both engines; `Option::VALIDATE_DATA = false` on `PhpParquetEngine` writes it unvalidated (a file other readers reject); arrow-ext always refuses | + +### 82) `flow-php/parquet` - `DELTA_BINARY_PACKED` follows the spec; affected files written by `PhpParquetEngine` must be rewritten + +| Before | After | +|--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|-----------------------------------------------------------------------------------------------------------------------------------------------------| +| `PhpParquetEngine` wrote `DELTA_BINARY_PACKED` blocks that other readers misread (a partial last block without all miniblock bit widths, zero-width miniblocks with 4 padding bytes) | spec-conformant blocks; DuckDB, arrow-rs and `flow_php` read the values back | +| it read other writers' `DELTA_BINARY_PACKED` `INT64` wrong after an outlier (`9223372036854645807` read as `9223372036788279061`, garbage after) | reads them as DuckDB and arrow-rs do | +| affected: `INT32`/`INT64` columns `PhpParquetEngine` wrote with `Option::WRITER_VERSION` `2` or `COLUMNS_ENCODINGS` `DELTA_BINARY_PACKED` (the default is version `1`, `PLAIN`) | no spec-conformant reader returns their values: read them with the Flow version that wrote them and write them again (or re-export from the source) | + +### 83) `flow-php/parquet` - `ParquetEngine::openForRead()` replaces `readColumns()` + +| Before | After | +|---------------------------------------------------------------------------------|----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| `$engine->readColumns($stream, $schema, $columns, $batchSize, $limit, $offset)` | `$engine->openForRead($stream)->readColumns($columns, $batchSize, $limit, $offset)` | +| a custom `ParquetEngine` implements `readColumns()` | it implements `openForRead(SourceStream): ParquetFileReader` - `metadata()`, `schema()`, `rowsNumber()`, `totalByteSize()`, `readColumns()`, `close()` (which closes the stream) | +| `new ParquetFile($stream, $byteOrder, $options, $engine)` | `new ParquetFile($stream, $options, $engine->openForRead($stream))` | +| a `ParquetFile` closed its stream only when destroyed | `ParquetFile::close()` closes its reader (once); `ParquetFile::reader()` returns it | + +### 84) `flow-php/parquet` - the arrow engine (`RustParquetEngine`, was `ArrowParquetEngine`) reads and writes through arrow-ext's new Parquet core + +| Before | After | +|----------------------------------------------------------------------------------------------------------------------------------|--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| `DATE` read as `DateTimeImmutable` in zone `+00:00` | zone `UTC`, as `PhpParquetEngine` | +| `TIME` of 25 hours read as `PT25H`, `days` `false` | `d` = 1, `h` = 1, `days` an int (the `diff()` interval `PhpParquetEngine` returns) | +| `INT96` outside 1677-2262 read wrapped (0001-01-01 → 1754-08-30) | read exactly, floored to the microsecond | +| `UINT_64` above `PHP_INT_MAX` read as a numeric string | refused, as by `PhpParquetEngine` | +| writes accepted an int into `FLOAT` / `DOUBLE` / `DATE` / `TIMESTAMP` / `TIME`, a numeric string into `DECIMAL` / `UINT_64` | refused - writes accept what `PhpParquetEngine` accepts; decimals are written from floats (or ints) as `PhpParquetEngine` writes them; an object with `toString()` into `UUID` is accepted | +| `TIME` writes dropped the interval's `d` | kept | +| an offset read decoded every row group before the offset; page headers were read to the end of the file | only the row groups a read needs, each column chunk read once | +| an `INTERVAL`, a path into a `LIST` / `MAP`, `LZO` or `INT96` under `Option::INT_96_AS_DATETIME = false` failed inside arrow-ext | refused before the first batch naming the column and `\Flow\Parquet\Reader::php()` | +| `Reader::read()` returned without touching the file; an invalid file failed at `metadata()` | the footer is read when the file is opened: `Reader::read()` / `readStream()` throw `InvalidArgumentException` `Given file is not valid Parquet file: …` | +| `Flow\Arrow\Parquet\Reader` / `Writer` / `Exception` | removed: `Flow\Parquet\Engine\{RustParquetFileReader, RustParquetFileWriter}`, `Flow\Arrow\Parquet\{RustColumnsReader, RustBatchReader}`; refusals are `Flow\Parquet\Exception\*` | +| `flow-php/parquet` installable with any arrow-ext version | `conflict: ext-arrow <0.45` | + +### 85) `flow-php/parquet` - `PhpParquetEngine` reads unsigned integers and INT96 exactly; `FlatColumn::enum()` writes the ENUM logical type; values it wrote wrongly are refused + +| Before | After | +|-----------------------------------------------------------------------------------------------------------------------------------------------------|----------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| `UINT_32` 4000000000 read as `-294967296`; `UINT_64` 18446744073709551615 read as `-1` | `4000000000`; a `UINT_64` above `PHP_INT_MAX` is refused | +| `INT96` 1960-01-01 23:59:59.999999999 read as 1960-01-02 00:00:00 (rounded) | 1960-01-01 23:59:59.999999 (floored, integer arithmetic) | +| `FlatColumn::enum()` wrote logical `STRING` + converted `ENUM`, a file arrow-rs refuses | logical `ENUM` + converted `ENUM` | +| `from_parquet()` refused a logical `ENUM` column (`ENUM is not supported`) | read as a `string` column, as `STRING` | +| an int outside an `INT_8/16/32` or `UINT_8/16/32/64` range written wrapped (4000000000 into `INT32` → `-294967296`), negatives into `UINT` accepted | `ValidationException` `Column "" row : … is out of the … range` | +| a `UUID` / `FIXED_LEN_BYTE_ARRAY(n)` value of another byte length written cut or short | refused naming column and row | +| invalid UTF-8 into `STRING` / `JSON` / `ENUM` written (other readers reject the file) | refused naming column and row | +| these checks | run under `Option::VALIDATE_DATA` (default `true`), as every other validator check | +| `Validator::validate(Column $column, mixed $data): void` | `Validator::validate(Column $column, mixed $data, int $row): void` - custom validators add the parameter (the 0-based row within the writer) | +| logical `INTEGER` read without its width and sign; written back as an `IntType` missing both (a footer arrow-rs refuses) | `LogicalType::integerData(): ?LogicalType\Integer` keeps them and writes them back; `LogicalType::integer()` → `LogicalType::integer(int $bitWidth, bool $isSigned)` | + +### 86) `flow-php/arrow-ext` - Parquet core replaced + +| Before | After | +|-------------------------------------------------------------------------------------------------|------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| `Flow\Arrow\Parquet\Reader` / `Writer` - PHP values per call (`readRowGroup()`, `writeBatch()`) | `Flow\Parquet\Engine\{RustParquetEngine, RustParquetFileReader, RustParquetFileWriter}` implement `ParquetEngine`, `ParquetFileReader`, `ParquetFileWriter` - the values `PhpParquetEngine` reads and accepts; `Flow\Arrow\Parquet\RustBatchReader` and `RustParquetFileWriter::writeArrowBatch()` hand batches to other extensions through the Arrow C Data Interface | + +### 87) `flow-php/etl-adapter-csv` - `to_csv()` writes a boolean as `true` / `false` + +| Before | After | +|------------------------------------------------------------------|--------------------------------------------------------------------| +| `true` → `1`, `false` → an empty field, the same bytes as `null` | `true` / `false`; `null` stays an empty field | +| a NOT NULL boolean column could not be read back by `from_csv()` | reads back equal; a nullable column keeps `false` and `null` apart | + +### 88) `flow-php/etl-adapter-csv`, `-xml`, `-text`, `-json` - a float is written as the shortest text that reads back as the same float + +| Before | After | +|-----------------------------------------------------------------------------------------------|---------------------------------------------------------------| +| CSV / XML / Text: `(string) $float`, 14 digits under the `precision` ini: `0.1 + 0.2` → `0.3` | `0.30000000000000004` | +| `1 / 3` → `0.33333333333333`, `123456789012345.678` → `1.2345678901235E+14` | `0.3333333333333333`, `123456789012345.67` | +| `1.0` → `1`, `1e25` → `1.0E+25`, `-0.0` → `-0` | `1.0`, `1.0e+25`, `-0.0` - always a fraction or an exponent | +| `NAN`, `INF`, `-INF` | unchanged; refused inside a nested CSV cell (`JsonException`) | +| JSON: the digits followed the `serialize_precision` ini (`17` → `0.10000000000000001`) | the shortest digits under any ini value | + +Writing a float column forces `serialize_precision = -1` for the render and restores the ini afterwards. When the ini +holds another value, `ini_set()` must be allowed; otherwise the writers throw +`RuntimeException('Writing floats requires serialize_precision = -1 and ini_set() cannot change it')`. + +### 89) `flow-php/etl-adapter-csv`, `-json`, `-xml`, `-text` - a nested value is written by its type, as a column of that type is + +| Before | After | +|---------------------------------------------------------------------------------------------------------|----------------------------------------------------------------| +| CSV `list`: `[{"date":"2026-01-02 03:04:05.000000","timezone_type":3,"timezone":"UTC"}]` | `["2026-01-02T03:04:05+00:00"]` (the loader's datetime format) | +| CSV `list`: `[{}]`; `list'], + ['node' => ''], + ['node' => '<p> & ©'], + ], + data_frame() + ->read(from_parquet($loaded, filesystem: $memory)) + ->withEntry('node', ref('node')->cast(type_string())) + ->fetch() + ->toArray(), + ); + } } diff --git a/src/adapter/etl-adapter-parquet/tests/Flow/ETL/Adapter/Parquet/Tests/Unit/AdaptiveParquetOpenSinkTest.php b/src/adapter/etl-adapter-parquet/tests/Flow/ETL/Adapter/Parquet/Tests/Unit/AdaptiveParquetOpenSinkTest.php new file mode 100644 index 0000000000..bc700258b9 --- /dev/null +++ b/src/adapter/etl-adapter-parquet/tests/Flow/ETL/Adapter/Parquet/Tests/Unit/AdaptiveParquetOpenSinkTest.php @@ -0,0 +1,149 @@ + + */ + public static function engines(): iterable + { + yield 'no engine' => [null]; + yield 'adaptive engine' => [new AdaptiveParquetEngine()]; + } + + #[DataProvider('engines')] + public function test_a_default_engine_writes_a_file_php_reads(?ParquetEngine $engine): void + { + $filesystem = memory_filesystem(); + $schema = schema(int_schema('id')); + $sink = new AdaptiveParquetOpenSink( + $filesystem->writeTo(path('memory://out.parquet')), + (new SchemaConverter())->toParquet($schema), + Compressions::GZIP, + Options::default(), + $engine, + ); + + $sink->write(array_to_rows([['id' => 1], ['id' => 2]], $schema)); + $sink->close(); + + static::assertSame( + [['id' => 1], ['id' => 2]], + ParquetFilesContext::phpEngineValues($filesystem, 'memory://out.parquet'), + ); + } + + /** + * The refusal of a string written into an int column names the lane that wrote: flow_php's native columns into + * arrow-ext, PHP values into arrow-ext, or PHP values into the PHP engine. + * + * @return iterable + */ + public static function lanes(): iterable + { + $arrow = extension_loaded('arrow'); + $rust = match (true) { + $arrow && extension_loaded('flow_php') => 'is not supported by the arrow Parquet writer', + $arrow => 'row 0: expected int, got string', + default => 'require integer as value', + }; + + yield 'no engine' => [static fn(): ?ParquetEngine => null, $rust]; + yield 'little-endian adaptive engine' => [static fn(): ParquetEngine => new AdaptiveParquetEngine(), $rust]; + yield 'big-endian adaptive engine' => [ + static fn(): ParquetEngine => new AdaptiveParquetEngine(ByteOrder::BIG_ENDIAN), + 'require integer as value', + ]; + yield 'php engine' => [static fn(): ParquetEngine => new PhpParquetEngine(), 'require integer as value']; + + if ($arrow) { + yield 'rust engine' => [static fn(): ParquetEngine => new RustParquetEngine(), $rust]; + } + } + + /** + * @param Closure(): ?ParquetEngine $engine + */ + #[DataProvider('lanes')] + public function test_the_engine_decides_which_lane_writes(Closure $engine, string $refusal): void + { + $sink = new AdaptiveParquetOpenSink( + memory_filesystem()->writeTo(path('memory://out.parquet')), + (new SchemaConverter())->toParquet(schema(int_schema('id'))), + Compressions::SNAPPY, + Options::default(), + $engine(), + ); + + $this->expectExceptionMessage($refusal); + + $sink->write(array_to_rows([['id' => 'one']], schema(str_schema('id')))); + } + + public function test_any_other_engine_writes_the_file(): void + { + $filesystem = memory_filesystem(); + $schema = schema(int_schema('id')); + $engine = new RecordingParquetEngine(); + $sink = new AdaptiveParquetOpenSink( + $filesystem->writeTo(path('memory://out.parquet')), + (new SchemaConverter())->toParquet($schema), + Compressions::SNAPPY, + Options::default(), + $engine, + ); + + $sink->write(array_to_rows([['id' => 1], ['id' => 2]], $schema)); + $sink->close(); + + static::assertSame(1, $engine->openedForWrite); + static::assertSame( + [['id' => 1], ['id' => 2]], + ParquetFilesContext::phpEngineValues($filesystem, 'memory://out.parquet'), + ); + } + + public function test_a_flow_php_of_another_abi_is_refused(): void + { + $this->expectException(RuntimeException::class); + $this->expectExceptionMessage('does not match flow-php/etl'); + + new AdaptiveParquetOpenSink( + memory_filesystem()->writeTo(path('memory://out.parquet')), + (new SchemaConverter())->toParquet(schema(int_schema('id'))), + Compressions::SNAPPY, + Options::default(), + new PhpParquetEngine(), + new FlowPhpExtension(true, FlowPhpExtension::ABI + 1, '0.46.0'), + ); + } +} diff --git a/src/adapter/etl-adapter-parquet/tests/Flow/ETL/Adapter/Parquet/Tests/Unit/AdaptiveParquetOpenSourceTest.php b/src/adapter/etl-adapter-parquet/tests/Flow/ETL/Adapter/Parquet/Tests/Unit/AdaptiveParquetOpenSourceTest.php new file mode 100644 index 0000000000..ce01e23bad --- /dev/null +++ b/src/adapter/etl-adapter-parquet/tests/Flow/ETL/Adapter/Parquet/Tests/Unit/AdaptiveParquetOpenSourceTest.php @@ -0,0 +1,88 @@ +file; + $source = new AdaptiveParquetOpenSource($file); + + static::assertInstanceOf(Generator::class, $source->batches( + schema(int_schema('id')), + 10, + null, + null, + new PhpBackend(), + )); + $source->close(); + } + + #[RequiresPhpExtension('flow_php')] + #[RequiresPhpExtension('arrow')] + public function test_with_flow_php_a_rust_engine_file_reads_through_rust(): void + { + // the file closes its reader when released, so it outlives the source as ParquetSourceFile does + $file = ParquetSourceFileContext::over(new NativeLocalFilesystem(), engine: new RustParquetEngine())->file; + $source = new AdaptiveParquetOpenSource($file); + + static::assertInstanceOf(RustIterator::class, $source->batches( + schema(int_schema('id')), + 10, + null, + null, + new PhpBackend(), + )); + $source->close(); + } + + #[RequiresPhpExtension('arrow')] + public function test_without_flow_php_a_rust_engine_file_reads_through_php(): void + { + if (extension_loaded('flow_php')) { + static::markTestSkipped('flow_php is loaded'); + } + + // the file closes its reader when released, so it outlives the source as ParquetSourceFile does + $file = ParquetSourceFileContext::over(new NativeLocalFilesystem(), engine: new RustParquetEngine())->file; + $source = new AdaptiveParquetOpenSource($file); + + static::assertInstanceOf(Generator::class, $source->batches( + schema(int_schema('id')), + 10, + null, + null, + new PhpBackend(), + )); + $source->close(); + } + + public function test_a_flow_php_of_another_abi_is_refused(): void + { + $file = ParquetSourceFileContext::over(new NativeLocalFilesystem())->file; + + $this->expectException(RuntimeException::class); + $this->expectExceptionMessage('does not match flow-php/etl'); + + new AdaptiveParquetOpenSource($file, new FlowPhpExtension(true, FlowPhpExtension::ABI + 1, '0.46.0')); + } +} diff --git a/src/adapter/etl-adapter-parquet/tests/Flow/ETL/Adapter/Parquet/Tests/Unit/FlowToParquetSchemaTest.php b/src/adapter/etl-adapter-parquet/tests/Flow/ETL/Adapter/Parquet/Tests/Unit/FlowToParquetSchemaTest.php index c1f41b9f04..8fce623443 100644 --- a/src/adapter/etl-adapter-parquet/tests/Flow/ETL/Adapter/Parquet/Tests/Unit/FlowToParquetSchemaTest.php +++ b/src/adapter/etl-adapter-parquet/tests/Flow/ETL/Adapter/Parquet/Tests/Unit/FlowToParquetSchemaTest.php @@ -38,7 +38,7 @@ public function test_convert_etl_entries_to_parquet_fields(): void FlatColumn::int64('integer', ParquetSchema\Repetition::REQUIRED), FlatColumn::boolean('boolean', ParquetSchema\Repetition::REQUIRED), FlatColumn::string('string', ParquetSchema\Repetition::REQUIRED), - FlatColumn::float('float', ParquetSchema\Repetition::REQUIRED), + FlatColumn::double('float', ParquetSchema\Repetition::REQUIRED), FlatColumn::dateTime('datetime', ParquetSchema\Repetition::REQUIRED), FlatColumn::json('json', ParquetSchema\Repetition::REQUIRED), NestedColumn::list('list', ParquetSchema\ListElement::string(true), ParquetSchema\Repetition::REQUIRED), diff --git a/src/adapter/etl-adapter-parquet/tests/Flow/ETL/Adapter/Parquet/Tests/Unit/ParquetEncoderTest.php b/src/adapter/etl-adapter-parquet/tests/Flow/ETL/Adapter/Parquet/Tests/Unit/ParquetEncoderTest.php index b643b39875..3a1440637b 100644 --- a/src/adapter/etl-adapter-parquet/tests/Flow/ETL/Adapter/Parquet/Tests/Unit/ParquetEncoderTest.php +++ b/src/adapter/etl-adapter-parquet/tests/Flow/ETL/Adapter/Parquet/Tests/Unit/ParquetEncoderTest.php @@ -6,7 +6,6 @@ use DateTimeImmutable; use Flow\ETL\Adapter\Parquet\ParquetEncoder; -use Flow\ETL\Row\TypedRowValues; use Flow\ETL\Tests\FlowTestCase; use Flow\Parquet\ParquetFile\Schema as ParquetSchema; use Flow\Parquet\ParquetFile\Schema\FlatColumn; @@ -17,7 +16,15 @@ use Flow\Types\Value\Json; use Flow\Types\Value\Uuid; -use function Flow\Types\DSL\type_datetime; +use function Flow\ETL\DSL\array_to_rows; +use function Flow\ETL\DSL\datetime_schema; +use function Flow\ETL\DSL\int_schema; +use function Flow\ETL\DSL\list_schema; +use function Flow\ETL\DSL\map_schema; +use function Flow\ETL\DSL\schema; +use function Flow\ETL\DSL\str_schema; +use function Flow\ETL\DSL\structure_schema; +use function Flow\ETL\DSL\uuid_schema; use function Flow\Types\DSL\type_integer; use function Flow\Types\DSL\type_json; use function Flow\Types\DSL\type_list; @@ -28,248 +35,127 @@ final class ParquetEncoderTest extends FlowTestCase { - public function test_decode_leaves_null_logical_values_untouched(): void - { - static::assertSame( - ['id' => null, 'payload' => null], - (new ParquetEncoder(ParquetSchema::with(FlatColumn::uuid('id'), FlatColumn::json('payload'))))->decode([[ - 'id' => null, - 'payload' => null, - ]])[0]->values, - ); - } - - public function test_decode_normalizes_json_columns_to_flow_native(): void - { - $decoded = (new ParquetEncoder(ParquetSchema::with(FlatColumn::json('payload'))))->decode([[ - 'payload' => '{"id":1,"status":"NEW"}', - ]]); - - static::assertInstanceOf(Json::class, $decoded[0]->values['payload']); - static::assertSame(['id' => 1, 'status' => 'NEW'], $decoded[0]->values['payload']->toArray()); - } - - public function test_decode_normalizes_uuid_columns_to_flow_native(): void - { - $decoded = (new ParquetEncoder(ParquetSchema::with(FlatColumn::uuid('id'))))->decode([[ - 'id' => 'f47ac10b-58cc-4372-a567-0e02b2c3d479', - ]]); - - static::assertInstanceOf(Uuid::class, $decoded[0]->values['id']); - static::assertSame('f47ac10b-58cc-4372-a567-0e02b2c3d479', $decoded[0]->values['id']->toString()); - } - - public function test_decode_passes_scalar_value_maps_through_unchanged(): void - { - static::assertSame( - ['id' => 1, 'name' => 'Alice'], - (new ParquetEncoder(ParquetSchema::with(FlatColumn::int64('id'), FlatColumn::string('name'))))->decode([[ - 'id' => 1, - 'name' => 'Alice', - ]])[0]->values, - ); - } - - public function test_encode_coerces_values_to_the_column_type(): void - { - static::assertSame( - [['id' => '1', 'name' => 'test']], - (new ParquetEncoder(ParquetSchema::with( - FlatColumn::string('id'), - FlatColumn::string('name'), - )))->encode([new TypedRowValues(['id' => 1, 'name' => 'test'], [ - 'id' => type_integer(), - 'name' => type_string(), - ])]), - ); - } - - public function test_encode_keeps_datetime_and_scalar_values(): void + public function test_columns_keeps_datetime_and_scalar_values(): void { $at = new DateTimeImmutable('2024-01-01 12:00:00 UTC'); - static::assertSame( - [['id' => 1, 'at' => $at]], + static::assertEquals( + ['id' => [1], 'at' => [$at]], (new ParquetEncoder(ParquetSchema::with( FlatColumn::int64('id'), FlatColumn::datetime('at'), - )))->encode([new TypedRowValues(['id' => 1, 'at' => $at], [ - 'id' => type_integer(), - 'at' => type_datetime(), - ])]), + )))->columns(array_to_rows([['id' => 1, 'at' => $at]], schema(int_schema('id'), datetime_schema('at')))), ); } - public function test_encode_leaves_null_values_untouched(): void + public function test_columns_leaves_null_values_untouched(): void { static::assertSame( - [['id' => null, 'name' => null]], + ['id' => [null], 'name' => [null]], (new ParquetEncoder(ParquetSchema::with( FlatColumn::int64('id'), FlatColumn::string('name'), - )))->encode([new TypedRowValues(['id' => null, 'name' => null], [ - 'id' => type_integer(), - 'name' => type_string(), - ])]), + )))->columns(array_to_rows( + [['id' => null, 'name' => null]], + schema(int_schema('id', nullable: true), str_schema('name', nullable: true)), + )), ); } - public function test_encode_renders_uuid_values_back_to_strings(): void + public function test_columns_renders_uuid_values_back_to_strings(): void { static::assertSame( - [['id' => 'f47ac10b-58cc-4372-a567-0e02b2c3d479']], - (new ParquetEncoder(ParquetSchema::with(FlatColumn::uuid('id'))))->encode([new TypedRowValues([ + ['id' => ['f47ac10b-58cc-4372-a567-0e02b2c3d479']], + (new ParquetEncoder(ParquetSchema::with(FlatColumn::uuid('id'))))->columns(array_to_rows([[ 'id' => new Uuid('f47ac10b-58cc-4372-a567-0e02b2c3d479'), - ], ['id' => type_uuid()])]), - ); - } - - public function test_decode_normalizes_json_and_uuid_nested_in_struct(): void - { - $decoded = (new ParquetEncoder(ParquetSchema::with(NestedColumn::struct('body', [ - FlatColumn::json('data'), - FlatColumn::uuid('id'), - FlatColumn::int64('n'), - ]))))->decode([[ - 'body' => ['data' => '{"a":1}', 'id' => 'f47ac10b-58cc-4372-a567-0e02b2c3d479', 'n' => 1], - ]]); - - // @mago-ignore analysis:mixed-assignment - $body = $decoded[0]->values['body']; - - static::assertIsArray($body); - static::assertInstanceOf(Json::class, $body['data']); - static::assertInstanceOf(Uuid::class, $body['id']); - static::assertSame(1, $body['n']); - } - - public function test_decode_normalizes_json_in_deep_struct(): void - { - $decoded = (new ParquetEncoder(ParquetSchema::with(NestedColumn::struct('outer', [NestedColumn::struct('inner', [FlatColumn::json( - 'deep', - )])]))))->decode([['outer' => ['inner' => ['deep' => '{"e":5}']]]]); - - // @mago-ignore analysis:mixed-assignment - $outer = $decoded[0]->values['outer']; - - static::assertIsArray($outer); - // @mago-ignore analysis:mixed-assignment - $inner = $outer['inner']; - static::assertIsArray($inner); - static::assertInstanceOf(Json::class, $inner['deep']); - } - - public function test_decode_normalizes_json_list_elements(): void - { - $decoded = (new ParquetEncoder(ParquetSchema::with(NestedColumn::list( - 'items', - ListElement::json(), - ))))->decode([['items' => ['{"c":3}', '{"d":4}']]]); - - // @mago-ignore analysis:mixed-assignment - $items = $decoded[0]->values['items']; - - static::assertIsArray($items); - static::assertContainsOnlyInstancesOf(Json::class, $items); - } - - public function test_decode_normalizes_nested_null_leaves_untouched(): void - { - static::assertSame( - ['body' => ['data' => null, 'n' => 1]], - (new ParquetEncoder(ParquetSchema::with(NestedColumn::struct('body', [ - FlatColumn::json('data'), - FlatColumn::int64('n'), - ]))))->decode([['body' => ['data' => null, 'n' => 1]]])[0]->values, + ]], schema(uuid_schema('id')))), ); } - public function test_decode_normalizes_uuid_map_values_keeping_keys(): void - { - $decoded = (new ParquetEncoder(ParquetSchema::with(NestedColumn::map( - 'ids', - MapKey::string(), - MapValue::uuid(), - ))))->decode([['ids' => ['a' => 'f47ac10b-58cc-4372-a567-0e02b2c3d479']]]); - - // @mago-ignore analysis:mixed-assignment - $ids = $decoded[0]->values['ids']; - - static::assertIsArray($ids); - static::assertSame(['a'], array_keys($ids)); - static::assertInstanceOf(Uuid::class, $ids['a']); - } - - public function test_encode_stringifies_json_and_uuid_nested_in_struct(): void + public function test_columns_stringifies_json_and_uuid_nested_in_struct(): void { static::assertSame( - [['body' => ['data' => '{"a":1}', 'id' => 'f47ac10b-58cc-4372-a567-0e02b2c3d479', 'n' => 1]]], + ['body' => [['data' => '{"a":1}', 'id' => 'f47ac10b-58cc-4372-a567-0e02b2c3d479', 'n' => 1]]], (new ParquetEncoder(ParquetSchema::with(NestedColumn::struct('body', [ FlatColumn::json('data'), FlatColumn::uuid('id'), FlatColumn::int64('n'), - ]))))->encode([new TypedRowValues([ + ]))))->columns(array_to_rows([[ 'body' => [ 'data' => Json::fromArray(['a' => 1]), 'id' => new Uuid('f47ac10b-58cc-4372-a567-0e02b2c3d479'), 'n' => 1, ], - ], ['body' => type_structure([ + ]], schema(structure_schema('body', type_structure([ 'data' => type_json(), 'id' => type_uuid(), 'n' => type_integer(), - ])])]), + ]))))), ); } - public function test_encode_stringifies_json_in_deep_struct(): void + public function test_columns_stringifies_json_in_deep_struct(): void { static::assertSame( - [['outer' => ['inner' => ['deep' => '{"e":5}']]]], + ['outer' => [['inner' => ['deep' => '{"e":5}']]]], (new ParquetEncoder(ParquetSchema::with(NestedColumn::struct('outer', [NestedColumn::struct('inner', [FlatColumn::json( 'deep', - )])]))))->encode([new TypedRowValues([ + )])]))))->columns(array_to_rows([[ 'outer' => ['inner' => ['deep' => Json::fromArray(['e' => 5])]], - ], ['outer' => type_structure(['inner' => type_structure(['deep' => type_json()])])])]), + ]], schema(structure_schema('outer', type_structure(['inner' => type_structure(['deep' => type_json()])]))))), ); } - public function test_encode_stringifies_uuid_list_elements(): void + public function test_columns_stringifies_uuid_list_elements(): void { static::assertSame( - [['ids' => ['f47ac10b-58cc-4372-a567-0e02b2c3d479']]], + ['ids' => [['f47ac10b-58cc-4372-a567-0e02b2c3d479']]], (new ParquetEncoder(ParquetSchema::with(NestedColumn::list( 'ids', ListElement::uuid(), - ))))->encode([new TypedRowValues([ - 'ids' => [new Uuid('f47ac10b-58cc-4372-a567-0e02b2c3d479')], - ], ['ids' => type_list(type_uuid())])]), + ))))->columns(array_to_rows([['ids' => [new Uuid( + 'f47ac10b-58cc-4372-a567-0e02b2c3d479', + )]]], schema(list_schema('ids', type_list(type_uuid()))))), ); } - public function test_encode_stringifies_uuid_map_values_keeping_keys(): void + public function test_columns_stringifies_uuid_map_values_keeping_keys(): void { static::assertSame( - [['ids' => ['a' => 'f47ac10b-58cc-4372-a567-0e02b2c3d479']]], + ['ids' => [['a' => 'f47ac10b-58cc-4372-a567-0e02b2c3d479']]], (new ParquetEncoder(ParquetSchema::with(NestedColumn::map( 'ids', MapKey::string(), MapValue::uuid(), - ))))->encode([new TypedRowValues([ - 'ids' => ['a' => new Uuid('f47ac10b-58cc-4372-a567-0e02b2c3d479')], - ], ['ids' => type_map(type_string(), type_uuid())])]), + ))))->columns(array_to_rows([['ids' => [ + 'a' => new Uuid('f47ac10b-58cc-4372-a567-0e02b2c3d479'), + ]]], schema(map_schema('ids', type_map(type_string(), type_uuid()))))), ); } - public function test_encode_stringifies_uuids_in_list_of_structs(): void + public function test_columns_stringifies_uuids_in_list_of_structs(): void { static::assertSame( - [['items' => [['x' => 'f47ac10b-58cc-4372-a567-0e02b2c3d479']]]], + ['items' => [[['x' => 'f47ac10b-58cc-4372-a567-0e02b2c3d479']]]], (new ParquetEncoder(ParquetSchema::with(NestedColumn::list('items', ListElement::structure([FlatColumn::uuid( 'x', - )])))))->encode([new TypedRowValues([ + )])))))->columns(array_to_rows([[ 'items' => [['x' => new Uuid('f47ac10b-58cc-4372-a567-0e02b2c3d479')]], - ], ['items' => type_list(type_structure(['x' => type_uuid()]))])]), + ]], schema(list_schema('items', type_list(type_structure(['x' => type_uuid()])))))), + ); + } + + public function test_columns_convert_only_the_non_null_values_of_a_converted_column(): void + { + static::assertSame( + ['id' => ['f47ac10b-58cc-4372-a567-0e02b2c3d479', null], 'n' => [1, 2]], + (new ParquetEncoder(ParquetSchema::with( + FlatColumn::uuid('id'), + FlatColumn::int64('n'), + )))->columns(array_to_rows( + [['id' => new Uuid('f47ac10b-58cc-4372-a567-0e02b2c3d479'), 'n' => 1], ['id' => null, 'n' => 2]], + schema(uuid_schema('id', nullable: true), int_schema('n')), + )), ); } } diff --git a/src/adapter/etl-adapter-parquet/tests/Flow/ETL/Adapter/Parquet/Tests/Unit/ParquetExtractorTest.php b/src/adapter/etl-adapter-parquet/tests/Flow/ETL/Adapter/Parquet/Tests/Unit/ParquetExtractorTest.php index 928019bc73..f4883a2542 100644 --- a/src/adapter/etl-adapter-parquet/tests/Flow/ETL/Adapter/Parquet/Tests/Unit/ParquetExtractorTest.php +++ b/src/adapter/etl-adapter-parquet/tests/Flow/ETL/Adapter/Parquet/Tests/Unit/ParquetExtractorTest.php @@ -7,17 +7,25 @@ use Closure; use Flow\ETL\Adapter\Parquet\ParquetExtractor; use Flow\ETL\Cardinality; +use Flow\ETL\Exception\InvalidArgumentException; use Flow\ETL\Tests\Double\CountingFilesystem; +use Flow\ETL\Tests\Double\SpyBackend; use Flow\ETL\Tests\FlowTestCase; use Flow\Filesystem\Local\NativeLocalFilesystem; use Flow\Parquet\Binary\ByteOrder; +use Flow\Parquet\Exception\InvalidArgumentException as NotParquet; use Flow\Parquet\Options; use Generator; use PHPUnit\Framework\Attributes\DataProvider; use function Flow\ETL\Adapter\Parquet\from_parquet; +use function Flow\ETL\DSL\config_builder; +use function Flow\ETL\DSL\flow_context; use function Flow\ETL\DSL\int_schema; use function Flow\ETL\DSL\schema; +use function Flow\Filesystem\DSL\memory_filesystem; +use function Flow\Filesystem\DSL\path; +use function gc_collect_cycles; final class ParquetExtractorTest extends FlowTestCase { @@ -90,6 +98,26 @@ public function test_statistics_are_computed_at_most_once(): void ); static::assertSame($extractor->statistics(), $extractor->statistics()); + static::assertSame(1, $filesystem->readFromCalls); + static::assertSame(0, $filesystem->closedStreams()); + + unset($extractor); + gc_collect_cycles(); + + static::assertSame(1, $filesystem->closedStreams()); + } + + public function test_a_file_that_is_not_parquet_closes_the_stream_it_opened(): void + { + $filesystem = new CountingFilesystem(memory_filesystem()); + $filesystem->writeTo(path('memory://not.parquet'))->append('id,name')->close(); + + try { + from_parquet(path('memory://not.parquet'), filesystem: $filesystem)->statistics(); + static::fail('a CSV file was read as Parquet'); + } catch (NotParquet) { + } + static::assertSame(1, $filesystem->readFromCalls); static::assertSame(1, $filesystem->closedStreams()); } @@ -161,14 +189,26 @@ public function test_schema_then_statistics_reads_the_footer_once(): void static::assertSame(1, $filesystem->readFromCalls); } - public function test_a_declared_schema_still_reads_the_footer_for_statistics(): void + public function test_with_schema_is_refused(): void { - static::assertEquals( - Cardinality::exact(1000), - from_parquet(__DIR__ . '/../Integration/Fixtures/Pagination/01_1000.parquet') - ->withSchema(schema(int_schema('id'))) - ->statistics() - ->rows, + $this->expectException(InvalidArgumentException::class); + $this->expectExceptionMessage( + 'Parquet is a self-describing format and does not accept a schema; declaring one is not supported yet.', ); + + from_parquet(__DIR__ . '/../Integration/Fixtures/Pagination/01_1000.parquet')->withSchema(schema(int_schema( + 'id', + ))); + } + + public function test_extract_builds_through_the_config_backend(): void + { + $backend = new SpyBackend(); + + from_parquet(__DIR__ . '/../Integration/Fixtures/Pagination/01_1000.parquet') + ->extract(flow_context(config_builder()->backend($backend)->build())) + ->current(); + + static::assertGreaterThanOrEqual(1, $backend->builders() + $backend->adopts()); } } diff --git a/src/adapter/etl-adapter-parquet/tests/Flow/ETL/Adapter/Parquet/Tests/Unit/ParquetFileBatchesTest.php b/src/adapter/etl-adapter-parquet/tests/Flow/ETL/Adapter/Parquet/Tests/Unit/ParquetFileBatchesTest.php new file mode 100644 index 0000000000..2d91a62f77 --- /dev/null +++ b/src/adapter/etl-adapter-parquet/tests/Flow/ETL/Adapter/Parquet/Tests/Unit/ParquetFileBatchesTest.php @@ -0,0 +1,206 @@ + array_to_rows([ + ['id' => 1], + ['id' => 2], + ], schema(int_schema('id')))]); + $opener = new ParquetSourceFileOpener( + $filesystem, + new PhpParquetEngine(), + Options::default(), + new SchemaConverter(), + [], + ); + $file = (new ParquetFileBatches($opener, FileColumnsContext::discovering(names: []), null, null))->batches( + new SourceFile(path('memory://a.parquet')), + schema(int_schema('id', nullable: true)), + 10, + new PhpBackend(), + new ReadWindow(2), + ); + + static::assertSame([], iterator_to_array($file, false)); + static::assertSame(2, $file->getReturn()); + } + + public function test_an_offset_inside_the_file_starts_the_read_there_and_is_consumed(): void + { + $filesystem = memory_filesystem(); + ParquetFilesContext::write($filesystem, ['memory://a.parquet' => array_to_rows([ + ['id' => 1], + ['id' => 2], + ['id' => 3], + ], schema(int_schema('id')))]); + $opener = new ParquetSourceFileOpener( + $filesystem, + new PhpParquetEngine(), + Options::default(), + new SchemaConverter(), + [], + ); + $file = (new ParquetFileBatches($opener, FileColumnsContext::discovering(names: []), null, null))->batches( + new SourceFile(path('memory://a.parquet')), + schema(int_schema('id', nullable: true)), + 10, + new PhpBackend(), + new ReadWindow(1), + ); + + static::assertSame( + [[['id' => 2], ['id' => 3]]], + array_map(static fn(Rows $rows): array => $rows->toArray(), iterator_to_array($file, false)), + ); + static::assertSame(1, $file->getReturn()); + } + + public function test_the_kept_first_file_is_read_instead_of_opened_again(): void + { + $filesystem = new CountingFilesystem(memory_filesystem()); + ParquetFilesContext::write($filesystem, ['memory://a.parquet' => array_to_rows([[ + 'id' => 1, + ]], schema(int_schema('id')))]); + $opener = new ParquetSourceFileOpener( + $filesystem, + new PhpParquetEngine(), + Options::default(), + new SchemaConverter(), + [], + ); + $kept = $opener->open(new SourceFile(path('memory://a.parquet'))); + $opened = $filesystem->readFromCalls; + + iterator_to_array((new ParquetFileBatches( + $opener, + FileColumnsContext::discovering(names: []), + $kept, + null, + ))->batches( + new SourceFile(path('memory://a.parquet')), + schema(int_schema('id', nullable: true)), + 10, + new PhpBackend(), + new ReadWindow(), + )); + + static::assertSame($opened, $filesystem->readFromCalls); + static::assertSame($filesystem->readFromCalls, $filesystem->closedStreams()); + } + + public function test_a_kept_first_file_of_another_path_is_closed_and_the_file_opened(): void + { + $filesystem = new CountingFilesystem(memory_filesystem()); + ParquetFilesContext::write($filesystem, [ + 'memory://a.parquet' => array_to_rows([['id' => 1]], schema(int_schema('id'))), + 'memory://b.parquet' => array_to_rows([['id' => 2]], schema(int_schema('id'))), + ]); + $opener = new ParquetSourceFileOpener( + $filesystem, + new PhpParquetEngine(), + Options::default(), + new SchemaConverter(), + [], + ); + $kept = $opener->open(new SourceFile(path('memory://a.parquet'))); + + $rows = iterator_to_array( + (new ParquetFileBatches($opener, FileColumnsContext::discovering(names: []), $kept, null))->batches( + new SourceFile(path('memory://b.parquet')), + schema(int_schema('id', nullable: true)), + 10, + new PhpBackend(), + new ReadWindow(), + ), + false, + ); + + static::assertSame([['id' => 2]], $rows[0]->toArray()); + static::assertSame($filesystem->readFromCalls, $filesystem->closedStreams()); + } + + public function test_close_releases_a_kept_first_file_the_read_never_reached(): void + { + $filesystem = new CountingFilesystem(memory_filesystem()); + ParquetFilesContext::write($filesystem, ['memory://a.parquet' => array_to_rows([[ + 'id' => 1, + ]], schema(int_schema('id')))]); + $opener = new ParquetSourceFileOpener( + $filesystem, + new PhpParquetEngine(), + Options::default(), + new SchemaConverter(), + [], + ); + $kept = $opener->open(new SourceFile(path('memory://a.parquet'))); + + (new ParquetFileBatches($opener, FileColumnsContext::discovering(names: []), $kept, null))->close(); + + static::assertSame($filesystem->readFromCalls, $filesystem->closedStreams()); + } + + public function test_a_file_that_diverges_from_the_derived_schema_is_refused_and_closed(): void + { + $filesystem = new CountingFilesystem(memory_filesystem()); + ParquetFilesContext::write($filesystem, ['memory://b.parquet' => array_to_rows([[ + 'name' => 'x', + ]], schema(str_schema('name')))]); + $opener = new ParquetSourceFileOpener( + $filesystem, + new PhpParquetEngine(), + Options::default(), + new SchemaConverter(), + [], + ); + + try { + iterator_to_array((new ParquetFileBatches( + $opener, + FileColumnsContext::discovering(names: []), + null, + new DerivedSchema(schema(int_schema('id', nullable: true)), 'memory://a.parquet'), + ))->batches( + new SourceFile(path('memory://b.parquet')), + schema(int_schema('id')), + 10, + new PhpBackend(), + new ReadWindow(), + )); + static::fail('a diverging file must be refused'); + } catch (InferredSchemaException) { + static::assertSame($filesystem->readFromCalls, $filesystem->closedStreams()); + } + } +} diff --git a/src/adapter/etl-adapter-parquet/tests/Flow/ETL/Adapter/Parquet/Tests/Unit/ParquetFileSinkTest.php b/src/adapter/etl-adapter-parquet/tests/Flow/ETL/Adapter/Parquet/Tests/Unit/ParquetFileSinkTest.php new file mode 100644 index 0000000000..4ff1478519 --- /dev/null +++ b/src/adapter/etl-adapter-parquet/tests/Flow/ETL/Adapter/Parquet/Tests/Unit/ParquetFileSinkTest.php @@ -0,0 +1,54 @@ +open(memory_filesystem()->writeTo(path('memory://a.parquet')), new PhpBackend()) + ->close(); + + static::assertSame(0, $engine->openedForWrite); + } + + public function test_the_first_write_opens_one_writer_for_every_write(): void + { + $filesystem = memory_filesystem(); + $engine = new RecordingParquetEngine(); + $sink = (new ParquetFileSinks( + null, + new SchemaConverter(), + Compressions::UNCOMPRESSED, + Options::default(), + $engine, + ))->open($filesystem->writeTo(path('memory://a.parquet')), new PhpBackend()); + + $sink->write(array_to_rows([['id' => 1]], schema(int_schema('id')))); + $sink->write(array_to_rows([['id' => 2]], schema(int_schema('id')))); + $sink->close(); + + static::assertSame(1, $engine->openedForWrite); + static::assertSame([['id' => 1], ['id' => 2]], ParquetFilesContext::values($filesystem, 'memory://a.parquet')); + } +} diff --git a/src/adapter/etl-adapter-parquet/tests/Flow/ETL/Adapter/Parquet/Tests/Unit/ParquetFileSinksTest.php b/src/adapter/etl-adapter-parquet/tests/Flow/ETL/Adapter/Parquet/Tests/Unit/ParquetFileSinksTest.php new file mode 100644 index 0000000000..a9e6a60da0 --- /dev/null +++ b/src/adapter/etl-adapter-parquet/tests/Flow/ETL/Adapter/Parquet/Tests/Unit/ParquetFileSinksTest.php @@ -0,0 +1,81 @@ +open($filesystem->writeTo(path('memory://a.parquet')), new PhpBackend()); + $second = $sinks->open($filesystem->writeTo(path('memory://b.parquet')), new PhpBackend()); + + $first->write(array_to_rows([['id' => 1, 'name' => 'a']], schema(int_schema('id'), str_schema('name')))); + $second->write(array_to_rows([['id' => 2]], schema(int_schema('id')))); + $first->close(); + $second->close(); + + static::assertSame(['id', 'name'], ParquetFilesContext::columnNames($filesystem, 'memory://b.parquet')); + static::assertSame( + [['id' => 2, 'name' => null]], + ParquetFilesContext::values($filesystem, 'memory://b.parquet'), + ); + } + + public function test_a_declared_schema_is_the_written_one(): void + { + $filesystem = memory_filesystem(); + $sink = (new ParquetFileSinks( + schema(int_schema('id', nullable: true), str_schema('note', nullable: true)), + new SchemaConverter(), + Compressions::UNCOMPRESSED, + Options::default(), + new PhpParquetEngine(), + ))->open($filesystem->writeTo(path('memory://a.parquet')), new PhpBackend()); + + $sink->write(array_to_rows([['id' => 1]], schema(int_schema('id')))); + $sink->close(); + + static::assertSame(['id', 'note'], ParquetFilesContext::columnNames($filesystem, 'memory://a.parquet')); + } + + public function test_a_writer_before_the_first_conform_is_refused(): void + { + $this->expectException(RuntimeException::class); + $this->expectExceptionMessage('The Parquet schema is fixed by the first conform()'); + + (new ParquetFileSinks( + null, + new SchemaConverter(), + Compressions::UNCOMPRESSED, + Options::default(), + new PhpParquetEngine(), + ))->writer(memory_filesystem()->writeTo(path('memory://a.parquet'))); + } +} diff --git a/src/adapter/etl-adapter-parquet/tests/Flow/ETL/Adapter/Parquet/Tests/Unit/ParquetLoaderTest.php b/src/adapter/etl-adapter-parquet/tests/Flow/ETL/Adapter/Parquet/Tests/Unit/ParquetLoaderTest.php index 0fa6c8f8f0..7bd214f679 100644 --- a/src/adapter/etl-adapter-parquet/tests/Flow/ETL/Adapter/Parquet/Tests/Unit/ParquetLoaderTest.php +++ b/src/adapter/etl-adapter-parquet/tests/Flow/ETL/Adapter/Parquet/Tests/Unit/ParquetLoaderTest.php @@ -5,14 +5,48 @@ namespace Flow\ETL\Adapter\Parquet\Tests\Unit; use Flow\ETL\Adapter\Parquet\ParquetLoader; +use Flow\ETL\Adapter\Parquet\Tests\Context\ParquetFilesContext; +use Flow\ETL\Filesystem\SaveMode; use Flow\Filesystem\Path\Option; use Flow\Filesystem\Path\Option\ContentType; use PHPUnit\Framework\TestCase; +use function Flow\ETL\Adapter\Parquet\to_parquet; +use function Flow\ETL\DSL\array_to_rows; +use function Flow\ETL\DSL\config; +use function Flow\ETL\DSL\flow_context; +use function Flow\ETL\DSL\int_schema; +use function Flow\ETL\DSL\schema; +use function Flow\ETL\DSL\str_schema; +use function Flow\Filesystem\DSL\memory_filesystem; use function Flow\Filesystem\DSL\path; +use function implode; +use function sort; final class ParquetLoaderTest extends TestCase { + public function test_a_second_run_writes_under_its_own_schema(): void + { + $context = flow_context(config()); + $memory = memory_filesystem(); + $loader = to_parquet(path('memory://runs/data.parquet'), filesystem: $memory)->saveMode(SaveMode::Append); + + $loader->load(array_to_rows([['id' => 1]], schema(int_schema('id'))), $context); + $loader->closure($context); + $loader->load(array_to_rows([['name' => 'a']], schema(str_schema('name'))), $context); + $loader->closure($context); + + $written = []; + + foreach ($memory->list(path('memory://runs/*.parquet')) as $file) { + $written[] = implode(',', ParquetFilesContext::columnNames($memory, $file->path->uri())); + } + + sort($written); + + static::assertSame(['id', 'name'], $written); + } + public function test_setting_content_type_on_path(): void { $loader = new ParquetLoader(path(__DIR__ . '/file.parquet')); diff --git a/src/adapter/etl-adapter-parquet/tests/Flow/ETL/Adapter/Parquet/Tests/Unit/ParquetSchemaConformanceTest.php b/src/adapter/etl-adapter-parquet/tests/Flow/ETL/Adapter/Parquet/Tests/Unit/ParquetSchemaConformanceTest.php new file mode 100644 index 0000000000..19229520fd --- /dev/null +++ b/src/adapter/etl-adapter-parquet/tests/Flow/ETL/Adapter/Parquet/Tests/Unit/ParquetSchemaConformanceTest.php @@ -0,0 +1,111 @@ +toParquet(schema( + str_schema('id'), + str_schema('name'), + ))))->conform( + array_to_rows( + [['id' => 1, 'name' => 'a'], ['id' => 2, 'name' => 'b']], + schema(int_schema('id'), str_schema('name')), + ), + new AdaptiveBackend(), + ); + + static::assertEquals(schema(str_schema('id'), str_schema('name')), $conformed->schema()); + static::assertSame(['1', '2'], $conformed->column('id')->values()); + } + + public function test_a_date_is_cast_to_a_datetime(): void + { + $conformed = (new ParquetSchemaConformance((new SchemaConverter())->toParquet(schema(datetime_schema( + 'at', + )))))->conform(array_to_rows([[ + 'at' => new DateTimeImmutable('2026-09-01'), + ]], schema(date_schema('at'))), new AdaptiveBackend()); + + static::assertEquals(schema(datetime_schema('at')), $conformed->schema()); + static::assertEquals([new DateTimeImmutable('2026-09-01 00:00:00 UTC')], $conformed->column('at')->values()); + } + + public function test_a_null_typed_column_is_cast_to_a_string_column(): void + { + $conformed = (new ParquetSchemaConformance((new SchemaConverter())->toParquet(schema(null_schema( + 'nothing', + )))))->conform(array_to_rows([['nothing' => null]], schema(null_schema('nothing'))), new AdaptiveBackend()); + + static::assertEquals(schema(str_schema('nothing', nullable: true)), $conformed->schema()); + static::assertSame([null], $conformed->column('nothing')->values()); + } + + public function test_a_matching_column_is_passed_on_as_it_is(): void + { + $rows = array_to_rows([['id' => 1, 'name' => 'a']], schema(int_schema('id'), str_schema('name'))); + + $conformed = (new ParquetSchemaConformance((new SchemaConverter())->toParquet(schema( + str_schema('id'), + str_schema('name'), + ))))->conform($rows, new AdaptiveBackend()); + + static::assertSame($rows->column('name'), $conformed->column('name')); + } + + public function test_a_zoned_datetime_column_is_passed_on_as_it_is(): void + { + $schema = schema( + datetime_schema('at', zone: 'Europe/Warsaw'), + list_schema('history', type_list(type_datetime('Asia/Tokyo'))), + ); + $rows = array_to_rows([[ + 'at' => new DateTimeImmutable('2026-09-01 10:00:00 UTC'), + 'history' => [new DateTimeImmutable('2026-09-01 11:00:00 UTC')], + ]], $schema); + + static::assertSame($rows, (new ParquetSchemaConformance((new SchemaConverter())->toParquet($schema)))->conform( + $rows, + new AdaptiveBackend(), + )); + } + + public function test_rows_matching_the_writer_schema_are_passed_on_as_they_are(): void + { + $rows = array_to_rows([['id' => 1]], schema(int_schema('id'))); + + static::assertSame($rows, (new ParquetSchemaConformance((new SchemaConverter())->toParquet(schema(int_schema( + 'id', + )))))->conform($rows, new AdaptiveBackend())); + } + + public function test_a_column_the_writer_does_not_know_is_passed_on_as_it_is(): void + { + $rows = array_to_rows([['id' => 1, 'extra' => 'x']], schema(int_schema('id'), str_schema('extra'))); + + static::assertSame($rows, (new ParquetSchemaConformance((new SchemaConverter())->toParquet(schema(int_schema( + 'id', + )))))->conform($rows, new AdaptiveBackend())); + } +} diff --git a/src/adapter/etl-adapter-parquet/tests/Flow/ETL/Adapter/Parquet/Tests/Unit/ParquetSourceFileOpenerTest.php b/src/adapter/etl-adapter-parquet/tests/Flow/ETL/Adapter/Parquet/Tests/Unit/ParquetSourceFileOpenerTest.php new file mode 100644 index 0000000000..4f917d8e52 --- /dev/null +++ b/src/adapter/etl-adapter-parquet/tests/Flow/ETL/Adapter/Parquet/Tests/Unit/ParquetSourceFileOpenerTest.php @@ -0,0 +1,57 @@ +open(new SourceFile(ParquetSourceFileContext::fixture())); + + static::assertEquals(schema(int_schema('id')), $file->schema()); + + $file->close(); + } + + public function test_a_stream_the_engine_refuses_is_closed(): void + { + $filesystem = new CountingFilesystem(new NativeLocalFilesystem()); + + try { + (new ParquetSourceFileOpener( + $filesystem, + new RefusingParquetEngine(), + Options::default(), + new SchemaConverter(), + [], + ))->open(new SourceFile(ParquetSourceFileContext::fixture())); + static::fail('the engine must refuse the stream'); + } catch (RuntimeException) { + static::assertSame(1, $filesystem->readFromCalls); + static::assertSame(1, $filesystem->closedStreams()); + } + } +} diff --git a/src/adapter/etl-adapter-parquet/tests/Flow/ETL/Adapter/Parquet/Tests/Unit/ParquetSourceFileTest.php b/src/adapter/etl-adapter-parquet/tests/Flow/ETL/Adapter/Parquet/Tests/Unit/ParquetSourceFileTest.php index 394a5f2515..2ffe2c9b15 100644 --- a/src/adapter/etl-adapter-parquet/tests/Flow/ETL/Adapter/Parquet/Tests/Unit/ParquetSourceFileTest.php +++ b/src/adapter/etl-adapter-parquet/tests/Flow/ETL/Adapter/Parquet/Tests/Unit/ParquetSourceFileTest.php @@ -5,7 +5,8 @@ namespace Flow\ETL\Adapter\Parquet\Tests\Unit; use Flow\ETL\Adapter\Parquet\Tests\Context\ParquetSourceFileContext; -use Flow\ETL\Extractor\SourceFile; +use Flow\ETL\Column\PhpBackend; +use Flow\ETL\Extractor\File\SourceFile; use Flow\ETL\Tests\Double\CountingFilesystem; use Flow\ETL\Tests\FlowTestCase; use Flow\Filesystem\Local\NativeLocalFilesystem; @@ -25,6 +26,21 @@ public function test_close_releases_the_stream(): void static::assertSame($filesystem->readFromCalls, $filesystem->closedStreams()); } + public function test_open_reads_the_batches_of_the_file(): void + { + $file = ParquetSourceFileContext::over(new NativeLocalFilesystem()); + $source = $file->open(); + $counts = []; + + foreach ($source->batches(schema(int_schema('id')), 400, 100, 850, new PhpBackend()) as $rows) { + $counts[] = $rows->count(); + } + + $source->close(); + + static::assertSame([400, 400, 50], $counts); + } + public function test_schema_comes_from_the_file_footer(): void { static::assertEquals( diff --git a/src/adapter/etl-adapter-parquet/tests/Flow/ETL/Adapter/Parquet/Tests/Unit/ParquetStoredTypeTest.php b/src/adapter/etl-adapter-parquet/tests/Flow/ETL/Adapter/Parquet/Tests/Unit/ParquetStoredTypeTest.php new file mode 100644 index 0000000000..36cfaf0e20 --- /dev/null +++ b/src/adapter/etl-adapter-parquet/tests/Flow/ETL/Adapter/Parquet/Tests/Unit/ParquetStoredTypeTest.php @@ -0,0 +1,54 @@ +normalize()), + ParquetStoredType::of(type_datetime('Europe/Warsaw')->normalize()), + ); + } + + public function test_a_nested_datetime_zone_is_not_stored(): void + { + static::assertSame( + ParquetStoredType::of( + type_structure([ + 'at' => type_list(type_datetime()), + 'by' => type_map(type_string(), type_datetime()), + ])->normalize(), + ), + ParquetStoredType::of( + type_structure([ + 'at' => type_list(type_datetime('America/New_York')), + 'by' => type_map(type_string(), type_datetime('Asia/Tokyo')), + ])->normalize(), + ), + ); + } + + public function test_other_types_keep_their_shape(): void + { + static::assertSame(type_time_zone()->normalize(), ParquetStoredType::of(type_time_zone()->normalize())); + static::assertNotSame( + ParquetStoredType::of(type_list(type_integer())->normalize()), + ParquetStoredType::of(type_list(type_string())->normalize()), + ); + } +} diff --git a/src/adapter/etl-adapter-parquet/tests/Flow/ETL/Adapter/Parquet/Tests/Unit/ParquetToFlowSchemaTest.php b/src/adapter/etl-adapter-parquet/tests/Flow/ETL/Adapter/Parquet/Tests/Unit/ParquetToFlowSchemaTest.php index b5c5bb3856..51d51ae828 100644 --- a/src/adapter/etl-adapter-parquet/tests/Flow/ETL/Adapter/Parquet/Tests/Unit/ParquetToFlowSchemaTest.php +++ b/src/adapter/etl-adapter-parquet/tests/Flow/ETL/Adapter/Parquet/Tests/Unit/ParquetToFlowSchemaTest.php @@ -54,6 +54,7 @@ public function test_converting_flat_fields_to_flow_schema(): void FlatColumn::dateTime('datetime'), FlatColumn::uuid('uuid'), FlatColumn::json('json'), + FlatColumn::enum('enum'), )); static::assertEquals( @@ -70,6 +71,7 @@ public function test_converting_flat_fields_to_flow_schema(): void datetime_schema('datetime', true), uuid_schema('uuid', true), json_schema('json', true), + str_schema('enum', true), ), $flowSchema, ); @@ -104,6 +106,7 @@ public function test_converting_struct_to_flow_schema(): void FlatColumn::uuid('uuid'), FlatColumn::string('name'), FlatColumn::boolean('active'), + FlatColumn::enum('color'), ]))); static::assertEquals( @@ -113,6 +116,7 @@ public function test_converting_struct_to_flow_schema(): void 'uuid' => type_optional(type_uuid()), 'name' => type_optional(type_string()), 'active' => type_optional(type_boolean()), + 'color' => type_optional(type_string()), ]), true, )), diff --git a/src/adapter/etl-adapter-parquet/tests/Flow/ETL/Adapter/Parquet/Tests/Unit/PhpParquetOpenSinkTest.php b/src/adapter/etl-adapter-parquet/tests/Flow/ETL/Adapter/Parquet/Tests/Unit/PhpParquetOpenSinkTest.php new file mode 100644 index 0000000000..7314563398 --- /dev/null +++ b/src/adapter/etl-adapter-parquet/tests/Flow/ETL/Adapter/Parquet/Tests/Unit/PhpParquetOpenSinkTest.php @@ -0,0 +1,49 @@ +toParquet($schema); + $sink = new PhpParquetOpenSink( + (new PhpParquetEngine())->openForWrite( + $filesystem->writeTo(path('memory://out.parquet')), + $parquetSchema, + Compressions::SNAPPY, + Options::default(), + ), + new ParquetEncoder($parquetSchema), + ); + + $sink->write(array_to_rows([['id' => 1, 'name' => 'a'], ['id' => 2, 'name' => null]], $schema)); + $sink->write(array_to_rows([['id' => 3, 'name' => 'c']], $schema)); + $sink->close(); + + static::assertSame( + [['id' => 1, 'name' => 'a'], ['id' => 2, 'name' => null], ['id' => 3, 'name' => 'c']], + ParquetFilesContext::phpEngineValues($filesystem, 'memory://out.parquet'), + ); + } +} diff --git a/src/adapter/etl-adapter-parquet/tests/Flow/ETL/Adapter/Parquet/Tests/Unit/PhpParquetOpenSourceTest.php b/src/adapter/etl-adapter-parquet/tests/Flow/ETL/Adapter/Parquet/Tests/Unit/PhpParquetOpenSourceTest.php new file mode 100644 index 0000000000..61e9591630 --- /dev/null +++ b/src/adapter/etl-adapter-parquet/tests/Flow/ETL/Adapter/Parquet/Tests/Unit/PhpParquetOpenSourceTest.php @@ -0,0 +1,74 @@ + $ids + */ + #[TestWith([null, null, [0, 1, 2, 3, 4, 5, 6, 7, 8, 9]])] + #[TestWith([3, 5, [3, 4, 5, 6, 7]])] + #[TestWith([6, null, [6, 7, 8, 9]])] + #[TestWith([0, 1, [0]])] + public function test_offset_and_limit_cross_row_groups(?int $offset, ?int $limit, array $ids): void + { + $filesystem = memory_filesystem(); + ParquetFilesContext::rowGroups($filesystem, 'memory://groups.parquet', 10, 4); + + $read = []; + + foreach ((new PhpParquetOpenSource((new Reader( + engine: new PhpParquetEngine(), + ))->readStream($filesystem->readFrom(path('memory://groups.parquet')))))->batches( + schema(int_schema('id')), + 3, + $offset, + $limit, + new PhpBackend(), + ) as $rows) { + static::assertLessThanOrEqual(3, $rows->count()); + $read = [...$read, ...$rows->column('id')->values()]; + } + + static::assertSame($ids, $read); + } + + public function test_batches_hold_the_schema_columns(): void + { + $filesystem = memory_filesystem(); + ParquetFilesContext::rowGroups($filesystem, 'memory://groups.parquet', 10, 4); + $source = new PhpParquetOpenSource((new Reader( + engine: new PhpParquetEngine(), + ))->readStream($filesystem->readFrom(path('memory://groups.parquet')))); + + $names = []; + + foreach ($source->batches(schema(str_schema('name')), 100, null, null, new PhpBackend()) as $rows) { + static::assertEquals(schema(str_schema('name')), $rows->schema()); + $names = [...$names, ...$rows->column('name')->values()]; + } + + $source->close(); + + static::assertSame(array_map(static fn(int $id): string => 'n' . $id, range(0, 9)), $names); + } +} diff --git a/src/adapter/etl-adapter-parquet/tests/Flow/ETL/Adapter/Parquet/Tests/Unit/RustParquetOpenSinkTest.php b/src/adapter/etl-adapter-parquet/tests/Flow/ETL/Adapter/Parquet/Tests/Unit/RustParquetOpenSinkTest.php new file mode 100644 index 0000000000..627e763458 --- /dev/null +++ b/src/adapter/etl-adapter-parquet/tests/Flow/ETL/Adapter/Parquet/Tests/Unit/RustParquetOpenSinkTest.php @@ -0,0 +1,87 @@ +write(array_to_rows( + [['id' => 1, 'name' => 'a', 'amount' => 0.14], ['id' => 2, 'name' => null, 'amount' => 1.5]], + $schema, + new PhpBackend(), + )); + $sink->close(); + + static::assertSame( + [['id' => 1, 'name' => 'a', 'amount' => 0.14], ['id' => 2, 'name' => null, 'amount' => 1.5]], + ParquetFilesContext::phpEngineValues($filesystem, 'memory://out.parquet'), + ); + } + + public function test_a_string_that_is_not_valid_utf8_is_refused_naming_column_and_row(): void + { + $schema = schema(str_schema('name')); + $sink = ParquetFilesContext::nativeSink(memory_filesystem(), 'memory://out.parquet', $schema); + + $this->expectException(RuntimeException::class); + $this->expectExceptionMessage( + 'Parquet column "name" row 1 holds a string that is not valid UTF-8; Parquet STRING columns require UTF-8', + ); + + $sink->write(array_to_rows([['name' => 'ok'], ['name' => "\xff\xfe"]], $schema, new PhpBackend())); + } + + public function test_an_exception_append_throws_surfaces_as_itself(): void + { + $filesystem = memory_filesystem(); + $schema = schema(int_schema('id')); + $sink = new RustParquetOpenSink( + new RustParquetFileWriter( + new FailingAppendDestinationStream($filesystem->writeTo(path('memory://out.parquet'))), + ArrowSchemaConverter::toExtension((new SchemaConverter())->toParquet($schema)), + Compressions::SNAPPY, + OptionsConverter::toExtension(Options::default()), + Options::default()->getInt(Option::ARROW_WRITE_BATCH_SIZE), + ), + ); + $sink->write(array_to_rows([['id' => 1]], $schema)); + + $this->expectException(FilesystemRuntimeException::class); + $this->expectExceptionMessage('Appending to "memory://out.parquet" failed'); + + $sink->close(); + } +} diff --git a/src/adapter/etl-adapter-parquet/tests/Flow/ETL/Adapter/Parquet/Tests/Unit/RustParquetOpenSourceTest.php b/src/adapter/etl-adapter-parquet/tests/Flow/ETL/Adapter/Parquet/Tests/Unit/RustParquetOpenSourceTest.php new file mode 100644 index 0000000000..dd5bfd380a --- /dev/null +++ b/src/adapter/etl-adapter-parquet/tests/Flow/ETL/Adapter/Parquet/Tests/Unit/RustParquetOpenSourceTest.php @@ -0,0 +1,123 @@ + $ids + */ + #[TestWith([null, null, [0, 1, 2, 3, 4, 5, 6, 7, 8, 9]])] + #[TestWith([3, 5, [3, 4, 5, 6, 7]])] + #[TestWith([6, null, [6, 7, 8, 9]])] + #[TestWith([0, 1, [0]])] + public function test_offset_and_limit_cross_row_groups(?int $offset, ?int $limit, array $ids): void + { + $filesystem = memory_filesystem(); + ParquetFilesContext::rowGroups($filesystem, 'memory://groups.parquet', 10, 4); + + $read = []; + + foreach ((new RustParquetOpenSource(new RustParquetFileReader($filesystem->readFrom(path( + 'memory://groups.parquet', + )))))->batches(schema(int_schema('id')), 3, $offset, $limit, new AdaptiveBackend()) as $rows) { + static::assertLessThanOrEqual(3, $rows->count()); + $read = [...$read, ...$rows->column('id')->values()]; + } + + static::assertSame($ids, $read); + } + + public function test_columns_are_adopted_by_the_backend(): void + { + $filesystem = memory_filesystem(); + ParquetFilesContext::rowGroups($filesystem, 'memory://groups.parquet', 10, 4); + $source = new RustParquetOpenSource(new RustParquetFileReader($filesystem->readFrom(path( + 'memory://groups.parquet', + )))); + + $names = []; + + foreach ($source->batches(schema(str_schema('name')), 100, null, null, new PhpBackend()) as $rows) { + static::assertEquals(schema(str_schema('name')), $rows->schema()); + static::assertInstanceOf(ScalarColumn::class, $rows->column('name')); + $names = [...$names, ...$rows->column('name')->values()]; + } + + $source->close(); + + static::assertSame(array_map(static fn(int $id): string => 'n' . $id, range(0, 9)), $names); + } + + public function test_a_second_batches_call_closes_the_first_reader(): void + { + $filesystem = memory_filesystem(); + ParquetFilesContext::rowGroups($filesystem, 'memory://groups.parquet', 10, 4); + $source = new RustParquetOpenSource(new RustParquetFileReader($filesystem->readFrom(path( + 'memory://groups.parquet', + )))); + $first = $source->batches(schema(int_schema('id')), 3, null, null, new AdaptiveBackend()); + $first->rewind(); + static::assertSame([0, 1, 2], $first->current()->column('id')->values()); + + $second = []; + + foreach ($source->batches(schema(int_schema('id')), 3, null, null, new AdaptiveBackend()) as $rows) { + $second = [...$second, ...$rows->column('id')->values()]; + } + + $first->next(); + + static::assertFalse($first->valid()); + static::assertSame(range(0, 9), $second); + } + + public function test_close_before_batches_is_harmless(): void + { + $filesystem = memory_filesystem(); + ParquetFilesContext::rowGroups($filesystem, 'memory://groups.parquet', 10, 4); + + (new RustParquetOpenSource(new RustParquetFileReader($filesystem->readFrom(path( + 'memory://groups.parquet', + )))))->close(); + + $this->expectNotToPerformAssertions(); + } + + public function test_an_exception_the_stream_throws_while_opening_surfaces_as_itself(): void + { + $filesystem = memory_filesystem(); + ParquetFilesContext::rowGroups($filesystem, 'memory://groups.parquet', 10, 4); + + $this->expectException(RuntimeException::class); + $this->expectExceptionMessage('Reading "memory://groups.parquet" failed'); + + new RustParquetOpenSource(new RustParquetFileReader(new FailingReadSourceStream($filesystem->readFrom(path( + 'memory://groups.parquet', + ))))); + } +} diff --git a/src/adapter/etl-adapter-parquet/tests/Flow/ETL/Adapter/Parquet/Tests/Unit/ValueConverter/ElementsValueConverterTest.php b/src/adapter/etl-adapter-parquet/tests/Flow/ETL/Adapter/Parquet/Tests/Unit/ValueConverter/ElementsValueConverterTest.php index 21676372e4..2c387da1ee 100644 --- a/src/adapter/etl-adapter-parquet/tests/Flow/ETL/Adapter/Parquet/Tests/Unit/ValueConverter/ElementsValueConverterTest.php +++ b/src/adapter/etl-adapter-parquet/tests/Flow/ETL/Adapter/Parquet/Tests/Unit/ValueConverter/ElementsValueConverterTest.php @@ -13,30 +13,6 @@ final class ElementsValueConverterTest extends FlowTestCase { - public function test_decode_converts_each_list_element(): void - { - // @mago-ignore analysis:mixed-assignment - $decoded = (new ElementsValueConverter(new UuidValueConverter()))->decode([ - 'f6d6e0e8-4b7e-4b0e-8d7a-ff0a0c9c9a5a', - null, - ]); - - static::assertIsArray($decoded); - static::assertInstanceOf(Uuid::class, $decoded[0]); - static::assertNull($decoded[1]); - } - - public function test_decode_converts_keyed_values_keeping_keys(): void - { - // @mago-ignore analysis:mixed-assignment - $decoded = (new ElementsValueConverter(new JsonValueConverter()))->decode(['a' => '{"x":1}', 'b' => null]); - - static::assertIsArray($decoded); - static::assertSame(['a', 'b'], array_keys($decoded)); - static::assertInstanceOf(Json::class, $decoded['a']); - static::assertNull($decoded['b']); - } - public function test_encode_stringifies_each_list_element(): void { static::assertSame( @@ -60,7 +36,6 @@ public function test_non_array_values_pass_through(): void { $converter = new ElementsValueConverter(new UuidValueConverter()); - static::assertNull($converter->decode(null)); static::assertSame('scalar', $converter->encode('scalar')); } } diff --git a/src/adapter/etl-adapter-parquet/tests/Flow/ETL/Adapter/Parquet/Tests/Unit/ValueConverter/JsonValueConverterTest.php b/src/adapter/etl-adapter-parquet/tests/Flow/ETL/Adapter/Parquet/Tests/Unit/ValueConverter/JsonValueConverterTest.php index ef549eee68..1ec807098c 100644 --- a/src/adapter/etl-adapter-parquet/tests/Flow/ETL/Adapter/Parquet/Tests/Unit/ValueConverter/JsonValueConverterTest.php +++ b/src/adapter/etl-adapter-parquet/tests/Flow/ETL/Adapter/Parquet/Tests/Unit/ValueConverter/JsonValueConverterTest.php @@ -10,20 +10,6 @@ final class JsonValueConverterTest extends FlowTestCase { - public function test_decode_casts_string_to_json(): void - { - // @mago-ignore analysis:mixed-assignment - $decoded = (new JsonValueConverter())->decode('{"a":1}'); - - static::assertInstanceOf(Json::class, $decoded); - static::assertSame(['a' => 1], $decoded->toArray()); - } - - public function test_decode_passes_null_through(): void - { - static::assertNull((new JsonValueConverter())->decode(null)); - } - public function test_encode_passes_scalar_through(): void { static::assertSame('{"a":1}', (new JsonValueConverter())->encode('{"a":1}')); diff --git a/src/adapter/etl-adapter-parquet/tests/Flow/ETL/Adapter/Parquet/Tests/Unit/ValueConverter/StructValueConverterTest.php b/src/adapter/etl-adapter-parquet/tests/Flow/ETL/Adapter/Parquet/Tests/Unit/ValueConverter/StructValueConverterTest.php index e738069a1f..795375531c 100644 --- a/src/adapter/etl-adapter-parquet/tests/Flow/ETL/Adapter/Parquet/Tests/Unit/ValueConverter/StructValueConverterTest.php +++ b/src/adapter/etl-adapter-parquet/tests/Flow/ETL/Adapter/Parquet/Tests/Unit/ValueConverter/StructValueConverterTest.php @@ -11,27 +11,6 @@ final class StructValueConverterTest extends FlowTestCase { - public function test_decode_converts_only_declared_children(): void - { - // @mago-ignore analysis:mixed-assignment - $decoded = (new StructValueConverter(['data' => new JsonValueConverter()]))->decode([ - 'data' => '{"a":1}', - 'n' => 1, - ]); - - static::assertIsArray($decoded); - static::assertInstanceOf(Json::class, $decoded['data']); - static::assertSame(1, $decoded['n']); - } - - public function test_decode_passes_missing_and_null_children_through(): void - { - $converter = new StructValueConverter(['data' => new JsonValueConverter()]); - - static::assertSame([], $converter->decode([])); - static::assertSame(['data' => null], $converter->decode(['data' => null])); - } - public function test_encode_stringifies_child_objects(): void { static::assertSame( @@ -47,7 +26,6 @@ public function test_non_array_values_pass_through(): void { $converter = new StructValueConverter(['data' => new JsonValueConverter()]); - static::assertNull($converter->decode(null)); static::assertSame('scalar', $converter->encode('scalar')); } } diff --git a/src/adapter/etl-adapter-parquet/tests/Flow/ETL/Adapter/Parquet/Tests/Unit/ValueConverter/UuidValueConverterTest.php b/src/adapter/etl-adapter-parquet/tests/Flow/ETL/Adapter/Parquet/Tests/Unit/ValueConverter/UuidValueConverterTest.php index 617a064c19..568eaafabe 100644 --- a/src/adapter/etl-adapter-parquet/tests/Flow/ETL/Adapter/Parquet/Tests/Unit/ValueConverter/UuidValueConverterTest.php +++ b/src/adapter/etl-adapter-parquet/tests/Flow/ETL/Adapter/Parquet/Tests/Unit/ValueConverter/UuidValueConverterTest.php @@ -10,20 +10,6 @@ final class UuidValueConverterTest extends FlowTestCase { - public function test_decode_casts_canonical_string_to_uuid(): void - { - // @mago-ignore analysis:mixed-assignment - $decoded = (new UuidValueConverter())->decode('f6d6e0e8-4b7e-4b0e-8d7a-ff0a0c9c9a5a'); - - static::assertInstanceOf(Uuid::class, $decoded); - static::assertSame('f6d6e0e8-4b7e-4b0e-8d7a-ff0a0c9c9a5a', $decoded->toString()); - } - - public function test_decode_passes_null_through(): void - { - static::assertNull((new UuidValueConverter())->decode(null)); - } - public function test_encode_passes_scalar_through(): void { static::assertSame( diff --git a/src/adapter/etl-adapter-postgresql/src/Flow/ETL/Adapter/PostgreSql/PostgreSqlCursorExtractor.php b/src/adapter/etl-adapter-postgresql/src/Flow/ETL/Adapter/PostgreSql/PostgreSqlCursorExtractor.php index 6575ba692a..afe288c2d1 100644 --- a/src/adapter/etl-adapter-postgresql/src/Flow/ETL/Adapter/PostgreSql/PostgreSqlCursorExtractor.php +++ b/src/adapter/etl-adapter-postgresql/src/Flow/ETL/Adapter/PostgreSql/PostgreSqlCursorExtractor.php @@ -14,6 +14,7 @@ use Flow\ETL\Extractor\Statistics; use Flow\ETL\FlowContext; use Flow\ETL\Rows; +use Flow\ETL\Rows\RowsBuilder; use Flow\ETL\Schema; use Flow\PostgreSql\Client\Client; use Flow\PostgreSql\QueryBuilder\Sql; @@ -72,7 +73,6 @@ public function extract(FlowContext $context, ?int $limit = null): Generator { $read = $this->read ??= ReadQuery::of($this->query, self::class); - $encoder = new PostgreSqlEncoder(); $cursorName = $this->cursorName ?? 'flow_cursor_' . bin2hex(random_bytes(8)); $schema = $this->schema(); @@ -119,11 +119,13 @@ public function extract(FlowContext $context, ?int $limit = null): Generator $cursor->free(); - $hydrated = $context->hydrator()->hydrate($encoder->decode($rawBatch), $schema); + $rows = (new RowsBuilder($schema, $context->backend())) + ->appendRows($rawBatch) + ->finish(); - $yielded += $hydrated->count(); + $yielded += $rows->count(); - $signal = yield $hydrated; + $signal = yield $rows; if ($signal === Signal::STOP) { return; diff --git a/src/adapter/etl-adapter-postgresql/src/Flow/ETL/Adapter/PostgreSql/PostgreSqlEncoder.php b/src/adapter/etl-adapter-postgresql/src/Flow/ETL/Adapter/PostgreSql/PostgreSqlEncoder.php index fa6b5df279..ea4abb3d51 100644 --- a/src/adapter/etl-adapter-postgresql/src/Flow/ETL/Adapter/PostgreSql/PostgreSqlEncoder.php +++ b/src/adapter/etl-adapter-postgresql/src/Flow/ETL/Adapter/PostgreSql/PostgreSqlEncoder.php @@ -4,31 +4,60 @@ namespace Flow\ETL\Adapter\PostgreSql; -use Flow\ETL\Row\Encoder; -use Flow\ETL\Row\RawRowValues; +use Flow\ETL\Column\Column; +use Flow\ETL\Column\TextValues; +use Flow\ETL\Rows; +use Flow\Types\Type; -/** - * @implements Encoder> - */ -final class PostgreSqlEncoder implements Encoder +final readonly class PostgreSqlEncoder { - public function decode(array $batch): array + public function __construct( + private TextValues $text = new TextValues(), + ) {} + + /** + * @param Type $type + * + * @return list + */ + public function column(Type $type, Column $column): array + { + return $this->text->bindable($type, $column); + } + + /** + * Every column's values, in schema order, as `column()` gives them. + * + * @return array> + */ + public function columns(Rows $rows): array { - $decoded = []; + $columns = []; - foreach ($batch as $values) { - $decoded[] = new RawRowValues($values); + foreach ($rows->schema()->definitions() as $definition) { + $name = $definition->entry()->name(); + $columns[$name] = $this->column($definition->type(), $rows->column($name)); } - return $decoded; + return $columns; } - public function encode(array $batch): array + /** + * @return list> + */ + public function encode(Rows $rows): array { + $columns = $this->columns($rows); $encoded = []; - foreach ($batch as $rowValues) { - $encoded[] = $rowValues->values; + for ($i = 0, $count = $rows->count(); $i < $count; $i++) { + $row = []; + + foreach ($columns as $name => $column) { + $row[$name] = $column[$i]; + } + + $encoded[] = $row; } return $encoded; diff --git a/src/adapter/etl-adapter-postgresql/src/Flow/ETL/Adapter/PostgreSql/PostgreSqlKeySetExtractor.php b/src/adapter/etl-adapter-postgresql/src/Flow/ETL/Adapter/PostgreSql/PostgreSqlKeySetExtractor.php index 037cb6b9d2..eef53f2c94 100644 --- a/src/adapter/etl-adapter-postgresql/src/Flow/ETL/Adapter/PostgreSql/PostgreSqlKeySetExtractor.php +++ b/src/adapter/etl-adapter-postgresql/src/Flow/ETL/Adapter/PostgreSql/PostgreSqlKeySetExtractor.php @@ -17,6 +17,7 @@ use Flow\ETL\Extractor\Statistics; use Flow\ETL\FlowContext; use Flow\ETL\Rows; +use Flow\ETL\Rows\RowsBuilder; use Flow\ETL\Schema; use Flow\PostgreSql\Client\Client; use Flow\PostgreSql\QueryBuilder\Sql; @@ -72,7 +73,6 @@ public function extract(FlowContext $context, ?int $limit = null): Generator $schema = $this->schema(); - $encoder = new PostgreSqlEncoder(); $yielded = 0; $cursorValues = null; $maximum = match (true) { @@ -114,11 +114,13 @@ public function extract(FlowContext $context, ?int $limit = null): Generator break; } - $hydrated = $context->hydrator()->hydrate($encoder->decode($rawBatch), $schema); + $rows = (new RowsBuilder($schema, $context->backend())) + ->appendRows($rawBatch) + ->finish(); - $yielded += $hydrated->count(); + $yielded += $rows->count(); - $signal = yield $hydrated; + $signal = yield $rows; if ($signal === Signal::STOP) { return; diff --git a/src/adapter/etl-adapter-postgresql/src/Flow/ETL/Adapter/PostgreSql/PostgreSqlLimitOffsetExtractor.php b/src/adapter/etl-adapter-postgresql/src/Flow/ETL/Adapter/PostgreSql/PostgreSqlLimitOffsetExtractor.php index 8d3ea01930..a65551ebbf 100644 --- a/src/adapter/etl-adapter-postgresql/src/Flow/ETL/Adapter/PostgreSql/PostgreSqlLimitOffsetExtractor.php +++ b/src/adapter/etl-adapter-postgresql/src/Flow/ETL/Adapter/PostgreSql/PostgreSqlLimitOffsetExtractor.php @@ -14,6 +14,7 @@ use Flow\ETL\Extractor\Statistics; use Flow\ETL\FlowContext; use Flow\ETL\Rows; +use Flow\ETL\Rows\RowsBuilder; use Flow\ETL\Schema; use Flow\PostgreSql\Client\Client; use Flow\PostgreSql\QueryBuilder\Sql; @@ -77,7 +78,6 @@ public function extract(FlowContext $context, ?int $limit = null): Generator return; } - $encoder = new PostgreSqlEncoder(); $yielded = 0; $pages = (int) ceil($total / $this->batchSize); $pageSql = $read->page(count($this->parameters) + 1); @@ -100,11 +100,13 @@ public function extract(FlowContext $context, ?int $limit = null): Generator return; } - $hydrated = $context->hydrator()->hydrate($encoder->decode($rawBatch), $schema); + $rows = (new RowsBuilder($schema, $context->backend())) + ->appendRows($rawBatch) + ->finish(); - $yielded += $hydrated->count(); + $yielded += $rows->count(); - $signal = yield $hydrated; + $signal = yield $rows; if ($signal === Signal::STOP) { return; diff --git a/src/adapter/etl-adapter-postgresql/src/Flow/ETL/Adapter/PostgreSql/PostgreSqlLoader.php b/src/adapter/etl-adapter-postgresql/src/Flow/ETL/Adapter/PostgreSql/PostgreSqlLoader.php index 8b2c036de3..1ea413e573 100644 --- a/src/adapter/etl-adapter-postgresql/src/Flow/ETL/Adapter/PostgreSql/PostgreSqlLoader.php +++ b/src/adapter/etl-adapter-postgresql/src/Flow/ETL/Adapter/PostgreSql/PostgreSqlLoader.php @@ -60,9 +60,9 @@ public function load(Rows $rows, FlowContext $context): void try { match ($this->operation) { - Operation::INSERT => $this->insertRows($rows, $context), - Operation::UPDATE => $this->updateRows($rows, $context), - Operation::DELETE => $this->deleteRows($rows, $context), + Operation::INSERT => $this->insertRows($rows), + Operation::UPDATE => $this->updateRows($rows), + Operation::DELETE => $this->deleteRows($rows), }; $context->telemetry()->loadingCompleted($this, [TelemetryAttributes::ATTR_LOADING_ROWS => $rows->count()]); @@ -113,7 +113,7 @@ private function encoder(): PostgreSqlEncoder return $this->encoder ??= new PostgreSqlEncoder(); } - private function deleteRows(Rows $rows, FlowContext $context): void + private function deleteRows(Rows $rows): void { $deleteOptions = $this->deleteOptions; @@ -124,13 +124,13 @@ private function deleteRows(Rows $rows, FlowContext $context): void $builder = new DeleteQueryBuilder($this->table, $this->typesMap); $schema = $rows->schema(); - foreach ($this->encoder()->encode($context->hydrator()->dehydrate($rows)) as $values) { + foreach ($this->encoder()->encode($rows) as $values) { [$query, $params] = $builder->build($values, $schema, $deleteOptions); $this->client->execute($query, $params); } } - private function insertRows(Rows $rows, FlowContext $context): void + private function insertRows(Rows $rows): void { // PQ_QUERY_PARAM_MAX_LIMIT: an INSERT binds one parameter per column per row $maxRows = max(1, intdiv(65_535, $rows->schema()->count())); @@ -138,7 +138,8 @@ private function insertRows(Rows $rows, FlowContext $context): void if ($rows->count() <= $maxRows) { $builder = new InsertQueryBuilder($this->table, $this->typesMap); [$query, $params] = $builder->build( - $this->encoder()->encode($context->hydrator()->dehydrate($rows)), + $this->encoder()->columns($rows), + $rows->count(), $rows->schema(), $this->client->converters(), $this->insertOptions, @@ -159,7 +160,8 @@ private function insertRows(Rows $rows, FlowContext $context): void foreach ($rows->chunks($maxRows) as $chunk) { $builder = new InsertQueryBuilder($this->table, $this->typesMap); [$query, $params] = $builder->build( - $this->encoder()->encode($context->hydrator()->dehydrate($chunk)), + $this->encoder()->columns($chunk), + $chunk->count(), $chunk->schema(), $this->client->converters(), $this->insertOptions, @@ -179,7 +181,7 @@ private function insertRows(Rows $rows, FlowContext $context): void } } - private function updateRows(Rows $rows, FlowContext $context): void + private function updateRows(Rows $rows): void { $updateOptions = $this->updateOptions; @@ -190,7 +192,7 @@ private function updateRows(Rows $rows, FlowContext $context): void $builder = new UpdateQueryBuilder($this->table, $this->typesMap); $schema = $rows->schema(); - foreach ($this->encoder()->encode($context->hydrator()->dehydrate($rows)) as $values) { + foreach ($this->encoder()->encode($rows) as $values) { [$query, $params] = $builder->build($values, $schema, $updateOptions); if ($query !== null) { diff --git a/src/adapter/etl-adapter-postgresql/src/Flow/ETL/Adapter/PostgreSql/QueryBuilder/DeleteQueryBuilder.php b/src/adapter/etl-adapter-postgresql/src/Flow/ETL/Adapter/PostgreSql/QueryBuilder/DeleteQueryBuilder.php index f8ee1a6064..7f061ec346 100644 --- a/src/adapter/etl-adapter-postgresql/src/Flow/ETL/Adapter/PostgreSql/QueryBuilder/DeleteQueryBuilder.php +++ b/src/adapter/etl-adapter-postgresql/src/Flow/ETL/Adapter/PostgreSql/QueryBuilder/DeleteQueryBuilder.php @@ -28,7 +28,7 @@ public function __construct( ) {} /** - * @param array $value dehydrated value map for a single row + * @param array $value encoded value map for a single row * * @return array{Sql, list} */ diff --git a/src/adapter/etl-adapter-postgresql/src/Flow/ETL/Adapter/PostgreSql/QueryBuilder/InsertQueryBuilder.php b/src/adapter/etl-adapter-postgresql/src/Flow/ETL/Adapter/PostgreSql/QueryBuilder/InsertQueryBuilder.php index 1854361c94..51a634be94 100644 --- a/src/adapter/etl-adapter-postgresql/src/Flow/ETL/Adapter/PostgreSql/QueryBuilder/InsertQueryBuilder.php +++ b/src/adapter/etl-adapter-postgresql/src/Flow/ETL/Adapter/PostgreSql/QueryBuilder/InsertQueryBuilder.php @@ -6,6 +6,7 @@ use Flow\ETL\Adapter\PostgreSql\EntryTypesMap; use Flow\ETL\Adapter\PostgreSql\LoaderOptions\InsertOptions; +use Flow\ETL\Exception\InvalidArgumentException; use Flow\ETL\Schema; use Flow\PostgreSql\Client\ConvertedParameters; use Flow\PostgreSql\Client\Types\ValueConverters; @@ -19,6 +20,7 @@ use function Flow\PostgreSql\DSL\bulk_insert; use function Flow\PostgreSql\DSL\conflict_columns; use function Flow\PostgreSql\DSL\conflict_constraint; +use function sprintf; final readonly class InsertQueryBuilder { @@ -29,32 +31,47 @@ public function __construct( /** * Every value already in PostgreSQL's text form. A column's converter is resolved once, on its first non-null - * value - so a column of an unmapped type holding only nulls passes. + * value - so a column of an unmapped type holding only nulls passes. Parameters are bound row by row, read + * straight from the columns. * - * @param list> $values pre-sorted dehydrated value maps + * @param array> $columns every column's encoded values, $count of them each + * + * @throws InvalidArgumentException a column that does not hold $count values * * @return array{Sql, ConvertedParameters} */ public function build( - array $values, + array $columns, + int $count, Schema $schema, ValueConverters $converters, ?InsertOptions $options = null, ): array { - $columns = $values === [] ? [] : array_keys($values[0]); + foreach ($columns as $name => $values) { + if (count($values) !== $count) { + throw new InvalidArgumentException(sprintf( + 'Column "%s" holds %d values, the insert %d rows', + $name, + count($values), + $count, + )); + } + } + $names = $count === 0 ? [] : array_keys($columns); $params = []; $types = []; $columnConverters = []; - // every row of a gated batch carries the same columns, so each column's type is resolved once - foreach ($columns as $column) { + foreach ($names as $column) { $types[$column] = $schema->get(ref($column))->type(); } - foreach ($values as $row) { - /** @var mixed $value */ - foreach ($row as $column => $value) { + for ($i = 0; $i < $count; $i++) { + foreach ($names as $column) { + // @mago-ignore analysis:mixed-assignment + $value = $columns[$column][$i]; + if ($value === null) { $params[] = null; @@ -72,7 +89,7 @@ public function build( } } - return [$this->query($columns, count($values), $options), new ConvertedParameters($params)]; + return [$this->query($names, $count, $options), new ConvertedParameters($params)]; } /** diff --git a/src/adapter/etl-adapter-postgresql/src/Flow/ETL/Adapter/PostgreSql/QueryBuilder/UpdateQueryBuilder.php b/src/adapter/etl-adapter-postgresql/src/Flow/ETL/Adapter/PostgreSql/QueryBuilder/UpdateQueryBuilder.php index 93f5c241db..b33488c9ff 100644 --- a/src/adapter/etl-adapter-postgresql/src/Flow/ETL/Adapter/PostgreSql/QueryBuilder/UpdateQueryBuilder.php +++ b/src/adapter/etl-adapter-postgresql/src/Flow/ETL/Adapter/PostgreSql/QueryBuilder/UpdateQueryBuilder.php @@ -29,7 +29,7 @@ public function __construct( ) {} /** - * @param array $value dehydrated value map for a single row + * @param array $value encoded value map for a single row * * @return array{null|Sql, list} */ diff --git a/src/adapter/etl-adapter-postgresql/tests/Flow/ETL/Adapter/PostgreSql/Tests/Integration/PostgreSqlCursorExtractorIntegrationTest.php b/src/adapter/etl-adapter-postgresql/tests/Flow/ETL/Adapter/PostgreSql/Tests/Integration/PostgreSqlCursorExtractorIntegrationTest.php index e19129d272..cfba0971a6 100644 --- a/src/adapter/etl-adapter-postgresql/tests/Flow/ETL/Adapter/PostgreSql/Tests/Integration/PostgreSqlCursorExtractorIntegrationTest.php +++ b/src/adapter/etl-adapter-postgresql/tests/Flow/ETL/Adapter/PostgreSql/Tests/Integration/PostgreSqlCursorExtractorIntegrationTest.php @@ -361,7 +361,7 @@ public function test_schema_and_extract_agree(): void $extractor = from_pgsql_cursor($this->client, sprintf('SELECT id, name FROM %s ORDER BY id', $this->tableName)); $batch = df()->read($extractor)->fetch(); - static::assertSame($extractor->schema()->references()->names(), $batch->first()->names()); + static::assertSame($extractor->schema()->references()->names(), array_keys($batch->values(0))); static::assertTrue($batch->schema()->isSame($extractor->schema())); } diff --git a/src/adapter/etl-adapter-postgresql/tests/Flow/ETL/Adapter/PostgreSql/Tests/Integration/PostgreSqlKeySetExtractorIntegrationTest.php b/src/adapter/etl-adapter-postgresql/tests/Flow/ETL/Adapter/PostgreSql/Tests/Integration/PostgreSqlKeySetExtractorIntegrationTest.php index 6a3be16c4c..e0389772a5 100644 --- a/src/adapter/etl-adapter-postgresql/tests/Flow/ETL/Adapter/PostgreSql/Tests/Integration/PostgreSqlKeySetExtractorIntegrationTest.php +++ b/src/adapter/etl-adapter-postgresql/tests/Flow/ETL/Adapter/PostgreSql/Tests/Integration/PostgreSqlKeySetExtractorIntegrationTest.php @@ -312,7 +312,7 @@ public function test_schema_and_extract_agree(): void ); $batch = df()->read($extractor)->fetch(); - static::assertSame($extractor->schema()->references()->names(), $batch->first()->names()); + static::assertSame($extractor->schema()->references()->names(), array_keys($batch->values(0))); static::assertTrue($batch->schema()->isSame($extractor->schema())); } diff --git a/src/adapter/etl-adapter-postgresql/tests/Flow/ETL/Adapter/PostgreSql/Tests/Integration/PostgreSqlLimitOffsetExtractorIntegrationTest.php b/src/adapter/etl-adapter-postgresql/tests/Flow/ETL/Adapter/PostgreSql/Tests/Integration/PostgreSqlLimitOffsetExtractorIntegrationTest.php index 6a03a59dc7..4a2785cb1a 100644 --- a/src/adapter/etl-adapter-postgresql/tests/Flow/ETL/Adapter/PostgreSql/Tests/Integration/PostgreSqlLimitOffsetExtractorIntegrationTest.php +++ b/src/adapter/etl-adapter-postgresql/tests/Flow/ETL/Adapter/PostgreSql/Tests/Integration/PostgreSqlLimitOffsetExtractorIntegrationTest.php @@ -172,7 +172,7 @@ public function test_schema_and_extract_agree(): void ); $batch = df()->read($extractor)->fetch(); - static::assertSame($extractor->schema()->references()->names(), $batch->first()->names()); + static::assertSame($extractor->schema()->references()->names(), array_keys($batch->values(0))); static::assertTrue($batch->schema()->isSame($extractor->schema())); } diff --git a/src/adapter/etl-adapter-postgresql/tests/Flow/ETL/Adapter/PostgreSql/Tests/Integration/PostgreSqlLoaderAllTypesIntegrationTest.php b/src/adapter/etl-adapter-postgresql/tests/Flow/ETL/Adapter/PostgreSql/Tests/Integration/PostgreSqlLoaderAllTypesIntegrationTest.php index 1e239884b9..b70b081632 100644 --- a/src/adapter/etl-adapter-postgresql/tests/Flow/ETL/Adapter/PostgreSql/Tests/Integration/PostgreSqlLoaderAllTypesIntegrationTest.php +++ b/src/adapter/etl-adapter-postgresql/tests/Flow/ETL/Adapter/PostgreSql/Tests/Integration/PostgreSqlLoaderAllTypesIntegrationTest.php @@ -14,6 +14,7 @@ use function Flow\ETL\Adapter\PostgreSql\from_pgsql_limit_offset; use function Flow\ETL\Adapter\PostgreSql\to_pgsql_table; +use function Flow\ETL\DSL\array_to_rows; use function Flow\ETL\DSL\bool_schema; use function Flow\ETL\DSL\date_schema; use function Flow\ETL\DSL\datetime_schema; @@ -25,8 +26,6 @@ use function Flow\ETL\DSL\json_schema; use function Flow\ETL\DSL\list_schema; use function Flow\ETL\DSL\map_schema; -use function Flow\ETL\DSL\row; -use function Flow\ETL\DSL\rows; use function Flow\ETL\DSL\schema; use function Flow\ETL\DSL\str_schema; use function Flow\ETL\DSL\structure_schema; @@ -110,7 +109,26 @@ public function test_inserts_all_entry_types(): void $xmlElement = $xmlElementDoc->getElementsByTagName('item')->item(0); df() - ->read(from_rows(rows( + ->read(from_rows(array_to_rows( + [[ + 'col_string' => 'test string', + 'col_integer' => 42, + 'col_float' => 3.14159, + 'col_boolean' => true, + 'col_date' => $date, + 'col_datetime' => $dateTime, + 'col_time' => $time, + 'col_uuid' => $uuid, + 'col_json' => type_json()->cast(['key' => 'value', 'number' => 123]), + 'col_xml' => $xmlDoc, + 'col_xml_element' => $xmlElement, + 'col_html' => '

HTML content

', + 'col_html_element' => 'element', + 'col_enum' => BackedStringEnum::one, + 'col_list' => [1, 2, 3], + 'col_map' => ['a' => 1, 'b' => 2], + 'col_structure' => ['name' => 'John', 'age' => 30], + ]], schema( str_schema('col_string'), int_schema('col_integer'), @@ -133,25 +151,6 @@ enum_schema('col_enum', BackedStringEnum::class), 'age' => type_integer(), ])), ), - row([ - 'col_string' => 'test string', - 'col_integer' => 42, - 'col_float' => 3.14159, - 'col_boolean' => true, - 'col_date' => $date, - 'col_datetime' => $dateTime, - 'col_time' => $time, - 'col_uuid' => $uuid, - 'col_json' => type_json()->cast(['key' => 'value', 'number' => 123]), - 'col_xml' => $xmlDoc, - 'col_xml_element' => $xmlElement, - 'col_html' => '

HTML content

', - 'col_html_element' => 'element', - 'col_enum' => BackedStringEnum::one, - 'col_list' => [1, 2, 3], - 'col_map' => ['a' => 1, 'b' => 2], - 'col_structure' => ['name' => 'John', 'age' => 30], - ]), ))) ->write(to_pgsql_table($this->client, $this->tableName)) ->run(); @@ -225,7 +224,26 @@ enum_schema('col_enum', BackedStringEnum::class), public function test_inserts_null_values_for_all_entry_types(): void { df() - ->read(from_rows(rows( + ->read(from_rows(array_to_rows( + [[ + 'col_string' => null, + 'col_integer' => null, + 'col_float' => null, + 'col_boolean' => null, + 'col_date' => null, + 'col_datetime' => null, + 'col_time' => null, + 'col_uuid' => null, + 'col_json' => null, + 'col_xml' => null, + 'col_xml_element' => null, + 'col_html' => null, + 'col_html_element' => null, + 'col_enum' => null, + 'col_list' => null, + 'col_map' => null, + 'col_structure' => null, + ]], schema( str_schema('col_string', nullable: true), int_schema('col_integer', nullable: true), @@ -252,25 +270,6 @@ public function test_inserts_null_values_for_all_entry_types(): void nullable: true, ), ), - row([ - 'col_string' => null, - 'col_integer' => null, - 'col_float' => null, - 'col_boolean' => null, - 'col_date' => null, - 'col_datetime' => null, - 'col_time' => null, - 'col_uuid' => null, - 'col_json' => null, - 'col_xml' => null, - 'col_xml_element' => null, - 'col_html' => null, - 'col_html_element' => null, - 'col_enum' => null, - 'col_list' => null, - 'col_map' => null, - 'col_structure' => null, - ]), ))) ->write(to_pgsql_table($this->client, $this->tableName)) ->run(); diff --git a/src/adapter/etl-adapter-postgresql/tests/Flow/ETL/Adapter/PostgreSql/Tests/Unit/PostgreSqlCursorExtractorTest.php b/src/adapter/etl-adapter-postgresql/tests/Flow/ETL/Adapter/PostgreSql/Tests/Unit/PostgreSqlCursorExtractorTest.php index 90d4cc7ec9..b1f9eb39c7 100644 --- a/src/adapter/etl-adapter-postgresql/tests/Flow/ETL/Adapter/PostgreSql/Tests/Unit/PostgreSqlCursorExtractorTest.php +++ b/src/adapter/etl-adapter-postgresql/tests/Flow/ETL/Adapter/PostgreSql/Tests/Unit/PostgreSqlCursorExtractorTest.php @@ -11,6 +11,7 @@ use Flow\ETL\Exception\InvalidArgumentException; use Flow\ETL\Exception\SchemaMismatchException; use Flow\ETL\Exception\SchemaNotDerivableException; +use Flow\ETL\Tests\Double\SpyBackend; use Flow\ETL\Tests\FlowTestCase; use Flow\PostgreSql\AST\Transformers\ExplainConfig; use Flow\PostgreSql\Client\Exception\PostgreSqlError; @@ -19,6 +20,7 @@ use function array_map; use function Flow\ETL\Adapter\PostgreSql\from_pgsql_cursor; +use function Flow\ETL\DSL\config_builder; use function Flow\ETL\DSL\df; use function Flow\ETL\DSL\flow_context; use function Flow\ETL\DSL\int_schema; @@ -260,7 +262,7 @@ public function test_extract_casts_an_array_column_through_the_derived_schema(): $batches = iterator_to_array(from_pgsql_cursor($client, 'SELECT tags FROM t')->extract(flow_context())); - static::assertSame(['a', 'b'], $batches[0]->first()->get('tags')); + static::assertSame(['a', 'b'], $batches[0]->column('tags')->value(0)); } public function test_extract_casts_through_the_derived_schema(): void @@ -269,12 +271,10 @@ public function test_extract_casts_through_the_derived_schema(): void ->willDescribe(ColumnMother::of(['id' => 'int8', 'amount' => 'numeric'])) ->willReturnCursors(new StubCursor([['id' => '1', 'amount' => '10.5']])); - $row = iterator_to_array( - from_pgsql_cursor($client, 'SELECT id, amount FROM t')->extract(flow_context()), - )[0]->first(); + $rows = iterator_to_array(from_pgsql_cursor($client, 'SELECT id, amount FROM t')->extract(flow_context()))[0]; - static::assertSame(1, $row->get('id')); - static::assertSame(10.5, $row->get('amount')); + static::assertSame(1, $rows->column('id')->value(0)); + static::assertSame(10.5, $rows->column('amount')->value(0)); } public function test_extract_derives_the_schema_once_and_reuses_it_across_batches(): void @@ -454,4 +454,21 @@ public function test_a_maximum_bounds_and_caps_the_estimate_and_a_new_one_drops_ $extractor->statistics()->rows, ); } + + public function test_extract_builds_through_the_config_backend(): void + { + $client = (new SpyClient()) + ->willDescribe(ColumnMother::of(['id' => 'int8'])) + ->willReturnCursors(new StubCursor([['id' => '1']]), new StubCursor()); + $backend = new SpyBackend(); + + iterator_to_array( + from_pgsql_cursor($client, 'SELECT id FROM t')->extract( + flow_context(config_builder()->backend($backend)->build()), + ), + false, + ); + + static::assertGreaterThanOrEqual(1, $backend->builders()); + } } diff --git a/src/adapter/etl-adapter-postgresql/tests/Flow/ETL/Adapter/PostgreSql/Tests/Unit/PostgreSqlEncoderTest.php b/src/adapter/etl-adapter-postgresql/tests/Flow/ETL/Adapter/PostgreSql/Tests/Unit/PostgreSqlEncoderTest.php index 850314c656..1bc0d68cce 100644 --- a/src/adapter/etl-adapter-postgresql/tests/Flow/ETL/Adapter/PostgreSql/Tests/Unit/PostgreSqlEncoderTest.php +++ b/src/adapter/etl-adapter-postgresql/tests/Flow/ETL/Adapter/PostgreSql/Tests/Unit/PostgreSqlEncoderTest.php @@ -4,60 +4,69 @@ namespace Flow\ETL\Adapter\PostgreSql\Tests\Unit; +use DOMDocument; use Flow\ETL\Adapter\PostgreSql\PostgreSqlEncoder; -use Flow\ETL\Row\RawRowValues; -use Flow\ETL\Row\TypedRowValues; use Flow\ETL\Tests\FlowTestCase; -use function Flow\Types\DSL\type_boolean; -use function Flow\Types\DSL\type_integer; -use function Flow\Types\DSL\type_string; +use function Flow\ETL\DSL\array_to_rows; +use function Flow\ETL\DSL\bool_schema; +use function Flow\ETL\DSL\int_schema; +use function Flow\ETL\DSL\schema; +use function Flow\ETL\DSL\str_schema; +use function Flow\ETL\DSL\xml_element_schema; +use function Flow\ETL\DSL\xml_schema; final class PostgreSqlEncoderTest extends FlowTestCase { - public function test_decode_wraps_each_value_map_in_row_values(): void + public function test_encode_returns_each_row_as_its_value_map(): void { - $decoded = (new PostgreSqlEncoder())->decode([ - ['id' => 1, 'name' => 'Norbert'], - ['id' => 2, 'name' => null], - ]); - - static::assertEquals( - [new RawRowValues(['id' => 1, 'name' => 'Norbert']), new RawRowValues(['id' => 2, 'name' => null])], - $decoded, + static::assertSame( + [['id' => 1, 'name' => 'Norbert'], ['id' => 2, 'name' => null]], + (new PostgreSqlEncoder())->encode(array_to_rows( + [['id' => 1, 'name' => 'Norbert'], ['id' => 2, 'name' => null]], + schema(int_schema('id'), str_schema('name', nullable: true)), + )), ); } - public function test_encode_unwraps_each_typed_row_values_to_its_value_map(): void + public function test_encode_returns_the_original_value_maps(): void { - $types = ['id' => type_integer(), 'name' => type_string()]; - - $encoded = (new PostgreSqlEncoder())->encode([ - new TypedRowValues(['id' => 1, 'name' => 'Norbert'], $types), - new TypedRowValues(['id' => 2, 'name' => null], $types), - ]); - - static::assertSame([['id' => 1, 'name' => 'Norbert'], ['id' => 2, 'name' => null]], $encoded); + static::assertSame( + [['id' => 1, 'active' => true], ['id' => 2, 'active' => false]], + (new PostgreSqlEncoder())->encode(array_to_rows( + [['id' => 1, 'active' => true], ['id' => 2, 'active' => false]], + schema(int_schema('id'), bool_schema('active')), + )), + ); } - public function test_encode_returns_the_original_value_maps(): void + public function test_encode_of_an_empty_batch_returns_empty(): void { - $types = ['id' => type_integer(), 'active' => type_boolean()]; + static::assertSame([], (new PostgreSqlEncoder())->encode(array_to_rows([], schema(int_schema('id'))))); + } + public function test_columns_returns_every_columns_values_in_schema_order(): void + { static::assertSame( - [['id' => 1, 'active' => true], ['id' => 2, 'active' => false]], - (new PostgreSqlEncoder())->encode([ - new TypedRowValues(['id' => 1, 'active' => true], $types), - new TypedRowValues(['id' => 2, 'active' => false], $types), - ]), + ['id' => [1, 2], 'name' => ['a', null]], + (new PostgreSqlEncoder())->columns(array_to_rows( + [['id' => 1, 'name' => 'a'], ['id' => 2, 'name' => null]], + schema(int_schema('id'), str_schema('name', nullable: true)), + )), ); } - public function test_encode_and_decode_of_an_empty_batch_return_empty(): void + public function test_columns_render_markup_from_the_physicals(): void { - $encoder = new PostgreSqlEncoder(); + $document = new DOMDocument(); + $document->loadXML(''); - static::assertSame([], $encoder->decode([])); - static::assertSame([], $encoder->encode([])); + static::assertSame( + ['x' => [''], 'e' => ['']], + (new PostgreSqlEncoder())->columns(array_to_rows( + [['x' => $document, 'e' => $document->getElementsByTagName('a')->item(0)]], + schema(xml_schema('x'), xml_element_schema('e')), + )), + ); } } diff --git a/src/adapter/etl-adapter-postgresql/tests/Flow/ETL/Adapter/PostgreSql/Tests/Unit/PostgreSqlKeySetExtractorTest.php b/src/adapter/etl-adapter-postgresql/tests/Flow/ETL/Adapter/PostgreSql/Tests/Unit/PostgreSqlKeySetExtractorTest.php index 9876dd9593..53c65bde83 100644 --- a/src/adapter/etl-adapter-postgresql/tests/Flow/ETL/Adapter/PostgreSql/Tests/Unit/PostgreSqlKeySetExtractorTest.php +++ b/src/adapter/etl-adapter-postgresql/tests/Flow/ETL/Adapter/PostgreSql/Tests/Unit/PostgreSqlKeySetExtractorTest.php @@ -10,6 +10,7 @@ use Flow\ETL\Cardinality; use Flow\ETL\Exception\InvalidArgumentException; use Flow\ETL\Exception\SchemaNotDerivableException; +use Flow\ETL\Tests\Double\SpyBackend; use Flow\ETL\Tests\FlowTestCase; use Flow\PostgreSql\AST\Transformers\ExplainConfig; use Flow\PostgreSql\Client\Exception\PostgreSqlError; @@ -20,6 +21,7 @@ use function Flow\ETL\Adapter\PostgreSql\from_pgsql_key_set; use function Flow\ETL\Adapter\PostgreSql\pgsql_pagination_key_asc; use function Flow\ETL\Adapter\PostgreSql\pgsql_pagination_key_set; +use function Flow\ETL\DSL\config_builder; use function Flow\ETL\DSL\df; use function Flow\ETL\DSL\flow_context; use function Flow\ETL\DSL\int_schema; @@ -145,16 +147,16 @@ public function test_extract_casts_through_the_derived_schema(): void ->willDescribe(ColumnMother::of(['id' => 'int8', 'amount' => 'numeric'])) ->willReturnCursors(new StubCursor([['id' => '1', 'amount' => '10.5']]), new StubCursor()); - $row = iterator_to_array( + $rows = iterator_to_array( from_pgsql_key_set( $client, 'SELECT id, amount FROM t', pgsql_pagination_key_set(pgsql_pagination_key_asc('id')), )->extract(flow_context()), - )[0]->first(); + )[0]; - static::assertSame(1, $row->get('id')); - static::assertSame(10.5, $row->get('amount')); + static::assertSame(1, $rows->column('id')->value(0)); + static::assertSame(10.5, $rows->column('amount')->value(0)); } public function test_extract_derives_the_schema_once_and_reuses_it_across_batches(): void @@ -395,4 +397,23 @@ public function test_a_maximum_bounds_and_caps_the_estimate_and_a_new_one_drops_ $extractor->statistics()->rows, ); } + + public function test_extract_builds_through_the_config_backend(): void + { + $client = (new SpyClient()) + ->willDescribe(ColumnMother::of(['id' => 'int8'])) + ->willReturnCursors(new StubCursor([['id' => '1']]), new StubCursor()); + $backend = new SpyBackend(); + + iterator_to_array( + from_pgsql_key_set( + $client, + 'SELECT id FROM t', + pgsql_pagination_key_set(pgsql_pagination_key_asc('id')), + )->extract(flow_context(config_builder()->backend($backend)->build())), + false, + ); + + static::assertGreaterThanOrEqual(1, $backend->builders()); + } } diff --git a/src/adapter/etl-adapter-postgresql/tests/Flow/ETL/Adapter/PostgreSql/Tests/Unit/PostgreSqlLimitOffsetExtractorTest.php b/src/adapter/etl-adapter-postgresql/tests/Flow/ETL/Adapter/PostgreSql/Tests/Unit/PostgreSqlLimitOffsetExtractorTest.php index 7d1909e5ae..17b3e1d473 100644 --- a/src/adapter/etl-adapter-postgresql/tests/Flow/ETL/Adapter/PostgreSql/Tests/Unit/PostgreSqlLimitOffsetExtractorTest.php +++ b/src/adapter/etl-adapter-postgresql/tests/Flow/ETL/Adapter/PostgreSql/Tests/Unit/PostgreSqlLimitOffsetExtractorTest.php @@ -11,6 +11,7 @@ use Flow\ETL\Exception\InvalidArgumentException; use Flow\ETL\Exception\SchemaNotDerivableException; use Flow\ETL\Rows; +use Flow\ETL\Tests\Double\SpyBackend; use Flow\ETL\Tests\FlowTestCase; use Flow\PostgreSql\AST\Transformers\ExplainConfig; use Flow\PostgreSql\Client\Exception\PostgreSqlError; @@ -19,6 +20,7 @@ use function array_map; use function extension_loaded; use function Flow\ETL\Adapter\PostgreSql\from_pgsql_limit_offset; +use function Flow\ETL\DSL\config_builder; use function Flow\ETL\DSL\df; use function Flow\ETL\DSL\flow_context; use function Flow\ETL\DSL\int_schema; @@ -168,12 +170,12 @@ public function test_extract_casts_through_the_derived_schema(): void ->willCountTotal(1) ->willReturnCursors(new StubCursor([['id' => '1', 'amount' => '10.5']])); - $row = iterator_to_array( + $rows = iterator_to_array( from_pgsql_limit_offset($client, 'SELECT id, amount FROM t ORDER BY id')->extract(flow_context()), - )[0]->first(); + )[0]; - static::assertSame(1, $row->get('id')); - static::assertSame(10.5, $row->get('amount')); + static::assertSame(1, $rows->column('id')->value(0)); + static::assertSame(10.5, $rows->column('amount')->value(0)); } public function test_extract_derives_the_schema_once_and_reuses_it_across_batches(): void @@ -365,4 +367,22 @@ public function test_a_maximum_bounds_and_caps_the_estimate_and_a_new_one_drops_ $extractor->statistics()->rows, ); } + + public function test_extract_builds_through_the_config_backend(): void + { + $client = (new SpyClient()) + ->willDescribe(ColumnMother::of(['id' => 'int8'])) + ->willCountTotal(1) + ->willReturnCursors(new StubCursor([['id' => '1']])); + $backend = new SpyBackend(); + + iterator_to_array( + from_pgsql_limit_offset($client, 'SELECT id FROM t ORDER BY id')->extract( + flow_context(config_builder()->backend($backend)->build()), + ), + false, + ); + + static::assertGreaterThanOrEqual(1, $backend->builders()); + } } diff --git a/src/adapter/etl-adapter-postgresql/tests/Flow/ETL/Adapter/PostgreSql/Tests/Unit/PostgreSqlLoaderTest.php b/src/adapter/etl-adapter-postgresql/tests/Flow/ETL/Adapter/PostgreSql/Tests/Unit/PostgreSqlLoaderTest.php index 46614585a9..cf1773228c 100644 --- a/src/adapter/etl-adapter-postgresql/tests/Flow/ETL/Adapter/PostgreSql/Tests/Unit/PostgreSqlLoaderTest.php +++ b/src/adapter/etl-adapter-postgresql/tests/Flow/ETL/Adapter/PostgreSql/Tests/Unit/PostgreSqlLoaderTest.php @@ -17,9 +17,9 @@ use PHPUnit\Framework\MockObject\MockObject; use PHPUnit\Framework\TestCase; +use function Flow\ETL\DSL\array_to_rows; use function Flow\ETL\DSL\flow_context; use function Flow\ETL\DSL\int_schema; -use function Flow\ETL\DSL\row; use function Flow\ETL\DSL\rows; use function Flow\ETL\DSL\schema; @@ -36,7 +36,7 @@ public function test_load_delete_requires_delete_options(): void $this->expectException(RuntimeException::class); $this->expectExceptionMessage('DeleteOptions must be set for DELETE operation'); - $loader->load(rows(schema(int_schema('id')), row(['id' => 1])), flow_context()); + $loader->load(array_to_rows([['id' => 1]], schema(int_schema('id'))), flow_context()); } public function test_load_delete_requires_primary_keys(): void @@ -51,7 +51,7 @@ public function test_load_delete_requires_primary_keys(): void $this->expectException(RuntimeException::class); $this->expectExceptionMessage('Primary keys must be specified for DELETE operation'); - $loader->load(rows(schema(int_schema('id')), row(['id' => 1])), flow_context()); + $loader->load(array_to_rows([['id' => 1]], schema(int_schema('id'))), flow_context()); } public function test_load_update_requires_primary_keys(): void @@ -66,7 +66,7 @@ public function test_load_update_requires_primary_keys(): void $this->expectException(RuntimeException::class); $this->expectExceptionMessage('Primary keys must be specified for UPDATE operation'); - $loader->load(rows(schema(int_schema('id')), row(['id' => 1])), flow_context()); + $loader->load(array_to_rows([['id' => 1]], schema(int_schema('id'))), flow_context()); } public function test_load_update_requires_update_options(): void @@ -80,7 +80,7 @@ public function test_load_update_requires_update_options(): void $this->expectException(RuntimeException::class); $this->expectExceptionMessage('UpdateOptions must be set for UPDATE operation'); - $loader->load(rows(schema(int_schema('id')), row(['id' => 1])), flow_context()); + $loader->load(array_to_rows([['id' => 1]], schema(int_schema('id'))), flow_context()); } public function test_a_chunked_insert_joins_a_callers_transaction(): void diff --git a/src/adapter/etl-adapter-postgresql/tests/Flow/ETL/Adapter/PostgreSql/Tests/Unit/QueryBuilder/InsertQueryBuilderTest.php b/src/adapter/etl-adapter-postgresql/tests/Flow/ETL/Adapter/PostgreSql/Tests/Unit/QueryBuilder/InsertQueryBuilderTest.php index aebc74c042..a094c8193d 100644 --- a/src/adapter/etl-adapter-postgresql/tests/Flow/ETL/Adapter/PostgreSql/Tests/Unit/QueryBuilder/InsertQueryBuilderTest.php +++ b/src/adapter/etl-adapter-postgresql/tests/Flow/ETL/Adapter/PostgreSql/Tests/Unit/QueryBuilder/InsertQueryBuilderTest.php @@ -7,6 +7,7 @@ use Flow\ETL\Adapter\PostgreSql\EntryTypesMap; use Flow\ETL\Adapter\PostgreSql\LoaderOptions\InsertOptions; use Flow\ETL\Adapter\PostgreSql\QueryBuilder\InsertQueryBuilder; +use Flow\ETL\Exception\InvalidArgumentException; use Flow\PostgreSql\Client\Types\ValueConverters; use PHPUnit\Framework\TestCase; @@ -20,7 +21,8 @@ final class InsertQueryBuilderTest extends TestCase public function test_build_converts_each_value_with_its_columns_converter(): void { [$_query, $params] = (new InsertQueryBuilder('users', new EntryTypesMap()))->build( - [['id' => 1, 'name' => 'Alice', 'active' => true], ['id' => 2, 'name' => 'Bob', 'active' => false]], + ['id' => [1, 2], 'name' => ['Alice', 'Bob'], 'active' => [true, false]], + 2, schema(int_schema('id'), str_schema('name'), bool_schema('active')), ValueConverters::create(), ); @@ -31,7 +33,8 @@ public function test_build_converts_each_value_with_its_columns_converter(): voi public function test_build_simple_insert(): void { [$query, $params] = (new InsertQueryBuilder('users', new EntryTypesMap()))->build( - [['id' => 1, 'name' => 'Alice'], ['id' => 2, 'name' => 'Bob']], + ['id' => [1, 2], 'name' => ['Alice', 'Bob']], + 2, schema(int_schema('id'), str_schema('name')), ValueConverters::create(), ); @@ -45,7 +48,8 @@ public function test_build_simple_insert(): void public function test_build_with_null_values(): void { [$_query, $params] = (new InsertQueryBuilder('users', new EntryTypesMap()))->build( - [['id' => 1, 'name' => null]], + ['id' => [1], 'name' => [null]], + 1, schema(int_schema('id'), str_schema('name', nullable: true)), ValueConverters::create(), ); @@ -56,7 +60,8 @@ public function test_build_with_null_values(): void public function test_build_with_skip_conflicts(): void { [$query, $_params] = (new InsertQueryBuilder('users', new EntryTypesMap()))->build( - [['id' => 1, 'name' => 'Alice']], + ['id' => [1], 'name' => ['Alice']], + 1, schema(int_schema('id'), str_schema('name')), ValueConverters::create(), InsertOptions::skipConflicts(), @@ -68,7 +73,8 @@ public function test_build_with_skip_conflicts(): void public function test_build_with_upsert_on_columns(): void { [$query, $_params] = (new InsertQueryBuilder('users', new EntryTypesMap()))->build( - [['id' => 1, 'name' => 'Alice']], + ['id' => [1], 'name' => ['Alice']], + 1, schema(int_schema('id'), str_schema('name')), ValueConverters::create(), InsertOptions::upsertOnColumns(['id']), @@ -82,7 +88,8 @@ public function test_build_with_upsert_on_columns(): void public function test_build_with_upsert_on_constraint(): void { [$query, $_params] = (new InsertQueryBuilder('users', new EntryTypesMap()))->build( - [['id' => 1, 'name' => 'Alice']], + ['id' => [1], 'name' => ['Alice']], + 1, schema(int_schema('id'), str_schema('name')), ValueConverters::create(), InsertOptions::upsertOnConstraint('users_pkey'), @@ -92,4 +99,17 @@ public function test_build_with_upsert_on_constraint(): void static::assertStringContainsString('ON CONSTRAINT "users_pkey"', $sql); static::assertStringContainsString('DO UPDATE SET', $sql); } + + public function test_build_refuses_a_column_of_another_length(): void + { + $this->expectException(InvalidArgumentException::class); + $this->expectExceptionMessage('Column "name" holds 1 values, the insert 2 rows'); + + (new InsertQueryBuilder('users', new EntryTypesMap()))->build( + ['id' => [1, 2], 'name' => ['Alice']], + 2, + schema(int_schema('id'), str_schema('name')), + ValueConverters::create(), + ); + } } diff --git a/src/adapter/etl-adapter-seal/src/Flow/ETL/Adapter/Seal/SealEncoder.php b/src/adapter/etl-adapter-seal/src/Flow/ETL/Adapter/Seal/SealEncoder.php index 92c7c58b0a..b45895187c 100644 --- a/src/adapter/etl-adapter-seal/src/Flow/ETL/Adapter/Seal/SealEncoder.php +++ b/src/adapter/etl-adapter-seal/src/Flow/ETL/Adapter/Seal/SealEncoder.php @@ -4,16 +4,12 @@ namespace Flow\ETL\Adapter\Seal; -use DateInterval; use DateTimeInterface; -use DateTimeZone; -use Dom\Element; -use Dom\XMLDocument; -use DOMDocument; -use DOMElement; -use Flow\ETL\Exception\RuntimeException; -use Flow\ETL\Row\Encoder; -use Flow\ETL\Row\RawRowValues; +use Flow\ETL\Column\Column; +use Flow\ETL\Column\Physical\IdentityPhysical; +use Flow\ETL\Column\Physical\PhysicalFor; +use Flow\ETL\Column\TextValues; +use Flow\ETL\Rows; use Flow\Types\Type; use Flow\Types\Type\Logical\DateTimeType; use Flow\Types\Type\Logical\DateType; @@ -22,55 +18,48 @@ use Flow\Types\Type\Logical\MapType; use Flow\Types\Type\Logical\StructureType; use Flow\Types\Type\Logical\TimeType; -use Flow\Types\Type\Logical\TimeZoneType; -use Flow\Types\Type\Logical\UuidType; -use Flow\Types\Type\Logical\XMLElementType; -use Flow\Types\Type\Logical\XMLType; use Flow\Types\Type\Native\ArrayType; -use Flow\Types\Type\Native\EnumType; +use Flow\Types\Type\Native\NullType; use Flow\Types\Value\Json; -use Flow\Types\Value\Uuid; -use UnitEnum; use function array_keys; -use function Flow\ETL\DSL\date_interval_to_microseconds; +use function Flow\Types\DSL\type_bare; use function is_array; use function is_bool; use function is_float; use function is_int; use function is_string; -/** - * @implements Encoder> - */ -final class SealEncoder implements Encoder +final class SealEncoder { + private readonly TextValues $text; + public function __construct( private readonly string $dateTimeFormat = DateTimeInterface::ATOM, private readonly string $dateFormat = 'Y-m-d', - ) {} + ) { + $this->text = new TextValues($dateTimeFormat, $dateFormat); + } - public function decode(array $batch): array + /** + * @return list> + */ + public function encode(Rows $rows): array { - $decoded = []; + $columns = []; - foreach ($batch as $values) { - $decoded[] = new RawRowValues($values); + foreach ($rows->schema()->definitions() as $definition) { + $name = $definition->entry()->name(); + $columns[$name] = $this->column($definition->type(), $rows->column($name)); } - return $decoded; - } - - public function encode(array $batch): array - { $documents = []; - foreach ($batch as $rowValues) { + for ($i = 0, $count = $rows->count(); $i < $count; $i++) { $document = []; - /** @var mixed $value */ - foreach ($rowValues->values as $name => $value) { - $document[$name] = $this->renderValue($rowValues->types[$name], $value); + foreach ($columns as $name => $column) { + $document[$name] = $column[$i]; } $documents[] = $document; @@ -79,24 +68,65 @@ public function encode(array $batch): array return $documents; } - private function renderValue(Type $type, mixed $value): string|float|int|bool|array|null + /** + * A column's document fields: datetime / date text from the physicals, json and containers as normalized arrays, + * physicals where the physical is the value (scalars, a time's microseconds, null), TextValues for every other type. + * + * @param Type $type + * + * @return list + */ + public function column(Type $type, Column $column): array { - if ($value === null) { - return null; + $bare = type_bare($type); + + if ($bare instanceof DateTimeType || $bare instanceof DateType) { + /** @var list $physicals */ + $physicals = $column->physicals(); + + return $this->text->dateTimes( + $bare, + $physicals, + $bare instanceof DateType ? $this->dateFormat : $this->dateTimeFormat, + ); } - return match ($type::class) { - DateType::class => $value instanceof DateTimeInterface ? $value->format($this->dateFormat) : null, - DateTimeType::class => $value instanceof DateTimeInterface ? $value->format($this->dateTimeFormat) : null, - TimeType::class => $value instanceof DateInterval ? date_interval_to_microseconds($value) : null, - UuidType::class => $value instanceof Uuid ? $value->toString() : null, - TimeZoneType::class => $value instanceof DateTimeZone ? $value->getName() : null, - EnumType::class => $value instanceof UnitEnum ? $value->name : null, - XMLType::class, XMLElementType::class => $this->xmlToString($value), - JsonType::class => $value instanceof Json ? $this->normalizeArray($value->toArray()) : null, - ListType::class, MapType::class, StructureType::class, ArrayType::class => $this->normalizeArray($value), - default => $this->normalizeValue($value), - }; + if ($bare instanceof JsonType) { + $fields = []; + + // @mago-ignore analysis:mixed-assignment + foreach ($column->values() as $value) { + $fields[] = $value instanceof Json ? $this->normalizeArray($value->toArray()) : null; + } + + return $fields; + } + + if ( + $bare instanceof ListType + || $bare instanceof MapType + || $bare instanceof StructureType + || $bare instanceof ArrayType + ) { + $fields = []; + + // @mago-ignore analysis:mixed-assignment + foreach ($column->values() as $value) { + $fields[] = $this->normalizeArray($value); + } + + return $fields; + } + + if ( + $bare instanceof TimeType + || $bare instanceof NullType + || (new PhysicalFor())->type($bare) instanceof IdentityPhysical + ) { + return $column->physicals(); + } + + return $this->text->texts($type, $column->physicals()); } /** @@ -133,50 +163,4 @@ private function normalizeValue(mixed $value): string|float|int|bool|array|null return null; } - - private function xmlToString(mixed $value): ?string - { - if ($value instanceof XMLDocument) { - return $this->domString($value->saveXml($value->documentElement)); - } - - if ($value instanceof DOMDocument) { - return $this->domString($value->saveXML($value->documentElement)); - } - - if ($value instanceof Element || $value instanceof DOMElement) { - return $this->elementToString($value); - } - - return null; - } - - private function elementToString(Element|DOMElement $value): string - { - $ownerDocument = $value->ownerDocument; - - if ($ownerDocument === null) { - return ''; - } - - if ($ownerDocument instanceof XMLDocument) { - // @mago-ignore analysis:possibly-invalid-argument - return $this->domString($ownerDocument->saveXml($value)); - } - - /** @var false|string $serialized */ - // @mago-ignore analysis:possibly-invalid-argument,non-existent-method - $serialized = $ownerDocument->saveXML($value); - - return $this->domString($serialized); - } - - private function domString(string|false $serialized): string - { - if ($serialized === false) { - throw new RuntimeException('Failed to serialize XML document.'); - } - - return $serialized; - } } diff --git a/src/adapter/etl-adapter-seal/src/Flow/ETL/Adapter/Seal/SealLoader.php b/src/adapter/etl-adapter-seal/src/Flow/ETL/Adapter/Seal/SealLoader.php index 6498324d39..960f2bb508 100644 --- a/src/adapter/etl-adapter-seal/src/Flow/ETL/Adapter/Seal/SealLoader.php +++ b/src/adapter/etl-adapter-seal/src/Flow/ETL/Adapter/Seal/SealLoader.php @@ -47,10 +47,8 @@ public function load(Rows $rows, FlowContext $context): void try { $this->engine->bulk( $this->index, - $this->operation === Operation::UPSERT - ? $this->encoder()->encode($context->hydrator()->dehydrate($rows)) - : [], - $this->operation === Operation::DELETE ? $this->deleteIdentifiers($rows, $context) : [], + $this->operation === Operation::UPSERT ? $this->encoder()->encode($rows) : [], + $this->operation === Operation::DELETE ? $this->deleteIdentifiers($rows) : [], $this->bulkSize, ); @@ -98,9 +96,9 @@ private function encoder(): SealEncoder /** * @return Generator */ - private function deleteIdentifiers(Rows $rows, FlowContext $context): Generator + private function deleteIdentifiers(Rows $rows): Generator { - foreach ($this->encoder()->encode($context->hydrator()->dehydrate($rows)) as $document) { + foreach ($this->encoder()->encode($rows) as $document) { // @mago-ignore analysis:mixed-assignment $identifier = array_key_exists($this->identifierEntry, $document) ? $document[$this->identifierEntry] diff --git a/src/adapter/etl-adapter-seal/tests/Flow/ETL/Adapter/Seal/Tests/Integration/SealLoaderTest.php b/src/adapter/etl-adapter-seal/tests/Flow/ETL/Adapter/Seal/Tests/Integration/SealLoaderTest.php index c07fee7e47..124bab1ac1 100644 --- a/src/adapter/etl-adapter-seal/tests/Flow/ETL/Adapter/Seal/Tests/Integration/SealLoaderTest.php +++ b/src/adapter/etl-adapter-seal/tests/Flow/ETL/Adapter/Seal/Tests/Integration/SealLoaderTest.php @@ -9,11 +9,10 @@ use function Flow\ETL\Adapter\Seal\to_seal_delete; use function Flow\ETL\Adapter\Seal\to_seal_schema; use function Flow\ETL\Adapter\Seal\to_seal_upsert; +use function Flow\ETL\DSL\array_to_rows; use function Flow\ETL\DSL\flow_context; use function Flow\ETL\DSL\int_schema; use function Flow\ETL\DSL\integer_schema; -use function Flow\ETL\DSL\row; -use function Flow\ETL\DSL\rows; use function Flow\ETL\DSL\schema; use function Flow\ETL\DSL\str_schema; use function Flow\ETL\DSL\string_schema; @@ -29,16 +28,17 @@ public function test_deleting_documents_by_identifier(): void )); to_seal_upsert($engine, 'users')->load( - rows( + array_to_rows( + [['id' => '1', 'name' => 'Alice'], ['id' => '2', 'name' => 'Bob']], schema(string_schema('id'), string_schema('name')), - row(['id' => '1', 'name' => 'Alice']), - row(['id' => '2', 'name' => 'Bob']), ), flow_context(), ); $this->sealContext()->refresh(); - to_seal_delete($engine, 'users')->load(rows(schema(string_schema('id')), row(['id' => '1'])), flow_context()); + to_seal_delete($engine, 'users')->load(array_to_rows([[ + 'id' => '1', + ]], schema(string_schema('id'))), flow_context()); $this->sealContext()->refresh(); static::assertSame(1, $engine->countDocuments('users')); @@ -54,14 +54,17 @@ public function test_deleting_documents_using_a_custom_identifier_entry(): void )); to_seal_upsert($engine, 'products')->load( - rows(schema(string_schema('sku'), string_schema('name')), row(['sku' => 'SKU_0001', 'name' => 'Keyboard'])), + array_to_rows( + [['sku' => 'SKU_0001', 'name' => 'Keyboard']], + schema(string_schema('sku'), string_schema('name')), + ), flow_context(), ); $this->sealContext()->refresh(); to_seal_delete($engine, 'products') ->withIdentifierEntry('sku') - ->load(rows(schema(string_schema('sku')), row(['sku' => 'SKU_0001'])), flow_context()); + ->load(array_to_rows([['sku' => 'SKU_0001']], schema(string_schema('sku'))), flow_context()); $this->sealContext()->refresh(); static::assertSame(0, $engine->countDocuments('products')); @@ -78,13 +81,13 @@ public function test_loading_respects_a_custom_bulk_size(): void $documents = []; for ($i = 1; $i <= 10; $i++) { - $documents[] = row(['id' => (string) $i, 'name' => 'User ' . $i, 'age' => 20 + $i]); + $documents[] = ['id' => (string) $i, 'name' => 'User ' . $i, 'age' => 20 + $i]; } to_seal_upsert($engine, 'users') ->withBulkSize(3) ->load( - rows(schema(str_schema('id'), str_schema('name'), int_schema('age')), ...$documents), + array_to_rows($documents, schema(str_schema('id'), str_schema('name'), int_schema('age'))), flow_context(), ); $this->sealContext()->refresh(); @@ -101,10 +104,9 @@ public function test_loading_rows_into_a_seal_index(): void )); to_seal_upsert($engine, 'users')->load( - rows( + array_to_rows( + [['id' => '1', 'name' => 'Alice', 'age' => 30], ['id' => '2', 'name' => 'Bob', 'age' => 25]], schema(string_schema('id'), string_schema('name'), integer_schema('age')), - row(['id' => '1', 'name' => 'Alice', 'age' => 30]), - row(['id' => '2', 'name' => 'Bob', 'age' => 25]), ), flow_context(), ); @@ -124,16 +126,16 @@ public function test_loading_updates_a_document_with_the_same_identifier(): void $loader = to_seal_upsert($engine, 'users'); $loader->load( - rows( + array_to_rows( + [['id' => '1', 'name' => 'Alice', 'age' => 30]], schema(string_schema('id'), string_schema('name'), integer_schema('age')), - row(['id' => '1', 'name' => 'Alice', 'age' => 30]), ), flow_context(), ); $loader->load( - rows( + array_to_rows( + [['id' => '1', 'name' => 'Alice Updated', 'age' => 31]], schema(string_schema('id'), string_schema('name'), integer_schema('age')), - row(['id' => '1', 'name' => 'Alice Updated', 'age' => 31]), ), flow_context(), ); diff --git a/src/adapter/etl-adapter-seal/tests/Flow/ETL/Adapter/Seal/Tests/Unit/SealEncoderTest.php b/src/adapter/etl-adapter-seal/tests/Flow/ETL/Adapter/Seal/Tests/Unit/SealEncoderTest.php index 1110731338..850b61bde8 100644 --- a/src/adapter/etl-adapter-seal/tests/Flow/ETL/Adapter/Seal/Tests/Unit/SealEncoderTest.php +++ b/src/adapter/etl-adapter-seal/tests/Flow/ETL/Adapter/Seal/Tests/Unit/SealEncoderTest.php @@ -9,7 +9,6 @@ use DateTimeZone; use DOMDocument; use Flow\ETL\Adapter\Seal\SealEncoder; -use Flow\ETL\Row\TypedRowValues; use Flow\ETL\Tests\Fixtures\Enum\BackedStringEnum; use Flow\ETL\Tests\FlowTestCase; use Flow\Types\Type; @@ -17,12 +16,24 @@ use Flow\Types\Value\Uuid; use Generator; use PHPUnit\Framework\Attributes\DataProvider; +use PHPUnit\Framework\Attributes\RequiresPhp; +use function Flow\ETL\DSL\array_to_rows; +use function Flow\ETL\DSL\date_schema; +use function Flow\ETL\DSL\datetime_schema; +use function Flow\ETL\DSL\definition_from_type; +use function Flow\ETL\DSL\html_element_schema; +use function Flow\ETL\DSL\html_schema; +use function Flow\ETL\DSL\int_schema; +use function Flow\ETL\DSL\schema; +use function Flow\ETL\DSL\str_schema; +use function Flow\ETL\DSL\xml_element_schema; +use function Flow\ETL\DSL\xml_schema; use function Flow\Types\DSL\type_boolean; -use function Flow\Types\DSL\type_date; use function Flow\Types\DSL\type_datetime; use function Flow\Types\DSL\type_enum; use function Flow\Types\DSL\type_float; +use function Flow\Types\DSL\type_html_element; use function Flow\Types\DSL\type_integer; use function Flow\Types\DSL\type_json; use function Flow\Types\DSL\type_list; @@ -32,7 +43,6 @@ use function Flow\Types\DSL\type_time; use function Flow\Types\DSL\type_time_zone; use function Flow\Types\DSL\type_uuid; -use function Flow\Types\DSL\type_xml; final class SealEncoderTest extends FlowTestCase { @@ -84,7 +94,9 @@ public function test_encodes_values_into_documents(mixed $value, mixed $expected { static::assertSame( $expected, - (new SealEncoder())->encode([new TypedRowValues(['field' => $value], ['field' => $type])])[0]['field'], + (new SealEncoder())->encode(array_to_rows([[ + 'field' => $value, + ]], schema(definition_from_type('field', $type, nullable: $value === null))))[0]['field'], ); } @@ -95,7 +107,7 @@ public function test_encodes_xml_document_to_string(): void static::assertSame( '1', - (new SealEncoder())->encode([new TypedRowValues(['xml' => $doc], ['xml' => type_xml()])])[0]['xml'], + (new SealEncoder())->encode(array_to_rows([['xml' => $doc]], schema(xml_schema('xml'))))[0]['xml'], ); } @@ -103,9 +115,9 @@ public function test_renders_date_with_the_date_format(): void { static::assertSame( '2023-10-01', - (new SealEncoder())->encode([ - new TypedRowValues(['date' => new DateTimeImmutable('2023-10-01 00:00:00')], ['date' => type_date()]), - ])[0]['date'], + (new SealEncoder())->encode(array_to_rows([[ + 'date' => new DateTimeImmutable('2023-10-01 00:00:00'), + ]], schema(date_schema('date'))))[0]['date'], ); } @@ -113,10 +125,56 @@ public function test_produces_documents_keyed_by_column_name(): void { static::assertSame( [['id' => 1, 'name' => 'Alice'], ['id' => 2, 'name' => 'Bob']], - (new SealEncoder())->encode([ - new TypedRowValues(['id' => 1, 'name' => 'Alice'], ['id' => type_integer(), 'name' => type_string()]), - new TypedRowValues(['id' => 2, 'name' => 'Bob'], ['id' => type_integer(), 'name' => type_string()]), - ]), + (new SealEncoder())->encode(array_to_rows( + [['id' => 1, 'name' => 'Alice'], ['id' => 2, 'name' => 'Bob']], + schema(int_schema('id'), str_schema('name')), + )), + ); + } + + public function test_encodes_an_xml_element_as_its_markup(): void + { + $document = new DOMDocument(); + $document->loadXML(''); + + static::assertSame( + [['e' => '']], + (new SealEncoder())->encode(array_to_rows([[ + 'e' => $document->getElementsByTagName('a')->item(0), + ]], schema(xml_element_schema('e')))), + ); + } + + public function test_encodes_a_datetime_column_with_nulls(): void + { + static::assertSame( + [['at' => '2026-01-02T03:04:05+00:00'], ['at' => null]], + (new SealEncoder())->encode(array_to_rows([ + ['at' => new DateTimeImmutable('2026-01-02 03:04:05 UTC')], + ['at' => null], + ], schema(datetime_schema('at', nullable: true)))), + ); + } + + #[RequiresPhp('>= 8.4.0')] + public function test_encodes_an_html_document_as_its_markup(): void + { + static::assertSame( + [['h' => '

a

']], + (new SealEncoder())->encode(array_to_rows([[ + 'h' => '

a

', + ]], schema(html_schema('h')))), + ); + } + + #[RequiresPhp('>= 8.4.0')] + public function test_encodes_an_html_element_with_a_parent_as_its_markup(): void + { + static::assertSame( + [['e' => '

a

']], + (new SealEncoder())->encode(array_to_rows([[ + 'e' => type_html_element()->cast('

a

')->firstElementChild, + ]], schema(html_element_schema('e')))), ); } } diff --git a/src/adapter/etl-adapter-seal/tests/Flow/ETL/Adapter/Seal/Tests/Unit/SealLoaderTest.php b/src/adapter/etl-adapter-seal/tests/Flow/ETL/Adapter/Seal/Tests/Unit/SealLoaderTest.php index c77a663d17..67819e1d9e 100644 --- a/src/adapter/etl-adapter-seal/tests/Flow/ETL/Adapter/Seal/Tests/Unit/SealLoaderTest.php +++ b/src/adapter/etl-adapter-seal/tests/Flow/ETL/Adapter/Seal/Tests/Unit/SealLoaderTest.php @@ -10,9 +10,9 @@ use function Flow\ETL\Adapter\Seal\to_seal_delete; use function Flow\ETL\Adapter\Seal\to_seal_schema; use function Flow\ETL\Adapter\Seal\to_seal_upsert; +use function Flow\ETL\DSL\array_to_rows; use function Flow\ETL\DSL\flow_context; use function Flow\ETL\DSL\list_schema; -use function Flow\ETL\DSL\row; use function Flow\ETL\DSL\rows; use function Flow\ETL\DSL\schema; use function Flow\ETL\DSL\str_schema; @@ -29,17 +29,19 @@ public function test_deleting_with_a_non_scalar_identifier_entry_throws(): void $this->expectException(RuntimeException::class); $this->expectExceptionMessage('Entry "id" cannot be used as a document identifier for DELETE operation'); - to_seal_delete($engine, 'users')->load( - rows(schema(list_schema('id', type_list(type_string()))), row(['id' => ['1', '2']])), - flow_context(), - ); + to_seal_delete($engine, 'users')->load(array_to_rows([['id' => [ + '1', + '2', + ]]], schema(list_schema('id', type_list(type_string())))), flow_context()); } public function test_deleting_empty_rows_removes_no_documents(): void { $engine = $this->sealContext()->engine(to_seal_schema(schema(str_schema('id')), 'users', 'id')); - to_seal_upsert($engine, 'users')->load(rows(schema(string_schema('id')), row(['id' => '1'])), flow_context()); + to_seal_upsert($engine, 'users')->load(array_to_rows([[ + 'id' => '1', + ]], schema(string_schema('id'))), flow_context()); to_seal_delete($engine, 'users')->load(rows(schema()), flow_context()); static::assertSame(1, $engine->countDocuments('users')); diff --git a/src/adapter/etl-adapter-text/src/Flow/ETL/Adapter/Text/TextEncoder.php b/src/adapter/etl-adapter-text/src/Flow/ETL/Adapter/Text/TextEncoder.php index a5ab22f982..39e7f456a3 100644 --- a/src/adapter/etl-adapter-text/src/Flow/ETL/Adapter/Text/TextEncoder.php +++ b/src/adapter/etl-adapter-text/src/Flow/ETL/Adapter/Text/TextEncoder.php @@ -4,71 +4,59 @@ namespace Flow\ETL\Adapter\Text; +use Flow\ETL\Column\TextValues; use Flow\ETL\Exception\RuntimeException; -use Flow\ETL\Row\Encoder; -use Flow\ETL\Row\RawRowValues; -use Stringable; +use Flow\ETL\Rows; +use Flow\Types\Type\Logical\ListType; +use Flow\Types\Type\Logical\MapType; +use Flow\Types\Type\Logical\StructureType; use function array_values; use function count; -use function get_debug_type; -use function is_bool; -use function is_scalar; -use function rtrim; +use function Flow\Types\DSL\type_bare; +use function implode; use function sprintf; +use function str_repeat; -/** - * @implements Encoder - */ -final class TextEncoder implements Encoder +final class TextEncoder { public function __construct( private readonly string $newLineSeparator = PHP_EOL, ) {} - public function decode(array $batch): array + public function encode(Rows $rows): string { - $maps = []; + $count = $rows->count(); - foreach ($batch as $line) { - $maps[] = new RawRowValues(['text' => rtrim($line)]); + if ($count < 1) { + return ''; } - return $maps; - } - - public function encode(array $batch): array - { - $lines = []; + $definitions = array_values($rows->schema()->definitions()); - foreach ($batch as $rowValues) { - $values = $rowValues->values; + if (count($definitions) > 1) { + throw new RuntimeException(sprintf( + 'Text data loader writes at most one column, the batch has %d columns.', + count($definitions), + )); + } - if (count($values) > 1) { - throw new RuntimeException(sprintf( - 'Text data loader supports only a single entry rows, and you have %d rows.', - count($values), - )); - } + if ($definitions === []) { + return str_repeat($this->newLineSeparator, $count); + } - /** @var mixed $value */ - $value = array_values($values)[0] ?? null; + $type = $definitions[0]->type(); + $bare = type_bare($type); - $lines[] = $this->renderValue($value) . $this->newLineSeparator; + if ($bare instanceof ListType || $bare instanceof MapType || $bare instanceof StructureType) { + throw new RuntimeException('Text data loader supports only scalar values, got array'); } - return $lines; - } - - private function renderValue(mixed $value): string - { - return match (true) { - $value === null => '', - is_bool($value) => $value ? 'true' : 'false', - is_scalar($value), $value instanceof Stringable => (string) $value, - default => throw new RuntimeException( - 'Text data loader supports only scalar values, got ' . get_debug_type($value), - ), - }; + return ( + implode( + $this->newLineSeparator, + (new TextValues())->texts($type, $rows->column($definitions[0]->entry()->name())->physicals()), + ) . $this->newLineSeparator + ); } } diff --git a/src/adapter/etl-adapter-text/src/Flow/ETL/Adapter/Text/TextExtractor.php b/src/adapter/etl-adapter-text/src/Flow/ETL/Adapter/Text/TextExtractor.php index 555626b1ef..6bbb6fbcba 100644 --- a/src/adapter/etl-adapter-text/src/Flow/ETL/Adapter/Text/TextExtractor.php +++ b/src/adapter/etl-adapter-text/src/Flow/ETL/Adapter/Text/TextExtractor.php @@ -8,10 +8,12 @@ use Flow\ETL\Extractor; use Flow\ETL\Extractor\BatchableExtractor; use Flow\ETL\Extractor\Batches; -use Flow\ETL\Extractor\FileExtractor; -use Flow\ETL\Extractor\FileReading; -use Flow\ETL\Extractor\ListedFiles; -use Flow\ETL\Extractor\MetadataColumnsExtractor; +use Flow\ETL\Extractor\File\FileExtractor; +use Flow\ETL\Extractor\File\FileReading; +use Flow\ETL\Extractor\File\FileReadLoop; +use Flow\ETL\Extractor\File\ListedFiles; +use Flow\ETL\Extractor\File\MetadataColumnsExtractor; +use Flow\ETL\Extractor\File\ReadWindow; use Flow\ETL\Extractor\RewindableExtractor; use Flow\ETL\Extractor\Signal; use Flow\ETL\Extractor\Statistics; @@ -25,9 +27,10 @@ use Flow\Filesystem\Path\Filter\OnlyFiles; use Generator; -use function count; use function Flow\ETL\DSL\schema; use function Flow\ETL\DSL\str_schema; +use function implode; +use function iterator_to_array; use function sprintf; final class TextExtractor implements @@ -73,69 +76,18 @@ public function isRepeatable(): bool */ public function extract(FlowContext $context, ?int $limit = null, Filter $pathFilter = new OnlyFiles()): Generator { - $hydrator = $context->hydrator(); - $batchSize = $this->batchSize(); - $yielded = 0; - $encoder = new TextEncoder(); - - $baseSchema = $this->schema ?? schema(str_schema('text')); - $fileColumns = $this->fileColumns($this->filesystem, $this->path); - $schema = $fileColumns->declare($baseSchema); - $body = $fileColumns->withoutTail($schema); - - foreach ($this->sourceFiles($this->filesystem, $this->path, $pathFilter) as $source) { - $stream = $this->filesystem->readFrom($source->path); - - try { - $constants = $fileColumns->forFile($source, $schema); - - $rawLines = []; - - foreach ($stream->readLines() as $line) { - $rawLines[] = $line; - - if (count($rawLines) >= $batchSize) { - $hydrated = $constants->fillRows( - $hydrator->hydrate($encoder->decode($rawLines), $body), - $schema, - ); - - $rawLines = []; - - $yielded += $hydrated->count(); - - $signal = yield $hydrated; - - if ($signal === Signal::STOP) { - return; - } - if ($limit !== null && $yielded >= $limit) { - return; - } - } - } - - if ($rawLines !== []) { - $hydrated = $constants->fillRows($hydrator->hydrate($encoder->decode($rawLines), $body), $schema); - - $yielded += $hydrated->count(); - - $signal = yield $hydrated; - - if ($signal === Signal::STOP) { - return; - } - - if ($limit !== null && $yielded >= $limit) { - return; - } - } - } finally { - $stream->close(); - } - } + yield from (new FileReadLoop( + $fileColumns, + $fileColumns->declare($this->schema ?? schema(str_schema('text'))), + ))->read( + iterator_to_array($this->sourceFiles($this->filesystem, $this->path, $pathFilter), false), + new TextFileBatches($this->filesystem), + $this->batchSize(), + $context->backend(), + new ReadWindow(limit: $limit), + ); } public function schema(): Schema @@ -160,8 +112,18 @@ public function statistics(): Statistics return new Statistics(size: $this->listed->bytes); } + /** + * @throws InvalidArgumentException when $schema has no "text" column, the one the lines are read into + */ public function withSchema(Schema $schema): static { + if ($schema->findDefinition('text') === null) { + throw new InvalidArgumentException(sprintf('from_text() reads every line into the "text" column, the schema declares no such column: %s', implode( + ', ', + $schema->references()->names(), + ))); + } + $this->schema = $schema; return $this; diff --git a/src/adapter/etl-adapter-text/src/Flow/ETL/Adapter/Text/TextFileBatches.php b/src/adapter/etl-adapter-text/src/Flow/ETL/Adapter/Text/TextFileBatches.php new file mode 100644 index 0000000000..4ca40d785b --- /dev/null +++ b/src/adapter/etl-adapter-text/src/Flow/ETL/Adapter/Text/TextFileBatches.php @@ -0,0 +1,75 @@ + $batchSize + * + * @return Generator + */ + public function batches( + SourceFile $source, + Schema $body, + int $batchSize, + Backend $backend, + ReadWindow $window, + ): Generator { + $text = new Schema($body->get('text')); + $stream = $this->filesystem->readFrom($source->path); + + try { + $lines = []; + + foreach ($stream->readLines() as $line) { + $lines[] = rtrim($line); + + if (count($lines) >= $batchSize) { + yield $this->batch($text, $lines, $body, $backend); + + $lines = []; + } + } + + if ($lines !== []) { + yield $this->batch($text, $lines, $body, $backend); + } + } finally { + $stream->close(); + } + + return 0; + } + + /** + * @param non-empty-list $lines + */ + public function batch(Schema $text, array $lines, Schema $body, Backend $backend): Rows + { + $builder = $backend->builder($text->get('text')); + $builder->appendMany($lines); + + return Rows::fromColumns($text, ['text' => $builder->finish()], count($lines))->matchTo($body, $backend); + } +} diff --git a/src/adapter/etl-adapter-text/src/Flow/ETL/Adapter/Text/TextLoader.php b/src/adapter/etl-adapter-text/src/Flow/ETL/Adapter/Text/TextLoader.php index 96e0ea18de..d96b15d485 100644 --- a/src/adapter/etl-adapter-text/src/Flow/ETL/Adapter/Text/TextLoader.php +++ b/src/adapter/etl-adapter-text/src/Flow/ETL/Adapter/Text/TextLoader.php @@ -4,39 +4,40 @@ namespace Flow\ETL\Adapter\Text; -use Flow\ETL\Config\Telemetry\TelemetryAttributes; +use Flow\ETL\Column\Backend; use Flow\ETL\Exception\InvalidArgumentException; -use Flow\ETL\Filesystem\FilesSink; use Flow\ETL\Filesystem\SaveMode; use Flow\ETL\FlowContext; use Flow\ETL\Loader; use Flow\ETL\Loader\Closure; use Flow\ETL\Loader\Discardable; -use Flow\ETL\Loader\FileLoader; +use Flow\ETL\Loader\File\FileLoader; +use Flow\ETL\Loader\File\FileSink; +use Flow\ETL\Loader\File\FileSinks; +use Flow\ETL\Loader\File\FileWriteFrame; +use Flow\ETL\Loader\File\PartitionRouter; use Flow\ETL\Loader\Partitioning; use Flow\ETL\Loader\PartitioningLoader; -use Flow\ETL\Loader\PartitionRouter; use Flow\ETL\Rows; +use Flow\Filesystem\DestinationStream; use Flow\Filesystem\Filesystem; use Flow\Filesystem\Local\NativeLocalFilesystem; use Flow\Filesystem\Path; use Flow\Filesystem\Path\Option; use Flow\Filesystem\Path\Option\ContentType; -use Throwable; -use function implode; use function sprintf; -final class TextLoader implements Closure, Discardable, FileLoader, Loader, PartitioningLoader +final class TextLoader implements Closure, Discardable, FileLoader, FileSinks, Loader, PartitioningLoader { + private ?FileWriteFrame $frame = null; + private PartitionRouter $router; private readonly Filesystem $filesystem; private SaveMode $saveMode = SaveMode::ExceptionIfExists; - private ?FilesSink $files = null; - private ?TextEncoder $encoder = null; private string $newLineSeparator = PHP_EOL; @@ -69,14 +70,14 @@ public function partitionBy(Partitioning $partitioning): static public function closure(FlowContext $context): void { - $this->files?->publish(); - $this->files = null; + $this->frame?->closure(); + $this->frame = null; } public function discard(FlowContext $context): void { - $this->files?->abandon(); - $this->files = null; + $this->frame?->discard(); + $this->frame = null; } public function destination(): Path @@ -90,23 +91,18 @@ public function load(Rows $rows, FlowContext $context): void return; } - $context->telemetry()->loadingStarted($this, [ - TelemetryAttributes::ATTR_LOADER_DESTINATION_URI => $this->path->uri(), - ]); - - try { - foreach ($this->router->route($rows) as [$partitions, $group]) { - ($this->files ??= new FilesSink($this->filesystem, $this->path, $this->saveMode)) - ->writeTo($partitions->toArray()) - ->append(implode('', $this->encoder()->encode($context->hydrator()->dehydrate($group)))); - } - - $context->telemetry()->loadingCompleted($this, [TelemetryAttributes::ATTR_LOADING_ROWS => $rows->count()]); - } catch (Throwable $e) { - $context->telemetry()->loadingFailed($this, $e); + ($this->frame ??= new FileWriteFrame( + $this->filesystem, + $this->path, + $this->saveMode, + $this->router, + $this, + ))->write($rows, $context, $this); + } - throw $e; - } + public function open(DestinationStream $stream, Backend $backend): FileSink + { + return new TextOpenSink($stream, $this->encoder()); } public function saveMode(SaveMode $mode): static diff --git a/src/adapter/etl-adapter-text/src/Flow/ETL/Adapter/Text/TextOpenSink.php b/src/adapter/etl-adapter-text/src/Flow/ETL/Adapter/Text/TextOpenSink.php new file mode 100644 index 0000000000..c392c611b6 --- /dev/null +++ b/src/adapter/etl-adapter-text/src/Flow/ETL/Adapter/Text/TextOpenSink.php @@ -0,0 +1,27 @@ +stream->append($this->encoder->encode($rows)); + } +} diff --git a/src/adapter/etl-adapter-text/tests/Flow/ETL/Adapter/Text/Tests/Integration/TextExtractorBatchContractTest.php b/src/adapter/etl-adapter-text/tests/Flow/ETL/Adapter/Text/Tests/Integration/TextExtractorBatchContractTest.php index da54154c10..fd4d76d7da 100644 --- a/src/adapter/etl-adapter-text/tests/Flow/ETL/Adapter/Text/Tests/Integration/TextExtractorBatchContractTest.php +++ b/src/adapter/etl-adapter-text/tests/Flow/ETL/Adapter/Text/Tests/Integration/TextExtractorBatchContractTest.php @@ -8,8 +8,7 @@ use Flow\ETL\Tests\FlowTestCase; use function Flow\ETL\Adapter\Text\from_text; -use function Flow\ETL\DSL\row; -use function Flow\ETL\DSL\rows; +use function Flow\ETL\DSL\array_to_rows; use function Flow\ETL\DSL\schema; use function Flow\ETL\DSL\str_schema; @@ -19,12 +18,7 @@ public function test_text_extractor_honours_the_batch_contract(): void { self::assertExtractorHonoursBatchContract( static fn(): TextExtractor => from_text(__DIR__ . '/../Fixtures/parity_lines.txt'), - rows( - schema(str_schema('text')), - row(['text' => 'alpha']), - row(['text' => 'beta']), - row(['text' => 'gamma']), - ), + array_to_rows([['text' => 'alpha'], ['text' => 'beta'], ['text' => 'gamma']], schema(str_schema('text'))), ); } } diff --git a/src/adapter/etl-adapter-text/tests/Flow/ETL/Adapter/Text/Tests/Integration/TextExtractorTest.php b/src/adapter/etl-adapter-text/tests/Flow/ETL/Adapter/Text/Tests/Integration/TextExtractorTest.php index f350ac449f..3a26ef3153 100644 --- a/src/adapter/etl-adapter-text/tests/Flow/ETL/Adapter/Text/Tests/Integration/TextExtractorTest.php +++ b/src/adapter/etl-adapter-text/tests/Flow/ETL/Adapter/Text/Tests/Integration/TextExtractorTest.php @@ -6,16 +6,23 @@ use Flow\ETL\Adapter\Text\TextExtractor; use Flow\ETL\Cardinality; +use Flow\ETL\Exception\InvalidArgumentException; use Flow\ETL\Extractor\Signal; use Flow\ETL\Tests\Context\ExtractedRows; use Flow\ETL\Tests\Context\MemoryFiles; use Flow\ETL\Tests\Double\CountingFilesystem; use Flow\ETL\Tests\Double\RecordingFilesystem; +use Flow\ETL\Tests\Double\SpyBackend; use Flow\ETL\Tests\Double\UnsizedFilesystem; use Flow\ETL\Tests\FlowTestCase; +use function array_keys; +use function array_unique; +use function array_values; +use function count; use function Flow\ETL\Adapter\Text\from_text; use function Flow\ETL\DSL\config; +use function Flow\ETL\DSL\config_builder; use function Flow\ETL\DSL\data_frame; use function Flow\ETL\DSL\flow_context; use function Flow\ETL\DSL\schema; @@ -109,9 +116,9 @@ public function test_limit_reached_on_the_first_file_tail_batch_skips_the_remain public function test_metadata_columns_extend_a_declared_schema(): void { static::assertEquals( - schema(str_schema('line'), str_schema('_input_file_uri')), + schema(str_schema('text'), str_schema('_input_file_uri')), from_text(__DIR__ . '/../Fixtures/annual-enterprise-survey-2019-financial-year-provisional-csv.csv') - ->withSchema(schema(str_schema('line'))) + ->withSchema(schema(str_schema('text'))) ->withMetadataColumns(true) ->schema(), ); @@ -186,4 +193,58 @@ public function test_the_listing_is_read_once(): void static::assertSame(1, $filesystem->listCalls); } + + public function test_extract_builds_through_the_config_backend(): void + { + $backend = new SpyBackend(); + + iterator_to_array( + from_text(path_real(__DIR__ . '/../Fixtures/orders_flow.csv')) + ->extract(flow_context(config_builder()->backend($backend)->build())), + false, + ); + + static::assertGreaterThanOrEqual(1, $backend->builders()); + } + + public function test_a_schema_without_the_text_column_is_refused(): void + { + $this->expectException(InvalidArgumentException::class); + $this->expectExceptionMessage( + 'from_text() reads every line into the "text" column, the schema declares no such column: line', + ); + + from_text(__DIR__ . '/../Fixtures/orders_flow.csv')->withSchema(schema(str_schema('line'))); + } + + public function test_columns_a_schema_declares_beside_text_are_padded_with_nulls(): void + { + $rows = data_frame() + ->read( + from_text(path_real(__DIR__ . '/../Fixtures/parity_lines.txt')) + ->withSchema(schema(str_schema('text'), str_schema('note', nullable: true))), + ) + ->fetch(); + + static::assertNotSame(0, $rows->count()); + static::assertSame([null], array_values(array_unique($rows->column('note')->values()))); + } + + public function test_one_extractor_read_twice_interleaved_gives_each_read_every_row_and_closes_every_stream(): void + { + $filesystem = new RecordingFilesystem(MemoryFiles::with([ + 'memory://in/a.txt' => "a\nb\n", + 'memory://in/b.txt' => "c\n", + ])); + [$first, $second] = ExtractedRows::interleaved(from_text(path('memory://in/*.txt'), $filesystem)->withBatchSize( + 1, + )); + + static::assertSame([['text' => 'a'], ['text' => 'b'], ['text' => 'c']], $first->toArray()); + static::assertSame($first->toArray(), $second->toArray()); + static::assertSame( + count(array_keys($filesystem->calls, 'readFrom', true)), + count(array_keys($filesystem->calls, 'closeSource', true)), + ); + } } diff --git a/src/adapter/etl-adapter-text/tests/Flow/ETL/Adapter/Text/Tests/Integration/TextExtractorTypedColumnsTest.php b/src/adapter/etl-adapter-text/tests/Flow/ETL/Adapter/Text/Tests/Integration/TextExtractorTypedColumnsTest.php new file mode 100644 index 0000000000..593956c1a6 --- /dev/null +++ b/src/adapter/etl-adapter-text/tests/Flow/ETL/Adapter/Text/Tests/Integration/TextExtractorTypedColumnsTest.php @@ -0,0 +1,55 @@ +extract(flow_context(Config::builder()->build())) as $rows) { + static::assertEquals(type_string(), $rows->schema()->get('text')->type()); + + foreach ($rows->toArray() as $row) { + $actual[] = $row; + } + } + + static::assertSame([['text' => 'alpha'], ['text' => 'beta'], ['text' => 'gamma']], $actual); + } + + public function test_appends_input_file_uri_when_metadata_columns_are_enabled(): void + { + $extractor = from_text($path = path_real(__DIR__ . '/../Fixtures/parity_lines.txt'))->withMetadataColumns(true); + + $actual = []; + + foreach ($extractor->extract(flow_context(Config::builder()->build())) as $rows) { + foreach ($rows->toArray() as $row) { + $actual[] = $row; + } + } + + static::assertSame( + [ + ['text' => 'alpha', '_input_file_uri' => $path->uri()], + ['text' => 'beta', '_input_file_uri' => $path->uri()], + ['text' => 'gamma', '_input_file_uri' => $path->uri()], + ], + $actual, + ); + } +} diff --git a/src/adapter/etl-adapter-text/tests/Flow/ETL/Adapter/Text/Tests/Integration/TextHydratorParityTest.php b/src/adapter/etl-adapter-text/tests/Flow/ETL/Adapter/Text/Tests/Integration/TextHydratorParityTest.php deleted file mode 100644 index 142867a760..0000000000 --- a/src/adapter/etl-adapter-text/tests/Flow/ETL/Adapter/Text/Tests/Integration/TextHydratorParityTest.php +++ /dev/null @@ -1,73 +0,0 @@ -extract(flow_context(Config::builder()->build())) as $rows) { - static::assertEquals(type_string(), $rows->schema()->get('text')->type()); - - foreach ($rows as $row) { - $actual[] = $row->toArray(); - } - } - - static::assertSame([['text' => 'alpha'], ['text' => 'beta'], ['text' => 'gamma']], $actual); - } - - public function test_appends_input_file_uri_when_metadata_columns_are_enabled(): void - { - $extractor = from_text($path = path_real(__DIR__ . '/../Fixtures/parity_lines.txt'))->withMetadataColumns(true); - - $actual = []; - - foreach ($extractor->extract(flow_context(Config::builder()->build())) as $rows) { - foreach ($rows as $row) { - $actual[] = $row->toArray(); - } - } - - static::assertSame( - [ - ['text' => 'alpha', '_input_file_uri' => $path->uri()], - ['text' => 'beta', '_input_file_uri' => $path->uri()], - ['text' => 'gamma', '_input_file_uri' => $path->uri()], - ], - $actual, - ); - } - - public function test_honours_a_hydrator_configured_on_the_context(): void - { - $extractor = from_text(path_real(__DIR__ . '/../Fixtures/parity_lines.txt')); - - $actual = []; - - foreach ($extractor->extract( - flow_context(Config::builder()->hydrator(new AdaptiveRowHydrator())->build()), - ) as $rows) { - foreach ($rows as $row) { - $actual[] = $row->toArray(); - } - } - - static::assertSame([['text' => 'alpha'], ['text' => 'beta'], ['text' => 'gamma']], $actual); - } -} diff --git a/src/adapter/etl-adapter-text/tests/Flow/ETL/Adapter/Text/Tests/Integration/TextLoaderTest.php b/src/adapter/etl-adapter-text/tests/Flow/ETL/Adapter/Text/Tests/Integration/TextLoaderTest.php index 0db7844297..bfa09ad7e9 100644 --- a/src/adapter/etl-adapter-text/tests/Flow/ETL/Adapter/Text/Tests/Integration/TextLoaderTest.php +++ b/src/adapter/etl-adapter-text/tests/Flow/ETL/Adapter/Text/Tests/Integration/TextLoaderTest.php @@ -5,13 +5,25 @@ namespace Flow\ETL\Adapter\Text\Tests\Integration; use Flow\ETL\Adapter\Text\TextLoader; +use Flow\ETL\Tests\Double\RecordingFilesystem; +use Flow\ETL\Tests\FlowTestCase; use Flow\Filesystem\Path\Option; use Flow\Filesystem\Path\Option\ContentType; -use PHPUnit\Framework\TestCase; +use function array_keys; +use function array_map; +use function count; +use function Flow\ETL\Adapter\Text\to_text; +use function Flow\ETL\DSL\array_to_rows; +use function Flow\ETL\DSL\float_schema; +use function Flow\ETL\DSL\flow_context; +use function Flow\ETL\DSL\partition_by; +use function Flow\ETL\DSL\schema; +use function Flow\ETL\DSL\str_schema; +use function Flow\Filesystem\DSL\memory_filesystem; use function Flow\Filesystem\DSL\path; -final class TextLoaderTest extends TestCase +final class TextLoaderTest extends FlowTestCase { public function test_setting_content_type_on_path(): void { @@ -19,4 +31,44 @@ public function test_setting_content_type_on_path(): void static::assertEquals($loader->destination()->getOption(Option::CONTENT_TYPE), ContentType::TEXT); } + + public function test_every_batch_is_one_append(): void + { + $filesystem = new RecordingFilesystem(memory_filesystem()); + $loader = to_text(path('memory://out.txt'), "\n", $filesystem); + + $loader->load(array_to_rows([ + ['price' => 0.1 + 0.2], + ['price' => 2.0], + ], schema(float_schema('price'))), flow_context()); + $loader->load(array_to_rows([['price' => 3.5]], schema(float_schema('price'))), flow_context()); + $loader->closure(flow_context()); + + static::assertSame( + "0.30000000000000004\n2.0\n3.5\n", + $filesystem->readFrom(path('memory://out.txt'))->content(), + ); + static::assertSame(2, count(array_keys($filesystem->calls, 'append', true))); + } + + public function test_two_runs_through_one_loader_write_independent_files(): void + { + $filesystem = memory_filesystem(); + $loader = to_text(path('memory://out.txt'), filesystem: $filesystem)->partitionBy(partition_by('group')); + $read = []; + + foreach (['a' => [1, 2], 'b' => [3]] as $group => $ids) { + $loader->load( + array_to_rows( + array_map(static fn(int $id): array => ['text' => (string) $id, 'group' => $group], $ids), + schema(str_schema('text'), str_schema('group')), + ), + flow_context(), + ); + $loader->closure(flow_context()); + $read[] = $filesystem->readFrom(path('memory://group=' . $group . '/out.txt'))->content(); + } + + static::assertSame(["1\n2\n", "3\n"], $read); + } } diff --git a/src/adapter/etl-adapter-text/tests/Flow/ETL/Adapter/Text/Tests/Integration/TextTest.php b/src/adapter/etl-adapter-text/tests/Flow/ETL/Adapter/Text/Tests/Integration/TextTest.php index c72321d43a..f78a1ba0a2 100644 --- a/src/adapter/etl-adapter-text/tests/Flow/ETL/Adapter/Text/Tests/Integration/TextTest.php +++ b/src/adapter/etl-adapter-text/tests/Flow/ETL/Adapter/Text/Tests/Integration/TextTest.php @@ -11,13 +11,12 @@ use function file_get_contents; use function Flow\ETL\Adapter\Text\from_text; use function Flow\ETL\Adapter\Text\to_text; +use function Flow\ETL\DSL\array_to_rows; use function Flow\ETL\DSL\data_frame; use function Flow\ETL\DSL\from_sequence_number; use function Flow\ETL\DSL\generate_random_string; use function Flow\ETL\DSL\lit; use function Flow\ETL\DSL\overwrite; -use function Flow\ETL\DSL\row; -use function Flow\ETL\DSL\rows; use function Flow\ETL\DSL\schema; use function Flow\ETL\DSL\select; use function Flow\ETL\DSL\string_schema; @@ -37,12 +36,11 @@ public function test_loading_text_files(): void $path = __DIR__ . '/var/flow_php_etl_csv_loader' . generate_random_string() . '.csv'; data_frame() - ->process(rows( - schema(string_schema('name')), - row(['name' => 'Norbert']), - row(['name' => 'Tomek']), - row(['name' => 'Dawid']), - )) + ->process(array_to_rows([ + ['name' => 'Norbert'], + ['name' => 'Tomek'], + ['name' => 'Dawid'], + ], schema(string_schema('name')))) ->write(to_text($path)) ->run(); diff --git a/src/adapter/etl-adapter-text/tests/Flow/ETL/Adapter/Text/Tests/Unit/TextEncoderTest.php b/src/adapter/etl-adapter-text/tests/Flow/ETL/Adapter/Text/Tests/Unit/TextEncoderTest.php index d68cae6cf5..ae8b1c8dca 100644 --- a/src/adapter/etl-adapter-text/tests/Flow/ETL/Adapter/Text/Tests/Unit/TextEncoderTest.php +++ b/src/adapter/etl-adapter-text/tests/Flow/ETL/Adapter/Text/Tests/Unit/TextEncoderTest.php @@ -6,53 +6,99 @@ use Flow\ETL\Adapter\Text\TextEncoder; use Flow\ETL\Exception\RuntimeException; -use Flow\ETL\Row\RawRowValues; -use Flow\ETL\Row\TypedRowValues; +use Flow\ETL\Schema\Definition; use Flow\ETL\Tests\FlowTestCase; +use Generator; +use PHPUnit\Framework\Attributes\DataProvider; -use function array_map; +use function Flow\ETL\DSL\array_to_rows; +use function Flow\ETL\DSL\float_schema; +use function Flow\ETL\DSL\int_schema; +use function Flow\ETL\DSL\list_schema; +use function Flow\ETL\DSL\map_schema; +use function Flow\ETL\DSL\schema; +use function Flow\ETL\DSL\str_schema; +use function Flow\ETL\DSL\structure_schema; use function Flow\Types\DSL\type_integer; +use function Flow\Types\DSL\type_list; +use function Flow\Types\DSL\type_map; use function Flow\Types\DSL\type_string; +use function Flow\Types\DSL\type_structure; final class TextEncoderTest extends FlowTestCase { - public function test_decode_trims_trailing_new_lines_into_a_single_text_column(): void + /** + * @return Generator, mixed}> + */ + public static function nested_columns(): Generator + { + yield 'list' => [list_schema('a', type_list(type_integer())), [1]]; + yield 'map' => [map_schema('a', type_map(type_string(), type_integer())), ['x' => 1]]; + yield 'structure' => [structure_schema('a', type_structure(['x' => type_integer()])), ['x' => 1]]; + } + + public function test_encode_writes_every_value_on_its_own_line_and_a_null_as_an_empty_line(): void { static::assertSame( - [['text' => 'first'], ['text' => 'second']], - array_map( - static fn(RawRowValues $rowValues): array => $rowValues->values, - (new TextEncoder())->decode(["first\n", "second\r\n"]), - ), + "first\n\nsecond\n", + (new TextEncoder("\n"))->encode(array_to_rows([ + ['text' => 'first'], + ['text' => null], + ['text' => 'second'], + ], schema(str_schema('text', nullable: true)))), ); } - public function test_encode_appends_the_new_line_separator_to_the_single_value(): void + public function test_encode_writes_the_text_of_a_value_that_is_no_string(): void { static::assertSame( - ["first\n", "second\n"], - (new TextEncoder("\n"))->encode([ - new TypedRowValues(['text' => 'first'], ['text' => type_string()]), - new TypedRowValues(['text' => 'second'], ['text' => type_string()]), - ]), + "0.30000000000000004\n1.0\n", + (new TextEncoder("\n"))->encode(array_to_rows([ + ['price' => 0.1 + 0.2], + ['price' => 1.0], + ], schema(float_schema('price')))), ); } - public function test_encode_renders_null_as_an_empty_line(): void + /** + * @param Definition $definition + */ + #[DataProvider('nested_columns')] + public function test_encode_refuses_a_nested_column(Definition $definition, mixed $value): void + { + $this->expectException(RuntimeException::class); + $this->expectExceptionMessage('Text data loader supports only scalar values, got array'); + + (new TextEncoder())->encode(array_to_rows([['a' => $value]], schema($definition))); + } + + public function test_encode_ends_every_line_with_the_configured_separator(): void { static::assertSame( - ["\n"], - (new TextEncoder("\n"))->encode([new TypedRowValues(['text' => null], ['text' => type_string()])]), + "first\r\nsecond\r\n", + (new TextEncoder("\r\n"))->encode(array_to_rows([ + ['text' => 'first'], + ['text' => 'second'], + ], schema(str_schema('text')))), ); } public function test_encode_throws_when_a_row_has_more_than_one_column(): void { $this->expectException(RuntimeException::class); - $this->expectExceptionMessage('Text data loader supports only a single entry rows, and you have 2 rows.'); + $this->expectExceptionMessage('Text data loader writes at most one column, the batch has 3 columns.'); + + (new TextEncoder())->encode(array_to_rows( + [['a' => 1, 'b' => 2, 'c' => 3], ['a' => 4, 'b' => 5, 'c' => 6]], + schema(int_schema('a'), int_schema('b'), int_schema('c')), + )); + } - (new TextEncoder())->encode([ - new TypedRowValues(['a' => 1, 'b' => 2], ['a' => type_integer(), 'b' => type_integer()]), - ]); + public function test_encode_of_an_empty_batch_with_more_than_one_column_returns_no_lines(): void + { + static::assertSame( + '', + (new TextEncoder())->encode(array_to_rows([], schema(int_schema('a'), int_schema('b')))), + ); } } diff --git a/src/adapter/etl-adapter-text/tests/Flow/ETL/Adapter/Text/Tests/Unit/TextFileBatchesTest.php b/src/adapter/etl-adapter-text/tests/Flow/ETL/Adapter/Text/Tests/Unit/TextFileBatchesTest.php new file mode 100644 index 0000000000..a4c8385b49 --- /dev/null +++ b/src/adapter/etl-adapter-text/tests/Flow/ETL/Adapter/Text/Tests/Unit/TextFileBatchesTest.php @@ -0,0 +1,58 @@ + "a\nb\nc\n"])); + $file = (new TextFileBatches($filesystem))->batches( + new SourceFile(path('memory://a.txt')), + schema(str_schema('text')), + 2, + new PhpBackend(), + new ReadWindow(), + ); + + static::assertSame( + [[['text' => 'a'], ['text' => 'b']], [['text' => 'c']]], + array_map(static fn(Rows $rows): array => $rows->toArray(), iterator_to_array($file, false)), + ); + static::assertSame(0, $file->getReturn()); + static::assertSame($filesystem->readFromCalls, $filesystem->closedStreams()); + } + + public function test_a_column_the_body_declares_beside_text_is_padded_with_nulls(): void + { + $file = (new TextFileBatches(MemoryFiles::with(['memory://a.txt' => "a\n"])))->batches( + new SourceFile(path('memory://a.txt')), + schema(str_schema('text'), str_schema('note', nullable: true)), + 10, + new PhpBackend(), + new ReadWindow(), + ); + + static::assertSame( + [[['text' => 'a', 'note' => null]]], + array_map(static fn(Rows $rows): array => $rows->toArray(), iterator_to_array($file, false)), + ); + } +} diff --git a/src/adapter/etl-adapter-text/tests/Flow/ETL/Adapter/Text/Tests/Unit/TextOpenSinkTest.php b/src/adapter/etl-adapter-text/tests/Flow/ETL/Adapter/Text/Tests/Unit/TextOpenSinkTest.php new file mode 100644 index 0000000000..b8a2dd5281 --- /dev/null +++ b/src/adapter/etl-adapter-text/tests/Flow/ETL/Adapter/Text/Tests/Unit/TextOpenSinkTest.php @@ -0,0 +1,32 @@ +writeTo(path('memory://out.txt')); + $sink = new TextOpenSink($stream, new TextEncoder("\n")); + + $sink->write(array_to_rows([['text' => 'a'], ['text' => 'b']], schema(str_schema('text')))); + $sink->write(array_to_rows([['text' => 'c']], schema(str_schema('text')))); + $sink->close(); + $stream->close(); + + static::assertSame("a\nb\nc\n", $filesystem->readFrom(path('memory://out.txt'))->content()); + } +} diff --git a/src/adapter/etl-adapter-xml/src/Flow/ETL/Adapter/XML/Loader/XMLLoader.php b/src/adapter/etl-adapter-xml/src/Flow/ETL/Adapter/XML/Loader/XMLLoader.php index 7f18345cc2..4ffd2be705 100644 --- a/src/adapter/etl-adapter-xml/src/Flow/ETL/Adapter/XML/Loader/XMLLoader.php +++ b/src/adapter/etl-adapter-xml/src/Flow/ETL/Adapter/XML/Loader/XMLLoader.php @@ -5,41 +5,43 @@ namespace Flow\ETL\Adapter\XML\Loader; use Flow\ETL\Adapter\XML\XMLEncoder; +use Flow\ETL\Adapter\XML\XMLOpenSink; use Flow\ETL\Adapter\XML\XMLWriter; -use Flow\ETL\Config\Telemetry\TelemetryAttributes; +use Flow\ETL\Column\Backend; use Flow\ETL\Exception\InvalidArgumentException; -use Flow\ETL\Filesystem\FilesSink; use Flow\ETL\Filesystem\SaveMode; use Flow\ETL\FlowContext; use Flow\ETL\Loader; use Flow\ETL\Loader\Closure; use Flow\ETL\Loader\Discardable; -use Flow\ETL\Loader\FileLoader; +use Flow\ETL\Loader\File\FileLoader; +use Flow\ETL\Loader\File\FileSink; +use Flow\ETL\Loader\File\FileSinks; +use Flow\ETL\Loader\File\FileWriteFrame; +use Flow\ETL\Loader\File\PartitionRouter; use Flow\ETL\Loader\Partitioning; use Flow\ETL\Loader\PartitioningLoader; -use Flow\ETL\Loader\PartitionRouter; use Flow\ETL\Rows; +use Flow\Filesystem\DestinationStream; use Flow\Filesystem\Filesystem; use Flow\Filesystem\Local\NativeLocalFilesystem; -use Flow\Filesystem\Partition; use Flow\Filesystem\Path; use Flow\Filesystem\Path\Option; use Flow\Filesystem\Path\Option\ContentType; -use Throwable; use function sprintf; use function trim; -final class XMLLoader implements Closure, Discardable, FileLoader, Loader, PartitioningLoader +final class XMLLoader implements Closure, Discardable, FileLoader, FileSinks, Loader, PartitioningLoader { + private ?FileWriteFrame $frame = null; + private PartitionRouter $router; private readonly Filesystem $filesystem; private SaveMode $saveMode = SaveMode::ExceptionIfExists; - private ?FilesSink $files = null; - private string $attributePrefix = '_'; private string $dateFormat = 'Y-m-d'; @@ -96,18 +98,14 @@ public function partitionBy(Partitioning $partitioning): static public function closure(FlowContext $context): void { - foreach ($this->files?->openStreams() ?? [] as $stream) { - $stream->append('rootElementName . '>'); - } - - $this->files?->publish(); - $this->files = null; + $this->frame?->closure(); + $this->frame = null; } public function discard(FlowContext $context): void { - $this->files?->abandon(); - $this->files = null; + $this->frame?->discard(); + $this->frame = null; } public function destination(): Path @@ -121,21 +119,29 @@ public function load(Rows $rows, FlowContext $context): void return; } - $context->telemetry()->loadingStarted($this, [ - TelemetryAttributes::ATTR_LOADER_DESTINATION_URI => $this->path->uri(), - ]); - - try { - foreach ($this->router->route($rows) as [$partitions, $group]) { - $this->write($group, $partitions->toArray(), $context); - } + ($this->frame ??= new FileWriteFrame( + $this->filesystem, + $this->path, + $this->saveMode, + $this->router, + $this, + ))->write($rows, $context, $this); + } - $context->telemetry()->loadingCompleted($this, [TelemetryAttributes::ATTR_LOADING_ROWS => $rows->count()]); - } catch (Throwable $e) { - $context->telemetry()->loadingFailed($this, $e); + public function open(DestinationStream $stream, Backend $backend): FileSink + { + $attributes = ''; - throw $e; + foreach ($this->xmlAttributes as $name => $value) { + $attributes .= $name . '="' . $value . '" '; } + + return new XMLOpenSink( + $stream, + $this->encoder(), + '\n<" . $this->rootElementName . ">\n", + $this->rootElementName, + ); } public function saveMode(SaveMode $mode): static @@ -218,30 +224,6 @@ public function withXMLAttributes(array $xmlAttributes): self return $this; } - /** - * @param array $partitions - */ - public function write(Rows $nextRows, array $partitions, FlowContext $context): void - { - $files = $this->files ??= new FilesSink($this->filesystem, $this->path, $this->saveMode); - $opening = !$files->touched($partitions); - $stream = $files->writeTo($partitions); - - if ($opening) { - $attributes = ''; - - foreach ($this->xmlAttributes as $name => $value) { - $attributes .= $name . '="' . $value . '" '; - } - - $stream->append('\n<" . $this->rootElementName . ">\n"); - } - - foreach ($this->encoder()->encode($context->hydrator()->dehydrate($nextRows)) as $node) { - $stream->append($node . "\n"); - } - } - private function encoder(): XMLEncoder { return $this->encoder ??= new XMLEncoder( diff --git a/src/adapter/etl-adapter-xml/src/Flow/ETL/Adapter/XML/XMLEncoder.php b/src/adapter/etl-adapter-xml/src/Flow/ETL/Adapter/XML/XMLEncoder.php index f069782601..e12925b440 100644 --- a/src/adapter/etl-adapter-xml/src/Flow/ETL/Adapter/XML/XMLEncoder.php +++ b/src/adapter/etl-adapter-xml/src/Flow/ETL/Adapter/XML/XMLEncoder.php @@ -4,234 +4,139 @@ namespace Flow\ETL\Adapter\XML; -use BackedEnum; -use Countable; -use DateInterval; -use DateTimeInterface; -use DateTimeZone; -use Dom\XMLDocument; -use DOMDocument; -use Flow\ETL\Adapter\XML\Abstraction\XMLAttribute; use Flow\ETL\Adapter\XML\Abstraction\XMLNode; -use Flow\ETL\Exception\InvalidArgumentException; +use Flow\ETL\Column\TextValues; use Flow\ETL\Exception\RuntimeException; -use Flow\ETL\Row\Encoder; -use Flow\ETL\Row\RawRowValues; -use Flow\Types\Type; -use Flow\Types\Type\Logical\DateTimeType; -use Flow\Types\Type\Logical\DateType; -use Flow\Types\Type\Logical\InstanceOfType; -use Flow\Types\Type\Logical\JsonType; +use Flow\ETL\Rows; use Flow\Types\Type\Logical\ListType; use Flow\Types\Type\Logical\MapType; use Flow\Types\Type\Logical\StructureType; -use Flow\Types\Type\Logical\TimeType; -use Flow\Types\Type\Logical\TimeZoneType; -use Flow\Types\Type\Logical\UuidType; -use Flow\Types\Type\Logical\XMLElementType; -use Flow\Types\Type\Logical\XMLType; -use Flow\Types\Type\Native\ArrayType; -use Flow\Types\Type\Native\BooleanType; -use Flow\Types\Type\Native\EnumType; -use Flow\Types\Type\Native\FloatType; -use Flow\Types\Type\Native\IntegerType; -use Flow\Types\Type\Native\StringType; -use Stringable; - -use function array_values; + +use function array_map; use function count; -use function Flow\ETL\DSL\date_interval_to_microseconds; -use function Flow\Types\DSL\type_string; -use function is_array; -use function is_iterable; -use function is_scalar; -use function json_encode; +use function Flow\Types\DSL\type_bare; +use function implode; +use function str_repeat; use function str_starts_with; use function strlen; use function substr; +use function substr_replace; -use const JSON_THROW_ON_ERROR; - -/** - * @implements Encoder - */ -final class XMLEncoder implements Encoder +final class XMLEncoder { + private bool $rowElementChecked = false; + + private readonly XMLValueNodes $nodes; + + private readonly TextValues $text; + public function __construct( private readonly ?XMLWriter $xmlWriter = null, private readonly string $attributePrefix = '_', - private readonly string $dateTimeFormat = 'Y-m-d\TH:i:s.uP', - private readonly string $dateFormat = 'Y-m-d', - private readonly string $listElementName = 'element', - private readonly string $mapElementName = 'element', - private readonly string $mapElementKeyName = 'key', - private readonly string $mapElementValueName = 'value', + string $dateTimeFormat = 'Y-m-d\TH:i:s.uP', + string $dateFormat = 'Y-m-d', + string $listElementName = 'element', + string $mapElementName = 'element', + string $mapElementKeyName = 'key', + string $mapElementValueName = 'value', private readonly string $rowElementName = 'row', - ) {} - - public function decode(array $batch): array - { - $maps = []; - - foreach ($batch as $xml) { - $maps[] = new RawRowValues(['node' => $xml]); - } - - return $maps; + ) { + $this->text = new TextValues($dateTimeFormat, $dateFormat); + $this->nodes = new XMLValueNodes( + $this->text, + $attributePrefix, + $listElementName, + $mapElementName, + $mapElementKeyName, + $mapElementValueName, + ); } - public function encode(array $batch): array + public function encode(Rows $rows): string { - $xmlWriter = $this->xmlWriter ?? throw new RuntimeException('XMLEncoder requires an XMLWriter to encode rows'); + $writer = $this->xmlWriter ?? throw new RuntimeException('XMLEncoder requires an XMLWriter to encode rows'); + $definitions = $rows->schema()->definitions(); - $lines = []; - - foreach ($batch as $rowValues) { - $elements = []; - - foreach ($rowValues->types as $name => $type) { - $elements[] = $this->normalize($name, $type, $rowValues->values[$name]); - } - - $lines[] = $xmlWriter->write(XMLNode::nested($this->rowElementName, ...$elements)); + foreach ($definitions as $definition) { + $this->nodes->refuseOptionalElements($definition->type()); } - return $lines; - } - - /** - * @param Type $type - * - * @throws InvalidArgumentException - */ - private function normalize(string $name, Type $type, mixed $value): XMLNode|XMLAttribute - { - if ($type instanceof StructureType) { - foreach ($type->elements() as $element) { - if ($element->optional) { - throw new RuntimeException(sprintf( - 'XML encoder does not support structure optional elements, given: %s', - $type->toString(), - )); - } - } + if (!$this->rowElementChecked) { + $writer->write(XMLNode::nestedNode($this->rowElementName)); + $this->rowElementChecked = true; } - if (str_starts_with($name, $this->attributePrefix)) { - return new XMLAttribute(substr($name, strlen($this->attributePrefix)), type_string()->cast($value)); - } + $count = $rows->count(); - if ($value === null) { - return XMLNode::flatNode($name, ''); + if ($count < 1) { + return ''; } - if ($type instanceof ListType) { - if (!is_array($value) && !$value instanceof Countable || !count($value) || !is_iterable($value)) { - return XMLNode::nestedNode($name); - } - - $elements = []; + /** @var list> $attributes */ + $attributes = []; + /** @var list> $children */ + $children = []; + + foreach ($definitions as $definition) { + $name = $definition->entry()->name(); + $type = $definition->type(); + $bare = type_bare($type); + $physicals = $rows->column($name)->physicals(); + + if (str_starts_with($name, $this->attributePrefix)) { + $attributes[] = $writer->attributes( + substr($name, strlen($this->attributePrefix)), + $this->text->texts($type, $physicals), + ); - // @mago-ignore analysis:mixed-assignment - foreach ($value as $elementValue) { - $elements[] = $this->normalize($this->listElementName, $type->element(), $elementValue); + continue; } - return XMLNode::nested($name, ...$elements); - } + if ($bare instanceof ListType || $bare instanceof MapType || $bare instanceof StructureType) { + $nodes = []; - if ($type instanceof MapType) { - if (!is_array($value) && !$value instanceof Countable || !count($value) || !is_iterable($value)) { - return XMLNode::nestedNode($name); - } + // @mago-ignore analysis:mixed-assignment + foreach ($physicals as $physical) { + $nodes[] = $writer->write($this->nodes->of($name, $type, $physical)); + } - $elements = []; + $children[] = $nodes; - // @mago-ignore analysis:mixed-assignment - foreach ($value as $key => $elementValue) { - $elements[] = XMLNode::nested( - $this->mapElementName, - $this->normalize($this->mapElementKeyName, $type->key(), $key), - $this->normalize($this->mapElementValueName, $type->value(), $elementValue), - ); + continue; } - return XMLNode::nested($name, ...$elements); + $children[] = $writer->elements($name, $this->text->texts($type, $physicals)); } - if ($type instanceof StructureType) { - $values = is_array($value) ? array_values($value) : []; - - if (count($values) > count($type->elements())) { - throw new RuntimeException(sprintf( - 'XML encoder received %d values for structure "%s" which declares %d elements - extra values would be silently lost', - count($values), - $type->toString(), - count($type->elements()), - )); - } + if ($attributes === [] && $children === []) { + return str_repeat('<' . $this->rowElementName . "/>\n", $count); + } - $elements = []; + $pieces = [...$attributes, ...$children]; + $pieces[0] = substr_replace($pieces[0], '<' . $this->rowElementName . ($attributes === [] ? '>' : ''), 0, 0); - foreach ($type->elements() as $position => $element) { - $elements[] = $this->normalize((string) $element->name, $element->type, $values[$position] ?? null); - } - - return XMLNode::nested($name, ...$elements); + if ($attributes !== [] && $children !== []) { + $pieces[count($attributes) - 1] = substr_replace($pieces[count($attributes) - 1], '>', PHP_INT_MAX, 0); } - return match ($type::class) { - StringType::class, IntegerType::class, BooleanType::class, FloatType::class => XMLNode::flatNode( - $name, - type_string()->cast($value), - ), - ArrayType::class => XMLNode::flatNode( - $name, - is_array($value) ? json_encode($value, JSON_THROW_ON_ERROR) : '', - ), - EnumType::class => XMLNode::flatNode($name, $value instanceof BackedEnum ? $value->name : ''), - InstanceOfType::class => XMLNode::flatNode($name, type_string()->cast($value)), - DateTimeType::class => XMLNode::flatNode( - $name, - type_string()->cast($value instanceof DateTimeInterface ? $value->format($this->dateTimeFormat) : ''), - ), - DateType::class => XMLNode::flatNode( - $name, - type_string()->cast($value instanceof DateTimeInterface ? $value->format($this->dateFormat) : ''), - ), - TimeType::class => XMLNode::flatNode( - $name, - $value instanceof DateInterval ? (string) date_interval_to_microseconds($value) : '', - ), - JsonType::class => XMLNode::flatNode($name, $value instanceof Stringable ? $value->__toString() : ''), - UuidType::class => XMLNode::flatNode( - $name, - is_scalar($value) || $value instanceof Stringable ? (string) $value : '', - ), - TimeZoneType::class => XMLNode::flatNode($name, $value instanceof DateTimeZone ? $value->getName() : ''), - XMLType::class, XMLElementType::class => XMLNode::flatNode($name, $this->xmlToString($value)), - default => throw new InvalidArgumentException( - "Given type can't be converted to node, given type: {$type->toString()}", - ), - }; - } + $last = count($pieces) - 1; + $pieces[$last] = substr_replace( + $pieces[$last], + $children === [] ? "/>\n" : 'rowElementName . ">\n", + PHP_INT_MAX, + 0, + ); - private function xmlToString(mixed $value): string - { - if ($value instanceof XMLDocument) { - $serialized = $value->saveXml($value->documentElement); - } elseif ($value instanceof DOMDocument) { - $serialized = $value->saveXML($value->documentElement); - } elseif ($value instanceof Stringable || is_scalar($value)) { - return (string) $value; - } else { - return ''; + if ($last === 0) { + return implode('', $pieces[0]); } - if ($serialized === false) { - throw new RuntimeException('Failed to serialize XML document.'); + $lines = ''; + + foreach (array_map(null, ...$pieces) as $row) { + $lines .= implode('', $row); } - return $serialized; + return $lines; } } diff --git a/src/adapter/etl-adapter-xml/src/Flow/ETL/Adapter/XML/XMLFileBatches.php b/src/adapter/etl-adapter-xml/src/Flow/ETL/Adapter/XML/XMLFileBatches.php new file mode 100644 index 0000000000..d04e0a57a2 --- /dev/null +++ b/src/adapter/etl-adapter-xml/src/Flow/ETL/Adapter/XML/XMLFileBatches.php @@ -0,0 +1,64 @@ + $bufferSize + */ + public function __construct( + private Filesystem $filesystem, + private string $xmlNodePath, + private int $bufferSize, + ) {} + + /** + * @param int<1, max> $batchSize + * + * @return Generator + */ + public function batches( + SourceFile $source, + Schema $body, + int $batchSize, + Backend $backend, + ReadWindow $window, + ): Generator { + $batches = new XMLNodeBatches(new XMLNodes($this->xmlNodePath), $batchSize, $this->bufferSize); + $node = $body->get('node'); + $stream = $this->filesystem->readFrom($source->path); + + try { + yield from match (true) { + $node instanceof XMLDefinition && count($body->definitions()) === 1 => $batches->physicals( + $stream, + $node, + $body, + $backend, + ), + $node instanceof StringDefinition => $batches->strings($stream, $body, $backend), + default => $batches->documents($stream, $body, $backend), + }; + } finally { + $stream->close(); + } + + return 0; + } +} diff --git a/src/adapter/etl-adapter-xml/src/Flow/ETL/Adapter/XML/XMLNodeBatches.php b/src/adapter/etl-adapter-xml/src/Flow/ETL/Adapter/XML/XMLNodeBatches.php new file mode 100644 index 0000000000..d48d696cd5 --- /dev/null +++ b/src/adapter/etl-adapter-xml/src/Flow/ETL/Adapter/XML/XMLNodeBatches.php @@ -0,0 +1,127 @@ + $batchSize + * @param int<1, max> $bufferSize bytes read from the stream at a time + */ + public function __construct( + private XMLNodes $nodes, + private int $batchSize, + private int $bufferSize, + ) {} + + /** + * @return Generator + */ + public function documents(SourceStream $stream, Schema $body, Backend $backend): Generator + { + $node = $body->get('node'); + $nodes = []; + + foreach ($this->nodes->documents($stream, $this->bufferSize) as $document) { + $nodes[] = $document; + + if (count($nodes) >= $this->batchSize) { + yield $this->batch($node, $nodes, $body, $backend); + + $nodes = []; + } + } + + if ($nodes !== []) { + yield $this->batch($node, $nodes, $body, $backend); + } + } + + /** + * @return Generator + */ + public function physicals(SourceStream $stream, XMLDefinition $node, Schema $body, Backend $backend): Generator + { + $documents = new XmlDocumentPhysical(); + $batch = []; + + foreach ($this->nodes->texts($stream, $this->bufferSize) as $text) { + $batch[] = $documents->physical($text); + + if (count($batch) >= $this->batchSize) { + $builder = $backend->builder($node); + $builder->appendPhysicals($batch, 0); + + yield Rows::fromColumns(new Schema($node), ['node' => $builder->finish()], count($batch))->matchTo( + $body, + $backend, + ); + + $batch = []; + } + } + + if ($batch !== []) { + $builder = $backend->builder($node); + $builder->appendPhysicals($batch, 0); + + yield Rows::fromColumns(new Schema($node), ['node' => $builder->finish()], count($batch))->matchTo( + $body, + $backend, + ); + } + } + + /** + * @return Generator + */ + public function strings(SourceStream $stream, Schema $body, Backend $backend): Generator + { + $node = $body->get('node'); + $nodes = []; + + foreach ($this->nodes->texts($stream, $this->bufferSize) as $text) { + $nodes[] = $text; + + if (count($nodes) >= $this->batchSize) { + yield $this->batch($node, $nodes, $body, $backend); + + $nodes = []; + } + } + + if ($nodes !== []) { + yield $this->batch($node, $nodes, $body, $backend); + } + } + + /** + * The node column built from $nodes; every other column $body declares is padded as matchTo() pads it. + * + * @param Definition $node + * @param non-empty-list $nodes + */ + public function batch(Definition $node, array $nodes, Schema $body, Backend $backend): Rows + { + $builder = $backend->builder($node); + $builder->appendMany($nodes); + + return Rows::fromColumns(new Schema($node), ['node' => $builder->finish()], count($nodes))->matchTo( + $body, + $backend, + ); + } +} diff --git a/src/adapter/etl-adapter-xml/src/Flow/ETL/Adapter/XML/XMLNodeCursor.php b/src/adapter/etl-adapter-xml/src/Flow/ETL/Adapter/XML/XMLNodeCursor.php new file mode 100644 index 0000000000..60a86d7745 --- /dev/null +++ b/src/adapter/etl-adapter-xml/src/Flow/ETL/Adapter/XML/XMLNodeCursor.php @@ -0,0 +1,122 @@ + + */ + private ?array $target; + + /** + * @param string $xmlNodePath slash-separated element names from the root, '' for the root itself + */ + public function __construct(string $xmlNodePath) + { + $this->target = $xmlNodePath === '' ? null : explode('/', $xmlNodePath); + } + + /** + * @param int<1, max> $bufferSize bytes read from the stream at a time + * + * @return Generator + */ + public function of(SourceStream $stream, int $bufferSize): Generator + { + $depth = $this->target === null ? 0 : count($this->target) - 1; + $reader = new XMLReader(); + // the buffer, and its last error, are cleared at every node so the end-of-document check sees this read's + // errors alone + $internalErrors = libxml_use_internal_errors(true); + libxml_clear_errors(); + + try { + $moved = $reader->open(SourceStreamWrapper::uri($stream, $bufferSize)) && $reader->read(); + + while ($moved) { + if ($reader->nodeType !== XMLReader::ELEMENT) { + $moved = $reader->read(); + + continue; + } + + // an element off the path is skipped whole, so every ancestor of the cursor is on the path + if ($this->target !== null && $reader->name !== $this->target[$reader->depth]) { + $moved = $reader->next(); + + continue; + } + + if ($reader->depth < $depth) { + $moved = $reader->read(); + + continue; + } + + yield $reader; + + libxml_use_internal_errors(true); + libxml_clear_errors(); + $moved = $reader->next(); + } + + $error = libxml_get_last_error(); + + if ( + $error !== false + && $error->level === LIBXML_ERR_FATAL + && $error->code !== self::DOCUMENT_EMPTY + && !($error->code === self::EXTRA_CONTENT && (new RootlessDocument())->matches($stream, $bufferSize)) + ) { + throw $this->failure(); + } + } finally { + $reader->close(); + libxml_use_internal_errors($internalErrors); + } + } + + public function failure(): RuntimeException + { + $error = libxml_get_last_error(); + + return new RuntimeException( + $error === false + ? 'XML Error: the document could not be read' + : sprintf('XML Error: %s at line %d', trim($error->message), $error->line), + ); + } +} diff --git a/src/adapter/etl-adapter-xml/src/Flow/ETL/Adapter/XML/XMLNodes.php b/src/adapter/etl-adapter-xml/src/Flow/ETL/Adapter/XML/XMLNodes.php index a03e4ff7cf..bc1e7b82f5 100644 --- a/src/adapter/etl-adapter-xml/src/Flow/ETL/Adapter/XML/XMLNodes.php +++ b/src/adapter/etl-adapter-xml/src/Flow/ETL/Adapter/XML/XMLNodes.php @@ -5,132 +5,76 @@ namespace Flow\ETL\Adapter\XML; use DOMDocument; -use Flow\ETL\Exception\RuntimeException; +use Flow\ETL\Column\Physical\XmlCharacterReferences; use Flow\Filesystem\SourceStream; -use Flow\Filesystem\Stream\SourceStreamWrapper; use Generator; -use XMLReader; -use function count; -use function explode; -use function libxml_clear_errors; -use function libxml_get_last_error; use function libxml_use_internal_errors; -use function sprintf; -use function trim; - -use const LIBXML_ERR_FATAL; /** - * libxml builds each node's DOM in C, so no element, attribute or text of the document passes through PHP. + * libxml builds each node in C, so no element, attribute or text of the document passes through PHP. */ final readonly class XMLNodes { - /** - * libxml's XML_ERR_DOCUMENT_EMPTY: no root element - an empty, blank or declaration-only file, which yields nothing - */ - private const int DOCUMENT_EMPTY = 4; - - /** - * libxml's XML_ERR_DOCUMENT_END, which libxml before 2.12 also reports for a document without a root - */ - private const int EXTRA_CONTENT = 5; - - /** - * @var null|non-empty-list - */ - private ?array $target; + private XMLNodeCursor $cursor; /** * @param string $xmlNodePath slash-separated element names from the root, '' for the root itself */ public function __construct(string $xmlNodePath) { - $this->target = $xmlNodePath === '' ? null : explode('/', $xmlNodePath); + $this->cursor = new XMLNodeCursor($xmlNodePath); } /** - * Each element on the path as a document of its own, declaring every namespace it uses. + * Each element on the path as a UTF-8 document of its own, declaring every namespace it uses. * * @param int<1, max> $bufferSize bytes read from the stream at a time * * @return Generator */ - public function of(SourceStream $stream, int $bufferSize): Generator + public function documents(SourceStream $stream, int $bufferSize): Generator { - $depth = $this->target === null ? 0 : count($this->target) - 1; - $reader = new XMLReader(); - // libxml reports into its own buffer only while this generator runs - never across a yield. The buffer, and - // its last error, are cleared each time so the end-of-document check sees this read's errors alone. - $internalErrors = libxml_use_internal_errors(true); - libxml_clear_errors(); - - try { - $moved = $reader->open(SourceStreamWrapper::uri($stream, $bufferSize)) && $reader->read(); - - while ($moved) { - if ($reader->nodeType !== XMLReader::ELEMENT) { - $moved = $reader->read(); - - continue; - } + $internalErrors = libxml_use_internal_errors(); - // an element off the path is skipped whole, so every ancestor of the cursor is on the path - if ($this->target !== null && $reader->name !== $this->target[$reader->depth]) { - $moved = $reader->next(); + foreach ($this->cursor->of($stream, $bufferSize) as $reader) { + $document = new DOMDocument('1.0', 'UTF-8'); + // its PHP warning repeats the libxml error failure() reports + $node = @$reader->expand($document); - continue; - } - - if ($reader->depth < $depth) { - $moved = $reader->read(); - - continue; - } - - $document = new DOMDocument(); - // its PHP warning repeats the libxml error failure() reports - $node = @$reader->expand($document); + if ($node === false) { + throw $this->cursor->failure(); + } - if ($node === false) { - throw $this->failure(); - } + $document->appendChild($node); + libxml_use_internal_errors($internalErrors); - $document->appendChild($node); + yield $document; + } + } - libxml_use_internal_errors($internalErrors); + /** + * Each element on the path as its outer XML: the text of the document documents() yields for it. + * + * @param int<1, max> $bufferSize bytes read from the stream at a time + * + * @return Generator + */ + public function texts(SourceStream $stream, int $bufferSize): Generator + { + $internalErrors = libxml_use_internal_errors(); + $references = new XmlCharacterReferences(); - yield $document; + foreach ($this->cursor->of($stream, $bufferSize) as $reader) { + $text = @$reader->readOuterXml(); - $internalErrors = libxml_use_internal_errors(true); - libxml_clear_errors(); - $moved = $reader->next(); + if ($text === '') { + throw $this->cursor->failure(); } - $error = libxml_get_last_error(); - - if ( - $error !== false - && $error->level === LIBXML_ERR_FATAL - && $error->code !== self::DOCUMENT_EMPTY - && !($error->code === self::EXTRA_CONTENT && (new RootlessDocument())->matches($stream, $bufferSize)) - ) { - throw $this->failure(); - } - } finally { - $reader->close(); libxml_use_internal_errors($internalErrors); - } - } - - private function failure(): RuntimeException - { - $error = libxml_get_last_error(); - return new RuntimeException( - $error === false - ? 'XML Error: the document could not be read' - : sprintf('XML Error: %s at line %d', trim($error->message), $error->line), - ); + yield $references->decoded($text); + } } } diff --git a/src/adapter/etl-adapter-xml/src/Flow/ETL/Adapter/XML/XMLOpenSink.php b/src/adapter/etl-adapter-xml/src/Flow/ETL/Adapter/XML/XMLOpenSink.php new file mode 100644 index 0000000000..58556d5985 --- /dev/null +++ b/src/adapter/etl-adapter-xml/src/Flow/ETL/Adapter/XML/XMLOpenSink.php @@ -0,0 +1,40 @@ +written) { + $this->stream->append('rootElementName . '>'); + } + } + + public function write(Rows $rows): void + { + $this->stream->append(($this->written ? '' : $this->head) . $this->encoder->encode($rows)); + $this->written = true; + } +} diff --git a/src/adapter/etl-adapter-xml/src/Flow/ETL/Adapter/XML/XMLParserExtractor.php b/src/adapter/etl-adapter-xml/src/Flow/ETL/Adapter/XML/XMLParserExtractor.php index 4d64eee9b0..08ac7ea654 100644 --- a/src/adapter/etl-adapter-xml/src/Flow/ETL/Adapter/XML/XMLParserExtractor.php +++ b/src/adapter/etl-adapter-xml/src/Flow/ETL/Adapter/XML/XMLParserExtractor.php @@ -8,15 +8,16 @@ use Flow\ETL\Extractor; use Flow\ETL\Extractor\BatchableExtractor; use Flow\ETL\Extractor\Batches; -use Flow\ETL\Extractor\FileExtractor; -use Flow\ETL\Extractor\FileReading; -use Flow\ETL\Extractor\ListedFiles; -use Flow\ETL\Extractor\MetadataColumnsExtractor; +use Flow\ETL\Extractor\File\FileExtractor; +use Flow\ETL\Extractor\File\FileReading; +use Flow\ETL\Extractor\File\FileReadLoop; +use Flow\ETL\Extractor\File\ListedFiles; +use Flow\ETL\Extractor\File\MetadataColumnsExtractor; +use Flow\ETL\Extractor\File\ReadWindow; use Flow\ETL\Extractor\RewindableExtractor; use Flow\ETL\Extractor\Signal; use Flow\ETL\Extractor\Statistics; use Flow\ETL\FlowContext; -use Flow\ETL\Row\RawRowValues; use Flow\ETL\Rows; use Flow\ETL\Schema; use Flow\Filesystem\Filesystem; @@ -26,9 +27,10 @@ use Flow\Filesystem\Path\Filter\OnlyFiles; use Generator; -use function count; use function Flow\ETL\DSL\schema; use function Flow\ETL\DSL\xml_schema; +use function implode; +use function iterator_to_array; use function sprintf; final class XMLParserExtractor implements @@ -96,66 +98,18 @@ public function isRepeatable(): bool */ public function extract(FlowContext $context, ?int $limit = null, Filter $pathFilter = new OnlyFiles()): Generator { - $hydrator = $context->hydrator(); - $batchSize = $this->batchSize(); - $yielded = 0; - $nodes = new XMLNodes($this->xmlNodePath); - - $baseSchema = $this->schema ?? schema(xml_schema('node')); - $fileColumns = $this->fileColumns($this->filesystem, $this->path); - $schema = $fileColumns->declare($baseSchema); - $body = $fileColumns->withoutTail($schema); - - foreach ($this->sourceFiles($this->filesystem, $this->path, $pathFilter) as $source) { - $stream = $this->filesystem->readFrom($source->path); - - try { - $constants = $fileColumns->forFile($source, $schema); - - $batch = []; - - foreach ($nodes->of($stream, $this->bufferSize) as $node) { - $batch[] = new RawRowValues(['node' => $node]); - - if (count($batch) >= $batchSize) { - $hydrated = $constants->fillRows($hydrator->hydrate($batch, $body), $schema); - - $batch = []; - - $yielded += $hydrated->count(); - - $signal = yield $hydrated; - - if ($signal === Signal::STOP) { - return; - } - if ($limit !== null && $yielded >= $limit) { - return; - } - } - } - - if ($batch !== []) { - $hydrated = $constants->fillRows($hydrator->hydrate($batch, $body), $schema); - - $yielded += $hydrated->count(); - - $signal = yield $hydrated; - - if ($signal === Signal::STOP) { - return; - } - - if ($limit !== null && $yielded >= $limit) { - return; - } - } - } finally { - $stream->close(); - } - } + yield from (new FileReadLoop( + $fileColumns, + $fileColumns->declare($this->schema ?? schema(xml_schema('node'))), + ))->read( + iterator_to_array($this->sourceFiles($this->filesystem, $this->path, $pathFilter), false), + new XMLFileBatches($this->filesystem, $this->xmlNodePath, $this->bufferSize), + $this->batchSize(), + $context->backend(), + new ReadWindow(limit: $limit), + ); } public function schema(): Schema @@ -190,8 +144,18 @@ public function withBufferSize(int $bufferSize): self return $this; } + /** + * @throws InvalidArgumentException when $schema has no "node" column, the one the matched elements are read into + */ public function withSchema(Schema $schema): static { + if ($schema->findDefinition('node') === null) { + throw new InvalidArgumentException(sprintf('from_xml() reads every matched element into the "node" column, the schema declares no such column: %s', implode( + ', ', + $schema->references()->names(), + ))); + } + $this->schema = $schema; return $this; diff --git a/src/adapter/etl-adapter-xml/src/Flow/ETL/Adapter/XML/XMLReaderExtractor.php b/src/adapter/etl-adapter-xml/src/Flow/ETL/Adapter/XML/XMLReaderExtractor.php deleted file mode 100644 index a6541813b4..0000000000 --- a/src/adapter/etl-adapter-xml/src/Flow/ETL/Adapter/XML/XMLReaderExtractor.php +++ /dev/null @@ -1,229 +0,0 @@ - nodes us root/elements/element. - * - * - * - * - * - * - * - * - * $xmlNodePath does not support attributes and it's not xpath, it is just a sequence - * of node names separated with slash. - * - * @param string $xmlNodePath - */ - public function __construct( - private readonly Path $path, - private readonly string $xmlNodePath = '', - Filesystem $filesystem = new NativeLocalFilesystem(), - ) { - if (!$filesystem->supports($path)) { - throw new InvalidArgumentException(sprintf( - 'Filesystem %s serves "%s://" paths, given: "%s". Pass the filesystem that handles ' - . 'this scheme, e.g. from_xml($path, filesystem: aws_s3_filesystem(...)).', - $filesystem::class, - $filesystem->mount()->protocol, - $path->uri(), - )); - } - - $this->filesystem = $filesystem; - - if (!$this->path->isLocal()) { - throw new InvalidArgumentException( - 'XMLReaderExtractor supports only local files, please use XMLParserExtractor that depends on php-xml extension.', - ); - } - } - - public function isRepeatable(): bool - { - return true; - } - - /** - * @return Generator - */ - public function extract(FlowContext $context, ?int $limit = null, Filter $pathFilter = new OnlyFiles()): Generator - { - $hydrator = $context->hydrator(); - $batchSize = $this->batchSize(); - $yielded = 0; - $encoder = new XMLEncoder(); - - $baseSchema = $this->schema ?? schema(xml_schema('node')); - - $fileColumns = $this->fileColumns($this->filesystem, $this->path); - $schema = $fileColumns->declare($baseSchema); - $body = $fileColumns->withoutTail($schema); - - foreach ($this->sourceFiles($this->filesystem, $this->path, $pathFilter) as $source) { - $constants = $fileColumns->forFile($source, $schema); - - $xmlReader = new XMLReader(); - $xmlReader->open($source->path->path()); - - try { - $previousDepth = 0; - $currentPathBreadCrumbs = []; - - $rawNodes = []; - - while ($xmlReader->read()) { - if ($xmlReader->nodeType === XMLReader::ELEMENT) { - if ($previousDepth === $xmlReader->depth) { - array_pop($currentPathBreadCrumbs); - $currentPathBreadCrumbs[] = $xmlReader->name; - } - - if ($xmlReader->depth > $previousDepth) { - $currentPathBreadCrumbs[] = $xmlReader->name; - } - - while ($xmlReader->depth < $previousDepth) { - array_pop($currentPathBreadCrumbs); - $previousDepth--; - } - - $currentPath = implode('/', array_map(strval(...), $currentPathBreadCrumbs)); - - if ( - $currentPath === $this->xmlNodePath - || $this->xmlNodePath === '' && $xmlReader->depth === 0 - ) { - $dom = new DOMDocument('1.0', ''); - $node = $xmlReader->expand($dom); - $rawNodes[] = $node === false ? '' : (string) $dom->saveXML($node); - - if (count($rawNodes) >= $batchSize) { - $hydrated = $constants->fillRows( - $hydrator->hydrate($encoder->decode($rawNodes), $body), - $schema, - ); - - $rawNodes = []; - - $yielded += $hydrated->count(); - - $signal = yield $hydrated; - - if ($signal === Signal::STOP) { - return; - } - - if ($limit !== null && $yielded >= $limit) { - return; - } - } - } - - $previousDepth = $xmlReader->depth; - } - } - - if ($rawNodes !== []) { - $hydrated = $constants->fillRows($hydrator->hydrate($encoder->decode($rawNodes), $body), $schema); - - $yielded += $hydrated->count(); - - $signal = yield $hydrated; - - if ($signal === Signal::STOP) { - return; - } - - if ($limit !== null && $yielded >= $limit) { - return; - } - } - } finally { - $xmlReader->close(); - } - } - } - - public function schema(): Schema - { - return $this->fileColumns($this->filesystem, $this->path)->declare($this->schema ?? schema(xml_schema('node'))); - } - - public function partitionSchema(): Schema - { - return $this->fileColumns($this->filesystem, $this->path)->partitions($this->schema ?? new Schema()); - } - - public function source(): Path - { - return $this->path; - } - - public function statistics(): Statistics - { - $this->listed ??= ListedFiles::of($this->sourceFiles($this->filesystem, $this->path)); - - return new Statistics(size: $this->listed->bytes); - } - - public function withSchema(Schema $schema): static - { - $this->schema = $schema; - - return $this; - } -} diff --git a/src/adapter/etl-adapter-xml/src/Flow/ETL/Adapter/XML/XMLValueNodes.php b/src/adapter/etl-adapter-xml/src/Flow/ETL/Adapter/XML/XMLValueNodes.php new file mode 100644 index 0000000000..c31835af37 --- /dev/null +++ b/src/adapter/etl-adapter-xml/src/Flow/ETL/Adapter/XML/XMLValueNodes.php @@ -0,0 +1,140 @@ + $type + */ + public function of(string $name, Type $type, mixed $physical): XMLNode + { + if ($physical === null) { + return XMLNode::flatNode($name, ''); + } + + $bare = type_bare($type); + + if ($bare instanceof ListType) { + /** @var list $physical */ + $elements = []; + + // @mago-ignore analysis:mixed-assignment + foreach ($physical as $element) { + $elements[] = $this->member($this->listElementName, $bare->element(), $element); + } + + return $elements === [] ? XMLNode::nestedNode($name) : XMLNode::nested($name, ...$elements); + } + + if ($bare instanceof MapType) { + /** @var array $physical */ + $elements = []; + + // @mago-ignore analysis:mixed-assignment + foreach ($physical as $key => $value) { + $elements[] = XMLNode::nested( + $this->mapElementName, + str_starts_with($this->mapElementKeyName, $this->attributePrefix) + ? new XMLAttribute( + substr($this->mapElementKeyName, strlen($this->attributePrefix)), + (string) $key, + ) + : XMLNode::flatNode($this->mapElementKeyName, (string) $key), + $this->member($this->mapElementValueName, $bare->value(), $value), + ); + } + + return $elements === [] ? XMLNode::nestedNode($name) : XMLNode::nested($name, ...$elements); + } + + if ($bare instanceof StructureType) { + /** @var array $physical */ + $elements = []; + + foreach ($bare->elements() as $element) { + $elements[] = $this->member( + (string) $element->name, + $element->type, + array_key_exists($element->name, $physical) ? $physical[$element->name] : null, + ); + } + + return XMLNode::nested($name, ...$elements); + } + + return XMLNode::flatNode($name, $this->text->texts($type, [$physical])[0] ?? ''); + } + + /** + * @param Type $type + */ + public function member(string $name, Type $type, mixed $physical): XMLAttribute|XMLNode + { + if (!str_starts_with($name, $this->attributePrefix)) { + return $this->of($name, $type, $physical); + } + + return new XMLAttribute( + substr($name, strlen($this->attributePrefix)), + type_string()->cast($this->text->texts($type, [$physical])[0] ?? null), + ); + } + + /** + * @param Type $type + */ + public function refuseOptionalElements(Type $type): void + { + $bare = type_bare($type); + + if ($bare instanceof ListType) { + $this->refuseOptionalElements($bare->element()); + } + + if ($bare instanceof MapType) { + $this->refuseOptionalElements($bare->value()); + } + + if ($bare instanceof StructureType) { + foreach ($bare->elements() as $element) { + if ($element->optional) { + throw new RuntimeException(sprintf( + 'XML encoder does not support structure optional elements, given: %s', + $bare->toString(), + )); + } + + $this->refuseOptionalElements($element->type); + } + } + } +} diff --git a/src/adapter/etl-adapter-xml/src/Flow/ETL/Adapter/XML/XMLWriter.php b/src/adapter/etl-adapter-xml/src/Flow/ETL/Adapter/XML/XMLWriter.php index e9383414e8..073a3d31b0 100644 --- a/src/adapter/etl-adapter-xml/src/Flow/ETL/Adapter/XML/XMLWriter.php +++ b/src/adapter/etl-adapter-xml/src/Flow/ETL/Adapter/XML/XMLWriter.php @@ -8,5 +8,19 @@ interface XMLWriter { + /** + * @param list $values + * + * @return list ` name="value"` per value + */ + public function attributes(string $name, array $values): array; + + /** + * @param list $values + * + * @return list `value` per value; null and '' give `` + */ + public function elements(string $name, array $values): array; + public function write(XMLNode $node): string; } diff --git a/src/adapter/etl-adapter-xml/src/Flow/ETL/Adapter/XML/XMLWriter/DOMDocumentWriter.php b/src/adapter/etl-adapter-xml/src/Flow/ETL/Adapter/XML/XMLWriter/DOMDocumentWriter.php index d65bc73b23..7448eca5c3 100644 --- a/src/adapter/etl-adapter-xml/src/Flow/ETL/Adapter/XML/XMLWriter/DOMDocumentWriter.php +++ b/src/adapter/etl-adapter-xml/src/Flow/ETL/Adapter/XML/XMLWriter/DOMDocumentWriter.php @@ -6,12 +6,31 @@ use DOMDocument; use DOMElement; +use Flow\ETL\Adapter\XML\Abstraction\XMLAttribute; use Flow\ETL\Adapter\XML\Abstraction\XMLNode; use Flow\ETL\Adapter\XML\XMLWriter; use Flow\ETL\Exception\RuntimeException; +use function array_map; +use function Flow\Types\DSL\type_string; +use function substr; + final class DOMDocumentWriter implements XMLWriter { + public function attributes(string $name, array $values): array + { + return array_map(fn(?string $value): string => substr( + $this->write(XMLNode::nested('x', new XMLAttribute($name, type_string()->cast($value)))), + 2, + -2, + ), $values); + } + + public function elements(string $name, array $values): array + { + return array_map(fn(?string $value): string => $this->write(XMLNode::flatNode($name, $value ?? '')), $values); + } + public function write(XMLNode $node): string { $dom = new DOMDocument(); diff --git a/src/adapter/etl-adapter-xml/src/Flow/ETL/Adapter/XML/XMLWriter/StringXMLWriter.php b/src/adapter/etl-adapter-xml/src/Flow/ETL/Adapter/XML/XMLWriter/StringXMLWriter.php index 971ffccba4..3975c7a8de 100644 --- a/src/adapter/etl-adapter-xml/src/Flow/ETL/Adapter/XML/XMLWriter/StringXMLWriter.php +++ b/src/adapter/etl-adapter-xml/src/Flow/ETL/Adapter/XML/XMLWriter/StringXMLWriter.php @@ -7,11 +7,17 @@ use DOMDocument; use Flow\ETL\Adapter\XML\Abstraction\XMLNode; use Flow\ETL\Adapter\XML\XMLWriter; +use Flow\Types\Exception\CastingException; use function array_key_exists; +use function array_keys; +use function Flow\Types\DSL\type_string; +use function in_array; +use function preg_grep; use function str_contains; use function strstr; use function strtr; +use function substr_replace; /** * DOMDocumentWriter's output on libxml 2.14 and later, byte for byte, without building a DOM for every row: libxml's @@ -109,6 +115,36 @@ final class StringXMLWriter implements XMLWriter private ?DOMDocument $validator = null; + public function attributes(string $name, array $values): array + { + $name = $this->attributeName($name); + + if (in_array(null, $values, true)) { + throw new CastingException(null, type_string()); + } + + /** @var list */ + return substr_replace( + substr_replace($this->escapeAll($values, self::ATTRIBUTE, '/[&<>"\x00-\x1F]/'), ' ' . $name . '="', 0, 0), + '"', + PHP_INT_MAX, + 0, + ); + } + + public function elements(string $name, array $values): array + { + $name = $this->elementName($name); + + /** @var list */ + return substr_replace( + substr_replace($this->escapeAll($values, self::TEXT, '/[&<>\x00-\x08\x0B-\x1F]/'), '<' . $name . '>', 0, 0), + '', + PHP_INT_MAX, + 0, + ); + } + public function write(XMLNode $node): string { $name = $this->elementName($node->name); @@ -155,6 +191,28 @@ private static function escape(string $value, array $map): string return strtr($value, $map); } + /** + * @param list $values null is '' + * @param array $map + * + * @return list + */ + private function escapeAll(array $values, array $map, string $escaped): array + { + foreach (array_keys($values, null, true) as $i) { + $values[$i] = ''; + } + + /** @var list $texts */ + $texts = $values; + + foreach (preg_grep($escaped, $texts) ?: [] as $i => $text) { + $texts[$i] = self::escape($text, $map); + } + + return $texts; + } + private function attributeName(string $name): string { if (!array_key_exists($name, $this->names)) { diff --git a/src/adapter/etl-adapter-xml/tests/Flow/ETL/Adapter/XML/Tests/Context/SerializedNodes.php b/src/adapter/etl-adapter-xml/tests/Flow/ETL/Adapter/XML/Tests/Context/SerializedNodes.php index 773a197585..42e72d9c75 100644 --- a/src/adapter/etl-adapter-xml/tests/Flow/ETL/Adapter/XML/Tests/Context/SerializedNodes.php +++ b/src/adapter/etl-adapter-xml/tests/Flow/ETL/Adapter/XML/Tests/Context/SerializedNodes.php @@ -24,7 +24,7 @@ public static function of(string $xmlNodePath, string $xml, int $bufferSize = 81 return array_map( static fn(DOMDocument $document): string => (string) $document->saveXML($document->documentElement), iterator_to_array( - (new XMLNodes($xmlNodePath))->of( + (new XMLNodes($xmlNodePath))->documents( new StringSourceStream(path('memory://document.xml'), $xml), $bufferSize, ), diff --git a/src/adapter/etl-adapter-xml/tests/Flow/ETL/Adapter/XML/Tests/Context/XMLWriterValues.php b/src/adapter/etl-adapter-xml/tests/Flow/ETL/Adapter/XML/Tests/Context/XMLWriterValues.php new file mode 100644 index 0000000000..aef9c8d1c6 --- /dev/null +++ b/src/adapter/etl-adapter-xml/tests/Flow/ETL/Adapter/XML/Tests/Context/XMLWriterValues.php @@ -0,0 +1,31 @@ + + */ + public static function values(): Generator + { + yield 'markup characters' => ['a & b < c > d " e \' f']; + yield 'an entity-looking text' => ['& <']; + yield 'line breaks and a tab' => ["a\rb\nc\td"]; + yield 'a value cut at its first NUL' => ["a\0b"]; + yield 'only a NUL' => ["\0"]; + yield 'an empty string' => ['']; + yield 'a multi-byte string' => ['zażółć gęślą jaźń ☃']; + yield 'plain text' => ['plain']; + + for ($byte = 1; $byte < 32; $byte++) { + yield 'the control byte ' . $byte => ['a' . chr($byte) . 'b']; + } + } +} diff --git a/src/adapter/etl-adapter-xml/tests/Flow/ETL/Adapter/XML/Tests/Integration/XMLExtractorBatchContractTest.php b/src/adapter/etl-adapter-xml/tests/Flow/ETL/Adapter/XML/Tests/Integration/XMLExtractorBatchContractTest.php index 95b342869b..cde649fac1 100644 --- a/src/adapter/etl-adapter-xml/tests/Flow/ETL/Adapter/XML/Tests/Integration/XMLExtractorBatchContractTest.php +++ b/src/adapter/etl-adapter-xml/tests/Flow/ETL/Adapter/XML/Tests/Integration/XMLExtractorBatchContractTest.php @@ -5,12 +5,10 @@ namespace Flow\ETL\Adapter\XML\Tests\Integration; use Flow\ETL\Adapter\XML\XMLParserExtractor; -use Flow\ETL\Adapter\XML\XMLReaderExtractor; use Flow\ETL\Tests\Context\ExtractedRows; use Flow\ETL\Tests\FlowTestCase; use function Flow\ETL\Adapter\XML\from_xml; -use function Flow\Filesystem\DSL\path_real; final class XMLExtractorBatchContractTest extends FlowTestCase { @@ -28,20 +26,4 @@ public function test_xml_parser_extractor_honours_the_batch_contract(): void ), ); } - - public function test_xml_reader_extractor_honours_the_batch_contract(): void - { - self::assertExtractorHonoursBatchContract( - // @mago-ignore analysis:deprecated-class - static fn(): XMLReaderExtractor => new XMLReaderExtractor( - path_real(__DIR__ . '/../Fixtures/simple_items.xml'), - 'root/items/item', - ), - // @mago-ignore analysis:deprecated-class - ExtractedRows::of((new XMLReaderExtractor( - path_real(__DIR__ . '/../Fixtures/simple_items.xml'), - 'root/items/item', - ))->withBatchSize(1)), - ); - } } diff --git a/src/adapter/etl-adapter-xml/tests/Flow/ETL/Adapter/XML/Tests/Integration/XMLExtractorTypedColumnsTest.php b/src/adapter/etl-adapter-xml/tests/Flow/ETL/Adapter/XML/Tests/Integration/XMLExtractorTypedColumnsTest.php new file mode 100644 index 0000000000..57c3927ad6 --- /dev/null +++ b/src/adapter/etl-adapter-xml/tests/Flow/ETL/Adapter/XML/Tests/Integration/XMLExtractorTypedColumnsTest.php @@ -0,0 +1,103 @@ +withXMLNodePath('root/items/item'); + + $rows = []; + + foreach ($extractor->extract(flow_context(Config::builder()->build())) as $batch) { + static::assertEquals(type_xml(), $batch->schema()->get('node')->type()); + + $rows = [...$rows, ...$batch->toArray()]; + } + + static::assertCount(5, $rows); + } + + public function test_appends_input_file_uri_when_metadata_columns_are_enabled(): void + { + $extractor = from_xml($path = path_real(__DIR__ . '/../Fixtures/simple_items.xml')) + ->withXMLNodePath('root/items/item') + ->withMetadataColumns(true); + + foreach ($extractor->extract(flow_context(Config::builder()->build())) as $batch) { + static::assertSame([$path->uri()], array_values(array_unique($batch->column('_input_file_uri')->values()))); + } + } + + public function test_an_xml_node_column_holds_the_utf8_document_of_each_node(): void + { + $physicals = []; + + foreach (from_xml(path_real(__DIR__ . '/../Fixtures/simple_items.xml')) + ->withXMLNodePath('root/items/item') + ->extract(flow_context()) as $batch) { + $physicals = [...$physicals, ...$batch->column('node')->physicals()]; + } + + static::assertCount(5, $physicals); + static::assertSame( + XmlDocumentPhysical::DECLARATION + . "\n 1\n \n", + $physicals[0], + ); + } + + public function test_a_string_node_column_holds_each_node_text(): void + { + $rows = []; + + foreach (from_xml(path_real(__DIR__ . '/../Fixtures/simple_items.xml')) + ->withXMLNodePath('root/items/item') + ->withSchema(schema(str_schema('node'))) + ->extract(flow_context()) as $batch) { + $rows = [...$rows, ...$batch->toArray()]; + } + + static::assertCount(5, $rows); + static::assertSame( + ['node' => "\n 5\n "], + $rows[4], + ); + } + + public function test_an_xml_node_column_beside_other_columns_reads_the_node_documents(): void + { + $nodes = []; + + foreach (from_xml(path_real(__DIR__ . '/../Fixtures/simple_items.xml')) + ->withXMLNodePath('root/items/item') + ->withSchema(schema(xml_schema('node'), str_schema('missing', nullable: true))) + ->extract(flow_context()) as $batch) { + $nodes = [...$nodes, ...$batch->column('node')->values()]; + } + + static::assertCount(5, $nodes); + static::assertInstanceOf(DOMDocument::class, $nodes[0]); + static::assertSame('UTF-8', $nodes[0]->encoding); + static::assertSame( + "\n 1\n ", + $nodes[0]->saveXML($nodes[0]->documentElement), + ); + } +} diff --git a/src/adapter/etl-adapter-xml/tests/Flow/ETL/Adapter/XML/Tests/Integration/XMLHydratorParityTest.php b/src/adapter/etl-adapter-xml/tests/Flow/ETL/Adapter/XML/Tests/Integration/XMLHydratorParityTest.php deleted file mode 100644 index e4fb5466de..0000000000 --- a/src/adapter/etl-adapter-xml/tests/Flow/ETL/Adapter/XML/Tests/Integration/XMLHydratorParityTest.php +++ /dev/null @@ -1,65 +0,0 @@ -withXMLNodePath('root/items/item'); - - $rows = []; - - foreach ($extractor->extract(flow_context(Config::builder()->build())) as $batch) { - static::assertEquals(type_xml(), $batch->schema()->get('node')->type()); - - foreach ($batch as $row) { - $rows[] = $row; - } - } - - static::assertCount(5, $rows); - } - - public function test_appends_input_file_uri_when_metadata_columns_are_enabled(): void - { - $extractor = from_xml($path = path_real(__DIR__ . '/../Fixtures/simple_items.xml')) - ->withXMLNodePath('root/items/item') - ->withMetadataColumns(true); - - foreach ($extractor->extract(flow_context(Config::builder()->build())) as $batch) { - foreach ($batch as $row) { - static::assertTrue($row->has('_input_file_uri')); - static::assertSame($path->uri(), $row->get('_input_file_uri')); - } - } - } - - public function test_honours_a_hydrator_configured_on_the_context(): void - { - $extractor = from_xml(path_real(__DIR__ . '/../Fixtures/simple_items.xml'))->withXMLNodePath('root/items/item'); - - $count = 0; - - foreach ($extractor->extract( - flow_context(Config::builder()->hydrator(new AdaptiveRowHydrator())->build()), - ) as $batch) { - static::assertEquals(type_xml(), $batch->schema()->get('node')->type()); - - $count += $batch->count(); - } - - static::assertSame(5, $count); - } -} diff --git a/src/adapter/etl-adapter-xml/tests/Flow/ETL/Adapter/XML/Tests/Integration/XMLParserExtractorTest.php b/src/adapter/etl-adapter-xml/tests/Flow/ETL/Adapter/XML/Tests/Integration/XMLParserExtractorTest.php index 262d9ad7c7..0efe23ef0e 100644 --- a/src/adapter/etl-adapter-xml/tests/Flow/ETL/Adapter/XML/Tests/Integration/XMLParserExtractorTest.php +++ b/src/adapter/etl-adapter-xml/tests/Flow/ETL/Adapter/XML/Tests/Integration/XMLParserExtractorTest.php @@ -7,6 +7,7 @@ use Flow\ETL\Adapter\XML\XMLParserExtractor; use Flow\ETL\Cardinality; use Flow\ETL\Config; +use Flow\ETL\Exception\InvalidArgumentException; use Flow\ETL\Exception\RuntimeException; use Flow\ETL\Exception\SchemaMismatchException; use Flow\ETL\Extractor\Signal; @@ -14,6 +15,7 @@ use Flow\ETL\Tests\Context\MemoryFiles; use Flow\ETL\Tests\Double\CountingFilesystem; use Flow\ETL\Tests\Double\RecordingFilesystem; +use Flow\ETL\Tests\Double\SpyBackend; use Flow\ETL\Tests\Double\UnsizedFilesystem; use Flow\ETL\Tests\FlowIntegrationTestCase; @@ -21,6 +23,7 @@ use function file_get_contents; use function Flow\ETL\Adapter\XML\from_xml; use function Flow\ETL\DSL\config; +use function Flow\ETL\DSL\config_builder; use function Flow\ETL\DSL\df; use function Flow\ETL\DSL\flow_context; use function Flow\ETL\DSL\ref; @@ -32,6 +35,7 @@ use function Flow\Filesystem\DSL\path; use function Flow\Filesystem\DSL\path_real; use function Flow\Types\DSL\type_string; +use function iterator_to_array; final class XMLParserExtractorTest extends FlowIntegrationTestCase { @@ -97,46 +101,67 @@ public function test_reading_xml(): void public function test_reading_xml_each_collection_item(): void { - static::assertXmlStringEqualsXmlString(<<<'XML' - - 1 - - XML, type_string()->cast(df() - ->read(from_xml(__DIR__ . '/../Fixtures/simple_items_flat.xml', 'root/items/item')) - ->fetch()[0]->get('node'))); + static::assertXmlStringEqualsXmlString( + <<<'XML' + + 1 + + XML, + type_string()->cast( + df() + ->read(from_xml(__DIR__ . '/../Fixtures/simple_items_flat.xml', 'root/items/item')) + ->fetch() + ->column('node') + ->value(0), + ), + ); - static::assertXmlStringEqualsXmlString(<<<'XML' - - 5 - - XML, type_string()->cast(df() - ->read(from_xml(__DIR__ . '/../Fixtures/simple_items_flat.xml', 'root/items/item')) - ->fetch()[4]->get('node'))); + static::assertXmlStringEqualsXmlString( + <<<'XML' + + 5 + + XML, + type_string()->cast( + df() + ->read(from_xml(__DIR__ . '/../Fixtures/simple_items_flat.xml', 'root/items/item')) + ->fetch() + ->column('node') + ->value(4), + ), + ); } public function test_reading_xml_from_path(): void { - static::assertXmlStringEqualsXmlString(<<<'XML' - - - 1 - - - 2 - - - 3 - - - 4 - - - 5 - - - XML, type_string()->cast(df() - ->read(from_xml(__DIR__ . '/../Fixtures/simple_items.xml', 'root/items')) - ->fetch()[0]->get('node'))); + static::assertXmlStringEqualsXmlString( + <<<'XML' + + + 1 + + + 2 + + + 3 + + + 4 + + + 5 + + + XML, + type_string()->cast( + df() + ->read(from_xml(__DIR__ . '/../Fixtures/simple_items.xml', 'root/items')) + ->fetch() + ->column('node') + ->value(0), + ), + ); } public function test_reading_xml_with_ancestor_namespace_declaration(): void @@ -146,9 +171,12 @@ public function test_reading_xml_with_ancestor_namespace_declaration(): void ->withEntry('title', ref('node')->xpath('/entry/g:title')->domElementValue()) ->fetch(); - static::assertSame(['Product 1', 'Product 2'], [$rows[0]->get('title'), $rows[1]->get('title')]); + static::assertSame(['Product 1', 'Product 2'], [ + $rows->column('title')->value(0), + $rows->column('title')->value(1), + ]); - $node = type_string()->cast($rows[0]->get('node')); + $node = type_string()->cast($rows->column('node')->value(0)); static::assertStringContainsString('xmlns:g="http://base.google.com/ns/1.0"', $node); static::assertStringContainsString('xmlns:c="http://example.com/custom"', $node); @@ -156,18 +184,26 @@ public function test_reading_xml_with_ancestor_namespace_declaration(): void public function test_reading_xml_with_default_namespace_declaration(): void { - $node = type_string()->cast(df() - ->read(from_xml(__DIR__ . '/../Fixtures/namespaced_default.xml', 'feed/entry')) - ->fetch()[0]->get('node')); + $node = type_string()->cast( + df() + ->read(from_xml(__DIR__ . '/../Fixtures/namespaced_default.xml', 'feed/entry')) + ->fetch() + ->column('node') + ->value(0), + ); static::assertStringContainsString('xmlns="http://example.com/default"', $node); } public function test_reading_xml_with_multi_ancestor_namespace_merge(): void { - $node = type_string()->cast(df() - ->read(from_xml(__DIR__ . '/../Fixtures/namespaced_multi_ancestor.xml', 'feed/group/entry')) - ->fetch()[0]->get('node')); + $node = type_string()->cast( + df() + ->read(from_xml(__DIR__ . '/../Fixtures/namespaced_multi_ancestor.xml', 'feed/group/entry')) + ->fetch() + ->column('node') + ->value(0), + ); static::assertStringContainsString('xmlns:a="http://example.com/a"', $node); static::assertStringContainsString('xmlns:b="http://example.com/b"', $node); @@ -180,18 +216,22 @@ public function test_reading_xml_with_namespace_on_captured_root(): void ->withEntry('title', ref('node')->xpath('/entry/g:title')->domElementValue()) ->fetch(); - static::assertSame('Product 1', $rows[0]->get('title')); + static::assertSame('Product 1', $rows->column('title')->value(0)); static::assertStringContainsString( 'xmlns:g="http://base.google.com/ns/1.0"', - type_string()->cast($rows[0]->get('node')), + type_string()->cast($rows->column('node')->value(0)), ); } public function test_reading_xml_with_prefixed_attribute(): void { - $node = type_string()->cast(df() - ->read(from_xml(__DIR__ . '/../Fixtures/namespaced_prefixed_attribute.xml', 'feed/entry')) - ->fetch()[0]->get('node')); + $node = type_string()->cast( + df() + ->read(from_xml(__DIR__ . '/../Fixtures/namespaced_prefixed_attribute.xml', 'feed/entry')) + ->fetch() + ->column('node') + ->value(0), + ); static::assertStringContainsString('xmlns:g="http://base.google.com/ns/1.0"', $node); static::assertStringContainsString('xml:lang="en"', $node); @@ -234,9 +274,13 @@ public function test_reading_xml_refuses_a_not_null_column_the_document_lacks(): public function test_reading_xml_with_shadowed_namespace_declaration(): void { - $node = type_string()->cast(df() - ->read(from_xml(__DIR__ . '/../Fixtures/namespaced_nested.xml', 'feed/group/entry')) - ->fetch()[0]->get('node')); + $node = type_string()->cast( + df() + ->read(from_xml(__DIR__ . '/../Fixtures/namespaced_nested.xml', 'feed/group/entry')) + ->fetch() + ->column('node') + ->value(0), + ); static::assertStringContainsString('xmlns:g="http://example.com/override"', $node); static::assertStringContainsString('xmlns:x="http://example.com/extra"', $node); @@ -260,13 +304,20 @@ public function test_reading_xml_with_sibling_namespace_isolation(): void public function test_reading_xml_without_namespaces_is_unchanged(): void { - static::assertXmlStringEqualsXmlString(<<<'XML' - - 1 - - XML, type_string()->cast(df() - ->read(from_xml(__DIR__ . '/../Fixtures/simple_items_flat.xml', 'root/items/item')) - ->fetch()[0]->get('node'))); + static::assertXmlStringEqualsXmlString( + <<<'XML' + + 1 + + XML, + type_string()->cast( + df() + ->read(from_xml(__DIR__ . '/../Fixtures/simple_items_flat.xml', 'root/items/item')) + ->fetch() + ->column('node') + ->value(0), + ), + ); } public function test_schema_appends_the_metadata_column(): void @@ -285,9 +336,19 @@ public function test_schema_defaults_to_a_single_node_column(): void public function test_schema_is_the_declared_one(): void { static::assertEquals( - schema(str_schema('name')), - from_xml(__DIR__ . '/../Fixtures/flow_orders.xml')->withSchema(schema(str_schema('name')))->schema(), + schema(str_schema('node')), + from_xml(__DIR__ . '/../Fixtures/flow_orders.xml')->withSchema(schema(str_schema('node')))->schema(), + ); + } + + public function test_a_schema_without_the_node_column_is_refused(): void + { + $this->expectException(InvalidArgumentException::class); + $this->expectExceptionMessage( + 'from_xml() reads every matched element into the "node" column, the schema declares no such column: name', ); + + from_xml(__DIR__ . '/../Fixtures/flow_orders.xml')->withSchema(schema(str_schema('name'))); } public function test_signal_stop(): void @@ -442,4 +503,18 @@ public function test_the_listing_is_read_once(): void static::assertSame(1, $filesystem->listCalls); } + + public function test_extract_builds_through_the_config_backend(): void + { + $backend = new SpyBackend(); + + iterator_to_array( + from_xml(path_real(__DIR__ . '/../Fixtures/one_line_items.xml')) + ->withXMLNodePath('root/item') + ->extract(flow_context(config_builder()->backend($backend)->build())), + false, + ); + + static::assertGreaterThanOrEqual(1, $backend->builders()); + } } diff --git a/src/adapter/etl-adapter-xml/tests/Flow/ETL/Adapter/XML/Tests/Integration/XMLReaderExtractorTest.php b/src/adapter/etl-adapter-xml/tests/Flow/ETL/Adapter/XML/Tests/Integration/XMLReaderExtractorTest.php deleted file mode 100644 index ee4304e0ee..0000000000 --- a/src/adapter/etl-adapter-xml/tests/Flow/ETL/Adapter/XML/Tests/Integration/XMLReaderExtractorTest.php +++ /dev/null @@ -1,233 +0,0 @@ -withBatchSize(1); - - self::assertExtractedRowsCount(2, $extractor, flow_context(config()), limit: 2); - } - - public function test_partition_columns_are_not_leaking_between_streams(): void - { - $rows = data_frame() - ->read( - // @mago-ignore analysis:deprecated-class - new XMLReaderExtractor(path(__DIR__ . '/../Fixtures/cross_stream/*/file.xml'), 'root/item'), - ) - ->fetch() - ->toArray(); - - static::assertSame(['node', 'date'], array_keys($rows[0])); - static::assertSame('2026-01-01', $rows[0]['date']); - static::assertSame(['node', 'date'], array_keys($rows[1])); - static::assertNull($rows[1]['date']); - } - - public function test_reading_deep_xml(): void - { - static::assertEquals( - 5, - data_frame() - ->read( - // @mago-ignore analysis:deprecated-class - new XMLReaderExtractor( - path(__DIR__ . '/../Fixtures/deepest_items_flat.xml'), - 'root/items/item/deep', - ), - ) - ->fetch() - ->count(), - ); - } - - public function test_reading_xml(): void - { - $xml = new DOMDocument(); - $xml->load(__DIR__ . '/../Fixtures/simple_items.xml'); - - static::assertEquals( - 1, - data_frame() - // @mago-ignore analysis:deprecated-class - ->read(new XMLReaderExtractor(path(__DIR__ . '/../Fixtures/simple_items.xml'))) - ->fetch() - ->count(), - ); - } - - public function test_reading_xml_each_collection_item(): void - { - // @mago-ignore analysis:deprecated-class - $extractor1 = new XMLReaderExtractor(path(__DIR__ . '/../Fixtures/simple_items_flat.xml'), 'root/items/item'); - static::assertXmlStringEqualsXmlString(<<<'XML' - - 1 - - XML, type_string()->cast(data_frame()->read($extractor1)->fetch()[0]->get('node'))); - - // @mago-ignore analysis:deprecated-class - $extractor2 = new XMLReaderExtractor(path(__DIR__ . '/../Fixtures/simple_items_flat.xml'), 'root/items/item'); - static::assertXmlStringEqualsXmlString(<<<'XML' - - 5 - - XML, type_string()->cast(data_frame()->read($extractor2)->fetch()[4]->get('node'))); - } - - public function test_reading_xml_from_path(): void - { - static::assertXmlStringEqualsXmlString( - <<<'XML' - - - 1 - - - 2 - - - 3 - - - 4 - - - 5 - - - XML, - type_string()->cast(data_frame() - // @mago-ignore analysis:deprecated-class - ->read(new XMLReaderExtractor(path(__DIR__ . '/../Fixtures/simple_items.xml'), 'root/items')) - ->fetch()[0]->get('node')), - ); - } - - public function test_schema_appends_the_metadata_column(): void - { - // @mago-ignore analysis:deprecated-class - $extractor = new XMLReaderExtractor(path_real(__DIR__ . '/../Fixtures/flow_orders.xml'), 'root/row'); - - static::assertEquals( - schema(xml_schema('node'), str_schema('_input_file_uri')), - $extractor->withMetadataColumns(true)->schema(), - ); - } - - public function test_schema_describes_a_single_node_column(): void - { - // @mago-ignore analysis:deprecated-class - $extractor = new XMLReaderExtractor(path_real(__DIR__ . '/../Fixtures/flow_orders.xml'), 'root/row'); - - static::assertEquals(schema(xml_schema('node')), $extractor->schema()); - } - - public function test_signal_stop(): void - { - // @mago-ignore analysis:deprecated-class - $extractor = new XMLReaderExtractor(path_real(__DIR__ . '/../Fixtures/flow_orders.xml'), 'root/row'); - - $generator = $extractor->extract(flow_context(config())); - - static::assertTrue($generator->valid()); - $generator->next(); - static::assertTrue($generator->valid()); - $generator->next(); - static::assertTrue($generator->valid()); - $generator->send(Signal::STOP); - static::assertFalse($generator->valid()); - } - - public function test_signal_stop_on_the_first_file_tail_batch_skips_the_remaining_files(): void - { - // @mago-ignore analysis:deprecated-class - $generator = (new XMLReaderExtractor(path(__DIR__ . '/../Fixtures/cross_stream/*/file.xml'), 'root/item')) - ->withBatchSize(10) - ->extract(flow_context(config())); - - static::assertTrue($generator->valid()); - $generator->send(Signal::STOP); - static::assertFalse($generator->valid()); - } - - public function test_limit_reached_on_the_first_file_tail_batch_skips_the_remaining_files(): void - { - // @mago-ignore analysis:deprecated-class - $extractor = (new XMLReaderExtractor( - path(__DIR__ . '/../Fixtures/cross_stream/*/file.xml'), - 'root/item', - ))->withBatchSize(10); - - static::assertCount(1, ExtractedRows::of($extractor, limit: 1)); - } - - public function test_is_repeatable(): void - { - static::assertTrue( - // @mago-ignore analysis:deprecated-class - (new XMLReaderExtractor(path_real(__DIR__ . '/../Fixtures/flow_orders.xml'), 'root/row'))->isRepeatable(), - ); - } - - public function test_it_declares_the_listed_byte_total_exactly(): void - { - // @mago-ignore analysis:deprecated-class - $statistics = (new XMLReaderExtractor(path_real(__DIR__ . '/../Fixtures/listed/*.xml')))->statistics(); - - static::assertEquals(Cardinality::exact(13), $statistics->size); - static::assertEquals(Cardinality::unknown(), $statistics->rows); - } - - public function test_a_member_without_a_size_makes_the_size_unknown(): void - { - $filesystem = new UnsizedFilesystem(native_local_filesystem(), [ - path_real(__DIR__ . '/../Fixtures/listed/b.xml')->uri(), - ]); - - // @mago-ignore analysis:deprecated-class - $extractor = new XMLReaderExtractor(path_real(__DIR__ . '/../Fixtures/listed/*.xml'), '', $filesystem); - - static::assertEquals(Cardinality::unknown(), $extractor->statistics()->size); - } - - public function test_the_listing_is_read_once(): void - { - $filesystem = new CountingFilesystem(native_local_filesystem()); - // @mago-ignore analysis:deprecated-class - $extractor = new XMLReaderExtractor(path_real(__DIR__ . '/../Fixtures/listed/*.xml'), '', $filesystem); - - $extractor->statistics(); - $extractor->statistics(); - - static::assertSame(1, $filesystem->listCalls); - } -} diff --git a/src/adapter/etl-adapter-xml/tests/Flow/ETL/Adapter/XML/Tests/Unit/XMLEncoderTest.php b/src/adapter/etl-adapter-xml/tests/Flow/ETL/Adapter/XML/Tests/Unit/XMLEncoderTest.php index ac02a45af8..078498d33a 100644 --- a/src/adapter/etl-adapter-xml/tests/Flow/ETL/Adapter/XML/Tests/Unit/XMLEncoderTest.php +++ b/src/adapter/etl-adapter-xml/tests/Flow/ETL/Adapter/XML/Tests/Unit/XMLEncoderTest.php @@ -4,170 +4,211 @@ namespace Flow\ETL\Adapter\XML\Tests\Unit; -use DateInterval; use DateTimeImmutable; -use DateTimeZone; +use DOMException; use Flow\ETL\Adapter\XML\XMLEncoder; use Flow\ETL\Adapter\XML\XMLWriter\DOMDocumentWriter; +use Flow\ETL\Adapter\XML\XMLWriter\StringXMLWriter; use Flow\ETL\Exception\RuntimeException; -use Flow\ETL\Row\TypedRowValues; -use Flow\ETL\Tests\Fixtures\Enum\BackedIntEnum; +use Flow\ETL\Schema\Definition; use Flow\ETL\Tests\FlowTestCase; -use Flow\Types\Value\Uuid; - +use Flow\Types\Exception\CastingException; +use Generator; +use PHPUnit\Framework\Attributes\DataProvider; + +use function Flow\ETL\DSL\array_to_rows; +use function Flow\ETL\DSL\bool_schema; +use function Flow\ETL\DSL\date_schema; +use function Flow\ETL\DSL\datetime_schema; +use function Flow\ETL\DSL\float_schema; +use function Flow\ETL\DSL\int_schema; +use function Flow\ETL\DSL\list_schema; +use function Flow\ETL\DSL\map_schema; +use function Flow\ETL\DSL\schema; +use function Flow\ETL\DSL\str_schema; +use function Flow\ETL\DSL\structure_schema; use function Flow\Types\DSL\structure_element; use function Flow\Types\DSL\type_date; use function Flow\Types\DSL\type_datetime; -use function Flow\Types\DSL\type_enum; +use function Flow\Types\DSL\type_float; use function Flow\Types\DSL\type_integer; +use function Flow\Types\DSL\type_json; use function Flow\Types\DSL\type_list; use function Flow\Types\DSL\type_map; +use function Flow\Types\DSL\type_optional; use function Flow\Types\DSL\type_string; use function Flow\Types\DSL\type_structure; -use function Flow\Types\DSL\type_time; -use function Flow\Types\DSL\type_time_zone; -use function Flow\Types\DSL\type_uuid; final class XMLEncoderTest extends FlowTestCase { - public function test_decode_wraps_raw_xml_into_a_node_column(): void + /** + * @return Generator>, list>, string}> + */ + public static function batches(): Generator + { + yield 'scalars' => [ + [int_schema('id'), str_schema('name')], + [['id' => 1, 'name' => 'Norbert'], ['id' => 2, 'name' => 'a < b & c']], + "1Norbert\n2a < b & c\n", + ]; + yield 'a prefixed column is an attribute' => [ + [int_schema('_id'), str_schema('name')], + [['_id' => 1, 'name' => 'Norbert']], + "Norbert\n", + ]; + yield 'attributes only' => [ + [int_schema('_id'), str_schema('_name'), bool_schema('_active'), float_schema('_price')], + [['_id' => 1, '_name' => 'a "b"', '_active' => true, '_price' => 1.0]], + "\n", + ]; + yield 'an attribute of a date, a datetime and a list' => [ + [date_schema('_on'), datetime_schema('_at'), list_schema('_tags', type_list(type_string()))], + [[ + '_on' => new DateTimeImmutable('2024-08-01'), + '_at' => new DateTimeImmutable('2024-08-01 10:00:00 UTC'), + '_tags' => ['a', 'b'], + ]], + "\n", + ]; + yield 'a null scalar is an empty element' => [ + [str_schema('name', nullable: true)], + [['name' => null]], + "\n", + ]; + yield 'list' => [ + [list_schema('tags', type_list(type_string()), nullable: true)], + [['tags' => ['a', 'b']], ['tags' => []], ['tags' => null]], + "ab\n\n\n", + ]; + yield 'B3 list' => [ + [list_schema('l', type_list(type_optional(type_integer())))], + [['l' => [1, null, 3]]], + "13\n", + ]; + yield 'B3 map' => [ + [map_schema('m', type_map(type_string(), type_optional(type_integer())))], + [['m' => ['x' => 1, 'y' => null]]], + "x1y\n", + ]; + yield 'map' => [ + [map_schema('m', type_map(type_string(), type_integer()))], + [['m' => ['x' => 1]], ['m' => []]], + "x1\n\n", + ]; + yield 'a map keyed by integers' => [ + [map_schema('m', type_map(type_integer(), type_string()))], + [['m' => [5 => 'a']]], + "5a\n", + ]; + yield 'structure' => [ + [structure_schema('address', type_structure(['city' => type_string(), 'zip' => type_string()]))], + [['address' => ['city' => 'Krakow', 'zip' => '31-021']]], + "
Krakow31-021
\n", + ]; + yield 'nested leaves render by their type' => [ + [ + list_schema('days', type_list(type_date())), + structure_schema('s', type_structure(['d' => type_datetime(), 'f' => type_float()])), + list_schema('docs', type_list(type_json())), + ], + [[ + 'days' => [new DateTimeImmutable('2024-08-01'), new DateTimeImmutable('2024-08-02')], + 's' => ['d' => new DateTimeImmutable('2024-08-01 10:00:00 UTC'), 'f' => 1.0], + 'docs' => ['{"a": 1}'], + ]], + '2024-08-012024-08-02' + . '2024-08-01T10:00:00.000000+00:001.0' + . "{\"a\": 1}\n", + ]; + yield 'a prefixed structure element is an attribute of its node' => [ + [structure_schema('address', type_structure(['_id' => type_integer(), 'city' => type_string()]))], + [['address' => ['_id' => 7, 'city' => 'Krakow']]], + "
Krakow
\n", + ]; + yield 'no rows' => [[int_schema('id')], [], '']; + } + + /** + * @param list> $definitions + * @param list> $rows + */ + #[DataProvider('batches')] + public function test_both_writers_encode_a_batch_to_the_same_bytes( + array $definitions, + array $rows, + string $expected, + ): void { + $batch = array_to_rows($rows, schema(...$definitions)); + + static::assertSame($expected, (new XMLEncoder(new StringXMLWriter()))->encode($batch)); + static::assertSame($expected, (new XMLEncoder(new DOMDocumentWriter()))->encode($batch)); + } + + public function test_encodes_datetime_and_date_with_the_configured_formats(): void { - $encoder = new XMLEncoder(new DOMDocumentWriter()); - static::assertSame( - ['node' => '1'], - $encoder->decode(['1'])[0]->values, - ); - } - - public function test_encodes_scalars_into_flat_nodes(): void - { - $encoder = new XMLEncoder(new DOMDocumentWriter()); - - static::assertXmlStringEqualsXmlString( - '1Norbert', - $encoder->encode([new TypedRowValues(['id' => 1, 'name' => 'Norbert'], [ - 'id' => type_integer(), - 'name' => type_string(), - ])])[0], - ); - } - - public function test_encodes_prefixed_columns_as_attributes(): void - { - $encoder = new XMLEncoder(new DOMDocumentWriter()); - - static::assertXmlStringEqualsXmlString( - 'Norbert', - $encoder->encode([new TypedRowValues(['_id' => 1, 'name' => 'Norbert'], [ - '_id' => type_integer(), - 'name' => type_string(), - ])])[0], - ); - } - - public function test_encodes_date_with_the_date_format(): void - { - $encoder = new XMLEncoder(new DOMDocumentWriter()); - - static::assertXmlStringEqualsXmlString( - '2024-08-01', - $encoder->encode([new TypedRowValues(['at' => new DateTimeImmutable('2024-08-01 10:00:00')], [ - 'at' => type_date(), - ])])[0], - ); - } - - public function test_encodes_datetime_with_the_configured_format(): void - { - $encoder = new XMLEncoder(new DOMDocumentWriter(), dateTimeFormat: 'Y-m-d'); - - static::assertXmlStringEqualsXmlString( - '2024-08-01', - $encoder->encode([new TypedRowValues(['at' => new DateTimeImmutable('2024-08-01 10:00:00')], [ - 'at' => type_datetime(), - ])])[0], - ); - } - - public function test_encodes_time_as_microseconds(): void - { - $encoder = new XMLEncoder(new DOMDocumentWriter()); - - static::assertXmlStringEqualsXmlString( - '3600000000', - $encoder->encode([new TypedRowValues(['d' => new DateInterval('PT1H')], ['d' => type_time()])])[0], - ); - } - - public function test_encodes_enum_as_its_name(): void - { - $encoder = new XMLEncoder(new DOMDocumentWriter()); - - static::assertXmlStringEqualsXmlString( - 'one', - $encoder->encode([new TypedRowValues(['e' => BackedIntEnum::one], [ - 'e' => type_enum(BackedIntEnum::class), - ])])[0], + "2024-08-0101/08/2024\n", + (new XMLEncoder(new StringXMLWriter(), dateTimeFormat: 'Y-m-d', dateFormat: 'd/m/Y'))->encode(array_to_rows( + [['at' => new DateTimeImmutable('2024-08-01 10:00:00'), 'on' => new DateTimeImmutable('2024-08-01')]], + schema(datetime_schema('at'), date_schema('on')), + )), ); } - public function test_encodes_a_timezone_as_its_iana_name(): void + public function test_uses_the_configured_element_names_and_attribute_prefix(): void { - static::assertXmlStringEqualsXmlString( - 'Europe/Warsaw', - (new XMLEncoder(new DOMDocumentWriter()))->encode([new TypedRowValues([ - 'tz' => new DateTimeZone('Europe/Warsaw'), - ], ['tz' => type_time_zone()])])[0], + static::assertSame( + 'ax1' . "\n", + (new XMLEncoder( + new StringXMLWriter(), + attributePrefix: '@', + listElementName: 'item', + mapElementName: 'entry', + mapElementKeyName: 'k', + mapElementValueName: 'v', + rowElementName: 'record', + ))->encode(array_to_rows( + [['@id' => 1, 'l' => ['a'], 'm' => ['x' => 1]]], + schema( + int_schema('@id'), + list_schema('l', type_list(type_string())), + map_schema('m', type_map(type_string(), type_integer())), + ), + )), ); } - public function test_encodes_uuid_as_string(): void + public function test_a_null_in_an_attribute_column_is_refused(): void { - $encoder = new XMLEncoder(new DOMDocumentWriter()); + $this->expectException(CastingException::class); - static::assertXmlStringEqualsXmlString( - 'f47ac10b-58cc-4372-a567-0e02b2c3d479', - $encoder->encode([new TypedRowValues(['id' => new Uuid('f47ac10b-58cc-4372-a567-0e02b2c3d479')], [ - 'id' => type_uuid(), - ])])[0], - ); + (new XMLEncoder(new StringXMLWriter()))->encode(array_to_rows([[ + '_id' => null, + ]], schema(int_schema('_id', nullable: true)))); } - public function test_encodes_list_into_element_nodes(): void + public function test_a_row_element_name_dom_refuses_is_refused_by_either_writer(): void { - $encoder = new XMLEncoder(new DOMDocumentWriter()); + $rows = array_to_rows([['id' => 1]], schema(int_schema('id'))); + $refused = 0; - static::assertXmlStringEqualsXmlString( - 'ab', - $encoder->encode([new TypedRowValues(['tags' => ['a', 'b']], ['tags' => type_list(type_string())])])[0], - ); - } - - public function test_encodes_map_into_key_value_element_nodes(): void - { - $encoder = new XMLEncoder(new DOMDocumentWriter()); + foreach ([new StringXMLWriter(), new DOMDocumentWriter()] as $writer) { + try { + (new XMLEncoder($writer, rowElementName: '1 row'))->encode($rows); + } catch (DOMException) { + $refused++; + } + } - static::assertXmlStringEqualsXmlString( - 'x1', - $encoder->encode([new TypedRowValues(['m' => ['x' => 1]], ['m' => type_map( - type_string(), - type_integer(), - )])])[0], - ); + static::assertSame(2, $refused); } - public function test_encodes_structure_into_nested_nodes(): void + public function test_encoding_without_a_writer_throws(): void { - $encoder = new XMLEncoder(new DOMDocumentWriter()); + $this->expectException(RuntimeException::class); + $this->expectExceptionMessage('XMLEncoder requires an XMLWriter to encode rows'); - static::assertXmlStringEqualsXmlString( - '
Krakow31-021
', - $encoder->encode([new TypedRowValues(['address' => [ - 'city' => 'Krakow', - 'zip' => '31-021', - ]], ['address' => type_structure(['city' => type_string(), 'zip' => type_string()])])])[0], - ); + (new XMLEncoder())->encode(array_to_rows([['id' => 1]], schema(int_schema('id')))); } public function test_encoding_structure_with_optional_elements_throws(): void @@ -179,10 +220,12 @@ public function test_encoding_structure_with_optional_elements_throws(): void 'XML encoder does not support structure optional elements, given: structure{city: string, zip?: string}', ); - $encoder->encode([new TypedRowValues(['address' => ['city' => 'Krakow']], ['address' => type_structure([ + $encoder->encode(array_to_rows([['address' => [ + 'city' => 'Krakow', + ]]], schema(structure_schema('address', type_structure([ 'city' => type_string(), 'zip' => structure_element('zip', type_string(), optional: true), - ])])]); + ]))))); } public function test_encoding_structure_with_an_interleaved_optional_element_throws(): void @@ -194,45 +237,11 @@ public function test_encoding_structure_with_an_interleaved_optional_element_thr 'XML encoder does not support structure optional elements, given: structure{zip?: string, city: string}', ); - $encoder->encode([new TypedRowValues(['address' => ['city' => 'Krakow']], ['address' => type_structure([ + $encoder->encode(array_to_rows([['address' => [ + 'city' => 'Krakow', + ]]], schema(structure_schema('address', type_structure([ 'zip' => structure_element('zip', type_string(), optional: true), 'city' => type_string(), - ])])]); - } - - public function test_encoding_structure_with_more_values_than_declared_elements_throws(): void - { - $encoder = new XMLEncoder(new DOMDocumentWriter()); - - $this->expectException(RuntimeException::class); - $this->expectExceptionMessage( - 'XML encoder received 3 values for structure "structure{city: string, zip: string}" which declares 2 elements', - ); - - $encoder->encode([new TypedRowValues(['address' => [ - 'city' => 'Krakow', - 'zip' => '31-021', - 'extra' => 'lost', - ]], ['address' => type_structure(['city' => type_string(), 'zip' => type_string()])])]); - } - - public function test_encodes_null_scalar_as_an_empty_node(): void - { - $encoder = new XMLEncoder(new DOMDocumentWriter()); - - static::assertXmlStringEqualsXmlString( - '', - $encoder->encode([new TypedRowValues(['name' => null], ['name' => type_string()])])[0], - ); - } - - public function test_uses_the_configured_row_element_name(): void - { - $encoder = new XMLEncoder(new DOMDocumentWriter(), rowElementName: 'record'); - - static::assertXmlStringEqualsXmlString( - '1', - $encoder->encode([new TypedRowValues(['id' => 1], ['id' => type_integer()])])[0], - ); + ]))))); } } diff --git a/src/adapter/etl-adapter-xml/tests/Flow/ETL/Adapter/XML/Tests/Unit/XMLFileBatchesTest.php b/src/adapter/etl-adapter-xml/tests/Flow/ETL/Adapter/XML/Tests/Unit/XMLFileBatchesTest.php new file mode 100644 index 0000000000..ed7de84b33 --- /dev/null +++ b/src/adapter/etl-adapter-xml/tests/Flow/ETL/Adapter/XML/Tests/Unit/XMLFileBatchesTest.php @@ -0,0 +1,64 @@ + '12', + ])); + $file = (new XMLFileBatches($filesystem, 'root/row', 8192))->batches( + new SourceFile(path('memory://a.xml')), + schema(str_schema('node')), + 10, + new PhpBackend(), + new ReadWindow(), + ); + + static::assertSame( + [[['node' => '1'], ['node' => '2']]], + array_map(static fn(Rows $rows): array => $rows->toArray(), iterator_to_array($file, false)), + ); + static::assertSame(0, $file->getReturn()); + static::assertSame($filesystem->readFromCalls, $filesystem->closedStreams()); + } + + public function test_a_column_the_body_declares_beside_node_is_padded_with_nulls(): void + { + $file = (new XMLFileBatches( + MemoryFiles::with(['memory://a.xml' => '1']), + 'root/row', + 8192, + ))->batches( + new SourceFile(path('memory://a.xml')), + schema(str_schema('node'), str_schema('note', nullable: true)), + 10, + new PhpBackend(), + new ReadWindow(), + ); + + static::assertSame( + [[['node' => '1', 'note' => null]]], + array_map(static fn(Rows $rows): array => $rows->toArray(), iterator_to_array($file, false)), + ); + } +} diff --git a/src/adapter/etl-adapter-xml/tests/Flow/ETL/Adapter/XML/Tests/Unit/XMLLoaderTest.php b/src/adapter/etl-adapter-xml/tests/Flow/ETL/Adapter/XML/Tests/Unit/XMLLoaderTest.php index 44bdeb7bd8..9f5d08dfd8 100644 --- a/src/adapter/etl-adapter-xml/tests/Flow/ETL/Adapter/XML/Tests/Unit/XMLLoaderTest.php +++ b/src/adapter/etl-adapter-xml/tests/Flow/ETL/Adapter/XML/Tests/Unit/XMLLoaderTest.php @@ -6,13 +6,25 @@ use Flow\ETL\Adapter\XML\Loader\XMLLoader; use Flow\ETL\Adapter\XML\XMLWriter\DOMDocumentWriter; +use Flow\ETL\Tests\Double\RecordingFilesystem; +use Flow\ETL\Tests\FlowTestCase; use Flow\Filesystem\Path\Option; use Flow\Filesystem\Path\Option\ContentType; -use PHPUnit\Framework\TestCase; +use function array_keys; +use function array_map; +use function count; +use function Flow\ETL\Adapter\XML\to_xml; +use function Flow\ETL\DSL\array_to_rows; +use function Flow\ETL\DSL\flow_context; +use function Flow\ETL\DSL\int_schema; +use function Flow\ETL\DSL\partition_by; +use function Flow\ETL\DSL\schema; +use function Flow\ETL\DSL\str_schema; +use function Flow\Filesystem\DSL\memory_filesystem; use function Flow\Filesystem\DSL\path; -final class XMLLoaderTest extends TestCase +final class XMLLoaderTest extends FlowTestCase { public function test_setting_content_type_on_path(): void { @@ -20,4 +32,76 @@ public function test_setting_content_type_on_path(): void static::assertEquals($loader->destination()->getOption(Option::CONTENT_TYPE), ContentType::XML); } + + public function test_every_batch_is_one_append_and_closure_closes_the_root(): void + { + $filesystem = new RecordingFilesystem(memory_filesystem()); + $schema = schema(int_schema('_id'), str_schema('name')); + $loader = to_xml(path('memory://out.xml'), filesystem: $filesystem); + + $loader->load(array_to_rows([['_id' => 1, 'name' => 'a']], $schema), flow_context()); + $loader->load(array_to_rows([ + ['_id' => 2, 'name' => 'b'], + ['_id' => 3, 'name' => 'c'], + ], $schema), flow_context()); + $loader->closure(flow_context()); + + static::assertSame( + "\n\n" + . "a\nb\nc\n" + . '', + $filesystem->readFrom(path('memory://out.xml'))->content(), + ); + static::assertSame(3, count(array_keys($filesystem->calls, 'append', true))); + } + + public function test_every_partition_file_has_its_own_root(): void + { + $filesystem = memory_filesystem(); + $schema = schema(int_schema('id'), str_schema('group')); + $loader = to_xml(path('memory://out/file.xml'), filesystem: $filesystem)->partitionBy(partition_by('group')); + + $loader->load(array_to_rows([ + ['id' => 1, 'group' => 'a'], + ['id' => 2, 'group' => 'b'], + ], $schema), flow_context()); + $loader->load(array_to_rows([['id' => 3, 'group' => 'a']], $schema), flow_context()); + $loader->closure(flow_context()); + + static::assertSame( + "\n\n1\n3\n", + $filesystem->readFrom(path('memory://out/group=a/file.xml'))->content(), + ); + static::assertSame( + "\n\n2\n", + $filesystem->readFrom(path('memory://out/group=b/file.xml'))->content(), + ); + } + + public function test_two_runs_through_one_loader_write_independent_files(): void + { + $filesystem = memory_filesystem(); + $loader = to_xml(path('memory://out.xml'), filesystem: $filesystem)->partitionBy(partition_by('group')); + $read = []; + + foreach (['a' => [1, 2], 'b' => [3]] as $group => $ids) { + $loader->load( + array_to_rows( + array_map(static fn(int $id): array => ['id' => $id, 'group' => $group], $ids), + schema(int_schema('id'), str_schema('group')), + ), + flow_context(), + ); + $loader->closure(flow_context()); + $read[] = $filesystem->readFrom(path('memory://group=' . $group . '/out.xml'))->content(); + } + + static::assertSame( + [ + "\n\n1\n2\n", + "\n\n3\n", + ], + $read, + ); + } } diff --git a/src/adapter/etl-adapter-xml/tests/Flow/ETL/Adapter/XML/Tests/Unit/XMLNodeBatchesTest.php b/src/adapter/etl-adapter-xml/tests/Flow/ETL/Adapter/XML/Tests/Unit/XMLNodeBatchesTest.php new file mode 100644 index 0000000000..afc2bdf992 --- /dev/null +++ b/src/adapter/etl-adapter-xml/tests/Flow/ETL/Adapter/XML/Tests/Unit/XMLNodeBatchesTest.php @@ -0,0 +1,88 @@ +documents( + new StringSourceStream(path('memory://a.xml'), 'ż23'), + schema(xml_schema('node'), str_schema('other', nullable: true)), + new PhpBackend(), + ), + false, + ); + + static::assertSame([2, 1], array_map(static fn(Rows $rows): int => $rows->count(), $batches)); + + $first = type_instance_of(DOMDocument::class)->assert($batches[0]->column('node')->value(0)); + + static::assertSame('UTF-8', $first->encoding); + static::assertSame('ż', $first->saveXML($first->documentElement)); + static::assertSame([null, null], $batches[0]->column('other')->values()); + } + + public function test_physicals_are_an_xml_column_of_node_document_bytes_in_batches(): void + { + $node = xml_schema('node'); + static::assertInstanceOf(XMLDefinition::class, $node); + + $batches = iterator_to_array( + (new XMLNodeBatches(new XMLNodes('rows/row'), 2, 8192))->physicals( + new StringSourceStream(path('memory://a.xml'), '23'), + $node, + schema($node), + new PhpBackend(), + ), + false, + ); + + static::assertSame([2, 1], array_map(static fn(Rows $rows): int => $rows->count(), $batches)); + static::assertSame( + [ + XmlDocumentPhysical::DECLARATION . "\n", + XmlDocumentPhysical::DECLARATION . "2\n", + ], + $batches[0]->column('node')->physicals(), + ); + } + + public function test_strings_are_rows_of_node_texts_in_batches(): void + { + $batches = iterator_to_array( + (new XMLNodeBatches(new XMLNodes('rows/row'), 2, 8192))->strings( + new StringSourceStream(path('memory://a.xml'), 'ż23'), + schema(str_schema('node')), + new PhpBackend(), + ), + false, + ); + + static::assertSame( + [[['node' => 'ż'], ['node' => '2']], [['node' => '3']]], + array_map(static fn(Rows $rows): array => $rows->toArray(), $batches), + ); + } +} diff --git a/src/adapter/etl-adapter-xml/tests/Flow/ETL/Adapter/XML/Tests/Unit/XMLNodeCursorTest.php b/src/adapter/etl-adapter-xml/tests/Flow/ETL/Adapter/XML/Tests/Unit/XMLNodeCursorTest.php new file mode 100644 index 0000000000..9de1175cfc --- /dev/null +++ b/src/adapter/etl-adapter-xml/tests/Flow/ETL/Adapter/XML/Tests/Unit/XMLNodeCursorTest.php @@ -0,0 +1,126 @@ + $bufferSize + * @param list $stops + */ + #[TestWith(['root/items/item', 1, ['1', '2']])] + #[TestWith(['root/items/item', 8192, ['1', '2']])] + #[TestWith(['', 8192, ['root']])] + #[TestWith(['feed/entry', 8192, []])] + public function test_stops_on_every_element_on_the_path(string $path, int $bufferSize, array $stops): void + { + $seen = []; + + foreach ((new XMLNodeCursor($path))->of( + new StringSourceStream( + path('memory://a.xml'), + '', + ), + $bufferSize, + ) as $reader) { + $seen[] = $reader->getAttribute('id') ?? $reader->name; + } + + static::assertSame($stops, $seen); + } + + #[TestWith([''])] + #[TestWith([""])] + #[TestWith([''])] + public function test_a_document_without_a_root_has_no_stops(string $xml): void + { + static::assertSame( + [], + iterator_to_array((new XMLNodeCursor('root/item'))->of( + new StringSourceStream(path('memory://a.xml'), $xml), + 8192, + )), + ); + } + + public function test_a_malformed_document_is_refused_at_its_end(): void + { + $this->expectException(RuntimeException::class); + $this->expectExceptionMessage('XML Error: Extra content at the end of the document at line 1'); + + iterator_to_array((new XMLNodeCursor('root/item'))->of( + new StringSourceStream(path('memory://a.xml'), 'junk'), + 8192, + )); + } + + public function test_libxml_reports_internally_at_every_stop_and_the_callers_mode_returns_after(): void + { + $previous = libxml_use_internal_errors(false); + + try { + $cursor = (new XMLNodeCursor('root/item'))->of( + new StringSourceStream(path('memory://a.xml'), ''), + 8192, + ); + + foreach ($cursor as $_reader) { + static::assertTrue(libxml_use_internal_errors()); + libxml_use_internal_errors(false); + } + + static::assertFalse(libxml_use_internal_errors()); + } finally { + libxml_use_internal_errors($previous); + } + } + + public function test_an_abandoned_walk_restores_the_callers_libxml_error_mode(): void + { + $previous = libxml_use_internal_errors(false); + + try { + $cursor = (new XMLNodeCursor('root/item'))->of( + new StringSourceStream(path('memory://a.xml'), ''), + 8192, + ); + $cursor->current(); + unset($cursor); + + static::assertFalse(libxml_use_internal_errors()); + } finally { + libxml_use_internal_errors($previous); + } + } + + public function test_failure_names_the_last_libxml_error(): void + { + $previous = libxml_use_internal_errors(true); + + try { + (new DOMDocument())->loadXML(''); + + static::assertMatchesRegularExpression( + '/^XML Error: .+ at line 1$/', + (new XMLNodeCursor('a'))->failure()->getMessage(), + ); + } finally { + libxml_clear_errors(); + libxml_use_internal_errors($previous); + } + } +} diff --git a/src/adapter/etl-adapter-xml/tests/Flow/ETL/Adapter/XML/Tests/Unit/XMLNodesTest.php b/src/adapter/etl-adapter-xml/tests/Flow/ETL/Adapter/XML/Tests/Unit/XMLNodesTest.php index f55436c61a..61028d3c2d 100644 --- a/src/adapter/etl-adapter-xml/tests/Flow/ETL/Adapter/XML/Tests/Unit/XMLNodesTest.php +++ b/src/adapter/etl-adapter-xml/tests/Flow/ETL/Adapter/XML/Tests/Unit/XMLNodesTest.php @@ -7,12 +7,17 @@ use DOMDocument; use Flow\ETL\Adapter\XML\Tests\Context\SerializedNodes; use Flow\ETL\Adapter\XML\XMLNodes; +use Flow\ETL\Column\Physical\XmlDocumentPhysical; use Flow\ETL\Exception\RuntimeException; use Flow\ETL\Tests\FlowTestCase; +use Flow\ETL\Tests\Mother\XmlNodeMother; use Flow\Filesystem\Stream\StringSourceStream; +use Generator; +use PHPUnit\Framework\Attributes\DataProvider; use PHPUnit\Framework\Attributes\TestWith; use function Flow\Filesystem\DSL\path; +use function iterator_to_array; use function libxml_use_internal_errors; final class XMLNodesTest extends FlowTestCase @@ -135,7 +140,7 @@ public function test_the_callers_libxml_error_mode_holds_at_every_yield_and_afte $previous = libxml_use_internal_errors($internalErrors); try { - $nodes = (new XMLNodes('root/item'))->of( + $nodes = (new XMLNodes('root/item'))->documents( new StringSourceStream(path('memory://a.xml'), '12'), 8192, ); @@ -155,7 +160,7 @@ public function test_an_abandoned_read_restores_the_callers_libxml_error_mode(): $previous = libxml_use_internal_errors(false); try { - $nodes = (new XMLNodes('root/item'))->of( + $nodes = (new XMLNodes('root/item'))->documents( new StringSourceStream(path('memory://a.xml'), '12'), 8192, ); @@ -167,4 +172,60 @@ public function test_an_abandoned_read_restores_the_callers_libxml_error_mode(): libxml_use_internal_errors($previous); } } + + public static function edge_documents(): Generator + { + foreach (XmlNodeMother::edges() as $label => ['document' => $document, 'node' => $node]) { + yield $label => [$document, $node]; + } + } + + #[DataProvider('edge_documents')] + public function test_a_node_text_is_the_physical_of_its_document(string $xml, string $node): void + { + $nodes = new XMLNodes('rows/row'); + $physical = new XmlDocumentPhysical(); + $documents = iterator_to_array( + $nodes->documents(new StringSourceStream(path('memory://a.xml'), $xml), 8192), + false, + ); + $texts = iterator_to_array($nodes->texts(new StringSourceStream(path('memory://a.xml'), $xml), 8192), false); + + static::assertSame([$node], $texts); + static::assertCount(1, $documents); + static::assertSame($physical->toPhysical($documents[0]), $physical->physical($texts[0])); + } + + public function test_texts_keep_the_callers_libxml_error_mode_at_every_yield(): void + { + $previous = libxml_use_internal_errors(false); + + try { + $texts = (new XMLNodes('root/item'))->texts( + new StringSourceStream(path('memory://a.xml'), '12'), + 8192, + ); + + foreach ($texts as $_text) { + static::assertFalse(libxml_use_internal_errors()); + } + + static::assertFalse(libxml_use_internal_errors()); + } finally { + libxml_use_internal_errors($previous); + } + } + + public function test_texts_refuse_a_malformed_document(): void + { + $this->expectException(RuntimeException::class); + $this->expectExceptionMessageMatches( + '/^XML Error: Opening and ending tag mismatch: item line 1 and wrong at line 1$/', + ); + + iterator_to_array((new XMLNodes('root/item'))->texts( + new StringSourceStream(path('memory://a.xml'), '12'), + 8192, + )); + } } diff --git a/src/adapter/etl-adapter-xml/tests/Flow/ETL/Adapter/XML/Tests/Unit/XMLOpenSinkTest.php b/src/adapter/etl-adapter-xml/tests/Flow/ETL/Adapter/XML/Tests/Unit/XMLOpenSinkTest.php new file mode 100644 index 0000000000..05f0f038c1 --- /dev/null +++ b/src/adapter/etl-adapter-xml/tests/Flow/ETL/Adapter/XML/Tests/Unit/XMLOpenSinkTest.php @@ -0,0 +1,47 @@ +writeTo(path('memory://out.xml')); + $sink = new XMLOpenSink($stream, new XMLEncoder(new StringXMLWriter()), "\n\n", 'root'); + + $sink->write(array_to_rows([['id' => 1]], schema(int_schema('id')))); + $sink->write(array_to_rows([['id' => 2]], schema(int_schema('id')))); + $sink->close(); + $stream->close(); + + static::assertSame( + "\n\n1\n2\n", + $filesystem->readFrom(path('memory://out.xml'))->content(), + ); + } + + public function test_a_sink_that_wrote_nothing_closes_without_a_root_end(): void + { + $filesystem = memory_filesystem(); + $stream = $filesystem->writeTo(path('memory://out.xml')); + + (new XMLOpenSink($stream, new XMLEncoder(new StringXMLWriter()), "\n\n", 'root'))->close(); + $stream->close(); + + static::assertSame('', $filesystem->readFrom(path('memory://out.xml'))->content()); + } +} diff --git a/src/adapter/etl-adapter-xml/tests/Flow/ETL/Adapter/XML/Tests/Unit/XMLValueNodesTest.php b/src/adapter/etl-adapter-xml/tests/Flow/ETL/Adapter/XML/Tests/Unit/XMLValueNodesTest.php new file mode 100644 index 0000000000..611ce22817 --- /dev/null +++ b/src/adapter/etl-adapter-xml/tests/Flow/ETL/Adapter/XML/Tests/Unit/XMLValueNodesTest.php @@ -0,0 +1,168 @@ +, mixed, XMLNode}> + */ + public static function nodes(): Generator + { + yield 'null' => [type_optional(type_list(type_integer())), null, XMLNode::flatNode('v', '')]; + yield 'a leaf' => [type_float(), 1.0, XMLNode::flatNode('v', '1.0')]; + yield 'list' => [ + type_list(type_string()), + ['a', 'b'], + XMLNode::nested('v', XMLNode::flatNode('element', 'a'), XMLNode::flatNode('element', 'b')), + ]; + yield 'an empty list' => [type_list(type_string()), [], XMLNode::nestedNode('v')]; + yield 'list' => [ + type_list(type_optional(type_integer())), + [1, null], + XMLNode::nested('v', XMLNode::flatNode('element', '1'), XMLNode::flatNode('element', '')), + ]; + yield 'list' => [ + type_list(type_date()), + [20_455], + XMLNode::nested('v', XMLNode::flatNode('element', '2026-01-02')), + ]; + yield 'list keeps the stored text' => [ + type_list(type_json()), + ['{"a": 1}'], + XMLNode::nested('v', XMLNode::flatNode('element', '{"a": 1}')), + ]; + yield 'map' => [ + type_map(type_string(), type_optional(type_integer())), + ['x' => 1, 'y' => null], + XMLNode::nested( + 'v', + XMLNode::nested('element', XMLNode::flatNode('key', 'x'), XMLNode::flatNode('value', '1')), + XMLNode::nested('element', XMLNode::flatNode('key', 'y'), XMLNode::flatNode('value', '')), + ), + ]; + yield 'a map keyed by integers' => [ + type_map(type_integer(), type_string()), + [5 => 'a'], + XMLNode::nested('v', XMLNode::nested( + 'element', + XMLNode::flatNode('key', '5'), + XMLNode::flatNode('value', 'a'), + )), + ]; + yield 'an empty map' => [type_map(type_string(), type_integer()), [], XMLNode::nestedNode('v')]; + yield 'structure' => [ + type_structure(['d' => type_datetime(), 'l' => type_list(type_integer())]), + ['d' => 1_767_323_045_000_000, 'l' => [1]], + XMLNode::nested( + 'v', + XMLNode::flatNode('d', '2026-01-02T03:04:05+00:00'), + XMLNode::nested('l', XMLNode::flatNode('element', '1')), + ), + ]; + yield 'a structure element whose name carries the prefix is an attribute' => [ + type_structure(['_id' => type_integer(), 'city' => type_string()]), + ['_id' => 7, 'city' => 'Krakow'], + XMLNode::nested('v', new XMLAttribute('id', '7'), XMLNode::flatNode('city', 'Krakow')), + ]; + } + + /** + * @param Type $type + */ + #[DataProvider('nodes')] + public function test_of(Type $type, mixed $physical, XMLNode $expected): void + { + static::assertEquals($expected, (new XMLValueNodes(new TextValues()))->of('v', $type, $physical)); + } + + public function test_of_uses_the_configured_names_and_prefix(): void + { + $nodes = new XMLValueNodes(new TextValues(), '@', 'item', 'entry', '@k', 'val'); + + static::assertEquals( + XMLNode::nested('v', XMLNode::flatNode('item', 'a')), + $nodes->of('v', type_list(type_string()), ['a']), + ); + static::assertEquals( + XMLNode::nested('v', XMLNode::nested('entry', new XMLAttribute('k', 'x'), XMLNode::flatNode('val', '1'))), + $nodes->of('v', type_map(type_string(), type_integer()), ['x' => 1]), + ); + } + + public function test_a_member_without_the_prefix_is_a_node(): void + { + static::assertEquals( + XMLNode::flatNode('id', '7'), + (new XMLValueNodes(new TextValues()))->member('id', type_integer(), 7), + ); + } + + public function test_a_null_member_that_is_an_attribute_is_refused(): void + { + $this->expectException(CastingException::class); + + (new XMLValueNodes(new TextValues()))->member('_id', type_optional(type_integer()), null); + } + + public function test_a_structure_with_an_optional_element_is_refused_at_any_depth(): void + { + $nodes = new XMLValueNodes(new TextValues()); + $optional = type_structure(['zip' => structure_element('zip', type_string(), optional: true)]); + $refused = 0; + + foreach ([ + $optional, + type_list($optional), + type_map(type_string(), $optional), + type_structure(['s' => $optional]), + ] as $type) { + try { + $nodes->refuseOptionalElements($type); + } catch (RuntimeException $refusal) { + static::assertSame( + 'XML encoder does not support structure optional elements, given: structure{zip?: string}', + $refusal->getMessage(), + ); + $refused++; + } + } + + static::assertSame(4, $refused); + } + + public function test_types_without_optional_elements_are_accepted(): void + { + $this->expectNotToPerformAssertions(); + + (new XMLValueNodes(new TextValues()))->refuseOptionalElements(type_list(type_structure([ + 'a' => type_optional(type_integer()), + 'm' => type_map(type_string(), type_date()), + ]))); + } +} diff --git a/src/adapter/etl-adapter-xml/tests/Flow/ETL/Adapter/XML/Tests/Unit/XMLWriter/DOMDocumentWriterTest.php b/src/adapter/etl-adapter-xml/tests/Flow/ETL/Adapter/XML/Tests/Unit/XMLWriter/DOMDocumentWriterTest.php index 44ed5a11bd..3e5d9c677a 100644 --- a/src/adapter/etl-adapter-xml/tests/Flow/ETL/Adapter/XML/Tests/Unit/XMLWriter/DOMDocumentWriterTest.php +++ b/src/adapter/etl-adapter-xml/tests/Flow/ETL/Adapter/XML/Tests/Unit/XMLWriter/DOMDocumentWriterTest.php @@ -4,10 +4,16 @@ namespace Flow\ETL\Adapter\XML\Tests\Unit\XMLWriter; +use DOMException; use Flow\ETL\Adapter\XML\Abstraction\XMLAttribute; use Flow\ETL\Adapter\XML\Abstraction\XMLNode; +use Flow\ETL\Adapter\XML\Tests\Context\XMLWriterValues; use Flow\ETL\Adapter\XML\XMLWriter\DOMDocumentWriter; use Flow\ETL\Tests\FlowTestCase; +use Flow\Types\Exception\CastingException; +use PHPUnit\Framework\Attributes\DataProviderExternal; + +use function substr; final class DOMDocumentWriterTest extends FlowTestCase { @@ -75,4 +81,64 @@ public function test_writing_xml_with_empty_attribute(): void $xmlWriter->write(XMLNode::flatNode('root', 'value')->appendAttribute(new XMLAttribute('attribute', ''))), ); } + + #[DataProviderExternal(XMLWriterValues::class, 'values')] + public function test_elements_are_what_write_renders_for_a_flat_node(string $value): void + { + $writer = new DOMDocumentWriter(); + + static::assertSame( + [$writer->write(XMLNode::flatNode('v', $value)), $writer->write(XMLNode::flatNode('v', 'second'))], + $writer->elements('v', [$value, 'second']), + ); + } + + #[DataProviderExternal(XMLWriterValues::class, 'values')] + public function test_attributes_are_what_write_renders_for_an_attribute(string $value): void + { + $writer = new DOMDocumentWriter(); + + static::assertSame( + [ + substr($writer->write(XMLNode::nested('x', new XMLAttribute('a', $value))), 2, -2), + ' a="second"', + ], + $writer->attributes('a', [$value, 'second']), + ); + } + + public function test_a_null_element_is_an_empty_element(): void + { + static::assertSame( + ['', 'x', ''], + (new DOMDocumentWriter())->elements('v', [null, 'x', '']), + ); + } + + public function test_a_null_attribute_is_refused(): void + { + $this->expectException(CastingException::class); + + (new DOMDocumentWriter())->attributes('a', ['x', null]); + } + + public function test_no_values_give_no_elements_and_no_attributes(): void + { + static::assertSame([], (new DOMDocumentWriter())->elements('v', [])); + static::assertSame([], (new DOMDocumentWriter())->attributes('a', [])); + } + + public function test_elements_of_a_name_dom_refuses_are_refused_with_its_exception(): void + { + $this->expectException(DOMException::class); + + (new DOMDocumentWriter())->elements('1 bad', ['x']); + } + + public function test_attributes_of_a_name_dom_refuses_are_refused_with_its_exception(): void + { + $this->expectException(DOMException::class); + + (new DOMDocumentWriter())->attributes('1 bad', ['x']); + } } diff --git a/src/adapter/etl-adapter-xml/tests/Flow/ETL/Adapter/XML/Tests/Unit/XMLWriter/StringXMLWriterTest.php b/src/adapter/etl-adapter-xml/tests/Flow/ETL/Adapter/XML/Tests/Unit/XMLWriter/StringXMLWriterTest.php index aa88635ae5..4362e2bcec 100644 --- a/src/adapter/etl-adapter-xml/tests/Flow/ETL/Adapter/XML/Tests/Unit/XMLWriter/StringXMLWriterTest.php +++ b/src/adapter/etl-adapter-xml/tests/Flow/ETL/Adapter/XML/Tests/Unit/XMLWriter/StringXMLWriterTest.php @@ -7,13 +7,18 @@ use DOMException; use Flow\ETL\Adapter\XML\Abstraction\XMLAttribute; use Flow\ETL\Adapter\XML\Abstraction\XMLNode; +use Flow\ETL\Adapter\XML\Tests\Context\XMLWriterValues; use Flow\ETL\Adapter\XML\XMLWriter\DOMDocumentWriter; use Flow\ETL\Adapter\XML\XMLWriter\StringXMLWriter; use Flow\ETL\Tests\FlowTestCase; +use Flow\Types\Exception\CastingException; use Generator; use PHPUnit\Framework\Attributes\DataProvider; +use PHPUnit\Framework\Attributes\DataProviderExternal; use PHPUnit\Framework\Attributes\TestWith; +use function substr; + final class StringXMLWriterTest extends FlowTestCase { /** @@ -134,4 +139,61 @@ public function test_writes_what_dom_document_writer_writes_on_libxml_2_14_and_l ): void { static::assertSame($xml, (new StringXMLWriter())->write($node)); } + + #[DataProviderExternal(XMLWriterValues::class, 'values')] + public function test_elements_are_what_write_renders_for_a_flat_node(string $value): void + { + $writer = new StringXMLWriter(); + + static::assertSame( + [$writer->write(XMLNode::flatNode('v', $value)), $writer->write(XMLNode::flatNode('v', 'second'))], + $writer->elements('v', [$value, 'second']), + ); + } + + #[DataProviderExternal(XMLWriterValues::class, 'values')] + public function test_attributes_are_what_write_renders_for_an_attribute(string $value): void + { + $writer = new StringXMLWriter(); + + static::assertSame( + [ + substr($writer->write(XMLNode::nested('x', new XMLAttribute('a', $value))), 2, -2), + ' a="second"', + ], + $writer->attributes('a', [$value, 'second']), + ); + } + + public function test_a_null_element_is_an_empty_element(): void + { + static::assertSame(['', 'x', ''], (new StringXMLWriter())->elements('v', [null, 'x', ''])); + } + + public function test_a_null_attribute_is_refused(): void + { + $this->expectException(CastingException::class); + + (new StringXMLWriter())->attributes('a', ['x', null]); + } + + public function test_no_values_give_no_elements_and_no_attributes(): void + { + static::assertSame([], (new StringXMLWriter())->elements('v', [])); + static::assertSame([], (new StringXMLWriter())->attributes('a', [])); + } + + public function test_elements_of_a_name_dom_refuses_are_refused_with_its_exception(): void + { + $this->expectException(DOMException::class); + + (new StringXMLWriter())->elements('1 bad', ['x']); + } + + public function test_attributes_of_a_name_dom_refuses_are_refused_with_its_exception(): void + { + $this->expectException(DOMException::class); + + (new StringXMLWriter())->attributes('1 bad', ['x']); + } } diff --git a/src/bridge/filesystem/sftp/tests/Flow/Filesystem/Bridge/SFTP/Tests/Integration/SFTPDataFrameTest.php b/src/bridge/filesystem/sftp/tests/Flow/Filesystem/Bridge/SFTP/Tests/Integration/SFTPDataFrameTest.php index dab8a20dbc..7e1bbe4a1f 100644 --- a/src/bridge/filesystem/sftp/tests/Flow/Filesystem/Bridge/SFTP/Tests/Integration/SFTPDataFrameTest.php +++ b/src/bridge/filesystem/sftp/tests/Flow/Filesystem/Bridge/SFTP/Tests/Integration/SFTPDataFrameTest.php @@ -34,8 +34,8 @@ public function test_orders_written_as_csv_are_read_back_from_sftp(): void $rows = data_frame()->read(from_csv(path('sftp:///upload/orders.csv'), filesystem: $filesystem))->fetch(); static::assertCount(1_000, $rows); - static::assertSame('user-0@example.com', $rows->first()->get('email')); - static::assertSame('user-999@example.com', $rows->last()?->get('email')); + static::assertSame('user-0@example.com', $rows->column('email')->value(0)); + static::assertSame('user-999@example.com', $rows->column('email')->value($rows->count() - 1)); } public function test_orders_written_as_parquet_survive_a_round_trip_with_every_entry_type(): void @@ -55,28 +55,29 @@ public function test_orders_written_as_parquet_survive_a_round_trip_with_every_e static::assertCount(1_000, $rows); - $first = $rows->first(); - - static::assertSame(0, $first->get('index')); + static::assertSame(0, $rows->column('index')->value(0)); static::assertSame( '254d61c5-22c8-4407-83a2-76f1cab53af2', - type_uuid()->assert($first->get('order_id'))->toString(), + type_uuid()->assert($rows->column('order_id')->value(0))->toString(), ); static::assertSame( '2025-01-01 12:00:00', - type_datetime()->assert($first->get('created_at'))->format('Y-m-d H:i:s'), + type_datetime()->assert($rows->column('created_at')->value(0))->format('Y-m-d H:i:s'), ); static::assertSame( ['street' => '123 Main St, Apt 0', 'city' => 'City ', 'zip' => '12345-0', 'country' => 'PL'], - $first->get('address'), + $rows->column('address')->value(0), + ); + static::assertSame( + ['Note 1 for order 0', 'Note 2 for order 0', 'Note 3 for order 0'], + $rows->column('notes')->value(0), ); - static::assertSame(['Note 1 for order 0', 'Note 2 for order 0', 'Note 3 for order 0'], $first->get('notes')); static::assertEqualsWithDelta( [ ['sku' => 'SKU_0001', 'quantity' => 1, 'price' => 0.14], ['sku' => 'SKU_0002', 'quantity' => 2, 'price' => 25.13], ], - $first->get('items'), + $rows->column('items')->value(0), 0.0001, ); } diff --git a/src/bridge/phpunit/postgresql/tests/Flow/Bridge/PHPUnit/PostgreSQL/Tests/Unit/SkipTransactionRollbackTest.php b/src/bridge/phpunit/postgresql/tests/Flow/Bridge/PHPUnit/PostgreSQL/Tests/Unit/SkipTransactionRollbackTest.php index 7f66ee1ada..3936d02e9c 100644 --- a/src/bridge/phpunit/postgresql/tests/Flow/Bridge/PHPUnit/PostgreSQL/Tests/Unit/SkipTransactionRollbackTest.php +++ b/src/bridge/phpunit/postgresql/tests/Flow/Bridge/PHPUnit/PostgreSQL/Tests/Unit/SkipTransactionRollbackTest.php @@ -17,21 +17,27 @@ final class SkipTransactionRollbackTest extends TestCase { public function test_attribute_can_target_class(): void { - $attribute = (new ReflectionClass(SkipTransactionRollback::class))->getAttributes(Attribute::class)[0]->newInstance(); + $attribute = (new ReflectionClass(SkipTransactionRollback::class))->getAttributes( + Attribute::class, + )[0]->newInstance(); static::assertNotSame(0, $attribute->flags & Attribute::TARGET_CLASS); } public function test_attribute_can_target_method(): void { - $attribute = (new ReflectionClass(SkipTransactionRollback::class))->getAttributes(Attribute::class)[0]->newInstance(); + $attribute = (new ReflectionClass(SkipTransactionRollback::class))->getAttributes( + Attribute::class, + )[0]->newInstance(); static::assertNotSame(0, $attribute->flags & Attribute::TARGET_METHOD); } public function test_attribute_detected_on_class(): void { - static::assertNotEmpty((new ReflectionClass(SkippedTestCase::class))->getAttributes(SkipTransactionRollback::class)); + static::assertNotEmpty((new ReflectionClass(SkippedTestCase::class))->getAttributes( + SkipTransactionRollback::class, + )); } public function test_attribute_detected_on_method(): void @@ -54,6 +60,8 @@ public function test_attribute_detected_on_parent_abstract_class(): void public function test_attribute_not_present_on_normal_class(): void { - static::assertEmpty((new ReflectionClass(NormalTestCase::class))->getAttributes(SkipTransactionRollback::class)); + static::assertEmpty((new ReflectionClass(NormalTestCase::class))->getAttributes( + SkipTransactionRollback::class, + )); } } diff --git a/src/bridge/symfony/filesystem-bundle/tests/Flow/Bridge/Symfony/FilesystemBundle/Tests/Unit/DependencyInjection/Compiler/RegisterFilesystemFactoriesPassTest.php b/src/bridge/symfony/filesystem-bundle/tests/Flow/Bridge/Symfony/FilesystemBundle/Tests/Unit/DependencyInjection/Compiler/RegisterFilesystemFactoriesPassTest.php index ff8b659e2a..5d2f40ba11 100644 --- a/src/bridge/symfony/filesystem-bundle/tests/Flow/Bridge/Symfony/FilesystemBundle/Tests/Unit/DependencyInjection/Compiler/RegisterFilesystemFactoriesPassTest.php +++ b/src/bridge/symfony/filesystem-bundle/tests/Flow/Bridge/Symfony/FilesystemBundle/Tests/Unit/DependencyInjection/Compiler/RegisterFilesystemFactoriesPassTest.php @@ -31,7 +31,9 @@ public function test_does_nothing_when_registry_definition_absent(): void public function test_injects_tagged_factories_as_references(): void { $container = new ContainerBuilder(); - $container->setDefinition('.flow.filesystem.factory_registry', (new Definition(FilesystemFactoryRegistry::class))->setArgument(0, [])); + $container->setDefinition('.flow.filesystem.factory_registry', (new Definition( + FilesystemFactoryRegistry::class, + ))->setArgument(0, [])); $memory = new Definition(MemoryFilesystemFactory::class); $memory->addTag('flow.filesystem.factory', ['type' => 'memory']); @@ -57,7 +59,9 @@ public function test_injects_tagged_factories_as_references(): void public function test_leaves_registry_empty_when_no_tagged_services(): void { $container = new ContainerBuilder(); - $container->setDefinition('.flow.filesystem.factory_registry', (new Definition(FilesystemFactoryRegistry::class))->setArgument(0, [])); + $container->setDefinition('.flow.filesystem.factory_registry', (new Definition( + FilesystemFactoryRegistry::class, + ))->setArgument(0, [])); (new RegisterFilesystemFactoriesPass())->process($container); @@ -67,7 +71,9 @@ public function test_leaves_registry_empty_when_no_tagged_services(): void public function test_throws_on_duplicate_type_across_services(): void { $container = new ContainerBuilder(); - $container->setDefinition('.flow.filesystem.factory_registry', (new Definition(FilesystemFactoryRegistry::class))->setArgument(0, [])); + $container->setDefinition('.flow.filesystem.factory_registry', (new Definition( + FilesystemFactoryRegistry::class, + ))->setArgument(0, [])); $a = new Definition(MemoryFilesystemFactory::class); $a->addTag('flow.filesystem.factory', ['type' => 'memory']); @@ -86,7 +92,9 @@ public function test_throws_on_duplicate_type_across_services(): void public function test_throws_on_tag_without_type_attribute(): void { $container = new ContainerBuilder(); - $container->setDefinition('.flow.filesystem.factory_registry', (new Definition(FilesystemFactoryRegistry::class))->setArgument(0, [])); + $container->setDefinition('.flow.filesystem.factory_registry', (new Definition( + FilesystemFactoryRegistry::class, + ))->setArgument(0, [])); $memory = new Definition(MemoryFilesystemFactory::class); $memory->addTag('flow.filesystem.factory'); diff --git a/src/bridge/symfony/http-foundation/tests/Flow/Bridge/Symfony/HttpFoundation/Tests/Unit/Response/FlowBufferedResponseTest.php b/src/bridge/symfony/http-foundation/tests/Flow/Bridge/Symfony/HttpFoundation/Tests/Unit/Response/FlowBufferedResponseTest.php index 461adee731..18a3230f87 100644 --- a/src/bridge/symfony/http-foundation/tests/Flow/Bridge/Symfony/HttpFoundation/Tests/Unit/Response/FlowBufferedResponseTest.php +++ b/src/bridge/symfony/http-foundation/tests/Flow/Bridge/Symfony/HttpFoundation/Tests/Unit/Response/FlowBufferedResponseTest.php @@ -9,10 +9,9 @@ use function Flow\Bridge\Symfony\HttpFoundation\http_json_output; use function Flow\Bridge\Symfony\HttpFoundation\http_stream_open; +use function Flow\ETL\DSL\array_to_rows; use function Flow\ETL\DSL\from_array; use function Flow\ETL\DSL\int_schema; -use function Flow\ETL\DSL\row; -use function Flow\ETL\DSL\rows; use function Flow\ETL\DSL\schema; final class FlowBufferedResponseTest extends FlowTestCase @@ -27,7 +26,10 @@ public function test_response_from_empty_dataset(): void public function test_response_is_buffered_only_once(): void { - $extractor = new CountingExtractor(schema(int_schema('id')), rows(schema(int_schema('id')), row(['id' => 1]))); + $extractor = new CountingExtractor( + schema(int_schema('id')), + array_to_rows([['id' => 1]], schema(int_schema('id'))), + ); $response = http_stream_open($extractor)->response(http_json_output()); diff --git a/src/bridge/symfony/postgresql-bundle/tests/Flow/Bridge/Symfony/PostgreSqlBundle/Tests/Fixtures/migrations/container_access/20260601000000_container_access/migration.php b/src/bridge/symfony/postgresql-bundle/tests/Flow/Bridge/Symfony/PostgreSqlBundle/Tests/Fixtures/migrations/container_access/20260601000000_container_access/migration.php index 63e15741a1..a218de7e70 100644 --- a/src/bridge/symfony/postgresql-bundle/tests/Flow/Bridge/Symfony/PostgreSqlBundle/Tests/Fixtures/migrations/container_access/20260601000000_container_access/migration.php +++ b/src/bridge/symfony/postgresql-bundle/tests/Flow/Bridge/Symfony/PostgreSqlBundle/Tests/Fixtures/migrations/container_access/20260601000000_container_access/migration.php @@ -14,7 +14,9 @@ return new class implements Migration { public function migrate(MigrationContext $context): void { - $container = type_instance_of(ContainerInterface::class)->assert($context->attribute(FlowPostgreSqlBundle::SERVICE_CONTAINER)); + $container = type_instance_of(ContainerInterface::class)->assert($context->attribute( + FlowPostgreSqlBundle::SERVICE_CONTAINER, + )); $table = type_string()->assert($container->getParameter('flow_test.container_table')); $seed = type_instance_of(MigrationSeedProvider::class) diff --git a/src/bridge/symfony/postgresql-messenger/tests/Flow/Bridge/Symfony/PostgreSQLMessenger/Tests/Unit/FlowPostgreSqlTransportTest.php b/src/bridge/symfony/postgresql-messenger/tests/Flow/Bridge/Symfony/PostgreSQLMessenger/Tests/Unit/FlowPostgreSqlTransportTest.php index de5b73efda..cde23cf3c0 100644 --- a/src/bridge/symfony/postgresql-messenger/tests/Flow/Bridge/Symfony/PostgreSQLMessenger/Tests/Unit/FlowPostgreSqlTransportTest.php +++ b/src/bridge/symfony/postgresql-messenger/tests/Flow/Bridge/Symfony/PostgreSQLMessenger/Tests/Unit/FlowPostgreSqlTransportTest.php @@ -47,7 +47,9 @@ public function test_all_delegates_to_receiver(): void public function test_does_not_implement_setupable_transport_interface(): void { - static::assertFalse((new ReflectionClass(FlowPostgreSqlTransport::class))->implementsInterface(SetupableTransportInterface::class)); + static::assertFalse((new ReflectionClass(FlowPostgreSqlTransport::class))->implementsInterface( + SetupableTransportInterface::class, + )); } public function test_find_delegates_to_receiver(): void diff --git a/src/bridge/symfony/telemetry-bundle/composer.json b/src/bridge/symfony/telemetry-bundle/composer.json index 545c85e3c8..0c9ad6d707 100644 --- a/src/bridge/symfony/telemetry-bundle/composer.json +++ b/src/bridge/symfony/telemetry-bundle/composer.json @@ -29,7 +29,7 @@ "require-dev": { "composer-runtime-api": "^2.2", "composer/semver": "^3.4", - "doctrine/dbal": "^4.4", + "doctrine/dbal": "^4.5", "flow-php/psr18-telemetry-bridge": "self.version", "flow-php/symfony-http-foundation-telemetry-bridge": "self.version", "flow-php/telemetry-otlp-bridge": "self.version", @@ -46,7 +46,7 @@ "twig/twig": "^3.0" }, "suggest": { - "doctrine/dbal": "Required for Doctrine DBAL tracing (^4.4)", + "doctrine/dbal": "Required for Doctrine DBAL tracing (^4.5)", "flow-php/psr18-telemetry-bridge": "Required for PSR-18 HTTP client tracing", "flow-php/symfony-http-foundation-telemetry-bridge": "Required for HTTP trace context propagation (extract incoming / inject outgoing W3C trace headers)", "flow-php/telemetry-otlp-bridge": "Required for OTLP exporter support", diff --git a/src/cli/src/Flow/CLI/Factory/LoaderFactory.php b/src/cli/src/Flow/CLI/Factory/LoaderFactory.php index 9d912b2748..8307cb9322 100644 --- a/src/cli/src/Flow/CLI/Factory/LoaderFactory.php +++ b/src/cli/src/Flow/CLI/Factory/LoaderFactory.php @@ -7,7 +7,7 @@ use Flow\CLI\Options\FileFormat; use Flow\ETL\Exception\InvalidArgumentException; use Flow\ETL\Loader; -use Flow\ETL\Loader\FileLoader; +use Flow\ETL\Loader\File\FileLoader; use Flow\Filesystem\Path; use Symfony\Component\Console\Input\InputInterface; diff --git a/src/cli/tests/Flow/CLI/Tests/Integration/DatabaseTableListCommandTest.php b/src/cli/tests/Flow/CLI/Tests/Integration/DatabaseTableListCommandTest.php index 75a5022dbf..40422c0130 100644 --- a/src/cli/tests/Flow/CLI/Tests/Integration/DatabaseTableListCommandTest.php +++ b/src/cli/tests/Flow/CLI/Tests/Integration/DatabaseTableListCommandTest.php @@ -38,21 +38,34 @@ protected function setUp(): void public function test_run_db_table_list(): void { - $this->dbContext()->createTable((new Table('table_01', [ - new Column('id', Type::getType(Types::INTEGER), ['notnull' => true]), - new Column('name', Type::getType(Types::STRING), ['notnull' => true, 'length' => 255]), - new Column('description', Type::getType(Types::STRING), ['notnull' => true, 'length' => 255]), - ]))->addPrimaryKeyConstraint( - new PrimaryKeyConstraint(null, [new UnqualifiedName(Identifier::unquoted('id'))], true), - )); - - $this->dbContext()->createTable((new Table('table_02', [ - new Column('id', Type::getType(Types::INTEGER), ['notnull' => true]), - new Column('created_at', Type::getType(Types::DATETIME_IMMUTABLE), ['notnull' => true]), - new Column('tags', Type::getType(Types::JSON), ['notnull' => true, 'platformOptions' => ['jsonb' => true]]), - ]))->addPrimaryKeyConstraint( - new PrimaryKeyConstraint(null, [new UnqualifiedName(Identifier::unquoted('id'))], true), - )); + $this->dbContext()->createTable( + (new Table('table_01', [ + new Column('id', Type::getType(Types::INTEGER), ['notnull' => true]), + new Column('name', Type::getType(Types::STRING), ['notnull' => true, 'length' => 255]), + new Column('description', Type::getType(Types::STRING), ['notnull' => true, 'length' => 255]), + ])) + ->edit() + ->addPrimaryKeyConstraint( + new PrimaryKeyConstraint(null, [new UnqualifiedName(Identifier::unquoted('id'))], true), + ) + ->create(), + ); + + $this->dbContext()->createTable( + (new Table('table_02', [ + new Column('id', Type::getType(Types::INTEGER), ['notnull' => true]), + new Column('created_at', Type::getType(Types::DATETIME_IMMUTABLE), ['notnull' => true]), + new Column('tags', Type::getType(Types::JSON), [ + 'notnull' => true, + 'platformOptions' => ['jsonb' => true], + ]), + ])) + ->edit() + ->addPrimaryKeyConstraint( + new PrimaryKeyConstraint(null, [new UnqualifiedName(Identifier::unquoted('id'))], true), + ) + ->create(), + ); $tester = new CommandTester(new DatabaseTableListCommand('db:table:list')); diff --git a/src/cli/tests/Flow/CLI/Tests/Integration/DatabaseTableSchemaCommandTest.php b/src/cli/tests/Flow/CLI/Tests/Integration/DatabaseTableSchemaCommandTest.php index dc2d58badf..905d3da6a9 100644 --- a/src/cli/tests/Flow/CLI/Tests/Integration/DatabaseTableSchemaCommandTest.php +++ b/src/cli/tests/Flow/CLI/Tests/Integration/DatabaseTableSchemaCommandTest.php @@ -40,21 +40,34 @@ protected function setUp(): void public function test_run_db_table_list(): void { - $this->dbContext()->createTable((new Table('table_01', [ - new Column('id', Type::getType(Types::INTEGER), ['notnull' => true]), - new Column('name', Type::getType(Types::STRING), ['notnull' => true, 'length' => 255]), - new Column('description', Type::getType(Types::STRING), ['notnull' => true, 'length' => 255]), - ]))->addPrimaryKeyConstraint( - new PrimaryKeyConstraint(null, [new UnqualifiedName(Identifier::unquoted('id'))], true), - )); - - $this->dbContext()->createTable((new Table('table_02', [ - new Column('id', Type::getType(Types::INTEGER), ['notnull' => true]), - new Column('created_at', Type::getType(Types::DATETIME_IMMUTABLE), ['notnull' => true]), - new Column('tags', Type::getType(Types::JSON), ['notnull' => true, 'platformOptions' => ['jsonb' => true]]), - ]))->addPrimaryKeyConstraint( - new PrimaryKeyConstraint(null, [new UnqualifiedName(Identifier::unquoted('id'))], true), - )); + $this->dbContext()->createTable( + (new Table('table_01', [ + new Column('id', Type::getType(Types::INTEGER), ['notnull' => true]), + new Column('name', Type::getType(Types::STRING), ['notnull' => true, 'length' => 255]), + new Column('description', Type::getType(Types::STRING), ['notnull' => true, 'length' => 255]), + ])) + ->edit() + ->addPrimaryKeyConstraint( + new PrimaryKeyConstraint(null, [new UnqualifiedName(Identifier::unquoted('id'))], true), + ) + ->create(), + ); + + $this->dbContext()->createTable( + (new Table('table_02', [ + new Column('id', Type::getType(Types::INTEGER), ['notnull' => true]), + new Column('created_at', Type::getType(Types::DATETIME_IMMUTABLE), ['notnull' => true]), + new Column('tags', Type::getType(Types::JSON), [ + 'notnull' => true, + 'platformOptions' => ['jsonb' => true], + ]), + ])) + ->edit() + ->addPrimaryKeyConstraint( + new PrimaryKeyConstraint(null, [new UnqualifiedName(Identifier::unquoted('id'))], true), + ) + ->create(), + ); $tester = new CommandTester(new DatabaseTableSchemaCommand('db:table:schema')); @@ -98,13 +111,18 @@ public function test_selecting_specific_columns_only(): void static::markTestSkipped('This test is not supported in doctrine/dbal 4.0'); } - $this->dbContext()->createTable((new Table('table_01', [ - new Column('id', Type::getType(Types::INTEGER), ['notnull' => true]), - new Column('name', Type::getType(Types::STRING), ['notnull' => true, 'length' => 255]), - new Column('description', Type::getType(Types::STRING), ['notnull' => true, 'length' => 255]), - ]))->addPrimaryKeyConstraint( - new PrimaryKeyConstraint(null, [new UnqualifiedName(Identifier::unquoted('id'))], true), - )); + $this->dbContext()->createTable( + (new Table('table_01', [ + new Column('id', Type::getType(Types::INTEGER), ['notnull' => true]), + new Column('name', Type::getType(Types::STRING), ['notnull' => true, 'length' => 255]), + new Column('description', Type::getType(Types::STRING), ['notnull' => true, 'length' => 255]), + ])) + ->edit() + ->addPrimaryKeyConstraint( + new PrimaryKeyConstraint(null, [new UnqualifiedName(Identifier::unquoted('id'))], true), + ) + ->create(), + ); $tester = new CommandTester(new DatabaseTableSchemaCommand('db:table:schema')); @@ -133,13 +151,18 @@ public function test_selecting_not_existing_column(): void static::markTestSkipped('This test is not supported in doctrine/dbal 4.0'); } - $this->dbContext()->createTable((new Table('table_01', [ - new Column('id', Type::getType(Types::INTEGER), ['notnull' => true]), - new Column('name', Type::getType(Types::STRING), ['notnull' => true, 'length' => 255]), - new Column('description', Type::getType(Types::STRING), ['notnull' => true, 'length' => 255]), - ]))->addPrimaryKeyConstraint( - new PrimaryKeyConstraint(null, [new UnqualifiedName(Identifier::unquoted('id'))], true), - )); + $this->dbContext()->createTable( + (new Table('table_01', [ + new Column('id', Type::getType(Types::INTEGER), ['notnull' => true]), + new Column('name', Type::getType(Types::STRING), ['notnull' => true, 'length' => 255]), + new Column('description', Type::getType(Types::STRING), ['notnull' => true, 'length' => 255]), + ])) + ->edit() + ->addPrimaryKeyConstraint( + new PrimaryKeyConstraint(null, [new UnqualifiedName(Identifier::unquoted('id'))], true), + ) + ->create(), + ); $tester = new CommandTester(new DatabaseTableSchemaCommand('db:table:schema')); diff --git a/src/cli/tests/Flow/CLI/Tests/Unit/Loader/ProgressBarLoaderTest.php b/src/cli/tests/Flow/CLI/Tests/Unit/Loader/ProgressBarLoaderTest.php index 78211be901..7b49c27c51 100644 --- a/src/cli/tests/Flow/CLI/Tests/Unit/Loader/ProgressBarLoaderTest.php +++ b/src/cli/tests/Flow/CLI/Tests/Unit/Loader/ProgressBarLoaderTest.php @@ -9,9 +9,9 @@ use Symfony\Component\Console\Helper\ProgressBar; use Symfony\Component\Console\Output\BufferedOutput; +use function Flow\ETL\DSL\array_to_rows; use function Flow\ETL\DSL\flow_context; use function Flow\ETL\DSL\int_schema; -use function Flow\ETL\DSL\row; use function Flow\ETL\DSL\rows; use function Flow\ETL\DSL\schema; @@ -21,10 +21,11 @@ public function test_it_advances_the_bar_by_the_batch_size(): void { $progressBar = new ProgressBar(new BufferedOutput()); - (new ProgressBarLoader($progressBar))->load( - rows(schema(int_schema('id')), row(['id' => 1]), row(['id' => 2]), row(['id' => 3])), - flow_context(), - ); + (new ProgressBarLoader($progressBar))->load(array_to_rows([ + ['id' => 1], + ['id' => 2], + ['id' => 3], + ], schema(int_schema('id'))), flow_context()); static::assertSame(3, $progressBar->getProgress()); } @@ -35,8 +36,8 @@ public function test_every_batch_adds_to_the_progress_so_far(): void $loader = new ProgressBarLoader($progressBar); $context = flow_context(); - $loader->load(rows(schema(int_schema('id')), row(['id' => 1]), row(['id' => 2])), $context); - $loader->load(rows(schema(int_schema('id')), row(['id' => 3])), $context); + $loader->load(array_to_rows([['id' => 1], ['id' => 2]], schema(int_schema('id'))), $context); + $loader->load(array_to_rows([['id' => 3]], schema(int_schema('id'))), $context); static::assertSame(3, $progressBar->getProgress()); } diff --git a/src/core/etl/composer.json b/src/core/etl/composer.json index 8ad3b01d4f..cf1598b099 100644 --- a/src/core/etl/composer.json +++ b/src/core/etl/composer.json @@ -21,6 +21,9 @@ "psr/simple-cache": "^1.0 || ^2.0 || ^3.0", "symfony/string": "^6.4 || ^7.4 || ^8.0" }, + "conflict": { + "ext-flow_php": "<0.45" + }, "suggest": { "ext-apcu": "Allows using ApcuCache as a pipeline cache implementation", "ext-bcmath": "BCMath extension for PHP for more precise calculations" diff --git a/src/core/etl/src/Flow/Calculator/RunningSum.php b/src/core/etl/src/Flow/Calculator/RunningSum.php index 2545c63227..34a0291f19 100644 --- a/src/core/etl/src/Flow/Calculator/RunningSum.php +++ b/src/core/etl/src/Flow/Calculator/RunningSum.php @@ -4,23 +4,71 @@ namespace Flow\Calculator; +use function abs; +use function is_finite; +use function is_float; +use function is_int; use function is_string; -final readonly class RunningSum +/** + * IEEE addition with Neumaier compensation: the rounding error of every float addition is carried and added back once, + * so a long run of floats does not drift. An exact addition is decimal (Calculator) over the compensated total. + */ +final class RunningSum { + private float $compensation = 0.0; + + private float|int $sum = 0; + public function __construct( - private Calculator $calculator = new Calculator(), + private readonly Calculator $calculator = new Calculator(), ) {} /** * @param float|int|numeric-string $value */ - public function add(float|int $sum, float|int|string $value, bool $exact): float|int + public function add(float|int|string $value, bool $exact): void + { + if ($exact) { + $this->sum = $this->calculator->add($this->value(), $value); + $this->compensation = 0.0; + + return; + } + + $value = is_string($value) ? (float) $value : $value; + + if (is_int($this->sum) && is_int($value)) { + // an int overflow promotes to float, as PHP's + does + $this->sum += $value; + + return; + } + + $sum = (float) $this->sum; + $total = $sum + $value; + + if (is_finite($total)) { + $this->compensation += abs($sum) >= abs($value) ? $sum - $total + $value : $value - $total + $sum; + } + + $this->sum = $total; + } + + public function merge(self $other, bool $exact): void { if ($exact) { - return $this->calculator->add($sum, $value); + $this->add($other->value(), true); + + return; } - return $sum + (is_string($value) ? (float) $value : $value); + $this->add($other->sum, false); + $this->compensation += $other->compensation; + } + + public function value(): float|int + { + return is_float($this->sum) && is_finite($this->sum) ? $this->sum + $this->compensation : $this->sum; } } diff --git a/src/core/etl/src/Flow/ETL/Bucketing/Bucket.php b/src/core/etl/src/Flow/ETL/Bucketing/Bucket.php index 4e86901e04..2d3fec16fc 100644 --- a/src/core/etl/src/Flow/ETL/Bucketing/Bucket.php +++ b/src/core/etl/src/Flow/ETL/Bucketing/Bucket.php @@ -4,14 +4,6 @@ namespace Flow\ETL\Bucketing; -use Flow\ETL\Row; -use Flow\ETL\Schema; - -use function Flow\ETL\DSL\int_schema; -use function Flow\ETL\DSL\row; -use function Flow\ETL\DSL\schema; -use function Flow\ETL\DSL\str_schema; - final readonly class Bucket { public function __construct( @@ -19,17 +11,4 @@ public function __construct( public int $totalRows, public int $index, ) {} - - public static function schema(): Schema - { - return schema(str_schema(BucketShape::id->value), int_schema(BucketShape::totalRows->value)); - } - - public function toRow(): Row - { - return row([ - BucketShape::id->value => $this->id, - BucketShape::totalRows->value => $this->totalRows, - ]); - } } diff --git a/src/core/etl/src/Flow/ETL/Bucketing/BucketShape.php b/src/core/etl/src/Flow/ETL/Bucketing/BucketShape.php deleted file mode 100644 index 72175df8dd..0000000000 --- a/src/core/etl/src/Flow/ETL/Bucketing/BucketShape.php +++ /dev/null @@ -1,14 +0,0 @@ - $rows - * - * @return Generator - */ - public function bucketize(Generator $rows, BucketsStorage $storage): Generator; -} diff --git a/src/core/etl/src/Flow/ETL/Bucketing/HashBucketing.php b/src/core/etl/src/Flow/ETL/Bucketing/HashBucketing.php index 4c87a5e71f..09c5382001 100644 --- a/src/core/etl/src/Flow/ETL/Bucketing/HashBucketing.php +++ b/src/core/etl/src/Flow/ETL/Bucketing/HashBucketing.php @@ -6,7 +6,6 @@ use Flow\ETL\Exception\InvalidArgumentException; use Flow\ETL\RandomValueGenerator; -use Flow\ETL\Row; use Flow\ETL\Row\Reference; use Flow\ETL\Rows; use Generator; @@ -16,7 +15,7 @@ use function sprintf; use function substr; -final class HashBucketing implements BucketingStrategy +final class HashBucketing { private readonly KeyValues $keyValues; @@ -40,6 +39,21 @@ public function __construct( $this->keyValues = new KeyValues($by); } + public function index(string $hash): int + { + return (int) hexdec(substr($hash, 0, 8)) % $this->bucketsCount; + } + + /** + * The bucket a row with these key values lands in. + * + * @param list $values in the order of the bucketing references + */ + public function indexOf(array $values): int + { + return $this->index($this->hasher->hash([$values])[0]); + } + /** * @param Generator $rows * @@ -58,20 +72,20 @@ public function bucketize(Generator $rows, BucketsStorage $storage): Generator foreach ($rows as $batch) { $hashes = $this->hasher->hash($this->keyValues->of($batch)); - /** @var array> $groups */ + /** @var array> $groups */ $groups = []; - foreach ($batch as $i => $row) { - $index = (int) hexdec(substr($hashes[$i], 0, 8)) % $this->bucketsCount; + foreach ($hashes as $i => $hash) { + $index = $this->index($hash); $id = sprintf('%s-%s-%d', $this->namespace, $runId, $index); - $groups[$id][] = $row; + $groups[$id][] = $i; $indexes[$id] = $index; } // a bucket is a subset of a batch that already passed the gate, under the same schema - foreach ($groups as $id => $groupRows) { - $storage->append($id, Rows::trusted($batch->schema(), $groupRows)); - $totals[$id] = ($totals[$id] ?? 0) + count($groupRows); + foreach ($groups as $id => $indices) { + $storage->append($id, $batch->gather($indices)); + $totals[$id] = ($totals[$id] ?? 0) + count($indices); } } diff --git a/src/core/etl/src/Flow/ETL/Bucketing/KeyGrouping.php b/src/core/etl/src/Flow/ETL/Bucketing/KeyGrouping.php index 2f8151a65b..77fae4e479 100644 --- a/src/core/etl/src/Flow/ETL/Bucketing/KeyGrouping.php +++ b/src/core/etl/src/Flow/ETL/Bucketing/KeyGrouping.php @@ -4,10 +4,11 @@ namespace Flow\ETL\Bucketing; +use Flow\ETL\Column\Backend; use Flow\ETL\Rows; use Generator; -use function array_values; +use function array_slice; /** * Collects rows sharing a key into one batch, so a downstream operator sees each key exactly once @@ -18,6 +19,7 @@ public function __construct( private KeyValues $keys, private Hasher $hasher, + private Backend $backend, ) {} /** @@ -27,30 +29,33 @@ public function __construct( */ public function group(Generator $rows): Generator { - /** @var array> $groups */ + /** @var array> $groups each key's rows, one gathered part per batch, in batch order */ $groups = []; $schema = null; - foreach ($rows as $batch) { - if (!$batch->count()) { + foreach ($rows as $input) { + if (!$input->count()) { continue; } - // bound once, like BucketAggregation: every batch out of one bucket shares a schema - $schema ??= $batch->schema(); - $hashes = $this->hasher->hash($this->keys->of($batch)); + // bound once: every batch out of one bucket shares a schema + $schema ??= $input->schema(); + $batch = $input->matchTo($schema, $this->backend); - foreach (array_values($batch->all()) as $index => $row) { - $groups[$hashes[$index]][] = $row; + /** @var array> $indices */ + $indices = []; + + foreach ($this->hasher->hash($this->keys->of($batch)) as $i => $hash) { + $indices[$hash][] = $i; } - } - if ($schema === null) { - return; + foreach ($indices as $hash => $rowsOfKey) { + $groups[$hash][] = $batch->gather($rowsOfKey); + } } - foreach ($groups as $rowsOfKey) { - yield new Rows($schema, ...$rowsOfKey); + foreach ($groups as $parts) { + yield $parts[0]->concat($this->backend, ...array_slice($parts, 1)); } } } diff --git a/src/core/etl/src/Flow/ETL/Bucketing/KeyValues.php b/src/core/etl/src/Flow/ETL/Bucketing/KeyValues.php index ad29bb53c6..57f652eadb 100644 --- a/src/core/etl/src/Flow/ETL/Bucketing/KeyValues.php +++ b/src/core/etl/src/Flow/ETL/Bucketing/KeyValues.php @@ -4,14 +4,14 @@ namespace Flow\ETL\Bucketing; -use Flow\ETL\Row; +use Flow\ETL\Column\ComparableValues; use Flow\ETL\Row\Reference; use Flow\ETL\Rows; -use Flow\ETL\Schema; /** * Extracts bucket-key values positionally, in reference order - position, not column name, defines * key identity, so two sides of a join extract hash-compatible values from differently named columns. + * Each value is its column's equality form (ComparableValues): === on it is value equality. */ final readonly class KeyValues { @@ -27,28 +27,25 @@ public function __construct( */ public function of(Rows $rows): array { + if ($rows->isEmpty()) { + return []; + } + $values = []; - $schema = $rows->schema(); - foreach ($rows as $row) { - $values[] = $this->ofRow($row, $schema); + for ($i = 0, $count = $rows->count(); $i < $count; $i++) { + $values[] = []; } - return $values; - } - - /** - * @return list - */ - public function ofRow(Row $row, Schema $schema): array - { - $values = []; + $comparable = new ComparableValues(); foreach ($this->refs as $ref) { - // absent under a nullable declaration is a legitimate null - it keys into a bucket that - // no comparison will match, which is what SQL says an unknown key does. Absent under NOT - // NULL is a row-shape violation, and Row::get() names it and lists the available columns. - $values[] = !$row->has($ref) && $schema->get($ref)->isNullable() ? null : $row->get($ref); + $column = $rows->column($rows->schema()->get($ref)->entry()->name()); + + // @mago-ignore analysis:mixed-assignment + foreach ($comparable->equality($column) as $i => $value) { + $values[$i][] = $value; + } } return $values; diff --git a/src/core/etl/src/Flow/ETL/Bucketing/NativeHasher.php b/src/core/etl/src/Flow/ETL/Bucketing/NativeHasher.php index 555f63c4ec..638f366ebf 100644 --- a/src/core/etl/src/Flow/ETL/Bucketing/NativeHasher.php +++ b/src/core/etl/src/Flow/ETL/Bucketing/NativeHasher.php @@ -8,12 +8,12 @@ use Flow\ETL\Hash\NativePHPHash; use Stringable; -use function array_map; use function is_bool; use function is_numeric; use function is_object; use function is_string; use function serialize; +use function strlen; final readonly class NativeHasher implements Hasher { @@ -56,7 +56,16 @@ public function hash(array $values): array $hashes = []; foreach ($values as $rowValues) { - $hashes[] = NativePHPHash::xxh128(serialize(array_map(self::normalize(...), $rowValues))); + $key = ''; + + // @mago-ignore analysis:mixed-assignment + foreach ($rowValues as $value) { + $normalized = self::normalize($value); + // length-prefixed, so a value cannot run into the next one + $key .= strlen($normalized) . ':' . $normalized; + } + + $hashes[] = NativePHPHash::xxh128($key); } return $hashes; diff --git a/src/core/etl/src/Flow/ETL/Bucketing/SortedRunBucketing.php b/src/core/etl/src/Flow/ETL/Bucketing/SortedRunBucketing.php deleted file mode 100644 index 2433817a56..0000000000 --- a/src/core/etl/src/Flow/ETL/Bucketing/SortedRunBucketing.php +++ /dev/null @@ -1,65 +0,0 @@ - $by - * @param int<1, max> $runSize - */ - public function __construct( - private readonly array $by, - private readonly int $runSize, - private readonly RandomValueGenerator $random, - ) { - // @mago-ignore analysis:invalid-operand - // @mago-ignore analysis:impossible-condition,redundant-comparison - if ($this->runSize < 1) { - throw new InvalidArgumentException('Run size must be greater than 0, given: ' . $this->runSize); - } - } - - /** - * @param Generator $rows - * - * @return Generator - */ - public function bucketize(Generator $rows, BucketsStorage $storage): Generator - { - $runId = $this->random->string(16); - $buffer = null; - $index = 0; - - foreach ($rows as $batch) { - $buffer = $buffer === null ? $batch : $buffer->merge($batch); - - while ($buffer->count() >= $this->runSize) { - yield $this->spill($buffer->take($this->runSize), $storage, $runId, $index++); - $buffer = $buffer->drop($this->runSize); - } - } - - if ($buffer !== null && !$buffer->empty()) { - yield $this->spill($buffer, $storage, $runId, $index); - } - } - - private function spill(Rows $run, BucketsStorage $storage, string $runId, int $index): Bucket - { - $bucketId = sprintf('sort-%s-%d', $runId, $index); - $storage->set($bucketId, $run->sortBy(...$this->by)); - - return new Bucket($bucketId, $run->count(), $index); - } -} diff --git a/src/core/etl/src/Flow/ETL/Bucketing/Storage/FilesystemBuckets.php b/src/core/etl/src/Flow/ETL/Bucketing/Storage/FilesystemBuckets.php index 3d6a3cf72c..39cc33e4ed 100644 --- a/src/core/etl/src/Flow/ETL/Bucketing/Storage/FilesystemBuckets.php +++ b/src/core/etl/src/Flow/ETL/Bucketing/Storage/FilesystemBuckets.php @@ -5,6 +5,7 @@ namespace Flow\ETL\Bucketing\Storage; use Flow\ETL\Bucketing\BucketsStorage; +use Flow\ETL\Column\Backend; use Flow\ETL\Exception\InvalidArgumentException; use Flow\ETL\Hash\NativePHPHash; use Flow\ETL\Rows; @@ -15,6 +16,8 @@ use Flow\Floe\Options; use Generator; +use function array_slice; + final class FilesystemBuckets implements BucketsStorage { private readonly Path $cacheDir; @@ -32,6 +35,7 @@ final class FilesystemBuckets implements BucketsStorage public function __construct( private readonly Filesystem $filesystem, Path $cacheDir, + private readonly Backend $backend, private readonly int $batchSize = 1000, ) { // @mago-ignore analysis:impossible-condition,redundant-comparison @@ -40,7 +44,7 @@ public function __construct( } $this->cacheDir = $cacheDir->suffix('/flow-php-buckets/'); - $this->reader = new FloeReader($this->filesystem); + $this->reader = new FloeReader($this->filesystem, $backend); } /** @@ -52,7 +56,7 @@ public function __construct( public function append(string $bucketId, Rows $rows): void { if (!isset($this->writers[$bucketId])) { - $writer = new FloeWriter($this->filesystem, $rows->schema(), new Options()); + $writer = new FloeWriter($this->filesystem, $rows->schema(), $this->backend, new Options()); $writer->append($this->keyPath($bucketId)); $this->writers[$bucketId] = $writer; } @@ -74,9 +78,25 @@ public function get(string $bucketId): Generator // finally, not a trailing close(): PHP runs it on generator destruction too, and a KWayMerge cursor is // destroyed rather than exhausted when a merge throws + // every append() is one Floe frame and a read never spans two, so small appends are coalesced back into + // batches of up to batchSize rows + $pending = []; + $pendingRows = 0; + try { foreach ($reader->rows($this->batchSize) as $batch) { - yield $batch; + if (($pendingRows + $batch->count()) > $this->batchSize) { + yield $pending[0]->concat($this->backend, ...array_slice($pending, 1)); + $pending = []; + $pendingRows = 0; + } + + $pending[] = $batch; + $pendingRows += $batch->count(); + } + + if ($pending !== []) { + yield $pending[0]->concat($this->backend, ...array_slice($pending, 1)); } } finally { $reader->close(); @@ -96,7 +116,7 @@ public function set(string $bucketId, Rows $rows): void $this->closeWriter($bucketId); // validation stays on until an upstream mechanism guarantees Rows match their schema - $writer = new FloeWriter($this->filesystem, $rows->schema(), new Options()); + $writer = new FloeWriter($this->filesystem, $rows->schema(), $this->backend, new Options()); $writer->create($this->keyPath($bucketId)); $writer->write($rows); $writer->close(); diff --git a/src/core/etl/src/Flow/ETL/Bucketing/Storage/MemoryBuckets.php b/src/core/etl/src/Flow/ETL/Bucketing/Storage/MemoryBuckets.php index e31a16e685..9ca5edfb62 100644 --- a/src/core/etl/src/Flow/ETL/Bucketing/Storage/MemoryBuckets.php +++ b/src/core/etl/src/Flow/ETL/Bucketing/Storage/MemoryBuckets.php @@ -17,7 +17,7 @@ final class MemoryBuckets implements ResidentBucketsStorage public function append(string $bucketId, Rows $rows): void { - if (!$rows->empty()) { + if (!$rows->isEmpty()) { $this->buckets[$bucketId][] = $rows; } } diff --git a/src/core/etl/src/Flow/ETL/Bucketing/Storage/PSRCacheBuckets.php b/src/core/etl/src/Flow/ETL/Bucketing/Storage/PSRCacheBuckets.php index b8e0ad39d0..cc59d6f887 100644 --- a/src/core/etl/src/Flow/ETL/Bucketing/Storage/PSRCacheBuckets.php +++ b/src/core/etl/src/Flow/ETL/Bucketing/Storage/PSRCacheBuckets.php @@ -7,7 +7,6 @@ use DateInterval; use Flow\ETL\Bucketing\BucketsStorage; use Flow\ETL\Rows; -use Flow\Floe\FloeSerializer; use Flow\Serializer\Serializer; use Generator; use Psr\SimpleCache\CacheInterface; @@ -21,9 +20,9 @@ { public function __construct( private CacheInterface $cache, + private Serializer $serializer, private string $prefix = 'flow:buckets', private ?DateInterval $ttl = null, - private Serializer $serializer = new FloeSerializer(), ) {} public function append(string $bucketId, Rows $rows): void diff --git a/src/core/etl/src/Flow/ETL/Bucketing/Storage/SpillingBuckets.php b/src/core/etl/src/Flow/ETL/Bucketing/Storage/SpillingBuckets.php new file mode 100644 index 0000000000..02ea02a79a --- /dev/null +++ b/src/core/etl/src/Flow/ETL/Bucketing/Storage/SpillingBuckets.php @@ -0,0 +1,161 @@ +> + */ + private array $buffers = []; + + /** + * @var array + */ + private array $buffered = []; + + private readonly MemoryBudget $budget; + + /** + * @var array + */ + private array $onDisk = []; + + private bool $spilling = false; + + /** + * @param int<1, max> $batchSize + */ + public function __construct( + public readonly BucketsStorage $disk, + public readonly Unit $memoryLimit, + private readonly Backend $backend, + private readonly int $batchSize = 1000, + ) { + // @mago-ignore analysis:invalid-operand + // @mago-ignore analysis:impossible-condition,redundant-comparison + if ($this->batchSize < 1) { + throw new InvalidArgumentException('Batch size must be greater than 0, given: ' . $this->batchSize); + } + + $this->budget = new MemoryBudget($this->backend, $this->memoryLimit); + } + + /** + * $bucketing's storage behind the memory limit - a resident storage already holds everything in memory. + */ + public static function around(BucketingConfig $bucketing, Unit $memoryLimit, Backend $backend): BucketsStorage + { + return $bucketing->storage instanceof ResidentBucketsStorage + ? $bucketing->storage + : new self($bucketing->storage, $memoryLimit, $backend, $bucketing->batchSize); + } + + public function append(string $bucketId, Rows $rows): void + { + if ($rows->isEmpty()) { + return; + } + + $this->buffers[$bucketId][] = $rows; + $this->buffered[$bucketId] = ($this->buffered[$bucketId] ?? 0) + $rows->count(); + + if (!$this->spilling && $this->budget->exceeded()) { + $this->spilling = true; + + foreach (array_keys($this->buffers) as $id) { + $this->writeFullBatches($id); + } + + return; + } + + if ($this->spilling) { + $this->writeFullBatches($bucketId); + } + } + + public function get(string $bucketId): Generator + { + if (array_key_exists($bucketId, $this->onDisk)) { + yield from $this->disk->get($bucketId); + } + + if (!array_key_exists($bucketId, $this->buffers)) { + return; + } + + $buffered = $this->buffers[$bucketId][0]->concat($this->backend, ...array_slice($this->buffers[$bucketId], 1)); + + for ($offset = 0, $count = $buffered->count(); $offset < $count; $offset += $this->batchSize) { + yield $buffered->slice($offset, min($this->batchSize, $count - $offset)); + } + } + + public function remove(string $bucketId): void + { + unset($this->buffers[$bucketId], $this->buffered[$bucketId]); + + if (array_key_exists($bucketId, $this->onDisk)) { + unset($this->onDisk[$bucketId]); + $this->disk->remove($bucketId); + } + } + + public function set(string $bucketId, Rows $rows): void + { + $this->remove($bucketId); + $this->append($bucketId, $rows); + } + + public function writeFullBatches(string $bucketId): void + { + $count = $this->buffered[$bucketId] ?? 0; + + if ($count < $this->batchSize) { + return; + } + + $buffered = $this->buffers[$bucketId][0]->concat($this->backend, ...array_slice($this->buffers[$bucketId], 1)); + $written = $count - ($count % $this->batchSize); + + for ($offset = 0; $offset < $written; $offset += $this->batchSize) { + $this->disk->append($bucketId, $buffered->slice($offset, $this->batchSize)); + } + + $this->onDisk[$bucketId] = true; + $this->budget->released(); + + if ($written === $count) { + unset($this->buffers[$bucketId], $this->buffered[$bucketId]); + + return; + } + + $this->buffers[$bucketId] = [$buffered->slice($written, $count - $written)]; + $this->buffered[$bucketId] = $count - $written; + } +} diff --git a/src/core/etl/src/Flow/ETL/Cache/CacheIndex.php b/src/core/etl/src/Flow/ETL/Cache/CacheIndex.php index 81c5aa46f2..7151eba4bc 100644 --- a/src/core/etl/src/Flow/ETL/Cache/CacheIndex.php +++ b/src/core/etl/src/Flow/ETL/Cache/CacheIndex.php @@ -5,14 +5,13 @@ namespace Flow\ETL\Cache; use Flow\ETL\Cardinality; +use Flow\ETL\Column\Backend; use Flow\ETL\Exception\InvalidArgumentException; -use Flow\ETL\Row; use Flow\ETL\Rows; use function array_map; +use function Flow\ETL\DSL\array_to_rows; use function Flow\ETL\DSL\int_schema; -use function Flow\ETL\DSL\row; -use function Flow\ETL\DSL\rows; use function Flow\ETL\DSL\schema; use function Flow\ETL\DSL\str_schema; use function get_debug_type; @@ -38,9 +37,16 @@ public static function fromRows(string $key, Rows $rows): self { $index = new self($key); - foreach ($rows->all() as $row) { - $value = $row->get('key'); + if ($rows->isEmpty()) { + return $index; + } + + $keys = $rows->column('key')->values(); + // an index written by 0.44.x has no "rows" column + $counts = $rows->schema()->findDefinition('rows') === null ? [] : $rows->column('rows')->values(); + // @mago-ignore analysis:mixed-assignment + foreach ($keys as $i => $value) { if (!is_string($value)) { throw new InvalidArgumentException(sprintf( 'CacheIndex expects rows with a string "key" entry, got: %s', @@ -48,8 +54,8 @@ public static function fromRows(string $key, Rows $rows): self )); } - // an index written by 0.44.x has no "rows" column - $chunkRows = $row->has('rows') ? $row->get('rows') : null; + // @mago-ignore analysis:mixed-assignment + $chunkRows = $counts[$i] ?? null; if ($chunkRows !== null && !is_int($chunkRows)) { throw new InvalidArgumentException(sprintf( @@ -84,12 +90,9 @@ public function rows(): Cardinality return Cardinality::exact($total); } - public function toRows(): Rows + public function toRows(Backend $backend): Rows { - return rows( - schema(str_schema('key'), int_schema('rows', nullable: true)), - ...array_map(static fn(array $chunk): Row => row($chunk), $this->chunks), - ); + return array_to_rows($this->chunks, schema(str_schema('key'), int_schema('rows', nullable: true)), $backend); } /** diff --git a/src/core/etl/src/Flow/ETL/Cache/Implementation/ApcuCache.php b/src/core/etl/src/Flow/ETL/Cache/Implementation/ApcuCache.php index 08b895ce71..1a8b69bd8e 100644 --- a/src/core/etl/src/Flow/ETL/Cache/Implementation/ApcuCache.php +++ b/src/core/etl/src/Flow/ETL/Cache/Implementation/ApcuCache.php @@ -10,6 +10,8 @@ use Flow\ETL\Exception\RuntimeException; use Flow\ETL\Rows; use Flow\ETL\Schema; +use Flow\Serializer\Exception\SerializationException; +use Flow\Serializer\Serializer; use function apcu_delete; use function apcu_enabled; @@ -17,12 +19,16 @@ use function apcu_fetch; use function apcu_store; use function extension_loaded; +use function Flow\Serializer\DSL\serialize_to_string; +use function Flow\Serializer\DSL\unserialize_from_string; +use function is_string; use function preg_quote; use function sprintf; final readonly class ApcuCache implements Cache { public function __construct( + private Serializer $serializer, private string $namespace = 'flow_php_cache', ) { if (!extension_loaded('apcu') || !apcu_enabled()) { @@ -56,14 +62,18 @@ public function get(string $key): Rows throw new KeyNotInCacheException($key); } - if (!$value instanceof Rows) { + if (!is_string($value)) { throw new RuntimeException(sprintf( 'Cache entry for key "%s" is corrupted or was not written by ApcuCache.', $key, )); } - return $value; + try { + return unserialize_from_string($this->serializer, $value); + } catch (SerializationException $e) { + throw new KeyNotInCacheException($key, $e); + } } public function has(string $key): bool @@ -95,7 +105,7 @@ public function schema(string $key): Schema public function set(string $key, Rows $value): void { if ( - apcu_store($this->namespacedKey($key), $value) === false + apcu_store($this->namespacedKey($key), serialize_to_string($this->serializer, $value)) === false || apcu_store($this->schemaKey($key), $value->schema()) === false ) { throw new RuntimeException(sprintf( diff --git a/src/core/etl/src/Flow/ETL/Cache/Implementation/FilesystemCache.php b/src/core/etl/src/Flow/ETL/Cache/Implementation/FilesystemCache.php index 62eb8cd556..19d8db3d4d 100644 --- a/src/core/etl/src/Flow/ETL/Cache/Implementation/FilesystemCache.php +++ b/src/core/etl/src/Flow/ETL/Cache/Implementation/FilesystemCache.php @@ -11,7 +11,6 @@ use Flow\ETL\Schema; use Flow\Filesystem\Filesystem; use Flow\Filesystem\Path; -use Flow\Floe\FloeSerializer; use Flow\Serializer\Exception\SerializationException; use Flow\Serializer\Serializer; use JsonException; @@ -27,8 +26,8 @@ public function __construct( private Filesystem $filesystem, + private Serializer $serializer, ?Path $cacheDir = null, - private Serializer $serializer = new FloeSerializer(), ) { $this->cacheDir = $cacheDir ?? $this->filesystem->getSystemTmpDir(); } diff --git a/src/core/etl/src/Flow/ETL/Cache/Implementation/PSRSimpleCache.php b/src/core/etl/src/Flow/ETL/Cache/Implementation/PSRSimpleCache.php index f47f042bd5..128f1925de 100644 --- a/src/core/etl/src/Flow/ETL/Cache/Implementation/PSRSimpleCache.php +++ b/src/core/etl/src/Flow/ETL/Cache/Implementation/PSRSimpleCache.php @@ -9,7 +9,6 @@ use Flow\ETL\Exception\KeyNotInCacheException; use Flow\ETL\Rows; use Flow\ETL\Schema; -use Flow\Floe\FloeSerializer; use Flow\Serializer\Exception\SerializationException; use Flow\Serializer\Serializer; use JsonException; @@ -26,8 +25,8 @@ { public function __construct( private CacheInterface $cache, + private Serializer $serializer, private int|DateInterval|null $ttl = null, - private Serializer $serializer = new FloeSerializer(), ) {} public function clear(): void diff --git a/src/core/etl/src/Flow/ETL/Column/AdaptiveBackend.php b/src/core/etl/src/Flow/ETL/Column/AdaptiveBackend.php new file mode 100644 index 0000000000..424a345c91 --- /dev/null +++ b/src/core/etl/src/Flow/ETL/Column/AdaptiveBackend.php @@ -0,0 +1,43 @@ +backend = ($extension ?? FlowPhpExtension::detect())->available() ? new RustBackend() : new PhpBackend(); + } + + public function builder(Definition $definition): ColumnBuilder + { + return $this->backend->builder($definition); + } + + public function constant(Definition $definition, mixed $value, int $count): Column + { + return $this->backend->constant($definition, $value, $count); + } + + public function decode(Definition $definition, array $buffers, int $count, int $nullCount): Column + { + return $this->backend->decode($definition, $buffers, $count, $nullCount); + } + + public function adopt(Definition $definition, Column $column): Column + { + return $this->backend->adopt($definition, $column); + } + + public function allocatedBytes(): int + { + return $this->backend->allocatedBytes(); + } +} diff --git a/src/core/etl/src/Flow/ETL/Column/Backend.php b/src/core/etl/src/Flow/ETL/Column/Backend.php new file mode 100644 index 0000000000..42b6799c74 --- /dev/null +++ b/src/core/etl/src/Flow/ETL/Column/Backend.php @@ -0,0 +1,53 @@ + $definition nullability, the enum class, the datetime zone and the cast plan + */ + public function builder(Definition $definition): ColumnBuilder; + + /** + * $value is logical or raw and is cast once through the Definition exactly like ColumnBuilder::append(). + * + * @param Definition $definition + */ + public function constant(Definition $definition, mixed $value, int $count): Column; + + /** + * Child node lengths and null counts are derived from the buffers, so only the top node travels. + * + * @param Definition $definition + * @param list $buffers the column's subtree buffers as Column::encode() produced them ('' = omitted validity) + */ + public function decode(Definition $definition, array $buffers, int $count, int $nullCount): Column; + + /** + * The same values in this backend's storage: $column itself when this backend owns it, otherwise one copy through + * builder($definition)->appendTake(). For producers that build columns outside the configured backend. + * + * @param Definition $definition + * + * @throws ColumnMismatchException a null under NOT NULL on the copy path (an own column passes through unchecked; + * Rows::fromColumns checks NOT NULL) + */ + public function adopt(Definition $definition, Column $column): Column; + + /** + * Bytes this backend holds outside PHP's memory manager (invisible to memory_get_usage() and memory_limit). + */ + public function allocatedBytes(): int; +} diff --git a/src/core/etl/src/Flow/ETL/Column/Column.php b/src/core/etl/src/Flow/ETL/Column/Column.php new file mode 100644 index 0000000000..8547fa1598 --- /dev/null +++ b/src/core/etl/src/Flow/ETL/Column/Column.php @@ -0,0 +1,80 @@ + + */ + public function type(): Type; + + public function count(): int; + + public function isNull(int $i): bool; + + public function nullCount(): int; + + /** + * PHYSICAL cell: int|float|bool|string|array|null - never a DateTimeImmutable, Uuid, Json … + */ + public function at(int $i): mixed; + + /** + * @return list physical, whole column - PHP returns its backing list (refcount), native copies once + */ + public function physicals(): array; + + /** + * LOGICAL cell - PHP: Physical::fromPhysical(at($i)); native: built in Rust. + */ + public function value(int $i): mixed; + + /** + * @return list logical, whole column (encoders, statistics, reduceToArray) + */ + public function values(): array; + + /** + * `self`, never `static`: the extension declares these too and ext-php-rs has no `static` return binding. + */ + public function slice(int $offset, int $length): self; + + /** + * @param list $indices every index in [0, count()) + */ + public function take(array $indices): self; + + /** + * The one type-changing operation: same buffers, a new Type of the same column kind, checked recursively. + * Dropping an OptionalType or an element's optionality at any depth requires that child column's + * nullCount() === 0. + * + * @param Type $type + */ + public function withType(Type $type): self; + + /** + * Same-class lane; a mixed-class concat never reaches here - Rows::concat routes it through PhpBackend. + */ + public function concat(self ...$others): self; + + /** + * @return list the column's frame buffers in the kind's buffer order, pre-order for nested kinds, in + * canonical form; an omitted validity buffer is '' + */ + public function encode(): array; +} diff --git a/src/core/etl/src/Flow/ETL/Column/ColumnBuilder.php b/src/core/etl/src/Flow/ETL/Column/ColumnBuilder.php new file mode 100644 index 0000000000..d86a5fe647 --- /dev/null +++ b/src/core/etl/src/Flow/ETL/Column/ColumnBuilder.php @@ -0,0 +1,57 @@ + $values one cast loop per column - the bulk door + * + * @throws SchemaMismatchException at the first refused value, its row the value's position in $values + */ + public function appendMany(array $values): void; + + /** + * Physical values exactly as Column::physicals() returns them - no cast, no conversion. + * + * @param list $physicals + * @param null|int $nullCount exactly the nulls among $physicals when the caller counted them, null otherwise - a + * count that is not exact is undefined behaviour + * + * @throws SchemaMismatchException at the first null under a NOT NULL definition + * @throws InvalidArgumentException for a physical of another kind than the definition's (a string in an + * integer column, an int in a float column); nothing is appended + */ + public function appendPhysicals(array $physicals, ?int $nullCount = null): void; + + /** + * Copies one physical cell, no materialisation. + */ + public function appendFrom(Column $column, int $i): void; + + /** + * @param list $indices gathered physically from $column + */ + public function appendTake(Column $column, array $indices): void; + + public function count(): int; + + public function finish(): Column; +} diff --git a/src/core/etl/src/Flow/ETL/Column/ComparableValues.php b/src/core/etl/src/Flow/ETL/Column/ComparableValues.php new file mode 100644 index 0000000000..2311f90966 --- /dev/null +++ b/src/core/etl/src/Flow/ETL/Column/ComparableValues.php @@ -0,0 +1,144 @@ + $type + */ + public function equalByPhysical(Type $type): bool + { + $bare = type_bare($type); + + return ( + $this->orderedByPhysical($bare) + || $bare instanceof UuidType + || $bare instanceof JsonType + || $bare instanceof EnumType + || $bare instanceof XMLType + || $bare instanceof XMLElementType + || $bare instanceof HTMLType + || $bare instanceof HTMLElementType + ); + } + + /** + * @return list === on these is value equality + */ + public function equality(Column $column): array + { + return $this->equalByPhysical($column->type()) + ? $this->equalities($column->type(), $column->physicals()) + : $column->values(); + } + + /** + * Element physicals carry their owner document, so elements, and lists of them, compare by their own markup. + * + * @param Type $type + * @param list $physicals + * + * @return list === on these is value equality + */ + public function equalities(Type $type, array $physicals): array + { + $bare = type_bare($type); + $element = $bare instanceof ListType ? type_bare($bare->element()) : $bare; + + if ($bare instanceof DateType) { + return $this->instants($physicals); + } + + if ($element instanceof DateType) { + $lists = []; + + // @mago-ignore analysis:mixed-assignment + foreach ($physicals as $list) { + $lists[] = $list === null ? null : $this->instants(type_list(type_mixed())->assert($list)); + } + + return $lists; + } + + return $element instanceof HTMLElementType || $element instanceof XMLElementType + ? (new TextValues())->of($type, $physicals) + : $physicals; + } + + /** + * A date's physical days on a datetime's microsecond scale, so a date and a datetime at the same instant compare + * and hash equal. + * + * @param list $days + * + * @return list + */ + public function instants(array $days): array + { + $instants = []; + + // @mago-ignore analysis:mixed-assignment + foreach ($days as $day) { + $instants[] = $day === null ? null : type_integer()->assert($day) * 86_400_000_000; + } + + return $instants; + } + + /** + * @param Type $type + */ + public function orderedByPhysical(Type $type): bool + { + $bare = type_bare($type); + + return ( + $bare instanceof IntegerType + || $bare instanceof FloatType + || $bare instanceof BooleanType + || $bare instanceof StringType + || $bare instanceof DateTimeType + || $bare instanceof DateType + || $bare instanceof TimeType + ); + } + + /** + * @return list PHP <, <=, <=> on these equal today's comparison of the values + */ + public function ordering(Column $column): array + { + if (!$this->orderedByPhysical($column->type())) { + return $column->values(); + } + + return type_bare($column->type()) instanceof DateType + ? $this->instants($column->physicals()) + : $column->physicals(); + } +} diff --git a/src/core/etl/src/Flow/ETL/Column/DateFormatPattern.php b/src/core/etl/src/Flow/ETL/Column/DateFormatPattern.php new file mode 100644 index 0000000000..f108335ced --- /dev/null +++ b/src/core/etl/src/Flow/ETL/Column/DateFormatPattern.php @@ -0,0 +1,72 @@ + the unescaped ASCII letters, in order + */ + public function letters(): array + { + $letters = []; + + for ($i = 0, $length = strlen($this->format); $i < $length; $i++) { + if ($this->format[$i] === '\\') { + $i++; + + continue; + } + + if (ctype_alpha($this->format[$i])) { + $letters[] = $this->format[$i]; + } + } + + return $letters; + } + + /** + * @return list the format cut at every unescaped u / v: [segment, the letter that ends it] + */ + public function segments(): array + { + $segments = []; + $segment = ''; + + for ($i = 0, $length = strlen($this->format); $i < $length; $i++) { + $character = $this->format[$i]; + + if ($character === '\\') { + $segment .= $character . ($this->format[$i + 1] ?? ''); + $i++; + + continue; + } + + if ($character === 'u' || $character === 'v') { + $segments[] = [$segment, $character]; + $segment = ''; + + continue; + } + + $segment .= $character; + } + + if ($segment !== '') { + $segments[] = [$segment, '']; + } + + return $segments; + } +} diff --git a/src/core/etl/src/Flow/ETL/Column/Elements.php b/src/core/etl/src/Flow/ETL/Column/Elements.php new file mode 100644 index 0000000000..7e82b47e4a --- /dev/null +++ b/src/core/etl/src/Flow/ETL/Column/Elements.php @@ -0,0 +1,98 @@ +> $containers + * + * @return list every element of every non-null container, in order + */ + public function flatten(array $containers): array + { + $flat = []; + + foreach ($containers as $container) { + if ($container === null) { + continue; + } + + // @mago-ignore analysis:mixed-assignment + foreach ($container as $element) { + $flat[] = $element; + } + } + + return $flat; + } + + /** + * @param list $flat + * @param list> $containers + * + * @return list> $flat cut back into the containers' shapes, keys kept, null stays null + */ + public function shape(array $flat, array $containers): array + { + $shaped = []; + $position = 0; + + foreach ($containers as $container) { + if ($container === null) { + $shaped[] = null; + + continue; + } + + foreach ($container as $key => $_) { + $container[$key] = $flat[$position++]; + } + + $shaped[] = $container; + } + + return $shaped; + } + + /** + * @param list> $structures + * + * @return list the element of every structure that has it + */ + public function field(array $structures, int|string $name): array + { + $values = []; + + foreach ($structures as $structure) { + if ($structure !== null && array_key_exists($name, $structure)) { + $values[] = $structure[$name]; + } + } + + return $values; + } + + /** + * @param list> $structures + * @param list $values field() rendered + * + * @return list> + */ + public function withField(array $structures, int|string $name, array $values): array + { + $position = 0; + + foreach ($structures as $i => $structure) { + if ($structure !== null && array_key_exists($name, $structure)) { + $structures[$i][$name] = $values[$position++]; + } + } + + return $structures; + } +} diff --git a/src/core/etl/src/Flow/ETL/Column/Layout/BooleanLayout.php b/src/core/etl/src/Flow/ETL/Column/Layout/BooleanLayout.php new file mode 100644 index 0000000000..f4e90cb2d3 --- /dev/null +++ b/src/core/etl/src/Flow/ETL/Column/Layout/BooleanLayout.php @@ -0,0 +1,60 @@ +next(); + + if (strlen($buffer) < intdiv($count + 7, 8)) { + throw new InvalidArgumentException(sprintf( + 'Boolean values buffer of %d bytes, expected %d for %d rows', + strlen($buffer), + intdiv($count + 7, 8), + $count, + )); + } + + $physicals = []; + + for ($i = 0; $i < $count; $i++) { + $physicals[] = $valid[$i] ? ((ord($buffer[$i >> 3]) >> ($i & 7)) & 1) === 1 : null; + } + + return $physicals; + } +} diff --git a/src/core/etl/src/Flow/ETL/Column/Layout/BufferLayout.php b/src/core/etl/src/Flow/ETL/Column/Layout/BufferLayout.php new file mode 100644 index 0000000000..9c801279fb --- /dev/null +++ b/src/core/etl/src/Flow/ETL/Column/Layout/BufferLayout.php @@ -0,0 +1,168 @@ + $type + */ + public function nodeCount(Type $type): int + { + $base = $type instanceof OptionalType ? $type->base() : $type; + + if ($base instanceof ListType) { + return 1 + $this->nodeCount($base->element()); + } + + if ($base instanceof MapType) { + return 2 + $this->nodeCount($base->key()) + $this->nodeCount($base->value()); + } + + if ($base instanceof StructureType) { + $nodes = 1; + + foreach ($base->elements() as $element) { + $nodes += $this->nodeCount($element->type); + } + + return $nodes; + } + + return 1; + } + + /** + * @param Type $type + */ + public function bufferCount(Type $type): int + { + $base = $type instanceof OptionalType ? $type->base() : $type; + + if ($base instanceof NullType) { + return 0; + } + + if ($base instanceof ListType) { + return 2 + $this->bufferCount($base->element()); + } + + if ($base instanceof MapType) { + return 3 + $this->bufferCount($base->key()) + $this->bufferCount($base->value()); + } + + if ($base instanceof StructureType) { + $buffers = 1; + + foreach ($base->elements() as $element) { + $buffers += $this->bufferCount($element->type); + } + + return $buffers; + } + + return (new LayoutFor())->type($type) instanceof Utf8Layout ? 3 : 2; + } + + /** + * Pre-order {length, nullCount} of one column's tree: the first node as given, every child derived from the + * buffers - a null kind is {length, length}, list and map children span the last offset, a map entries node + * holds no nulls and structure children span their parent. + * + * @param Type $type + * @param list $buffers + * + * @throws InvalidArgumentException + * + * @return list + */ + public function nodes(Type $type, int $length, int $nullCount, array $buffers): array + { + if (count($buffers) !== $this->bufferCount($type)) { + throw new InvalidArgumentException(sprintf( + 'Column of type %s holds %d buffers, its layout needs %d', + $type->toString(), + count($buffers), + $this->bufferCount($type), + )); + } + + $base = $type instanceof OptionalType ? $type->base() : $type; + $nodes = [[$length, $nullCount]]; + $children = []; + $position = 1; + + if ($base instanceof ListType || $base instanceof MapType) { + if (strlen($buffers[1]) !== (($length + 1) * 4)) { + throw new InvalidArgumentException(sprintf( + '%s offsets buffer of %d bytes, expected %d for %d rows', + $base instanceof ListType ? 'List' : 'Map', + strlen($buffers[1]), + ($length + 1) * 4, + $length, + )); + } + + /** @var int $childLength */ + $childLength = unpack('V', $buffers[1], $length * 4)[1]; + + if ($base instanceof ListType) { + $children[] = [$base->element(), $childLength]; + $position = 2; + } else { + if ($buffers[2] !== '') { + throw new InvalidArgumentException(sprintf( + 'Map entries validity must be omitted, got %d bytes', + strlen($buffers[2]), + )); + } + + $nodes[] = [$childLength, 0]; + $children[] = [$base->key(), $childLength]; + $children[] = [$base->value(), $childLength]; + $position = 3; + } + } elseif ($base instanceof StructureType) { + foreach ($base->elements() as $element) { + $children[] = [$element->type, $length]; + } + } + + foreach ($children as [$childType, $childLength]) { + $childBuffers = array_slice($buffers, $position, $this->bufferCount($childType)); + $position += count($childBuffers); + $childBase = $childType instanceof OptionalType ? $childType->base() : $childType; + + array_push($nodes, ...$this->nodes( + $childType, + $childLength, + $childBase instanceof NullType + ? $childLength + : (new Validity())->nullCount($childBuffers[0], $childLength), + $childBuffers, + )); + } + + return $nodes; + } +} diff --git a/src/core/etl/src/Flow/ETL/Column/Layout/Buffers.php b/src/core/etl/src/Flow/ETL/Column/Layout/Buffers.php new file mode 100644 index 0000000000..31dd9eddd8 --- /dev/null +++ b/src/core/etl/src/Flow/ETL/Column/Layout/Buffers.php @@ -0,0 +1,35 @@ + $buffers + */ + public function __construct( + private readonly array $buffers, + ) {} + + public function next(): string + { + if ($this->position >= count($this->buffers)) { + throw new InvalidArgumentException('Column buffers exhausted: the column needs more buffers than given'); + } + + return $this->buffers[$this->position++]; + } + + public function remaining(): int + { + return count($this->buffers) - $this->position; + } +} diff --git a/src/core/etl/src/Flow/ETL/Column/Layout/FixedBinary16Layout.php b/src/core/etl/src/Flow/ETL/Column/Layout/FixedBinary16Layout.php new file mode 100644 index 0000000000..fd9e105a5d --- /dev/null +++ b/src/core/etl/src/Flow/ETL/Column/Layout/FixedBinary16Layout.php @@ -0,0 +1,57 @@ +next(); + + if (strlen($buffer) !== (16 * $count)) { + throw new InvalidArgumentException(sprintf( + 'FixedBinary16 values buffer of %d bytes, expected %d for %d rows', + strlen($buffer), + 16 * $count, + $count, + )); + } + + if ($count === 0) { + return []; + } + + $physicals = []; + + foreach (str_split($buffer, 16) as $i => $value) { + $physicals[] = $valid[$i] ? $value : null; + } + + return $physicals; + } +} diff --git a/src/core/etl/src/Flow/ETL/Column/Layout/Float64Layout.php b/src/core/etl/src/Flow/ETL/Column/Layout/Float64Layout.php new file mode 100644 index 0000000000..e0445ff90a --- /dev/null +++ b/src/core/etl/src/Flow/ETL/Column/Layout/Float64Layout.php @@ -0,0 +1,57 @@ +next(); + + if (strlen($buffer) !== (8 * $count)) { + throw new InvalidArgumentException(sprintf( + 'Float64 values buffer of %d bytes, expected %d for %d rows', + strlen($buffer), + 8 * $count, + $count, + )); + } + + if ($count === 0) { + return []; + } + + /** @var list $physicals */ + $physicals = array_values((array) unpack('e*', $buffer)); + + foreach ($valid as $i => $isValid) { + if (!$isValid) { + $physicals[$i] = null; + } + } + + return $physicals; + } +} diff --git a/src/core/etl/src/Flow/ETL/Column/Layout/Int32Layout.php b/src/core/etl/src/Flow/ETL/Column/Layout/Int32Layout.php new file mode 100644 index 0000000000..1d9e99072a --- /dev/null +++ b/src/core/etl/src/Flow/ETL/Column/Layout/Int32Layout.php @@ -0,0 +1,61 @@ +next(); + + if (strlen($buffer) !== (4 * $count)) { + throw new InvalidArgumentException(sprintf( + 'Int32 values buffer of %d bytes, expected %d for %d rows', + strlen($buffer), + 4 * $count, + $count, + )); + } + + if ($count === 0) { + return []; + } + + $physicals = []; + + foreach (array_values((array) unpack('V*', $buffer)) as $i => $value) { + if (!$valid[$i]) { + $physicals[] = null; + + continue; + } + + $unsigned = (int) $value; + $physicals[] = $unsigned >= 2_147_483_648 ? $unsigned - 4_294_967_296 : $unsigned; + } + + return $physicals; + } +} diff --git a/src/core/etl/src/Flow/ETL/Column/Layout/Int64Layout.php b/src/core/etl/src/Flow/ETL/Column/Layout/Int64Layout.php new file mode 100644 index 0000000000..22d6950fc9 --- /dev/null +++ b/src/core/etl/src/Flow/ETL/Column/Layout/Int64Layout.php @@ -0,0 +1,57 @@ +next(); + + if (strlen($buffer) !== (8 * $count)) { + throw new InvalidArgumentException(sprintf( + 'Int64 values buffer of %d bytes, expected %d for %d rows', + strlen($buffer), + 8 * $count, + $count, + )); + } + + if ($count === 0) { + return []; + } + + /** @var list $physicals */ + $physicals = array_values((array) unpack('P*', $buffer)); + + foreach ($valid as $i => $isValid) { + if (!$isValid) { + $physicals[$i] = null; + } + } + + return $physicals; + } +} diff --git a/src/core/etl/src/Flow/ETL/Column/Layout/Layout.php b/src/core/etl/src/Flow/ETL/Column/Layout/Layout.php new file mode 100644 index 0000000000..2e31d5a45b --- /dev/null +++ b/src/core/etl/src/Flow/ETL/Column/Layout/Layout.php @@ -0,0 +1,22 @@ + $physicals nulls allowed + * + * @return list value buffers, null slots in canonical form + */ + public function encode(array $physicals): array; + + /** + * @param list $valid + * + * @return list physicals, null where !$valid[$i] + */ + public function decode(Buffers $buffers, int $count, array $valid): array; +} diff --git a/src/core/etl/src/Flow/ETL/Column/Layout/LayoutFor.php b/src/core/etl/src/Flow/ETL/Column/Layout/LayoutFor.php new file mode 100644 index 0000000000..7fee2e178c --- /dev/null +++ b/src/core/etl/src/Flow/ETL/Column/Layout/LayoutFor.php @@ -0,0 +1,66 @@ + $type + */ + public function type(Type $type): Layout + { + $base = $type instanceof OptionalType ? $type->base() : $type; + + return match (true) { + $base instanceof IntegerType, + $base instanceof PositiveIntegerType, + $base instanceof DateTimeType, + $base instanceof TimeType, + => new Int64Layout(), + $base instanceof FloatType => new Float64Layout(), + $base instanceof DateType => new Int32Layout(), + $base instanceof BooleanType => new BooleanLayout(), + $base instanceof StringType, + $base instanceof NonEmptyStringType, + $base instanceof NumericStringType, + $base instanceof ClassStringType, + $base instanceof TimeZoneType, + $base instanceof JsonType, + $base instanceof EnumType, + $base instanceof XMLType, + $base instanceof XMLElementType, + $base instanceof HTMLType, + $base instanceof HTMLElementType, + => new Utf8Layout(), + $base instanceof UuidType => new FixedBinary16Layout(), + default => throw new InvalidArgumentException(sprintf('type %s has no scalar layout', $base->toString())), + }; + } +} diff --git a/src/core/etl/src/Flow/ETL/Column/Layout/NullCount.php b/src/core/etl/src/Flow/ETL/Column/Layout/NullCount.php new file mode 100644 index 0000000000..4225bf6c7d --- /dev/null +++ b/src/core/etl/src/Flow/ETL/Column/Layout/NullCount.php @@ -0,0 +1,25 @@ + $values + */ + public function of(array $values): int + { + $nulls = 0; + + // @mago-ignore analysis:mixed-assignment + foreach ($values as $value) { + if ($value === null) { + $nulls++; + } + } + + return $nulls; + } +} diff --git a/src/core/etl/src/Flow/ETL/Column/Layout/Offsets.php b/src/core/etl/src/Flow/ETL/Column/Layout/Offsets.php new file mode 100644 index 0000000000..771192ae2f --- /dev/null +++ b/src/core/etl/src/Flow/ETL/Column/Layout/Offsets.php @@ -0,0 +1,73 @@ + $offsets + */ + public function pack(array $offsets): string + { + $last = $offsets[count($offsets) - 1]; + + if ($last > 2_147_483_647) { + throw new OffsetOverflow(sprintf( + 'Offsets exceed the i32 range of a batch buffer: the last offset is %d', + $last, + )); + } + + return pack('V*', ...$offsets); + } + + /** + * @return list $count + 1 offsets, the first 0, monotonic + */ + public function unpack(string $buffer, int $count, string $layout): array + { + if (strlen($buffer) !== (($count + 1) * 4)) { + throw new InvalidArgumentException(sprintf( + '%s offsets buffer of %d bytes, expected %d for %d rows', + $layout, + strlen($buffer), + ($count + 1) * 4, + $count, + )); + } + + /** @var list $offsets */ + $offsets = array_values((array) unpack('V*', $buffer)); + + if ($offsets[0] !== 0) { + throw new InvalidArgumentException(sprintf('%s offsets start at %d, not 0', $layout, $offsets[0])); + } + + for ($i = 1; $i <= $count; $i++) { + if ($offsets[$i] < $offsets[$i - 1]) { + throw new InvalidArgumentException(sprintf( + '%s offsets are not monotonic: offset %d is %d, offset %d is %d', + $layout, + $i - 1, + $offsets[$i - 1], + $i, + $offsets[$i], + )); + } + } + + return $offsets; + } +} diff --git a/src/core/etl/src/Flow/ETL/Column/Layout/Utf8Layout.php b/src/core/etl/src/Flow/ETL/Column/Layout/Utf8Layout.php new file mode 100644 index 0000000000..49e6dc2e13 --- /dev/null +++ b/src/core/etl/src/Flow/ETL/Column/Layout/Utf8Layout.php @@ -0,0 +1,63 @@ +offsets->pack($offsets), implode('', $values)]; + } + + public function decode(Buffers $buffers, int $count, array $valid): array + { + $offsets = $this->offsets->unpack($buffers->next(), $count, 'Utf8'); + $data = $buffers->next(); + + if ($offsets[$count] !== strlen($data)) { + throw new InvalidArgumentException(sprintf( + 'Utf8 data buffer of %d bytes, the last offset is %d', + strlen($data), + $offsets[$count], + )); + } + + $physicals = []; + + for ($i = 0; $i < $count; $i++) { + $physicals[] = $valid[$i] ? substr($data, $offsets[$i], $offsets[$i + 1] - $offsets[$i]) : null; + } + + return $physicals; + } +} diff --git a/src/core/etl/src/Flow/ETL/Column/Layout/Validity.php b/src/core/etl/src/Flow/ETL/Column/Layout/Validity.php new file mode 100644 index 0000000000..d809503962 --- /dev/null +++ b/src/core/etl/src/Flow/ETL/Column/Layout/Validity.php @@ -0,0 +1,137 @@ + $values + */ + public function fromValues(array $values): string + { + $bitmap = ''; + $byte = 0; + $bit = 0; + $hasNull = false; + + // @mago-ignore analysis:mixed-assignment + foreach ($values as $value) { + if ($value === null) { + $hasNull = true; + } else { + $byte |= 1 << $bit; + } + + if (++$bit === 8) { + $bitmap .= chr($byte); + $byte = 0; + $bit = 0; + } + } + + if (!$hasNull) { + return ''; + } + + return $bit === 0 ? $bitmap : $bitmap . chr($byte); + } + + /** + * @param array $nulls + */ + public function fromNulls(array $nulls, int $count): string + { + if ($nulls === []) { + return ''; + } + + // @mago-ignore analysis:possibly-invalid-argument + $bitmap = str_repeat("\xFF", intdiv($count, 8)); + $tail = $count % 8; + + if ($tail !== 0) { + $bitmap .= chr((1 << $tail) - 1); + } + + foreach ($nulls as $index => $_) { + $byte = intdiv($index, 8); + $bitmap[$byte] = chr(ord($bitmap[$byte]) & ~(1 << ($index % 8)) & 0xFF); + } + + return $bitmap; + } + + /** + * @return list + */ + public function valid(string $bitmap, int $count): array + { + if ($bitmap === '') { + // @mago-ignore analysis:possibly-invalid-argument + return $count === 0 ? [] : array_fill(0, $count, true); + } + + if (strlen($bitmap) < intdiv($count + 7, 8)) { + throw new InvalidArgumentException(sprintf( + 'Validity bitmap of %d bytes is too short for %d rows', + strlen($bitmap), + $count, + )); + } + + $valid = []; + + for ($i = 0; $i < $count; $i++) { + $valid[] = ((ord($bitmap[$i >> 3]) >> ($i & 7)) & 1) === 1; + } + + return $valid; + } + + public function nullCount(string $bitmap, int $count): int + { + if ($bitmap === '') { + return 0; + } + + $nulls = 0; + + foreach ($this->valid($bitmap, $count) as $valid) { + if (!$valid) { + $nulls++; + } + } + + return $nulls; + } + + /** + * @param list $valid + * + * @return array + */ + public function nulls(array $valid): array + { + $nulls = []; + + foreach ($valid as $index => $isValid) { + if (!$isValid) { + $nulls[$index] = true; + } + } + + return $nulls; + } +} diff --git a/src/core/etl/src/Flow/ETL/Column/Php/CastingColumnBuilder.php b/src/core/etl/src/Flow/ETL/Column/Php/CastingColumnBuilder.php new file mode 100644 index 0000000000..4e73b0e468 --- /dev/null +++ b/src/core/etl/src/Flow/ETL/Column/Php/CastingColumnBuilder.php @@ -0,0 +1,201 @@ + + */ + private readonly Type $type; + + /** + * @param Definition $definition + */ + public function __construct( + private readonly Definition $definition, + private readonly Physical $physical, + private readonly PhpColumnBuilder $physicals, + ) { + $this->identity = $physical instanceof IdentityPhysical; + $this->nullable = $definition->isNullable() || $definition instanceof NullDefinition; + $this->type = $definition->type(); + $this->native = $this->identity + ? match ($this->type::class) { + BooleanType::class => 'bool', + IntegerType::class => 'int', + FloatType::class => 'float', + StringType::class => 'string', + default => null, + } + : null; + } + + public function append(mixed $value): void + { + if ($value === null) { + if (!$this->nullable) { + throw ColumnMismatchException::valueDoesNotMatch($this->definition, null); + } + + $this->physicals->appendPhysical(null); + + return; + } + + $this->physicals->appendPhysical($this->physical->toPhysical($this->type->cast($value))); + } + + public function appendFrom(Column $column, int $i): void + { + if (!$this->nullable && $column->isNull($i)) { + throw ColumnMismatchException::valueDoesNotMatch($this->definition, null); + } + + $this->physicals->appendPhysical($column->at($i)); + } + + public function appendMany(array $values): void + { + $physicals = []; + $nulls = 0; + $position = 0; + + try { + // @mago-ignore analysis:mixed-assignment + foreach ($values as $position => $value) { + if ($value === null) { + if (!$this->nullable) { + throw ColumnMismatchException::valueDoesNotMatch($this->definition, null); + } + + $physicals[] = null; + $nulls++; + + continue; + } + + if (get_debug_type($value) === $this->native) { + $physicals[] = $value; + + continue; + } + + // @mago-ignore analysis:mixed-assignment + $cast = $this->type->cast($value); + + if ($this->identity) { + $physicals[] = $cast; + + continue; + } + + try { + $physicals[] = $this->physical->toPhysical($cast); + } catch (TypesException $e) { + // the value cast, so only the physical form's reason (a range) tells the user why it was refused + throw ColumnMismatchException::valueDoesNotMatch($this->definition, $value, $e); + } + } + } catch (ColumnMismatchException $e) { + throw new SchemaMismatchException($position, $e); + } catch (TypesException) { + throw new SchemaMismatchException($position, ColumnMismatchException::valueDoesNotMatch( + $this->definition, + $values[$position], + )); + } + + $this->physicals->appendPhysicals($physicals, $nulls); + } + + /** + * @param list $physicals + * + * @throws SchemaMismatchException + */ + public function appendPhysicals(array $physicals, ?int $nullCount = null): void + { + $kind = new PhysicalKind(); + $type = $this->definition->type(); + $checkNulls = !$this->nullable && $nullCount !== 0; + + // @mago-ignore analysis:mixed-assignment + foreach ($physicals as $position => $physical) { + if ($physical === null) { + if ($checkNulls) { + throw new SchemaMismatchException($position, ColumnMismatchException::valueDoesNotMatch( + $this->definition, + null, + )); + } + + continue; + } + + if (!$kind->accepts($type, $physical)) { + throw new InvalidArgumentException(sprintf( + 'Cannot store a %s physical in a %s column', + get_debug_type($physical), + $type->toString(), + )); + } + } + + $this->physicals->appendPhysicals($physicals, $nullCount); + } + + public function appendTake(Column $column, array $indices): void + { + $all = $column->physicals(); + $physicals = []; + + foreach ($indices as $index) { + if (!$this->nullable && $all[$index] === null) { + throw ColumnMismatchException::valueDoesNotMatch($this->definition, null); + } + + $physicals[] = $all[$index]; + } + + $this->physicals->appendPhysicals($physicals); + } + + public function count(): int + { + return $this->physicals->count(); + } + + public function finish(): Column + { + return $this->physicals->finish(); + } +} diff --git a/src/core/etl/src/Flow/ETL/Column/Php/ColumnDecoder.php b/src/core/etl/src/Flow/ETL/Column/Php/ColumnDecoder.php new file mode 100644 index 0000000000..bffbb0e153 --- /dev/null +++ b/src/core/etl/src/Flow/ETL/Column/Php/ColumnDecoder.php @@ -0,0 +1,159 @@ + $type + * + * @throws InvalidArgumentException + */ + public function decode(Type $type, Buffers $buffers, int $count, int $nullCount): Column + { + $base = $type instanceof OptionalType ? $type->base() : $type; + $nodes = []; + $own = $buffers; + + // only a container has child nodes to derive; a flat column reads its buffers straight off the cursor + if ($base instanceof ListType || $base instanceof MapType || $base instanceof StructureType) { + $subtree = []; + + for ($i = $this->layout->bufferCount($type); $i > 0; $i--) { + $subtree[] = $buffers->next(); + } + + $nodes = $this->layout->nodes($type, $count, $nullCount, $subtree); + $own = new Buffers($subtree); + } + + if ($base instanceof NullType) { + if ($nullCount !== $count) { + throw new InvalidArgumentException(sprintf( + 'Null column of %d rows carries a null count of %d', + $count, + $nullCount, + )); + } + + return new ConstantColumn($type, new NullPhysical(), null, $count); + } + + $validity = new Validity(); + $bitmap = $own->next(); + $valid = $validity->valid($bitmap, $count); + $derived = $validity->nullCount($bitmap, $count); + + if ($nullCount !== $derived) { + throw new InvalidArgumentException(sprintf( + 'Column null count %d disagrees with its validity bitmap (%d nulls in %d rows)', + $nullCount, + $derived, + $count, + )); + } + + if ($base instanceof ListType) { + $offsets = (new Offsets())->unpack($own->next(), $count, 'List'); + $element = $base->element(); + [$length, $nulls] = $nodes[1]; + + if ($nulls > 0 && !$element instanceof OptionalType && !$element instanceof NullType) { + throw new InvalidArgumentException(sprintf( + 'List child holds %d nulls in a non-nullable %s', + $nulls, + $element->toString(), + )); + } + + return new ListColumn( + $type, + $offsets, + $this->decode($element, $own, $length, $nulls), + $validity->nulls($valid), + ); + } + + if ($base instanceof MapType) { + $offsets = (new Offsets())->unpack($own->next(), $count, 'Map'); + $own->next(); + $key = $base->key(); + $value = $base->value(); + [$length, $keyNulls] = $nodes[2]; + [, $valueNulls] = $nodes[2 + $this->layout->nodeCount($key)]; + + if ($keyNulls > 0) { + throw new InvalidArgumentException(sprintf( + 'Map key child holds %d nulls in a non-nullable %s', + $keyNulls, + $key->toString(), + )); + } + + if ($valueNulls > 0 && !$value instanceof OptionalType && !$value instanceof NullType) { + throw new InvalidArgumentException(sprintf( + 'Map value child holds %d nulls in a non-nullable %s', + $valueNulls, + $value->toString(), + )); + } + + return new MapColumn( + $type, + $offsets, + $this->decode($key, $own, $length, $keyNulls), + $this->decode($value, $own, $length, $valueNulls), + $validity->nulls($valid), + ); + } + + if ($base instanceof StructureType) { + $children = []; + $node = 1; + + foreach ($base->elements() as $element) { + [$length, $nulls] = $nodes[$node]; + $node += $this->layout->nodeCount($element->type); + $children[$element->name] = $this->decode($element->type, $own, $length, $nulls); + } + + return new StructColumn($type, $children, $validity->nulls($valid)); + } + + return new ScalarColumn( + $type, + (new PhysicalFor())->type($type), + (new LayoutFor()) + ->type($type) + ->decode($own, $count, $valid), + $derived, + ); + } +} diff --git a/src/core/etl/src/Flow/ETL/Column/Php/ConstantColumn.php b/src/core/etl/src/Flow/ETL/Column/Php/ConstantColumn.php new file mode 100644 index 0000000000..7db6a65123 --- /dev/null +++ b/src/core/etl/src/Flow/ETL/Column/Php/ConstantColumn.php @@ -0,0 +1,129 @@ + $type + */ + public function __construct( + public Type $type, + public Physical $physical, + public mixed $physicalValue, + public int $count, + ) {} + + public function at(int $i): mixed + { + return $this->physicalValue; + } + + public function concat(Column ...$others): Column + { + $count = $this->count; + + foreach ($others as $other) { + if (!$other instanceof self || $other->physicalValue !== $this->physicalValue) { + return $this->expand()->concat(...array_map(static fn(Column $column): Column => $column instanceof self + ? $column->expand() + : $column, $others)); + } + + $count += $other->count; + } + + return new self($this->type, $this->physical, $this->physicalValue, $count); + } + + public function count(): int + { + return $this->count; + } + + public function encode(): array + { + return $this->isNullKind() ? [] : $this->expand()->encode(); + } + + public function expand(): Column + { + if ($this->isNullKind()) { + return $this; + } + + $builder = (new PhysicalBuilderFor())->type($this->type); + $builder->appendPhysicals($this->physicals()); + + return $builder->finish(); + } + + public function isNull(int $i): bool + { + return $this->physicalValue === null; + } + + public function isNullKind(): bool + { + return ($this->type instanceof OptionalType ? $this->type->base() : $this->type) instanceof NullType; + } + + public function nullCount(): int + { + return $this->physicalValue === null ? $this->count : 0; + } + + public function physicals(): array + { + // @mago-ignore analysis:possibly-invalid-argument + return $this->count === 0 ? [] : array_fill(0, $this->count, $this->physicalValue); + } + + public function slice(int $offset, int $length): Column + { + return new self($this->type, $this->physical, $this->physicalValue, $length); + } + + public function take(array $indices): Column + { + return new self($this->type, $this->physical, $this->physicalValue, count($indices)); + } + + public function type(): Type + { + return $this->type; + } + + public function value(int $i): mixed + { + return $this->physicalValue === null ? null : $this->physical->fromPhysical($this->physicalValue); + } + + public function values(): array + { + // @mago-ignore analysis:possibly-invalid-argument + return $this->count === 0 ? [] : array_fill(0, $this->count, $this->value(0)); + } + + public function withType(Type $type): Column + { + (new Retype())->assert($this->type, $type, $this->nullCount()); + + return new self($type, (new PhysicalFor())->type($type), $this->physicalValue, $this->count); + } +} diff --git a/src/core/etl/src/Flow/ETL/Column/Php/ListColumn.php b/src/core/etl/src/Flow/ETL/Column/Php/ListColumn.php new file mode 100644 index 0000000000..ebc75a359e --- /dev/null +++ b/src/core/etl/src/Flow/ETL/Column/Php/ListColumn.php @@ -0,0 +1,176 @@ + $type a ListType, or an OptionalType over one for a child column + * @param list $offsets count() + 1 offsets, from 0 + * @param array $nulls + */ + public function __construct( + public Type $type, + public array $offsets, + public Column $element, + public array $nulls, + ) {} + + public function at(int $i): mixed + { + if (array_key_exists($i, $this->nulls)) { + return null; + } + + return $this->element->slice($this->offsets[$i], $this->offsets[$i + 1] - $this->offsets[$i])->physicals(); + } + + public function concat(Column ...$others): Column + { + $offsets = $this->offsets; + $nulls = $this->nulls; + $rows = $this->count(); + $elements = []; + + foreach ($others as $other) { + if (!$other instanceof self) { + throw new InvalidArgumentException(sprintf('%s cannot concat %s', self::class, $other::class)); + } + + $base = $offsets[count($offsets) - 1]; + + for ($k = 1, $last = count($other->offsets); $k < $last; $k++) { + $offsets[] = $base + $other->offsets[$k]; + } + + foreach ($other->nulls as $index => $_) { + $nulls[$rows + $index] = true; + } + + $rows += $other->count(); + $elements[] = $other->element; + } + + return new self($this->type, $offsets, $this->element->concat(...$elements), $nulls); + } + + public function count(): int + { + return count($this->offsets) - 1; + } + + public function encode(): array + { + return [ + (new Validity())->fromNulls($this->nulls, $this->count()), + (new Offsets())->pack($this->offsets), + ...$this->element->encode(), + ]; + } + + public function isNull(int $i): bool + { + return array_key_exists($i, $this->nulls); + } + + public function nullCount(): int + { + return count($this->nulls); + } + + public function physicals(): array + { + $elements = $this->element->physicals(); + $physicals = []; + + for ($i = 0, $count = $this->count(); $i < $count; $i++) { + $physicals[] = array_key_exists($i, $this->nulls) + ? null + : array_slice($elements, $this->offsets[$i], $this->offsets[$i + 1] - $this->offsets[$i]); + } + + return $physicals; + } + + public function slice(int $offset, int $length): Column + { + return $this->take($length === 0 ? [] : range($offset, $offset + $length - 1)); + } + + public function take(array $indices): Column + { + $offsets = [0]; + $nulls = []; + $elementIndices = []; + + foreach ($indices as $position => $index) { + if (array_key_exists($index, $this->nulls)) { + $nulls[$position] = true; + } + + for ($j = $this->offsets[$index], $end = $this->offsets[$index + 1]; $j < $end; $j++) { + $elementIndices[] = $j; + } + + $offsets[] = count($elementIndices); + } + + return new self($this->type, $offsets, $this->element->take($elementIndices), $nulls); + } + + public function type(): Type + { + return $this->type; + } + + public function value(int $i): mixed + { + if (array_key_exists($i, $this->nulls)) { + return null; + } + + return $this->element->slice($this->offsets[$i], $this->offsets[$i + 1] - $this->offsets[$i])->values(); + } + + public function values(): array + { + $elements = $this->element->values(); + $values = []; + + for ($i = 0, $count = $this->count(); $i < $count; $i++) { + $values[] = array_key_exists($i, $this->nulls) + ? null + : array_slice($elements, $this->offsets[$i], $this->offsets[$i + 1] - $this->offsets[$i]); + } + + return $values; + } + + public function withType(Type $type): Column + { + (new Retype())->assert($this->type, $type, count($this->nulls)); + + $list = $type instanceof OptionalType ? $type->base() : $type; + assert($list instanceof ListType); + + return new self($type, $this->offsets, $this->element->withType($list->element()), $this->nulls); + } +} diff --git a/src/core/etl/src/Flow/ETL/Column/Php/ListColumnBuilder.php b/src/core/etl/src/Flow/ETL/Column/Php/ListColumnBuilder.php new file mode 100644 index 0000000000..64ae96d973 --- /dev/null +++ b/src/core/etl/src/Flow/ETL/Column/Php/ListColumnBuilder.php @@ -0,0 +1,86 @@ + + */ + private array $nulls = []; + + /** + * @var list + */ + private array $offsets = [0]; + + /** + * @param Type $type + */ + public function __construct( + private readonly Type $type, + private readonly PhpColumnBuilder $element, + ) {} + + public function appendPhysical(mixed $physical): void + { + if ($physical === null) { + $this->nulls[count($this->offsets) - 1] = true; + $this->offsets[] = $this->offsets[count($this->offsets) - 1]; + + return; + } + + assert(is_array($physical)); + $this->element->appendPhysicals(array_values($physical)); + $this->offsets[] = $this->element->count(); + } + + public function appendPhysicals(array $physicals, ?int $nullCount = null): void + { + $elements = []; + $end = $this->offsets[count($this->offsets) - 1]; + + // @mago-ignore analysis:mixed-assignment + foreach ($physicals as $physical) { + if ($physical === null) { + $this->nulls[count($this->offsets) - 1] = true; + $this->offsets[] = $end; + + continue; + } + + assert(is_array($physical)); + + // @mago-ignore analysis:mixed-assignment + foreach ($physical as $element) { + $elements[] = $element; + } + + $end += count($physical); + $this->offsets[] = $end; + } + + $this->element->appendPhysicals($elements); + } + + public function count(): int + { + return count($this->offsets) - 1; + } + + public function finish(): Column + { + return new ListColumn($this->type, $this->offsets, $this->element->finish(), $this->nulls); + } +} diff --git a/src/core/etl/src/Flow/ETL/Column/Php/MapColumn.php b/src/core/etl/src/Flow/ETL/Column/Php/MapColumn.php new file mode 100644 index 0000000000..fdff9f9197 --- /dev/null +++ b/src/core/etl/src/Flow/ETL/Column/Php/MapColumn.php @@ -0,0 +1,228 @@ + $type a MapType, or an OptionalType over one for a child column + * @param list $offsets count() + 1 offsets, from 0 + * @param array $nulls + */ + public function __construct( + public Type $type, + public array $offsets, + public Column $keys, + public Column $values, + public array $nulls, + ) {} + + public function at(int $i): mixed + { + if (array_key_exists($i, $this->nulls)) { + return null; + } + + $length = $this->offsets[$i + 1] - $this->offsets[$i]; + + /** @var list $keys */ + $keys = $this->keys->slice($this->offsets[$i], $length)->physicals(); + + return array_combine($keys, $this->values->slice($this->offsets[$i], $length)->physicals()); + } + + public function concat(Column ...$others): Column + { + $offsets = $this->offsets; + $nulls = $this->nulls; + $rows = $this->count(); + $keys = []; + $values = []; + + foreach ($others as $other) { + if (!$other instanceof self) { + throw new InvalidArgumentException(sprintf('%s cannot concat %s', self::class, $other::class)); + } + + $base = $offsets[count($offsets) - 1]; + + for ($k = 1, $last = count($other->offsets); $k < $last; $k++) { + $offsets[] = $base + $other->offsets[$k]; + } + + foreach ($other->nulls as $index => $_) { + $nulls[$rows + $index] = true; + } + + $rows += $other->count(); + $keys[] = $other->keys; + $values[] = $other->values; + } + + return new self( + $this->type, + $offsets, + $this->keys->concat(...$keys), + $this->values->concat(...$values), + $nulls, + ); + } + + public function count(): int + { + return count($this->offsets) - 1; + } + + public function encode(): array + { + return [ + (new Validity())->fromNulls($this->nulls, $this->count()), + (new Offsets())->pack($this->offsets), + '', + ...$this->keys->encode(), + ...$this->values->encode(), + ]; + } + + public function isNull(int $i): bool + { + return array_key_exists($i, $this->nulls); + } + + public function nullCount(): int + { + return count($this->nulls); + } + + public function physicals(): array + { + $keys = $this->keys->physicals(); + $values = $this->values->physicals(); + $physicals = []; + + for ($i = 0, $count = $this->count(); $i < $count; $i++) { + if (array_key_exists($i, $this->nulls)) { + $physicals[] = null; + + continue; + } + + $length = $this->offsets[$i + 1] - $this->offsets[$i]; + + /** @var list $rowKeys */ + $rowKeys = array_slice($keys, $this->offsets[$i], $length); + $physicals[] = array_combine($rowKeys, array_slice($values, $this->offsets[$i], $length)); + } + + return $physicals; + } + + public function slice(int $offset, int $length): Column + { + return $this->take($length === 0 ? [] : range($offset, $offset + $length - 1)); + } + + public function take(array $indices): Column + { + $offsets = [0]; + $nulls = []; + $entryIndices = []; + + foreach ($indices as $position => $index) { + if (array_key_exists($index, $this->nulls)) { + $nulls[$position] = true; + } + + for ($j = $this->offsets[$index], $end = $this->offsets[$index + 1]; $j < $end; $j++) { + $entryIndices[] = $j; + } + + $offsets[] = count($entryIndices); + } + + return new self( + $this->type, + $offsets, + $this->keys->take($entryIndices), + $this->values->take($entryIndices), + $nulls, + ); + } + + public function type(): Type + { + return $this->type; + } + + public function value(int $i): mixed + { + if (array_key_exists($i, $this->nulls)) { + return null; + } + + $length = $this->offsets[$i + 1] - $this->offsets[$i]; + + /** @var list $keys */ + $keys = $this->keys->slice($this->offsets[$i], $length)->values(); + + return array_combine($keys, $this->values->slice($this->offsets[$i], $length)->values()); + } + + public function values(): array + { + $keys = $this->keys->values(); + $values = $this->values->values(); + $result = []; + + for ($i = 0, $count = $this->count(); $i < $count; $i++) { + if (array_key_exists($i, $this->nulls)) { + $result[] = null; + + continue; + } + + $length = $this->offsets[$i + 1] - $this->offsets[$i]; + + /** @var list $rowKeys */ + $rowKeys = array_slice($keys, $this->offsets[$i], $length); + $result[] = array_combine($rowKeys, array_slice($values, $this->offsets[$i], $length)); + } + + return $result; + } + + public function withType(Type $type): Column + { + (new Retype())->assert($this->type, $type, count($this->nulls)); + + $map = $type instanceof OptionalType ? $type->base() : $type; + assert($map instanceof MapType); + + return new self( + $type, + $this->offsets, + $this->keys->withType($map->key()), + $this->values->withType($map->value()), + $this->nulls, + ); + } +} diff --git a/src/core/etl/src/Flow/ETL/Column/Php/MapColumnBuilder.php b/src/core/etl/src/Flow/ETL/Column/Php/MapColumnBuilder.php new file mode 100644 index 0000000000..492c985189 --- /dev/null +++ b/src/core/etl/src/Flow/ETL/Column/Php/MapColumnBuilder.php @@ -0,0 +1,69 @@ + + */ + private array $nulls = []; + + /** + * @var list + */ + private array $offsets = [0]; + + /** + * @param Type $type + */ + public function __construct( + private readonly Type $type, + private readonly PhpColumnBuilder $keys, + private readonly PhpColumnBuilder $values, + ) {} + + public function appendPhysical(mixed $physical): void + { + if ($physical === null) { + $this->nulls[count($this->offsets) - 1] = true; + $this->offsets[] = $this->offsets[count($this->offsets) - 1]; + + return; + } + + assert(is_array($physical)); + $this->keys->appendPhysicals(array_keys($physical)); + $this->values->appendPhysicals(array_values($physical)); + $this->offsets[] = $this->keys->count(); + } + + public function appendPhysicals(array $physicals, ?int $nullCount = null): void + { + // @mago-ignore analysis:mixed-assignment + foreach ($physicals as $physical) { + $this->appendPhysical($physical); + } + } + + public function count(): int + { + return count($this->offsets) - 1; + } + + public function finish(): Column + { + return new MapColumn($this->type, $this->offsets, $this->keys->finish(), $this->values->finish(), $this->nulls); + } +} diff --git a/src/core/etl/src/Flow/ETL/Column/Php/NullColumnBuilder.php b/src/core/etl/src/Flow/ETL/Column/Php/NullColumnBuilder.php new file mode 100644 index 0000000000..7971fc8a25 --- /dev/null +++ b/src/core/etl/src/Flow/ETL/Column/Php/NullColumnBuilder.php @@ -0,0 +1,43 @@ + $type + */ + public function __construct( + private readonly Type $type, + ) {} + + public function appendPhysical(mixed $physical): void + { + $this->count++; + } + + public function appendPhysicals(array $physicals, ?int $nullCount = null): void + { + $this->count += count($physicals); + } + + public function count(): int + { + return $this->count; + } + + public function finish(): Column + { + return new ConstantColumn($this->type, new NullPhysical(), null, $this->count); + } +} diff --git a/src/core/etl/src/Flow/ETL/Column/Php/PhpColumnBuilder.php b/src/core/etl/src/Flow/ETL/Column/Php/PhpColumnBuilder.php new file mode 100644 index 0000000000..9b3072d4e3 --- /dev/null +++ b/src/core/etl/src/Flow/ETL/Column/Php/PhpColumnBuilder.php @@ -0,0 +1,25 @@ + $physicals + * @param null|int $nullCount the nulls among $physicals when the caller already counted them + */ + public function appendPhysicals(array $physicals, ?int $nullCount = null): void; + + public function count(): int; + + public function finish(): Column; +} diff --git a/src/core/etl/src/Flow/ETL/Column/Php/ScalarColumn.php b/src/core/etl/src/Flow/ETL/Column/Php/ScalarColumn.php new file mode 100644 index 0000000000..0fc13c5f31 --- /dev/null +++ b/src/core/etl/src/Flow/ETL/Column/Php/ScalarColumn.php @@ -0,0 +1,136 @@ + $type + * @param list $values physicals, null for a null cell + */ + public function __construct( + public Type $type, + public Physical $physical, + public array $values, + public int $nullCount, + ) {} + + public function at(int $i): mixed + { + return $this->values[$i]; + } + + public function concat(Column ...$others): Column + { + $values = [$this->values]; + $nullCount = $this->nullCount; + + foreach ($others as $other) { + if (!$other instanceof self) { + throw new InvalidArgumentException(sprintf('%s cannot concat %s', self::class, $other::class)); + } + + $values[] = $other->values; + $nullCount += $other->nullCount; + } + + return new self($this->type, $this->physical, array_merge(...$values), $nullCount); + } + + public function count(): int + { + return count($this->values); + } + + public function encode(): array + { + return [ + (new Validity())->fromValues($this->values), + ...(new LayoutFor()) + ->type($this->type) + ->encode($this->values), + ]; + } + + public function isNull(int $i): bool + { + return $this->values[$i] === null; + } + + public function nullCount(): int + { + return $this->nullCount; + } + + public function physicals(): array + { + return $this->values; + } + + public function slice(int $offset, int $length): Column + { + $values = array_slice($this->values, $offset, $length); + + return new self( + $this->type, + $this->physical, + $values, + $this->nullCount === 0 ? 0 : (new NullCount())->of($values), + ); + } + + public function take(array $indices): Column + { + $values = []; + + foreach ($indices as $index) { + $values[] = $this->values[$index]; + } + + return new self( + $this->type, + $this->physical, + $values, + $this->nullCount === 0 ? 0 : (new NullCount())->of($values), + ); + } + + public function type(): Type + { + return $this->type; + } + + public function value(int $i): mixed + { + return $this->values[$i] === null ? null : $this->physical->fromPhysical($this->values[$i]); + } + + public function values(): array + { + return $this->physical->fromPhysicalAll($this->values); + } + + public function withType(Type $type): Column + { + (new Retype())->assert($this->type, $type, $this->nullCount); + + return new self($type, (new PhysicalFor())->type($type), $this->values, $this->nullCount); + } +} diff --git a/src/core/etl/src/Flow/ETL/Column/Php/ScalarColumnBuilder.php b/src/core/etl/src/Flow/ETL/Column/Php/ScalarColumnBuilder.php new file mode 100644 index 0000000000..d2a92470c9 --- /dev/null +++ b/src/core/etl/src/Flow/ETL/Column/Php/ScalarColumnBuilder.php @@ -0,0 +1,60 @@ + + */ + private array $values = []; + + /** + * @param Type $type + */ + public function __construct( + private readonly Type $type, + private readonly Physical $physical, + ) {} + + public function appendPhysical(mixed $physical): void + { + assert(!is_array($physical) && !is_object($physical)); + + if ($physical === null) { + $this->nullCount++; + } + + $this->values[] = $physical; + } + + public function appendPhysicals(array $physicals, ?int $nullCount = null): void + { + $this->nullCount += $nullCount ?? count(array_keys($physicals, null, true)); + array_push($this->values, ...$physicals); + } + + public function count(): int + { + return count($this->values); + } + + public function finish(): Column + { + return new ScalarColumn($this->type, $this->physical, $this->values, $this->nullCount); + } +} diff --git a/src/core/etl/src/Flow/ETL/Column/Php/StructColumn.php b/src/core/etl/src/Flow/ETL/Column/Php/StructColumn.php new file mode 100644 index 0000000000..2be89158f4 --- /dev/null +++ b/src/core/etl/src/Flow/ETL/Column/Php/StructColumn.php @@ -0,0 +1,291 @@ + $type a StructureType, or an OptionalType over one for a child column + * @param non-empty-array $children keyed by element name, in element order + * @param array $nulls + */ + public function __construct( + public Type $type, + public array $children, + public array $nulls, + ) {} + + public function at(int $i): mixed + { + if (array_key_exists($i, $this->nulls)) { + return null; + } + + $physical = []; + + foreach ($this->structure()->elements() as $element) { + $child = $this->children[$element->name]; + + if ($child->isNull($i)) { + if (!$element->optional) { + $physical[$element->name] = null; + } + + continue; + } + + $physical[$element->name] = $child->at($i); + } + + return $physical; + } + + public function concat(Column ...$others): Column + { + $nulls = $this->nulls; + $rows = $this->count(); + $children = []; + + foreach ($others as $other) { + if (!$other instanceof self) { + throw new InvalidArgumentException(sprintf('%s cannot concat %s', self::class, $other::class)); + } + + foreach ($other->nulls as $index => $_) { + $nulls[$rows + $index] = true; + } + + $rows += $other->count(); + + foreach ($other->children as $name => $child) { + $children[$name][] = $child; + } + } + + $concatenated = []; + + foreach ($this->children as $name => $child) { + $concatenated[$name] = $child->concat(...$children[$name] ?? []); + } + + return new self($this->type, $concatenated, $nulls); + } + + public function count(): int + { + return $this->children[array_key_first($this->children)]->count(); + } + + public function encode(): array + { + $buffers = [(new Validity())->fromNulls($this->nulls, $this->count())]; + + foreach ($this->children as $child) { + foreach ($child->encode() as $buffer) { + $buffers[] = $buffer; + } + } + + return $buffers; + } + + public function isNull(int $i): bool + { + return array_key_exists($i, $this->nulls); + } + + public function nullCount(): int + { + return count($this->nulls); + } + + public function physicals(): array + { + $elements = $this->structure()->elements(); + $children = []; + + foreach ($elements as $element) { + $children[$element->name] = $this->children[$element->name]->physicals(); + } + + $physicals = []; + + for ($i = 0, $count = $this->count(); $i < $count; $i++) { + if (array_key_exists($i, $this->nulls)) { + $physicals[] = null; + + continue; + } + + $physical = []; + + foreach ($elements as $element) { + // @mago-ignore analysis:mixed-assignment + $cell = $children[$element->name][$i]; + + if ($cell === null && $element->optional) { + continue; + } + + $physical[$element->name] = $cell; + } + + $physicals[] = $physical; + } + + return $physicals; + } + + public function slice(int $offset, int $length): Column + { + return $this->take($length === 0 ? [] : range($offset, $offset + $length - 1)); + } + + public function structure(): StructureType + { + $structure = $this->type instanceof OptionalType ? $this->type->base() : $this->type; + assert($structure instanceof StructureType); + + return $structure; + } + + public function take(array $indices): Column + { + $nulls = []; + + foreach ($indices as $position => $index) { + if (array_key_exists($index, $this->nulls)) { + $nulls[$position] = true; + } + } + + $children = []; + + foreach ($this->children as $name => $child) { + $children[$name] = $child->take($indices); + } + + return new self($this->type, $children, $nulls); + } + + public function type(): Type + { + return $this->type; + } + + public function value(int $i): mixed + { + if (array_key_exists($i, $this->nulls)) { + return null; + } + + $value = []; + + foreach ($this->structure()->elements() as $element) { + $child = $this->children[$element->name]; + + if ($child->isNull($i)) { + if (!$element->optional) { + $value[$element->name] = null; + } + + continue; + } + + $value[$element->name] = $child->value($i); + } + + return $value; + } + + public function values(): array + { + $elements = $this->structure()->elements(); + $children = []; + + foreach ($elements as $element) { + $children[$element->name] = $this->children[$element->name]->values(); + } + + $values = []; + + for ($i = 0, $count = $this->count(); $i < $count; $i++) { + if (array_key_exists($i, $this->nulls)) { + $values[] = null; + + continue; + } + + $value = []; + + foreach ($elements as $element) { + // @mago-ignore analysis:mixed-assignment + $cell = $children[$element->name][$i]; + + if ($cell === null && $element->optional) { + continue; + } + + $value[$element->name] = $cell; + } + + $values[] = $value; + } + + return $values; + } + + public function withType(Type $type): Column + { + (new Retype())->assert($this->type, $type, count($this->nulls)); + + $structure = $type instanceof OptionalType ? $type->base() : $type; + assert($structure instanceof StructureType); + + $from = $this->structure()->elements(); + $children = []; + + foreach ($structure->elements() as $position => $element) { + $child = $this->children[$element->name]; + + if ($from[$position]->optional && !$element->optional) { + $absent = 0; + + for ($i = 0, $count = $child->count(); $i < $count; $i++) { + if (!array_key_exists($i, $this->nulls) && $child->isNull($i)) { + $absent++; + } + } + + if ($absent > 0) { + throw new InvalidArgumentException(sprintf( + '%d absent values under the required element "%s"', + $absent, + $element->name, + )); + } + } + + $children[$element->name] = $child->withType($element->type); + } + + return new self($type, $children, $this->nulls); + } +} diff --git a/src/core/etl/src/Flow/ETL/Column/Php/StructColumnBuilder.php b/src/core/etl/src/Flow/ETL/Column/Php/StructColumnBuilder.php new file mode 100644 index 0000000000..16ab249e2d --- /dev/null +++ b/src/core/etl/src/Flow/ETL/Column/Php/StructColumnBuilder.php @@ -0,0 +1,90 @@ + + */ + private array $nulls = []; + + /** + * @param Type $type + * @param non-empty-array $children keyed by element name, in element order + */ + public function __construct( + private readonly Type $type, + private readonly array $children, + ) {} + + public function appendPhysical(mixed $physical): void + { + if ($physical === null) { + $this->nulls[$this->count++] = true; + + foreach ($this->children as $child) { + $child->appendPhysical(null); + } + + return; + } + + assert(is_array($physical)); + $this->count++; + + foreach ($this->children as $name => $child) { + $child->appendPhysical(array_key_exists($name, $physical) ? $physical[$name] : null); + } + } + + public function appendPhysicals(array $physicals, ?int $nullCount = null): void + { + $names = array_keys($this->children); + $columns = array_fill_keys($names, []); + + // @mago-ignore analysis:mixed-assignment + foreach ($physicals as $physical) { + if ($physical === null) { + $this->nulls[$this->count] = true; + } + + $this->count++; + + foreach ($names as $name) { + $columns[$name][] = $physical[$name] ?? null; + } + } + + foreach ($this->children as $name => $child) { + $child->appendPhysicals($columns[$name]); + } + } + + public function count(): int + { + return $this->count; + } + + public function finish(): Column + { + $children = []; + + foreach ($this->children as $name => $child) { + $children[$name] = $child->finish(); + } + + return new StructColumn($this->type, $children, $this->nulls); + } +} diff --git a/src/core/etl/src/Flow/ETL/Column/PhpBackend.php b/src/core/etl/src/Flow/ETL/Column/PhpBackend.php new file mode 100644 index 0000000000..1662a4f33c --- /dev/null +++ b/src/core/etl/src/Flow/ETL/Column/PhpBackend.php @@ -0,0 +1,101 @@ +definition($definition), + (new PhysicalBuilderFor())->type($definition->type()), + ); + } + + public function constant(Definition $definition, mixed $value, int $count): Column + { + $physical = (new PhysicalFor())->definition($definition); + + if ($value === null) { + if (!$definition->isNullable() && !$definition instanceof NullDefinition) { + throw ColumnMismatchException::valueDoesNotMatch($definition, null); + } + + return new ConstantColumn($definition->type(), $physical, null, $count); + } + + return new ConstantColumn( + $definition->type(), + $physical, + $physical->toPhysical($definition->type()->cast($value)), + $count, + ); + } + + public function decode(Definition $definition, array $buffers, int $count, int $nullCount): Column + { + $cursor = new Buffers($buffers); + $column = (new ColumnDecoder())->decode($definition->type(), $cursor, $count, $nullCount); + + if ($cursor->remaining() !== 0) { + throw new InvalidArgumentException(sprintf( + 'Column "%s": %d buffers left after decoding', + $definition->entry()->name(), + $cursor->remaining(), + )); + } + + return $column; + } + + public function adopt(Definition $definition, Column $column): Column + { + if ($column instanceof ValueColumn) { + throw ColumnMismatchException::untypedColumn($definition); + } + + if ( + $column instanceof ScalarColumn + || $column instanceof ConstantColumn + || $column instanceof ListColumn + || $column instanceof MapColumn + || $column instanceof StructColumn + ) { + return $column; + } + + $builder = $this->builder($definition); + + if ($column->count() > 0) { + $builder->appendTake($column, range(0, $column->count() - 1)); + } + + return $builder->finish(); + } + + public function allocatedBytes(): int + { + return 0; + } +} diff --git a/src/core/etl/src/Flow/ETL/Column/Physical/DatePhysical.php b/src/core/etl/src/Flow/ETL/Column/Physical/DatePhysical.php new file mode 100644 index 0000000000..2a67ef38fb --- /dev/null +++ b/src/core/etl/src/Flow/ETL/Column/Physical/DatePhysical.php @@ -0,0 +1,66 @@ +days = new DaysFromCivil(); + $this->utc = new DateTimeZone('UTC'); + } + + public function toPhysical(mixed $value): mixed + { + assert($value instanceof DateTimeInterface); + + $days = $this->days->of((int) $value->format('Y'), (int) $value->format('n'), (int) $value->format('j')); + + if ($days < -2_147_483_648 || $days > 2_147_483_647) { + throw new CastingException($value, type_date(), reason: 'outside the day range of a date column'); + } + + return $days; + } + + public function fromPhysical(mixed $physical): mixed + { + assert(is_int($physical)); + + return (new DateTimeImmutable('@' . ($physical * 86_400)))->setTimezone($this->utc); + } + + public function fromPhysicalAll(array $physicals): array + { + $values = []; + + // @mago-ignore analysis:mixed-assignment + foreach ($physicals as $physical) { + if ($physical === null) { + $values[] = null; + + continue; + } + + assert(is_int($physical)); + $values[] = (new DateTimeImmutable('@' . ($physical * 86_400)))->setTimezone($this->utc); + } + + return $values; + } +} diff --git a/src/core/etl/src/Flow/ETL/Column/Physical/DateTimePhysical.php b/src/core/etl/src/Flow/ETL/Column/Physical/DateTimePhysical.php new file mode 100644 index 0000000000..43c85b4c26 --- /dev/null +++ b/src/core/etl/src/Flow/ETL/Column/Physical/DateTimePhysical.php @@ -0,0 +1,116 @@ +getTimestamp() * 1_000_000) + (int) $value->format('u'); + + // an i64 overflow of the multiplication or the addition yields a float at runtime + // @mago-ignore analysis:impossible-condition,redundant-type-comparison + if (!is_int($us)) { + throw new CastingException( + $value, + type_datetime(), + reason: 'outside the microsecond range of a datetime column', + ); + } + + return $us; + } + + public function fromPhysical(mixed $physical): mixed + { + assert(is_int($physical)); + + if (PHP_VERSION_ID >= 80400) { + $seconds = intdiv($physical, 1_000_000); + $micros = $physical % 1_000_000; + + if ($micros < 0) { + $micros += 1_000_000; + $seconds--; + } + + // @mago-ignore analysis:unavailable-method + $instant = DateTimeImmutable::createFromTimestamp($seconds); + + // @mago-ignore analysis:unavailable-method + return $instant->setMicrosecond($micros)->setTimezone($this->zone); + } + + return (new DateTimeImmutable(sprintf( + '@%s%d.%06d', + $physical < 0 ? '-' : '', + intdiv(abs($physical), 1_000_000), + abs($physical) % 1_000_000, + )))->setTimezone($this->zone); + } + + public function fromPhysicalAll(array $physicals): array + { + $values = []; + $modern = PHP_VERSION_ID >= 80400; + + // @mago-ignore analysis:mixed-assignment + foreach ($physicals as $physical) { + if ($physical === null) { + $values[] = null; + + continue; + } + + assert(is_int($physical)); + + if ($modern) { + $seconds = intdiv($physical, 1_000_000); + $micros = $physical % 1_000_000; + + if ($micros < 0) { + $micros += 1_000_000; + $seconds--; + } + + // @mago-ignore analysis:unavailable-method + $instant = DateTimeImmutable::createFromTimestamp($seconds); + // @mago-ignore analysis:unavailable-method + $values[] = $instant->setMicrosecond($micros)->setTimezone($this->zone); + + continue; + } + + $values[] = (new DateTimeImmutable(sprintf( + '@%s%d.%06d', + $physical < 0 ? '-' : '', + intdiv(abs($physical), 1_000_000), + abs($physical) % 1_000_000, + )))->setTimezone($this->zone); + } + + return $values; + } +} diff --git a/src/core/etl/src/Flow/ETL/Column/Physical/DaysFromCivil.php b/src/core/etl/src/Flow/ETL/Column/Physical/DaysFromCivil.php new file mode 100644 index 0000000000..6f2079dc09 --- /dev/null +++ b/src/core/etl/src/Flow/ETL/Column/Physical/DaysFromCivil.php @@ -0,0 +1,24 @@ += 0 ? $year : $year - 399, 400); + $yearOfEra = $year - ($era * 400); + $dayOfYear = intdiv((153 * ($month > 2 ? $month - 3 : $month + 9)) + 2, 5) + $day - 1; + $dayOfEra = ($yearOfEra * 365) + intdiv($yearOfEra, 4) - intdiv($yearOfEra, 100) + $dayOfYear; + + return ($era * 146_097) + $dayOfEra - 719_468; + } +} diff --git a/src/core/etl/src/Flow/ETL/Column/Physical/ElementPosition.php b/src/core/etl/src/Flow/ETL/Column/Physical/ElementPosition.php new file mode 100644 index 0000000000..0eafe0448d --- /dev/null +++ b/src/core/etl/src/Flow/ETL/Column/Physical/ElementPosition.php @@ -0,0 +1,121 @@ +\0\0", the path + * counting element children from the document element. Earlier builds wrote "\0\0", and + * before that bare element markup. + */ +final readonly class ElementPosition +{ + /** + * @param \DOMElement|\Dom\Element $element + * + * @return ?string null for an element outside its owner document's tree + */ + public function path(object $element): ?string + { + $indices = []; + $node = $element; + + while ($node->parentNode !== null && $node->parentNode->nodeType === XML_ELEMENT_NODE) { + $index = 0; + + for ( + $sibling = $node->previousElementSibling; + $sibling !== null; + $sibling = $sibling->previousElementSibling + ) { + $index++; + } + + $indices[] = $index; + /** @var \DOMElement|\Dom\Element $node */ + $node = $node->parentNode; + } + + return $node === $element->ownerDocument?->documentElement ? implode('/', array_reverse($indices)) : null; + } + + public function physical(string $path, string $markup, string $document): string + { + return "\x01" . $path . "\0" . strlen($markup) . "\0" . $markup . $document; + } + + /** + * @return ?string null for bare element markup + */ + public function pathOf(string $physical): ?string + { + return str_starts_with($physical, "\x01") || str_starts_with($physical, "\0") + ? substr($physical, 1, (int) strpos($physical, "\0", 1) - 1) + : null; + } + + /** + * @return ?string null for bytes that do not carry the element markup + */ + public function markupOf(string $physical): ?string + { + if (!str_starts_with($physical, "\x01")) { + return null; + } + + $pathEnd = (int) strpos($physical, "\0", 1); + $lengthEnd = (int) strpos($physical, "\0", $pathEnd + 1); + + return substr($physical, $lengthEnd + 1, (int) substr($physical, $pathEnd + 1, $lengthEnd - $pathEnd - 1)); + } + + public function documentOf(string $physical): string + { + $pathEnd = (int) strpos($physical, "\0", 1); + + if (!str_starts_with($physical, "\x01")) { + return substr($physical, $pathEnd + 1); + } + + $lengthEnd = (int) strpos($physical, "\0", $pathEnd + 1); + + return substr($physical, $lengthEnd + 1 + (int) substr($physical, $pathEnd + 1, $lengthEnd - $pathEnd - 1)); + } + + /** + * @param null|\DOMElement|\Dom\Element $documentElement + * + * @return null|\DOMElement|\Dom\Element + */ + public function element(?object $documentElement, string $path): ?object + { + $node = $documentElement; + + foreach ($path === '' ? [] : array_map(intval(...), explode('/', $path)) as $index) { + $node = $node?->firstElementChild; + + for ($i = 0; $node !== null && $i < $index; $i++) { + $node = $node->nextElementSibling; + } + + if ($node === null) { + return null; + } + } + + return $node; + } +} diff --git a/src/core/etl/src/Flow/ETL/Column/Physical/EnumPhysical.php b/src/core/etl/src/Flow/ETL/Column/Physical/EnumPhysical.php new file mode 100644 index 0000000000..5c6487dc67 --- /dev/null +++ b/src/core/etl/src/Flow/ETL/Column/Physical/EnumPhysical.php @@ -0,0 +1,48 @@ + $class + */ + public function __construct( + private string $class, + ) {} + + public function toPhysical(mixed $value): mixed + { + assert($value instanceof UnitEnum); + + return $value->name; + } + + public function fromPhysical(mixed $physical): mixed + { + assert(is_string($physical)); + + return constant($this->class . '::' . $physical); + } + + public function fromPhysicalAll(array $physicals): array + { + $values = []; + + // @mago-ignore analysis:mixed-assignment + foreach ($physicals as $physical) { + assert($physical === null || is_string($physical)); + $values[] = $physical === null ? null : constant($this->class . '::' . $physical); + } + + return $values; + } +} diff --git a/src/core/etl/src/Flow/ETL/Column/Physical/HtmlDocumentPhysical.php b/src/core/etl/src/Flow/ETL/Column/Physical/HtmlDocumentPhysical.php new file mode 100644 index 0000000000..7d6e87037a --- /dev/null +++ b/src/core/etl/src/Flow/ETL/Column/Physical/HtmlDocumentPhysical.php @@ -0,0 +1,51 @@ +saveHtml(); + } + + public function fromPhysical(mixed $physical): mixed + { + assert(is_string($physical)); + + if (!class_exists('\Dom\HTMLDocument')) { + throw new InvalidArgumentException('Floe cannot restore HTML values, \Dom\HTMLDocument requires PHP 8.4+'); + } + + // @mago-expect analysis:unavailable-method + return HTMLDocument::createFromString($physical, LIBXML_NOERROR); + } + + public function fromPhysicalAll(array $physicals): array + { + $values = []; + + // @mago-ignore analysis:mixed-assignment + foreach ($physicals as $physical) { + $values[] = $physical === null ? null : $this->fromPhysical($physical); + } + + return $values; + } +} diff --git a/src/core/etl/src/Flow/ETL/Column/Physical/HtmlElementPhysical.php b/src/core/etl/src/Flow/ETL/Column/Physical/HtmlElementPhysical.php new file mode 100644 index 0000000000..bdcac7e7d8 --- /dev/null +++ b/src/core/etl/src/Flow/ETL/Column/Physical/HtmlElementPhysical.php @@ -0,0 +1,117 @@ +ownerDocument?->saveHtml($value); + + if (!is_string($html)) { + throw new InvalidArgumentException('Floe failed to convert HTMLElement to HTML string'); + } + + $path = $this->position->path($value); + + if ($path === null) { + return $html; + } + + // @mago-ignore analysis:non-existent-method + // @mago-ignore analysis:mixed-assignment + $document = $value->ownerDocument?->saveHtml(); + assert(is_string($document)); + + return $this->position->physical($path, $html, $document); + } + + public function fromPhysical(mixed $physical): mixed + { + assert(is_string($physical)); + + if (!class_exists('\Dom\HTMLDocument')) { + throw new InvalidArgumentException('Floe cannot restore HTML values, \Dom\HTMLDocument requires PHP 8.4+'); + } + + $path = $this->position->pathOf($physical); + + // @mago-expect analysis:unavailable-method + $element = $path === null + ? HTMLDocument::createFromString( + '' . $physical . '', + LIBXML_NOERROR, + )->body?->firstElementChild + : $this->position->element( + // @mago-expect analysis:unavailable-method + HTMLDocument::createFromString( + $this->position->documentOf($physical), + LIBXML_HTML_NOIMPLIED | LIBXML_NOERROR, + )->documentElement, + $path, + ); + + if ($element === null) { + throw new InvalidArgumentException(sprintf('Floe failed to restore HTMLElement from "%s"', $physical)); + } + + return $element; + } + + public function markup(string $physical): string + { + $markup = $this->position->markupOf($physical); + + if ($markup !== null) { + return $markup; + } + + if ($this->position->pathOf($physical) === null) { + return $physical; + } + + /** @var \Dom\HTMLElement $element */ + $element = $this->fromPhysical($physical); + // @mago-ignore analysis:non-existent-method + // @mago-ignore analysis:mixed-assignment + $html = $element->ownerDocument?->saveHtml($element); + assert(is_string($html)); + + return $html; + } + + public function fromPhysicalAll(array $physicals): array + { + $values = []; + + // @mago-ignore analysis:mixed-assignment + foreach ($physicals as $physical) { + $values[] = $physical === null ? null : $this->fromPhysical($physical); + } + + return $values; + } +} diff --git a/src/core/etl/src/Flow/ETL/Column/Physical/IdentityPhysical.php b/src/core/etl/src/Flow/ETL/Column/Physical/IdentityPhysical.php new file mode 100644 index 0000000000..5f964cc81c --- /dev/null +++ b/src/core/etl/src/Flow/ETL/Column/Physical/IdentityPhysical.php @@ -0,0 +1,23 @@ +newInstanceWithoutConstructor(); + $json->value = $value; + $json->isObject = $value[0] === '{'; + + return $json; + }, + null, + Json::class, + ); + + $this->create = $create; + } + + public function create(string $json): Json + { + return ($this->create)($json); + } +} diff --git a/src/core/etl/src/Flow/ETL/Column/Physical/JsonPhysical.php b/src/core/etl/src/Flow/ETL/Column/Physical/JsonPhysical.php new file mode 100644 index 0000000000..285fbe8d7f --- /dev/null +++ b/src/core/etl/src/Flow/ETL/Column/Physical/JsonPhysical.php @@ -0,0 +1,44 @@ +toString(); + } + + public function fromPhysical(mixed $physical): mixed + { + assert(is_string($physical)); + + return $this->factory->create($physical); + } + + public function fromPhysicalAll(array $physicals): array + { + $values = []; + + // @mago-ignore analysis:mixed-assignment + foreach ($physicals as $physical) { + assert($physical === null || is_string($physical)); + $values[] = $physical === null ? null : $this->factory->create($physical); + } + + return $values; + } +} diff --git a/src/core/etl/src/Flow/ETL/Column/Physical/ListPhysical.php b/src/core/etl/src/Flow/ETL/Column/Physical/ListPhysical.php new file mode 100644 index 0000000000..9b7fbc66d5 --- /dev/null +++ b/src/core/etl/src/Flow/ETL/Column/Physical/ListPhysical.php @@ -0,0 +1,52 @@ +element->toPhysical($element); + } + + return $physicals; + } + + public function fromPhysical(mixed $physical): mixed + { + assert(is_array($physical)); + + /** @var list $physical */ + return $this->element->fromPhysicalAll($physical); + } + + public function fromPhysicalAll(array $physicals): array + { + $values = []; + + // @mago-ignore analysis:mixed-assignment + foreach ($physicals as $physical) { + assert($physical === null || is_array($physical)); + + /** @var null|list $physical */ + $values[] = $physical === null ? null : $this->element->fromPhysicalAll($physical); + } + + return $values; + } +} diff --git a/src/core/etl/src/Flow/ETL/Column/Physical/MapPhysical.php b/src/core/etl/src/Flow/ETL/Column/Physical/MapPhysical.php new file mode 100644 index 0000000000..ed710e4dd0 --- /dev/null +++ b/src/core/etl/src/Flow/ETL/Column/Physical/MapPhysical.php @@ -0,0 +1,54 @@ + $element) { + $physicals[$key] = $element === null ? null : $this->value->toPhysical($element); + } + + return $physicals; + } + + public function fromPhysical(mixed $physical): mixed + { + assert(is_array($physical)); + + return array_combine(array_keys($physical), $this->value->fromPhysicalAll(array_values($physical))); + } + + public function fromPhysicalAll(array $physicals): array + { + $values = []; + + // @mago-ignore analysis:mixed-assignment + foreach ($physicals as $physical) { + assert($physical === null || is_array($physical)); + $values[] = $physical === null + ? null + : array_combine(array_keys($physical), $this->value->fromPhysicalAll(array_values($physical))); + } + + return $values; + } +} diff --git a/src/core/etl/src/Flow/ETL/Column/Physical/NullPhysical.php b/src/core/etl/src/Flow/ETL/Column/Physical/NullPhysical.php new file mode 100644 index 0000000000..699f862c17 --- /dev/null +++ b/src/core/etl/src/Flow/ETL/Column/Physical/NullPhysical.php @@ -0,0 +1,26 @@ + $physicals nulls stay null + * + * @return list + */ + public function fromPhysicalAll(array $physicals): array; +} diff --git a/src/core/etl/src/Flow/ETL/Column/Physical/PhysicalBuilderFor.php b/src/core/etl/src/Flow/ETL/Column/Physical/PhysicalBuilderFor.php new file mode 100644 index 0000000000..9636117f56 --- /dev/null +++ b/src/core/etl/src/Flow/ETL/Column/Physical/PhysicalBuilderFor.php @@ -0,0 +1,56 @@ + $type + * + * @throws InvalidArgumentException + */ + public function type(Type $type): PhpColumnBuilder + { + $base = $type instanceof OptionalType ? $type->base() : $type; + + if ($base instanceof ListType) { + return new ListColumnBuilder($type, $this->type($base->element())); + } + + if ($base instanceof MapType) { + return new MapColumnBuilder($type, $this->type($base->key()), $this->type($base->value())); + } + + if ($base instanceof StructureType) { + $children = []; + + foreach ($base->elements() as $element) { + $children[$element->name] = $this->type($element->type); + } + + return new StructColumnBuilder($type, $children); + } + + if ($base instanceof NullType) { + return new NullColumnBuilder($type); + } + + return new ScalarColumnBuilder($type, (new PhysicalFor())->type($type)); + } +} diff --git a/src/core/etl/src/Flow/ETL/Column/Physical/PhysicalFor.php b/src/core/etl/src/Flow/ETL/Column/Physical/PhysicalFor.php new file mode 100644 index 0000000000..ccd58a6b90 --- /dev/null +++ b/src/core/etl/src/Flow/ETL/Column/Physical/PhysicalFor.php @@ -0,0 +1,251 @@ + $definition + * + * @throws ColumnMismatchException + */ + public function definition(Definition $definition): Physical + { + if (!in_array($definition::class, self::DEFINITIONS, true)) { + throw ColumnMismatchException::unsupportedType( + $definition, + 'only the 19 Flow definitions have a column kind', + ); + } + + try { + return $this->type($definition->type()); + } catch (InvalidArgumentException $e) { + throw ColumnMismatchException::unsupportedType($definition, $e->getMessage()); + } + } + + /** + * A column of $type exists: definition_from_type() accepts it and definition() gives it a Physical. + * + * @param Type $type + */ + public function supports(Type $type): bool + { + $base = type_bare($type); + + return match (true) { + $base instanceof ArrayType => true, + $base instanceof EmptyArrayType, + $base instanceof PositiveIntegerType, + $base instanceof NonEmptyStringType, + $base instanceof NumericStringType, + $base instanceof ClassStringType, + => false, + default => $this->supportsChild($base), + }; + } + + /** + * A child (list element, map value, structure element) of $type exists once TypeProjection stored it as json + * (an array) or as optional (a nullable union): type() gives it a Physical. + * + * @param Type $type + */ + public function supportsChild(Type $type): bool + { + $base = type_bare($type); + + return match (true) { + $base instanceof ArrayType, $base instanceof EmptyArrayType => true, + $base instanceof IntegerType, + $base instanceof PositiveIntegerType, + $base instanceof FloatType, + $base instanceof BooleanType, + $base instanceof StringType, + $base instanceof NonEmptyStringType, + $base instanceof NumericStringType, + $base instanceof ClassStringType, + $base instanceof DateTimeType, + $base instanceof DateType, + $base instanceof TimeType, + $base instanceof UuidType, + $base instanceof TimeZoneType, + $base instanceof JsonType, + $base instanceof XMLType, + $base instanceof XMLElementType, + $base instanceof HTMLType, + $base instanceof HTMLElementType, + $base instanceof NullType, + => true, + $base instanceof EnumType => $base->class !== UnitEnum::class && $base->class !== BackedEnum::class, + $base instanceof ListType => $this->supportsChild($base->element()), + $base instanceof MapType => $this->supportsChild($base->value()), + $base instanceof StructureType => $this->supportsElements($base), + default => false, + }; + } + + /** + * @param StructureType $type + */ + public function supportsElements(StructureType $type): bool + { + foreach ($type->elements() as $element) { + if (!$this->supportsChild($element->type)) { + return false; + } + } + + return true; + } + + /** + * @param Type $type + * + * @throws InvalidArgumentException + */ + public function type(Type $type): Physical + { + $base = $type instanceof OptionalType ? $type->base() : $type; + + return match (true) { + $base instanceof IntegerType, + $base instanceof PositiveIntegerType, + $base instanceof FloatType, + $base instanceof BooleanType, + $base instanceof StringType, + $base instanceof NonEmptyStringType, + $base instanceof NumericStringType, + $base instanceof ClassStringType, + => new IdentityPhysical(), + $base instanceof DateTimeType => new DateTimePhysical($base->zone()), + $base instanceof DateType => new DatePhysical(), + $base instanceof TimeType => new TimePhysical(), + $base instanceof UuidType => new UuidPhysical(), + $base instanceof TimeZoneType => new TimeZonePhysical(), + $base instanceof JsonType => new JsonPhysical(), + $base instanceof EnumType => $base->class === UnitEnum::class || $base->class === BackedEnum::class + ? throw new InvalidArgumentException('declare the concrete enum class') + : new EnumPhysical($base->class), + $base instanceof XMLType => new XmlDocumentPhysical(), + $base instanceof XMLElementType => new XmlElementPhysical(), + $base instanceof HTMLType => new HtmlDocumentPhysical(), + $base instanceof HTMLElementType => new HtmlElementPhysical(), + $base instanceof ListType => ($element = $this->type($base->element())) instanceof IdentityPhysical + ? $element + : new ListPhysical($element), + $base instanceof MapType => ($value = $this->type($base->value())) instanceof IdentityPhysical + ? $value + : new MapPhysical($value), + $base instanceof StructureType => array_filter( + $elements = $this->elements($base), + static fn(Physical $element): bool => !$element instanceof IdentityPhysical, + ) === [] + ? new IdentityPhysical() + : new StructPhysical($elements), + $base instanceof NullType => new NullPhysical(), + $base instanceof MixedType => throw new InvalidArgumentException( + 'a mixed element has no column kind, declare it or use json', + ), + default => throw new InvalidArgumentException(sprintf('type %s has no column kind', $base->toString())), + }; + } + + /** + * @param StructureType $type + * + * @return array + */ + public function elements(StructureType $type): array + { + $elements = []; + + foreach ($type->elements() as $element) { + $elements[$element->name] = $this->type($element->type); + } + + return $elements; + } +} diff --git a/src/core/etl/src/Flow/ETL/Column/Physical/PhysicalKind.php b/src/core/etl/src/Flow/ETL/Column/Physical/PhysicalKind.php new file mode 100644 index 0000000000..019db9a397 --- /dev/null +++ b/src/core/etl/src/Flow/ETL/Column/Physical/PhysicalKind.php @@ -0,0 +1,100 @@ + $type + */ + public function accepts(Type $type, mixed $physical): bool + { + if ($physical === null) { + return true; + } + + $base = $type instanceof OptionalType ? $type->base() : $type; + + return match (true) { + $base instanceof IntegerType, + $base instanceof PositiveIntegerType, + $base instanceof DateTimeType, + $base instanceof DateType, + $base instanceof TimeType, + => is_int($physical), + $base instanceof FloatType => is_float($physical), + $base instanceof BooleanType => is_bool($physical), + $base instanceof UuidType => is_string($physical) && strlen($physical) === 16, + $base instanceof NullType => true, + $base instanceof ListType => is_array($physical) && $this->all($base->element(), $physical), + $base instanceof MapType => is_array($physical) && $this->all($base->value(), $physical), + $base instanceof StructureType => is_array($physical) && $this->elements($base, $physical), + $base instanceof ArrayType => is_array($physical), + default => is_string($physical), + }; + } + + /** + * @param Type $type + * @param array $physicals + */ + public function all(Type $type, array $physicals): bool + { + // @mago-ignore analysis:mixed-assignment + foreach ($physicals as $physical) { + if (!$this->accepts($type, $physical)) { + return false; + } + } + + return true; + } + + /** + * @param StructureType $type + * @param array $physical + */ + public function elements(StructureType $type, array $physical): bool + { + foreach ($type->elements() as $element) { + if ( + array_key_exists($element->name, $physical) + && !$this->accepts($element->type, $physical[$element->name]) + ) { + return false; + } + } + + return true; + } +} diff --git a/src/core/etl/src/Flow/ETL/Column/Physical/StructPhysical.php b/src/core/etl/src/Flow/ETL/Column/Physical/StructPhysical.php new file mode 100644 index 0000000000..776d42e6f4 --- /dev/null +++ b/src/core/etl/src/Flow/ETL/Column/Physical/StructPhysical.php @@ -0,0 +1,77 @@ + $elements in element order + */ + public function __construct( + private array $elements, + ) {} + + public function toPhysical(mixed $value): mixed + { + assert(is_array($value)); + + $physicals = []; + + foreach ($this->elements as $name => $element) { + if (array_key_exists($name, $value)) { + $physicals[$name] = $value[$name] === null ? null : $element->toPhysical($value[$name]); + } + } + + return $physicals; + } + + public function fromPhysical(mixed $physical): mixed + { + assert(is_array($physical)); + + $values = []; + + foreach ($this->elements as $name => $element) { + if (array_key_exists($name, $physical)) { + $values[$name] = $physical[$name] === null ? null : $element->fromPhysical($physical[$name]); + } + } + + return $values; + } + + public function fromPhysicalAll(array $physicals): array + { + $values = []; + + // @mago-ignore analysis:mixed-assignment + foreach ($physicals as $physical) { + if ($physical === null) { + $values[] = null; + + continue; + } + + assert(is_array($physical)); + + $value = []; + + foreach ($this->elements as $name => $element) { + if (array_key_exists($name, $physical)) { + $value[$name] = $physical[$name] === null ? null : $element->fromPhysical($physical[$name]); + } + } + + $values[] = $value; + } + + return $values; + } +} diff --git a/src/core/etl/src/Flow/ETL/Column/Physical/TimePhysical.php b/src/core/etl/src/Flow/ETL/Column/Physical/TimePhysical.php new file mode 100644 index 0000000000..1f8db67dbd --- /dev/null +++ b/src/core/etl/src/Flow/ETL/Column/Physical/TimePhysical.php @@ -0,0 +1,94 @@ +y !== 0 || $value->m !== 0) { + throw new CastingException( + $value, + type_time(), + reason: "Relative DateInterval (with months/years) can't be stored in a time column", + ); + } + + $seconds = ((($value->d * 24) + $value->h) * 3600) + ($value->i * 60) + $value->s; + // an i64 overflow yields a float at runtime + // @mago-ignore analysis:redundant-type-comparison,redundant-condition + $us = is_int($seconds) ? ($seconds * 1_000_000) + (int) round($value->f * 1_000_000) : null; + + // @mago-ignore analysis:impossible-condition,redundant-type-comparison + if (!is_int($us)) { + throw new CastingException($value, type_time(), reason: 'outside the microsecond range of a time column'); + } + + return $value->invert === 1 ? -$us : $us; + } + + public function fromPhysical(mixed $physical): mixed + { + assert(is_int($physical)); + + $magnitude = abs($physical); + $interval = new DateInterval(sprintf( + 'PT%dH%dM%dS', + intdiv($magnitude, 3_600_000_000), + intdiv($magnitude, 60_000_000) % 60, + intdiv($magnitude, 1_000_000) % 60, + )); + // @mago-ignore analysis:invalid-property-write + $interval->f = ($magnitude % 1_000_000) / 1_000_000; + // @mago-ignore analysis:invalid-property-write + $interval->invert = $physical < 0 ? 1 : 0; + + return $interval; + } + + public function fromPhysicalAll(array $physicals): array + { + $values = []; + + // @mago-ignore analysis:mixed-assignment + foreach ($physicals as $physical) { + if ($physical === null) { + $values[] = null; + + continue; + } + + assert(is_int($physical)); + + $magnitude = abs($physical); + $interval = new DateInterval(sprintf( + 'PT%dH%dM%dS', + intdiv($magnitude, 3_600_000_000), + intdiv($magnitude, 60_000_000) % 60, + intdiv($magnitude, 1_000_000) % 60, + )); + // @mago-ignore analysis:invalid-property-write + $interval->f = ($magnitude % 1_000_000) / 1_000_000; + // @mago-ignore analysis:invalid-property-write + $interval->invert = $physical < 0 ? 1 : 0; + $values[] = $interval; + } + + return $values; + } +} diff --git a/src/core/etl/src/Flow/ETL/Column/Physical/TimeZonePhysical.php b/src/core/etl/src/Flow/ETL/Column/Physical/TimeZonePhysical.php new file mode 100644 index 0000000000..e4d2a4e73d --- /dev/null +++ b/src/core/etl/src/Flow/ETL/Column/Physical/TimeZonePhysical.php @@ -0,0 +1,44 @@ +getName(); + } + + public function fromPhysical(mixed $physical): mixed + { + assert(is_string($physical)); + + return $this->zones->get($physical); + } + + public function fromPhysicalAll(array $physicals): array + { + $values = []; + + // @mago-ignore analysis:mixed-assignment + foreach ($physicals as $physical) { + assert($physical === null || is_string($physical)); + $values[] = $physical === null ? null : $this->zones->get($physical); + } + + return $values; + } +} diff --git a/src/core/etl/src/Flow/Floe/Decoding/TimeZones.php b/src/core/etl/src/Flow/ETL/Column/Physical/TimeZones.php similarity index 89% rename from src/core/etl/src/Flow/Floe/Decoding/TimeZones.php rename to src/core/etl/src/Flow/ETL/Column/Physical/TimeZones.php index c247298c3c..4614933b18 100644 --- a/src/core/etl/src/Flow/Floe/Decoding/TimeZones.php +++ b/src/core/etl/src/Flow/ETL/Column/Physical/TimeZones.php @@ -2,7 +2,7 @@ declare(strict_types=1); -namespace Flow\Floe\Decoding; +namespace Flow\ETL\Column\Physical; use DateTimeZone; diff --git a/src/core/etl/src/Flow/ETL/Column/Physical/UuidPhysical.php b/src/core/etl/src/Flow/ETL/Column/Physical/UuidPhysical.php new file mode 100644 index 0000000000..0de84e7668 --- /dev/null +++ b/src/core/etl/src/Flow/ETL/Column/Physical/UuidPhysical.php @@ -0,0 +1,42 @@ +toString())); + } + + public function fromPhysical(mixed $physical): mixed + { + assert(is_string($physical)); + + return Uuid::fromBytes($physical); + } + + public function fromPhysicalAll(array $physicals): array + { + $values = []; + + // @mago-ignore analysis:mixed-assignment + foreach ($physicals as $physical) { + assert($physical === null || is_string($physical)); + $values[] = $physical === null ? null : Uuid::fromBytes($physical); + } + + return $values; + } +} diff --git a/src/core/etl/src/Flow/ETL/Column/Physical/XmlCharacterReferences.php b/src/core/etl/src/Flow/ETL/Column/Physical/XmlCharacterReferences.php new file mode 100644 index 0000000000..ad0eee99a2 --- /dev/null +++ b/src/core/etl/src/Flow/ETL/Column/Physical/XmlCharacterReferences.php @@ -0,0 +1,43 @@ +||&#(x[0-9A-Fa-f]+|[0-9]+);/s', + static function (array $match): string { + if (!array_key_exists(1, $match)) { + return $match[0]; + } + + $codepoint = str_starts_with($match[1], 'x') ? hexdec(substr($match[1], 1)) : intval($match[1]); + + return $codepoint < 0x80 ? $match[0] : html_entity_decode($match[0], ENT_XML1 | ENT_QUOTES, 'UTF-8'); + }, + $xml, + ); + } +} diff --git a/src/core/etl/src/Flow/ETL/Column/Physical/XmlDocumentPhysical.php b/src/core/etl/src/Flow/ETL/Column/Physical/XmlDocumentPhysical.php new file mode 100644 index 0000000000..9f97ee11c6 --- /dev/null +++ b/src/core/etl/src/Flow/ETL/Column/Physical/XmlDocumentPhysical.php @@ -0,0 +1,89 @@ +\n"; + + public function physical(string $outerXml): string + { + return self::DECLARATION . $outerXml . "\n"; + } + + /** + * The outer XML of bytes physical() produced, null for any other bytes: an older declaration, or a comment, PI, + * DOCTYPE or declaration beside the root. + */ + public function text(string $physical): ?string + { + if (!str_starts_with($physical, self::DECLARATION) || !str_ends_with($physical, ">\n")) { + return null; + } + + $outer = substr($physical, strlen(self::DECLARATION), -1); + + return $outer[0] === '<' + && !in_array($outer[1] ?? '', ['!', '?'], true) + && !str_ends_with($outer, '-->') + && !str_ends_with($outer, '?>') + && !str_ends_with($outer, ']]>') + ? $outer + : null; + } + + public function toPhysical(mixed $value): mixed + { + assert($value instanceof DOMDocument); + + $xml = $value->saveXML(); + + if ($xml === false) { + throw new InvalidArgumentException('Floe failed to convert DOMDocument to XML string'); + } + + return (new XmlCharacterReferences())->decoded($xml); + } + + public function fromPhysical(mixed $physical): mixed + { + assert(is_string($physical)); + + $document = new DOMDocument(); + + if (!@$document->loadXML($physical)) { + throw new InvalidArgumentException(sprintf('Floe failed to restore DOMDocument from "%s"', $physical)); + } + + // libxml2 before 2.13 writes non-ASCII attribute characters of a document without an encoding as references + $document->encoding ??= 'UTF-8'; + + return $document; + } + + public function fromPhysicalAll(array $physicals): array + { + $values = []; + + // @mago-ignore analysis:mixed-assignment + foreach ($physicals as $physical) { + $values[] = $physical === null ? null : $this->fromPhysical($physical); + } + + return $values; + } +} diff --git a/src/core/etl/src/Flow/ETL/Column/Physical/XmlElementPhysical.php b/src/core/etl/src/Flow/ETL/Column/Physical/XmlElementPhysical.php new file mode 100644 index 0000000000..aa443b8032 --- /dev/null +++ b/src/core/etl/src/Flow/ETL/Column/Physical/XmlElementPhysical.php @@ -0,0 +1,104 @@ +position->path($value); + $xml = $value->ownerDocument?->saveXML($path === null ? $value : null); + + if ($xml === null || $xml === false) { + throw new InvalidArgumentException('Floe failed to convert DOMElement to XML string'); + } + + $xml = (new XmlCharacterReferences())->decoded($xml); + + return $path === null ? $xml : $this->position->physical($path, $this->canonical($value), $xml); + } + + /** @var \Dom\Element $value */ + $path = $this->position->path($value); + // @mago-ignore analysis:non-existent-method + // @mago-ignore analysis:mixed-assignment + $markup = $value->ownerDocument?->saveXml($path === null ? $value : null); + assert(is_string($markup)); + $markup = (new XmlCharacterReferences())->decoded($markup); + + return $path === null ? $markup : $this->position->physical($path, $this->canonical($value), $markup); + } + + public function fromPhysical(mixed $physical): mixed + { + assert(is_string($physical)); + + $path = $this->position->pathOf($physical); + $root = type_instance_of(DOMDocument::class)->assert($this->document->fromPhysical( + $path === null ? $physical : $this->position->documentOf($physical), + ))->documentElement; + $element = $path === null ? $root : $this->position->element($root, $path); + + if ($element === null) { + throw new InvalidArgumentException(sprintf('Floe failed to restore DOMElement from "%s"', $physical)); + } + + return $element; + } + + public function markup(string $physical): string + { + return ( + $this->position->markupOf($physical) ?? $this->canonical(type_instance_of( + DOMElement::class, + )->assert($this->fromPhysical($physical))) + ); + } + + /** + * @param DOMElement|\Dom\Element $element + */ + public function canonical(object $element): string + { + $markup = $element->C14N(); + + if (!is_string($markup)) { + throw new InvalidArgumentException('Floe failed to canonicalize XML element'); + } + + return $markup; + } + + public function fromPhysicalAll(array $physicals): array + { + $values = []; + + // @mago-ignore analysis:mixed-assignment + foreach ($physicals as $physical) { + $values[] = $physical === null ? null : $this->fromPhysical($physical); + } + + return $values; + } +} diff --git a/src/core/etl/src/Flow/ETL/Column/Retype.php b/src/core/etl/src/Flow/ETL/Column/Retype.php new file mode 100644 index 0000000000..a015b7b749 --- /dev/null +++ b/src/core/etl/src/Flow/ETL/Column/Retype.php @@ -0,0 +1,119 @@ + $from + * @param Type $to + * + * @throws InvalidArgumentException + */ + public function assert(Type $from, Type $to, int $nullCount): void + { + if ($from instanceof OptionalType && !$to instanceof OptionalType && $nullCount > 0) { + throw new InvalidArgumentException(sprintf('%d null values under a required type', $nullCount)); + } + + if (!$this->sameKind($from, $to)) { + throw new InvalidArgumentException(sprintf( + '%s and %s are different column kinds', + $from->toString(), + $to->toString(), + )); + } + } + + /** + * Whether withType($to) on a column of $from can only fail on top-level nulls: the same kind at every level, and no + * nested level goes from optional to required (list element, map key / value, structure element by name - + * optional flag and type). + * + * @param Type $from + * @param Type $to + */ + public function proves(Type $from, Type $to): bool + { + if (!$this->sameKind($from, $to)) { + return false; + } + + $fromBase = $from instanceof OptionalType ? $from->base() : $from; + $toBase = $to instanceof OptionalType ? $to->base() : $to; + + if ($fromBase instanceof ListType && $toBase instanceof ListType) { + return $this->provesNested($fromBase->element(), $toBase->element()); + } + + if ($fromBase instanceof MapType && $toBase instanceof MapType) { + return ( + $this->provesNested($fromBase->key(), $toBase->key()) + && $this->provesNested($fromBase->value(), $toBase->value()) + ); + } + + if ($fromBase instanceof StructureType && $toBase instanceof StructureType) { + foreach ($toBase->elements() as $element) { + $own = $fromBase->element($element->name); + + if ( + $own === null + || $own->optional && !$element->optional + || !$this->provesNested($own->type, $element->type) + ) { + return false; + } + } + } + + return true; + } + + /** + * proves() one level down, where a null cannot be counted: optional to required is never proven. + * + * @param Type $from + * @param Type $to + */ + public function provesNested(Type $from, Type $to): bool + { + return !($from instanceof OptionalType && !$to instanceof OptionalType) && $this->proves($from, $to); + } + + /** + * The column kind rule: the same type class, an enum of the same class, a structure of the same element names. + * + * @param Type $from + * @param Type $to + */ + public function sameKind(Type $from, Type $to): bool + { + $fromBase = $from instanceof OptionalType ? $from->base() : $from; + $toBase = $to instanceof OptionalType ? $to->base() : $to; + + return match (true) { + $fromBase::class !== $toBase::class => false, + $fromBase instanceof EnumType && $toBase instanceof EnumType => $fromBase->class === $toBase->class, + $fromBase instanceof StructureType && $toBase instanceof StructureType => array_map( + static fn(StructureElement $element): int|string => $element->name, + $fromBase->elements(), + ) === array_map(static fn(StructureElement $element): int|string => $element->name, $toBase->elements()), + default => true, + }; + } +} diff --git a/src/core/etl/src/Flow/ETL/Column/RoundTripPrecision.php b/src/core/etl/src/Flow/ETL/Column/RoundTripPrecision.php new file mode 100644 index 0000000000..7a38f4d352 --- /dev/null +++ b/src/core/etl/src/Flow/ETL/Column/RoundTripPrecision.php @@ -0,0 +1,40 @@ +previous = (string) $current; + } + + public function restore(): void + { + if ($this->previous !== null) { + ini_set('serialize_precision', $this->previous); + $this->previous = null; + } + } +} diff --git a/src/core/etl/src/Flow/ETL/Column/TextValues.php b/src/core/etl/src/Flow/ETL/Column/TextValues.php new file mode 100644 index 0000000000..76138623dc --- /dev/null +++ b/src/core/etl/src/Flow/ETL/Column/TextValues.php @@ -0,0 +1,459 @@ + $type + * @param list $physicals + * + * @return list + */ + public function of(Type $type, array $physicals): array + { + if ($this->keepsPhysical($type)) { + return $physicals; + } + + $bare = type_bare($type); + + if ($bare instanceof DateTimeType) { + /** @var list $instants */ + $instants = $physicals; + + return $this->dateTimes($bare, $instants, $this->dateTimeFormat); + } + + if ($bare instanceof DateType) { + /** @var list $days */ + $days = $physicals; + + return $this->dateTimes($bare, $days, $this->dateFormat); + } + + if ($bare instanceof UuidType) { + /** @var list $bytes */ + $bytes = $physicals; + + return $this->uuids($bytes); + } + + if ($bare instanceof JsonType) { + /** @var list $texts */ + $texts = $physicals; + $values = []; + + foreach ($texts as $text) { + $values[] = $text === null ? null : json_decode($text, false, 512, JSON_THROW_ON_ERROR); + } + + return $values; + } + + if ($bare instanceof HTMLElementType || $bare instanceof XMLElementType) { + $elements = $bare instanceof HTMLElementType ? new HtmlElementPhysical() : new XmlElementPhysical(); + $values = []; + + // @mago-ignore analysis:mixed-assignment + foreach ($physicals as $physical) { + $values[] = $physical === null ? null : $elements->markup(type_string()->assert($physical)); + } + + return $values; + } + + if ($bare instanceof XMLType) { + $nodes = (new PhysicalFor())->type($bare); + $documents = new XmlDocumentPhysical(); + $references = new XmlCharacterReferences(); + $values = []; + + // @mago-ignore analysis:mixed-assignment + foreach ($physicals as $physical) { + if ($physical === null) { + $values[] = null; + + continue; + } + + $text = $documents->text(type_string()->assert($physical)); + + if ($text === null) { + $document = type_instance_of(DOMDocument::class)->assert($nodes->fromPhysical($physical)); + $text = $document->saveXML($document->documentElement); + } + + if ($text === false) { + throw new RuntimeException('Failed to serialize XML document.'); + } + + $values[] = $references->decoded($text); + } + + return $values; + } + + /** @var list> $containers */ + $containers = $physicals; + $elements = new Elements(); + + if ($bare instanceof ListType) { + return $elements->shape($this->of($bare->element(), $elements->flatten($containers)), $containers); + } + + if ($bare instanceof MapType) { + if (!$this->keepsPhysical($bare->value())) { + $containers = $elements->shape($this->of($bare->value(), $elements->flatten($containers)), $containers); + } + } elseif ($bare instanceof StructureType) { + foreach ($bare->elements() as $element) { + if (!$this->keepsPhysical($element->type)) { + $containers = $elements->withField( + $containers, + $element->name, + $this->of($element->type, $elements->field($containers, $element->name)), + ); + } + } + } + + $values = []; + + foreach ($containers as $container) { + $values[] = $container === null ? null : (object) $container; + } + + return $values; + } + + /** + * @param Type $type + */ + public function keepsPhysical(Type $type): bool + { + $bare = type_bare($type); + + return match (true) { + $bare instanceof DateTimeType, + $bare instanceof DateType, + $bare instanceof UuidType, + $bare instanceof JsonType, + $bare instanceof XMLType, + $bare instanceof XMLElementType, + $bare instanceof HTMLElementType, + $bare instanceof MapType, + $bare instanceof StructureType, + => false, + $bare instanceof ListType => $this->keepsPhysical($bare->element()), + default => true, + }; + } + + /** + * The values a database driver binds for a column: markup (xml, xml_element, html, html_element) as the text its + * physicals render, every other type as its logical values - the driver's types convert those themselves. + * + * @param Type $type + * + * @return list + */ + public function bindable(Type $type, Column $column): array + { + $bare = type_bare($type); + + return $bare instanceof XMLType + || $bare instanceof XMLElementType + || $bare instanceof HTMLType + || $bare instanceof HTMLElementType + ? $this->texts($type, $column->physicals()) + : $column->values(); + } + + /** + * @param Type $type + * @param list $physicals + * + * @return list + */ + public function texts(Type $type, array $physicals): array + { + $bare = type_bare($type); + + if ($bare instanceof JsonType) { + /** @var list */ + return $physicals; + } + + if ($bare instanceof BooleanType) { + $texts = []; + + // @mago-ignore analysis:mixed-assignment + foreach ($physicals as $physical) { + $texts[] = $physical === null ? null : ($physical === true ? 'true' : 'false'); + } + + return $texts; + } + + if ($bare instanceof FloatType) { + /** @var list $floats */ + $floats = $physicals; + + return $this->floats($floats); + } + + if ($bare instanceof ListType || $bare instanceof MapType || $bare instanceof StructureType) { + $texts = []; + $precision = new RoundTripPrecision(); + $precision->force(); + + try { + // @mago-ignore analysis:mixed-assignment + foreach ($this->of($type, $physicals) as $value) { + $texts[] = $value === null + ? null + : json_encode($value, JSON_PRESERVE_ZERO_FRACTION | JSON_THROW_ON_ERROR); + } + } finally { + $precision->restore(); + } + + return $texts; + } + + if ($bare instanceof IntegerType || $bare instanceof TimeType) { + $texts = []; + + /** @var list $integers */ + $integers = $physicals; + + foreach ($integers as $integer) { + $texts[] = $integer === null ? null : (string) $integer; + } + + return $texts; + } + + /** @var list */ + return $this->of($type, $physicals); + } + + /** + * @param list $physicals + * + * @return list + */ + public function dateTimes(DateTimeType|DateType $type, array $physicals, string $format): array + { + $pattern = new DateFormatPattern($format); + $texts = []; + + // gmdate() names the UTC zone "GMT" under T, where a DateTimeInterface in UTC names it "UTC" + if (($type instanceof DateType || $type->zoneName() === 'UTC') && !in_array('T', $pattern->letters(), true)) { + $segments = $pattern->segments(); + + foreach ($physicals as $physical) { + if ($physical === null) { + $texts[] = null; + + continue; + } + + if ($type instanceof DateType) { + $second = $physical * 86_400; + $micro = 0; + } else { + $micro = $physical % 1_000_000; + + if ($micro < 0) { + $micro += 1_000_000; + } + + $second = intdiv($physical - $micro, 1_000_000); + } + + $text = ''; + + foreach ($segments as [$segment, $letter]) { + $text .= gmdate($segment, $second) . match ($letter) { + 'u' => sprintf('%06d', $micro), + 'v' => sprintf('%03d', intdiv($micro, 1000)), + '' => '', + }; + } + + $texts[] = $text; + } + + return $texts; + } + + $values = (new PhysicalFor())->type($type); + + foreach ($physicals as $physical) { + $texts[] = $physical === null + ? null + : type_instance_of(DateTimeInterface::class)->assert($values->fromPhysical($physical))->format($format); + } + + return $texts; + } + + /** + * @param list $physicals 16 raw bytes each + * + * @return list + */ + public function uuids(array $physicals): array + { + if ($physicals === []) { + return []; + } + + if (!in_array(null, $physicals, true)) { + /** @var list */ + return substr_replace( + substr_replace( + substr_replace( + substr_replace(str_split(bin2hex(implode('', $physicals)), 32), '-', 8, 0), + '-', + 13, + 0, + ), + '-', + 18, + 0, + ), + '-', + 23, + 0, + ); + } + + $texts = []; + + foreach ($physicals as $physical) { + if ($physical === null) { + $texts[] = null; + + continue; + } + + $hex = bin2hex($physical); + $texts[] = + substr($hex, 0, 8) + . '-' + . substr($hex, 8, 4) + . '-' + . substr($hex, 12, 4) + . '-' + . substr($hex, 16, 4) + . '-' + . substr($hex, 20); + } + + return $texts; + } + + /** + * @param list $physicals + * + * @return list + */ + public function floats(array $physicals): array + { + if ($physicals === []) { + return []; + } + + $precision = new RoundTripPrecision(); + $precision->force(); + + try { + try { + $texts = explode(',', substr( + json_encode($physicals, JSON_PRESERVE_ZERO_FRACTION | JSON_THROW_ON_ERROR), + 1, + -1, + )); + + foreach (array_keys($texts, 'null', true) as $i) { + $texts[$i] = null; + } + + return $texts; + } catch (JsonException) { + $texts = []; + + foreach ($physicals as $physical) { + $texts[] = match (true) { + $physical === null => null, + default => NonFiniteFloat::text($physical) ?? json_encode( + $physical, + JSON_PRESERVE_ZERO_FRACTION | JSON_THROW_ON_ERROR, + ), + }; + } + + return $texts; + } + } finally { + $precision->restore(); + } + } +} diff --git a/src/core/etl/src/Flow/ETL/Column/ValueColumn.php b/src/core/etl/src/Flow/ETL/Column/ValueColumn.php new file mode 100644 index 0000000000..57c5a8d599 --- /dev/null +++ b/src/core/etl/src/Flow/ETL/Column/ValueColumn.php @@ -0,0 +1,107 @@ + $values logical PHP values, null = null + */ + public function __construct( + private array $values, + ) {} + + public function at(int $i): mixed + { + return $this->values[$i]; + } + + public function concat(Column ...$others): Column + { + throw new InvalidArgumentException( + 'An untyped function result (mixed) cannot be concatenated; it exists only inside function evaluation', + ); + } + + public function count(): int + { + return count($this->values); + } + + public function encode(): array + { + throw new RuntimeException( + 'An untyped function result (mixed) cannot be encoded; it exists only inside function evaluation', + ); + } + + public function isNull(int $i): bool + { + return $this->values[$i] === null; + } + + public function nullCount(): int + { + return count(array_filter($this->values, static fn(mixed $value): bool => $value === null)); + } + + public function physicals(): array + { + return $this->values; + } + + public function slice(int $offset, int $length): Column + { + return new self(array_slice($this->values, $offset, $length)); + } + + public function take(array $indices): Column + { + $values = []; + + foreach ($indices as $index) { + $values[] = $this->values[$index]; + } + + return new self($values); + } + + /** + * @return MixedType + */ + public function type(): Type + { + return type_mixed(); + } + + public function value(int $i): mixed + { + return $this->values[$i]; + } + + public function values(): array + { + return $this->values; + } + + public function withType(Type $type): Column + { + throw new InvalidArgumentException(sprintf( + 'An untyped function result (mixed) cannot be retyped to %s; it exists only inside function evaluation', + $type->toString(), + )); + } +} diff --git a/src/core/etl/src/Flow/ETL/Config.php b/src/core/etl/src/Flow/ETL/Config.php index 11d2f8015e..da08289996 100644 --- a/src/core/etl/src/Flow/ETL/Config.php +++ b/src/core/etl/src/Flow/ETL/Config.php @@ -5,6 +5,7 @@ namespace Flow\ETL; use Flow\Calculator\Calculator; +use Flow\ETL\Column\Backend; use Flow\ETL\Config\Cache\CacheConfig; use Flow\ETL\Config\ConfigBuilder; use Flow\ETL\Config\Grouping\HashGroupByConfig; @@ -13,7 +14,6 @@ use Flow\ETL\Config\Sort\ExternalSortConfig; use Flow\ETL\Config\Sort\MemorySortConfig; use Flow\ETL\Config\Telemetry\TelemetryConfig; -use Flow\ETL\Row\Hydrator; use Flow\Serializer\Serializer; use Psr\Clock\ClockInterface; @@ -25,9 +25,6 @@ { private Planner $planner; - /** - * @param Hydrator $hydrator - */ public function __construct( private string $id, private string $name, @@ -36,7 +33,6 @@ public function __construct( private ClockInterface $clock, private Optimizer $optimizer, private Executor $executor, - private Hydrator $hydrator, public CacheConfig $cache, public MemorySortConfig|ExternalSortConfig $sort, private ?Analyze $analyze, @@ -44,6 +40,7 @@ public function __construct( public HashGroupByConfig $grouping, public HashJoinConfig $join, public HashRepartitionConfig $repartition, + private Backend $backend, private Calculator $calculator = new Calculator(), private RandomValueGenerator $randomValueGenerator = new NativePHPRandomValueGenerator(), ) { @@ -65,6 +62,11 @@ public function analyze(): ?Analyze return $this->analyze; } + public function backend(): Backend + { + return $this->backend; + } + public function calculator(): Calculator { return $this->calculator; @@ -80,14 +82,6 @@ public function executor(): Executor return $this->executor; } - /** - * @return Hydrator - */ - public function hydrator(): Hydrator - { - return $this->hydrator; - } - public function id(): string { return $this->id; diff --git a/src/core/etl/src/Flow/ETL/Config/Bucketing/BucketingConfigBuilder.php b/src/core/etl/src/Flow/ETL/Config/Bucketing/BucketingConfigBuilder.php index b1df735e32..500d2aa490 100644 --- a/src/core/etl/src/Flow/ETL/Config/Bucketing/BucketingConfigBuilder.php +++ b/src/core/etl/src/Flow/ETL/Config/Bucketing/BucketingConfigBuilder.php @@ -6,6 +6,7 @@ use Flow\ETL\Bucketing\BucketsStorage; use Flow\ETL\Bucketing\Storage\FilesystemBuckets; +use Flow\ETL\Column\Backend; use Flow\ETL\Exception\InvalidArgumentException; use Flow\Filesystem\Local\NativeLocalFilesystem; use Flow\Filesystem\Path; @@ -49,12 +50,13 @@ public function batchSize(int $batchSize): self return $this; } - public function build(Path $spillRoot): BucketingConfig + public function build(Path $spillRoot, Backend $backend): BucketingConfig { return new BucketingConfig( $this->storage ?? new FilesystemBuckets( new NativeLocalFilesystem(), $spillRoot->suffix($this->spillDirectory), + $backend, $this->batchSize, ), $this->bucketsCount, diff --git a/src/core/etl/src/Flow/ETL/Config/ConfigBuilder.php b/src/core/etl/src/Flow/ETL/Config/ConfigBuilder.php index d6f23317ea..cc3d0ff8d1 100644 --- a/src/core/etl/src/Flow/ETL/Config/ConfigBuilder.php +++ b/src/core/etl/src/Flow/ETL/Config/ConfigBuilder.php @@ -7,6 +7,8 @@ use Flow\Clock\SystemClock; use Flow\ETL\Analyze; use Flow\ETL\Cache; +use Flow\ETL\Column\AdaptiveBackend; +use Flow\ETL\Column\Backend; use Flow\ETL\Config; use Flow\ETL\Config\Cache\CacheConfigBuilder; use Flow\ETL\Config\Grouping\GroupByAlgorithmBuilder; @@ -23,8 +25,6 @@ use Flow\ETL\NativePHPRandomValueGenerator; use Flow\ETL\Optimizer; use Flow\ETL\RandomValueGenerator; -use Flow\ETL\Row\AdaptiveRowHydrator; -use Flow\ETL\Row\Hydrator; use Flow\Filesystem\Path; use Flow\Floe\FloeSerializer; use Flow\Serializer\Serializer; @@ -40,12 +40,9 @@ final class ConfigBuilder private ?ClockInterface $clock; - private ?GroupByAlgorithmBuilder $groupBy; + private ?Backend $backend; - /** - * @var null|Hydrator - */ - private ?Hydrator $hydrator; + private ?GroupByAlgorithmBuilder $groupBy; private ?JoinAlgorithmBuilder $join; @@ -74,7 +71,7 @@ public function __construct() $this->id = null; $this->name = null; $this->serializer = null; - $this->hydrator = null; + $this->backend = null; $this->optimizer = null; $this->executor = null; $this->clock = null; @@ -98,19 +95,24 @@ public function analyze(Analyze $analyze): self return $this; } + public function backend(Backend $backend): self + { + $this->backend = $backend; + + return $this; + } + public function build(): Config { $id = $this->id ??= 'flow-php-' . $this->randomValueGenerator->string(32); + // built here, not in the constructor: a backend() opt-out must not depend on the default's extension check + $backend = $this->backend ?? new AdaptiveBackend(); $this->optimizer ??= Optimizer::default(); $this->executor ??= new Executor(); - $this->hydrator ??= new AdaptiveRowHydrator(); - // the default serializer shares the context hydrator - one source of Row objects - $this->serializer ??= new FloeSerializer(hydrator: $this->hydrator); - - $serializer = $this->serializer; + // built per build(), not stored: a later backend() call must reach the next build's serializer + $serializer = $this->serializer ?? new FloeSerializer($backend); $optimizer = $this->optimizer; $executor = $this->executor; - $hydrator = $this->hydrator; $dataframeName = $this->name ?? 'flow_dataframe'; $cacheConfig = $this->cache->build($serializer, $this->telemetryConfig, $dataframeName); @@ -123,15 +125,18 @@ public function build(): Config $this->getClock(), $optimizer, $executor, - $hydrator, $cacheConfig, - ($this->sort ?? new ExternalSortBuilder())->build($cacheConfig->localFilesystemCacheDir), + ($this->sort ?? new ExternalSortBuilder())->build($cacheConfig->localFilesystemCacheDir, $backend), $this->analyze, $this->telemetryConfig ?? TelemetryConfig::default($this->getClock()), - ($this->groupBy ?? new HashGroupByBuilder())->build($cacheConfig->localFilesystemCacheDir), - ($this->join ?? new HashJoinBuilder())->build($cacheConfig->localFilesystemCacheDir), - ($this->repartition ?? new HashRepartitionBuilder())->build($cacheConfig->localFilesystemCacheDir), + ($this->groupBy ?? new HashGroupByBuilder())->build($cacheConfig->localFilesystemCacheDir, $backend), + ($this->join ?? new HashJoinBuilder())->build($cacheConfig->localFilesystemCacheDir, $backend), + ($this->repartition ?? new HashRepartitionBuilder())->build( + $cacheConfig->localFilesystemCacheDir, + $backend, + ), randomValueGenerator: $this->randomValueGenerator, + backend: $backend, ); } @@ -163,16 +168,6 @@ public function groupBy(GroupByAlgorithmBuilder $algorithm): self return $this; } - /** - * @param Hydrator $hydrator - */ - public function hydrator(Hydrator $hydrator): self - { - $this->hydrator = $hydrator; - - return $this; - } - public function id(string $id): self { $this->id = $id; diff --git a/src/core/etl/src/Flow/ETL/Config/Grouping/GroupByAlgorithmBuilder.php b/src/core/etl/src/Flow/ETL/Config/Grouping/GroupByAlgorithmBuilder.php index 0b7453a799..8bcc145f44 100644 --- a/src/core/etl/src/Flow/ETL/Config/Grouping/GroupByAlgorithmBuilder.php +++ b/src/core/etl/src/Flow/ETL/Config/Grouping/GroupByAlgorithmBuilder.php @@ -4,9 +4,10 @@ namespace Flow\ETL\Config\Grouping; +use Flow\ETL\Column\Backend; use Flow\Filesystem\Path; interface GroupByAlgorithmBuilder { - public function build(Path $spillRoot): HashGroupByConfig; + public function build(Path $spillRoot, Backend $backend): HashGroupByConfig; } diff --git a/src/core/etl/src/Flow/ETL/Config/Grouping/HashGroupByBuilder.php b/src/core/etl/src/Flow/ETL/Config/Grouping/HashGroupByBuilder.php index 193f180cef..e0f2b09271 100644 --- a/src/core/etl/src/Flow/ETL/Config/Grouping/HashGroupByBuilder.php +++ b/src/core/etl/src/Flow/ETL/Config/Grouping/HashGroupByBuilder.php @@ -5,13 +5,19 @@ namespace Flow\ETL\Config\Grouping; use Flow\ETL\Bucketing\BucketsStorage; +use Flow\ETL\Column\Backend; use Flow\ETL\Config\Bucketing\BucketingConfigBuilder; +use Flow\ETL\Config\MemoryLimit; +use Flow\ETL\Dataset\Memory\Unit; +use Flow\ETL\Exception\InvalidArgumentException; use Flow\Filesystem\Path; final class HashGroupByBuilder implements GroupByAlgorithmBuilder { private readonly BucketingConfigBuilder $bucketing; + private ?Unit $memoryLimit = null; + public function __construct() { $this->bucketing = new BucketingConfigBuilder('/flow-php-group-by/', 64); @@ -27,9 +33,12 @@ public function batchSize(int $batchSize): self return $this; } - public function build(Path $spillRoot): HashGroupByConfig + public function build(Path $spillRoot, Backend $backend): HashGroupByConfig { - return new HashGroupByConfig($this->bucketing->build($spillRoot)); + return new HashGroupByConfig( + $this->bucketing->build($spillRoot, $backend), + $this->memoryLimit ?? MemoryLimit::default(), + ); } /** @@ -42,6 +51,20 @@ public function bucketsCount(int $bucketsCount): self return $this; } + /** + * The process memory past which buckets move to the storage; defaults to MemoryLimit::default(). + */ + public function memoryLimit(Unit $memoryLimit): self + { + if ($memoryLimit->inBytes() < 1) { + throw new InvalidArgumentException('Group by memory limit must be greater than 0 bytes'); + } + + $this->memoryLimit = $memoryLimit; + + return $this; + } + public function storage(BucketsStorage $storage): self { $this->bucketing->storage($storage); diff --git a/src/core/etl/src/Flow/ETL/Config/Grouping/HashGroupByConfig.php b/src/core/etl/src/Flow/ETL/Config/Grouping/HashGroupByConfig.php index e3726c3629..6442002dd7 100644 --- a/src/core/etl/src/Flow/ETL/Config/Grouping/HashGroupByConfig.php +++ b/src/core/etl/src/Flow/ETL/Config/Grouping/HashGroupByConfig.php @@ -5,10 +5,12 @@ namespace Flow\ETL\Config\Grouping; use Flow\ETL\Config\Bucketing\BucketingConfig; +use Flow\ETL\Dataset\Memory\Unit; final readonly class HashGroupByConfig { public function __construct( public BucketingConfig $bucketing, + public Unit $memoryLimit, ) {} } diff --git a/src/core/etl/src/Flow/ETL/Config/Join/HashJoinBuilder.php b/src/core/etl/src/Flow/ETL/Config/Join/HashJoinBuilder.php index a8e4f238c5..77d9c775f4 100644 --- a/src/core/etl/src/Flow/ETL/Config/Join/HashJoinBuilder.php +++ b/src/core/etl/src/Flow/ETL/Config/Join/HashJoinBuilder.php @@ -5,13 +5,19 @@ namespace Flow\ETL\Config\Join; use Flow\ETL\Bucketing\BucketsStorage; +use Flow\ETL\Column\Backend; use Flow\ETL\Config\Bucketing\BucketingConfigBuilder; +use Flow\ETL\Config\MemoryLimit; +use Flow\ETL\Dataset\Memory\Unit; +use Flow\ETL\Exception\InvalidArgumentException; use Flow\Filesystem\Path; final class HashJoinBuilder implements JoinAlgorithmBuilder { private readonly BucketingConfigBuilder $bucketing; + private ?Unit $memoryLimit = null; + public function __construct() { $this->bucketing = new BucketingConfigBuilder('/flow-php-join/', 64); @@ -27,9 +33,12 @@ public function batchSize(int $batchSize): self return $this; } - public function build(Path $spillRoot): HashJoinConfig + public function build(Path $spillRoot, Backend $backend): HashJoinConfig { - return new HashJoinConfig($this->bucketing->build($spillRoot)); + return new HashJoinConfig( + $this->bucketing->build($spillRoot, $backend), + $this->memoryLimit ?? MemoryLimit::default(), + ); } /** @@ -42,6 +51,21 @@ public function bucketsCount(int $bucketsCount): self return $this; } + /** + * The process memory past which the build side and the buckets move to the storage; defaults to + * MemoryLimit::default(). + */ + public function memoryLimit(Unit $memoryLimit): self + { + if ($memoryLimit->inBytes() < 1) { + throw new InvalidArgumentException('Join memory limit must be greater than 0 bytes'); + } + + $this->memoryLimit = $memoryLimit; + + return $this; + } + public function storage(BucketsStorage $storage): self { $this->bucketing->storage($storage); diff --git a/src/core/etl/src/Flow/ETL/Config/Join/HashJoinConfig.php b/src/core/etl/src/Flow/ETL/Config/Join/HashJoinConfig.php index 3f73c02318..9249a875d3 100644 --- a/src/core/etl/src/Flow/ETL/Config/Join/HashJoinConfig.php +++ b/src/core/etl/src/Flow/ETL/Config/Join/HashJoinConfig.php @@ -5,10 +5,12 @@ namespace Flow\ETL\Config\Join; use Flow\ETL\Config\Bucketing\BucketingConfig; +use Flow\ETL\Dataset\Memory\Unit; final readonly class HashJoinConfig { public function __construct( public BucketingConfig $bucketing, + public Unit $memoryLimit, ) {} } diff --git a/src/core/etl/src/Flow/ETL/Config/Join/JoinAlgorithmBuilder.php b/src/core/etl/src/Flow/ETL/Config/Join/JoinAlgorithmBuilder.php index 84d653d62d..66e79241ca 100644 --- a/src/core/etl/src/Flow/ETL/Config/Join/JoinAlgorithmBuilder.php +++ b/src/core/etl/src/Flow/ETL/Config/Join/JoinAlgorithmBuilder.php @@ -4,9 +4,10 @@ namespace Flow\ETL\Config\Join; +use Flow\ETL\Column\Backend; use Flow\Filesystem\Path; interface JoinAlgorithmBuilder { - public function build(Path $spillRoot): HashJoinConfig; + public function build(Path $spillRoot, Backend $backend): HashJoinConfig; } diff --git a/src/core/etl/src/Flow/ETL/Config/MemoryLimit.php b/src/core/etl/src/Flow/ETL/Config/MemoryLimit.php new file mode 100644 index 0000000000..c256473a02 --- /dev/null +++ b/src/core/etl/src/Flow/ETL/Config/MemoryLimit.php @@ -0,0 +1,41 @@ +limit() ?? Unit::fromGb( + self::WITHOUT_MEMORY_LIMIT_GB, + ) + ); + } +} diff --git a/src/core/etl/src/Flow/ETL/Config/Repartition/HashRepartitionBuilder.php b/src/core/etl/src/Flow/ETL/Config/Repartition/HashRepartitionBuilder.php index 3639363cea..fde31e314d 100644 --- a/src/core/etl/src/Flow/ETL/Config/Repartition/HashRepartitionBuilder.php +++ b/src/core/etl/src/Flow/ETL/Config/Repartition/HashRepartitionBuilder.php @@ -5,13 +5,19 @@ namespace Flow\ETL\Config\Repartition; use Flow\ETL\Bucketing\BucketsStorage; +use Flow\ETL\Column\Backend; use Flow\ETL\Config\Bucketing\BucketingConfigBuilder; +use Flow\ETL\Config\MemoryLimit; +use Flow\ETL\Dataset\Memory\Unit; +use Flow\ETL\Exception\InvalidArgumentException; use Flow\Filesystem\Path; final class HashRepartitionBuilder implements RepartitionAlgorithmBuilder { private readonly BucketingConfigBuilder $bucketing; + private ?Unit $memoryLimit = null; + public function __construct() { $this->bucketing = new BucketingConfigBuilder('/flow-php-repartition/', 64); @@ -27,9 +33,12 @@ public function batchSize(int $batchSize): self return $this; } - public function build(Path $spillRoot): HashRepartitionConfig + public function build(Path $spillRoot, Backend $backend): HashRepartitionConfig { - return new HashRepartitionConfig($this->bucketing->build($spillRoot)); + return new HashRepartitionConfig( + $this->bucketing->build($spillRoot, $backend), + $this->memoryLimit ?? MemoryLimit::default(), + ); } /** @@ -42,6 +51,20 @@ public function bucketsCount(int $bucketsCount): self return $this; } + /** + * The process memory past which buckets move to the storage; defaults to MemoryLimit::default(). + */ + public function memoryLimit(Unit $memoryLimit): self + { + if ($memoryLimit->inBytes() < 1) { + throw new InvalidArgumentException('Repartition memory limit must be greater than 0 bytes'); + } + + $this->memoryLimit = $memoryLimit; + + return $this; + } + public function storage(BucketsStorage $storage): self { $this->bucketing->storage($storage); diff --git a/src/core/etl/src/Flow/ETL/Config/Repartition/HashRepartitionConfig.php b/src/core/etl/src/Flow/ETL/Config/Repartition/HashRepartitionConfig.php index 33a7009206..c5edeff752 100644 --- a/src/core/etl/src/Flow/ETL/Config/Repartition/HashRepartitionConfig.php +++ b/src/core/etl/src/Flow/ETL/Config/Repartition/HashRepartitionConfig.php @@ -5,10 +5,12 @@ namespace Flow\ETL\Config\Repartition; use Flow\ETL\Config\Bucketing\BucketingConfig; +use Flow\ETL\Dataset\Memory\Unit; final readonly class HashRepartitionConfig { public function __construct( public BucketingConfig $bucketing, + public Unit $memoryLimit, ) {} } diff --git a/src/core/etl/src/Flow/ETL/Config/Repartition/RepartitionAlgorithmBuilder.php b/src/core/etl/src/Flow/ETL/Config/Repartition/RepartitionAlgorithmBuilder.php index e9f8364ac6..c4d185a9fe 100644 --- a/src/core/etl/src/Flow/ETL/Config/Repartition/RepartitionAlgorithmBuilder.php +++ b/src/core/etl/src/Flow/ETL/Config/Repartition/RepartitionAlgorithmBuilder.php @@ -4,9 +4,10 @@ namespace Flow\ETL\Config\Repartition; +use Flow\ETL\Column\Backend; use Flow\Filesystem\Path; interface RepartitionAlgorithmBuilder { - public function build(Path $spillRoot): HashRepartitionConfig; + public function build(Path $spillRoot, Backend $backend): HashRepartitionConfig; } diff --git a/src/core/etl/src/Flow/ETL/Config/Sort/ExternalSortBuilder.php b/src/core/etl/src/Flow/ETL/Config/Sort/ExternalSortBuilder.php index 63b3d4a5b3..957c363118 100644 --- a/src/core/etl/src/Flow/ETL/Config/Sort/ExternalSortBuilder.php +++ b/src/core/etl/src/Flow/ETL/Config/Sort/ExternalSortBuilder.php @@ -5,7 +5,10 @@ namespace Flow\ETL\Config\Sort; use Flow\ETL\Bucketing\BucketsStorage; +use Flow\ETL\Column\Backend; use Flow\ETL\Config\Bucketing\BucketingConfigBuilder; +use Flow\ETL\Config\MemoryLimit; +use Flow\ETL\Dataset\Memory\Unit; use Flow\ETL\Exception\InvalidArgumentException; use Flow\Filesystem\Path; @@ -15,10 +18,7 @@ final class ExternalSortBuilder implements SortAlgorithmBuilder private ?BucketsStorage $mergeStorage = null; - /** - * @var int<1, max> - */ - private int $runSize = 10_000; + private ?Unit $memoryLimit = null; public function __construct() { @@ -35,9 +35,13 @@ public function batchSize(int $batchSize): self return $this; } - public function build(Path $spillRoot): ExternalSortConfig + public function build(Path $spillRoot, Backend $backend): ExternalSortConfig { - return new ExternalSortConfig($this->bucketing->build($spillRoot), $this->mergeStorage, $this->runSize); + return new ExternalSortConfig( + $this->bucketing->build($spillRoot, $backend), + $this->memoryLimit ?? MemoryLimit::default(), + $this->mergeStorage, + ); } /** @@ -61,16 +65,15 @@ public function mergeStorage(BucketsStorage $storage): self } /** - * @param int<1, max> $runSize + * The process memory past which the sort spills sorted runs; defaults to MemoryLimit::default(). */ - public function runSize(int $runSize): self + public function memoryLimit(Unit $memoryLimit): self { - // @mago-ignore analysis:impossible-condition,redundant-comparison - if ($runSize < 1) { - throw new InvalidArgumentException('Run size must be greater than 0'); + if ($memoryLimit->inBytes() < 1) { + throw new InvalidArgumentException('Sort memory limit must be greater than 0 bytes'); } - $this->runSize = $runSize; + $this->memoryLimit = $memoryLimit; return $this; } diff --git a/src/core/etl/src/Flow/ETL/Config/Sort/ExternalSortConfig.php b/src/core/etl/src/Flow/ETL/Config/Sort/ExternalSortConfig.php index eb87958672..52a74737d6 100644 --- a/src/core/etl/src/Flow/ETL/Config/Sort/ExternalSortConfig.php +++ b/src/core/etl/src/Flow/ETL/Config/Sort/ExternalSortConfig.php @@ -6,17 +6,18 @@ use Flow\ETL\Bucketing\BucketsStorage; use Flow\ETL\Config\Bucketing\BucketingConfig; +use Flow\ETL\Dataset\Memory\Unit; final readonly class ExternalSortConfig { /** * @param BucketingConfig $bucketing - the spill side + * @param Unit $memoryLimit - memory the sort holds (sorting in memory) before it spills sorted runs * @param null|BucketsStorage $merge - null means the merged runs go to the same storage the spill uses - * @param int<1, max> $runSize - rows buffered and sorted in memory before they are spilled as one run */ public function __construct( public BucketingConfig $bucketing, + public Unit $memoryLimit, public ?BucketsStorage $merge = null, - public int $runSize = 10_000, ) {} } diff --git a/src/core/etl/src/Flow/ETL/Config/Sort/MemorySortBuilder.php b/src/core/etl/src/Flow/ETL/Config/Sort/MemorySortBuilder.php index 33c8469131..47704f62dc 100644 --- a/src/core/etl/src/Flow/ETL/Config/Sort/MemorySortBuilder.php +++ b/src/core/etl/src/Flow/ETL/Config/Sort/MemorySortBuilder.php @@ -4,11 +4,12 @@ namespace Flow\ETL\Config\Sort; +use Flow\ETL\Column\Backend; use Flow\Filesystem\Path; final class MemorySortBuilder implements SortAlgorithmBuilder { - public function build(Path $spillRoot): MemorySortConfig + public function build(Path $spillRoot, Backend $backend): MemorySortConfig { return new MemorySortConfig(); } diff --git a/src/core/etl/src/Flow/ETL/Config/Sort/SortAlgorithmBuilder.php b/src/core/etl/src/Flow/ETL/Config/Sort/SortAlgorithmBuilder.php index abd8a96484..e6d72e80f2 100644 --- a/src/core/etl/src/Flow/ETL/Config/Sort/SortAlgorithmBuilder.php +++ b/src/core/etl/src/Flow/ETL/Config/Sort/SortAlgorithmBuilder.php @@ -4,9 +4,10 @@ namespace Flow\ETL\Config\Sort; +use Flow\ETL\Column\Backend; use Flow\Filesystem\Path; interface SortAlgorithmBuilder { - public function build(Path $spillRoot): MemorySortConfig|ExternalSortConfig; + public function build(Path $spillRoot, Backend $backend): MemorySortConfig|ExternalSortConfig; } diff --git a/src/core/etl/src/Flow/ETL/Config/Telemetry/TelemetryContext.php b/src/core/etl/src/Flow/ETL/Config/Telemetry/TelemetryContext.php index 34dc7e072b..915656131d 100644 --- a/src/core/etl/src/Flow/ETL/Config/Telemetry/TelemetryContext.php +++ b/src/core/etl/src/Flow/ETL/Config/Telemetry/TelemetryContext.php @@ -4,6 +4,7 @@ namespace Flow\ETL\Config\Telemetry; +use Flow\ETL\Column\Backend; use Flow\ETL\Config\Sort\ExternalSortConfig; use Flow\ETL\Config\Sort\MemorySortConfig; use Flow\ETL\Dataset\Memory\Consumption; @@ -60,12 +61,13 @@ final class TelemetryContext private readonly SpanStack $transformationSpans; public function __construct( + private readonly Backend $backend, private readonly Logger $logger, private readonly Tracer $tracer, private readonly Meter $meter, public readonly TelemetryOptions $options, ) { - $this->memory = new Consumption(); + $this->memory = new Consumption($this->backend); $this->loadingSpans = new SpanStack(); $this->transformationSpans = new SpanStack(); } @@ -141,7 +143,7 @@ public function dataFrameCompleted(FlowContext $context, array $attributes = []) $this->dataFrameExecutionTime = null; $this->totalRowsProcessed = 0; $this->dataFrameSpan = null; - $this->memory = new Consumption(); + $this->memory = new Consumption($this->backend); } /** @@ -200,7 +202,7 @@ public function dataFrameFailed(FlowContext $context, Throwable $exception, arra $this->dataFrameExecutionTime = null; $this->totalRowsProcessed = 0; $this->dataFrameSpan = null; - $this->memory = new Consumption(); + $this->memory = new Consumption($this->backend); } public function dataFrameStarted(FlowContext $context): void diff --git a/src/core/etl/src/Flow/ETL/Constraint.php b/src/core/etl/src/Flow/ETL/Constraint.php index a6a0c7cf34..12eca63381 100644 --- a/src/core/etl/src/Flow/ETL/Constraint.php +++ b/src/core/etl/src/Flow/ETL/Constraint.php @@ -6,9 +6,12 @@ interface Constraint { - public function isSatisfiedBy(Row $row, Schema $schema): bool; + /** + * Index of the first row that violates; state advances over the rows before it. + */ + public function firstViolation(Rows $rows): ?int; public function toString(): string; - public function violation(Row $row, Schema $schema): string; + public function violation(Rows $rows, int $index): string; } diff --git a/src/core/etl/src/Flow/ETL/Constraint/SortedByConstraint.php b/src/core/etl/src/Flow/ETL/Constraint/SortedByConstraint.php index 548564301f..8f90d265ee 100644 --- a/src/core/etl/src/Flow/ETL/Constraint/SortedByConstraint.php +++ b/src/core/etl/src/Flow/ETL/Constraint/SortedByConstraint.php @@ -4,30 +4,23 @@ namespace Flow\ETL\Constraint; -use DateInterval; -use DateTimeInterface; use Flow\ETL\Constraint; -use Flow\ETL\Row; +use Flow\ETL\Row\NullsOrder; use Flow\ETL\Row\Reference; use Flow\ETL\Row\References; -use Flow\ETL\Row\SortOrder; -use Flow\ETL\Schema; +use Flow\ETL\Rows; +use Flow\ETL\Sort\RowOrder; use Flow\Types\Type\TypedValueFormatter; use function implode; -use function is_array; -use function is_numeric; -use function is_string; use function var_export; final class SortedByConstraint implements Constraint { - private bool $firstRow = true; - /** - * @var array|bool|float|int|object|string> + * The last accepted row, one-row batch - the "previous" of the next batch's first row. */ - private array $previousValues = []; + private ?Rows $previous = null; private readonly References $references; @@ -36,71 +29,33 @@ public function __construct(Reference $column, Reference ...$columns) $this->references = new References($column, ...$columns); } - public function isSatisfiedBy(Row $row, Schema $schema): bool + public function firstViolation(Rows $rows): ?int { - if ($this->firstRow) { - foreach ($this->references->all() as $reference) { - $this->previousValues[$reference->name()] = $row->get($reference); - } - $this->firstRow = false; - - return true; + if ($rows->isEmpty()) { + return null; } - foreach ($this->references->all() as $reference) { - $currentValue = $row->get($reference); - $previousValue = $this->previousValues[$reference->name()]; - - $direction = $reference->sort(); - - if ($previousValue === null && $currentValue === null) { - $comparison = 0; - } elseif ($previousValue === null) { - $comparison = $direction === SortOrder::ASC ? -1 : 1; - } elseif ($currentValue === null) { - $comparison = $direction === SortOrder::ASC ? 1 : -1; - } elseif (is_numeric($previousValue) && is_numeric($currentValue)) { - $prev = (float) $previousValue; - $curr = (float) $currentValue; - $comparison = $direction === SortOrder::ASC ? $prev <=> $curr : $curr <=> $prev; - } elseif (is_string($previousValue) && is_string($currentValue)) { - $comparison = $direction === SortOrder::ASC - ? $previousValue <=> $currentValue - : $currentValue <=> $previousValue; - } elseif ($previousValue instanceof DateTimeInterface && $currentValue instanceof DateTimeInterface) { - $comparison = $direction === SortOrder::ASC - ? $previousValue <=> $currentValue - : $currentValue <=> $previousValue; - } elseif ($previousValue instanceof DateInterval && $currentValue instanceof DateInterval) { - $comparison = $direction === SortOrder::ASC - ? $previousValue <=> $currentValue - : $currentValue <=> $previousValue; - } elseif (is_array($previousValue) && is_array($currentValue)) { - $comparison = $direction === SortOrder::ASC - ? $previousValue <=> $currentValue - : $currentValue <=> $previousValue; - } else { - $comparison = 0; - } - - if ($comparison < 0) { - foreach ($this->references->all() as $ref) { - $this->previousValues[$ref->name()] = $row->get($ref); - } + $order = new RowOrder($this->references->all()); + $keys = $order->keys($rows); + $violation = null; - return true; - } + if ($this->previous !== null && $order->compare($order->keys($this->previous), 0, $keys, 0) > 0) { + $violation = 0; + } - if ($comparison > 0) { - return false; + for ($i = 1, $count = $rows->count(); $violation === null && $i < $count; $i++) { + if ($order->compare($keys, $i - 1, $keys, $i) > 0) { + $violation = $i; } } - foreach ($this->references->all() as $reference) { - $this->previousValues[$reference->name()] = $row->get($reference); + $last = ($violation ?? $rows->count()) - 1; + + if ($last >= 0) { + $this->previous = $rows->slice($last, 1); } - return true; + return $violation; } public function toString(): string @@ -108,27 +63,36 @@ public function toString(): string $columns = []; foreach ($this->references->all() as $reference) { - $columns[] = $reference->name() . ' ' . $reference->sort()->name; + $columns[] = + $reference->name() + . ' ' + . $reference->sort()->name + . ( + $reference->nulls() === NullsOrder::defaultFor($reference->sort()) + ? '' + : ' NULLS ' . $reference->nulls()->name + ); } return sprintf('Sorted constraint on [%s]', implode(', ', $columns)); } - public function violation(Row $row, Schema $schema): string + public function violation(Rows $rows, int $index): string { $formatter = new TypedValueFormatter(); $violations = []; foreach ($this->references->all() as $reference) { - $definition = $schema->get($reference); - $previousValue = $this->previousValues[$reference->name()] ?? null; + $definition = $rows->schema()->get($reference); + // @mago-ignore analysis:mixed-assignment + $previousValue = $this->previous?->column($reference->base())->value(0); $violations[] = sprintf( '%s<%s> expected %s order, current: %s, previous: %s', $reference->name(), $definition->type()->toString(), $reference->sort()->name, - $formatter->format($definition->type(), $row->get($reference)), + $formatter->format($definition->type(), $rows->column($reference->base())->value($index)), $previousValue === null ? 'null' : var_export($previousValue, true), ); } diff --git a/src/core/etl/src/Flow/ETL/Constraint/UniqueConstraint.php b/src/core/etl/src/Flow/ETL/Constraint/UniqueConstraint.php index e244eb41b6..2d5430ccf2 100644 --- a/src/core/etl/src/Flow/ETL/Constraint/UniqueConstraint.php +++ b/src/core/etl/src/Flow/ETL/Constraint/UniqueConstraint.php @@ -7,10 +7,10 @@ use Flow\ETL\Constraint; use Flow\ETL\Constraint\UniqueConstraint\InMemoryStorage; use Flow\ETL\Constraint\UniqueConstraint\Storage; -use Flow\ETL\Row; use Flow\ETL\Row\Reference; use Flow\ETL\Row\References; -use Flow\ETL\Schema; +use Flow\ETL\Rows; +use Flow\ETL\Rows\RowHashes; use Flow\Types\Type\TypedValueFormatter; use function Flow\ETL\DSL\refs; @@ -21,36 +21,27 @@ final class UniqueConstraint implements Constraint private Storage $storage; - /** - * Rows arrive batch by batch and Rows::schema() hands out the same instance for a whole batch, - * so the kept schema is folded once per batch instead of once per row. - */ - private ?Schema $keptSchema = null; - - private ?Schema $keptSchemaSource = null; - public function __construct(string|Reference $column, string|Reference ...$columns) { $this->reference = refs($column, ...$columns); $this->storage = new InMemoryStorage(); } - public function isSatisfiedBy(Row $row, Schema $schema): bool + public function firstViolation(Rows $rows): ?int { - if ($this->keptSchema === null || $this->keptSchemaSource !== $schema) { - $this->keptSchema = $schema->keep(...$this->reference); - $this->keptSchemaSource = $schema; + if ($rows->isEmpty()) { + return null; } - $key = $row->hash($this->keptSchema); + foreach ((new RowHashes())->of($rows->select(...$this->reference->names())) as $i => $key) { + if ($this->storage->has($key)) { + return $i; + } - if ($this->storage->has($key)) { - return false; + $this->storage->set($key); } - $this->storage->set($key); - - return true; + return null; } public function toString(): string @@ -61,19 +52,19 @@ public function toString(): string ))); } - public function violation(Row $row, Schema $schema): string + public function violation(Rows $rows, int $index): string { $formatter = new TypedValueFormatter(); $violations = []; - foreach ($schema->keep(...$this->reference)->definitions() as $definition) { + foreach ($rows->schema()->keep(...$this->reference)->definitions() as $definition) { $name = $definition->entry()->name(); $violations[] = $name . '<' . $definition->type()->toString() . '> = ' - . $formatter->format($definition->type(), $row->get($name)); + . $formatter->format($definition->type(), $rows->column($name)->value($index)); } return sprintf('Values: [%s]', implode(', ', $violations)); diff --git a/src/core/etl/src/Flow/ETL/DSL/functions.php b/src/core/etl/src/Flow/ETL/DSL/functions.php index ae47b26c35..1ef78d2326 100644 --- a/src/core/etl/src/Flow/ETL/DSL/functions.php +++ b/src/core/etl/src/Flow/ETL/DSL/functions.php @@ -17,6 +17,8 @@ use Flow\ETL\Analyze; use Flow\ETL\Cache; use Flow\ETL\Cache\Implementation\FilesystemCache; +use Flow\ETL\Column\AdaptiveBackend; +use Flow\ETL\Column\Backend; use Flow\ETL\Config; use Flow\ETL\Config\ConfigBuilder; use Flow\ETL\Config\Grouping\HashGroupByBuilder; @@ -41,9 +43,9 @@ use Flow\ETL\Extractor\CacheExtractor; use Flow\ETL\Extractor\ChainExtractor; use Flow\ETL\Extractor\DataFrameExtractor; +use Flow\ETL\Extractor\File\PartitionTypes; use Flow\ETL\Extractor\FilesExtractor; use Flow\ETL\Extractor\MemoryExtractor; -use Flow\ETL\Extractor\PartitionTypes; use Flow\ETL\Extractor\PathPartitionsExtractor; use Flow\ETL\Extractor\RowsExtractor; use Flow\ETL\Extractor\SequenceExtractor; @@ -145,17 +147,13 @@ use Flow\ETL\Memory\Memory; use Flow\ETL\NativePHPRandomValueGenerator; use Flow\ETL\RandomValueGenerator; -use Flow\ETL\Row; -use Flow\ETL\Row\AdaptiveRowHydrator; -use Flow\ETL\Row\ColumnName; use Flow\ETL\Row\Formatter\ASCIISchemaFormatter; -use Flow\ETL\Row\Hydrator; -use Flow\ETL\Row\RawRowValues; use Flow\ETL\Row\Reference; use Flow\ETL\Row\References; use Flow\ETL\Row\SortOrder; use Flow\ETL\Row\UnresolvedReference; use Flow\ETL\Rows; +use Flow\ETL\Rows\RowsBuilder; use Flow\ETL\Schema; use Flow\ETL\Schema\Definition; use Flow\ETL\Schema\Definition\BooleanDefinition; @@ -217,11 +215,8 @@ use Flow\ETL\WithEntry; use Flow\Filesystem\Filesystem; use Flow\Filesystem\Local\NativeLocalFilesystem; -use Flow\Filesystem\Partition; -use Flow\Filesystem\Partitions; use Flow\Filesystem\Path; use Flow\Filesystem\Stream\Mode; -use Flow\Floe\FloeSerializer; use Flow\Serializer\Serializer; use Flow\Types\Type; use Flow\Types\Type\Logical\DateTimeType; @@ -386,11 +381,11 @@ function files(string|Path $directory, Filesystem $filesystem = new NativeLocalF #[DocumentationDSL(module: Module::CORE, type: DSLType::DATA_FRAME)] function filesystem_cache( + Serializer $serializer, Path|string|null $cache_dir = null, Filesystem $filesystem = new NativeLocalFilesystem(), - Serializer $serializer = new FloeSerializer(), ): FilesystemCache { - return new FilesystemCache($filesystem, is_string($cache_dir) ? path_real($cache_dir) : $cache_dir, $serializer); + return new FilesystemCache($filesystem, $serializer, is_string($cache_dir) ? path_real($cache_dir) : $cache_dir); } /** @@ -569,19 +564,10 @@ function rename_map(array $renames): RenameMapEntryStrategy return new RenameMapEntryStrategy($renames); } -/** - * @param array $values - */ #[DocumentationDSL(module: Module::CORE, type: DSLType::DATA_FRAME)] -function row(array $values): Row +function rows(Schema $schema, Backend $backend = new AdaptiveBackend()): Rows { - return new Row($values); -} - -#[DocumentationDSL(module: Module::CORE, type: DSLType::DATA_FRAME)] -function rows(Schema $schema, Row ...$row): Rows -{ - return new Rows($schema, ...$row); + return Rows::empty($schema, $backend); } /** @@ -1173,69 +1159,13 @@ function number_format( /** * @param array>|array $data - * @param array|Partitions $partitions */ #[DocumentationDSL(module: Module::CORE, type: DSLType::DATA_FRAME)] -function array_to_row( - array $data, - Schema $schema, - Hydrator $hydrator = new AdaptiveRowHydrator(), - array|Partitions $partitions = [], -): Row { - $map = []; - - // @mago-ignore analysis:mixed-assignment - foreach ($data as $key => $value) { - $map[(new ColumnName())->of($key)] = $value; - } - - foreach ($partitions as $partition) { - if (!array_key_exists($partition->name, $map)) { - $map[$partition->name] = $partition->value; - } - } - - return $hydrator->hydrate([new RawRowValues($map)], $schema)->first(); -} - -/** - * @param array>|array $data - */ -#[DocumentationDSL(module: Module::CORE, type: DSLType::DATA_FRAME)] -function array_to_rows(array $data, Schema $schema, Hydrator $hydrator = new AdaptiveRowHydrator()): Rows +function array_to_rows(array $data, Schema $schema, Backend $backend = new AdaptiveBackend()): Rows { - $isRows = true; - - // @mago-ignore analysis:mixed-assignment - foreach ($data as $v) { - if (!is_array($v)) { - $isRows = false; - - break; - } - } - - $rawRows = $isRows ? $data : [$data]; - $maps = []; - - // @mago-ignore analysis:mixed-assignment - foreach ($rawRows as $row) { - $row = type_array()->assert($row); - $map = []; - - // @mago-ignore analysis:mixed-assignment - foreach ($row as $key => $value) { - // PHP gives back a numeric-string column name as an int key, which the positional rule - // would rename to eNN. A declared schema naming that column settles which one it is. - $declared = $schema->findDefinition((string) $key); - - $map[$declared === null ? (new ColumnName())->of($key) : (string) $key] = $value; - } - - $maps[] = new RawRowValues($map); - } - - return $hydrator->hydrate($maps, $schema); + return (new RowsBuilder($schema, $backend)) + ->appendRecords($data) + ->finish(); } #[DocumentationDSL(module: Module::CORE, type: DSLType::WINDOW_FUNCTION)] @@ -1257,9 +1187,13 @@ function dense_rank(): DenseRank } #[DocumentationDSL(module: Module::CORE, type: DSLType::AGGREGATING_FUNCTION)] -function average(Reference|string $ref, int $scale = 2, Rounding $rounding = Rounding::HALF_UP): Average -{ - return new Average(is_string($ref) ? ref($ref) : $ref, $scale, $rounding); +function average( + Reference|string $ref, + int $scale = 2, + Rounding $rounding = Rounding::HALF_UP, + bool $exact = false, +): Average { + return new Average(is_string($ref) ? ref($ref) : $ref, $scale, $rounding, $exact); } #[DocumentationDSL(module: Module::CORE, type: DSLType::SCALAR_FUNCTION)] @@ -1329,7 +1263,7 @@ function unbounded_following(): FrameBound } #[DocumentationDSL(module: Module::CORE, type: DSLType::AGGREGATING_FUNCTION)] -function sum(Reference|string $ref, ScalarFunction|bool $exact = false): Sum +function sum(Reference|string $ref, bool $exact = false): Sum { return new Sum(is_string($ref) ? ref($ref) : $ref, $exact); } diff --git a/src/core/etl/src/Flow/ETL/DataFrame.php b/src/core/etl/src/Flow/ETL/DataFrame.php index 168075b601..67a3e7de7b 100644 --- a/src/core/etl/src/Flow/ETL/DataFrame.php +++ b/src/core/etl/src/Flow/ETL/DataFrame.php @@ -213,8 +213,8 @@ public function count(): int ->planner() ->plan(Trigger::count->plan($this->root, $this->sinks), $this->context), ) - ->first() - ->get('count'), + ->column('count') + ->value(0), ); } @@ -299,7 +299,6 @@ public function duplicateRow(mixed $condition, WithEntry ...$entries): self * * DataFrame::get() : \Generator * DataFrame::getAsArray() : \Generator - * DataFrame::getEach() : \Generator * DataFrame::getEachAsArray() : \Generator * * @trigger @@ -401,27 +400,6 @@ public function getAsArray(): Generator } } - /** - * Yield each row as an instance of Row. - * - * @trigger - * - * @return \Generator - */ - public function getEach(): Generator - { - foreach ($this->context - ->config - ->executor() - ->execute( - $this->context->config->planner()->plan(Trigger::rows->plan($this->root, $this->sinks), $this->context), - ) as $rows) { - foreach ($rows as $row) { - yield $row; - } - } - } - /** * Yield each row as an array. * @@ -437,8 +415,8 @@ public function getEachAsArray(): Generator ->execute( $this->context->config->planner()->plan(Trigger::rows->plan($this->root, $this->sinks), $this->context), ) as $rows) { - foreach ($rows as $row) { - yield $row->toArray(); + foreach ($rows->toArray() as $row) { + yield $row; } } } diff --git a/src/core/etl/src/Flow/ETL/Dataset/Memory/BoundedRead.php b/src/core/etl/src/Flow/ETL/Dataset/Memory/BoundedRead.php new file mode 100644 index 0000000000..cb2972cc34 --- /dev/null +++ b/src/core/etl/src/Flow/ETL/Dataset/Memory/BoundedRead.php @@ -0,0 +1,59 @@ + $read + * @param Generator $rest + * + * @return Generator the batches already read, then what is left of $rest + */ + public function followedBy(array $read, Generator $rest): Generator + { + yield from $read; + + // a started generator is never rewound, and PHP refuses to yield from one that already finished + if ($rest->valid()) { + yield from $rest; + } + } + + /** + * Reads $rows while the process stays under the limit; $rows is left at the first batch not read. + * + * @param Generator $rows + * + * @return array{list, bool} the batches read, and whether $rows ended within the limit + */ + public function read(Generator $rows): array + { + $budget = new MemoryBudget($this->backend, $this->limit); + $read = []; + + while ($rows->valid()) { + /** @var Rows $batch valid() holds, so current() is a batch */ + $batch = $rows->current(); + $read[] = $batch; + $rows->next(); + + if ($budget->exceeded()) { + return [$read, false]; + } + } + + return [$read, true]; + } +} diff --git a/src/core/etl/src/Flow/ETL/Dataset/Memory/Consumption.php b/src/core/etl/src/Flow/ETL/Dataset/Memory/Consumption.php index 8c2a3b8136..88d1cb0e6d 100644 --- a/src/core/etl/src/Flow/ETL/Dataset/Memory/Consumption.php +++ b/src/core/etl/src/Flow/ETL/Dataset/Memory/Consumption.php @@ -4,6 +4,8 @@ namespace Flow\ETL\Dataset\Memory; +use Flow\ETL\Column\Backend; + use function memory_get_usage; final class Consumption @@ -15,16 +17,17 @@ final class Consumption private Unit $min; public function __construct( + private readonly Backend $backend, private readonly bool $realMemory = true, ) { - $this->initial = Unit::fromBytes(memory_get_usage($this->realMemory)); + $this->initial = Unit::fromBytes(memory_get_usage($this->realMemory) + $this->backend->allocatedBytes()); $this->min = $this->initial; $this->max = $this->initial; } public function capture(): Unit { - $current = Unit::fromBytes(memory_get_usage($this->realMemory)); + $current = Unit::fromBytes(memory_get_usage($this->realMemory) + $this->backend->allocatedBytes()); if ($current->isGreaterThan($this->max)) { $this->max = $current; diff --git a/src/core/etl/src/Flow/ETL/Dataset/Memory/MemoryBudget.php b/src/core/etl/src/Flow/ETL/Dataset/Memory/MemoryBudget.php new file mode 100644 index 0000000000..080154c23c --- /dev/null +++ b/src/core/etl/src/Flow/ETL/Dataset/Memory/MemoryBudget.php @@ -0,0 +1,36 @@ +consumption = new Consumption($backend, realMemory: false); + } + + public function exceeded(): bool + { + return $this->consumption->capture()->isGreaterThan($this->limit); + } + + /** + * After a spill freed its buffer: hands the memory PHP keeps cached back to the OS. + */ + public function released(): void + { + gc_mem_caches(); + } +} diff --git a/src/core/etl/src/Flow/ETL/Dataset/Statistics/Column.php b/src/core/etl/src/Flow/ETL/Dataset/Statistics/Column.php index 7afb82c907..7811d15af3 100644 --- a/src/core/etl/src/Flow/ETL/Dataset/Statistics/Column.php +++ b/src/core/etl/src/Flow/ETL/Dataset/Statistics/Column.php @@ -6,8 +6,10 @@ use DateTimeInterface; use DateTimeZone; +use Flow\ETL\Column\Column as ColumnData; use Flow\ETL\Row\Reference; use Flow\ETL\Schema\Definition; +use Flow\Types\Type; use Flow\Types\Type\Logical\DateTimeType; use Flow\Types\Type\Logical\DateType; use Flow\Types\Type\Logical\ListType; @@ -23,11 +25,13 @@ use function count; use function Flow\Types\DSL\type_instance_of; +use function intdiv; use function is_bool; use function is_countable; use function is_float; use function is_int; use function is_scalar; +use function is_string; use function json_encode; use function max; use function mb_strlen; @@ -58,84 +62,41 @@ final class Column * * @throws \JsonException */ - public function __construct(Definition $definition, mixed $value) + public function __construct(Definition $definition, ColumnData $column) { $this->reference = $definition->entry(); $this->distinctCounter = new DistinctCounter(); - $this->add($definition, $value); + $this->add($definition, $column); } /** * @param Definition $definition + * + * @throws \JsonException */ - public function add(Definition $definition, mixed $value): void + public function add(Definition $definition, ColumnData $column): void { if ($this->reference->name() !== $definition->entry()->name()) { return; } - $type = $definition->type(); - - if ($value === null) { - $this->nullsCount++; - - return; - } - - if ($type instanceof UuidType) { - $this->distinctCounter->add(type_instance_of(Uuid::class)->assert($value)->toString()); + $this->nullsCount += $column->nullCount(); + if ($column->nullCount() === $column->count()) { return; } - if ($type instanceof TimeZoneType) { - $this->distinctCounter->add(type_instance_of(DateTimeZone::class)->assert($value)->getName()); - - return; - } - - if ($type instanceof StructureType) { - $this->distinctCounter->add(json_encode($value, JSON_THROW_ON_ERROR)); - - return; - } - - if ($type instanceof ListType || $type instanceof MapType) { - $this->distinctCounter->add(json_encode($value, JSON_THROW_ON_ERROR)); - $elementsCount = is_countable($value) ? count($value) : 0; - $this->maxElementsCount = max($this->maxElementsCount ?? $elementsCount, $elementsCount); - $this->minElementsCount = min($this->minElementsCount ?? $elementsCount, $elementsCount); - - return; - } - - if (is_scalar($value) || $value instanceof DateTimeInterface) { - $this->distinctCounter->add($value); - } - - if ($type instanceof StringType) { - $valueLength = mb_strlen(is_scalar($value) ? (string) $value : ''); - $this->maxLength = max($this->maxLength ?? $valueLength, $valueLength); - $this->minLength = min($this->minLength ?? $valueLength, $valueLength); - - return; - } - - if ($type instanceof DateType || $type instanceof DateTimeType) { - if ($value instanceof DateTimeInterface) { - $this->max = max($this->max ?? $value, $value); - $this->min = min($this->min ?? $value, $value); - } - - return; - } + $type = $definition->type(); - if ($type instanceof IntegerType || $type instanceof FloatType || $type instanceof BooleanType) { - if (is_int($value) || is_float($value) || is_bool($value)) { - $this->min = min($this->min ?? $value, $value); - $this->max = max($this->max ?? $value, $value); - } - } + match (true) { + $type instanceof StringType => $this->strings($column->physicals()), + $type instanceof IntegerType, $type instanceof FloatType, $type instanceof BooleanType => $this->scalars( + $column->physicals(), + ), + $type instanceof DateTimeType => $this->instants($column, 1_000_000), + $type instanceof DateType => $this->instants($column, 1), + default => $this->values($type, $column->values()), + }; } public function distinctCount(): int @@ -187,4 +148,128 @@ public function reference(): Reference { return $this->reference; } + + /** + * Date physicals are days, datetime physicals microseconds: the distinct key is the Unix second, as + * DateTimeInterface::getTimestamp() gives it, and only the two extremes are materialised. + * + * @param int $perSecond physical units per second, or 1 for days + */ + private function instants(ColumnData $column, int $perSecond): void + { + $min = null; + $max = null; + + // @mago-ignore analysis:mixed-assignment + foreach ($column->physicals() as $i => $physical) { + if (!is_int($physical)) { + continue; + } + + $seconds = $perSecond === 1 ? $physical * 86_400 : intdiv($physical, $perSecond); + + if ($perSecond !== 1 && ($physical % $perSecond) < 0) { + $seconds--; + } + + $this->distinctCounter->add($seconds); + + if ($min === null || $physical < $min[1]) { + $min = [$i, $physical]; + } + + if ($max === null || $physical > $max[1]) { + $max = [$i, $physical]; + } + } + + if ($min === null || $max === null) { + return; + } + + $minValue = type_instance_of(DateTimeInterface::class)->assert($column->value($min[0])); + $maxValue = type_instance_of(DateTimeInterface::class)->assert($column->value($max[0])); + $this->min = min($this->min ?? $minValue, $minValue); + $this->max = max($this->max ?? $maxValue, $maxValue); + } + + /** + * @param list $physicals + */ + private function scalars(array $physicals): void + { + // @mago-ignore analysis:mixed-assignment + foreach ($physicals as $value) { + if (is_int($value) || is_float($value) || is_bool($value)) { + $this->distinctCounter->add($value); + $this->min = min($this->min ?? $value, $value); + $this->max = max($this->max ?? $value, $value); + } + } + } + + /** + * @param list $physicals + */ + private function strings(array $physicals): void + { + // @mago-ignore analysis:mixed-assignment + foreach ($physicals as $value) { + if (!is_string($value)) { + continue; + } + + $this->distinctCounter->add($value); + $valueLength = mb_strlen($value); + $this->maxLength = max($this->maxLength ?? $valueLength, $valueLength); + $this->minLength = min($this->minLength ?? $valueLength, $valueLength); + } + } + + /** + * @param Type $type + * @param list $values + * + * @throws \JsonException + */ + private function values(Type $type, array $values): void + { + // @mago-ignore analysis:mixed-assignment + foreach ($values as $value) { + if ($value === null) { + continue; + } + + if ($type instanceof UuidType) { + $this->distinctCounter->add(type_instance_of(Uuid::class)->assert($value)->toString()); + + continue; + } + + if ($type instanceof TimeZoneType) { + $this->distinctCounter->add(type_instance_of(DateTimeZone::class)->assert($value)->getName()); + + continue; + } + + if ($type instanceof StructureType) { + $this->distinctCounter->add(json_encode($value, JSON_THROW_ON_ERROR)); + + continue; + } + + if ($type instanceof ListType || $type instanceof MapType) { + $this->distinctCounter->add(json_encode($value, JSON_THROW_ON_ERROR)); + $elementsCount = is_countable($value) ? count($value) : 0; + $this->maxElementsCount = max($this->maxElementsCount ?? $elementsCount, $elementsCount); + $this->minElementsCount = min($this->minElementsCount ?? $elementsCount, $elementsCount); + + continue; + } + + if (is_scalar($value) || $value instanceof DateTimeInterface) { + $this->distinctCounter->add($value); + } + } + } } diff --git a/src/core/etl/src/Flow/ETL/Dataset/Statistics/Columns.php b/src/core/etl/src/Flow/ETL/Dataset/Statistics/Columns.php index 61a54bc116..3434db6223 100644 --- a/src/core/etl/src/Flow/ETL/Dataset/Statistics/Columns.php +++ b/src/core/etl/src/Flow/ETL/Dataset/Statistics/Columns.php @@ -4,6 +4,7 @@ namespace Flow\ETL\Dataset\Statistics; +use Flow\ETL\Column\Column as ColumnData; use Flow\ETL\Row\Reference; use Flow\ETL\Schema\Definition; use InvalidArgumentException; @@ -21,23 +22,20 @@ final class Columns public function __construct() {} - /** - * @param Entry $entry - */ /** * @param Definition $definition */ - public function add(Definition $definition, mixed $value): void + public function add(Definition $definition, ColumnData $column): void { $name = $definition->entry()->name(); if (!array_key_exists($name, $this->columns)) { - $this->columns[$name] = new Column($definition, $value); + $this->columns[$name] = new Column($definition, $column); return; } - $this->columns[$name]->add($definition, $value); + $this->columns[$name]->add($definition, $column); } /** diff --git a/src/core/etl/src/Flow/ETL/Exception/ColumnMismatchException.php b/src/core/etl/src/Flow/ETL/Exception/ColumnMismatchException.php index b2ff42e379..b6c1988cbe 100644 --- a/src/core/etl/src/Flow/ETL/Exception/ColumnMismatchException.php +++ b/src/core/etl/src/Flow/ETL/Exception/ColumnMismatchException.php @@ -4,13 +4,18 @@ namespace Flow\ETL\Exception; +use DateTimeInterface; use Flow\ETL\Schema\Definition; +use Flow\Types\Exception\Exception as TypesException; use Flow\Types\Type\TypeDetector; +use Throwable; use function get_debug_type; use function is_bool; use function is_float; +use function is_infinite; use function is_int; +use function is_nan; use function is_string; use function sprintf; use function strlen; @@ -21,8 +26,9 @@ final class ColumnMismatchException extends InvalidArgumentException private function __construct( public readonly string $column, public readonly string $detail, + ?Throwable $previous = null, ) { - parent::__construct('Row does not match its schema: column "' . $column . '"' . $detail); + parent::__construct('Row does not match its schema: column "' . $column . '"' . $detail, 0, $previous); } /** @@ -41,20 +47,58 @@ public static function unexpectedColumn(string $column): self /** * @param Definition $definition */ - public static function valueDoesNotMatch(Definition $definition, mixed $value): self + public static function retype(Definition $definition, string $reason): self + { + return new self($definition->entry()->name(), sprintf( + ': cannot retype the column to %s, %s', + $definition->type()->toString(), + $reason, + )); + } + + /** + * @param Definition $definition + */ + public static function unsupportedType(Definition $definition, string $reason): self + { + return new self($definition->entry()->name(), sprintf( + ': %s cannot be a batch column, %s', + $definition->type()->toString(), + $reason, + )); + } + + /** + * @param Definition $definition + */ + public static function untypedColumn(Definition $definition): self + { + return new self( + $definition->entry()->name(), + ': mixed cannot be a batch column, an untyped function result exists only inside function evaluation', + ); + } + + /** + * @param Definition $definition + */ + public static function valueDoesNotMatch(Definition $definition, mixed $value, ?TypesException $reason = null): self { return new self( $definition->entry()->name(), - $value === null - ? sprintf(': could not convert null to %s, column is not nullable', $definition->type()->toString()) - : sprintf( - ': could not convert %s (%s) to %s', - self::describe($value), - (new TypeDetector()) - ->detectType($value) - ->toString(), - $definition->type()->toString(), - ), + ( + $value === null + ? sprintf(': could not convert null to %s, column is not nullable', $definition->type()->toString()) + : sprintf( + ': could not convert %s (%s) to %s', + self::describe($value), + (new TypeDetector()) + ->detectType($value) + ->toString(), + $definition->type()->toString(), + ) + ) . ($reason === null ? '' : ' - ' . $reason->getMessage()), + $reason, ); } @@ -63,7 +107,10 @@ private static function describe(mixed $value): string return match (true) { is_string($value) => "'" . (strlen($value) > 32 ? substr($value, 0, 32) . '...' : $value) . "'", is_bool($value) => $value ? 'true' : 'false', + is_float($value) && is_nan($value) => 'NAN', + is_float($value) && is_infinite($value) => $value > 0 ? 'INF' : '-INF', is_int($value), is_float($value) => (string) $value, + $value instanceof DateTimeInterface => $value->format('Y-m-d\TH:i:s.uP'), default => get_debug_type($value), }; } diff --git a/src/core/etl/src/Flow/ETL/Exception/EvaluationException.php b/src/core/etl/src/Flow/ETL/Exception/EvaluationException.php new file mode 100644 index 0000000000..a31932763a --- /dev/null +++ b/src/core/etl/src/Flow/ETL/Exception/EvaluationException.php @@ -0,0 +1,30 @@ +getMessage(), $rowIndex), 0, $cause); + } + + /** + * $cause at $rowIndex of the caller's batch; an EvaluationException cause is re-indexed, never nested. + */ + public static function at(int $rowIndex, BaseException $cause): self + { + return $cause instanceof self + ? new self($rowIndex, type_instance_of(BaseException::class)->assert($cause->getPrevious())) + : new self($rowIndex, $cause); + } +} diff --git a/src/core/etl/src/Flow/ETL/Exception/InferredSchemaException.php b/src/core/etl/src/Flow/ETL/Exception/InferredSchemaException.php index 2603d00223..743bf6c26d 100644 --- a/src/core/etl/src/Flow/ETL/Exception/InferredSchemaException.php +++ b/src/core/etl/src/Flow/ETL/Exception/InferredSchemaException.php @@ -46,7 +46,7 @@ public static function filesDiverge(string $source, string $readFrom, Validation { return new self(sprintf( "Columns of %s do not match the schema read from %s:\n%sRead the files as one wider schema with " - . '->unionByName(), or declare the schema with ->withSchema(...).', + . '->unionByName().', $source, $readFrom, $validation->toString(), diff --git a/src/core/etl/src/Flow/ETL/Exception/OffsetOverflow.php b/src/core/etl/src/Flow/ETL/Exception/OffsetOverflow.php new file mode 100644 index 0000000000..26cb8bcca2 --- /dev/null +++ b/src/core/etl/src/Flow/ETL/Exception/OffsetOverflow.php @@ -0,0 +1,7 @@ +merge($nextRows); + foreach ($batches as $batch) { + $last = $batch; + + if (!$batch->isEmpty()) { + $parts[] = $batch; + } + } + + if ($parts !== []) { + return $parts[0]->concat($plan->root()->context()->backend(), ...array_slice($parts, 1)); } - if ($rows !== null) { - return $rows; + if ($last !== null) { + return $last; } try { - return new Rows($plan->schema()); + return Rows::empty($plan->schema(), $plan->root()->context()->backend()); } catch (SchemaNotDerivableException) { - return new Rows(new Schema()); + return Rows::empty(new Schema(), $plan->root()->context()->backend()); } } @@ -131,9 +142,11 @@ public function executePipeline(Pipeline $pipeline): Generator 'pipeline #%d', $leaf->id, )); - $generator = $source instanceof FileExtractor - ? $source->extract($leaf->context(), $leaf->limit(), $leaf->pathFilter()) - : $source->extract($leaf->context(), $leaf->limit()); + $generator = (new AdoptedBatches($leaf->context()->backend()))->of( + $source instanceof FileExtractor + ? $source->extract($leaf->context(), $leaf->limit(), $leaf->pathFilter()) + : $source->extract($leaf->context(), $leaf->limit()), + ); $sources = $leaf->sources(); if ($sources !== null) { diff --git a/src/core/etl/src/Flow/ETL/Executor/AdoptedBatches.php b/src/core/etl/src/Flow/ETL/Executor/AdoptedBatches.php new file mode 100644 index 0000000000..e9e57dc389 --- /dev/null +++ b/src/core/etl/src/Flow/ETL/Executor/AdoptedBatches.php @@ -0,0 +1,37 @@ + $batches what an extractor yields + * + * @return Generator + */ + public function of(Generator $batches): Generator + { + foreach ($batches as $batch) { + $signal = yield $batch->adoptedBy($this->backend); + + if ($signal === Signal::STOP) { + $batches->send(Signal::STOP); + + return; + } + } + } +} diff --git a/src/core/etl/src/Flow/ETL/Executor/Segment.php b/src/core/etl/src/Flow/ETL/Executor/Segment.php index 09eb6554be..14d57bb2c5 100644 --- a/src/core/etl/src/Flow/ETL/Executor/Segment.php +++ b/src/core/etl/src/Flow/ETL/Executor/Segment.php @@ -130,7 +130,7 @@ public function execute(Generator $input, FlowContext $context): Generator } catch (LimitReachedException $limit) { $context->telemetry()->limitReached(['limit' => $limit->limit]); // the remaining steps still run, on the trimmed batch - $rows = $limit->rows ?? new Rows($rows->schema()); + $rows = $limit->rows ?? Rows::empty($rows->schema(), $context->backend()); $stop = true; } catch (Throwable $failure) { if ($failure instanceof SinkFailure) { diff --git a/src/core/etl/src/Flow/ETL/Executor/StatisticsCollector.php b/src/core/etl/src/Flow/ETL/Executor/StatisticsCollector.php index ea9d8ad556..b94a1a9e7e 100644 --- a/src/core/etl/src/Flow/ETL/Executor/StatisticsCollector.php +++ b/src/core/etl/src/Flow/ETL/Executor/StatisticsCollector.php @@ -61,7 +61,7 @@ public function __construct( } gc_collect_cycles(); - $this->memory = new Consumption(); + $this->memory = new Consumption($this->context->backend()); $this->startedAt = $this->context->config->clock()->now(); $this->startTime = HighResolutionTime::now(); } @@ -82,15 +82,9 @@ public function capture(Rows $rows): void $this->schema = $this->schema->merge($rows->schema()); } - if ($this->columnStatistics !== null) { - foreach ($rows->all() as $row) { - foreach ($rows->schema()->definitions() as $definition) { - $name = $definition->entry()->name(); - - if ($row->has($name)) { - $this->columnStatistics->add($definition, $row->get($name)); - } - } + if ($this->columnStatistics !== null && !$rows->isEmpty()) { + foreach ($rows->schema()->definitions() as $name => $definition) { + $this->columnStatistics->add($definition, $rows->column($name)); } } } diff --git a/src/core/etl/src/Flow/ETL/Extractor/ArrayExtractor.php b/src/core/etl/src/Flow/ETL/Extractor/ArrayExtractor.php index 4e44843471..72fcd2079c 100644 --- a/src/core/etl/src/Flow/ETL/Extractor/ArrayExtractor.php +++ b/src/core/etl/src/Flow/ETL/Extractor/ArrayExtractor.php @@ -7,6 +7,9 @@ use Flow\ETL\Cardinality; use Flow\ETL\Exception\InvalidLogicException; use Flow\ETL\Extractor; +use Flow\ETL\Extractor\Memory\InferredRows; +use Flow\ETL\Extractor\Memory\InMemoryRows; +use Flow\ETL\Extractor\Memory\SpilledRows; use Flow\ETL\FlowContext; use Flow\ETL\Schema; use Flow\ETL\Schema\Inference\SchemaInference; @@ -86,7 +89,7 @@ public function extract(FlowContext $context, ?int $limit = null): Generator continue; } - $signal = yield $batches->of($buffer, $schema, $context->hydrator()); + $signal = yield $batches->of($buffer, $schema, $context->backend()); if ($signal === Signal::STOP) { return; @@ -96,7 +99,7 @@ public function extract(FlowContext $context, ?int $limit = null): Generator } if ($buffer !== []) { - yield $batches->of($buffer, $schema, $context->hydrator()); + yield $batches->of($buffer, $schema, $context->backend()); } } diff --git a/src/core/etl/src/Flow/ETL/Extractor/BatchByExtractor.php b/src/core/etl/src/Flow/ETL/Extractor/BatchByExtractor.php index 81ae95d6f2..7ef17eaf19 100644 --- a/src/core/etl/src/Flow/ETL/Extractor/BatchByExtractor.php +++ b/src/core/etl/src/Flow/ETL/Extractor/BatchByExtractor.php @@ -4,16 +4,16 @@ namespace Flow\ETL\Extractor; +use Flow\ETL\Column\Backend; use Flow\ETL\Exception\InvalidArgumentException; use Flow\ETL\Extractor; use Flow\ETL\FlowContext; +use Flow\ETL\Processor\BatchingByProcessor; use Flow\ETL\Row\Reference; use Flow\ETL\Rows; use Flow\ETL\Schema; use Generator; -use function count; - final class BatchByExtractor implements Extractor, OverridingExtractor, RewindableExtractor { private ?Schema $schema = null; @@ -44,42 +44,27 @@ public function isRepeatable(): bool */ public function extract(FlowContext $context, ?int $limit = null): Generator { - // pinned from the declaration or the first child batch, then every later batch is matched to - // it - a buffer spans child batches, so its rows must all answer to one schema before trusted() - $schema = $this->schema; - - $buffer = []; - $currentGroupValue = null; - - foreach ($this->extractor->extract($context) as $rows) { - $schema ??= $rows->schema(); - $rows = $rows->matchTo($schema); - - foreach ($rows->all() as $row) { - $groupValue = $row->get($this->column); - - if ($currentGroupValue === null) { - $currentGroupValue = $groupValue; - } elseif ($currentGroupValue !== $groupValue) { - if ($this->minSize === null || count($buffer) >= $this->minSize) { - $signal = yield Rows::trusted($schema, $buffer); - - if ($signal === Signal::STOP) { - return; - } - - $buffer = []; - } + return (new BatchingByProcessor($this->column, $this->minSize))->process( + $this->declared($this->extractor->extract($context), $context->backend()), + $context, + ); + } - $currentGroupValue = $groupValue; - } + /** + * @param Generator $batches + * + * @return Generator + */ + public function declared(Generator $batches, Backend $backend): Generator + { + if ($this->schema === null) { + yield from $batches; - $buffer[] = $row; - } + return; } - if (count($buffer) > 0) { - yield Rows::trusted($schema ?? $this->schema(), $buffer); + foreach ($batches as $batch) { + yield $batch->matchTo($this->schema, $backend); } } diff --git a/src/core/etl/src/Flow/ETL/Extractor/BatchExtractor.php b/src/core/etl/src/Flow/ETL/Extractor/BatchExtractor.php index ccf19c498b..962f956c61 100644 --- a/src/core/etl/src/Flow/ETL/Extractor/BatchExtractor.php +++ b/src/core/etl/src/Flow/ETL/Extractor/BatchExtractor.php @@ -10,8 +10,6 @@ use Flow\ETL\Schema; use Generator; -use function count; - final class BatchExtractor implements BatchableExtractor, Extractor, OverridingExtractor, RewindableExtractor { use Batches; @@ -30,33 +28,41 @@ public function __construct( */ public function extract(FlowContext $context, ?int $limit = null): Generator { - // pinned from the declaration or the first child batch, then every later batch is matched to - // it - a buffer spans child batches, so its rows must all answer to one schema before trusted() + // pinned from the declaration or the first child batch, then every later batch is matched to it - a window + // spans child batches, so they must all answer to one schema before they are concatenated $schema = $this->schema; - $buffer = []; + /** @var list $pending */ + $pending = []; + $pendingCount = 0; foreach ($this->extractor->extract($context) as $rows) { $schema ??= $rows->schema(); - $rows = $rows->matchTo($schema); - - foreach ($rows->all() as $row) { - $buffer[] = $row; + $pending[] = $rows->matchTo($schema, $context->backend()); + $pendingCount += $rows->count(); - if (count($buffer) === $this->batchSize) { - $signal = yield Rows::trusted($schema, $buffer); + while ($pendingCount >= $this->batchSize) { + // one variadic concat per window copies each row once; a pairwise concat per child batch would + // re-copy everything pending + $window = Rows::empty($schema, $context->backend())->concat($context->backend(), ...$pending); + $signal = yield $window->slice(0, $this->batchSize); - if ($signal === Signal::STOP) { - return; - } - - $buffer = []; + if ($signal === Signal::STOP) { + return; } + + $pending = $window->count() > $this->batchSize + ? [$window->slice($this->batchSize, $window->count() - $this->batchSize)] + : []; + $pendingCount -= $this->batchSize; } } - if ($buffer !== []) { - yield Rows::trusted($schema ?? $this->schema(), $buffer); + if ($pendingCount > 0) { + yield Rows::empty($schema ?? $this->schema(), $context->backend())->concat( + $context->backend(), + ...$pending, + ); } } diff --git a/src/core/etl/src/Flow/ETL/Extractor/CacheExtractor.php b/src/core/etl/src/Flow/ETL/Extractor/CacheExtractor.php index 488727bbd4..a2488e42ab 100644 --- a/src/core/etl/src/Flow/ETL/Extractor/CacheExtractor.php +++ b/src/core/etl/src/Flow/ETL/Extractor/CacheExtractor.php @@ -47,7 +47,7 @@ public function extract(FlowContext $context, ?int $limit = null): Generator if (!$cache->has($this->id)) { if ($this->fallbackExtractor !== null) { foreach ($this->fallbackExtractor->extract($context) as $rows) { - $signal = yield $declared === null ? $rows : $rows->matchTo($declared); + $signal = yield $declared === null ? $rows : $rows->matchTo($declared, $context->backend()); if ($signal === Signal::STOP) { return; @@ -60,7 +60,7 @@ public function extract(FlowContext $context, ?int $limit = null): Generator foreach ($index->values() as $cacheKey) { $cached = $cache->get($cacheKey); - $signal = yield $folded === null ? $cached : $cached->matchTo($folded); + $signal = yield $folded === null ? $cached : $cached->matchTo($folded, $context->backend()); if ($signal === Signal::STOP) { return; diff --git a/src/core/etl/src/Flow/ETL/Extractor/ChainExtractor.php b/src/core/etl/src/Flow/ETL/Extractor/ChainExtractor.php index 6f4a444d11..df3a827f23 100644 --- a/src/core/etl/src/Flow/ETL/Extractor/ChainExtractor.php +++ b/src/core/etl/src/Flow/ETL/Extractor/ChainExtractor.php @@ -45,7 +45,7 @@ public function extract(FlowContext $context, ?int $limit = null): Generator foreach ($this->extractors as $extractor) { foreach ($extractor->extract($context) as $rows) { - $signal = yield $rows->matchTo($schema); + $signal = yield $rows->matchTo($schema, $context->backend()); if ($signal === Signal::STOP) { return; diff --git a/src/core/etl/src/Flow/ETL/Extractor/CollectingExtractor.php b/src/core/etl/src/Flow/ETL/Extractor/CollectingExtractor.php index 159e1c1602..b6be4394b0 100644 --- a/src/core/etl/src/Flow/ETL/Extractor/CollectingExtractor.php +++ b/src/core/etl/src/Flow/ETL/Extractor/CollectingExtractor.php @@ -26,13 +26,18 @@ public function extract(FlowContext $context, ?int $limit = null): Generator foreach ($this->extractor->extract($context) as $rows) { if ($schema !== null) { - $rows = $rows->matchTo($schema); + $rows = $rows->matchTo($schema, $context->backend()); } - $collectedRows = $collectedRows === null ? $rows : $collectedRows->merge($rows); + // an empty part adds nothing, whatever its schema + $collectedRows = match (true) { + $collectedRows === null, $collectedRows->isEmpty() => $rows, + $rows->isEmpty() => $collectedRows, + default => $collectedRows->concat($context->backend(), $rows), + }; } - yield $collectedRows ?? new Rows($this->schema()); + yield $collectedRows ?? Rows::empty($this->schema(), $context->backend()); } public function isRepeatable(): bool diff --git a/src/core/etl/src/Flow/ETL/Extractor/DataFrameExtractor.php b/src/core/etl/src/Flow/ETL/Extractor/DataFrameExtractor.php index cf3c7a86ea..febb950e9c 100644 --- a/src/core/etl/src/Flow/ETL/Extractor/DataFrameExtractor.php +++ b/src/core/etl/src/Flow/ETL/Extractor/DataFrameExtractor.php @@ -11,11 +11,10 @@ use Flow\ETL\Plan\Node\Limit; use Flow\ETL\Plan\Trigger; use Flow\ETL\Rows; +use Flow\ETL\Rows\ColumnRetyping; use Flow\ETL\Schema; use Generator; -use function Flow\ETL\DSL\array_to_rows; - final class DataFrameExtractor implements RewindableExtractor { private ?Schema $schema = null; @@ -46,7 +45,19 @@ public function extract(FlowContext $context, ?int $limit = null): Generator foreach ($config->executor()->execute($config->planner()->plan($logical, $this->plan->context)) as $rows) { if ($this->schema !== null) { - $rows = array_to_rows($rows->toArray(), $this->schema, $context->hydrator()); + $backend = $context->backend(); + $retyping = new ColumnRetyping(); + $columns = []; + + foreach ($this->schema->definitions() as $name => $definition) { + $own = $rows->schema()->findDefinition($definition->entry()->name()); + + $columns[$name] = $own === null + ? $retyping->absent($definition, $rows->count(), $backend) + : $retyping->cast($rows->column($definition->entry()->name()), $own, $definition, $backend); + } + + $rows = Rows::fromColumns($this->schema, $columns, $rows->count()); } $signal = yield $rows; diff --git a/src/core/etl/src/Flow/ETL/Extractor/FeedExtractor.php b/src/core/etl/src/Flow/ETL/Extractor/FeedExtractor.php index e242faafd5..769f446420 100644 --- a/src/core/etl/src/Flow/ETL/Extractor/FeedExtractor.php +++ b/src/core/etl/src/Flow/ETL/Extractor/FeedExtractor.php @@ -59,7 +59,7 @@ public function extract(FlowContext $context, ?int $limit = null): Generator // is parked, so both operands can change between the yield and this check. if ($this->batch === null && !$this->finished) { // a downstream projection still sees the columns it was built against - $signal = yield new Rows($this->schema); + $signal = yield Rows::empty($this->schema, $context->backend()); if ($signal === Signal::STOP) { return; diff --git a/src/core/etl/src/Flow/ETL/Extractor/DeclaresPartitionTypes.php b/src/core/etl/src/Flow/ETL/Extractor/File/DeclaresPartitionTypes.php similarity index 91% rename from src/core/etl/src/Flow/ETL/Extractor/DeclaresPartitionTypes.php rename to src/core/etl/src/Flow/ETL/Extractor/File/DeclaresPartitionTypes.php index 6ab9804563..fe46d0c6b0 100644 --- a/src/core/etl/src/Flow/ETL/Extractor/DeclaresPartitionTypes.php +++ b/src/core/etl/src/Flow/ETL/Extractor/File/DeclaresPartitionTypes.php @@ -2,7 +2,7 @@ declare(strict_types=1); -namespace Flow\ETL\Extractor; +namespace Flow\ETL\Extractor\File; trait DeclaresPartitionTypes { diff --git a/src/core/etl/src/Flow/ETL/Extractor/File/DerivedSchema.php b/src/core/etl/src/Flow/ETL/Extractor/File/DerivedSchema.php new file mode 100644 index 0000000000..12a34851d0 --- /dev/null +++ b/src/core/etl/src/Flow/ETL/Extractor/File/DerivedSchema.php @@ -0,0 +1,35 @@ +validate($this->derived, $schema); + + if (!$validation->isValid()) { + throw InferredSchemaException::filesDiverge($source, $this->derivedFrom, $validation); + } + } +} diff --git a/src/core/etl/src/Flow/ETL/Extractor/File/FileBatches.php b/src/core/etl/src/Flow/ETL/Extractor/File/FileBatches.php new file mode 100644 index 0000000000..6e33736842 --- /dev/null +++ b/src/core/etl/src/Flow/ETL/Extractor/File/FileBatches.php @@ -0,0 +1,38 @@ + $batchSize + * + * @return Generator + */ + public function batches( + SourceFile $source, + Schema $body, + int $batchSize, + Backend $backend, + ReadWindow $window, + ): Generator; +} diff --git a/src/core/etl/src/Flow/ETL/Extractor/FileColumns.php b/src/core/etl/src/Flow/ETL/Extractor/File/FileColumns.php similarity index 88% rename from src/core/etl/src/Flow/ETL/Extractor/FileColumns.php rename to src/core/etl/src/Flow/ETL/Extractor/File/FileColumns.php index 9d86a02afb..96f1aca713 100644 --- a/src/core/etl/src/Flow/ETL/Extractor/FileColumns.php +++ b/src/core/etl/src/Flow/ETL/Extractor/File/FileColumns.php @@ -2,10 +2,9 @@ declare(strict_types=1); -namespace Flow\ETL\Extractor; +namespace Flow\ETL\Extractor\File; use Flow\ETL\Exception\InvalidArgumentException; -use Flow\ETL\Rows; use Flow\ETL\Schema; use Flow\Filesystem\Partition; use Throwable; @@ -31,11 +30,6 @@ public function __construct( private bool $metadataColumns, ) {} - public function apply(Rows $rows): Rows - { - return $this->partitionColumns->apply($rows, $this->partitionNames, $this->partitionTypes); - } - public function declare(Schema $schema): Schema { return $this->partitionColumns->declare( @@ -94,12 +88,7 @@ public function forFile(SourceFile $source, Schema $declared): FileConstants } } - return new FileConstants( - $this->partitionColumns, - $this->metadataColumns ? $source->uri() : null, - $this->partitionNames, - $values, - ); + return new FileConstants($this->metadataColumns ? $source->uri() : null, $this->partitionNames, $values); } /** diff --git a/src/core/etl/src/Flow/ETL/Extractor/File/FileConstants.php b/src/core/etl/src/Flow/ETL/Extractor/File/FileConstants.php new file mode 100644 index 0000000000..b9a5bb8462 --- /dev/null +++ b/src/core/etl/src/Flow/ETL/Extractor/File/FileConstants.php @@ -0,0 +1,62 @@ + $partitionNames + * @param array $partitionValues already the type the declared schema gives them + */ + public function __construct( + private ?string $uri, + private array $partitionNames, + private array $partitionValues, + ) {} + + /** + * The file's value of every column this file adds - `_input_file_uri` when metadata columns are on, then each + * partition column (null when the path does not carry it). + * + * @return array + */ + public function values(): array + { + $values = $this->uri === null ? [] : ['_input_file_uri' => $this->uri]; + + foreach ($this->partitionNames as $name => $_) { + $values[$name] = array_key_exists($name, $this->partitionValues) ? $this->partitionValues[$name] : null; + } + + return $values; + } + + /** + * values() over a batch the reader already matched to the file's body schema, adopting $declared - the schema + * FileColumns::declare() built over that body schema. A batch with nothing to add is returned as it is. + */ + public function fillRows(Rows $rows, Schema $declared, Backend $backend): Rows + { + if ($this->uri === null && $this->partitionNames === []) { + return $rows; + } + + $added = []; + + // @mago-ignore analysis:mixed-assignment + foreach ($this->values() as $name => $value) { + $added[$name] = $backend->constant($declared->get($name), $value, $rows->count()); + } + + return $rows->withColumns($declared, $added); + } +} diff --git a/src/core/etl/src/Flow/ETL/Extractor/FileExtractor.php b/src/core/etl/src/Flow/ETL/Extractor/File/FileExtractor.php similarity index 95% rename from src/core/etl/src/Flow/ETL/Extractor/FileExtractor.php rename to src/core/etl/src/Flow/ETL/Extractor/File/FileExtractor.php index a86f5ded4a..f8b5369cc0 100644 --- a/src/core/etl/src/Flow/ETL/Extractor/FileExtractor.php +++ b/src/core/etl/src/Flow/ETL/Extractor/File/FileExtractor.php @@ -2,9 +2,10 @@ declare(strict_types=1); -namespace Flow\ETL\Extractor; +namespace Flow\ETL\Extractor\File; use Flow\ETL\Extractor; +use Flow\ETL\Extractor\Signal; use Flow\ETL\FlowContext; use Flow\ETL\Rows; use Flow\ETL\Schema; diff --git a/src/core/etl/src/Flow/ETL/Extractor/File/FileReadLoop.php b/src/core/etl/src/Flow/ETL/Extractor/File/FileReadLoop.php new file mode 100644 index 0000000000..3375c2b4d1 --- /dev/null +++ b/src/core/etl/src/Flow/ETL/Extractor/File/FileReadLoop.php @@ -0,0 +1,86 @@ + $sources + * @param int<1, max> $batchSize + * + * @return Generator + */ + public function read( + array $sources, + FileBatches $batches, + int $batchSize, + Backend $backend, + ReadWindow $window, + ): Generator { + $body = $this->fileColumns->withoutTail($this->schema); + $offset = $window->offset; + $skips = $batches instanceof OffsetSkippingFileBatches; + $yielded = 0; + + foreach ($sources as $source) { + $constants = $this->fileColumns->forFile($source, $this->schema); + $wanted = $window->limit === null ? null : $window->limit - $yielded; + $file = $batches->batches( + $source, + $body, + $batchSize, + $backend, + // a format that cannot skip has to produce the offset it leaves to the loop before the rows wanted + $skips + ? new ReadWindow($offset, $wanted) + : new ReadWindow(0, $wanted === null ? null : $wanted + $offset), + ); + + foreach ($file as $batch) { + if (!$skips && $offset > 0) { + $skipped = min($offset, $batch->count()); + $offset -= $skipped; + + if ($skipped === $batch->count()) { + continue; + } + + $batch = $batch->slice($skipped, $batch->count() - $skipped); + } + + if ($window->limit !== null && $batch->count() > ($window->limit - $yielded)) { + $batch = $batch->slice(0, $window->limit - $yielded); + } + + $rows = $constants->fillRows($batch, $this->schema, $backend); + $yielded += $rows->count(); + $signal = yield $rows; + + if ($signal === Signal::STOP) { + return; + } + + if ($window->limit !== null && $yielded >= $window->limit) { + return; + } + } + + $offset -= $file->getReturn(); + } + } +} diff --git a/src/core/etl/src/Flow/ETL/Extractor/FileReading.php b/src/core/etl/src/Flow/ETL/Extractor/File/FileReading.php similarity index 96% rename from src/core/etl/src/Flow/ETL/Extractor/FileReading.php rename to src/core/etl/src/Flow/ETL/Extractor/File/FileReading.php index bd57eb3735..361e85a034 100644 --- a/src/core/etl/src/Flow/ETL/Extractor/FileReading.php +++ b/src/core/etl/src/Flow/ETL/Extractor/File/FileReading.php @@ -2,7 +2,7 @@ declare(strict_types=1); -namespace Flow\ETL\Extractor; +namespace Flow\ETL\Extractor\File; use Flow\Filesystem\FileListing; use Flow\Filesystem\Filesystem; diff --git a/src/core/etl/src/Flow/ETL/Extractor/FooterStatistics.php b/src/core/etl/src/Flow/ETL/Extractor/File/FooterStatistics.php similarity index 96% rename from src/core/etl/src/Flow/ETL/Extractor/FooterStatistics.php rename to src/core/etl/src/Flow/ETL/Extractor/File/FooterStatistics.php index ecfb822ab0..cfa7a287d3 100644 --- a/src/core/etl/src/Flow/ETL/Extractor/FooterStatistics.php +++ b/src/core/etl/src/Flow/ETL/Extractor/File/FooterStatistics.php @@ -2,10 +2,11 @@ declare(strict_types=1); -namespace Flow\ETL\Extractor; +namespace Flow\ETL\Extractor\File; use Flow\ETL\Cardinality; use Flow\ETL\Exception\InvalidArgumentException; +use Flow\ETL\Extractor\Statistics; use function max; use function round; diff --git a/src/core/etl/src/Flow/ETL/Extractor/File/InferredColumns.php b/src/core/etl/src/Flow/ETL/Extractor/File/InferredColumns.php new file mode 100644 index 0000000000..f4117cca6f --- /dev/null +++ b/src/core/etl/src/Flow/ETL/Extractor/File/InferredColumns.php @@ -0,0 +1,54 @@ + $tail the file constant columns, never part of a file's own columns + * @param string $inferredFrom the source the inferred header was read from + */ + public function __construct( + private Schema $inferred, + private array $tail, + private SchemaInference $inference, + private string $inferredFrom, + ) {} + + /** + * Refuses a file whose own columns (header) are not the inferred ones; a file with no columns passes. + * + * @param list $columns the file's columns, constant columns and duplicates allowed + * + * @throws InferredSchemaException + */ + public function refuseDivergence(string $source, array $columns): void + { + $expected = $this->inferred->references()->names(); + $columns = array_values(array_diff(array_unique($columns), $this->tail)); + + if ($columns !== [] && (array_diff($columns, $expected) !== [] || array_diff($expected, $columns) !== [])) { + throw InferredSchemaException::columnsDiverge( + $source, + $this->inferredFrom, + $this->inferred, + $columns, + $this->inference, + ); + } + } +} diff --git a/src/core/etl/src/Flow/ETL/Extractor/ListedFiles.php b/src/core/etl/src/Flow/ETL/Extractor/File/ListedFiles.php similarity index 97% rename from src/core/etl/src/Flow/ETL/Extractor/ListedFiles.php rename to src/core/etl/src/Flow/ETL/Extractor/File/ListedFiles.php index c6c7c6e9bd..2b5c0b3733 100644 --- a/src/core/etl/src/Flow/ETL/Extractor/ListedFiles.php +++ b/src/core/etl/src/Flow/ETL/Extractor/File/ListedFiles.php @@ -2,7 +2,7 @@ declare(strict_types=1); -namespace Flow\ETL\Extractor; +namespace Flow\ETL\Extractor\File; use Flow\ETL\Cardinality; use Flow\ETL\Exception\InvalidArgumentException; diff --git a/src/core/etl/src/Flow/ETL/Extractor/ListingPartitions.php b/src/core/etl/src/Flow/ETL/Extractor/File/ListingPartitions.php similarity index 95% rename from src/core/etl/src/Flow/ETL/Extractor/ListingPartitions.php rename to src/core/etl/src/Flow/ETL/Extractor/File/ListingPartitions.php index 5bd6f8eabb..a52430dd9e 100644 --- a/src/core/etl/src/Flow/ETL/Extractor/ListingPartitions.php +++ b/src/core/etl/src/Flow/ETL/Extractor/File/ListingPartitions.php @@ -2,7 +2,7 @@ declare(strict_types=1); -namespace Flow\ETL\Extractor; +namespace Flow\ETL\Extractor\File; use Flow\Filesystem\Filesystem; use Flow\Filesystem\Path; diff --git a/src/core/etl/src/Flow/ETL/Extractor/MetadataColumns.php b/src/core/etl/src/Flow/ETL/Extractor/File/MetadataColumns.php similarity index 88% rename from src/core/etl/src/Flow/ETL/Extractor/MetadataColumns.php rename to src/core/etl/src/Flow/ETL/Extractor/File/MetadataColumns.php index de3a9247fe..6d77ae383d 100644 --- a/src/core/etl/src/Flow/ETL/Extractor/MetadataColumns.php +++ b/src/core/etl/src/Flow/ETL/Extractor/File/MetadataColumns.php @@ -2,7 +2,7 @@ declare(strict_types=1); -namespace Flow\ETL\Extractor; +namespace Flow\ETL\Extractor\File; trait MetadataColumns { diff --git a/src/core/etl/src/Flow/ETL/Extractor/MetadataColumnsExtractor.php b/src/core/etl/src/Flow/ETL/Extractor/File/MetadataColumnsExtractor.php similarity index 80% rename from src/core/etl/src/Flow/ETL/Extractor/MetadataColumnsExtractor.php rename to src/core/etl/src/Flow/ETL/Extractor/File/MetadataColumnsExtractor.php index e93259fd0a..891f0ec6dc 100644 --- a/src/core/etl/src/Flow/ETL/Extractor/MetadataColumnsExtractor.php +++ b/src/core/etl/src/Flow/ETL/Extractor/File/MetadataColumnsExtractor.php @@ -2,7 +2,7 @@ declare(strict_types=1); -namespace Flow\ETL\Extractor; +namespace Flow\ETL\Extractor\File; interface MetadataColumnsExtractor { diff --git a/src/core/etl/src/Flow/ETL/Extractor/File/OffsetSkippingFileBatches.php b/src/core/etl/src/Flow/ETL/Extractor/File/OffsetSkippingFileBatches.php new file mode 100644 index 0000000000..90fd4375a1 --- /dev/null +++ b/src/core/etl/src/Flow/ETL/Extractor/File/OffsetSkippingFileBatches.php @@ -0,0 +1,11 @@ + partition name => nullable + */ + public function names(Path $path, Filter $filter): array + { + $counts = []; + $nullable = []; + $paths = 0; + + foreach ((new FileListing($this->filesystem))->list($path, $filter) as $status) { + $paths++; + + foreach ($status->path->partitions() as $partition) { + $counts[$partition->name] = ($counts[$partition->name] ?? 0) + 1; + + if ($partition->value === null) { + $nullable[$partition->name] = true; + } + } + } + + $names = []; + + foreach ($counts as $name => $count) { + $names[$name] = $count < $paths || array_key_exists($name, $nullable); + } + + ksort($names); + + return $names; + } + + /** + * @param array $names + */ + public function declare(Schema $schema, array $names, PartitionTypes $types = new PartitionTypes()): Schema + { + $types->assertEveryNameIsAPartition($names); + + if ($names === []) { + return $schema; + } + + $definitions = []; + + foreach ($names as $name => $nullable) { + // 1) a declared schema wins, 2) then a declared partition type, 3) then string + $definitions[] = + $schema->findDefinition($name) + ?? ( + $types->has($name) + ? definition_from_type($name, $types->get($name), nullable: $nullable) + : str_schema($name, nullable: $nullable) + ); + } + + return $schema->gracefulRemove(...array_keys($names))->add(...$definitions); + } +} diff --git a/src/core/etl/src/Flow/ETL/Extractor/PartitionTypes.php b/src/core/etl/src/Flow/ETL/Extractor/File/PartitionTypes.php similarity index 97% rename from src/core/etl/src/Flow/ETL/Extractor/PartitionTypes.php rename to src/core/etl/src/Flow/ETL/Extractor/File/PartitionTypes.php index ddd4bcf8cb..fee4ac9b32 100644 --- a/src/core/etl/src/Flow/ETL/Extractor/PartitionTypes.php +++ b/src/core/etl/src/Flow/ETL/Extractor/File/PartitionTypes.php @@ -2,7 +2,7 @@ declare(strict_types=1); -namespace Flow\ETL\Extractor; +namespace Flow\ETL\Extractor\File; use Flow\ETL\Exception\InvalidArgumentException; use Flow\Types\Type; diff --git a/src/core/etl/src/Flow/ETL/Extractor/PathFiltering.php b/src/core/etl/src/Flow/ETL/Extractor/File/PathFiltering.php similarity index 76% rename from src/core/etl/src/Flow/ETL/Extractor/PathFiltering.php rename to src/core/etl/src/Flow/ETL/Extractor/File/PathFiltering.php index 27772816f5..88db0dea72 100644 --- a/src/core/etl/src/Flow/ETL/Extractor/PathFiltering.php +++ b/src/core/etl/src/Flow/ETL/Extractor/File/PathFiltering.php @@ -2,9 +2,10 @@ declare(strict_types=1); -namespace Flow\ETL\Extractor; +namespace Flow\ETL\Extractor\File; use Flow\ETL\Cardinality; +use Flow\ETL\Extractor\Statistics; use Flow\ETL\Schema; use Flow\Filesystem\Path; use Flow\Filesystem\Path\Filter\OnlyFiles; @@ -83,32 +84,6 @@ private function derivedFooters(Generator $files, bool $unionByName): FooterStat return $this->derivedFooters ?? new FooterStatistics(new Statistics(), 0); } - /** - * The same footers without deriving a schema, for a read whose schema was declared: nothing else opens them. - * - * @param Generator $files an unstarted generator - */ - private function footers(Generator $files, bool $unionByName): FooterStatistics - { - $statistics = new Statistics(rows: Cardinality::exact(0), size: Cardinality::exact(0)); - $read = 0; - - foreach ($files as $file) { - try { - $statistics = $statistics->merge($file->statistics()); - $read++; - } finally { - $file->close(); - } - - if (!$unionByName) { - break; - } - } - - return new FooterStatistics($statistics, $read); - } - /** * @return array */ diff --git a/src/core/etl/src/Flow/ETL/Extractor/File/ReadWindow.php b/src/core/etl/src/Flow/ETL/Extractor/File/ReadWindow.php new file mode 100644 index 0000000000..61b531ae5e --- /dev/null +++ b/src/core/etl/src/Flow/ETL/Extractor/File/ReadWindow.php @@ -0,0 +1,29 @@ + $partitionNames - * @param array $partitionValues already the type the declared schema gives them - */ - public function __construct( - private PartitionColumns $partitionColumns, - private ?string $uri, - private array $partitionNames, - private array $partitionValues, - ) {} - - /** - * @param array $row - * - * @return array - */ - public function fill(array $row): array - { - // plain assignment, unlike the partition columns below: a body column named _input_file_uri - // keeps its position and is overwritten in place - if ($this->uri !== null) { - $row['_input_file_uri'] = $this->uri; - } - - return $this->partitionColumns->fill($row, $this->partitionNames, $this->partitionValues); - } - - /** - * fill() over a batch the reader already matched to the file's body schema, adopting $declared - the schema - * FileColumns::declare() built over that body schema. A batch with nothing to add is returned as it is. - */ - public function fillRows(Rows $rows, Schema $declared): Rows - { - if ($this->uri === null && $this->partitionNames === []) { - return $rows; - } - - $filled = []; - - foreach ($rows->all() as $row) { - $filled[] = new Row($this->fill($row->values())); - } - - // declare() appends the tail in the order fill() writes it, so the rows need no second check - return Rows::trusted($declared, $filled); - } -} diff --git a/src/core/etl/src/Flow/ETL/Extractor/FilesExtractor.php b/src/core/etl/src/Flow/ETL/Extractor/FilesExtractor.php index 3f4383ab28..41295a8a14 100644 --- a/src/core/etl/src/Flow/ETL/Extractor/FilesExtractor.php +++ b/src/core/etl/src/Flow/ETL/Extractor/FilesExtractor.php @@ -7,7 +7,12 @@ use Flow\ETL\Cardinality; use Flow\ETL\Exception\InvalidArgumentException; use Flow\ETL\Extractor; +use Flow\ETL\Extractor\File\FileExtractor; +use Flow\ETL\Extractor\File\ListedFiles; +use Flow\ETL\Extractor\File\ListingPartitions; +use Flow\ETL\Extractor\File\SourceFile; use Flow\ETL\FlowContext; +use Flow\ETL\Rows\RowsBuilder; use Flow\ETL\Schema; use Flow\Filesystem\FileListing; use Flow\Filesystem\Filesystem; @@ -17,8 +22,6 @@ use Flow\Filesystem\Path\Filter\OnlyFiles; use Generator; -use function count; -use function Flow\ETL\DSL\array_to_rows; use function Flow\ETL\DSL\bool_schema; use function Flow\ETL\DSL\schema; use function Flow\ETL\DSL\str_schema; @@ -63,16 +66,16 @@ public function isRepeatable(): bool public function extract(FlowContext $context, ?int $limit = null, Filter $pathFilter = new OnlyFiles()): Generator { $batchSize = $this->batchSize(); + $backend = $context->backend(); $fileColumns = $this->fileColumns($this->filesystem, $this->path); $schema = $this->schema(); - $buffer = []; + $builder = new RowsBuilder($schema, $backend); $yielded = 0; foreach ((new FileListing($this->filesystem))->list($this->path, $pathFilter) as $fileStatus) { - $constants = $fileColumns->forFile(new SourceFile($fileStatus->path, $fileStatus->size), $schema); $extension = $fileStatus->path->extension(); - $buffer[] = $constants->fill([ + $builder->append([ 'path' => $fileStatus->path->path(), 'protocol' => $fileStatus->path->protocol(), 'file_name' => $fileStatus->path->filename(), @@ -82,29 +85,31 @@ public function extract(FlowContext $context, ?int $limit = null, Filter $pathFi // Path::extension() answers false for an extensionless file; the column is one // type, so the absence is spelled null rather than a boolean in a string column. 'extension' => $extension === false ? null : $extension, + ...$fileColumns->forFile(new SourceFile($fileStatus->path, $fileStatus->size), $schema)->values(), ]); - if (count($buffer) < $batchSize) { + if ($builder->count() < $batchSize) { continue; } - $yielded += count($buffer); + $rows = $builder->finish(); + $yielded += $rows->count(); - $signal = yield array_to_rows($buffer, $schema, $context->hydrator()); + $signal = yield $rows; if ($signal === Signal::STOP) { return; } - $buffer = []; + $builder = new RowsBuilder($schema, $backend); if ($limit !== null && $yielded >= $limit) { return; } } - if ($buffer !== []) { - yield array_to_rows($buffer, $schema, $context->hydrator()); + if ($builder->count() > 0) { + yield $builder->finish(); } } diff --git a/src/core/etl/src/Flow/ETL/Extractor/Grid/HeaderNames.php b/src/core/etl/src/Flow/ETL/Extractor/Grid/HeaderNames.php new file mode 100644 index 0000000000..aef355d841 --- /dev/null +++ b/src/core/etl/src/Flow/ETL/Extractor/Grid/HeaderNames.php @@ -0,0 +1,34 @@ + $cells + * + * @return list + */ + public function of(array $cells): array + { + $names = []; + + // @mago-ignore analysis:mixed-assignment + foreach ($cells as $index => $cell) { + $name = trim(is_scalar($cell) ? (string) $cell : ''); + $names[] = $name !== '' ? $name : 'e' . str_pad((string) $index, 2, '0', STR_PAD_LEFT); + } + + return $names; + } +} diff --git a/src/core/etl/src/Flow/ETL/Extractor/Grid/RecordDecoder.php b/src/core/etl/src/Flow/ETL/Extractor/Grid/RecordDecoder.php new file mode 100644 index 0000000000..b02c610d2f --- /dev/null +++ b/src/core/etl/src/Flow/ETL/Extractor/Grid/RecordDecoder.php @@ -0,0 +1,89 @@ + + */ + private ?array $headers = null; + + public function __construct( + private readonly bool $withHeader, + private readonly bool $emptyToNull, + private readonly bool $dropExtraColumns = true, + ) {} + + /** + * Records before the header is resolved that hold no cell are skipped. A cell a record lacks is null; a present + * '' cell is null under emptyToNull. + * + * @param list> $batch + * + * @throws InvalidArgumentException a record with more cells than headers, unless extra columns are dropped + * + * @return list> + */ + public function decode(array $batch): array + { + $decoded = []; + + foreach ($batch as $record) { + $cells = array_values($record); + + if ($this->headers === null) { + if ($cells === []) { + continue; + } + + $this->headers = (new HeaderNames())->of( + $this->withHeader ? $cells : array_fill(0, count($cells), null), + ); + + if ($this->withHeader) { + continue; + } + } + + if (!$this->dropExtraColumns && count($cells) > count($this->headers)) { + throw InvalidArgumentException::because( + 'Row has more columns (%d) than headers (%d)', + count($cells), + count($this->headers), + ); + } + + $values = []; + + foreach ($this->headers as $index => $name) { + // @mago-ignore analysis:mixed-assignment + $cell = $cells[$index] ?? null; + $values[$name] = $this->emptyToNull && $cell === '' ? null : $cell; + } + + $decoded[] = $values; + } + + return $decoded; + } + + /** + * @return null|list null before the first decode() resolved them + */ + public function headers(): ?array + { + return $this->headers; + } +} diff --git a/src/core/etl/src/Flow/ETL/Extractor/InMemoryRows.php b/src/core/etl/src/Flow/ETL/Extractor/Memory/InMemoryRows.php similarity index 85% rename from src/core/etl/src/Flow/ETL/Extractor/InMemoryRows.php rename to src/core/etl/src/Flow/ETL/Extractor/Memory/InMemoryRows.php index a917981dc5..64b1ba9103 100644 --- a/src/core/etl/src/Flow/ETL/Extractor/InMemoryRows.php +++ b/src/core/etl/src/Flow/ETL/Extractor/Memory/InMemoryRows.php @@ -2,10 +2,9 @@ declare(strict_types=1); -namespace Flow\ETL\Extractor; +namespace Flow\ETL\Extractor\Memory; use Flow\ETL\Row\ColumnName; -use Flow\ETL\Row\RawRowValues; use Flow\ETL\Schema\Inference\SchemaSampler; use Generator; @@ -25,7 +24,7 @@ public function __construct( * $rowBudget is never rationed: the inner generator is lazy and SchemaInferrer stops advancing it * when the budget is spent. * - * @return iterable> + * @return iterable>> */ public function samples(int $rowBudget): iterable { @@ -36,9 +35,9 @@ public function samples(int $rowBudget): iterable /** * The same ColumnName normalisation array_to_rows() applies, so the fold's names are the - * hydrator's names. + * batch builder's names. * - * @return Generator + * @return Generator> */ public function values(): Generator { @@ -52,7 +51,7 @@ public function values(): Generator $values[$columnName->of($key)] = $value; } - yield new RawRowValues($values); + yield $values; } } } diff --git a/src/core/etl/src/Flow/ETL/Extractor/InferredRows.php b/src/core/etl/src/Flow/ETL/Extractor/Memory/InferredRows.php similarity index 82% rename from src/core/etl/src/Flow/ETL/Extractor/InferredRows.php rename to src/core/etl/src/Flow/ETL/Extractor/Memory/InferredRows.php index 4ffd696729..159ff6919f 100644 --- a/src/core/etl/src/Flow/ETL/Extractor/InferredRows.php +++ b/src/core/etl/src/Flow/ETL/Extractor/Memory/InferredRows.php @@ -2,17 +2,17 @@ declare(strict_types=1); -namespace Flow\ETL\Extractor; +namespace Flow\ETL\Extractor\Memory; +use Flow\ETL\Column\Backend; use Flow\ETL\Exception\InferredSchemaException; use Flow\ETL\Exception\SchemaMismatchException; -use Flow\ETL\Row\Hydrator; use Flow\ETL\Rows; +use Flow\ETL\Rows\RowsBuilder; use Flow\ETL\Schema; use Flow\ETL\Schema\Inference\SchemaInference; use function count; -use function Flow\ETL\DSL\array_to_rows; /** * The gate places a refused row in its batch; only the source knows where that batch started, so a row past a @@ -35,10 +35,12 @@ public function __construct( * * @throws InferredSchemaException|SchemaMismatchException */ - public function of(array $rows, Schema $schema, Hydrator $hydrator): Rows + public function of(array $rows, Schema $schema, Backend $backend): Rows { try { - $batch = array_to_rows($rows, $schema, $hydrator); + $batch = (new RowsBuilder($schema, $backend)) + ->appendProjected($rows) + ->finish(); } catch (SchemaMismatchException $mismatch) { $row = $this->offset + $mismatch->rowIndex; diff --git a/src/core/etl/src/Flow/ETL/Extractor/SpillState.php b/src/core/etl/src/Flow/ETL/Extractor/Memory/SpillState.php similarity index 92% rename from src/core/etl/src/Flow/ETL/Extractor/SpillState.php rename to src/core/etl/src/Flow/ETL/Extractor/Memory/SpillState.php index 129b952cdc..9a36a8c6d4 100644 --- a/src/core/etl/src/Flow/ETL/Extractor/SpillState.php +++ b/src/core/etl/src/Flow/ETL/Extractor/Memory/SpillState.php @@ -2,7 +2,7 @@ declare(strict_types=1); -namespace Flow\ETL\Extractor; +namespace Flow\ETL\Extractor\Memory; /** * The lifecycle of one SpilledRows. Split into states rather than carried as booleans because the diff --git a/src/core/etl/src/Flow/ETL/Extractor/SpilledRows.php b/src/core/etl/src/Flow/ETL/Extractor/Memory/SpilledRows.php similarity index 96% rename from src/core/etl/src/Flow/ETL/Extractor/SpilledRows.php rename to src/core/etl/src/Flow/ETL/Extractor/Memory/SpilledRows.php index d5a0caa454..b4562a732d 100644 --- a/src/core/etl/src/Flow/ETL/Extractor/SpilledRows.php +++ b/src/core/etl/src/Flow/ETL/Extractor/Memory/SpilledRows.php @@ -2,10 +2,9 @@ declare(strict_types=1); -namespace Flow\ETL\Extractor; +namespace Flow\ETL\Extractor\Memory; use Flow\ETL\Exception\InvalidLogicException; -use Flow\ETL\Row\RawRowValues; use Flow\ETL\Schema\Inference\SchemaSampler; use Flow\Filesystem\Filesystem; use Flow\Filesystem\Path; @@ -143,7 +142,7 @@ public function rows(): Generator * $rowBudget is IGNORED: every row is written regardless, which is what makes a partially written * spill unreachable by construction rather than merely unlikely. * - * @return iterable> + * @return iterable>> * * @throws InvalidLogicException when the source was already read */ @@ -192,8 +191,8 @@ public function samples(int $rowBudget): iterable } }; - // InMemoryRows owns the raw-row to RawRowValues naming; keeping a second copy here would put the - // "same names array_to_rows() gives" invariant in two places. + // InMemoryRows owns the raw-row naming; keeping a second copy here would put the "same names array_to_rows() + // gives" invariant in two places. return (new InMemoryRows($spilling()))->samples($rowBudget); } diff --git a/src/core/etl/src/Flow/ETL/Extractor/MemoryExtractor.php b/src/core/etl/src/Flow/ETL/Extractor/MemoryExtractor.php index 2c62187509..1161a8b1a1 100644 --- a/src/core/etl/src/Flow/ETL/Extractor/MemoryExtractor.php +++ b/src/core/etl/src/Flow/ETL/Extractor/MemoryExtractor.php @@ -5,6 +5,8 @@ namespace Flow\ETL\Extractor; use Flow\ETL\Extractor; +use Flow\ETL\Extractor\Memory\InferredRows; +use Flow\ETL\Extractor\Memory\InMemoryRows; use Flow\ETL\FlowContext; use Flow\ETL\Memory\Memory; use Flow\ETL\Schema; @@ -54,7 +56,7 @@ public function extract(FlowContext $context, ?int $limit = null): Generator continue; } - $signal = yield $batches->of($buffer, $schema, $context->hydrator()); + $signal = yield $batches->of($buffer, $schema, $context->backend()); if ($signal === Signal::STOP) { return; @@ -64,7 +66,7 @@ public function extract(FlowContext $context, ?int $limit = null): Generator } if ($buffer !== []) { - yield $batches->of($buffer, $schema, $context->hydrator()); + yield $batches->of($buffer, $schema, $context->backend()); } } diff --git a/src/core/etl/src/Flow/ETL/Extractor/PartitionColumns.php b/src/core/etl/src/Flow/ETL/Extractor/PartitionColumns.php deleted file mode 100644 index 3c3bc991f3..0000000000 --- a/src/core/etl/src/Flow/ETL/Extractor/PartitionColumns.php +++ /dev/null @@ -1,123 +0,0 @@ - partition name => nullable - */ - public function names(Path $path, Filter $filter): array - { - $counts = []; - $nullable = []; - $paths = 0; - - foreach ((new FileListing($this->filesystem))->list($path, $filter) as $status) { - $paths++; - - foreach ($status->path->partitions() as $partition) { - $counts[$partition->name] = ($counts[$partition->name] ?? 0) + 1; - - if ($partition->value === null) { - $nullable[$partition->name] = true; - } - } - } - - $names = []; - - foreach ($counts as $name => $count) { - $names[$name] = $count < $paths || array_key_exists($name, $nullable); - } - - ksort($names); - - return $names; - } - - /** - * @param array $names - */ - public function declare(Schema $schema, array $names, PartitionTypes $types = new PartitionTypes()): Schema - { - $types->assertEveryNameIsAPartition($names); - - if ($names === []) { - return $schema; - } - - $definitions = []; - - foreach ($names as $name => $nullable) { - // 1) a declared schema wins, 2) then a declared partition type, 3) then string - $definitions[] = - $schema->findDefinition($name) - ?? ( - $types->has($name) - ? definition_from_type($name, $types->get($name), nullable: $nullable) - : str_schema($name, nullable: $nullable) - ); - } - - return $schema->gracefulRemove(...array_keys($names))->add(...$definitions); - } - - /** - * @param array $names - */ - public function apply(Rows $rows, array $names, PartitionTypes $types = new PartitionTypes()): Rows - { - $types->assertEveryNameIsAPartition($names); - - if ($names === []) { - return $rows; - } - - $definitions = []; - - foreach ($names as $name => $nullable) { - $definitions[] = $types->has($name) - ? definition_from_type($name, $types->get($name), nullable: $nullable) - : str_schema($name, nullable: $nullable); - } - - return new Rows($rows->schema()->gracefulRemove(...array_keys($names))->add(...$definitions), ...$rows->all()); - } - - /** - * @param array $names - * @param array $values partition name => value, for the path this row came from - * @param array $row - * - * @return array - */ - public function fill(array $row, array $names, array $values): array - { - foreach ($names as $name => $_) { - unset($row[$name]); - $row[$name] = array_key_exists($name, $values) ? $values[$name] : null; - } - - return $row; - } -} diff --git a/src/core/etl/src/Flow/ETL/Extractor/PathPartitionsExtractor.php b/src/core/etl/src/Flow/ETL/Extractor/PathPartitionsExtractor.php index 567f4a823b..d9a9fd0220 100644 --- a/src/core/etl/src/Flow/ETL/Extractor/PathPartitionsExtractor.php +++ b/src/core/etl/src/Flow/ETL/Extractor/PathPartitionsExtractor.php @@ -7,7 +7,11 @@ use Flow\ETL\Cardinality; use Flow\ETL\Exception\InvalidArgumentException; use Flow\ETL\Extractor; +use Flow\ETL\Extractor\File\FileExtractor; +use Flow\ETL\Extractor\File\ListingPartitions; +use Flow\ETL\Extractor\File\SourceFile; use Flow\ETL\FlowContext; +use Flow\ETL\Rows\RowsBuilder; use Flow\ETL\Schema; use Flow\Filesystem\FileListing; use Flow\Filesystem\Filesystem; @@ -21,8 +25,6 @@ use function array_map; use function array_merge; use function array_values; -use function count; -use function Flow\ETL\DSL\array_to_rows; use function Flow\ETL\DSL\map_schema; use function Flow\ETL\DSL\schema; use function Flow\ETL\DSL\str_schema; @@ -70,41 +72,43 @@ public function isRepeatable(): bool public function extract(FlowContext $context, ?int $limit = null, Filter $pathFilter = new OnlyFiles()): Generator { $batchSize = $this->batchSize(); + $backend = $context->backend(); $fileColumns = $this->fileColumns($this->filesystem, $this->path); $schema = $this->schema(); - $buffer = []; + $builder = new RowsBuilder($schema, $backend); $yielded = 0; foreach ((new FileListing($this->filesystem))->list($this->path, $pathFilter) as $fileStatus) { - $constants = $fileColumns->forFile(new SourceFile($fileStatus->path, $fileStatus->size), $schema); - $buffer[] = $constants->fill([ + $builder->append([ 'path' => $fileStatus->path->uri(), 'partitions' => array_merge(...array_values(array_map(static fn(Partition $p) => [ $p->name => $p->value, ], $fileStatus->path->partitions()->toArray()))), + ...$fileColumns->forFile(new SourceFile($fileStatus->path, $fileStatus->size), $schema)->values(), ]); - if (count($buffer) < $batchSize) { + if ($builder->count() < $batchSize) { continue; } - $yielded += count($buffer); + $rows = $builder->finish(); + $yielded += $rows->count(); - $signal = yield array_to_rows($buffer, $schema, $context->hydrator()); + $signal = yield $rows; if ($signal === Signal::STOP) { return; } - $buffer = []; + $builder = new RowsBuilder($schema, $backend); if ($limit !== null && $yielded >= $limit) { return; } } - if ($buffer !== []) { - yield array_to_rows($buffer, $schema, $context->hydrator()); + if ($builder->count() > 0) { + yield $builder->finish(); } } diff --git a/src/core/etl/src/Flow/ETL/Extractor/RowsExtractor.php b/src/core/etl/src/Flow/ETL/Extractor/RowsExtractor.php index a21dcb9dd2..89d1539c66 100644 --- a/src/core/etl/src/Flow/ETL/Extractor/RowsExtractor.php +++ b/src/core/etl/src/Flow/ETL/Extractor/RowsExtractor.php @@ -40,7 +40,7 @@ public function extract(FlowContext $context, ?int $limit = null): Generator $schema = $this->schema(); foreach ($this->rows as $rows) { - $signal = yield $rows->matchTo($schema); + $signal = yield $rows->matchTo($schema, $context->backend()); if ($signal === Signal::STOP) { return; diff --git a/src/core/etl/src/Flow/ETL/Extractor/SequenceExtractor.php b/src/core/etl/src/Flow/ETL/Extractor/SequenceExtractor.php index 80b299ef89..4ef18782bf 100644 --- a/src/core/etl/src/Flow/ETL/Extractor/SequenceExtractor.php +++ b/src/core/etl/src/Flow/ETL/Extractor/SequenceExtractor.php @@ -5,6 +5,8 @@ namespace Flow\ETL\Extractor; use Flow\ETL\Extractor; +use Flow\ETL\Extractor\Memory\InferredRows; +use Flow\ETL\Extractor\Memory\InMemoryRows; use Flow\ETL\Extractor\SequenceGenerator\SequenceGenerator; use Flow\ETL\FlowContext; use Flow\ETL\Rows; @@ -59,7 +61,7 @@ public function extract(FlowContext $context, ?int $limit = null): Generator continue; } - $signal = yield $batches->of($buffer, $schema, $context->hydrator()); + $signal = yield $batches->of($buffer, $schema, $context->backend()); if ($signal === Signal::STOP) { return; @@ -69,7 +71,7 @@ public function extract(FlowContext $context, ?int $limit = null): Generator } if ($buffer !== []) { - yield $batches->of($buffer, $schema, $context->hydrator()); + yield $batches->of($buffer, $schema, $context->backend()); } } diff --git a/src/core/etl/src/Flow/ETL/Filesystem/ScalarFunctionFilter.php b/src/core/etl/src/Flow/ETL/Filesystem/ScalarFunctionFilter.php index 6288a731be..8f0f12f621 100644 --- a/src/core/etl/src/Flow/ETL/Filesystem/ScalarFunctionFilter.php +++ b/src/core/etl/src/Flow/ETL/Filesystem/ScalarFunctionFilter.php @@ -13,7 +13,9 @@ use Flow\Filesystem\Path\Filter; use Throwable; -use function Flow\ETL\DSL\row; +use function Flow\ETL\DSL\array_to_rows; +use function Flow\ETL\DSL\schema; +use function Flow\ETL\DSL\str_schema; use function sprintf; final readonly class ScalarFunctionFilter implements Filter @@ -39,11 +41,13 @@ public function __construct( public function accept(FileStatus $status): bool { $values = []; + $definitions = []; foreach ($status->path->partitions()->toArray() as $partition) { // findDefinition(), not get(): a source that declares no partition columns hands an empty schema // here, and Schema::get() would throw. $definition = $this->partitions->findDefinition($partition->name); + $definitions[] = $definition ?? str_schema($partition->name, nullable: true); if ($definition === null || $definition->matches($partition->value)) { $values[$partition->name] = $partition->value; @@ -68,7 +72,13 @@ public function accept(FileStatus $status): bool } } + $schema = schema(...$definitions); + + // an undeclared partition column resolves against the file's own path value (a str column) // @mago-ignore analysis:mixed-operand - return (bool) $this->resolved->eval(row($values), $this->context); + return (bool) (new ReferenceResolver()) + ->resolve($this->resolved, $schema) + ->eval(array_to_rows([$values], $schema, $this->context->backend()), $this->context) + ->value(0); } } diff --git a/src/core/etl/src/Flow/ETL/FlowContext.php b/src/core/etl/src/Flow/ETL/FlowContext.php index 9a81873b84..53bef1f5bd 100644 --- a/src/core/etl/src/Flow/ETL/FlowContext.php +++ b/src/core/etl/src/Flow/ETL/FlowContext.php @@ -5,9 +5,9 @@ namespace Flow\ETL; use Flow\Calculator\Calculator; +use Flow\ETL\Column\Backend; use Flow\ETL\Config\Telemetry\TelemetryContext; use Flow\ETL\ErrorHandler\ThrowError; -use Flow\ETL\Row\Hydrator; /** * Mutable Flow execution context. @@ -40,12 +40,9 @@ public function errorHandler(): ErrorHandler return $this->errorHandler; } - /** - * @return Hydrator - */ - public function hydrator(): Hydrator + public function backend(): Backend { - return $this->config->hydrator(); + return $this->config->backend(); } public function setErrorHandler(ErrorHandler $handler): self @@ -66,6 +63,7 @@ public function withErrorHandler(ErrorHandler $handler): self public function telemetry(): TelemetryContext { return $this->telemetryContext ??= new TelemetryContext( + $this->config->backend(), $this->config->telemetry->telemetry->logger('flow_php_dataframe', $this->config->version()), $this->config->telemetry->telemetry->tracer('flow_php_dataframe', $this->config->version()), $this->config->telemetry->telemetry->meter('flow_php_dataframe', $this->config->version()), diff --git a/src/core/etl/src/Flow/ETL/FlowPhpExtension.php b/src/core/etl/src/Flow/ETL/FlowPhpExtension.php new file mode 100644 index 0000000000..20844015ee --- /dev/null +++ b/src/core/etl/src/Flow/ETL/FlowPhpExtension.php @@ -0,0 +1,63 @@ +loaded) { + return false; + } + + if ($this->abi !== self::ABI) { + throw new RuntimeException(sprintf( + 'The loaded flow_php extension (%s, ABI %s) does not match flow-php/etl (ABI %d). Rebuild or upgrade ' + . 'flow_php to the version of flow-php/etl, or stop loading it to use the PHP implementation.', + $this->version, + $this->abi ?? 'none', + self::ABI, + )); + } + + return true; + } +} diff --git a/src/core/etl/src/Flow/ETL/Formatter/ASCII/ASCIIBody.php b/src/core/etl/src/Flow/ETL/Formatter/ASCII/ASCIIBody.php index 4a25f58abd..d811f30185 100644 --- a/src/core/etl/src/Flow/ETL/Formatter/ASCII/ASCIIBody.php +++ b/src/core/etl/src/Flow/ETL/Formatter/ASCII/ASCIIBody.php @@ -21,14 +21,17 @@ public function print(int|bool $truncate = 20): string { $buffer = ''; - foreach ($this->body->rows() as $row) { + for ($index = 0, $count = $this->body->count(); $index < $count; $index++) { $buffer .= '|'; foreach ($this->headers->names() as $name) { $header = new ASCIIValue(type_string(), $name); try { - $value = new ASCIIValue($this->body->schema()->get($name)->type(), $row->get($name)); + $value = new ASCIIValue( + $this->body->schema()->get($name)->type(), + $this->body->value($name, $index), + ); } catch (InvalidArgumentException) { $value = new ASCIIValue(type_string(), ''); } diff --git a/src/core/etl/src/Flow/ETL/Formatter/ASCII/Body.php b/src/core/etl/src/Flow/ETL/Formatter/ASCII/Body.php index b502bfbde0..03be77bc0f 100644 --- a/src/core/etl/src/Flow/ETL/Formatter/ASCII/Body.php +++ b/src/core/etl/src/Flow/ETL/Formatter/ASCII/Body.php @@ -5,7 +5,6 @@ namespace Flow\ETL\Formatter\ASCII; use Flow\ETL\Exception\InvalidArgumentException; -use Flow\ETL\Row; use Flow\ETL\Rows; use Flow\ETL\Schema; @@ -25,33 +24,36 @@ public function schema(): Schema */ public function maximumLength(string $entry, int|bool $truncate = 20): int { + $definition = $this->schema()->findDefinition($entry); + + if ($definition === null) { + return 0; + } + $max = 0; - foreach ($this->rows as $row) { - try { - $value = new ASCIIValue($this->schema()->get($entry)->type(), $row->get($entry)); + // @mago-ignore analysis:mixed-assignment + foreach ($this->rows->column($definition->entry()->name())->values() as $cell) { + $value = new ASCIIValue($definition->type(), $cell); - if ($value->length($truncate) >= $max) { - $max = $value->length($truncate); - } - } catch (InvalidArgumentException) { + if ($value->length($truncate) >= $max) { + $max = $value->length($truncate); } } return $max; } + public function count(): int + { + return $this->rows->count(); + } + /** - * @return array + * @throws InvalidArgumentException the batch has no $entry column */ - public function rows(): array + public function value(string $entry, int $index): mixed { - $rows = []; - - foreach ($this->rows as $row) { - $rows[] = $row; - } - - return $rows; + return $this->rows->column($entry)->value($index); } } diff --git a/src/core/etl/src/Flow/ETL/Function/AggregatingFunction.php b/src/core/etl/src/Flow/ETL/Function/AggregatingFunction.php index b27b1e4061..41b313624d 100644 --- a/src/core/etl/src/Flow/ETL/Function/AggregatingFunction.php +++ b/src/core/etl/src/Flow/ETL/Function/AggregatingFunction.php @@ -4,14 +4,25 @@ namespace Flow\ETL\Function; +use Flow\ETL\Exception\InvalidArgumentException; use Flow\ETL\FlowContext; -use Flow\ETL\Row; use Flow\ETL\Row\Reference; +use Flow\ETL\Rows; use Flow\Types\Type; interface AggregatingFunction extends FunctionTree { - public function aggregate(Row $row, FlowContext $context): void; + /** + * @param list $indices the rows of $rows that belong to this aggregate, ascending + */ + public function aggregate(Rows $rows, array $indices, FlowContext $context): void; + + /** + * Folds $other's partial state into this one; $other aggregated rows that come after this one's. + * + * @throws InvalidArgumentException $other is not the same class + */ + public function merge(AggregatingFunction $other, FlowContext $context): void; /** * Decided in the constructor, never mutated. diff --git a/src/core/etl/src/Flow/ETL/Function/All.php b/src/core/etl/src/Flow/ETL/Function/All.php index 5715aaca43..30ee225ae9 100644 --- a/src/core/etl/src/Flow/ETL/Function/All.php +++ b/src/core/etl/src/Flow/ETL/Function/All.php @@ -4,14 +4,19 @@ namespace Flow\ETL\Function; +use Flow\ETL\Column\Column; use Flow\ETL\FlowContext; -use Flow\ETL\Row; +use Flow\ETL\Function\Evaluation\ResultColumn; +use Flow\ETL\Function\Evaluation\Selection; +use Flow\ETL\Rows; use Flow\Types\Type; use Flow\Types\Type\Nullability; +use function array_key_exists; use function array_map; use function array_values; use function Flow\Types\DSL\type_boolean; +use function range; final readonly class All implements ScalarFunction { @@ -65,25 +70,56 @@ public function returns(): Type )); } - public function eval(Row $row, FlowContext $context): ?bool + public function eval(Rows $rows, FlowContext $context): Column { - $sawNull = false; + $undecided = $rows->isEmpty() ? [] : range(0, $rows->count() - 1); + $results = []; - foreach ($this->functions as $ref) { - $value = (new Parameter($ref))->eval($row, $context); + /** @var array $sawNull */ + $sawNull = []; - if ($value === null) { - $sawNull = true; - - continue; + foreach ($this->functions as $function) { + if ($undecided === []) { + break; } - if (!$value) { - return false; + $column = (new Selection($undecided))->evaluate($function, $rows, $context); + $next = []; + + // @mago-ignore analysis:mixed-assignment + foreach ($column->values() as $k => $value) { + $i = $undecided[$k]; + + if ($value === null) { + $sawNull[$i] = true; + $next[] = $i; + + continue; + } + + if (!$value) { + $results[$i] = false; + + continue; + } + + $next[] = $i; } + + $undecided = $next; + } + + foreach ($undecided as $i) { + $results[$i] = array_key_exists($i, $sawNull) ? null : true; + } + + $ordered = []; + + for ($i = 0, $count = $rows->count(); $i < $count; $i++) { + $ordered[] = $results[$i]; } - return $sawNull ? null : true; + return (new ResultColumn($context->backend()))->of($this, $ordered); } public function or(ScalarFunction $scalarFunction): Any diff --git a/src/core/etl/src/Flow/ETL/Function/Any.php b/src/core/etl/src/Flow/ETL/Function/Any.php index d0941ca813..0aa8cefbaa 100644 --- a/src/core/etl/src/Flow/ETL/Function/Any.php +++ b/src/core/etl/src/Flow/ETL/Function/Any.php @@ -4,14 +4,19 @@ namespace Flow\ETL\Function; +use Flow\ETL\Column\Column; use Flow\ETL\FlowContext; -use Flow\ETL\Row; +use Flow\ETL\Function\Evaluation\ResultColumn; +use Flow\ETL\Function\Evaluation\Selection; +use Flow\ETL\Rows; use Flow\Types\Type; use Flow\Types\Type\Nullability; +use function array_key_exists; use function array_map; use function array_values; use function Flow\Types\DSL\type_boolean; +use function range; final readonly class Any implements ScalarFunction { @@ -65,25 +70,56 @@ public function returns(): Type )); } - public function eval(Row $row, FlowContext $context): ?bool + public function eval(Rows $rows, FlowContext $context): Column { - $sawNull = false; + $undecided = $rows->isEmpty() ? [] : range(0, $rows->count() - 1); + $results = []; - foreach ($this->functions as $ref) { - $value = (new Parameter($ref))->eval($row, $context); + /** @var array $sawNull */ + $sawNull = []; - if ($value === null) { - $sawNull = true; - - continue; + foreach ($this->functions as $function) { + if ($undecided === []) { + break; } - if ($value) { - return true; + $column = (new Selection($undecided))->evaluate($function, $rows, $context); + $next = []; + + // @mago-ignore analysis:mixed-assignment + foreach ($column->values() as $k => $value) { + $i = $undecided[$k]; + + if ($value === null) { + $sawNull[$i] = true; + $next[] = $i; + + continue; + } + + if ($value) { + $results[$i] = true; + + continue; + } + + $next[] = $i; } + + $undecided = $next; + } + + foreach ($undecided as $i) { + $results[$i] = array_key_exists($i, $sawNull) ? null : false; + } + + $ordered = []; + + for ($i = 0, $count = $rows->count(); $i < $count; $i++) { + $ordered[] = $results[$i]; } - return $sawNull ? null : false; + return (new ResultColumn($context->backend()))->of($this, $ordered); } public function or(ScalarFunction $scalarFunction): self diff --git a/src/core/etl/src/Flow/ETL/Function/Append.php b/src/core/etl/src/Flow/ETL/Function/Append.php index 884139d90f..4efca1292f 100644 --- a/src/core/etl/src/Flow/ETL/Function/Append.php +++ b/src/core/etl/src/Flow/ETL/Function/Append.php @@ -4,9 +4,13 @@ namespace Flow\ETL\Function; +use Exception; +use Flow\ETL\Column\Column; +use Flow\ETL\Exception\EvaluationException; use Flow\ETL\Exception\InvalidArgumentException; use Flow\ETL\FlowContext; -use Flow\ETL\Row; +use Flow\ETL\Function\Evaluation\ResultColumn; +use Flow\ETL\Rows; use Flow\Types\Type; use function Flow\ETL\DSL\lit; @@ -51,19 +55,33 @@ public function returns(): Type return type_string(); } - public function eval(Row $row, FlowContext $context): ?string + public function eval(Rows $rows, FlowContext $context): Column { - $value = (new Parameter($this->value))->asString($row, $context); - $suffix = (new Parameter($this->suffix))->asString($row, $context); + $values = (new Parameter($this->value))->asStrings($rows, $context); + $suffixes = (new Parameter($this->suffix))->asStrings($rows, $context); + $results = []; + $i = 0; - if ($value === null) { - throw new InvalidArgumentException('Append function requires non-null value'); - } + try { + foreach ($values as $i => $value) { + $suffix = $suffixes[$i]; + + if ($value === null) { + throw new InvalidArgumentException('Append function requires non-null value'); + } + + if ($suffix === null) { + $results[] = $value; + + continue; + } - if ($suffix === null) { - return $value; + $results[] = s($value)->append($suffix)->toString(); + } + } catch (Exception $e) { + throw EvaluationException::at($i, $e); } - return s($value)->append($suffix)->toString(); + return (new ResultColumn($context->backend()))->of($this, $results); } } diff --git a/src/core/etl/src/Flow/ETL/Function/ArrayExpand.php b/src/core/etl/src/Flow/ETL/Function/ArrayExpand.php index f700854f1f..f0c37015bf 100644 --- a/src/core/etl/src/Flow/ETL/Function/ArrayExpand.php +++ b/src/core/etl/src/Flow/ETL/Function/ArrayExpand.php @@ -4,10 +4,12 @@ namespace Flow\ETL\Function; +use Flow\ETL\Column\Column; use Flow\ETL\Exception\SchemaNotDerivableException; use Flow\ETL\FlowContext; +use Flow\ETL\Function\Evaluation\ResultColumn; use Flow\ETL\Function\ScalarFunction\ExpandResults; -use Flow\ETL\Row; +use Flow\ETL\Rows; use Flow\Types\Type; use Flow\Types\Type\Logical\ListType; use Flow\Types\Type\Logical\MapType; @@ -15,6 +17,7 @@ use function array_keys; use function array_map; +use function array_values; use function Flow\Types\DSL\type_bare; use function Flow\Types\DSL\type_integer; use function Flow\Types\DSL\type_map; @@ -80,25 +83,28 @@ public function returns(): Type }; } - /** - * @return array - */ - public function eval(Row $row, FlowContext $context): array + public function eval(Rows $rows, FlowContext $context): Column { - $array = (new Parameter($this->ref))->asArray($row, $context); - - if ($array === null) { - return []; - } - - if ($this->expand === ArrayExpand\ArrayExpand::KEYS) { - return array_keys($array); - } - - if ($this->expand === ArrayExpand\ArrayExpand::BOTH) { - return array_map(static fn($key, $value) => [$key => $value], array_keys($array), $array); + $lists = []; + + foreach ((new Parameter($this->ref))->asArrays($rows, $context) as $array) { + if ($array === null) { + $lists[] = []; + + continue; + } + + $lists[] = match ($this->expand) { + ArrayExpand\ArrayExpand::KEYS => array_keys($array), + ArrayExpand\ArrayExpand::BOTH => array_map( + static fn($key, $value) => [$key => $value], + array_keys($array), + $array, + ), + default => array_values($array), + }; } - return $array; + return (new ResultColumn($context->backend()))->lists($this, $lists); } } diff --git a/src/core/etl/src/Flow/ETL/Function/ArrayFilter.php b/src/core/etl/src/Flow/ETL/Function/ArrayFilter.php index 8ab235a16e..7163bd84ab 100644 --- a/src/core/etl/src/Flow/ETL/Function/ArrayFilter.php +++ b/src/core/etl/src/Flow/ETL/Function/ArrayFilter.php @@ -4,9 +4,13 @@ namespace Flow\ETL\Function; +use Exception; +use Flow\ETL\Column\Column; +use Flow\ETL\Exception\EvaluationException; use Flow\ETL\Exception\InvalidArgumentException; use Flow\ETL\FlowContext; -use Flow\ETL\Row; +use Flow\ETL\Function\Evaluation\ResultColumn; +use Flow\ETL\Rows; use Flow\Types\Type; use Flow\Types\Type\Logical\ListType; @@ -63,16 +67,28 @@ public function returns(): Type return $array; } - public function eval(Row $row, FlowContext $context): mixed + public function eval(Rows $rows, FlowContext $context): Column { - $array = (new Parameter($this->array))->asArray($row, $context); - - if (null === $array) { - throw new InvalidArgumentException('ArrayFilter function requires non-null array'); + $arrays = (new Parameter($this->array))->asArrays($rows, $context); + $values = (new Parameter($this->value))->values($rows, $context); + $results = []; + $i = 0; + + try { + foreach ($arrays as $i => $array) { + // @mago-ignore analysis:mixed-assignment + $value = $values[$i]; + + if (null === $array) { + throw new InvalidArgumentException('ArrayFilter function requires non-null array'); + } + + $results[] = array_filter($array, static fn($item) => $item !== $value); + } + } catch (Exception $e) { + throw EvaluationException::at($i, $e); } - $value = (new Parameter($this->value))->eval($row, $context); - - return array_filter($array, static fn($item) => $item !== $value); + return (new ResultColumn($context->backend()))->of($this, $results); } } diff --git a/src/core/etl/src/Flow/ETL/Function/ArrayGet.php b/src/core/etl/src/Flow/ETL/Function/ArrayGet.php index e69d5346bd..bd2cd9155b 100644 --- a/src/core/etl/src/Flow/ETL/Function/ArrayGet.php +++ b/src/core/etl/src/Flow/ETL/Function/ArrayGet.php @@ -4,13 +4,17 @@ namespace Flow\ETL\Function; +use Exception; use Flow\ArrayDot\Exception\InvalidPathException; use Flow\ArrayDot\Path; use Flow\ArrayDot\Step\Key; +use Flow\ETL\Column\Column; +use Flow\ETL\Exception\EvaluationException; use Flow\ETL\Exception\InvalidArgumentException; use Flow\ETL\Exception\SchemaNotDerivableException; use Flow\ETL\FlowContext; -use Flow\ETL\Row; +use Flow\ETL\Function\Evaluation\ResultColumn; +use Flow\ETL\Rows; use Flow\Types\Type; use Flow\Types\Type\ArrayKey; use Flow\Types\Type\Logical\StructureType; @@ -100,18 +104,36 @@ public function returns(): Type return $nullable ? (new TypeWidener())->nullable($type) : $type; } - public function eval(Row $row, FlowContext $context): mixed + public function eval(Rows $rows, FlowContext $context): Column { try { - $value = (new Parameter($this->ref))->asArray($row, $context); + $values = (new Parameter($this->ref))->asArrays($rows, $context); + } catch (EvaluationException $e) { + throw new EvaluationException( + $e->rowIndex, + new InvalidArgumentException('ArrayGet function failed to get value from array.', 0, $e), + ); + } - if ($value === null) { - throw new InvalidArgumentException('ArrayGet function requires non-null array'); - } + $results = []; + $i = 0; - return array_dot_get($value, $this->path); - } catch (InvalidArgumentException $e) { - throw new InvalidArgumentException('ArrayGet function failed to get value from array.', 0, $e); + try { + foreach ($values as $i => $value) { + try { + if ($value === null) { + throw new InvalidArgumentException('ArrayGet function requires non-null array'); + } + + $results[] = array_dot_get($value, $this->path); + } catch (InvalidArgumentException $e) { + throw new InvalidArgumentException('ArrayGet function failed to get value from array.', 0, $e); + } + } + } catch (Exception $e) { + throw EvaluationException::at($i, $e); } + + return (new ResultColumn($context->backend()))->of($this, $results); } } diff --git a/src/core/etl/src/Flow/ETL/Function/ArrayGetCollection.php b/src/core/etl/src/Flow/ETL/Function/ArrayGetCollection.php index 87a4e5b075..9fb6bf8e70 100644 --- a/src/core/etl/src/Flow/ETL/Function/ArrayGetCollection.php +++ b/src/core/etl/src/Flow/ETL/Function/ArrayGetCollection.php @@ -4,13 +4,17 @@ namespace Flow\ETL\Function; +use Exception; use Flow\ArrayDot\Exception\InvalidPathException; use Flow\ArrayDot\Path; use Flow\ArrayDot\Step\Key; use Flow\ArrayDot\Step\Multimatch; +use Flow\ETL\Column\Column; +use Flow\ETL\Exception\EvaluationException; use Flow\ETL\Exception\InvalidArgumentException; use Flow\ETL\FlowContext; -use Flow\ETL\Row; +use Flow\ETL\Function\Evaluation\ResultColumn; +use Flow\ETL\Rows; use Flow\Types\Type; use function array_map; @@ -76,54 +80,81 @@ public static function fromFirst(ScalarFunction $ref, ScalarFunction|array $keys return new self($ref, $keys, '0'); } - public function eval(Row $row, FlowContext $context): mixed + public function eval(Rows $rows, FlowContext $context): Column { try { - $value = (new Parameter($this->ref))->asArray($row, $context); - $index = (new Parameter($this->index))->asString($row, $context); - $keys = (new Parameter($this->keys))->asArray($row, $context); - - if ($value === null || $index === null || $keys === null) { - throw new InvalidArgumentException( - 'ArrayGetCollection function requires non-null array, index, and keys', - ); - } - - // the keys are literal, so they become Key steps and are never parsed as a path - $keyPaths = array_map(static function (mixed $key): Path { - if (!is_scalar($key)) { - throw new InvalidArgumentException(sprintf( - 'ArrayGetCollection keys must be scalar, got "%s".', - get_debug_type($key), - )); - } + $values = (new Parameter($this->ref))->asArrays($rows, $context); + $indexes = (new Parameter($this->index))->asStrings($rows, $context); + $keysList = (new Parameter($this->keys))->asArrays($rows, $context); + } catch (EvaluationException $e) { + throw new EvaluationException( + $e->rowIndex, + new InvalidArgumentException('ArrayGetCollection function failed to evaluate parameters.', 0, $e), + ); + } - return new Path([new Key((string) $key, nullsafe: true)]); - }, array_values($keys)); - - try { - $array = $index === '0' ? array_values($value) : $value; - - // @mago-ignore analysis:mixed-assignment - $extractedValues = array_dot_get($array, new Path([ - ...Path::fromString($index)->steps, - new Multimatch($keyPaths), - ])); - } catch (InvalidPathException $e) { - throw new InvalidArgumentException( - 'ArrayGetCollection function failed to get values from array.', - 0, - $e, - ); - } + $results = []; + $i = 0; - if (!is_array($extractedValues)) { - throw new InvalidArgumentException('ArrayGetCollection function requires the result to be an array'); + try { + foreach ($values as $i => $value) { + $index = $indexes[$i]; + $keys = $keysList[$i]; + + try { + if ($value === null || $index === null || $keys === null) { + throw new InvalidArgumentException( + 'ArrayGetCollection function requires non-null array, index, and keys', + ); + } + + // the keys are literal, so they become Key steps and are never parsed as a path + $keyPaths = array_map(static function (mixed $key): Path { + if (!is_scalar($key)) { + throw new InvalidArgumentException(sprintf( + 'ArrayGetCollection keys must be scalar, got "%s".', + get_debug_type($key), + )); + } + + return new Path([new Key((string) $key, nullsafe: true)]); + }, array_values($keys)); + + try { + $array = $index === '0' ? array_values($value) : $value; + + // @mago-ignore analysis:mixed-assignment + $extractedValues = array_dot_get($array, new Path([ + ...Path::fromString($index)->steps, + new Multimatch($keyPaths), + ])); + } catch (InvalidPathException $e) { + throw new InvalidArgumentException( + 'ArrayGetCollection function failed to get values from array.', + 0, + $e, + ); + } + + if (!is_array($extractedValues)) { + throw new InvalidArgumentException( + 'ArrayGetCollection function requires the result to be an array', + ); + } + + $results[] = $extractedValues; + } catch (InvalidArgumentException $e) { + throw new InvalidArgumentException( + 'ArrayGetCollection function failed to evaluate parameters.', + 0, + $e, + ); + } } - - return $extractedValues; - } catch (InvalidArgumentException $e) { - throw new InvalidArgumentException('ArrayGetCollection function failed to evaluate parameters.', 0, $e); + } catch (Exception $e) { + throw EvaluationException::at($i, $e); } + + return (new ResultColumn($context->backend()))->of($this, $results); } } diff --git a/src/core/etl/src/Flow/ETL/Function/ArrayKeep.php b/src/core/etl/src/Flow/ETL/Function/ArrayKeep.php index 140f1ad770..4dc931c48d 100644 --- a/src/core/etl/src/Flow/ETL/Function/ArrayKeep.php +++ b/src/core/etl/src/Flow/ETL/Function/ArrayKeep.php @@ -4,9 +4,13 @@ namespace Flow\ETL\Function; +use Exception; +use Flow\ETL\Column\Column; +use Flow\ETL\Exception\EvaluationException; use Flow\ETL\Exception\InvalidArgumentException; use Flow\ETL\FlowContext; -use Flow\ETL\Row; +use Flow\ETL\Function\Evaluation\ResultColumn; +use Flow\ETL\Rows; use Flow\Types\Type; use Flow\Types\Type\Logical\ListType; @@ -63,16 +67,28 @@ public function returns(): Type return $array; } - public function eval(Row $row, FlowContext $context): mixed + public function eval(Rows $rows, FlowContext $context): Column { - $array = (new Parameter($this->array))->asArray($row, $context); - - if (null === $array) { - throw new InvalidArgumentException('ArrayKeep function requires non-null array'); + $arrays = (new Parameter($this->array))->asArrays($rows, $context); + $values = (new Parameter($this->value))->values($rows, $context); + $results = []; + $i = 0; + + try { + foreach ($arrays as $i => $array) { + // @mago-ignore analysis:mixed-assignment + $value = $values[$i]; + + if (null === $array) { + throw new InvalidArgumentException('ArrayKeep function requires non-null array'); + } + + $results[] = array_filter($array, static fn($item) => $item === $value); + } + } catch (Exception $e) { + throw EvaluationException::at($i, $e); } - $value = (new Parameter($this->value))->eval($row, $context); - - return array_filter($array, static fn($item) => $item === $value); + return (new ResultColumn($context->backend()))->of($this, $results); } } diff --git a/src/core/etl/src/Flow/ETL/Function/ArrayKeyRename.php b/src/core/etl/src/Flow/ETL/Function/ArrayKeyRename.php index 657ff8be3b..80cb9b64ce 100644 --- a/src/core/etl/src/Flow/ETL/Function/ArrayKeyRename.php +++ b/src/core/etl/src/Flow/ETL/Function/ArrayKeyRename.php @@ -4,9 +4,13 @@ namespace Flow\ETL\Function; +use Exception; +use Flow\ETL\Column\Column; +use Flow\ETL\Exception\EvaluationException; use Flow\ETL\Exception\InvalidArgumentException; use Flow\ETL\FlowContext; -use Flow\ETL\Row; +use Flow\ETL\Function\Evaluation\ResultColumn; +use Flow\ETL\Rows; use Flow\Types\Type; use function Flow\ArrayDot\array_dot_rename; @@ -47,14 +51,24 @@ public function returns(): Type return type_array(); } - public function eval(Row $row, FlowContext $context): mixed + public function eval(Rows $rows, FlowContext $context): Column { - $value = (new Parameter($this->ref))->asArray($row, $context); + $values = (new Parameter($this->ref))->asArrays($rows, $context); + $results = []; + $i = 0; - if ($value === null) { - throw new InvalidArgumentException('ArrayKeyRename function requires non-null array'); + try { + foreach ($values as $i => $value) { + if ($value === null) { + throw new InvalidArgumentException('ArrayKeyRename function requires non-null array'); + } + + $results[] = array_dot_rename($value, $this->path, $this->newName); + } + } catch (Exception $e) { + throw EvaluationException::at($i, $e); } - return array_dot_rename($value, $this->path, $this->newName); + return (new ResultColumn($context->backend()))->of($this, $results); } } diff --git a/src/core/etl/src/Flow/ETL/Function/ArrayKeys.php b/src/core/etl/src/Flow/ETL/Function/ArrayKeys.php index 0521acdc95..2e43de00b6 100644 --- a/src/core/etl/src/Flow/ETL/Function/ArrayKeys.php +++ b/src/core/etl/src/Flow/ETL/Function/ArrayKeys.php @@ -4,10 +4,14 @@ namespace Flow\ETL\Function; +use Exception; +use Flow\ETL\Column\Column; +use Flow\ETL\Exception\EvaluationException; use Flow\ETL\Exception\InvalidArgumentException; use Flow\ETL\Exception\SchemaNotDerivableException; use Flow\ETL\FlowContext; -use Flow\ETL\Row; +use Flow\ETL\Function\Evaluation\ResultColumn; +use Flow\ETL\Rows; use Flow\Types\Type; use Flow\Types\Type\Logical\ListType; use Flow\Types\Type\Logical\MapType; @@ -70,17 +74,24 @@ public function returns(): Type }; } - /** - * @return null|array - */ - public function eval(Row $row, FlowContext $context): mixed + public function eval(Rows $rows, FlowContext $context): Column { - $array = (new Parameter($this->array))->asArray($row, $context); + $arrays = (new Parameter($this->array))->asArrays($rows, $context); + $results = []; + $i = 0; + + try { + foreach ($arrays as $i => $array) { + if (!is_array($array)) { + throw new InvalidArgumentException('ArrayKeys function requires non-null array'); + } - if (!is_array($array)) { - throw new InvalidArgumentException('ArrayKeys function requires non-null array'); + $results[] = array_keys($array); + } + } catch (Exception $e) { + throw EvaluationException::at($i, $e); } - return array_keys($array); + return (new ResultColumn($context->backend()))->of($this, $results); } } diff --git a/src/core/etl/src/Flow/ETL/Function/ArrayKeysStyleConvert.php b/src/core/etl/src/Flow/ETL/Function/ArrayKeysStyleConvert.php index c2c14b3316..76562e43af 100644 --- a/src/core/etl/src/Flow/ETL/Function/ArrayKeysStyleConvert.php +++ b/src/core/etl/src/Flow/ETL/Function/ArrayKeysStyleConvert.php @@ -4,18 +4,28 @@ namespace Flow\ETL\Function; +use Exception; +use Flow\ETL\Column\Column; +use Flow\ETL\Exception\EvaluationException; use Flow\ETL\Exception\InvalidArgumentException; use Flow\ETL\Exception\SchemaNotDerivableException; use Flow\ETL\FlowContext; +use Flow\ETL\Function\Evaluation\ResultColumn; use Flow\ETL\Function\StyleConverter\ArrayKeyConverter; -use Flow\ETL\Row; +use Flow\ETL\Rows; use Flow\ETL\String\StringStyles; use Flow\Types\Type; +use Flow\Types\Type\Logical\ListType; +use Flow\Types\Type\Logical\MapType; +use Flow\Types\Type\Logical\OptionalType; use Flow\Types\Type\Logical\StructureType; use function array_key_exists; use function Flow\Types\DSL\structure_element; use function Flow\Types\DSL\type_bare; +use function Flow\Types\DSL\type_list; +use function Flow\Types\DSL\type_map; +use function Flow\Types\DSL\type_optional; final class ArrayKeysStyleConvert implements ScalarFunction { @@ -57,10 +67,39 @@ public function returns(): Type ); } + return $this->converted($array); + } + + /** + * $type with every structure field renamed the way eval() renames keys - at every depth, as ArrayKeyConverter + * recurses into nested arrays. + * + * @param Type $type + * + * @return Type + */ + public function converted(Type $type): Type + { + if ($type instanceof OptionalType) { + return type_optional($this->converted($type->base())); + } + + if ($type instanceof ListType) { + return type_list($this->converted($type->element())); + } + + if ($type instanceof MapType) { + return type_map($type->key(), $this->converted($type->value())); + } + + if (!$type instanceof StructureType) { + return $type; + } + $elements = []; $sources = []; - foreach ($array->elements() as $element) { + foreach ($type->elements() as $element) { $converted = $this->style->convert((string) $element->name); if (array_key_exists($converted, $sources)) { @@ -71,7 +110,7 @@ public function returns(): Type . '" and "' . $element->name . '" of "' - . $array->toString() + . $type->toString() . '" both convert to "' . $converted . '"', @@ -79,22 +118,31 @@ public function returns(): Type } $sources[$converted] = $element->name; - $elements[] = structure_element($converted, $element->type, $element->optional); + $elements[] = structure_element($converted, $this->converted($element->type), $element->optional); } return new StructureType($elements); } - public function eval(Row $row, FlowContext $context): mixed + public function eval(Rows $rows, FlowContext $context): Column { - $array = (new Parameter($this->ref))->asArray($row, $context); + $arrays = (new Parameter($this->ref))->asArrays($rows, $context); + $converter = new ArrayKeyConverter(fn(string $key): string => $this->style->convert($key)); + $results = []; + $i = 0; - if ($array === null) { - throw new InvalidArgumentException('ArrayKeysStyleConvert function requires non-null array'); - } + try { + foreach ($arrays as $i => $array) { + if ($array === null) { + throw new InvalidArgumentException('ArrayKeysStyleConvert function requires non-null array'); + } - $converter = new ArrayKeyConverter(fn(string $key): string => $this->style->convert($key)); + $results[] = $converter->convert($array); + } + } catch (Exception $e) { + throw EvaluationException::at($i, $e); + } - return $converter->convert($array); + return (new ResultColumn($context->backend()))->of($this, $results); } } diff --git a/src/core/etl/src/Flow/ETL/Function/ArrayMerge.php b/src/core/etl/src/Flow/ETL/Function/ArrayMerge.php index 6e7f97fb82..0cdc4186ba 100644 --- a/src/core/etl/src/Flow/ETL/Function/ArrayMerge.php +++ b/src/core/etl/src/Flow/ETL/Function/ArrayMerge.php @@ -4,9 +4,13 @@ namespace Flow\ETL\Function; +use Exception; +use Flow\ETL\Column\Column; +use Flow\ETL\Exception\EvaluationException; use Flow\ETL\Exception\InvalidArgumentException; use Flow\ETL\FlowContext; -use Flow\ETL\Row; +use Flow\ETL\Function\Evaluation\ResultColumn; +use Flow\ETL\Rows; use Flow\Types\Exception\InvalidTypeException; use Flow\Types\Type; use Flow\Types\Type\Logical\ListType; @@ -75,18 +79,27 @@ public function returns(): Type return type_array(); } - /** - * @return null|array - */ - public function eval(Row $row, FlowContext $context): mixed + public function eval(Rows $rows, FlowContext $context): Column { - $left = (new Parameter($this->left))->asArray($row, $context); - $right = (new Parameter($this->right))->asArray($row, $context); + $lefts = (new Parameter($this->left))->asArrays($rows, $context); + $rights = (new Parameter($this->right))->asArrays($rows, $context); + $results = []; + $i = 0; + + try { + foreach ($lefts as $i => $left) { + $right = $rights[$i]; + + if ($left === null || $right === null) { + throw new InvalidArgumentException('ArrayMerge function requires two non-null arrays'); + } - if ($left === null || $right === null) { - throw new InvalidArgumentException('ArrayMerge function requires two non-null arrays'); + $results[] = array_merge($left, $right); + } + } catch (Exception $e) { + throw EvaluationException::at($i, $e); } - return array_merge($left, $right); + return (new ResultColumn($context->backend()))->of($this, $results); } } diff --git a/src/core/etl/src/Flow/ETL/Function/ArrayMergeCollection.php b/src/core/etl/src/Flow/ETL/Function/ArrayMergeCollection.php index 35d1b10144..211af046c6 100644 --- a/src/core/etl/src/Flow/ETL/Function/ArrayMergeCollection.php +++ b/src/core/etl/src/Flow/ETL/Function/ArrayMergeCollection.php @@ -4,10 +4,14 @@ namespace Flow\ETL\Function; +use Exception; +use Flow\ETL\Column\Column; +use Flow\ETL\Exception\EvaluationException; use Flow\ETL\Exception\InvalidArgumentException; use Flow\ETL\Exception\SchemaNotDerivableException; use Flow\ETL\FlowContext; -use Flow\ETL\Row; +use Flow\ETL\Function\Evaluation\ResultColumn; +use Flow\ETL\Rows; use Flow\Types\Type; use Flow\Types\Type\Logical\ListType; use Flow\Types\Type\Logical\StructureType; @@ -83,27 +87,34 @@ public function returns(): Type ); } - /** - * @return null|array - */ - public function eval(Row $row, FlowContext $context): mixed + public function eval(Rows $rows, FlowContext $context): Column { - $array = (new Parameter($this->array))->asArray($row, $context); + $arrays = (new Parameter($this->array))->asArrays($rows, $context); + $results = []; + $i = 0; + + try { + foreach ($arrays as $i => $array) { + if ($array === null) { + throw new InvalidArgumentException('ArrayMergeCollection function requires non-null array'); + } - if ($array === null) { - throw new InvalidArgumentException('ArrayMergeCollection function requires non-null array'); - } + // @mago-ignore analysis:mixed-assignment + foreach ($array as $element) { + if (!is_array($element)) { + throw new InvalidArgumentException( + 'ArrayMergeCollection function requires array elements to be arrays', + ); + } + } - // @mago-ignore analysis:mixed-assignment - foreach ($array as $element) { - if (!is_array($element)) { - throw new InvalidArgumentException( - 'ArrayMergeCollection function requires array elements to be arrays', - ); + /** @var array> $array */ + $results[] = array_merge(...array_values($array)); } + } catch (Exception $e) { + throw EvaluationException::at($i, $e); } - /** @var array> $array */ - return array_merge(...array_values($array)); + return (new ResultColumn($context->backend()))->of($this, $results); } } diff --git a/src/core/etl/src/Flow/ETL/Function/ArrayPathExists.php b/src/core/etl/src/Flow/ETL/Function/ArrayPathExists.php index 2b35289a8b..a36ad8566b 100644 --- a/src/core/etl/src/Flow/ETL/Function/ArrayPathExists.php +++ b/src/core/etl/src/Flow/ETL/Function/ArrayPathExists.php @@ -4,9 +4,13 @@ namespace Flow\ETL\Function; +use Exception; +use Flow\ETL\Column\Column; +use Flow\ETL\Exception\EvaluationException; use Flow\ETL\Exception\InvalidArgumentException; use Flow\ETL\FlowContext; -use Flow\ETL\Row; +use Flow\ETL\Function\Evaluation\ResultColumn; +use Flow\ETL\Rows; use Flow\Types\Type; use Flow\Types\Type\Nullability; @@ -55,19 +59,43 @@ public function returns(): Type return (new Nullability())->any(type_boolean(), $this->array->returns(), $this->path->returns()); } - public function eval(Row $row, FlowContext $context): ?bool + public function eval(Rows $rows, FlowContext $context): Column { try { - $array = (new Parameter($this->array))->asArray($row, $context); - $path = (new Parameter($this->path))->asString($row, $context); + $arrays = (new Parameter($this->array))->asArrays($rows, $context); + $paths = (new Parameter($this->path))->asStrings($rows, $context); + } catch (EvaluationException $e) { + throw new EvaluationException( + $e->rowIndex, + new InvalidArgumentException( + 'ArrayPathExists error: ' . ($e->getPrevious()?->getMessage() ?? $e->getMessage()), + ), + ); + } - if ($array === null || $path === null) { - return null; - } + $results = []; + $i = 0; + + try { + foreach ($arrays as $i => $array) { + $path = $paths[$i]; - return array_dot_exists($array, $path); - } catch (InvalidArgumentException $e) { - throw new InvalidArgumentException('ArrayPathExists error: ' . $e->getMessage()); + try { + if ($array === null || $path === null) { + $results[] = null; + + continue; + } + + $results[] = array_dot_exists($array, $path); + } catch (InvalidArgumentException $e) { + throw new InvalidArgumentException('ArrayPathExists error: ' . $e->getMessage(), 0, $e); + } + } + } catch (Exception $e) { + throw EvaluationException::at($i, $e); } + + return (new ResultColumn($context->backend()))->of($this, $results); } } diff --git a/src/core/etl/src/Flow/ETL/Function/ArrayReverse.php b/src/core/etl/src/Flow/ETL/Function/ArrayReverse.php index ce4d484d88..c84ef64c11 100644 --- a/src/core/etl/src/Flow/ETL/Function/ArrayReverse.php +++ b/src/core/etl/src/Flow/ETL/Function/ArrayReverse.php @@ -4,9 +4,13 @@ namespace Flow\ETL\Function; +use Exception; +use Flow\ETL\Column\Column; +use Flow\ETL\Exception\EvaluationException; use Flow\ETL\Exception\InvalidArgumentException; use Flow\ETL\FlowContext; -use Flow\ETL\Row; +use Flow\ETL\Function\Evaluation\ResultColumn; +use Flow\ETL\Rows; use Flow\Types\Type; use Flow\Types\Type\Logical\StructureType; @@ -62,17 +66,24 @@ public function returns(): Type return $array; } - /** - * @return null|array - */ - public function eval(Row $row, FlowContext $context): mixed + public function eval(Rows $rows, FlowContext $context): Column { - $array = (new Parameter($this->array))->asArray($row, $context); + $arrays = (new Parameter($this->array))->asArrays($rows, $context); + $results = []; + $i = 0; + + try { + foreach ($arrays as $i => $array) { + if ($array === null) { + throw new InvalidArgumentException('ArrayReverse function requires non-null array'); + } - if ($array === null) { - throw new InvalidArgumentException('ArrayReverse function requires non-null array'); + $results[] = array_reverse($array, $this->preserveKeys); + } + } catch (Exception $e) { + throw EvaluationException::at($i, $e); } - return array_reverse($array, $this->preserveKeys); + return (new ResultColumn($context->backend()))->of($this, $results); } } diff --git a/src/core/etl/src/Flow/ETL/Function/ArraySort.php b/src/core/etl/src/Flow/ETL/Function/ArraySort.php index 36daec8b2c..5c3d6efc17 100644 --- a/src/core/etl/src/Flow/ETL/Function/ArraySort.php +++ b/src/core/etl/src/Flow/ETL/Function/ArraySort.php @@ -4,10 +4,14 @@ namespace Flow\ETL\Function; +use Exception; +use Flow\ETL\Column\Column; +use Flow\ETL\Exception\EvaluationException; use Flow\ETL\Exception\InvalidArgumentException; use Flow\ETL\FlowContext; use Flow\ETL\Function\ArraySort\Sort; -use Flow\ETL\Row; +use Flow\ETL\Function\Evaluation\ResultColumn; +use Flow\ETL\Rows; use Flow\Types\Type; use Flow\Types\Type\Logical\StructureType; @@ -84,22 +88,34 @@ public function returns(): Type return $array; } - /** - * @return null|array - */ - public function eval(Row $row, FlowContext $context): mixed + public function eval(Rows $rows, FlowContext $context): Column { - $array = (new Parameter($this->ref))->asArray($row, $context); - $flags = (new Parameter($this->flags))->asInt($row, $context); - $recursive = (new Parameter($this->recursive))->asBoolean($row, $context) ?? false; + $arrays = (new Parameter($this->ref))->asArrays($rows, $context); + $flagsList = (new Parameter($this->flags))->asInts($rows, $context); + $recursives = (new Parameter($this->recursive))->asBooleans($rows, $context); + $results = []; + $i = 0; - if ($array === null) { - throw new InvalidArgumentException('ArraySort function requires non-null array'); - } + try { + foreach ($arrays as $i => $array) { + $flags = $flagsList[$i]; + $recursive = $recursives[$i]; - $this->recursiveSort($array, $this->sortFunction->value, $flags, $recursive); + $recursive ??= false; - return $array; + if ($array === null) { + throw new InvalidArgumentException('ArraySort function requires non-null array'); + } + + $this->recursiveSort($array, $this->sortFunction->value, $flags, $recursive); + + $results[] = $array; + } + } catch (Exception $e) { + throw EvaluationException::at($i, $e); + } + + return (new ResultColumn($context->backend()))->of($this, $results); } /** diff --git a/src/core/etl/src/Flow/ETL/Function/ArrayUnpack.php b/src/core/etl/src/Flow/ETL/Function/ArrayUnpack.php index 240d958210..a7d0f21b42 100644 --- a/src/core/etl/src/Flow/ETL/Function/ArrayUnpack.php +++ b/src/core/etl/src/Flow/ETL/Function/ArrayUnpack.php @@ -4,10 +4,14 @@ namespace Flow\ETL\Function; +use Exception; +use Flow\ETL\Column\Column; +use Flow\ETL\Exception\EvaluationException; use Flow\ETL\Exception\InvalidArgumentException; use Flow\ETL\FlowContext; +use Flow\ETL\Function\Evaluation\ResultColumn; use Flow\ETL\Function\ScalarFunction\UnpackResults; -use Flow\ETL\Row; +use Flow\ETL\Rows; use Flow\ETL\Schema; use Flow\Types\Type; @@ -45,16 +49,20 @@ public function children(): array return [$this->array]; } - /** - * @return array - */ - public function eval(Row $row, FlowContext $context): array + public function eval(Rows $rows, FlowContext $context): Column { - return ( - (new Parameter($this->array))->asArray($row, $context) ?? throw new InvalidArgumentException( - 'array_unpack() requires a non-null array', - ) - ); + $results = []; + $i = 0; + + try { + foreach ((new Parameter($this->array))->asArrays($rows, $context) as $i => $array) { + $results[] = $array ?? throw new InvalidArgumentException('array_unpack() requires a non-null array'); + } + } catch (Exception $e) { + throw EvaluationException::at($i, $e); + } + + return (new ResultColumn($context->backend()))->of($this, $results); } /** diff --git a/src/core/etl/src/Flow/ETL/Function/ArrayValues.php b/src/core/etl/src/Flow/ETL/Function/ArrayValues.php index aff2c6a507..07bc1cec19 100644 --- a/src/core/etl/src/Flow/ETL/Function/ArrayValues.php +++ b/src/core/etl/src/Flow/ETL/Function/ArrayValues.php @@ -4,10 +4,14 @@ namespace Flow\ETL\Function; +use Exception; +use Flow\ETL\Column\Column; +use Flow\ETL\Exception\EvaluationException; use Flow\ETL\Exception\InvalidArgumentException; use Flow\ETL\Exception\SchemaNotDerivableException; use Flow\ETL\FlowContext; -use Flow\ETL\Row; +use Flow\ETL\Function\Evaluation\ResultColumn; +use Flow\ETL\Rows; use Flow\Types\Type; use Flow\Types\Type\Logical\ListType; use Flow\Types\Type\Logical\MapType; @@ -67,17 +71,24 @@ public function returns(): Type }; } - /** - * @return null|array - */ - public function eval(Row $row, FlowContext $context): mixed + public function eval(Rows $rows, FlowContext $context): Column { - $array = (new Parameter($this->array))->asArray($row, $context); + $arrays = (new Parameter($this->array))->asArrays($rows, $context); + $results = []; + $i = 0; + + try { + foreach ($arrays as $i => $array) { + if (!is_array($array)) { + throw new InvalidArgumentException('ArrayValues function requires non-null array'); + } - if (!is_array($array)) { - throw new InvalidArgumentException('ArrayValues function requires non-null array'); + $results[] = array_values($array); + } + } catch (Exception $e) { + throw EvaluationException::at($i, $e); } - return array_values($array); + return (new ResultColumn($context->backend()))->of($this, $results); } } diff --git a/src/core/etl/src/Flow/ETL/Function/Ascii.php b/src/core/etl/src/Flow/ETL/Function/Ascii.php index 1b0abaf2ff..9a22561f53 100644 --- a/src/core/etl/src/Flow/ETL/Function/Ascii.php +++ b/src/core/etl/src/Flow/ETL/Function/Ascii.php @@ -4,9 +4,13 @@ namespace Flow\ETL\Function; +use Exception; +use Flow\ETL\Column\Column; +use Flow\ETL\Exception\EvaluationException; use Flow\ETL\Exception\InvalidArgumentException; use Flow\ETL\FlowContext; -use Flow\ETL\Row; +use Flow\ETL\Function\Evaluation\ResultColumn; +use Flow\ETL\Rows; use Flow\Types\Type; use function Flow\ETL\DSL\lit; @@ -49,14 +53,24 @@ public function returns(): Type return type_string(); } - public function eval(Row $row, FlowContext $context): ?string + public function eval(Rows $rows, FlowContext $context): Column { - $string = (new Parameter($this->string))->asString($row, $context); + $strings = (new Parameter($this->string))->asStrings($rows, $context); + $results = []; + $i = 0; - if ($string === null) { - throw new InvalidArgumentException('Ascii function requires non-null value'); + try { + foreach ($strings as $i => $string) { + if ($string === null) { + throw new InvalidArgumentException('Ascii function requires non-null value'); + } + + $results[] = u($string)->ascii()->toString(); + } + } catch (Exception $e) { + throw EvaluationException::at($i, $e); } - return u($string)->ascii()->toString(); + return (new ResultColumn($context->backend()))->of($this, $results); } } diff --git a/src/core/etl/src/Flow/ETL/Function/Average.php b/src/core/etl/src/Flow/ETL/Function/Average.php index c4ccc35118..585e45529e 100644 --- a/src/core/etl/src/Flow/ETL/Function/Average.php +++ b/src/core/etl/src/Flow/ETL/Function/Average.php @@ -6,10 +6,12 @@ use Flow\Calculator\Calculator; use Flow\Calculator\Rounding; +use Flow\Calculator\RunningSum; +use Flow\ETL\Exception\InvalidArgumentException; use Flow\ETL\Exception\RuntimeException; use Flow\ETL\FlowContext; -use Flow\ETL\Row; use Flow\ETL\Row\Reference; +use Flow\ETL\Rows; use Flow\ETL\Window; use Flow\ETL\Window\Accumulator\AverageAccumulator; use Flow\ETL\Window\FrameAccumulator; @@ -19,26 +21,26 @@ use function Flow\Types\DSL\type_float; use function Flow\Types\DSL\type_optional; use function is_numeric; +use function sprintf; final class Average implements AggregatingFunction, FrameAccumulating, WindowFunction { use ResolvesFromChildren; - private int $count; + private int $count = 0; private readonly string $outputName; - private float $sum; + private ?RunningSum $runningSum = null; public function __construct( private readonly Reference $ref, private readonly int $scale = 2, private readonly Rounding $rounding = Rounding::HALF_UP, + private readonly bool $exact = false, private readonly ?Window $window = null, ) { $this->outputName = $ref->hasAlias() ? $ref->name() : $ref->to() . '_avg'; - $this->count = 0; - $this->sum = 0; } /** @@ -55,36 +57,54 @@ public function children(): array public function withChildren(array $children): static { /** @var list $children */ - return new self($children[0], $this->scale, $this->rounding, $this->window); + return new self($children[0], $this->scale, $this->rounding, $this->exact, $this->window); } - public function aggregate(Row $row, FlowContext $context): void + public function aggregate(Rows $rows, array $indices, FlowContext $context): void { - if (!$row->has($this->ref)) { + if ($rows->schema()->findDefinition($this->ref->base()) === null) { return; } - /** @var mixed $value */ - $value = $row->get($this->ref); + // @mago-ignore analysis:mixed-assignment + foreach ($rows->column($this->ref->base())->take($indices)->values() as $value) { + if (is_numeric($value)) { + $this->runningSum ??= new RunningSum($context->calculator()); + // @mago-ignore analysis:possibly-invalid-argument + $this->runningSum->add($value, $this->exact); + $this->count++; + } + } + } + + public function merge(AggregatingFunction $other, FlowContext $context): void + { + if (!$other instanceof self) { + throw new InvalidArgumentException(sprintf('%s cannot merge %s', self::class, $other::class)); + } - if (is_numeric($value)) { - // @mago-ignore analysis:possibly-invalid-argument - $this->sum = $context->calculator()->add($this->sum, $value); - $this->count++; + if ($other->runningSum === null) { + return; } + + $this->runningSum ??= new RunningSum($context->calculator()); + $this->runningSum->merge($other->runningSum, $this->exact); + $this->count += $other->count; } public function accumulator(FlowContext $context): FrameAccumulator { - return new AverageAccumulator($this->ref, $this->scale, $this->rounding, $context); + return new AverageAccumulator($this->ref, $this->scale, $this->rounding, $this->exact, $context); } public function apply(WindowContext $window): mixed { $accumulator = $this->accumulator($window->flowContext()); - foreach ($window->frame() as $frameRow) { - $accumulator->accumulate($frameRow); + $frame = $window->frame(); + + for ($index = 0, $count = $frame->count(); $index < $count; $index++) { + $accumulator->accumulate($frame, $index); } return $accumulator->value(); @@ -92,7 +112,7 @@ public function apply(WindowContext $window): mixed public function over(Window $window): static { - return new self($this->ref, $this->scale, $this->rounding, $window); + return new self($this->ref, $this->scale, $this->rounding, $this->exact, $window); } public function outputName(): string @@ -118,11 +138,11 @@ public function returns(): Type public function value(): ?float { - if (0 === $this->count) { + if (0 === $this->count || $this->runningSum === null) { return null; } - return (new Calculator())->divide($this->sum, $this->count, $this->scale, $this->rounding); + return (new Calculator())->divide($this->runningSum->value(), $this->count, $this->scale, $this->rounding); } public function toString(): string diff --git a/src/core/etl/src/Flow/ETL/Function/Between.php b/src/core/etl/src/Flow/ETL/Function/Between.php index e390334434..29b09dbfd4 100644 --- a/src/core/etl/src/Flow/ETL/Function/Between.php +++ b/src/core/etl/src/Flow/ETL/Function/Between.php @@ -4,14 +4,21 @@ namespace Flow\ETL\Function; +use Exception; +use Flow\ETL\Column\Column; +use Flow\ETL\Column\ComparableValues; +use Flow\ETL\Exception\EvaluationException; use Flow\ETL\Exception\InvalidArgumentException; use Flow\ETL\FlowContext; use Flow\ETL\Function\Between\Boundary; -use Flow\ETL\Row; +use Flow\ETL\Function\Evaluation\ResultColumn; +use Flow\ETL\Rows; use Flow\Types\Type; use Flow\Types\Type\Nullability; +use function array_map; use function Flow\ETL\DSL\lit; +use function Flow\Types\DSL\type_bare; use function Flow\Types\DSL\type_boolean; final class Between implements ScalarFunction @@ -66,18 +73,35 @@ public function returns(): Type ); } - public function eval(Row $row, FlowContext $context): ?bool + public function eval(Rows $rows, FlowContext $context): Column { - $boundary = (new Parameter($this->boundary))->asEnum($row, $context, Boundary::class); + $boundaries = (new Parameter($this->boundary))->asEnums($rows, $context, Boundary::class); + $columns = array_map( + static fn(ScalarFunction $function): Column => (new Parameter($function))->column($rows, $context), + [$this->value, $this->lowerBoundRef, $this->upperBoundRef], + ); + $comparable = new ComparableValues(); + $sameType = + type_bare($columns[0]->type())::class === type_bare($columns[1]->type())::class + && type_bare($columns[0]->type())::class === type_bare($columns[2]->type())::class; + [$values, $lowerBounds, $upperBounds] = array_map(static fn(Column $column): array => $sameType + ? $comparable->ordering($column) + : $column->values(), $columns); + $results = []; + $i = 0; + + try { + foreach ($boundaries as $i => $boundary) { + if (!$boundary instanceof Boundary) { + throw new InvalidArgumentException('Between function requires valid boundary'); + } - if (!$boundary instanceof Boundary) { - throw new InvalidArgumentException('Between function requires valid boundary'); + $results[] = $boundary->compare($values[$i], $lowerBounds[$i], $upperBounds[$i]); + } + } catch (Exception $e) { + throw EvaluationException::at($i, $e); } - return $boundary->compare( - (new Parameter($this->value))->eval($row, $context), - (new Parameter($this->lowerBoundRef))->eval($row, $context), - (new Parameter($this->upperBoundRef))->eval($row, $context), - ); + return (new ResultColumn($context->backend()))->of($this, $results); } } diff --git a/src/core/etl/src/Flow/ETL/Function/BinaryLength.php b/src/core/etl/src/Flow/ETL/Function/BinaryLength.php index c46e29f925..163fe989b9 100644 --- a/src/core/etl/src/Flow/ETL/Function/BinaryLength.php +++ b/src/core/etl/src/Flow/ETL/Function/BinaryLength.php @@ -4,9 +4,13 @@ namespace Flow\ETL\Function; +use Exception; +use Flow\ETL\Column\Column; +use Flow\ETL\Exception\EvaluationException; use Flow\ETL\Exception\InvalidArgumentException; use Flow\ETL\FlowContext; -use Flow\ETL\Row; +use Flow\ETL\Function\Evaluation\ResultColumn; +use Flow\ETL\Rows; use Flow\Types\Type; use function Flow\ETL\DSL\lit; @@ -49,14 +53,24 @@ public function returns(): Type return type_integer(); } - public function eval(Row $row, FlowContext $context): ?int + public function eval(Rows $rows, FlowContext $context): Column { - $value = (new Parameter($this->value))->asString($row, $context); + $values = (new Parameter($this->value))->asStrings($rows, $context); + $results = []; + $i = 0; - if ($value === null) { - throw new InvalidArgumentException('BinaryLength function requires non-null value'); + try { + foreach ($values as $i => $value) { + if ($value === null) { + throw new InvalidArgumentException('BinaryLength function requires non-null value'); + } + + $results[] = b($value)->length(); + } + } catch (Exception $e) { + throw EvaluationException::at($i, $e); } - return b($value)->length(); + return (new ResultColumn($context->backend()))->of($this, $results); } } diff --git a/src/core/etl/src/Flow/ETL/Function/CallUserFunc.php b/src/core/etl/src/Flow/ETL/Function/CallUserFunc.php index 3404632072..ca3e20e57c 100644 --- a/src/core/etl/src/Flow/ETL/Function/CallUserFunc.php +++ b/src/core/etl/src/Flow/ETL/Function/CallUserFunc.php @@ -4,9 +4,13 @@ namespace Flow\ETL\Function; +use Exception; +use Flow\ETL\Column\Column; +use Flow\ETL\Exception\EvaluationException; use Flow\ETL\Exception\InvalidArgumentException; use Flow\ETL\FlowContext; -use Flow\ETL\Row; +use Flow\ETL\Function\Evaluation\ResultColumn; +use Flow\ETL\Rows; use Flow\Types\Type; use function array_combine; @@ -85,24 +89,39 @@ public function returns(): Type return type_optional($this->returnType); } - public function eval(Row $row, FlowContext $context): mixed + public function eval(Rows $rows, FlowContext $context): Column { - $callable = (new Parameter($this->callable))->eval($row, $context); - - if (!is_callable($callable)) { - throw new InvalidArgumentException('CallUserFunc requires a valid callable'); + $callables = (new Parameter($this->callable))->values($rows, $context); + $arguments = array_map(static fn(ScalarFunction $parameter): array => (new Parameter($parameter))->values( + $rows, + $context, + ), $this->parameters); + $results = []; + $i = 0; + + try { + // @mago-ignore analysis:mixed-assignment + foreach ($callables as $i => $callable) { + if (!is_callable($callable)) { + throw new InvalidArgumentException('CallUserFunc requires a valid callable'); + } + + $parameters = []; + + foreach ($arguments as $key => $argument) { + $parameters[$key] = $argument[$i]; + } + + // The callable's output may not match the declared type - coerce it before trusting it. + // @mago-ignore analysis:mixed-assignment + $result = call_user_func($callable, ...$parameters); + + $results[] = $result === null ? null : $this->returnType->cast($result); + } + } catch (Exception $e) { + throw EvaluationException::at($i, $e); } - $parameters = []; - - foreach ($this->parameters as $key => $parameter) { - $parameters[$key] = (new Parameter($parameter))->eval($row, $context); - } - - // The callable's output may not match the declared type - coerce it before trusting it. - // @mago-ignore analysis:mixed-assignment - $result = call_user_func($callable, ...$parameters); - - return $result === null ? null : $this->returnType->cast($result); + return (new ResultColumn($context->backend()))->of($this, $results); } } diff --git a/src/core/etl/src/Flow/ETL/Function/Capitalize.php b/src/core/etl/src/Flow/ETL/Function/Capitalize.php index b69ffa1864..b514286a1b 100644 --- a/src/core/etl/src/Flow/ETL/Function/Capitalize.php +++ b/src/core/etl/src/Flow/ETL/Function/Capitalize.php @@ -4,9 +4,13 @@ namespace Flow\ETL\Function; +use Exception; +use Flow\ETL\Column\Column; +use Flow\ETL\Exception\EvaluationException; use Flow\ETL\Exception\InvalidArgumentException; use Flow\ETL\FlowContext; -use Flow\ETL\Row; +use Flow\ETL\Function\Evaluation\ResultColumn; +use Flow\ETL\Rows; use Flow\Types\Type; use function Flow\ETL\DSL\lit; @@ -54,18 +58,31 @@ public function returns(): Type return type_string(); } - public function eval(Row $row, FlowContext $context): ?string + public function eval(Rows $rows, FlowContext $context): Column { - $string = (new Parameter($this->string))->eval($row, $context); + $strings = (new Parameter($this->string))->values($rows, $context); + $results = []; + $i = 0; - if ($string === null) { - throw new InvalidArgumentException('Capitalize function requires non-null value'); - } + try { + // @mago-ignore analysis:mixed-assignment + foreach ($strings as $i => $string) { + if ($string === null) { + throw new InvalidArgumentException('Capitalize function requires non-null value'); + } + + if (function_exists('mb_convert_case')) { + $results[] = mb_convert_case(is_scalar($string) ? (string) $string : '', MB_CASE_TITLE); + + continue; + } - if (function_exists('mb_convert_case')) { - return mb_convert_case(is_scalar($string) ? (string) $string : '', MB_CASE_TITLE); + $results[] = ucwords(is_scalar($string) ? (string) $string : ''); + } + } catch (Exception $e) { + throw EvaluationException::at($i, $e); } - return ucwords(is_scalar($string) ? (string) $string : ''); + return (new ResultColumn($context->backend()))->of($this, $results); } } diff --git a/src/core/etl/src/Flow/ETL/Function/Cast.php b/src/core/etl/src/Flow/ETL/Function/Cast.php index 7d629ac112..9a20cdc757 100644 --- a/src/core/etl/src/Flow/ETL/Function/Cast.php +++ b/src/core/etl/src/Flow/ETL/Function/Cast.php @@ -4,19 +4,33 @@ namespace Flow\ETL\Function; +use DateTimeInterface; +use Exception; +use Flow\ETL\Column\Column; +use Flow\ETL\Column\Physical\XmlDocumentPhysical; +use Flow\ETL\Column\TextValues; +use Flow\ETL\Exception\EvaluationException; use Flow\ETL\Exception\InvalidArgumentException; use Flow\ETL\Exception\RuntimeException; use Flow\ETL\Exception\UnsupportedUnionTypeException; use Flow\ETL\FlowContext; -use Flow\ETL\Row; +use Flow\ETL\Function\Evaluation\ResultColumn; +use Flow\ETL\Rows; use Flow\Types\Exception\CastingException; use Flow\Types\Exception\InvalidArgumentException as TypesInvalidArgumentException; use Flow\Types\Type; +use Flow\Types\Type\Logical\DateTimeType; +use Flow\Types\Type\Logical\DateType; +use Flow\Types\Type\Logical\XMLType; +use Flow\Types\Type\Native\StringType; use Flow\Types\Type\TypeFactory; +use function array_search; use function Flow\ETL\DSL\definition_from_type; use function Flow\ETL\DSL\lit; +use function Flow\Types\DSL\type_bare; use function Flow\Types\DSL\type_is_nullable; +use function is_string; use function sprintf; final class Cast implements ScalarFunction @@ -79,18 +93,64 @@ public function returns(): Type return $this->type; } - public function eval(Row $row, FlowContext $context): mixed + public function eval(Rows $rows, FlowContext $context): Column { - $value = (new Parameter($this->value))->eval($row, $context); - - if (null === $value && !type_is_nullable($this->type)) { - throw new InvalidArgumentException('Cast function requires non-null value'); + $values = null; + $documents = null; + + if (type_bare($this->type) instanceof StringType) { + $column = (new Parameter($this->value))->column($rows, $context); + $type = type_bare($column->type()); + + if ($type instanceof DateTimeType || $type instanceof DateType) { + /** @var list $physicals */ + $physicals = $column->physicals(); + + if ($column->nullCount() > 0 && !type_is_nullable($this->type)) { + throw EvaluationException::at( + (int) array_search(null, $physicals, true), + new InvalidArgumentException('Cast function requires non-null value'), + ); + } + + return (new ResultColumn($context->backend()))->of($this, (new TextValues())->dateTimes( + $type, + $physicals, + DateTimeInterface::RFC3339, + )); + } + + if ($type instanceof XMLType) { + $values = $column->physicals(); + $documents = new XmlDocumentPhysical(); + } } + $results = []; + $i = 0; + try { - return $this->type->cast($value); - } catch (CastingException $e) { - throw new InvalidArgumentException('Cast function failed: ' . $e->getMessage()); + // @mago-ignore analysis:mixed-assignment + foreach ($values ?? (new Parameter($this->value))->values($rows, $context) as $i => $value) { + if ($documents !== null && is_string($value)) { + // @mago-ignore analysis:mixed-assignment + $value = $documents->text($value) ?? $documents->fromPhysical($value); + } + + if (null === $value && !type_is_nullable($this->type)) { + throw new InvalidArgumentException('Cast function requires non-null value'); + } + + try { + $results[] = $this->type->cast($value); + } catch (CastingException $e) { + throw new InvalidArgumentException('Cast function failed: ' . $e->getMessage(), 0, $e); + } + } + } catch (Exception $e) { + throw EvaluationException::at($i, $e); } + + return (new ResultColumn($context->backend()))->of($this, $results); } } diff --git a/src/core/etl/src/Flow/ETL/Function/Chunk.php b/src/core/etl/src/Flow/ETL/Function/Chunk.php index 610ab231f5..b61f84160c 100644 --- a/src/core/etl/src/Flow/ETL/Function/Chunk.php +++ b/src/core/etl/src/Flow/ETL/Function/Chunk.php @@ -4,9 +4,13 @@ namespace Flow\ETL\Function; +use Exception; +use Flow\ETL\Column\Column; +use Flow\ETL\Exception\EvaluationException; use Flow\ETL\Exception\InvalidArgumentException; use Flow\ETL\FlowContext; -use Flow\ETL\Row; +use Flow\ETL\Function\Evaluation\ResultColumn; +use Flow\ETL\Rows; use Flow\Types\Type; use Symfony\Component\String\AbstractString; @@ -53,27 +57,36 @@ public function returns(): Type return type_list(type_string()); } - /** - * @return array - */ - public function eval(Row $row, FlowContext $context): array + public function eval(Rows $rows, FlowContext $context): Column { - $value = (new Parameter($this->value))->asString($row, $context); - $size = (new Parameter($this->size))->asInt($row, $context); + $values = (new Parameter($this->value))->asStrings($rows, $context); + $sizes = (new Parameter($this->size))->asInts($rows, $context); + $results = []; + $i = 0; - if ($value === null) { - throw new InvalidArgumentException('Chunk function requires non-null value'); - } + try { + foreach ($values as $i => $value) { + $size = $sizes[$i]; - if ($size === null || $size <= 0) { - throw new InvalidArgumentException('Chunk function requires non-null, positive size'); - } + if ($value === null) { + throw new InvalidArgumentException('Chunk function requires non-null value'); + } - $chunks = s($value)->chunk($size); + if ($size === null || $size <= 0) { + throw new InvalidArgumentException('Chunk function requires non-null, positive size'); + } + + $chunks = s($value)->chunk($size); + + $results[] = array_map( + static fn(AbstractString $chunk): string => $chunk->toString(), + iterator_to_array($chunks, false), + ); + } + } catch (Exception $e) { + throw EvaluationException::at($i, $e); + } - return array_map( - static fn(AbstractString $chunk): string => $chunk->toString(), - iterator_to_array($chunks, false), - ); + return (new ResultColumn($context->backend()))->of($this, $results); } } diff --git a/src/core/etl/src/Flow/ETL/Function/Coalesce.php b/src/core/etl/src/Flow/ETL/Function/Coalesce.php index b1841fa2c1..973bc9daa2 100644 --- a/src/core/etl/src/Flow/ETL/Function/Coalesce.php +++ b/src/core/etl/src/Flow/ETL/Function/Coalesce.php @@ -4,8 +4,11 @@ namespace Flow\ETL\Function; +use Flow\ETL\Column\Column; use Flow\ETL\FlowContext; -use Flow\ETL\Row; +use Flow\ETL\Function\Evaluation\ResultColumn; +use Flow\ETL\Function\Evaluation\Selection; +use Flow\ETL\Rows; use Flow\Types\Exception\InvalidTypeException; use Flow\Types\Type; use Flow\Types\Type\Unifier\NullabilityRule; @@ -13,6 +16,7 @@ use function array_map; use function array_values; +use function range; final class Coalesce implements ScalarFunction { @@ -61,16 +65,39 @@ public function returns(): Type ); } - public function eval(Row $row, FlowContext $context): mixed + public function eval(Rows $rows, FlowContext $context): Column { + $pending = $rows->isEmpty() ? [] : range(0, $rows->count() - 1); + $values = []; + foreach ($this->values as $value) { - $result = (new Parameter($value))->eval($row, $context); + if ($pending === []) { + break; + } + + $column = (new Selection($pending))->evaluate($value, $rows, $context); + $stillNull = []; + + // @mago-ignore analysis:mixed-assignment + foreach ($column->values() as $k => $result) { + if ($result === null) { + $stillNull[] = $pending[$k]; + + continue; + } - if ($result !== null) { - return $result; + $values[$pending[$k]] = $result; } + + $pending = $stillNull; + } + + $results = []; + + for ($i = 0, $count = $rows->count(); $i < $count; $i++) { + $results[] = $values[$i] ?? null; } - return null; + return (new ResultColumn($context->backend()))->of($this, $results); } } diff --git a/src/core/etl/src/Flow/ETL/Function/CodePointLength.php b/src/core/etl/src/Flow/ETL/Function/CodePointLength.php index f697a4add5..bb5bb9f3a2 100644 --- a/src/core/etl/src/Flow/ETL/Function/CodePointLength.php +++ b/src/core/etl/src/Flow/ETL/Function/CodePointLength.php @@ -4,9 +4,13 @@ namespace Flow\ETL\Function; +use Exception; +use Flow\ETL\Column\Column; +use Flow\ETL\Exception\EvaluationException; use Flow\ETL\Exception\InvalidArgumentException; use Flow\ETL\FlowContext; -use Flow\ETL\Row; +use Flow\ETL\Function\Evaluation\ResultColumn; +use Flow\ETL\Rows; use Flow\Types\Type; use function Flow\ETL\DSL\lit; @@ -49,14 +53,24 @@ public function returns(): Type return type_integer(); } - public function eval(Row $row, FlowContext $context): ?int + public function eval(Rows $rows, FlowContext $context): Column { - $value = (new Parameter($this->value))->asString($row, $context); + $values = (new Parameter($this->value))->asStrings($rows, $context); + $results = []; + $i = 0; - if ($value === null) { - throw new InvalidArgumentException('CodePointLength function requires non-null value'); + try { + foreach ($values as $i => $value) { + if ($value === null) { + throw new InvalidArgumentException('CodePointLength function requires non-null value'); + } + + $results[] = s($value)->toCodePointString()->length(); + } + } catch (Exception $e) { + throw EvaluationException::at($i, $e); } - return s($value)->toCodePointString()->length(); + return (new ResultColumn($context->backend()))->of($this, $results); } } diff --git a/src/core/etl/src/Flow/ETL/Function/CollapseWhitespace.php b/src/core/etl/src/Flow/ETL/Function/CollapseWhitespace.php index 02e8053d3b..a5c69eb708 100644 --- a/src/core/etl/src/Flow/ETL/Function/CollapseWhitespace.php +++ b/src/core/etl/src/Flow/ETL/Function/CollapseWhitespace.php @@ -4,9 +4,13 @@ namespace Flow\ETL\Function; +use Exception; +use Flow\ETL\Column\Column; +use Flow\ETL\Exception\EvaluationException; use Flow\ETL\Exception\InvalidArgumentException; use Flow\ETL\FlowContext; -use Flow\ETL\Row; +use Flow\ETL\Function\Evaluation\ResultColumn; +use Flow\ETL\Rows; use Flow\Types\Type; use function Flow\ETL\DSL\lit; @@ -49,14 +53,24 @@ public function returns(): Type return type_string(); } - public function eval(Row $row, FlowContext $context): ?string + public function eval(Rows $rows, FlowContext $context): Column { - $value = (new Parameter($this->value))->asString($row, $context); + $values = (new Parameter($this->value))->asStrings($rows, $context); + $results = []; + $i = 0; - if ($value === null) { - throw new InvalidArgumentException('CollapseWhitespace function requires non-null value'); + try { + foreach ($values as $i => $value) { + if ($value === null) { + throw new InvalidArgumentException('CollapseWhitespace function requires non-null value'); + } + + $results[] = s($value)->collapseWhitespace()->toString(); + } + } catch (Exception $e) { + throw EvaluationException::at($i, $e); } - return s($value)->collapseWhitespace()->toString(); + return (new ResultColumn($context->backend()))->of($this, $results); } } diff --git a/src/core/etl/src/Flow/ETL/Function/Collect.php b/src/core/etl/src/Flow/ETL/Function/Collect.php index c06c2f4a13..80b4b37625 100644 --- a/src/core/etl/src/Flow/ETL/Function/Collect.php +++ b/src/core/etl/src/Flow/ETL/Function/Collect.php @@ -4,14 +4,15 @@ namespace Flow\ETL\Function; +use Flow\ETL\Exception\InvalidArgumentException; use Flow\ETL\FlowContext; -use Flow\ETL\Row; use Flow\ETL\Row\Reference; +use Flow\ETL\Rows; use Flow\Types\Type; -use function current; use function Flow\Types\DSL\type_list; use function Flow\Types\DSL\type_optional; +use function sprintf; final class Collect implements AggregatingFunction { @@ -48,18 +49,25 @@ public function withChildren(array $children): static return new self($children[0]); } - public function aggregate(Row $row, FlowContext $context): void + public function aggregate(Rows $rows, array $indices, FlowContext $context): void { - if (!$row->has($this->ref)) { + if ($rows->schema()->findDefinition($this->ref->base()) === null) { return; } - /** @var array $values */ - $values = []; + // @mago-ignore analysis:mixed-assignment + foreach ($rows->column($this->ref->base())->take($indices)->values() as $value) { + $this->collection[] = $value; + } + } - $values[$this->ref->name()] = $row->get($this->ref); + public function merge(AggregatingFunction $other, FlowContext $context): void + { + if (!$other instanceof self) { + throw new InvalidArgumentException(sprintf('%s cannot merge %s', self::class, $other::class)); + } - $this->collection[] = current($values); + $this->collection = [...$this->collection, ...$other->collection]; } public function outputName(): string diff --git a/src/core/etl/src/Flow/ETL/Function/CollectUnique.php b/src/core/etl/src/Flow/ETL/Function/CollectUnique.php index e7354dd0b8..02577165a8 100644 --- a/src/core/etl/src/Flow/ETL/Function/CollectUnique.php +++ b/src/core/etl/src/Flow/ETL/Function/CollectUnique.php @@ -4,15 +4,16 @@ namespace Flow\ETL\Function; +use Flow\ETL\Exception\InvalidArgumentException; use Flow\ETL\FlowContext; -use Flow\ETL\Row; use Flow\ETL\Row\Reference; +use Flow\ETL\Rows; use Flow\Types\Type; -use function current; use function Flow\Types\DSL\type_list; use function Flow\Types\DSL\type_optional; use function in_array; +use function sprintf; final class CollectUnique implements AggregatingFunction { @@ -49,22 +50,31 @@ public function withChildren(array $children): static return new self($children[0]); } - public function aggregate(Row $row, FlowContext $context): void + public function aggregate(Rows $rows, array $indices, FlowContext $context): void { - if (!$row->has($this->ref)) { + if ($rows->schema()->findDefinition($this->ref->base()) === null) { return; } - /** @var array $values */ - $values = []; - - $values[$this->ref->name()] = $row->get($this->ref); + // @mago-ignore analysis:mixed-assignment + foreach ($rows->column($this->ref->base())->take($indices)->values() as $value) { + if (!in_array($value, $this->collection, true)) { + $this->collection[] = $value; + } + } + } - /** @var mixed $value */ - $value = current($values); + public function merge(AggregatingFunction $other, FlowContext $context): void + { + if (!$other instanceof self) { + throw new InvalidArgumentException(sprintf('%s cannot merge %s', self::class, $other::class)); + } - if (!in_array($value, $this->collection, true)) { - $this->collection[] = $value; + // @mago-ignore analysis:mixed-assignment + foreach ($other->collection as $value) { + if (!in_array($value, $this->collection, true)) { + $this->collection[] = $value; + } } } diff --git a/src/core/etl/src/Flow/ETL/Function/Combine.php b/src/core/etl/src/Flow/ETL/Function/Combine.php index d4a7c3e54b..2abb530a7e 100644 --- a/src/core/etl/src/Flow/ETL/Function/Combine.php +++ b/src/core/etl/src/Flow/ETL/Function/Combine.php @@ -4,10 +4,14 @@ namespace Flow\ETL\Function; +use Exception; +use Flow\ETL\Column\Column; +use Flow\ETL\Exception\EvaluationException; use Flow\ETL\Exception\InvalidArgumentException; use Flow\ETL\Exception\SchemaNotDerivableException; use Flow\ETL\FlowContext; -use Flow\ETL\Row; +use Flow\ETL\Function\Evaluation\ResultColumn; +use Flow\ETL\Rows; use Flow\Types\Type; use Flow\Types\Type\Logical\ListType; @@ -72,37 +76,48 @@ public function returns(): Type return type_map($keys->element(), $values->element()); } - /** - * @return null|array - */ - public function eval(Row $row, FlowContext $context): ?array + public function eval(Rows $rows, FlowContext $context): Column { - $keys = (new Parameter($this->keys))->asArray($row, $context); - $values = (new Parameter($this->values))->asArray($row, $context); - - if (null === $keys || null === $values) { - throw new InvalidArgumentException('Combine function requires non-null arrays'); - } - - if ([] === $keys) { - return []; - } - - if (!array_is_list($keys)) { - throw new InvalidArgumentException('Combine function requires keys to be a list'); - } - - if (count($keys) !== count($values)) { - throw new InvalidArgumentException( - 'Combine function requires keys and values arrays to have the same length', - ); - } - - if (!is_string($keys[0] ?? null) && !is_int($keys[0] ?? null)) { - throw new InvalidArgumentException('Combine function requires keys to be strings or integers'); + $keysList = (new Parameter($this->keys))->asArrays($rows, $context); + $valuesList = (new Parameter($this->values))->asArrays($rows, $context); + $results = []; + $i = 0; + + try { + foreach ($keysList as $i => $keys) { + $values = $valuesList[$i]; + + if (null === $keys || null === $values) { + throw new InvalidArgumentException('Combine function requires non-null arrays'); + } + + if ([] === $keys) { + $results[] = []; + + continue; + } + + if (!array_is_list($keys)) { + throw new InvalidArgumentException('Combine function requires keys to be a list'); + } + + if (count($keys) !== count($values)) { + throw new InvalidArgumentException( + 'Combine function requires keys and values arrays to have the same length', + ); + } + + if (!is_string($keys[0] ?? null) && !is_int($keys[0] ?? null)) { + throw new InvalidArgumentException('Combine function requires keys to be strings or integers'); + } + + /** @var array $keys */ + $results[] = array_combine($keys, $values); + } + } catch (Exception $e) { + throw EvaluationException::at($i, $e); } - /** @var array $keys */ - return array_combine($keys, $values); + return (new ResultColumn($context->backend()))->of($this, $results); } } diff --git a/src/core/etl/src/Flow/ETL/Function/Concat.php b/src/core/etl/src/Flow/ETL/Function/Concat.php index 84319f0885..433e2d45f4 100644 --- a/src/core/etl/src/Flow/ETL/Function/Concat.php +++ b/src/core/etl/src/Flow/ETL/Function/Concat.php @@ -4,8 +4,12 @@ namespace Flow\ETL\Function; +use Exception; +use Flow\ETL\Column\Column; +use Flow\ETL\Exception\EvaluationException; use Flow\ETL\FlowContext; -use Flow\ETL\Row; +use Flow\ETL\Function\Evaluation\ResultColumn; +use Flow\ETL\Rows; use Flow\Types\Type; use function array_map; @@ -56,21 +60,34 @@ public function returns(): Type return type_string(); } - public function eval(Row $row, FlowContext $context): string + public function eval(Rows $rows, FlowContext $context): Column { - /** @var array $concatValues */ - $concatValues = []; + $arguments = array_map(static fn(ScalarFunction $ref): array => (new Parameter($ref))->values( + $rows, + $context, + ), $this->refs); + $results = []; + $i = 0; - foreach ($this->refs as $ref) { - $value = (new Parameter($ref))->eval($row, $context); + try { + for ($count = $rows->count(); $i < $count; $i++) { + /** @var array $concatValues */ + $concatValues = []; - if ($value === null) { - continue; - } + foreach ($arguments as $argument) { + if ($argument[$i] === null) { + continue; + } + + $concatValues[] = type_string()->cast($argument[$i]); + } - $concatValues[] = type_string()->cast($value); + $results[] = implode('', $concatValues); + } + } catch (Exception $e) { + throw EvaluationException::at($i, $e); } - return implode('', $concatValues); + return (new ResultColumn($context->backend()))->of($this, $results); } } diff --git a/src/core/etl/src/Flow/ETL/Function/ConcatWithSeparator.php b/src/core/etl/src/Flow/ETL/Function/ConcatWithSeparator.php index ab2a99a8f0..204e347fdc 100644 --- a/src/core/etl/src/Flow/ETL/Function/ConcatWithSeparator.php +++ b/src/core/etl/src/Flow/ETL/Function/ConcatWithSeparator.php @@ -4,9 +4,13 @@ namespace Flow\ETL\Function; +use Exception; +use Flow\ETL\Column\Column; +use Flow\ETL\Exception\EvaluationException; use Flow\ETL\Exception\InvalidArgumentException; use Flow\ETL\FlowContext; -use Flow\ETL\Row; +use Flow\ETL\Function\Evaluation\ResultColumn; +use Flow\ETL\Rows; use Flow\Types\Type; use function array_map; @@ -65,28 +69,43 @@ public function returns(): Type return type_string(); } - public function eval(Row $row, FlowContext $context): string + public function eval(Rows $rows, FlowContext $context): Column { - $separator = (new Parameter($this->separator))->asString($row, $context); - - if (!is_string($separator)) { - throw new InvalidArgumentException('ConcatWithSeparator function requires non-null separator'); - } - - /** @var array $concatValues */ - $concatValues = []; - - foreach ($this->refs as $value) { - $value = (new Parameter($value))->eval($row, $context); - - if (is_type(type_list(type_string()), $value)) { - /** @var list $value */ - $concatValues = array_merge($concatValues, $value); - } elseif ($value !== null) { - $concatValues[] = type_string()->cast($value); + $separators = (new Parameter($this->separator))->asStrings($rows, $context); + $arguments = array_map(static fn(ScalarFunction $ref): array => (new Parameter($ref))->values( + $rows, + $context, + ), $this->refs); + $results = []; + $i = 0; + + try { + foreach ($separators as $i => $separator) { + if (!is_string($separator)) { + throw new InvalidArgumentException('ConcatWithSeparator function requires non-null separator'); + } + + /** @var array $concatValues */ + $concatValues = []; + + foreach ($arguments as $argument) { + // @mago-ignore analysis:mixed-assignment + $value = $argument[$i]; + + if (is_type(type_list(type_string()), $value)) { + /** @var list $value */ + $concatValues = array_merge($concatValues, $value); + } elseif ($value !== null) { + $concatValues[] = type_string()->cast($value); + } + } + + $results[] = implode($separator, $concatValues); } + } catch (Exception $e) { + throw EvaluationException::at($i, $e); } - return implode($separator, $concatValues); + return (new ResultColumn($context->backend()))->of($this, $results); } } diff --git a/src/core/etl/src/Flow/ETL/Function/Contains.php b/src/core/etl/src/Flow/ETL/Function/Contains.php index 27818935c5..e7a926de31 100644 --- a/src/core/etl/src/Flow/ETL/Function/Contains.php +++ b/src/core/etl/src/Flow/ETL/Function/Contains.php @@ -4,8 +4,12 @@ namespace Flow\ETL\Function; +use Exception; +use Flow\ETL\Column\Column; +use Flow\ETL\Exception\EvaluationException; use Flow\ETL\FlowContext; -use Flow\ETL\Row; +use Flow\ETL\Function\Evaluation\ResultColumn; +use Flow\ETL\Rows; use Flow\Types\Type; use Flow\Types\Type\Nullability; @@ -55,19 +59,35 @@ public function returns(): Type return (new Nullability())->any(type_boolean(), $this->haystack->returns(), $this->needle->returns()); } - public function eval(Row $row, FlowContext $context): ?bool + public function eval(Rows $rows, FlowContext $context): Column { - $haystack = (new Parameter($this->haystack))->as($row, $context, type_string(), type_array()); - $needle = (new Parameter($this->needle))->asString($row, $context); + $haystacks = (new Parameter($this->haystack))->asTypes($rows, $context, type_string(), type_array()); + $needles = (new Parameter($this->needle))->asStrings($rows, $context); + $results = []; + $i = 0; - if ($haystack === null || $needle === null) { - return null; - } + try { + foreach ($haystacks as $i => $haystack) { + $needle = $needles[$i]; + + if ($haystack === null || $needle === null) { + $results[] = null; + + continue; + } + + if (is_string($haystack)) { + $results[] = str_contains($haystack, $needle); + + continue; + } - if (is_string($haystack)) { - return str_contains($haystack, $needle); + $results[] = in_array($needle, $haystack, true); + } + } catch (Exception $e) { + throw EvaluationException::at($i, $e); } - return in_array($needle, $haystack, true); + return (new ResultColumn($context->backend()))->of($this, $results); } } diff --git a/src/core/etl/src/Flow/ETL/Function/Count.php b/src/core/etl/src/Flow/ETL/Function/Count.php index e3cf739df2..754c6fad84 100644 --- a/src/core/etl/src/Flow/ETL/Function/Count.php +++ b/src/core/etl/src/Flow/ETL/Function/Count.php @@ -4,18 +4,21 @@ namespace Flow\ETL\Function; +use Flow\ETL\Exception\InvalidArgumentException; use Flow\ETL\Exception\InvalidLogicException; use Flow\ETL\Exception\RuntimeException; use Flow\ETL\FlowContext; -use Flow\ETL\Row; use Flow\ETL\Row\Reference; +use Flow\ETL\Rows; use Flow\ETL\Window; use Flow\ETL\Window\Accumulator\CountAccumulator; use Flow\ETL\Window\FrameAccumulator; use Flow\ETL\Window\WindowContext; use Flow\Types\Type; +use function count; use function Flow\Types\DSL\type_integer; +use function sprintf; final class Count implements AggregatingFunction, FrameAccumulating, WindowFunction { @@ -70,16 +73,22 @@ public function withChildren(array $children): static return new self($children[0], $this->window); } - public function aggregate(Row $row, FlowContext $context): void + public function aggregate(Rows $rows, array $indices, FlowContext $context): void { - if ($this->ref !== null && !$row->has($this->ref)) { + if ($this->ref !== null && $rows->schema()->findDefinition($this->ref->base()) === null) { return; } - if ($this->ref) { - $row->get($this->ref); + $this->count += count($indices); + } + + public function merge(AggregatingFunction $other, FlowContext $context): void + { + if (!$other instanceof self) { + throw new InvalidArgumentException(sprintf('%s cannot merge %s', self::class, $other::class)); } - $this->count++; + + $this->count += $other->count; } public function accumulator(FlowContext $context): FrameAccumulator @@ -95,8 +104,10 @@ public function apply(WindowContext $window): mixed $accumulator = $this->accumulator($window->flowContext()); - foreach ($window->frame() as $frameRow) { - $accumulator->accumulate($frameRow); + $frame = $window->frame(); + + for ($index = 0, $count = $frame->count(); $index < $count; $index++) { + $accumulator->accumulate($frame, $index); } return $accumulator->value(); diff --git a/src/core/etl/src/Flow/ETL/Function/DOMElementAttributeValue.php b/src/core/etl/src/Flow/ETL/Function/DOMElementAttributeValue.php index 1514416abe..c269fbfa3f 100644 --- a/src/core/etl/src/Flow/ETL/Function/DOMElementAttributeValue.php +++ b/src/core/etl/src/Flow/ETL/Function/DOMElementAttributeValue.php @@ -7,9 +7,13 @@ use Dom\HTMLElement; use DOMDocument; use DOMNode; +use Exception; +use Flow\ETL\Column\Column; +use Flow\ETL\Exception\EvaluationException; use Flow\ETL\Exception\InvalidArgumentException; use Flow\ETL\FlowContext; -use Flow\ETL\Row; +use Flow\ETL\Function\Evaluation\ResultColumn; +use Flow\ETL\Rows; use Flow\Types\Type; use function class_exists; @@ -60,7 +64,7 @@ public function returns(): Type return type_optional(type_string()); } - public function eval(Row $row, FlowContext $context): ?string + public function eval(Rows $rows, FlowContext $context): Column { $types = [ type_instance_of(DOMNode::class), @@ -72,42 +76,59 @@ public function eval(Row $row, FlowContext $context): ?string $types[] = type_list(type_instance_of(HTMLElement::class)); } - $node = (new Parameter($this->domElement))->as($row, $context, ...$types); + $nodes = (new Parameter($this->domElement))->asTypes($rows, $context, ...$types); + $attributeNames = (new Parameter($this->attribute))->asStrings($rows, $context); + $results = []; + $i = 0; - if ($node instanceof DOMDocument) { - $node = $node->documentElement; - } + try { + foreach ($nodes as $i => $node) { + $attributeName = $attributeNames[$i]; - if (is_array($node) && count($node)) { - $node = reset($node); - } + if ($node instanceof DOMDocument) { + $node = $node->documentElement; + } - $attributeName = (new Parameter($this->attribute))->asString($row, $context); + if (is_array($node) && count($node)) { + $node = reset($node); + } - if ($node === null) { - throw new InvalidArgumentException('DOMElementAttributeValue requires non-null DOMNode'); - } + if ($node === null) { + throw new InvalidArgumentException('DOMElementAttributeValue requires non-null DOMNode'); + } - if ($attributeName === null) { - throw new InvalidArgumentException('DOMElementAttributeValue requires non-null attribute name'); - } + if ($attributeName === null) { + throw new InvalidArgumentException('DOMElementAttributeValue requires non-null attribute name'); + } - if (!$node instanceof DOMNode && !$node instanceof HTMLElement) { - return null; - } + if (!$node instanceof DOMNode && !$node instanceof HTMLElement) { + $results[] = null; - $attributes = $node->attributes; + continue; + } - if ($attributes === null) { - return null; - } + $attributes = $node->attributes; + + if ($attributes === null) { + $results[] = null; + + continue; + } + + $namedItem = $attributes->getNamedItem($attributeName); + + if ($namedItem === null) { + $results[] = null; - $namedItem = $attributes->getNamedItem($attributeName); + continue; + } - if ($namedItem === null) { - return null; + $results[] = $namedItem->nodeValue; + } + } catch (Exception $e) { + throw EvaluationException::at($i, $e); } - return $namedItem->nodeValue; + return (new ResultColumn($context->backend()))->of($this, $results); } } diff --git a/src/core/etl/src/Flow/ETL/Function/DOMElementAttributesCount.php b/src/core/etl/src/Flow/ETL/Function/DOMElementAttributesCount.php index 81a5040a24..ad48955c94 100644 --- a/src/core/etl/src/Flow/ETL/Function/DOMElementAttributesCount.php +++ b/src/core/etl/src/Flow/ETL/Function/DOMElementAttributesCount.php @@ -7,9 +7,13 @@ use Dom\HTMlElement; use DOMElement; use DOMNode; +use Exception; +use Flow\ETL\Column\Column; +use Flow\ETL\Exception\EvaluationException; use Flow\ETL\Exception\InvalidArgumentException; use Flow\ETL\FlowContext; -use Flow\ETL\Row; +use Flow\ETL\Function\Evaluation\ResultColumn; +use Flow\ETL\Rows; use Flow\Types\Type; use function class_exists; @@ -53,7 +57,7 @@ public function returns(): Type return type_integer(); } - public function eval(Row $row, FlowContext $context): ?int + public function eval(Rows $rows, FlowContext $context): Column { $types = [ type_instance_of(DOMElement::class), @@ -63,12 +67,22 @@ public function eval(Row $row, FlowContext $context): ?int $types[] = type_instance_of(HTMLElement::class); } - $domElement = (new Parameter($this->domElement))->as($row, $context, ...$types); + $domElements = (new Parameter($this->domElement))->asTypes($rows, $context, ...$types); + $results = []; + $i = 0; - if ($domElement === null) { - throw new InvalidArgumentException('DOMElementAttributesCount requires non-null DOMElement'); + try { + foreach ($domElements as $i => $domElement) { + if ($domElement === null) { + throw new InvalidArgumentException('DOMElementAttributesCount requires non-null DOMElement'); + } + + $results[] = $domElement->attributes->length ?? 0; + } + } catch (Exception $e) { + throw EvaluationException::at($i, $e); } - return $domElement->attributes->length ?? 0; + return (new ResultColumn($context->backend()))->of($this, $results); } } diff --git a/src/core/etl/src/Flow/ETL/Function/DOMElementNamespaceValue.php b/src/core/etl/src/Flow/ETL/Function/DOMElementNamespaceValue.php index 7477752e06..98e34bf146 100644 --- a/src/core/etl/src/Flow/ETL/Function/DOMElementNamespaceValue.php +++ b/src/core/etl/src/Flow/ETL/Function/DOMElementNamespaceValue.php @@ -11,9 +11,13 @@ use DOMDocument; use DOMNode; use DOMXPath; +use Exception; +use Flow\ETL\Column\Column; +use Flow\ETL\Exception\EvaluationException; use Flow\ETL\Exception\InvalidArgumentException; use Flow\ETL\FlowContext; -use Flow\ETL\Row; +use Flow\ETL\Function\Evaluation\ResultColumn; +use Flow\ETL\Rows; use Flow\Types\Type; use function class_exists; @@ -64,7 +68,7 @@ public function returns(): Type return type_optional(type_string()); } - public function eval(Row $row, FlowContext $context): ?string + public function eval(Rows $rows, FlowContext $context): Column { $types = [ type_instance_of(DOMNode::class), @@ -76,8 +80,28 @@ public function eval(Row $row, FlowContext $context): ?string $types[] = type_list(type_instance_of(Element::class)); } - $node = (new Parameter($this->domElement))->as($row, $context, ...$types); + $nodes = (new Parameter($this->domElement))->asTypes($rows, $context, ...$types); + $attributeNames = (new Parameter($this->attribute))->asStrings($rows, $context); + $results = []; + $i = 0; + try { + // @mago-ignore analysis:mixed-assignment + foreach ($nodes as $i => $node) { + $results[] = $this->namespaceValue($node, $attributeNames[$i]); + } + } catch (Exception $e) { + throw EvaluationException::at($i, $e); + } + + return (new ResultColumn($context->backend()))->of($this, $results); + } + + /** + * @param mixed $node the evaluated DOM operand of one row + */ + public function namespaceValue(mixed $node, ?string $attributeName): ?string + { if ($node instanceof DOMDocument) { $node = $node->documentElement; } @@ -94,8 +118,6 @@ public function eval(Row $row, FlowContext $context): ?string return null; } - $attributeName = (new Parameter($this->attribute))->asString($row, $context); - if ($attributeName === null) { return $node->namespaceURI; } diff --git a/src/core/etl/src/Flow/ETL/Function/DOMElementNextSibling.php b/src/core/etl/src/Flow/ETL/Function/DOMElementNextSibling.php index 86f3d1faae..97d9f5bacd 100644 --- a/src/core/etl/src/Flow/ETL/Function/DOMElementNextSibling.php +++ b/src/core/etl/src/Flow/ETL/Function/DOMElementNextSibling.php @@ -9,9 +9,13 @@ use DOMDocument; use DOMElement; use DOMNode; +use Exception; +use Flow\ETL\Column\Column; +use Flow\ETL\Exception\EvaluationException; use Flow\ETL\Exception\InvalidArgumentException; use Flow\ETL\FlowContext; -use Flow\ETL\Row; +use Flow\ETL\Function\Evaluation\ResultColumn; +use Flow\ETL\Rows; use Flow\Types\Type; use function class_exists; @@ -56,7 +60,7 @@ public function returns(): Type return type_optional(type_xml_element()); } - public function eval(Row $row, FlowContext $context): ?DOMElement + public function eval(Rows $rows, FlowContext $context): Column { $types = [ type_instance_of(DOMNode::class), @@ -67,21 +71,31 @@ public function eval(Row $row, FlowContext $context): ?DOMElement $types[] = type_instance_of(HTMLElement::class); } - $node = (new Parameter($this->element))->as($row, $context, ...$types); - - if ($node instanceof DOMDocument) { - $node = $node->documentElement; - } - - if (!$node instanceof DOMElement) { - throw new InvalidArgumentException('DOMElementNextSibling requires DOMElement.'); - } - - // @mago-ignore analysis:impossible-condition - if ($node instanceof CharacterData) { - throw new InvalidArgumentException('DOMElementNextSibling requires HTMLElement.'); + $nodes = (new Parameter($this->element))->asTypes($rows, $context, ...$types); + $results = []; + $i = 0; + + try { + foreach ($nodes as $i => $node) { + if ($node instanceof DOMDocument) { + $node = $node->documentElement; + } + + if (!$node instanceof DOMElement) { + throw new InvalidArgumentException('DOMElementNextSibling requires DOMElement.'); + } + + // @mago-ignore analysis:impossible-condition + if ($node instanceof CharacterData) { + throw new InvalidArgumentException('DOMElementNextSibling requires HTMLElement.'); + } + + $results[] = $node->nextElementSibling; + } + } catch (Exception $e) { + throw EvaluationException::at($i, $e); } - return $node->nextElementSibling; + return (new ResultColumn($context->backend()))->of($this, $results); } } diff --git a/src/core/etl/src/Flow/ETL/Function/DOMElementParent.php b/src/core/etl/src/Flow/ETL/Function/DOMElementParent.php index d703f9d709..8815ebd412 100644 --- a/src/core/etl/src/Flow/ETL/Function/DOMElementParent.php +++ b/src/core/etl/src/Flow/ETL/Function/DOMElementParent.php @@ -8,9 +8,13 @@ use DOMDocument; use DOMElement; use DOMNode; +use Exception; +use Flow\ETL\Column\Column; +use Flow\ETL\Exception\EvaluationException; use Flow\ETL\Exception\InvalidArgumentException; use Flow\ETL\FlowContext; -use Flow\ETL\Row; +use Flow\ETL\Function\Evaluation\ResultColumn; +use Flow\ETL\Rows; use Flow\Types\Type; use function class_exists; @@ -55,7 +59,7 @@ public function returns(): Type return type_optional(type_xml_element()); } - public function eval(Row $row, FlowContext $context): DOMNode|HTMLElement|null + public function eval(Rows $rows, FlowContext $context): Column { $types = [ type_instance_of(DOMNode::class), @@ -65,22 +69,33 @@ public function eval(Row $row, FlowContext $context): DOMNode|HTMLElement|null $types[] = type_instance_of(HTMLElement::class); } - $node = (new Parameter($this->element))->as($row, $context, ...$types); + $nodes = (new Parameter($this->element))->asTypes($rows, $context, ...$types); + $results = []; + $i = 0; - if ($node instanceof DOMDocument) { - $node = $node->documentElement; - } + try { + foreach ($nodes as $i => $node) { + if ($node instanceof DOMDocument) { + $node = $node->documentElement; + } - if ($node === null) { - throw new InvalidArgumentException('DOMElementParent requires non-null DOMNode or HTMLElement.'); - } + if ($node === null) { + throw new InvalidArgumentException('DOMElementParent requires non-null DOMNode or HTMLElement.'); + } + + if ($node instanceof HTMLElement) { + $results[] = $node->parentElement; + + continue; + } - if ($node instanceof HTMLElement) { - // @mago-ignore analysis:less-specific-return-statement - return $node->parentElement; + // parentNode of a root element is the DOMDocument, which is not an xml_element. + $results[] = $node->parentNode instanceof DOMElement ? $node->parentNode : null; + } + } catch (Exception $e) { + throw EvaluationException::at($i, $e); } - // parentNode of a root element is the DOMDocument, which is not an xml_element. - return $node->parentNode instanceof DOMElement ? $node->parentNode : null; + return (new ResultColumn($context->backend()))->of($this, $results); } } diff --git a/src/core/etl/src/Flow/ETL/Function/DOMElementPreviousSibling.php b/src/core/etl/src/Flow/ETL/Function/DOMElementPreviousSibling.php index 5047db8ea3..478284313e 100644 --- a/src/core/etl/src/Flow/ETL/Function/DOMElementPreviousSibling.php +++ b/src/core/etl/src/Flow/ETL/Function/DOMElementPreviousSibling.php @@ -9,9 +9,13 @@ use DOMDocument; use DOMElement; use DOMNode; +use Exception; +use Flow\ETL\Column\Column; +use Flow\ETL\Exception\EvaluationException; use Flow\ETL\Exception\InvalidArgumentException; use Flow\ETL\FlowContext; -use Flow\ETL\Row; +use Flow\ETL\Function\Evaluation\ResultColumn; +use Flow\ETL\Rows; use Flow\Types\Type; use function class_exists; @@ -56,7 +60,7 @@ public function returns(): Type return type_optional(type_xml_element()); } - public function eval(Row $row, FlowContext $context): ?DOMElement + public function eval(Rows $rows, FlowContext $context): Column { $types = [ type_instance_of(DOMNode::class), @@ -67,21 +71,31 @@ public function eval(Row $row, FlowContext $context): ?DOMElement $types[] = type_instance_of(HTMLElement::class); } - $node = (new Parameter($this->element))->as($row, $context, ...$types); - - if ($node instanceof DOMDocument) { - $node = $node->documentElement; - } - - if (!$node instanceof DOMElement) { - throw new InvalidArgumentException('DOMElementPreviousSibling requires DOMElement.'); - } - - // @mago-ignore analysis:impossible-condition - if ($node instanceof CharacterData) { - throw new InvalidArgumentException('DOMElementPreviousSibling requires HTMLElement.'); + $nodes = (new Parameter($this->element))->asTypes($rows, $context, ...$types); + $results = []; + $i = 0; + + try { + foreach ($nodes as $i => $node) { + if ($node instanceof DOMDocument) { + $node = $node->documentElement; + } + + if (!$node instanceof DOMElement) { + throw new InvalidArgumentException('DOMElementPreviousSibling requires DOMElement.'); + } + + // @mago-ignore analysis:impossible-condition + if ($node instanceof CharacterData) { + throw new InvalidArgumentException('DOMElementPreviousSibling requires HTMLElement.'); + } + + $results[] = $node->previousElementSibling; + } + } catch (Exception $e) { + throw EvaluationException::at($i, $e); } - return $node->previousElementSibling; + return (new ResultColumn($context->backend()))->of($this, $results); } } diff --git a/src/core/etl/src/Flow/ETL/Function/DOMElementValue.php b/src/core/etl/src/Flow/ETL/Function/DOMElementValue.php index 3f5a45472c..7015fba879 100644 --- a/src/core/etl/src/Flow/ETL/Function/DOMElementValue.php +++ b/src/core/etl/src/Flow/ETL/Function/DOMElementValue.php @@ -9,8 +9,12 @@ use DOMDocument; use DOMElement; use DOMNode; +use Exception; +use Flow\ETL\Column\Column; +use Flow\ETL\Exception\EvaluationException; use Flow\ETL\FlowContext; -use Flow\ETL\Row; +use Flow\ETL\Function\Evaluation\ResultColumn; +use Flow\ETL\Rows; use Flow\Types\Type; use function class_exists; @@ -59,7 +63,7 @@ public function returns(): Type return type_optional(type_string()); } - public function eval(Row $row, FlowContext $context): mixed + public function eval(Rows $rows, FlowContext $context): Column { $types = [ type_instance_of(DOMNode::class), @@ -72,24 +76,38 @@ public function eval(Row $row, FlowContext $context): mixed $types[] = type_list(type_instance_of(HTMLElement::class)); } - $node = (new Parameter($this->node))->as($row, $context, ...$types); + $nodes = (new Parameter($this->node))->asTypes($rows, $context, ...$types); + $results = []; + $i = 0; - if (is_array($node) && count($node)) { - $node = reset($node); - } + try { + foreach ($nodes as $i => $node) { + if (is_array($node) && count($node)) { + $node = reset($node); + } - if ($node instanceof DOMDocument) { - $node = $node->documentElement; - } + if ($node instanceof DOMDocument) { + $node = $node->documentElement; + } - if ($node instanceof DOMElement) { - return $node->nodeValue; - } + if ($node instanceof DOMElement) { + $results[] = $node->nodeValue; + + continue; + } + + if ($node instanceof CharacterData || $node instanceof HTMLElement) { + $results[] = $node->textContent; + + continue; + } - if ($node instanceof CharacterData || $node instanceof HTMLElement) { - return $node->textContent; + $results[] = null; + } + } catch (Exception $e) { + throw EvaluationException::at($i, $e); } - return null; + return (new ResultColumn($context->backend()))->of($this, $results); } } diff --git a/src/core/etl/src/Flow/ETL/Function/DateTimeFormat.php b/src/core/etl/src/Flow/ETL/Function/DateTimeFormat.php index 95d580be2d..855b131510 100644 --- a/src/core/etl/src/Flow/ETL/Function/DateTimeFormat.php +++ b/src/core/etl/src/Flow/ETL/Function/DateTimeFormat.php @@ -5,13 +5,23 @@ namespace Flow\ETL\Function; use DateTimeInterface; +use Exception; +use Flow\ETL\Column\Column; +use Flow\ETL\Column\TextValues; +use Flow\ETL\Exception\EvaluationException; use Flow\ETL\Exception\InvalidArgumentException; use Flow\ETL\FlowContext; -use Flow\ETL\Row; +use Flow\ETL\Function\Evaluation\ResultColumn; +use Flow\ETL\Rows; use Flow\Types\Type; +use Flow\Types\Type\Logical\DateTimeType; +use Flow\Types\Type\Logical\DateType; +use function array_search; use function Flow\ETL\DSL\lit; +use function Flow\Types\DSL\type_bare; use function Flow\Types\DSL\type_string; +use function is_string; final class DateTimeFormat implements ScalarFunction { @@ -51,15 +61,53 @@ public function returns(): Type return type_string(); } - public function eval(Row $row, FlowContext $context): mixed + public function eval(Rows $rows, FlowContext $context): Column { - $value = (new Parameter($this->dateTime))->asInstanceOf($row, $context, DateTimeInterface::class); - $format = (new Parameter($this->format))->asString($row, $context); + // @mago-ignore analysis:mixed-assignment + $format = $this->format instanceof Literal ? $this->format->value() : null; - if ($value === null || $format === null) { - throw new InvalidArgumentException('DateTimeFormat function requires non-null values'); + if (is_string($format)) { + $column = (new Parameter($this->dateTime))->column($rows, $context); + $type = type_bare($column->type()); + + if ($type instanceof DateTimeType || $type instanceof DateType) { + /** @var list $physicals */ + $physicals = $column->physicals(); + + if ($column->nullCount() > 0) { + throw EvaluationException::at( + (int) array_search(null, $physicals, true), + new InvalidArgumentException('DateTimeFormat function requires non-null values'), + ); + } + + return (new ResultColumn($context->backend()))->of($this, (new TextValues())->dateTimes( + $type, + $physicals, + $format, + )); + } + } + + $values = (new Parameter($this->dateTime))->asInstancesOf($rows, $context, DateTimeInterface::class); + $formats = (new Parameter($this->format))->asStrings($rows, $context); + $results = []; + $i = 0; + + try { + foreach ($values as $i => $value) { + $format = $formats[$i]; + + if ($value === null || $format === null) { + throw new InvalidArgumentException('DateTimeFormat function requires non-null values'); + } + + $results[] = $value->format($format); + } + } catch (Exception $e) { + throw EvaluationException::at($i, $e); } - return $value->format($format); + return (new ResultColumn($context->backend()))->of($this, $results); } } diff --git a/src/core/etl/src/Flow/ETL/Function/DenseRank.php b/src/core/etl/src/Flow/ETL/Function/DenseRank.php index d0ecd17f16..6f192d9423 100644 --- a/src/core/etl/src/Flow/ETL/Function/DenseRank.php +++ b/src/core/etl/src/Flow/ETL/Function/DenseRank.php @@ -55,18 +55,15 @@ public function rankPartition(Rows $partition): array throw new BaseRuntimeException('Dens Rank window function requires to be ordered by one column'); } - $comparator = new PeerComparator($orderBy); $ranks = []; $rank = 1; - $previous = null; - foreach ($partition as $row) { - if ($previous !== null && !$comparator->arePeers($previous, $row, $partition->schema())) { + foreach ((new PeerComparator($orderBy))->peersOfPrevious($partition) as $index => $peer) { + if ($index > 0 && !$peer) { $rank++; } $ranks[] = $rank; - $previous = $row; } return $ranks; diff --git a/src/core/etl/src/Flow/ETL/Function/Divide.php b/src/core/etl/src/Flow/ETL/Function/Divide.php index 544fd71f09..37c01c26b4 100644 --- a/src/core/etl/src/Flow/ETL/Function/Divide.php +++ b/src/core/etl/src/Flow/ETL/Function/Divide.php @@ -4,11 +4,14 @@ namespace Flow\ETL\Function; -use Flow\Calculator\Calculator; +use Exception; use Flow\Calculator\Rounding; +use Flow\ETL\Column\Column; +use Flow\ETL\Exception\EvaluationException; use Flow\ETL\Exception\InvalidArgumentException; use Flow\ETL\FlowContext; -use Flow\ETL\Row; +use Flow\ETL\Function\Evaluation\ResultColumn; +use Flow\ETL\Rows; use Flow\Types\Type; use function Flow\ETL\DSL\lit; @@ -60,21 +63,34 @@ public function returns(): Type return type_float(); } - public function eval(Row $row, FlowContext $context): int|float|null + public function eval(Rows $rows, FlowContext $context): Column { - $leftValue = (new Parameter($this->left))->asNumber($row, $context); - $rightValue = (new Parameter($this->right))->asNumber($row, $context); - $scale = (new Parameter($this->scale))->asInt($row, $context); - $rounding = (new Parameter($this->rounding))->asEnum($row, $context, Rounding::class); + $lefts = (new Parameter($this->left))->asNumbers($rows, $context); + $rights = (new Parameter($this->right))->asNumbers($rows, $context); + $scales = (new Parameter($this->scale))->asInts($rows, $context); + $roundings = (new Parameter($this->rounding))->asEnums($rows, $context, Rounding::class); + $calculator = $context->calculator(); + $results = []; + $i = 0; - if ($leftValue === null || $rightValue === null) { - throw new InvalidArgumentException('Divide function requires non-null values'); - } + try { + foreach ($lefts as $i => $leftValue) { + $rightValue = $rights[$i]; + + if ($leftValue === null || $rightValue === null) { + throw new InvalidArgumentException('Divide function requires non-null values'); + } + + if ($rightValue === 0) { + throw new InvalidArgumentException('Divide function cannot divide by zero'); + } - if ($rightValue === 0) { - throw new InvalidArgumentException('Divide function cannot divide by zero'); + $results[] = $calculator->divide($leftValue, $rightValue, $scales[$i], $roundings[$i]); + } + } catch (Exception $e) { + throw EvaluationException::at($i, $e); } - return (new Calculator())->divide($leftValue, $rightValue, $scale, $rounding); + return (new ResultColumn($context->backend()))->of($this, $results); } } diff --git a/src/core/etl/src/Flow/ETL/Function/EndsWith.php b/src/core/etl/src/Flow/ETL/Function/EndsWith.php index 220956a0c4..3b2fb9066c 100644 --- a/src/core/etl/src/Flow/ETL/Function/EndsWith.php +++ b/src/core/etl/src/Flow/ETL/Function/EndsWith.php @@ -4,8 +4,12 @@ namespace Flow\ETL\Function; +use Exception; +use Flow\ETL\Column\Column; +use Flow\ETL\Exception\EvaluationException; use Flow\ETL\FlowContext; -use Flow\ETL\Row; +use Flow\ETL\Function\Evaluation\ResultColumn; +use Flow\ETL\Rows; use Flow\Types\Type; use Flow\Types\Type\Nullability; @@ -51,15 +55,29 @@ public function returns(): Type return (new Nullability())->any(type_boolean(), $this->haystack->returns(), $this->needle->returns()); } - public function eval(Row $row, FlowContext $context): ?bool + public function eval(Rows $rows, FlowContext $context): Column { - $haystack = (new Parameter($this->haystack))->asString($row, $context); - $needle = (new Parameter($this->needle))->asString($row, $context); + $haystacks = (new Parameter($this->haystack))->asStrings($rows, $context); + $needles = (new Parameter($this->needle))->asStrings($rows, $context); + $results = []; + $i = 0; - if ($haystack === null || $needle === null) { - return null; + try { + foreach ($haystacks as $i => $haystack) { + $needle = $needles[$i]; + + if ($haystack === null || $needle === null) { + $results[] = null; + + continue; + } + + $results[] = str_ends_with($haystack, $needle); + } + } catch (Exception $e) { + throw EvaluationException::at($i, $e); } - return str_ends_with($haystack, $needle); + return (new ResultColumn($context->backend()))->of($this, $results); } } diff --git a/src/core/etl/src/Flow/ETL/Function/EnsureEnd.php b/src/core/etl/src/Flow/ETL/Function/EnsureEnd.php index e622b1b0b8..d95ce27100 100644 --- a/src/core/etl/src/Flow/ETL/Function/EnsureEnd.php +++ b/src/core/etl/src/Flow/ETL/Function/EnsureEnd.php @@ -4,9 +4,13 @@ namespace Flow\ETL\Function; +use Exception; +use Flow\ETL\Column\Column; +use Flow\ETL\Exception\EvaluationException; use Flow\ETL\Exception\InvalidArgumentException; use Flow\ETL\FlowContext; -use Flow\ETL\Row; +use Flow\ETL\Function\Evaluation\ResultColumn; +use Flow\ETL\Rows; use Flow\Types\Type; use function Flow\ETL\DSL\lit; @@ -51,19 +55,33 @@ public function returns(): Type return type_string(); } - public function eval(Row $row, FlowContext $context): ?string + public function eval(Rows $rows, FlowContext $context): Column { - $value = (new Parameter($this->value))->asString($row, $context); - $suffix = (new Parameter($this->suffix))->asString($row, $context); + $values = (new Parameter($this->value))->asStrings($rows, $context); + $suffixes = (new Parameter($this->suffix))->asStrings($rows, $context); + $results = []; + $i = 0; - if ($value === null) { - throw new InvalidArgumentException('EnsureEnd function requires non-null value'); - } + try { + foreach ($values as $i => $value) { + $suffix = $suffixes[$i]; + + if ($value === null) { + throw new InvalidArgumentException('EnsureEnd function requires non-null value'); + } + + if ($suffix === null || $suffix === '') { + $results[] = $value; + + continue; + } - if ($suffix === null || $suffix === '') { - return $value; + $results[] = s($value)->ensureEnd($suffix)->toString(); + } + } catch (Exception $e) { + throw EvaluationException::at($i, $e); } - return s($value)->ensureEnd($suffix)->toString(); + return (new ResultColumn($context->backend()))->of($this, $results); } } diff --git a/src/core/etl/src/Flow/ETL/Function/EnsureStart.php b/src/core/etl/src/Flow/ETL/Function/EnsureStart.php index 785928e340..6568e62336 100644 --- a/src/core/etl/src/Flow/ETL/Function/EnsureStart.php +++ b/src/core/etl/src/Flow/ETL/Function/EnsureStart.php @@ -4,9 +4,13 @@ namespace Flow\ETL\Function; +use Exception; +use Flow\ETL\Column\Column; +use Flow\ETL\Exception\EvaluationException; use Flow\ETL\Exception\InvalidArgumentException; use Flow\ETL\FlowContext; -use Flow\ETL\Row; +use Flow\ETL\Function\Evaluation\ResultColumn; +use Flow\ETL\Rows; use Flow\Types\Type; use function Flow\ETL\DSL\lit; @@ -51,19 +55,33 @@ public function returns(): Type return type_string(); } - public function eval(Row $row, FlowContext $context): ?string + public function eval(Rows $rows, FlowContext $context): Column { - $value = (new Parameter($this->value))->asString($row, $context); - $prefix = (new Parameter($this->prefix))->asString($row, $context); + $values = (new Parameter($this->value))->asStrings($rows, $context); + $prefixes = (new Parameter($this->prefix))->asStrings($rows, $context); + $results = []; + $i = 0; - if ($value === null) { - throw new InvalidArgumentException('EnsureStart function requires non-null value'); - } + try { + foreach ($values as $i => $value) { + $prefix = $prefixes[$i]; + + if ($value === null) { + throw new InvalidArgumentException('EnsureStart function requires non-null value'); + } + + if ($prefix === null || $prefix === '') { + $results[] = $value; + + continue; + } - if ($prefix === null || $prefix === '') { - return $value; + $results[] = s($value)->ensureStart($prefix)->toString(); + } + } catch (Exception $e) { + throw EvaluationException::at($i, $e); } - return s($value)->ensureStart($prefix)->toString(); + return (new ResultColumn($context->backend()))->of($this, $results); } } diff --git a/src/core/etl/src/Flow/ETL/Function/EnumName.php b/src/core/etl/src/Flow/ETL/Function/EnumName.php index 1a6e4399d3..7336ef8421 100644 --- a/src/core/etl/src/Flow/ETL/Function/EnumName.php +++ b/src/core/etl/src/Flow/ETL/Function/EnumName.php @@ -4,9 +4,13 @@ namespace Flow\ETL\Function; +use Exception; +use Flow\ETL\Column\Column; +use Flow\ETL\Exception\EvaluationException; use Flow\ETL\Exception\InvalidArgumentException; use Flow\ETL\FlowContext; -use Flow\ETL\Row; +use Flow\ETL\Function\Evaluation\ResultColumn; +use Flow\ETL\Rows; use Flow\Types\Type; use UnitEnum; @@ -49,14 +53,25 @@ public function returns(): Type return type_string(); } - public function eval(Row $row, FlowContext $context): string + public function eval(Rows $rows, FlowContext $context): Column { - $enum = (new Parameter($this->value))->eval($row, $context); + $enums = (new Parameter($this->value))->values($rows, $context); + $results = []; + $i = 0; - if (!$enum instanceof UnitEnum) { - throw new InvalidArgumentException('EnumName function requires a UnitEnum value'); + try { + // @mago-ignore analysis:mixed-assignment + foreach ($enums as $i => $enum) { + if (!$enum instanceof UnitEnum) { + throw new InvalidArgumentException('EnumName function requires a UnitEnum value'); + } + + $results[] = $enum->name; + } + } catch (Exception $e) { + throw EvaluationException::at($i, $e); } - return $enum->name; + return (new ResultColumn($context->backend()))->of($this, $results); } } diff --git a/src/core/etl/src/Flow/ETL/Function/EnumValue.php b/src/core/etl/src/Flow/ETL/Function/EnumValue.php index 3fed477765..56ef1af74f 100644 --- a/src/core/etl/src/Flow/ETL/Function/EnumValue.php +++ b/src/core/etl/src/Flow/ETL/Function/EnumValue.php @@ -5,10 +5,14 @@ namespace Flow\ETL\Function; use BackedEnum; +use Exception; +use Flow\ETL\Column\Column; +use Flow\ETL\Exception\EvaluationException; use Flow\ETL\Exception\InvalidArgumentException; use Flow\ETL\Exception\SchemaNotDerivableException; use Flow\ETL\FlowContext; -use Flow\ETL\Row; +use Flow\ETL\Function\Evaluation\ResultColumn; +use Flow\ETL\Rows; use Flow\Types\Type; use Flow\Types\Type\Native\EnumType; use ReflectionEnum; @@ -65,14 +69,25 @@ public function returns(): Type return $backingType !== null && $backingType->getName() === 'int' ? type_integer() : type_string(); } - public function eval(Row $row, FlowContext $context): int|string + public function eval(Rows $rows, FlowContext $context): Column { - $enum = (new Parameter($this->value))->eval($row, $context); - - if (!$enum instanceof BackedEnum) { - throw new InvalidArgumentException('EnumValue function requires a BackedEnum value'); + $enums = (new Parameter($this->value))->values($rows, $context); + $results = []; + $i = 0; + + try { + // @mago-ignore analysis:mixed-assignment + foreach ($enums as $i => $enum) { + if (!$enum instanceof BackedEnum) { + throw new InvalidArgumentException('EnumValue function requires a BackedEnum value'); + } + + $results[] = $enum->value; + } + } catch (Exception $e) { + throw EvaluationException::at($i, $e); } - return $enum->value; + return (new ResultColumn($context->backend()))->of($this, $results); } } diff --git a/src/core/etl/src/Flow/ETL/Function/Equals.php b/src/core/etl/src/Flow/ETL/Function/Equals.php index af9c492343..16f3ab93ab 100644 --- a/src/core/etl/src/Flow/ETL/Function/Equals.php +++ b/src/core/etl/src/Flow/ETL/Function/Equals.php @@ -7,13 +7,19 @@ use DateInterval; use DateTimeImmutable; use DateTimeInterface; +use Exception; +use Flow\ETL\Column\Column; +use Flow\ETL\Column\ComparableValues; +use Flow\ETL\Exception\EvaluationException; use Flow\ETL\FlowContext; -use Flow\ETL\Row; +use Flow\ETL\Function\Evaluation\ResultColumn; +use Flow\ETL\Rows; use Flow\Types\Type; use Flow\Types\Type\Nullability; use Flow\Types\Type\ValueComparator; use function Flow\ETL\DSL\lit; +use function Flow\Types\DSL\type_bare; use function Flow\Types\DSL\type_boolean; final class Equals implements ScalarFunction @@ -56,24 +62,56 @@ public function returns(): Type return (new Nullability())->any(type_boolean(), $this->left->returns(), $this->right->returns()); } - public function eval(Row $row, FlowContext $context): ?bool + public function eval(Rows $rows, FlowContext $context): Column { - $left = (new Parameter($this->left))->eval($row, $context); - $right = (new Parameter($this->right))->eval($row, $context); + $leftColumn = (new Parameter($this->left))->column($rows, $context); + $rightColumn = (new Parameter($this->right))->column($rows, $context); + $comparable = new ComparableValues(); + $sameType = type_bare($leftColumn->type())::class === type_bare($rightColumn->type())::class; + [$lefts, $rights] = $sameType + ? [$comparable->equality($leftColumn), $comparable->equality($rightColumn)] + : [$leftColumn->values(), $rightColumn->values()]; + $results = []; + $i = 0; - if ($left === null || $right === null) { - return null; + // same-typed physicals: every arm of the dispatch below is === on them + if ($sameType && $comparable->equalByPhysical($leftColumn->type())) { + // @mago-ignore analysis:mixed-assignment + foreach ($lefts as $i => $left) { + $results[] = $left === null || $rights[$i] === null ? null : $left === $rights[$i]; + } + + return (new ResultColumn($context->backend()))->of($this, $results); + } + + try { + // @mago-ignore analysis:mixed-assignment + foreach ($lefts as $i => $left) { + // @mago-ignore analysis:mixed-assignment + $right = $rights[$i]; + + if ($left === null || $right === null) { + $results[] = null; + + continue; + } + + // The dispatch picks a comparison strategy from the values, never a column type - bind has + // already proved the pair comparable in returns(). + // @mago-expect analysis:mixed-operand(2) + $results[] = match (true) { + is_int($left) || is_float($left) || is_int($right) || is_float($right) => $left == $right, + $left instanceof DateTimeInterface && $right instanceof DateTimeInterface => $left == $right, + $left instanceof DateInterval && $right instanceof DateInterval => (new DateTimeImmutable( + '@0', + ))->add($left) == (new DateTimeImmutable('@0'))->add($right), + default => $left === $right, + }; + } + } catch (Exception $e) { + throw EvaluationException::at($i, $e); } - // The dispatch picks a comparison strategy from the values, never a column type - bind has - // already proved the pair comparable in returns(). - return match (true) { - is_int($left) || is_float($left) || is_int($right) || is_float($right) => $left == $right, - $left instanceof DateTimeInterface && $right instanceof DateTimeInterface => $left == $right, - $left instanceof DateInterval && $right instanceof DateInterval => (new DateTimeImmutable('@0'))->add( - $left, - ) == (new DateTimeImmutable('@0'))->add($right), - default => $left === $right, - }; + return (new ResultColumn($context->backend()))->of($this, $results); } } diff --git a/src/core/etl/src/Flow/ETL/Function/Evaluation/ResultColumn.php b/src/core/etl/src/Flow/ETL/Function/Evaluation/ResultColumn.php new file mode 100644 index 0000000000..456e4ee78e --- /dev/null +++ b/src/core/etl/src/Flow/ETL/Function/Evaluation/ResultColumn.php @@ -0,0 +1,166 @@ +` column of an ExpandResults function: one list per row. + * + * @param list> $lists + */ + public function lists(ScalarFunction $function, array $lists): Column + { + $element = ReturnTypes::of($function); + + if ($element === null) { + return new ValueColumn($lists); + } + + return $this->typed(type_optional(type_list($element)), $lists); + } + + /** + * @param list $values logical or raw, cast once by the builder + */ + public function of(ScalarFunction $function, array $values): Column + { + $returns = ReturnTypes::of($function); + + if ($returns === null) { + return new ValueColumn($values); + } + + // nullable on purpose: only the step that stores the column (withEntry/filter gate) enforces NOT NULL, as today + return $this->typed($returns instanceof NullType ? $returns : type_optional(type_bare($returns)), $values); + } + + /** + * An intermediate keeps its values in a typed column only when that column holds them losslessly: a type with a + * column kind, no xml/xml_element/html/html_element (a column keeps markup, not the live node) and no untyped + * array (a column keeps it as json) at any depth. + * + * @param Type $type + */ + public function holdsLosslessly(Type $type): bool + { + return (new PhysicalFor())->supports($type) && $this->keepsValues($type); + } + + /** + * @param Type $type + */ + public function keepsValues(Type $type): bool + { + $base = type_bare($type); + + if ( + $base instanceof ArrayType + || $base instanceof XMLType + || $base instanceof XMLElementType + || $base instanceof HTMLType + || $base instanceof HTMLElementType + ) { + return false; + } + + if ($base instanceof ListType) { + return $this->keepsValues($base->element()); + } + + if ($base instanceof MapType) { + return $this->keepsValues($base->value()); + } + + if ($base instanceof StructureType) { + foreach ($base->elements() as $element) { + if (!$this->keepsValues($element->type)) { + return false; + } + } + } + + return true; + } + + /** + * @param Type $type + * @param list $values + */ + public function typed(Type $type, array $values): Column + { + $bare = type_bare($type); + $native = match ($bare::class) { + BooleanType::class => 'bool', + IntegerType::class => 'int', + FloatType::class => 'float', + StringType::class => 'string', + default => null, + }; + + // values that already hold the column's PHP type are its physicals: no definition, no casting builder + if ($native !== null && $type instanceof OptionalType) { + $nulls = 0; + + // @mago-ignore analysis:mixed-assignment + foreach ($values as $value) { + if ($value === null) { + $nulls++; + } elseif (get_debug_type($value) !== $native) { + $native = null; + + break; + } + } + + if ($native !== null) { + $builder = $this->backend->builder(definition_from_type('value', $type)); + $builder->appendPhysicals($values, $nulls); + + return $builder->finish(); + } + } + + if (!$this->holdsLosslessly($type)) { + return new ValueColumn($values); + } + + $builder = $this->backend->builder(definition_from_type('value', $type)); + $builder->appendMany($values); + + return $builder->finish(); + } +} diff --git a/src/core/etl/src/Flow/ETL/Function/Evaluation/ReturnTypes.php b/src/core/etl/src/Flow/ETL/Function/Evaluation/ReturnTypes.php new file mode 100644 index 0000000000..7b29aeeb88 --- /dev/null +++ b/src/core/etl/src/Flow/ETL/Function/Evaluation/ReturnTypes.php @@ -0,0 +1,42 @@ +> false: the result schema is not derivable + */ + private static ?WeakMap $types = null; + + /** + * @return null|Type null when the function's result schema is not derivable + */ + public static function of(ScalarFunction $function): ?Type + { + self::$types ??= new WeakMap(); + + if (!self::$types->offsetExists($function)) { + try { + self::$types[$function] = $function->returns(); + } catch (SchemaNotDerivableException) { + self::$types[$function] = false; + } + } + + $type = self::$types[$function]; + + return $type === false ? null : $type; + } +} diff --git a/src/core/etl/src/Flow/ETL/Function/Evaluation/Selection.php b/src/core/etl/src/Flow/ETL/Function/Evaluation/Selection.php new file mode 100644 index 0000000000..ac650c8817 --- /dev/null +++ b/src/core/etl/src/Flow/ETL/Function/Evaluation/Selection.php @@ -0,0 +1,52 @@ + $indices ascending rows of the batch passed to evaluate() + */ + public function __construct( + private array $indices, + ) {} + + /** + * $function over the selected rows only; an error names the row of $rows it failed on. + */ + public function evaluate(ScalarFunction $function, Rows $rows, FlowContext $context): Column + { + if (count($this->indices) === $rows->count()) { + return $function->eval($rows, $context); + } + + $schema = $rows->schema(); + $names = []; + + // only the columns $function reads are copied; a reference the batch lacks (e.g. inside on_each()) is not one + foreach ((new ReferencedColumns())->in($function)->all() as $ref) { + if ($schema->findDefinition($ref->to()) !== null) { + $names[] = $ref->to(); + } + } + + $subset = $rows->project($schema->keep(...$names), $context->backend())->gather($this->indices); + + try { + return $function->eval($subset, $context); + } catch (EvaluationException $e) { + throw EvaluationException::at($this->indices[$e->rowIndex], $e); + } + } +} diff --git a/src/core/etl/src/Flow/ETL/Function/Evaluation/TolerantEvaluation.php b/src/core/etl/src/Flow/ETL/Function/Evaluation/TolerantEvaluation.php new file mode 100644 index 0000000000..018556453a --- /dev/null +++ b/src/core/etl/src/Flow/ETL/Function/Evaluation/TolerantEvaluation.php @@ -0,0 +1,81 @@ +, values: array} values of every row that did not fail + */ + public function evaluate(ScalarFunction $function, Rows $rows, FlowContext $context): array + { + $live = $rows->isEmpty() ? [] : range(0, $rows->count() - 1); + $failed = []; + $values = []; + $retries = 0; + + while (true) { + try { + // @mago-ignore analysis:mixed-assignment + foreach ((new Selection($live)) + ->evaluate($function, $rows, $context) + ->values() as $k => $value) { + $values[$live[$k]] = $value; + } + + break; + } catch (EvaluationException $e) { + $failed[$e->rowIndex] = true; + $live = array_values(array_diff($live, [$e->rowIndex])); + + if (++$retries === self::RETRIES || $live === []) { + $lane = $this->rowByRow($function, $rows, $live, $context); + + return ['failed' => $failed + $lane['failed'], 'values' => $values + $lane['values']]; + } + } catch (Exception) { + // no row coordinate: nothing to exclude + $lane = $this->rowByRow($function, $rows, $live, $context); + + return ['failed' => $failed + $lane['failed'], 'values' => $values + $lane['values']]; + } + } + + return ['failed' => $failed, 'values' => $values]; + } + + /** + * @param list $indices + * + * @return array{failed: array, values: array} + */ + public function rowByRow(ScalarFunction $function, Rows $rows, array $indices, FlowContext $context): array + { + $failed = []; + $values = []; + + foreach ($indices as $i) { + try { + $values[$i] = $function->eval($rows->slice($i, 1), $context)->values()[0]; + } catch (Exception) { + $failed[$i] = true; + } + } + + return ['failed' => $failed, 'values' => $values]; + } +} diff --git a/src/core/etl/src/Flow/ETL/Function/Exists.php b/src/core/etl/src/Flow/ETL/Function/Exists.php index 295720b486..efdb64eb18 100644 --- a/src/core/etl/src/Flow/ETL/Function/Exists.php +++ b/src/core/etl/src/Flow/ETL/Function/Exists.php @@ -4,12 +4,16 @@ namespace Flow\ETL\Function; -use Exception; +use Flow\ETL\Column\Column; use Flow\ETL\FlowContext; -use Flow\ETL\Row; +use Flow\ETL\Function\Evaluation\ResultColumn; +use Flow\ETL\Function\Evaluation\TolerantEvaluation; use Flow\ETL\Row\Reference; +use Flow\ETL\Rows; use Flow\Types\Type; +use function array_fill; +use function array_key_exists; use function Flow\Types\DSL\type_boolean; final class Exists implements ScalarFunction @@ -48,20 +52,31 @@ public function returns(): Type return type_boolean(); } - public function eval(Row $row, FlowContext $context): bool + public function eval(Rows $rows, FlowContext $context): Column { - try { - if ($this->ref instanceof Reference) { - return $row->has($this->ref->name()); - } + if ($this->ref instanceof Reference) { + $exists = $rows->schema()->findDefinition($this->ref->name()) !== null; - (new Parameter($this->ref))->eval($row, $context); + // @mago-ignore analysis:possibly-invalid-argument + return (new ResultColumn($context->backend()))->of( + $this, + $rows->isEmpty() ? [] : array_fill(0, $rows->count(), $exists), + ); + } + + // the operand is left unresolved by the gate (children() is empty): a reference the batch lacks stays + // unresolved and throws, which is this function's "no" + $failed = (new TolerantEvaluation())->evaluate( + (new ReferenceResolver())->resolve($this->ref, $rows->schema()), + $rows, + $context, + )['failed']; + $results = []; - return true; - } catch (Exception) { - // "Can this reference be reached" is this function's whole question - a throwing - // operand legitimately means "no". Deliberately kept after the LENIENT removal. - return false; + for ($i = 0, $count = $rows->count(); $i < $count; $i++) { + $results[] = !array_key_exists($i, $failed); } + + return (new ResultColumn($context->backend()))->of($this, $results); } } diff --git a/src/core/etl/src/Flow/ETL/Function/First.php b/src/core/etl/src/Flow/ETL/Function/First.php index f828d818e0..e1bd1c8808 100644 --- a/src/core/etl/src/Flow/ETL/Function/First.php +++ b/src/core/etl/src/Flow/ETL/Function/First.php @@ -4,12 +4,14 @@ namespace Flow\ETL\Function; +use Flow\ETL\Exception\InvalidArgumentException; use Flow\ETL\FlowContext; -use Flow\ETL\Row; use Flow\ETL\Row\Reference; +use Flow\ETL\Rows; use Flow\Types\Type; use function Flow\Types\DSL\type_optional; +use function sprintf; final class First implements AggregatingFunction { @@ -49,16 +51,29 @@ public function withChildren(array $children): static return new self($children[0]); } - public function aggregate(Row $row, FlowContext $context): void + public function aggregate(Rows $rows, array $indices, FlowContext $context): void { - if (!$row->has($this->ref)) { + if ($this->found || $indices === [] || $rows->schema()->findDefinition($this->ref->base()) === null) { return; } - if (!$this->found) { - $this->first = $row->get($this->ref); - $this->found = true; + // @mago-ignore analysis:mixed-property-type-coercion + $this->first = $rows->column($this->ref->base())->value($indices[0]); + $this->found = true; + } + + public function merge(AggregatingFunction $other, FlowContext $context): void + { + if (!$other instanceof self) { + throw new InvalidArgumentException(sprintf('%s cannot merge %s', self::class, $other::class)); } + + if ($this->found || !$other->found) { + return; + } + + $this->first = $other->first; + $this->found = true; } public function outputName(): string diff --git a/src/core/etl/src/Flow/ETL/Function/GreaterThan.php b/src/core/etl/src/Flow/ETL/Function/GreaterThan.php index b6b024a158..14fd765b0d 100644 --- a/src/core/etl/src/Flow/ETL/Function/GreaterThan.php +++ b/src/core/etl/src/Flow/ETL/Function/GreaterThan.php @@ -6,13 +6,19 @@ use DateInterval; use DateTimeImmutable; +use Exception; +use Flow\ETL\Column\Column; +use Flow\ETL\Column\ComparableValues; +use Flow\ETL\Exception\EvaluationException; use Flow\ETL\FlowContext; -use Flow\ETL\Row; +use Flow\ETL\Function\Evaluation\ResultColumn; +use Flow\ETL\Rows; use Flow\Types\Type; use Flow\Types\Type\Nullability; use Flow\Types\Type\ValueComparator; use function Flow\ETL\DSL\lit; +use function Flow\Types\DSL\type_bare; use function Flow\Types\DSL\type_boolean; final class GreaterThan implements ScalarFunction @@ -55,23 +61,45 @@ public function returns(): Type return (new Nullability())->any(type_boolean(), $this->left->returns(), $this->right->returns()); } - public function eval(Row $row, FlowContext $context): ?bool + public function eval(Rows $rows, FlowContext $context): Column { - $left = (new Parameter($this->left))->eval($row, $context); - $right = (new Parameter($this->right))->eval($row, $context); - - if ($left === null || $right === null) { - return null; - } - - // The dispatch picks a comparison strategy from the values, never a column type - bind has - // already proved the pair comparable in returns(). - if ($left instanceof DateInterval && $right instanceof DateInterval) { - $reference = new DateTimeImmutable('@0'); - - return $reference->add($left) > $reference->add($right); + $leftColumn = (new Parameter($this->left))->column($rows, $context); + $rightColumn = (new Parameter($this->right))->column($rows, $context); + $comparable = new ComparableValues(); + [$lefts, $rights] = type_bare($leftColumn->type())::class === type_bare($rightColumn->type())::class + ? [$comparable->ordering($leftColumn), $comparable->ordering($rightColumn)] + : [$leftColumn->values(), $rightColumn->values()]; + $results = []; + $i = 0; + + try { + // @mago-ignore analysis:mixed-assignment + foreach ($lefts as $i => $left) { + // @mago-ignore analysis:mixed-assignment + $right = $rights[$i]; + + if ($left === null || $right === null) { + $results[] = null; + + continue; + } + + // The dispatch picks a comparison strategy from the values, never a column type - bind has + // already proved the pair comparable in returns(). + if ($left instanceof DateInterval && $right instanceof DateInterval) { + $reference = new DateTimeImmutable('@0'); + $results[] = $reference->add($left) > $reference->add($right); + + continue; + } + + // @mago-expect analysis:mixed-operand(2) + $results[] = $left > $right; + } + } catch (Exception $e) { + throw EvaluationException::at($i, $e); } - return $left > $right; + return (new ResultColumn($context->backend()))->of($this, $results); } } diff --git a/src/core/etl/src/Flow/ETL/Function/GreaterThanEqual.php b/src/core/etl/src/Flow/ETL/Function/GreaterThanEqual.php index 32d6f68883..ef0a84a058 100644 --- a/src/core/etl/src/Flow/ETL/Function/GreaterThanEqual.php +++ b/src/core/etl/src/Flow/ETL/Function/GreaterThanEqual.php @@ -6,13 +6,19 @@ use DateInterval; use DateTimeImmutable; +use Exception; +use Flow\ETL\Column\Column; +use Flow\ETL\Column\ComparableValues; +use Flow\ETL\Exception\EvaluationException; use Flow\ETL\FlowContext; -use Flow\ETL\Row; +use Flow\ETL\Function\Evaluation\ResultColumn; +use Flow\ETL\Rows; use Flow\Types\Type; use Flow\Types\Type\Nullability; use Flow\Types\Type\ValueComparator; use function Flow\ETL\DSL\lit; +use function Flow\Types\DSL\type_bare; use function Flow\Types\DSL\type_boolean; final class GreaterThanEqual implements ScalarFunction @@ -55,23 +61,45 @@ public function returns(): Type return (new Nullability())->any(type_boolean(), $this->left->returns(), $this->right->returns()); } - public function eval(Row $row, FlowContext $context): ?bool + public function eval(Rows $rows, FlowContext $context): Column { - $left = (new Parameter($this->left))->eval($row, $context); - $right = (new Parameter($this->right))->eval($row, $context); - - if ($left === null || $right === null) { - return null; - } - - // The dispatch picks a comparison strategy from the values, never a column type - bind has - // already proved the pair comparable in returns(). - if ($left instanceof DateInterval && $right instanceof DateInterval) { - $reference = new DateTimeImmutable('@0'); - - return $reference->add($left) >= $reference->add($right); + $leftColumn = (new Parameter($this->left))->column($rows, $context); + $rightColumn = (new Parameter($this->right))->column($rows, $context); + $comparable = new ComparableValues(); + [$lefts, $rights] = type_bare($leftColumn->type())::class === type_bare($rightColumn->type())::class + ? [$comparable->ordering($leftColumn), $comparable->ordering($rightColumn)] + : [$leftColumn->values(), $rightColumn->values()]; + $results = []; + $i = 0; + + try { + // @mago-ignore analysis:mixed-assignment + foreach ($lefts as $i => $left) { + // @mago-ignore analysis:mixed-assignment + $right = $rights[$i]; + + if ($left === null || $right === null) { + $results[] = null; + + continue; + } + + // The dispatch picks a comparison strategy from the values, never a column type - bind has + // already proved the pair comparable in returns(). + if ($left instanceof DateInterval && $right instanceof DateInterval) { + $reference = new DateTimeImmutable('@0'); + $results[] = $reference->add($left) >= $reference->add($right); + + continue; + } + + // @mago-expect analysis:mixed-operand(2) + $results[] = $left >= $right; + } + } catch (Exception $e) { + throw EvaluationException::at($i, $e); } - return $left >= $right; + return (new ResultColumn($context->backend()))->of($this, $results); } } diff --git a/src/core/etl/src/Flow/ETL/Function/Greatest.php b/src/core/etl/src/Flow/ETL/Function/Greatest.php index 67da939ddb..ac6f45fa1e 100644 --- a/src/core/etl/src/Flow/ETL/Function/Greatest.php +++ b/src/core/etl/src/Flow/ETL/Function/Greatest.php @@ -4,8 +4,10 @@ namespace Flow\ETL\Function; +use Flow\ETL\Column\Column; use Flow\ETL\FlowContext; -use Flow\ETL\Row; +use Flow\ETL\Function\Evaluation\ResultColumn; +use Flow\ETL\Rows; use Flow\Types\Exception\InvalidTypeException; use Flow\Types\Type; use Flow\Types\Type\Unifier\NullabilityRule; @@ -69,23 +71,27 @@ public function returns(): Type ); } - public function eval(Row $row, FlowContext $context): mixed + public function eval(Rows $rows, FlowContext $context): Column { - // NULL iff every argument is NULL - nulls are skipped, never compared - $values = []; - - foreach ($this->values as $value) { - $evaluated = (new Parameter($value))->eval($row, $context); - - if ($evaluated !== null) { - $values[] = $evaluated; + $arguments = array_map(static fn(ScalarFunction $value): array => (new Parameter($value))->values( + $rows, + $context, + ), $this->values); + $results = []; + + for ($i = 0, $count = $rows->count(); $i < $count; $i++) { + // NULL iff every argument is NULL - nulls are skipped, never compared + $values = []; + + foreach ($arguments as $argument) { + if ($argument[$i] !== null) { + $values[] = $argument[$i]; + } } - } - if ($values === []) { - return null; + $results[] = $values === [] ? null : max($values); } - return max($values); + return (new ResultColumn($context->backend()))->of($this, $results); } } diff --git a/src/core/etl/src/Flow/ETL/Function/HTMLQuerySelector.php b/src/core/etl/src/Flow/ETL/Function/HTMLQuerySelector.php index f7eebc79c6..916a900f1b 100644 --- a/src/core/etl/src/Flow/ETL/Function/HTMLQuerySelector.php +++ b/src/core/etl/src/Flow/ETL/Function/HTMLQuerySelector.php @@ -4,13 +4,16 @@ namespace Flow\ETL\Function; -use Dom\Element; use Dom\HTMLDocument; use Dom\HTMLElement; +use Exception; +use Flow\ETL\Column\Column; +use Flow\ETL\Exception\EvaluationException; use Flow\ETL\Exception\InvalidArgumentException; use Flow\ETL\Exception\RequiredPHPVersionException; use Flow\ETL\FlowContext; -use Flow\ETL\Row; +use Flow\ETL\Function\Evaluation\ResultColumn; +use Flow\ETL\Rows; use Flow\Types\Type; use function class_exists; @@ -61,24 +64,36 @@ public function returns(): Type return type_optional(type_html_element()); } - public function eval(Row $row, FlowContext $context): ?Element + public function eval(Rows $rows, FlowContext $context): Column { - $value = (new Parameter($this->value))->as( - $row, + $values = (new Parameter($this->value))->asTypes( + $rows, $context, type_instance_of(HTMLDocument::class), type_instance_of(HTMLElement::class), ); - $selector = (new Parameter($this->selector))->asString($row, $context); + $selectors = (new Parameter($this->selector))->asStrings($rows, $context); + $results = []; + $i = 0; - if (null === $value) { - throw new InvalidArgumentException('HTMLQuerySelector requires non-null HTMLDocument'); - } + try { + foreach ($values as $i => $value) { + $selector = $selectors[$i]; + + if (null === $value) { + throw new InvalidArgumentException('HTMLQuerySelector requires non-null HTMLDocument'); + } + + if (null === $selector) { + throw new InvalidArgumentException('HTMLQuerySelector requires non-null selector'); + } - if (null === $selector) { - throw new InvalidArgumentException('HTMLQuerySelector requires non-null selector'); + $results[] = $value->querySelector($selector); + } + } catch (Exception $e) { + throw EvaluationException::at($i, $e); } - return $value->querySelector($selector); + return (new ResultColumn($context->backend()))->of($this, $results); } } diff --git a/src/core/etl/src/Flow/ETL/Function/HTMLQuerySelectorAll.php b/src/core/etl/src/Flow/ETL/Function/HTMLQuerySelectorAll.php index e086960e85..fcfee6954c 100644 --- a/src/core/etl/src/Flow/ETL/Function/HTMLQuerySelectorAll.php +++ b/src/core/etl/src/Flow/ETL/Function/HTMLQuerySelectorAll.php @@ -7,10 +7,14 @@ use DOM\Element; use DOM\HTMLDocument; use Dom\HTMLElement; +use Exception; +use Flow\ETL\Column\Column; +use Flow\ETL\Exception\EvaluationException; use Flow\ETL\Exception\InvalidArgumentException; use Flow\ETL\Exception\RequiredPHPVersionException; use Flow\ETL\FlowContext; -use Flow\ETL\Row; +use Flow\ETL\Function\Evaluation\ResultColumn; +use Flow\ETL\Rows; use Flow\Types\Type; use function class_exists; @@ -62,43 +66,54 @@ public function returns(): Type return type_optional(type_list(type_html_element())); } - /** - * @return null|array - */ - public function eval(Row $row, FlowContext $context): ?array + public function eval(Rows $rows, FlowContext $context): Column { - $value = (new Parameter($this->value))->as( - $row, + $values = (new Parameter($this->value))->asTypes( + $rows, $context, type_instance_of(HTMLDocument::class), type_instance_of(HTMLElement::class), ); - $selector = (new Parameter($this->selector))->asString($row, $context); + $selectors = (new Parameter($this->selector))->asStrings($rows, $context); + $results = []; + $i = 0; - if (null === $value) { - throw new InvalidArgumentException('HTMLQuerySelectorAll requires non-null HTMLDocument'); - } + try { + foreach ($values as $i => $value) { + $selector = $selectors[$i]; - if (null === $selector) { - throw new InvalidArgumentException('HTMLQuerySelectorAll requires non-null selector'); - } + if (null === $value) { + throw new InvalidArgumentException('HTMLQuerySelectorAll requires non-null HTMLDocument'); + } - $result = $value->querySelectorAll($selector); + if (null === $selector) { + throw new InvalidArgumentException('HTMLQuerySelectorAll requires non-null selector'); + } - if (0 === $result->count()) { - return null; - } + $result = $value->querySelectorAll($selector); - $nodes = []; + if (0 === $result->count()) { + $results[] = null; - foreach ($result as $node) { - if (!$node instanceof Element) { - continue; - } + continue; + } + + $nodes = []; + + foreach ($result as $node) { + if (!$node instanceof Element) { + continue; + } - $nodes[] = $node; + $nodes[] = $node; + } + + $results[] = $nodes; + } + } catch (Exception $e) { + throw EvaluationException::at($i, $e); } - return $nodes; + return (new ResultColumn($context->backend()))->of($this, $results); } } diff --git a/src/core/etl/src/Flow/ETL/Function/Hash.php b/src/core/etl/src/Flow/ETL/Function/Hash.php index 8f5fcfdf40..4278743ae9 100644 --- a/src/core/etl/src/Flow/ETL/Function/Hash.php +++ b/src/core/etl/src/Flow/ETL/Function/Hash.php @@ -4,10 +4,14 @@ namespace Flow\ETL\Function; +use Exception; +use Flow\ETL\Column\Column; +use Flow\ETL\Exception\EvaluationException; use Flow\ETL\FlowContext; +use Flow\ETL\Function\Evaluation\ResultColumn; use Flow\ETL\Hash\Algorithm; use Flow\ETL\Hash\NativePHPHash; -use Flow\ETL\Row; +use Flow\ETL\Rows; use Flow\Types\Type; use Flow\Types\Value\Json; @@ -56,20 +60,31 @@ public function returns(): Type return type_optional(type_string()); } - public function eval(Row $row, FlowContext $context): ?string + public function eval(Rows $rows, FlowContext $context): Column { - $value = (new Parameter($this->value))->eval($row, $context); + $values = (new Parameter($this->value))->values($rows, $context); + $results = []; + $i = 0; - if ($value instanceof Json) { - $value = $value->toArray(); + try { + // @mago-ignore analysis:mixed-assignment + foreach ($values as $i => $value) { + if ($value instanceof Json) { + $value = $value->toArray(); + } + + $results[] = match ($value) { + null => null, + default => match (gettype($value)) { + 'array', 'object' => $this->algorithm->hash(serialize($value)), + default => $this->algorithm->hash(is_scalar($value) ? (string) $value : ''), + }, + }; + } + } catch (Exception $e) { + throw EvaluationException::at($i, $e); } - return match ($value) { - null => null, - default => match (gettype($value)) { - 'array', 'object' => $this->algorithm->hash(serialize($value)), - default => $this->algorithm->hash(is_scalar($value) ? (string) $value : ''), - }, - }; + return (new ResultColumn($context->backend()))->of($this, $results); } } diff --git a/src/core/etl/src/Flow/ETL/Function/IndexOf.php b/src/core/etl/src/Flow/ETL/Function/IndexOf.php index a496787bb7..433afe2463 100644 --- a/src/core/etl/src/Flow/ETL/Function/IndexOf.php +++ b/src/core/etl/src/Flow/ETL/Function/IndexOf.php @@ -4,9 +4,13 @@ namespace Flow\ETL\Function; +use Exception; +use Flow\ETL\Column\Column; +use Flow\ETL\Exception\EvaluationException; use Flow\ETL\Exception\InvalidArgumentException; use Flow\ETL\FlowContext; -use Flow\ETL\Row; +use Flow\ETL\Function\Evaluation\ResultColumn; +use Flow\ETL\Rows; use Flow\Types\Type; use function Flow\ETL\DSL\lit; @@ -60,21 +64,40 @@ public function returns(): Type return type_optional(type_integer()); } - public function eval(Row $row, FlowContext $context): ?int + public function eval(Rows $rows, FlowContext $context): Column { - $string = (new Parameter($this->string))->asString($row, $context); - $needle = (new Parameter($this->needle))->asString($row, $context); - $offset = type_integer()->assert((new Parameter($this->offset))->as($row, $context, type_integer())); - $ignoreCase = (new Parameter($this->ignoreCase))->asBoolean($row, $context) ?? false; + $strings = (new Parameter($this->string))->asStrings($rows, $context); + $needles = (new Parameter($this->needle))->asStrings($rows, $context); + $offsets = (new Parameter($this->offset))->asTypes($rows, $context, type_integer()); + $ignoreCases = (new Parameter($this->ignoreCase))->asBooleans($rows, $context); + $results = []; + $i = 0; - if ($string === null || $needle === null) { - throw new InvalidArgumentException('IndexOf function requires non-null string and needle'); - } + try { + foreach ($strings as $i => $string) { + $needle = $needles[$i]; + $offset = $offsets[$i]; + $ignoreCase = $ignoreCases[$i]; + + $offset = type_integer()->assert($offset); + $ignoreCase ??= false; + + if ($string === null || $needle === null) { + throw new InvalidArgumentException('IndexOf function requires non-null string and needle'); + } + + if ($ignoreCase) { + $results[] = u($string)->ignoreCase()->indexOf($needle, $offset); + + continue; + } - if ($ignoreCase) { - return u($string)->ignoreCase()->indexOf($needle, $offset); + $results[] = u($string)->indexOf($needle, $offset); + } + } catch (Exception $e) { + throw EvaluationException::at($i, $e); } - return u($string)->indexOf($needle, $offset); + return (new ResultColumn($context->backend()))->of($this, $results); } } diff --git a/src/core/etl/src/Flow/ETL/Function/IndexOfLast.php b/src/core/etl/src/Flow/ETL/Function/IndexOfLast.php index 28cd3845c4..b2852c454c 100644 --- a/src/core/etl/src/Flow/ETL/Function/IndexOfLast.php +++ b/src/core/etl/src/Flow/ETL/Function/IndexOfLast.php @@ -4,9 +4,13 @@ namespace Flow\ETL\Function; +use Exception; +use Flow\ETL\Column\Column; +use Flow\ETL\Exception\EvaluationException; use Flow\ETL\Exception\InvalidArgumentException; use Flow\ETL\FlowContext; -use Flow\ETL\Row; +use Flow\ETL\Function\Evaluation\ResultColumn; +use Flow\ETL\Rows; use Flow\Types\Type; use function Flow\ETL\DSL\lit; @@ -60,21 +64,40 @@ public function returns(): Type return type_optional(type_integer()); } - public function eval(Row $row, FlowContext $context): ?int + public function eval(Rows $rows, FlowContext $context): Column { - $string = (new Parameter($this->string))->asString($row, $context); - $needle = (new Parameter($this->needle))->asString($row, $context); - $offset = type_integer()->assert((new Parameter($this->offset))->as($row, $context, type_integer())); - $ignoreCase = (new Parameter($this->ignoreCase))->asBoolean($row, $context) ?? false; + $strings = (new Parameter($this->string))->asStrings($rows, $context); + $needles = (new Parameter($this->needle))->asStrings($rows, $context); + $offsets = (new Parameter($this->offset))->asTypes($rows, $context, type_integer()); + $ignoreCases = (new Parameter($this->ignoreCase))->asBooleans($rows, $context); + $results = []; + $i = 0; - if ($string === null || $needle === null) { - throw new InvalidArgumentException('IndexOfLast function requires non-null string and needle'); - } + try { + foreach ($strings as $i => $string) { + $needle = $needles[$i]; + $offset = $offsets[$i]; + $ignoreCase = $ignoreCases[$i]; + + $offset = type_integer()->assert($offset); + $ignoreCase ??= false; + + if ($string === null || $needle === null) { + throw new InvalidArgumentException('IndexOfLast function requires non-null string and needle'); + } + + if ($ignoreCase) { + $results[] = u($string)->ignoreCase()->indexOfLast($needle, $offset); + + continue; + } - if ($ignoreCase) { - return u($string)->ignoreCase()->indexOfLast($needle, $offset); + $results[] = u($string)->indexOfLast($needle, $offset); + } + } catch (Exception $e) { + throw EvaluationException::at($i, $e); } - return u($string)->indexOfLast($needle, $offset); + return (new ResultColumn($context->backend()))->of($this, $results); } } diff --git a/src/core/etl/src/Flow/ETL/Function/IsEmpty.php b/src/core/etl/src/Flow/ETL/Function/IsEmpty.php index 91d8a4287e..24a5557dfa 100644 --- a/src/core/etl/src/Flow/ETL/Function/IsEmpty.php +++ b/src/core/etl/src/Flow/ETL/Function/IsEmpty.php @@ -4,8 +4,12 @@ namespace Flow\ETL\Function; +use Exception; +use Flow\ETL\Column\Column; +use Flow\ETL\Exception\EvaluationException; use Flow\ETL\FlowContext; -use Flow\ETL\Row; +use Flow\ETL\Function\Evaluation\ResultColumn; +use Flow\ETL\Rows; use Flow\Types\Type; use Flow\Types\Type\Nullability; @@ -49,14 +53,26 @@ public function returns(): Type return (new Nullability())->any(type_boolean(), $this->value->returns()); } - public function eval(Row $row, FlowContext $context): ?bool + public function eval(Rows $rows, FlowContext $context): Column { - $value = (new Parameter($this->value))->asString($row, $context); + $values = (new Parameter($this->value))->asStrings($rows, $context); + $results = []; + $i = 0; - if ($value === null) { - return null; + try { + foreach ($values as $i => $value) { + if ($value === null) { + $results[] = null; + + continue; + } + + $results[] = s($value)->isEmpty(); + } + } catch (Exception $e) { + throw EvaluationException::at($i, $e); } - return s($value)->isEmpty(); + return (new ResultColumn($context->backend()))->of($this, $results); } } diff --git a/src/core/etl/src/Flow/ETL/Function/IsIn.php b/src/core/etl/src/Flow/ETL/Function/IsIn.php index 19ff851ed0..02851f2e5e 100644 --- a/src/core/etl/src/Flow/ETL/Function/IsIn.php +++ b/src/core/etl/src/Flow/ETL/Function/IsIn.php @@ -5,8 +5,14 @@ namespace Flow\ETL\Function; use DateTimeInterface; +use Exception; +use Flow\ETL\Column\Column; +use Flow\ETL\Column\ComparableValues; +use Flow\ETL\Column\Physical\PhysicalFor; +use Flow\ETL\Exception\EvaluationException; use Flow\ETL\FlowContext; -use Flow\ETL\Row; +use Flow\ETL\Function\Evaluation\ResultColumn; +use Flow\ETL\Rows; use Flow\Types\Type; use Flow\Types\Type\Logical\ListType; use Flow\Types\Type\Logical\MapType; @@ -14,7 +20,9 @@ use Flow\Types\Type\ValueComparator; use function Flow\ETL\DSL\lit; +use function Flow\Types\DSL\type_bare; use function Flow\Types\DSL\type_boolean; +use function get_debug_type; use function in_array; final class IsIn implements ScalarFunction @@ -76,37 +84,80 @@ public function returns(): Type return $nullability->any(type_boolean(), $haystack, $needle); } - public function eval(Row $row, FlowContext $context): ?bool + public function eval(Rows $rows, FlowContext $context): Column { - $haystack = (new Parameter($this->haystack))->asArray($row, $context); - $needle = (new Parameter($this->needle))->eval($row, $context); - - if ($haystack === null || $needle === null) { - return null; + $haystackParameter = new Parameter($this->haystack); + $haystackColumn = $haystackParameter->column($rows, $context); + $needleColumn = (new Parameter($this->needle))->column($rows, $context); + $haystackType = type_bare($haystackColumn->type()); + $comparable = new ComparableValues(); + + // a list of the needle's type compares by the physical form of its elements: list physicals are the + // element physicals + if ( + $haystackType instanceof ListType + && type_bare($haystackType->element())::class === type_bare($needleColumn->type())::class + && $comparable->equalByPhysical($needleColumn->type()) + ) { + /** @var list> $haystacks */ + $haystacks = $comparable->equalities($haystackColumn->type(), $haystackColumn->physicals()); + $needles = $comparable->equality($needleColumn); + $physical = null; + } else { + $haystacks = $haystackParameter->arraysOf($haystackColumn); + $needles = $needleColumn->values(); + // a candidate of the needle's kind compares by its physical form, as the list path above does + $physical = $comparable->equalByPhysical($needleColumn->type()) + ? (new PhysicalFor())->type(type_bare($needleColumn->type())) + : null; } - // Equals::eval()'s dispatch minus its DateInterval arm, which IsIn never had: two equal - // DateIntervals still compare equal under equals() and NOT equal under isIn(). Bind has - // already proved the pair comparable in returns(). - // @mago-ignore analysis:mixed-assignment - foreach ($haystack as $candidate) { - if ($candidate === null) { - continue; - } - - // @mago-ignore analysis:mixed-operand - if (match (true) { - is_int($needle) || is_float($needle) || is_int($candidate) || is_float($candidate) => $needle - == $candidate, - $needle instanceof DateTimeInterface && $candidate instanceof DateTimeInterface => $needle - == $candidate, - default => $needle === $candidate, - }) { - return true; + $results = []; + $i = 0; + + try { + // @mago-ignore analysis:mixed-assignment + foreach ($needles as $i => $needle) { + $haystack = $haystacks[$i]; + + if ($haystack === null || $needle === null) { + $results[] = null; + + continue; + } + + // Bind has already proved the pair comparable in returns(). + // @mago-ignore analysis:mixed-assignment + foreach ($haystack as $candidate) { + if ($candidate === null) { + continue; + } + + // @mago-expect analysis:mixed-operand(2) + if (match (true) { + is_int($needle) || is_float($needle) || is_int($candidate) || is_float($candidate) => $needle + == $candidate, + $needle instanceof DateTimeInterface && $candidate instanceof DateTimeInterface => $needle + == $candidate, + $physical !== null && get_debug_type($needle) === get_debug_type($candidate) + => $comparable->equalities($needleColumn->type(), [$physical->toPhysical( + $needle, + )]) === $comparable->equalities($needleColumn->type(), [$physical->toPhysical($candidate)]), + default => $needle === $candidate, + }) { + $results[] = true; + + continue 2; + } + } + + // A match beats a NULL element; no match with a NULL element is unknowable + $results[] = in_array(null, $haystack, true) ? null : false; } + } catch (Exception $e) { + throw EvaluationException::at($i, $e); } - // A match beats a NULL element; no match with a NULL element is unknowable - return in_array(null, $haystack, true) ? null : false; + return (new ResultColumn($context->backend()))->of($this, $results); } } diff --git a/src/core/etl/src/Flow/ETL/Function/IsNotNull.php b/src/core/etl/src/Flow/ETL/Function/IsNotNull.php index 4d1e6afbe3..94ca82a2b7 100644 --- a/src/core/etl/src/Flow/ETL/Function/IsNotNull.php +++ b/src/core/etl/src/Flow/ETL/Function/IsNotNull.php @@ -4,10 +4,13 @@ namespace Flow\ETL\Function; +use Flow\ETL\Column\Column; use Flow\ETL\FlowContext; -use Flow\ETL\Row; +use Flow\ETL\Function\Evaluation\ResultColumn; +use Flow\ETL\Rows; use Flow\Types\Type; +use function array_fill; use function Flow\ETL\DSL\lit; use function Flow\Types\DSL\type_boolean; @@ -47,8 +50,24 @@ public function returns(): Type return type_boolean(); } - public function eval(Row $row, FlowContext $context): bool + public function eval(Rows $rows, FlowContext $context): Column { - return (new Parameter($this->value))->eval($row, $context) !== null; + $column = (new Parameter($this->value))->column($rows, $context); + + if ($column->nullCount() === 0) { + // @mago-ignore analysis:possibly-invalid-argument + return (new ResultColumn($context->backend()))->of( + $this, + $rows->isEmpty() ? [] : array_fill(0, $rows->count(), true), + ); + } + + $results = []; + + for ($i = 0, $count = $column->count(); $i < $count; $i++) { + $results[] = !$column->isNull($i); + } + + return (new ResultColumn($context->backend()))->of($this, $results); } } diff --git a/src/core/etl/src/Flow/ETL/Function/IsNotNumeric.php b/src/core/etl/src/Flow/ETL/Function/IsNotNumeric.php index 100ab7e3b6..eab45a1066 100644 --- a/src/core/etl/src/Flow/ETL/Function/IsNotNumeric.php +++ b/src/core/etl/src/Flow/ETL/Function/IsNotNumeric.php @@ -4,8 +4,12 @@ namespace Flow\ETL\Function; +use Exception; +use Flow\ETL\Column\Column; +use Flow\ETL\Exception\EvaluationException; use Flow\ETL\FlowContext; -use Flow\ETL\Row; +use Flow\ETL\Function\Evaluation\ResultColumn; +use Flow\ETL\Rows; use Flow\Types\Type; use Flow\Types\Type\Nullability; @@ -49,14 +53,27 @@ public function returns(): Type return (new Nullability())->any(type_boolean(), $this->value->returns()); } - public function eval(Row $row, FlowContext $context): ?bool + public function eval(Rows $rows, FlowContext $context): Column { - $value = (new Parameter($this->value))->eval($row, $context); + $values = (new Parameter($this->value))->values($rows, $context); + $results = []; + $i = 0; - if ($value === null) { - return null; + try { + // @mago-ignore analysis:mixed-assignment + foreach ($values as $i => $value) { + if ($value === null) { + $results[] = null; + + continue; + } + + $results[] = !is_numeric($value); + } + } catch (Exception $e) { + throw EvaluationException::at($i, $e); } - return !is_numeric($value); + return (new ResultColumn($context->backend()))->of($this, $results); } } diff --git a/src/core/etl/src/Flow/ETL/Function/IsNull.php b/src/core/etl/src/Flow/ETL/Function/IsNull.php index 0349acf157..063cb1fec3 100644 --- a/src/core/etl/src/Flow/ETL/Function/IsNull.php +++ b/src/core/etl/src/Flow/ETL/Function/IsNull.php @@ -4,10 +4,13 @@ namespace Flow\ETL\Function; +use Flow\ETL\Column\Column; use Flow\ETL\FlowContext; -use Flow\ETL\Row; +use Flow\ETL\Function\Evaluation\ResultColumn; +use Flow\ETL\Rows; use Flow\Types\Type; +use function array_fill; use function Flow\ETL\DSL\lit; use function Flow\Types\DSL\type_boolean; @@ -47,8 +50,24 @@ public function returns(): Type return type_boolean(); } - public function eval(Row $row, FlowContext $context): bool + public function eval(Rows $rows, FlowContext $context): Column { - return (new Parameter($this->value))->eval($row, $context) === null; + $column = (new Parameter($this->value))->column($rows, $context); + + if ($column->nullCount() === 0) { + // @mago-ignore analysis:possibly-invalid-argument + return (new ResultColumn($context->backend()))->of( + $this, + $rows->isEmpty() ? [] : array_fill(0, $rows->count(), false), + ); + } + + $results = []; + + for ($i = 0, $count = $column->count(); $i < $count; $i++) { + $results[] = $column->isNull($i); + } + + return (new ResultColumn($context->backend()))->of($this, $results); } } diff --git a/src/core/etl/src/Flow/ETL/Function/IsNumeric.php b/src/core/etl/src/Flow/ETL/Function/IsNumeric.php index aa41618711..36dd3eb6d5 100644 --- a/src/core/etl/src/Flow/ETL/Function/IsNumeric.php +++ b/src/core/etl/src/Flow/ETL/Function/IsNumeric.php @@ -4,8 +4,12 @@ namespace Flow\ETL\Function; +use Exception; +use Flow\ETL\Column\Column; +use Flow\ETL\Exception\EvaluationException; use Flow\ETL\FlowContext; -use Flow\ETL\Row; +use Flow\ETL\Function\Evaluation\ResultColumn; +use Flow\ETL\Rows; use Flow\Types\Type; use Flow\Types\Type\Nullability; @@ -49,14 +53,27 @@ public function returns(): Type return (new Nullability())->any(type_boolean(), $this->value->returns()); } - public function eval(Row $row, FlowContext $context): ?bool + public function eval(Rows $rows, FlowContext $context): Column { - $value = (new Parameter($this->value))->eval($row, $context); + $values = (new Parameter($this->value))->values($rows, $context); + $results = []; + $i = 0; - if ($value === null) { - return null; + try { + // @mago-ignore analysis:mixed-assignment + foreach ($values as $i => $value) { + if ($value === null) { + $results[] = null; + + continue; + } + + $results[] = is_numeric($value); + } + } catch (Exception $e) { + throw EvaluationException::at($i, $e); } - return is_numeric($value); + return (new ResultColumn($context->backend()))->of($this, $results); } } diff --git a/src/core/etl/src/Flow/ETL/Function/IsType.php b/src/core/etl/src/Flow/ETL/Function/IsType.php index f1c1b132b9..3b1c0c13bc 100644 --- a/src/core/etl/src/Flow/ETL/Function/IsType.php +++ b/src/core/etl/src/Flow/ETL/Function/IsType.php @@ -4,8 +4,10 @@ namespace Flow\ETL\Function; +use Flow\ETL\Column\Column; use Flow\ETL\FlowContext; -use Flow\ETL\Row; +use Flow\ETL\Function\Evaluation\ResultColumn; +use Flow\ETL\Rows; use Flow\Types\Type; use Flow\Types\Type\Nullability; use Flow\Types\Type\TypeFactory; @@ -61,22 +63,30 @@ public function returns(): Type return (new Nullability())->any(type_boolean(), $this->value->returns()); } - public function eval(Row $row, FlowContext $context): ?bool + public function eval(Rows $rows, FlowContext $context): Column { - $value = (new Parameter($this->value))->eval($row, $context); + $results = []; - // A NULL operand is unknowable, even against type_null() - ->isNull() is the null test. - if ($value === null) { - return null; - } + // @mago-ignore analysis:mixed-assignment + foreach ((new Parameter($this->value))->values($rows, $context) as $value) { + // A NULL operand is unknowable, even against type_null() - ->isNull() is the null test. + if ($value === null) { + $results[] = null; - foreach ($this->types as $type) { - // @mago-ignore analysis:redundant-type-comparison - if ($type->isValid($value)) { - return true; + continue; } + + foreach ($this->types as $type) { + if ($type->isValid($value)) { + $results[] = true; + + continue 2; + } + } + + $results[] = false; } - return false; + return (new ResultColumn($context->backend()))->of($this, $results); } } diff --git a/src/core/etl/src/Flow/ETL/Function/IsUtf8.php b/src/core/etl/src/Flow/ETL/Function/IsUtf8.php index b078eec7dd..0b68c63b30 100644 --- a/src/core/etl/src/Flow/ETL/Function/IsUtf8.php +++ b/src/core/etl/src/Flow/ETL/Function/IsUtf8.php @@ -4,8 +4,12 @@ namespace Flow\ETL\Function; +use Exception; +use Flow\ETL\Column\Column; +use Flow\ETL\Exception\EvaluationException; use Flow\ETL\FlowContext; -use Flow\ETL\Row; +use Flow\ETL\Function\Evaluation\ResultColumn; +use Flow\ETL\Rows; use Flow\Types\Type; use Flow\Types\Type\Nullability; @@ -49,14 +53,26 @@ public function returns(): Type return (new Nullability())->any(type_boolean(), $this->string->returns()); } - public function eval(Row $row, FlowContext $context): ?bool + public function eval(Rows $rows, FlowContext $context): Column { - $string = (new Parameter($this->string))->asString($row, $context); + $strings = (new Parameter($this->string))->asStrings($rows, $context); + $results = []; + $i = 0; - if ($string === null) { - return null; + try { + foreach ($strings as $i => $string) { + if ($string === null) { + $results[] = null; + + continue; + } + + $results[] = b($string)->isUtf8(); + } + } catch (Exception $e) { + throw EvaluationException::at($i, $e); } - return b($string)->isUtf8(); + return (new ResultColumn($context->backend()))->of($this, $results); } } diff --git a/src/core/etl/src/Flow/ETL/Function/JsonDecode.php b/src/core/etl/src/Flow/ETL/Function/JsonDecode.php index 69b51a897c..fc0362b1e6 100644 --- a/src/core/etl/src/Flow/ETL/Function/JsonDecode.php +++ b/src/core/etl/src/Flow/ETL/Function/JsonDecode.php @@ -4,9 +4,13 @@ namespace Flow\ETL\Function; +use Exception; +use Flow\ETL\Column\Column; +use Flow\ETL\Exception\EvaluationException; use Flow\ETL\Exception\InvalidArgumentException; use Flow\ETL\FlowContext; -use Flow\ETL\Row; +use Flow\ETL\Function\Evaluation\ResultColumn; +use Flow\ETL\Rows; use Flow\Types\Type; use Flow\Types\Value\Json; use JsonException; @@ -55,40 +59,59 @@ public function returns(): Type return type_array(); } - public function eval(Row $row, FlowContext $context): mixed + public function eval(Rows $rows, FlowContext $context): Column { - $value = (new Parameter($this->value))->eval($row, $context); - $flags = (int) (new Parameter($this->flags))->asInt($row, $context); + $values = (new Parameter($this->value))->values($rows, $context); + $flagsList = (new Parameter($this->flags))->asInts($rows, $context); + $results = []; + $i = 0; - if ($value === null) { - throw new InvalidArgumentException('JsonDecode function requires non-null value'); - } + try { + // @mago-ignore analysis:mixed-assignment + foreach ($values as $i => $value) { + $flags = $flagsList[$i]; - if ($value instanceof Json) { - return $value->toArray(); - } + $flags = (int) $flags; - if (is_array($value)) { - return $value; - } + if ($value === null) { + throw new InvalidArgumentException('JsonDecode function requires non-null value'); + } - if (!is_string($value)) { - throw new InvalidArgumentException('JsonDecode function requires string, array, or Json value'); - } + if ($value instanceof Json) { + $results[] = $value->toArray(); - try { - // @mago-ignore analysis:mixed-assignment - $decoded = json_decode($value, true, 512, $flags); - } catch (JsonException $e) { - throw new InvalidArgumentException('JsonDecode error: ' . $e->getMessage()); - } + continue; + } + + if (is_array($value)) { + $results[] = $value; + + continue; + } + + if (!is_string($value)) { + throw new InvalidArgumentException('JsonDecode function requires string, array, or Json value'); + } + + try { + // @mago-ignore analysis:mixed-assignment + $decoded = json_decode($value, true, 512, $flags); + } catch (JsonException $e) { + throw new InvalidArgumentException('JsonDecode error: ' . $e->getMessage(), 0, $e); + } + + if (!is_array($decoded)) { + throw new InvalidArgumentException( + 'JsonDecode function requires JSON that decodes to an array, cast scalar JSON instead', + ); + } - if (!is_array($decoded)) { - throw new InvalidArgumentException( - 'JsonDecode function requires JSON that decodes to an array, cast scalar JSON instead', - ); + $results[] = $decoded; + } + } catch (Exception $e) { + throw EvaluationException::at($i, $e); } - return $decoded; + return (new ResultColumn($context->backend()))->of($this, $results); } } diff --git a/src/core/etl/src/Flow/ETL/Function/JsonEncode.php b/src/core/etl/src/Flow/ETL/Function/JsonEncode.php index b4c6142dbd..2870052a8a 100644 --- a/src/core/etl/src/Flow/ETL/Function/JsonEncode.php +++ b/src/core/etl/src/Flow/ETL/Function/JsonEncode.php @@ -4,9 +4,13 @@ namespace Flow\ETL\Function; +use Exception; +use Flow\ETL\Column\Column; +use Flow\ETL\Exception\EvaluationException; use Flow\ETL\Exception\InvalidArgumentException; use Flow\ETL\FlowContext; -use Flow\ETL\Row; +use Flow\ETL\Function\Evaluation\ResultColumn; +use Flow\ETL\Rows; use Flow\Types\Type; use Flow\Types\Type\Logical\JsonType; use Flow\Types\Type\Logical\ListType; @@ -72,29 +76,48 @@ public function returns(): Type return type_optional($isContainer ? type_json() : type_string()); } - public function eval(Row $row, FlowContext $context): Json|string|null + public function eval(Rows $rows, FlowContext $context): Column { - $value = (new Parameter($this->value))->eval($row, $context); - $flags = (int) (new Parameter($this->flags))->asInt($row, $context); - - if ($value === null) { - return null; - } + $values = (new Parameter($this->value))->values($rows, $context); + $flagsList = (new Parameter($this->flags))->asInts($rows, $context); + $results = []; + $i = 0; try { - $encoded = json_encode($value, $flags); + // @mago-ignore analysis:mixed-assignment + foreach ($values as $i => $value) { + $flags = $flagsList[$i]; - if ($encoded === false) { - throw new InvalidArgumentException('JsonEncode error: json_encode returned false'); - } + $flags = (int) $flags; - if (is_array($value) || is_object($value)) { - return new Json($encoded); - } + if ($value === null) { + $results[] = null; + + continue; + } + + try { + $encoded = json_encode($value, $flags); - return $encoded; - } catch (JsonException $e) { - throw new InvalidArgumentException('JsonEncode error: ' . $e->getMessage()); + if ($encoded === false) { + throw new InvalidArgumentException('JsonEncode error: json_encode returned false'); + } + + if (is_array($value) || is_object($value)) { + $results[] = new Json($encoded); + + continue; + } + + $results[] = $encoded; + } catch (JsonException $e) { + throw new InvalidArgumentException('JsonEncode error: ' . $e->getMessage()); + } + } + } catch (Exception $e) { + throw EvaluationException::at($i, $e); } + + return (new ResultColumn($context->backend()))->of($this, $results); } } diff --git a/src/core/etl/src/Flow/ETL/Function/Last.php b/src/core/etl/src/Flow/ETL/Function/Last.php index 4e0147450a..eb52ddbad4 100644 --- a/src/core/etl/src/Flow/ETL/Function/Last.php +++ b/src/core/etl/src/Flow/ETL/Function/Last.php @@ -4,17 +4,22 @@ namespace Flow\ETL\Function; +use Flow\ETL\Exception\InvalidArgumentException; use Flow\ETL\FlowContext; -use Flow\ETL\Row; use Flow\ETL\Row\Reference; +use Flow\ETL\Rows; use Flow\Types\Type; +use function count; use function Flow\Types\DSL\type_optional; +use function sprintf; final class Last implements AggregatingFunction { use ResolvesFromChildren; + private bool $found = false; + /** * @var null|array|bool|float|int|object|string */ @@ -46,13 +51,29 @@ public function withChildren(array $children): static return new self($children[0]); } - public function aggregate(Row $row, FlowContext $context): void + public function aggregate(Rows $rows, array $indices, FlowContext $context): void { - if (!$row->has($this->ref)) { + if ($indices === [] || $rows->schema()->findDefinition($this->ref->base()) === null) { + return; + } + + // @mago-ignore analysis:mixed-property-type-coercion + $this->last = $rows->column($this->ref->base())->value($indices[count($indices) - 1]); + $this->found = true; + } + + public function merge(AggregatingFunction $other, FlowContext $context): void + { + if (!$other instanceof self) { + throw new InvalidArgumentException(sprintf('%s cannot merge %s', self::class, $other::class)); + } + + if (!$other->found) { return; } - $this->last = $row->get($this->ref); + $this->last = $other->last; + $this->found = true; } public function outputName(): string diff --git a/src/core/etl/src/Flow/ETL/Function/Least.php b/src/core/etl/src/Flow/ETL/Function/Least.php index f10a4d10a3..c89fd37a4e 100644 --- a/src/core/etl/src/Flow/ETL/Function/Least.php +++ b/src/core/etl/src/Flow/ETL/Function/Least.php @@ -4,8 +4,10 @@ namespace Flow\ETL\Function; +use Flow\ETL\Column\Column; use Flow\ETL\FlowContext; -use Flow\ETL\Row; +use Flow\ETL\Function\Evaluation\ResultColumn; +use Flow\ETL\Rows; use Flow\Types\Exception\InvalidTypeException; use Flow\Types\Type; use Flow\Types\Type\Unifier\NullabilityRule; @@ -69,23 +71,27 @@ public function returns(): Type ); } - public function eval(Row $row, FlowContext $context): mixed + public function eval(Rows $rows, FlowContext $context): Column { - // NULL iff every argument is NULL - nulls are skipped, never compared - $values = []; - - foreach ($this->values as $value) { - $evaluated = (new Parameter($value))->eval($row, $context); - - if ($evaluated !== null) { - $values[] = $evaluated; + $arguments = array_map(static fn(ScalarFunction $value): array => (new Parameter($value))->values( + $rows, + $context, + ), $this->values); + $results = []; + + for ($i = 0, $count = $rows->count(); $i < $count; $i++) { + // NULL iff every argument is NULL - nulls are skipped, never compared + $values = []; + + foreach ($arguments as $argument) { + if ($argument[$i] !== null) { + $values[] = $argument[$i]; + } } - } - if ($values === []) { - return null; + $results[] = $values === [] ? null : min($values); } - return min($values); + return (new ResultColumn($context->backend()))->of($this, $results); } } diff --git a/src/core/etl/src/Flow/ETL/Function/LessThan.php b/src/core/etl/src/Flow/ETL/Function/LessThan.php index 6187efc934..f8a228c619 100644 --- a/src/core/etl/src/Flow/ETL/Function/LessThan.php +++ b/src/core/etl/src/Flow/ETL/Function/LessThan.php @@ -6,13 +6,19 @@ use DateInterval; use DateTimeImmutable; +use Exception; +use Flow\ETL\Column\Column; +use Flow\ETL\Column\ComparableValues; +use Flow\ETL\Exception\EvaluationException; use Flow\ETL\FlowContext; -use Flow\ETL\Row; +use Flow\ETL\Function\Evaluation\ResultColumn; +use Flow\ETL\Rows; use Flow\Types\Type; use Flow\Types\Type\Nullability; use Flow\Types\Type\ValueComparator; use function Flow\ETL\DSL\lit; +use function Flow\Types\DSL\type_bare; use function Flow\Types\DSL\type_boolean; final class LessThan implements ScalarFunction @@ -55,23 +61,45 @@ public function returns(): Type return (new Nullability())->any(type_boolean(), $this->left->returns(), $this->right->returns()); } - public function eval(Row $row, FlowContext $context): ?bool + public function eval(Rows $rows, FlowContext $context): Column { - $left = (new Parameter($this->left))->eval($row, $context); - $right = (new Parameter($this->right))->eval($row, $context); - - if ($left === null || $right === null) { - return null; - } - - // The dispatch picks a comparison strategy from the values, never a column type - bind has - // already proved the pair comparable in returns(). - if ($left instanceof DateInterval && $right instanceof DateInterval) { - $reference = new DateTimeImmutable('@0'); - - return $reference->add($left) < $reference->add($right); + $leftColumn = (new Parameter($this->left))->column($rows, $context); + $rightColumn = (new Parameter($this->right))->column($rows, $context); + $comparable = new ComparableValues(); + [$lefts, $rights] = type_bare($leftColumn->type())::class === type_bare($rightColumn->type())::class + ? [$comparable->ordering($leftColumn), $comparable->ordering($rightColumn)] + : [$leftColumn->values(), $rightColumn->values()]; + $results = []; + $i = 0; + + try { + // @mago-ignore analysis:mixed-assignment + foreach ($lefts as $i => $left) { + // @mago-ignore analysis:mixed-assignment + $right = $rights[$i]; + + if ($left === null || $right === null) { + $results[] = null; + + continue; + } + + // The dispatch picks a comparison strategy from the values, never a column type - bind has + // already proved the pair comparable in returns(). + if ($left instanceof DateInterval && $right instanceof DateInterval) { + $reference = new DateTimeImmutable('@0'); + $results[] = $reference->add($left) < $reference->add($right); + + continue; + } + + // @mago-expect analysis:mixed-operand(2) + $results[] = $left < $right; + } + } catch (Exception $e) { + throw EvaluationException::at($i, $e); } - return $left < $right; + return (new ResultColumn($context->backend()))->of($this, $results); } } diff --git a/src/core/etl/src/Flow/ETL/Function/LessThanEqual.php b/src/core/etl/src/Flow/ETL/Function/LessThanEqual.php index c482637658..de7af8e23e 100644 --- a/src/core/etl/src/Flow/ETL/Function/LessThanEqual.php +++ b/src/core/etl/src/Flow/ETL/Function/LessThanEqual.php @@ -6,13 +6,19 @@ use DateInterval; use DateTimeImmutable; +use Exception; +use Flow\ETL\Column\Column; +use Flow\ETL\Column\ComparableValues; +use Flow\ETL\Exception\EvaluationException; use Flow\ETL\FlowContext; -use Flow\ETL\Row; +use Flow\ETL\Function\Evaluation\ResultColumn; +use Flow\ETL\Rows; use Flow\Types\Type; use Flow\Types\Type\Nullability; use Flow\Types\Type\ValueComparator; use function Flow\ETL\DSL\lit; +use function Flow\Types\DSL\type_bare; use function Flow\Types\DSL\type_boolean; final class LessThanEqual implements ScalarFunction @@ -55,23 +61,45 @@ public function returns(): Type return (new Nullability())->any(type_boolean(), $this->left->returns(), $this->right->returns()); } - public function eval(Row $row, FlowContext $context): ?bool + public function eval(Rows $rows, FlowContext $context): Column { - $left = (new Parameter($this->left))->eval($row, $context); - $right = (new Parameter($this->right))->eval($row, $context); - - if ($left === null || $right === null) { - return null; - } - - // The dispatch picks a comparison strategy from the values, never a column type - bind has - // already proved the pair comparable in returns(). - if ($left instanceof DateInterval && $right instanceof DateInterval) { - $reference = new DateTimeImmutable('@0'); - - return $reference->add($left) <= $reference->add($right); + $leftColumn = (new Parameter($this->left))->column($rows, $context); + $rightColumn = (new Parameter($this->right))->column($rows, $context); + $comparable = new ComparableValues(); + [$lefts, $rights] = type_bare($leftColumn->type())::class === type_bare($rightColumn->type())::class + ? [$comparable->ordering($leftColumn), $comparable->ordering($rightColumn)] + : [$leftColumn->values(), $rightColumn->values()]; + $results = []; + $i = 0; + + try { + // @mago-ignore analysis:mixed-assignment + foreach ($lefts as $i => $left) { + // @mago-ignore analysis:mixed-assignment + $right = $rights[$i]; + + if ($left === null || $right === null) { + $results[] = null; + + continue; + } + + // The dispatch picks a comparison strategy from the values, never a column type - bind has + // already proved the pair comparable in returns(). + if ($left instanceof DateInterval && $right instanceof DateInterval) { + $reference = new DateTimeImmutable('@0'); + $results[] = $reference->add($left) <= $reference->add($right); + + continue; + } + + // @mago-expect analysis:mixed-operand(2) + $results[] = $left <= $right; + } + } catch (Exception $e) { + throw EvaluationException::at($i, $e); } - return $left <= $right; + return (new ResultColumn($context->backend()))->of($this, $results); } } diff --git a/src/core/etl/src/Flow/ETL/Function/ListSelect.php b/src/core/etl/src/Flow/ETL/Function/ListSelect.php index a01cfd1a8c..45c5b54dbc 100644 --- a/src/core/etl/src/Flow/ETL/Function/ListSelect.php +++ b/src/core/etl/src/Flow/ETL/Function/ListSelect.php @@ -4,13 +4,15 @@ namespace Flow\ETL\Function; +use Flow\ETL\Column\Column; use Flow\ETL\Exception\InvalidLogicException; use Flow\ETL\Exception\SchemaNotDerivableException; use Flow\ETL\FlowContext; -use Flow\ETL\Row; +use Flow\ETL\Function\Evaluation\ResultColumn; use Flow\ETL\Row\Reference; use Flow\ETL\Row\References; use Flow\ETL\Row\UnresolvedReference; +use Flow\ETL\Rows; use Flow\Types\Type; use Flow\Types\Type\ArrayKey; use Flow\Types\Type\Logical\ListType; @@ -18,6 +20,7 @@ use Flow\Types\Type\Logical\StructureType; use Flow\Types\Type\TypeWidener; +use function array_fill; use function array_key_exists; use function array_values; use function Flow\Types\DSL\structure_element; @@ -100,36 +103,44 @@ public function returns(): Type return type_optional(type_list(new StructureType($elements))); } - /** - * @return null|array> - */ - public function eval(Row $row, FlowContext $context): ?array + public function eval(Rows $rows, FlowContext $context): Column { - if (!$row->has($this->ref)) { - return null; + if ($rows->schema()->findDefinition($this->ref->base()) === null) { + // @mago-ignore analysis:possibly-invalid-argument + return (new ResultColumn($context->backend()))->of( + $this, + $rows->isEmpty() ? [] : array_fill(0, $rows->count(), null), + ); } - $list = $row->get($this->ref); + $results = []; - if (!is_array($list)) { - return null; - } + // @mago-ignore analysis:mixed-assignment + foreach ($rows->column($this->ref->base())->values() as $value) { + if (!is_array($value)) { + $results[] = null; - $output = []; + continue; + } - // @mago-ignore analysis:mixed-assignment - foreach (array_values($list) as $index => $element) { - $output[$index] = []; - - foreach ($this->refs as $ref) { - if (is_array($element) && array_key_exists($ref->to(), $element)) { - $output[$index][$ref->name()] = $element[$ref->to()]; - } else { - $output[$index][$ref->name()] = null; + $output = []; + + // @mago-ignore analysis:mixed-assignment + foreach (array_values($value) as $index => $element) { + $output[$index] = []; + + foreach ($this->refs as $ref) { + if (is_array($element) && array_key_exists($ref->to(), $element)) { + $output[$index][$ref->name()] = $element[$ref->to()]; + } else { + $output[$index][$ref->name()] = null; + } } } + + $results[] = $output; } - return $output; + return (new ResultColumn($context->backend()))->of($this, $results); } } diff --git a/src/core/etl/src/Flow/ETL/Function/Literal.php b/src/core/etl/src/Flow/ETL/Function/Literal.php index 1191700f2e..cfeefc3764 100644 --- a/src/core/etl/src/Flow/ETL/Function/Literal.php +++ b/src/core/etl/src/Flow/ETL/Function/Literal.php @@ -5,13 +5,21 @@ namespace Flow\ETL\Function; use Closure; +use Flow\ETL\Column\Backend; +use Flow\ETL\Column\Column; +use Flow\ETL\Column\ValueColumn; use Flow\ETL\Exception\InvalidArgumentException; use Flow\ETL\FlowContext; -use Flow\ETL\Row; +use Flow\ETL\Function\Evaluation\ResultColumn; +use Flow\ETL\Rows; use Flow\Types\Type; +use Flow\Types\Type\Native\NullType; use Flow\Types\Type\TypeDetector; +use function array_fill; use function array_walk_recursive; +use function Flow\ETL\DSL\definition_from_type; +use function Flow\Types\DSL\type_bare; use function Flow\Types\DSL\type_null; use function Flow\Types\DSL\type_optional; use function is_array; @@ -20,6 +28,16 @@ final class Literal implements ScalarFunction { use ScalarFunctionChain; + /** + * The column of the last evaluated batch, handed out again for a batch of the same size. + */ + private ?Column $column = null; + + /** + * The backend $column was built in: a column is reused only for the same row count in the same backend. + */ + private ?Backend $columnBackend = null; + public function __construct( private readonly mixed $value, ) { @@ -68,9 +86,20 @@ public function returns(): Type return $this->value === null ? type_optional(type_null()) : (new TypeDetector())->detectType($this->value); } - public function eval(Row $row, FlowContext $context): mixed + public function eval(Rows $rows, FlowContext $context): Column { - return $this->value; + if ($this->column?->count() === $rows->count() && $this->columnBackend === $context->backend()) { + return $this->column; + } + + $this->columnBackend = $context->backend(); + $type = type_bare($this->returns()); + $type = $type instanceof NullType ? $type : type_optional($type); + + // @mago-ignore analysis:possibly-invalid-argument + return $this->column = (new ResultColumn($context->backend()))->holdsLosslessly($type) + ? $context->backend()->constant(definition_from_type('value', $type), $this->value, $rows->count()) + : new ValueColumn($rows->isEmpty() ? [] : array_fill(0, $rows->count(), $this->value)); } public function value(): mixed diff --git a/src/core/etl/src/Flow/ETL/Function/MatchCases.php b/src/core/etl/src/Flow/ETL/Function/MatchCases.php index 17214fb6eb..15b3b567af 100644 --- a/src/core/etl/src/Flow/ETL/Function/MatchCases.php +++ b/src/core/etl/src/Flow/ETL/Function/MatchCases.php @@ -4,11 +4,15 @@ namespace Flow\ETL\Function; +use Flow\ETL\Column\Column; +use Flow\ETL\Exception\EvaluationException; use Flow\ETL\Exception\InvalidArgumentException; use Flow\ETL\Exception\InvalidLogicException; use Flow\ETL\FlowContext; +use Flow\ETL\Function\Evaluation\ResultColumn; +use Flow\ETL\Function\Evaluation\Selection; use Flow\ETL\Function\MatchCases\MatchCondition; -use Flow\ETL\Row; +use Flow\ETL\Rows; use Flow\Types\Exception\InvalidTypeException; use Flow\Types\Type; use Flow\Types\Type\Unifier\NullabilityRule; @@ -20,6 +24,7 @@ use function count; use function Flow\ETL\DSL\lit; use function json_encode; +use function range; final class MatchCases implements ScalarFunction { @@ -94,21 +99,64 @@ public function returns(): Type ); } - public function eval(Row $row, FlowContext $context): mixed + public function eval(Rows $rows, FlowContext $context): Column { - foreach ($this->cases as $condition) { - if ($condition->valid($row, $context)) { - return $condition->eval($row, $context); + $pending = $rows->isEmpty() ? [] : range(0, $rows->count() - 1); + $values = []; + + foreach ($this->cases as $case) { + if ($pending === []) { + break; + } + + $taken = []; + $rest = []; + + foreach ($case->valid($rows, $pending, $context) as $k => $valid) { + if ($valid) { + $taken[] = $pending[$k]; + } else { + $rest[] = $pending[$k]; + } } + + if ($taken !== []) { + // @mago-ignore analysis:mixed-assignment + foreach ((new Selection($taken)) + ->evaluate($case, $rows, $context) + ->values() as $k => $value) { + $values[$taken[$k]] = $value; + } + } + + $pending = $rest; } - if ($this->default !== null) { - return (new Parameter($this->default))->eval($row, $context); + if ($pending !== []) { + if ($this->default === null) { + throw new EvaluationException( + $pending[0], + new InvalidArgumentException( + 'Not a single case matches row, consider using default parameter, row: ' + . json_encode($rows->values($pending[0]), JSON_THROW_ON_ERROR), + ), + ); + } + + // @mago-ignore analysis:mixed-assignment + foreach ((new Selection($pending)) + ->evaluate($this->default, $rows, $context) + ->values() as $k => $value) { + $values[$pending[$k]] = $value; + } } - throw new InvalidArgumentException( - 'Not a single case matches row, consider using default parameter, row: ' - . json_encode($row->toArray(), JSON_THROW_ON_ERROR), - ); + $results = []; + + for ($i = 0, $count = $rows->count(); $i < $count; $i++) { + $results[] = $values[$i] ?? null; + } + + return (new ResultColumn($context->backend()))->of($this, $results); } } diff --git a/src/core/etl/src/Flow/ETL/Function/MatchCases/MatchCondition.php b/src/core/etl/src/Flow/ETL/Function/MatchCases/MatchCondition.php index 7560ef4daa..e6d23d9d6f 100644 --- a/src/core/etl/src/Flow/ETL/Function/MatchCases/MatchCondition.php +++ b/src/core/etl/src/Flow/ETL/Function/MatchCases/MatchCondition.php @@ -4,15 +4,22 @@ namespace Flow\ETL\Function\MatchCases; +use Flow\ETL\Column\Column; +use Flow\ETL\Exception\EvaluationException; +use Flow\ETL\Exception\InvalidArgumentException; use Flow\ETL\FlowContext; +use Flow\ETL\Function\Evaluation\Selection; use Flow\ETL\Function\FunctionTree; use Flow\ETL\Function\Parameter; use Flow\ETL\Function\ResolvesFromChildren; use Flow\ETL\Function\ScalarFunction; -use Flow\ETL\Row; +use Flow\ETL\Rows; use Flow\Types\Type; use function Flow\ETL\DSL\lit; +use function get_debug_type; +use function is_scalar; +use function sprintf; final readonly class MatchCondition implements ScalarFunction { @@ -53,13 +60,33 @@ public function returns(): Type return $this->then->returns(); } - public function eval(Row $row, FlowContext $context): mixed + public function eval(Rows $rows, FlowContext $context): Column { - return (new Parameter($this->then))->eval($row, $context); + return (new Parameter($this->then))->column($rows, $context); } - public function valid(Row $row, FlowContext $context): bool + /** + * @param list $indices ascending rows of $rows + * + * @return list one per index + */ + public function valid(Rows $rows, array $indices, FlowContext $context): array { - return (new Parameter($this->condition))->asBoolean($row, $context) ?? false; + $column = (new Selection($indices))->evaluate($this->condition, $rows, $context); + $valid = []; + + // @mago-ignore analysis:mixed-assignment + foreach ($column->values() as $k => $value) { + if ($value !== null && !is_scalar($value)) { + throw new EvaluationException( + $indices[$k], + new InvalidArgumentException(sprintf('Expected type "boolean", got "%s".', get_debug_type($value))), + ); + } + + $valid[] = (bool) $value; + } + + return $valid; } } diff --git a/src/core/etl/src/Flow/ETL/Function/Max.php b/src/core/etl/src/Flow/ETL/Function/Max.php index 27b43c3144..fc89efbafa 100644 --- a/src/core/etl/src/Flow/ETL/Function/Max.php +++ b/src/core/etl/src/Flow/ETL/Function/Max.php @@ -5,14 +5,16 @@ namespace Flow\ETL\Function; use DateTimeInterface; +use Flow\ETL\Exception\InvalidArgumentException; use Flow\ETL\FlowContext; -use Flow\ETL\Row; use Flow\ETL\Row\Reference; +use Flow\ETL\Rows; use Flow\Types\Type; use function Flow\Types\DSL\type_optional; use function is_numeric; use function max; +use function sprintf; final class Max implements AggregatingFunction { @@ -46,30 +48,43 @@ public function withChildren(array $children): static return new self($children[0]); } - public function aggregate(Row $row, FlowContext $context): void + public function aggregate(Rows $rows, array $indices, FlowContext $context): void { - if (!$row->has($this->ref)) { + if ($rows->schema()->findDefinition($this->ref->base()) === null) { return; } - /** @var mixed $value */ - $value = $row->get($this->ref); - - if ($this->max === null) { - if (is_numeric($value)) { - $this->max = (float) $value; - } elseif ($value instanceof DateTimeInterface) { - $this->max = $value; - } - } else { - if (is_numeric($value)) { - $this->max = max($this->max, (float) $value); - } elseif ($value instanceof DateTimeInterface) { - $this->max = max($this->max, $value); + // @mago-ignore analysis:mixed-assignment + foreach ($rows->column($this->ref->base())->take($indices)->values() as $value) { + if ($this->max === null) { + if (is_numeric($value)) { + $this->max = (float) $value; + } elseif ($value instanceof DateTimeInterface) { + $this->max = $value; + } + } else { + if (is_numeric($value)) { + $this->max = max($this->max, (float) $value); + } elseif ($value instanceof DateTimeInterface) { + $this->max = max($this->max, $value); + } } } } + public function merge(AggregatingFunction $other, FlowContext $context): void + { + if (!$other instanceof self) { + throw new InvalidArgumentException(sprintf('%s cannot merge %s', self::class, $other::class)); + } + + if ($other->max === null) { + return; + } + + $this->max = $this->max === null ? $other->max : max($this->max, $other->max); + } + public function outputName(): string { return $this->outputName; diff --git a/src/core/etl/src/Flow/ETL/Function/Min.php b/src/core/etl/src/Flow/ETL/Function/Min.php index 9a1d2b496e..7b061a988d 100644 --- a/src/core/etl/src/Flow/ETL/Function/Min.php +++ b/src/core/etl/src/Flow/ETL/Function/Min.php @@ -5,14 +5,16 @@ namespace Flow\ETL\Function; use DateTimeInterface; +use Flow\ETL\Exception\InvalidArgumentException; use Flow\ETL\FlowContext; -use Flow\ETL\Row; use Flow\ETL\Row\Reference; +use Flow\ETL\Rows; use Flow\Types\Type; use function Flow\Types\DSL\type_optional; use function is_numeric; use function min; +use function sprintf; final class Min implements AggregatingFunction { @@ -46,30 +48,43 @@ public function withChildren(array $children): static return new self($children[0]); } - public function aggregate(Row $row, FlowContext $context): void + public function aggregate(Rows $rows, array $indices, FlowContext $context): void { - if (!$row->has($this->ref)) { + if ($rows->schema()->findDefinition($this->ref->base()) === null) { return; } - /** @var mixed $value */ - $value = $row->get($this->ref); - - if ($this->min === null) { - if (is_numeric($value)) { - $this->min = (float) $value; - } elseif ($value instanceof DateTimeInterface) { - $this->min = $value; - } - } else { - if (is_numeric($value)) { - $this->min = min($this->min, (float) $value); - } elseif ($value instanceof DateTimeInterface) { - $this->min = min($this->min, $value); + // @mago-ignore analysis:mixed-assignment + foreach ($rows->column($this->ref->base())->take($indices)->values() as $value) { + if ($this->min === null) { + if (is_numeric($value)) { + $this->min = (float) $value; + } elseif ($value instanceof DateTimeInterface) { + $this->min = $value; + } + } else { + if (is_numeric($value)) { + $this->min = min($this->min, (float) $value); + } elseif ($value instanceof DateTimeInterface) { + $this->min = min($this->min, $value); + } } } } + public function merge(AggregatingFunction $other, FlowContext $context): void + { + if (!$other instanceof self) { + throw new InvalidArgumentException(sprintf('%s cannot merge %s', self::class, $other::class)); + } + + if ($other->min === null) { + return; + } + + $this->min = $this->min === null ? $other->min : min($this->min, $other->min); + } + public function outputName(): string { return $this->outputName; diff --git a/src/core/etl/src/Flow/ETL/Function/Minus.php b/src/core/etl/src/Flow/ETL/Function/Minus.php index 7d42f54c5b..6cd84d48a3 100644 --- a/src/core/etl/src/Flow/ETL/Function/Minus.php +++ b/src/core/etl/src/Flow/ETL/Function/Minus.php @@ -4,10 +4,13 @@ namespace Flow\ETL\Function; -use Flow\Calculator\Calculator; +use Exception; +use Flow\ETL\Column\Column; +use Flow\ETL\Exception\EvaluationException; use Flow\ETL\Exception\InvalidArgumentException; use Flow\ETL\FlowContext; -use Flow\ETL\Row; +use Flow\ETL\Function\Evaluation\ResultColumn; +use Flow\ETL\Rows; use Flow\Types\Exception\InvalidTypeException; use Flow\Types\Type; use Flow\Types\Type\Native\FloatType; @@ -17,6 +20,8 @@ use function Flow\Types\DSL\type_bare; use function Flow\Types\DSL\type_float; use function Flow\Types\DSL\type_integer; +use function is_float; +use function is_int; final class Minus implements ScalarFunction { @@ -25,8 +30,11 @@ final class Minus implements ScalarFunction private readonly ScalarFunction $left; private readonly ScalarFunction $right; - public function __construct(ScalarFunction|int|float $left, ScalarFunction|int|float $right) - { + public function __construct( + ScalarFunction|int|float $left, + ScalarFunction|int|float $right, + private readonly bool $exact = false, + ) { $this->left = $left instanceof ScalarFunction ? $left : lit($left); $this->right = $right instanceof ScalarFunction ? $right : lit($right); } @@ -45,7 +53,7 @@ public function children(): array public function withChildren(array $children): static { /** @var list $children */ - return new self($children[0], $children[1]); + return new self($children[0], $children[1], $this->exact); } /** @@ -66,15 +74,39 @@ public function returns(): Type return $left instanceof IntegerType && $right instanceof IntegerType ? type_integer() : type_float(); } - public function eval(Row $row, FlowContext $context): int|float|null + public function eval(Rows $rows, FlowContext $context): Column { - $leftValue = (new Parameter($this->left))->asNumber($row, $context); - $rightValue = (new Parameter($this->right))->asNumber($row, $context); + $lefts = (new Parameter($this->left))->asNumbers($rows, $context); + $rights = (new Parameter($this->right))->asNumbers($rows, $context); + $results = []; + $i = 0; + + try { + foreach ($lefts as $i => $left) { + $right = $rights[$i]; + + if ($left === null || $right === null) { + throw new InvalidArgumentException('Minus function requires non-null values'); + } + + if ($this->exact) { + $results[] = $context->calculator()->subtract($left, $right); + + continue; + } + + $result = $left - $right; + + if (is_int($left) && is_int($right) && is_float($result)) { + throw new InvalidArgumentException('Minus function integer overflow'); + } - if ($leftValue === null || $rightValue === null) { - throw new InvalidArgumentException('Minus function requires non-null values'); + $results[] = $result; + } + } catch (Exception $e) { + throw EvaluationException::at($i, $e); } - return (new Calculator())->subtract($leftValue, $rightValue); + return (new ResultColumn($context->backend()))->of($this, $results); } } diff --git a/src/core/etl/src/Flow/ETL/Function/Mod.php b/src/core/etl/src/Flow/ETL/Function/Mod.php index 126a305747..10be6b5f37 100644 --- a/src/core/etl/src/Flow/ETL/Function/Mod.php +++ b/src/core/etl/src/Flow/ETL/Function/Mod.php @@ -4,10 +4,13 @@ namespace Flow\ETL\Function; -use Flow\Calculator\Calculator; +use Exception; +use Flow\ETL\Column\Column; +use Flow\ETL\Exception\EvaluationException; use Flow\ETL\Exception\InvalidArgumentException; use Flow\ETL\FlowContext; -use Flow\ETL\Row; +use Flow\ETL\Function\Evaluation\ResultColumn; +use Flow\ETL\Rows; use Flow\Types\Type; use function Flow\ETL\DSL\lit; @@ -51,19 +54,32 @@ public function returns(): Type return type_integer(); } - public function eval(Row $row, FlowContext $context): ?int + public function eval(Rows $rows, FlowContext $context): Column { - $leftValue = (new Parameter($this->left))->asInt($row, $context); - $rightValue = (new Parameter($this->right))->asInt($row, $context); + $lefts = (new Parameter($this->left))->asInts($rows, $context); + $rights = (new Parameter($this->right))->asInts($rows, $context); + $calculator = $context->calculator(); + $results = []; + $i = 0; - if ($leftValue === null || $rightValue === null) { - throw new InvalidArgumentException('Mod function requires non-null values'); - } + try { + foreach ($lefts as $i => $leftValue) { + $rightValue = $rights[$i]; + + if ($leftValue === null || $rightValue === null) { + throw new InvalidArgumentException('Mod function requires non-null values'); + } + + if ($rightValue === 0) { + throw new InvalidArgumentException('Mod function cannot perform modulo by zero'); + } - if ($rightValue === 0) { - throw new InvalidArgumentException('Mod function cannot perform modulo by zero'); + $results[] = $calculator->modulus($leftValue, $rightValue); + } + } catch (Exception $e) { + throw EvaluationException::at($i, $e); } - return (new Calculator())->modulus($leftValue, $rightValue); + return (new ResultColumn($context->backend()))->of($this, $results); } } diff --git a/src/core/etl/src/Flow/ETL/Function/ModifyDateTime.php b/src/core/etl/src/Flow/ETL/Function/ModifyDateTime.php index 3d48320ecb..ed2ffdecb6 100644 --- a/src/core/etl/src/Flow/ETL/Function/ModifyDateTime.php +++ b/src/core/etl/src/Flow/ETL/Function/ModifyDateTime.php @@ -7,9 +7,13 @@ use DateTime; use DateTimeImmutable; use DateTimeInterface; +use Exception; +use Flow\ETL\Column\Column; +use Flow\ETL\Exception\EvaluationException; use Flow\ETL\Exception\InvalidArgumentException; use Flow\ETL\FlowContext; -use Flow\ETL\Row; +use Flow\ETL\Function\Evaluation\ResultColumn; +use Flow\ETL\Rows; use Flow\Types\Type; use Flow\Types\Type\Logical\DateTimeType; use Flow\Types\Type\Logical\OptionalType; @@ -58,19 +62,33 @@ public function returns(): Type return $base instanceof DateTimeType ? $base : type_datetime(); } - public function eval(Row $row, FlowContext $context): mixed + public function eval(Rows $rows, FlowContext $context): Column { - $value = (new Parameter($this->reference))->asInstanceOf($row, $context, DateTimeInterface::class); - $modifier = (new Parameter($this->modifier))->asString($row, $context); + $values = (new Parameter($this->reference))->asInstancesOf($rows, $context, DateTimeInterface::class); + $modifiers = (new Parameter($this->modifier))->asStrings($rows, $context); + $results = []; + $i = 0; - if ($modifier === null || $value === null) { - throw new InvalidArgumentException('ModifyDateTime function requires non-null values'); - } + try { + foreach ($values as $i => $value) { + $modifier = $modifiers[$i]; + + if ($modifier === null || $value === null) { + throw new InvalidArgumentException('ModifyDateTime function requires non-null values'); + } + + if (!$value instanceof DateTime && !$value instanceof DateTimeImmutable) { + throw new InvalidArgumentException( + 'ModifyDateTime function requires DateTime or DateTimeImmutable object', + ); + } - if (!$value instanceof DateTime && !$value instanceof DateTimeImmutable) { - throw new InvalidArgumentException('ModifyDateTime function requires DateTime or DateTimeImmutable object'); + $results[] = $value->modify($modifier); + } + } catch (Exception $e) { + throw EvaluationException::at($i, $e); } - return $value->modify($modifier); + return (new ResultColumn($context->backend()))->of($this, $results); } } diff --git a/src/core/etl/src/Flow/ETL/Function/Multiply.php b/src/core/etl/src/Flow/ETL/Function/Multiply.php index 266569c541..f52699ee87 100644 --- a/src/core/etl/src/Flow/ETL/Function/Multiply.php +++ b/src/core/etl/src/Flow/ETL/Function/Multiply.php @@ -4,10 +4,13 @@ namespace Flow\ETL\Function; -use Flow\Calculator\Calculator; +use Exception; +use Flow\ETL\Column\Column; +use Flow\ETL\Exception\EvaluationException; use Flow\ETL\Exception\InvalidArgumentException; use Flow\ETL\FlowContext; -use Flow\ETL\Row; +use Flow\ETL\Function\Evaluation\ResultColumn; +use Flow\ETL\Rows; use Flow\Types\Exception\InvalidTypeException; use Flow\Types\Type; use Flow\Types\Type\Native\FloatType; @@ -17,6 +20,8 @@ use function Flow\Types\DSL\type_bare; use function Flow\Types\DSL\type_float; use function Flow\Types\DSL\type_integer; +use function is_float; +use function is_int; final class Multiply implements ScalarFunction { @@ -25,8 +30,11 @@ final class Multiply implements ScalarFunction private readonly ScalarFunction $left; private readonly ScalarFunction $right; - public function __construct(ScalarFunction|int|float $left, ScalarFunction|int|float $right) - { + public function __construct( + ScalarFunction|int|float $left, + ScalarFunction|int|float $right, + private readonly bool $exact = false, + ) { $this->left = $left instanceof ScalarFunction ? $left : lit($left); $this->right = $right instanceof ScalarFunction ? $right : lit($right); } @@ -45,7 +53,7 @@ public function children(): array public function withChildren(array $children): static { /** @var list $children */ - return new self($children[0], $children[1]); + return new self($children[0], $children[1], $this->exact); } /** @@ -66,15 +74,39 @@ public function returns(): Type return $left instanceof IntegerType && $right instanceof IntegerType ? type_integer() : type_float(); } - public function eval(Row $row, FlowContext $context): int|float|null + public function eval(Rows $rows, FlowContext $context): Column { - $leftValue = (new Parameter($this->left))->asNumber($row, $context); - $rightValue = (new Parameter($this->right))->asNumber($row, $context); + $lefts = (new Parameter($this->left))->asNumbers($rows, $context); + $rights = (new Parameter($this->right))->asNumbers($rows, $context); + $results = []; + $i = 0; + + try { + foreach ($lefts as $i => $left) { + $right = $rights[$i]; + + if ($left === null || $right === null) { + throw new InvalidArgumentException('Multiply function requires non-null values'); + } + + if ($this->exact) { + $results[] = $context->calculator()->multiply($left, $right); + + continue; + } + + $result = $left * $right; + + if (is_int($left) && is_int($right) && is_float($result)) { + throw new InvalidArgumentException('Multiply function integer overflow'); + } - if ($leftValue === null || $rightValue === null) { - throw new InvalidArgumentException('Multiply function requires non-null values'); + $results[] = $result; + } + } catch (Exception $e) { + throw EvaluationException::at($i, $e); } - return (new Calculator())->multiply($leftValue, $rightValue); + return (new ResultColumn($context->backend()))->of($this, $results); } } diff --git a/src/core/etl/src/Flow/ETL/Function/Not.php b/src/core/etl/src/Flow/ETL/Function/Not.php index 35ae6c7b3a..09899bcc88 100644 --- a/src/core/etl/src/Flow/ETL/Function/Not.php +++ b/src/core/etl/src/Flow/ETL/Function/Not.php @@ -4,8 +4,10 @@ namespace Flow\ETL\Function; +use Flow\ETL\Column\Column; use Flow\ETL\FlowContext; -use Flow\ETL\Row; +use Flow\ETL\Function\Evaluation\ResultColumn; +use Flow\ETL\Rows; use Flow\Types\Type; use Flow\Types\Type\Nullability; @@ -44,11 +46,16 @@ public function returns(): Type return (new Nullability())->any(type_boolean(), $this->value->returns()); } - public function eval(Row $row, FlowContext $context): ?bool + public function eval(Rows $rows, FlowContext $context): Column { - $value = (new Parameter($this->value))->eval($row, $context); + $results = []; - // SQL NOT NULL is NULL - the row must drop, not flip to true. - return $value === null ? null : !$value; + // @mago-ignore analysis:mixed-assignment + foreach ((new Parameter($this->value))->values($rows, $context) as $value) { + // SQL NOT NULL is NULL - the row must drop, not flip to true. + $results[] = $value === null ? null : !$value; + } + + return (new ResultColumn($context->backend()))->of($this, $results); } } diff --git a/src/core/etl/src/Flow/ETL/Function/NotEquals.php b/src/core/etl/src/Flow/ETL/Function/NotEquals.php index 564d186d2d..75b702ff35 100644 --- a/src/core/etl/src/Flow/ETL/Function/NotEquals.php +++ b/src/core/etl/src/Flow/ETL/Function/NotEquals.php @@ -4,8 +4,10 @@ namespace Flow\ETL\Function; +use Flow\ETL\Column\Column; use Flow\ETL\FlowContext; -use Flow\ETL\Row; +use Flow\ETL\Function\Evaluation\ResultColumn; +use Flow\ETL\Rows; use Flow\Types\Type; use Flow\Types\Type\Nullability; use Flow\Types\Type\ValueComparator; @@ -53,10 +55,17 @@ public function returns(): Type return (new Nullability())->any(type_boolean(), $this->left->returns(), $this->right->returns()); } - public function eval(Row $row, FlowContext $context): ?bool + public function eval(Rows $rows, FlowContext $context): Column { - $equals = (new Equals($this->left, $this->right))->eval($row, $context); + $results = []; - return $equals === null ? null : !$equals; + // @mago-ignore analysis:mixed-assignment + foreach ((new Equals($this->left, $this->right)) + ->eval($rows, $context) + ->physicals() as $equals) { + $results[] = $equals === null ? null : !$equals; + } + + return (new ResultColumn($context->backend()))->of($this, $results); } } diff --git a/src/core/etl/src/Flow/ETL/Function/NotSame.php b/src/core/etl/src/Flow/ETL/Function/NotSame.php index fcf0bcd056..a47ee83a15 100644 --- a/src/core/etl/src/Flow/ETL/Function/NotSame.php +++ b/src/core/etl/src/Flow/ETL/Function/NotSame.php @@ -4,12 +4,18 @@ namespace Flow\ETL\Function; +use Exception; +use Flow\ETL\Column\Column; +use Flow\ETL\Column\ComparableValues; +use Flow\ETL\Exception\EvaluationException; use Flow\ETL\FlowContext; -use Flow\ETL\Row; +use Flow\ETL\Function\Evaluation\ResultColumn; +use Flow\ETL\Rows; use Flow\Types\Type; use Flow\Types\Type\ValueComparator; use function Flow\ETL\DSL\lit; +use function Flow\Types\DSL\type_bare; use function Flow\Types\DSL\type_boolean; final class NotSame implements ScalarFunction @@ -52,11 +58,30 @@ public function returns(): Type return type_boolean(); } - public function eval(Row $row, FlowContext $context): bool + public function eval(Rows $rows, FlowContext $context): Column { - // PHP identity, not SQL equality: null !== null is a defined, useful answer - return ( - (new Parameter($this->left))->eval($row, $context) !== (new Parameter($this->right))->eval($row, $context) - ); + $leftColumn = (new Parameter($this->left))->column($rows, $context); + $rightColumn = (new Parameter($this->right))->column($rows, $context); + $comparable = new ComparableValues(); + [$lefts, $rights] = type_bare($leftColumn->type())::class === type_bare($rightColumn->type())::class + ? [$comparable->equality($leftColumn), $comparable->equality($rightColumn)] + : [$leftColumn->values(), $rightColumn->values()]; + $results = []; + $i = 0; + + try { + // @mago-ignore analysis:mixed-assignment + foreach ($lefts as $i => $left) { + // @mago-ignore analysis:mixed-assignment + $right = $rights[$i]; + + // PHP identity, not SQL equality: null !== null is a defined, useful answer + $results[] = $left !== $right; + } + } catch (Exception $e) { + throw EvaluationException::at($i, $e); + } + + return (new ResultColumn($context->backend()))->of($this, $results); } } diff --git a/src/core/etl/src/Flow/ETL/Function/Now.php b/src/core/etl/src/Flow/ETL/Function/Now.php index 8fbb62af99..1e94f520e7 100644 --- a/src/core/etl/src/Flow/ETL/Function/Now.php +++ b/src/core/etl/src/Flow/ETL/Function/Now.php @@ -6,9 +6,13 @@ use DateTimeImmutable; use DateTimeZone; +use Exception; +use Flow\ETL\Column\Column; +use Flow\ETL\Exception\EvaluationException; use Flow\ETL\Exception\InvalidArgumentException; use Flow\ETL\FlowContext; -use Flow\ETL\Row; +use Flow\ETL\Function\Evaluation\ResultColumn; +use Flow\ETL\Rows; use Flow\Types\Type; use Flow\Types\Type\Logical\DateTimeType; @@ -64,14 +68,24 @@ public function returns(): Type return $this->type; } - public function eval(Row $row, FlowContext $context): ?DateTimeImmutable + public function eval(Rows $rows, FlowContext $context): Column { - $tz = (new Parameter($this->timeZone))->asInstanceOf($row, $context, DateTimeZone::class); + $tzes = (new Parameter($this->timeZone))->asInstancesOf($rows, $context, DateTimeZone::class); + $results = []; + $i = 0; - if ($tz === null) { - throw new InvalidArgumentException('Now function requires valid DateTimeZone'); + try { + foreach ($tzes as $i => $tz) { + if ($tz === null) { + throw new InvalidArgumentException('Now function requires valid DateTimeZone'); + } + + $results[] = $this->type->cast(new DateTimeImmutable('now', $tz)); + } + } catch (Exception $e) { + throw EvaluationException::at($i, $e); } - return $this->type->cast(new DateTimeImmutable('now', $tz)); + return (new ResultColumn($context->backend()))->of($this, $results); } } diff --git a/src/core/etl/src/Flow/ETL/Function/NumberFormat.php b/src/core/etl/src/Flow/ETL/Function/NumberFormat.php index 915d162253..f3e78fb7e6 100644 --- a/src/core/etl/src/Flow/ETL/Function/NumberFormat.php +++ b/src/core/etl/src/Flow/ETL/Function/NumberFormat.php @@ -4,9 +4,13 @@ namespace Flow\ETL\Function; +use Exception; +use Flow\ETL\Column\Column; +use Flow\ETL\Exception\EvaluationException; use Flow\ETL\Exception\InvalidArgumentException; use Flow\ETL\FlowContext; -use Flow\ETL\Row; +use Flow\ETL\Function\Evaluation\ResultColumn; +use Flow\ETL\Rows; use Flow\Types\Type; use function Flow\ETL\DSL\lit; @@ -63,17 +67,36 @@ public function returns(): Type return type_string(); } - public function eval(Row $row, FlowContext $context): ?string + public function eval(Rows $rows, FlowContext $context): Column { - $value = (new Parameter($this->value))->asNumber($row, $context); - $decimals = (new Parameter($this->decimals))->asInt($row, $context); - $decimalSeparator = (new Parameter($this->decimalSeparator))->asString($row, $context); - $thousandsSeparator = (new Parameter($this->thousandsSeparator))->asString($row, $context); + $values = (new Parameter($this->value))->asNumbers($rows, $context); + $decimalsList = (new Parameter($this->decimals))->asInts($rows, $context); + $decimalSeparators = (new Parameter($this->decimalSeparator))->asStrings($rows, $context); + $thousandsSeparators = (new Parameter($this->thousandsSeparator))->asStrings($rows, $context); + $results = []; + $i = 0; - if ($value === null || $decimals === null || $decimalSeparator === null || $thousandsSeparator === null) { - throw new InvalidArgumentException('NumberFormat function requires non-null values'); + try { + foreach ($values as $i => $value) { + $decimals = $decimalsList[$i]; + $decimalSeparator = $decimalSeparators[$i]; + $thousandsSeparator = $thousandsSeparators[$i]; + + if ( + $value === null + || $decimals === null + || $decimalSeparator === null + || $thousandsSeparator === null + ) { + throw new InvalidArgumentException('NumberFormat function requires non-null values'); + } + + $results[] = number_format((float) $value, $decimals, $decimalSeparator, $thousandsSeparator); + } + } catch (Exception $e) { + throw EvaluationException::at($i, $e); } - return number_format((float) $value, $decimals, $decimalSeparator, $thousandsSeparator); + return (new ResultColumn($context->backend()))->of($this, $results); } } diff --git a/src/core/etl/src/Flow/ETL/Function/OnEach.php b/src/core/etl/src/Flow/ETL/Function/OnEach.php index 2e531ed8e8..453a4e91a7 100644 --- a/src/core/etl/src/Flow/ETL/Function/OnEach.php +++ b/src/core/etl/src/Flow/ETL/Function/OnEach.php @@ -4,20 +4,26 @@ namespace Flow\ETL\Function; +use Flow\ETL\Column\Column; +use Flow\ETL\Exception\EvaluationException; use Flow\ETL\Exception\InvalidArgumentException; use Flow\ETL\FlowContext; -use Flow\ETL\Row; +use Flow\ETL\Function\Evaluation\ResultColumn; +use Flow\ETL\Rows; +use Flow\ETL\Schema; use Flow\Types\Type; use Flow\Types\Type\Logical\MapType; use Flow\Types\Type\Logical\StructureElement; use Flow\Types\Type\Logical\StructureType; -use function Flow\ETL\DSL\array_to_row; +use function array_keys; +use function Flow\ETL\DSL\array_to_rows; use function Flow\ETL\DSL\lit; use function Flow\Types\DSL\structure_element; use function Flow\Types\DSL\type_bare; use function Flow\Types\DSL\type_list; use function Flow\Types\DSL\type_map; +use function Flow\Types\DSL\type_optional; final class OnEach implements ScalarFunction { @@ -25,6 +31,11 @@ final class OnEach implements ScalarFunction private readonly ScalarFunction $array; + /** + * @var null|array{Schema, ScalarFunction, Type} + */ + private ?array $bound = null; + /** * @param array|ScalarFunction $array */ @@ -59,12 +70,44 @@ public function withChildren(array $children): static return new self($children[0], $this->function, $this->preserveKeys, $this->elementSchema); } + /** + * The element schema, the body resolved against it and the result type - derived once per node, not per batch. + * + * @return array{Schema, ScalarFunction, Type} + */ + public function bound(): array + { + if ($this->bound === null) { + $arrayType = type_bare($this->array->returns()); + $schema = $this->elementSchema->of($arrayType); + + $this->bound = [ + $schema, + (new ReferenceResolver())->resolve($this->function, $schema), + type_optional($this->typeOver($arrayType)), + ]; + } + + return $this->bound; + } + /** * @return Type */ public function returns(): Type { - $arrayType = type_bare($this->array->returns()); + return $this->typeOver(type_bare($this->array->returns())); + } + + /** + * The produced type for an operand of $arrayType - one read of the operand's returns() serves eval() and bind. + * + * @param Type $arrayType + * + * @return Type + */ + public function typeOver(Type $arrayType): Type + { $inner = $this->elementSchema->of($arrayType); $body = (new ReferenceResolver())->resolve($this->function, $inner); @@ -91,34 +134,54 @@ public function returns(): Type return type_list($bodyType); } - public function eval(Row $row, FlowContext $context): mixed + public function eval(Rows $rows, FlowContext $context): Column { - $value = (new Parameter($this->array))->asArray($row, $context); + $arrays = (new Parameter($this->array))->asArrays($rows, $context); + $elements = []; - if ($value === null) { - throw new InvalidArgumentException('OnEach requires non-null array'); - } + /** @var list $rowOf the row of $rows every element comes from */ + $rowOf = []; - $output = []; + foreach ($arrays as $i => $array) { + if ($array === null) { + throw new EvaluationException($i, new InvalidArgumentException('OnEach requires non-null array')); + } - $hydrator = $context->hydrator(); - // hoisted: array_to_row() runs per element, and the schema is the same for every one of them - $elementSchema = $this->elementSchema->of(type_bare($this->array->returns())); + // @mago-ignore analysis:mixed-assignment + foreach ($array as $item) { + $elements[] = ['element' => $item]; + $rowOf[] = $i; + } + } + + [$elementSchema, $body, $type] = $this->bound(); - // @mago-ignore analysis:mixed-assignment - foreach ($value as $key => $item) { - $result = (new Parameter($this->function))->eval( - array_to_row(['element' => $item], $elementSchema, $hydrator), + try { + $results = (new Parameter($body))->values( + array_to_rows($elements, $elementSchema, $context->backend()), $context, ); + } catch (EvaluationException $e) { + throw EvaluationException::at($rowOf[$e->rowIndex], $e); + } - if ($this->preserveKeys) { - $output[$key] = $result; - } else { - $output[] = $result; + $output = []; + $position = 0; + + foreach ($arrays as $array) { + $values = []; + + foreach (array_keys($array) as $key) { + if ($this->preserveKeys) { + $values[$key] = $results[$position++]; + } else { + $values[] = $results[$position++]; + } } + + $output[] = $values; } - return $output; + return (new ResultColumn($context->backend()))->typed($type, $output); } } diff --git a/src/core/etl/src/Flow/ETL/Function/Optional.php b/src/core/etl/src/Flow/ETL/Function/Optional.php index ca75b1d37e..d90583e559 100644 --- a/src/core/etl/src/Flow/ETL/Function/Optional.php +++ b/src/core/etl/src/Flow/ETL/Function/Optional.php @@ -4,11 +4,14 @@ namespace Flow\ETL\Function; -use Exception; +use Flow\ETL\Column\Column; use Flow\ETL\FlowContext; -use Flow\ETL\Row; +use Flow\ETL\Function\Evaluation\ResultColumn; +use Flow\ETL\Function\Evaluation\TolerantEvaluation; +use Flow\ETL\Rows; use Flow\Types\Type; +use function array_key_exists; use function Flow\Types\DSL\type_optional; final class Optional implements ScalarFunction @@ -44,12 +47,15 @@ public function returns(): Type return type_optional($this->function->returns()); } - public function eval(Row $row, FlowContext $context): mixed + public function eval(Rows $rows, FlowContext $context): Column { - try { - return (new Parameter($this->function))->eval($row, $context); - } catch (Exception) { - return null; + $evaluated = (new TolerantEvaluation())->evaluate($this->function, $rows, $context); + $results = []; + + for ($i = 0, $count = $rows->count(); $i < $count; $i++) { + $results[] = array_key_exists($i, $evaluated['failed']) ? null : $evaluated['values'][$i]; } + + return (new ResultColumn($context->backend()))->of($this, $results); } } diff --git a/src/core/etl/src/Flow/ETL/Function/Parameter.php b/src/core/etl/src/Flow/ETL/Function/Parameter.php index 3a7680f8c2..3547f35b79 100644 --- a/src/core/etl/src/Flow/ETL/Function/Parameter.php +++ b/src/core/etl/src/Flow/ETL/Function/Parameter.php @@ -4,29 +4,39 @@ namespace Flow\ETL\Function; +use Flow\ETL\Column\Column; +use Flow\ETL\Exception\EvaluationException; use Flow\ETL\Exception\InvalidArgumentException; use Flow\ETL\FlowContext; -use Flow\ETL\Row; -use Flow\ETL\Row\Reference; +use Flow\ETL\Rows; use Flow\Types\Exception\InvalidTypeException; use Flow\Types\Type; +use Flow\Types\Type\Native\BooleanType; +use Flow\Types\Type\Native\FloatType; +use Flow\Types\Type\Native\IntegerType; +use Flow\Types\Type\Native\StringType; use Flow\Types\Value\Json; use UnitEnum; +use function array_map; use function Flow\ETL\DSL\lit; use function Flow\Types\DSL\type_array; -use function Flow\Types\DSL\type_float; +use function Flow\Types\DSL\type_bare; use function Flow\Types\DSL\type_instance_of; use function Flow\Types\DSL\type_integer; -use function Flow\Types\DSL\type_object; use function Flow\Types\DSL\type_string; +use function get_debug_type; use function implode; +use function is_float; +use function is_int; +use function is_numeric; +use function is_scalar; use function sprintf; /** - * Every as*() arm separates the two axes: a genuine NULL flowing through is not an error and - * propagates (or takes the arm's null-input default), while a non-null value that cannot coerce - * throws - optional() is the door for pipelines that want the old silent tolerance. + * Every as*() reader separates the two axes: a genuine NULL flowing through is not an error and + * propagates (or takes the reader's null-input default), while a non-null value that cannot coerce + * throws, naming its row - optional() is the door for pipelines that want the old silent tolerance. */ final readonly class Parameter { @@ -38,82 +48,81 @@ public function __construct(mixed $function) } /** - * @template T - * - * @param Type ...$types - * - * @return null|T + * @return list> */ - public function as(Row $row, FlowContext $context, Type ...$types): mixed + public function asArrays(Rows $rows, FlowContext $context): array { - $value = $this->eval($row, $context); - - if ($value === null) { - return null; - } - - foreach ($types as $nextType) { - if ($nextType->isValid($value)) { - return $value; - } - } - - throw new InvalidArgumentException(sprintf( - 'Expected one of "%s", got "%s".', - implode('", "', array_map(static fn(Type $type): string => $type->toString(), $types)), - get_debug_type($value), - )); + return $this->arraysOf($this->column($rows, $context)); } /** - * @return null|array + * @return list> */ - public function asArray(Row $row, FlowContext $context): ?array + public function arraysOf(Column $column): array { - $result = $this->eval($row, $context); + $type = type_array(); + $arrays = []; - if ($result === null) { - return null; - } + // @mago-ignore analysis:mixed-assignment + foreach ($column->values() as $i => $value) { + if ($value === null) { + $arrays[] = null; - if ($result instanceof Json) { - return $result->toArray(); - } + continue; + } - $type = type_array(); + if ($value instanceof Json) { + $arrays[] = $value->toArray(); + + continue; + } + + if (!$type->isValid($value)) { + throw new EvaluationException( + $i, + new InvalidArgumentException(InvalidTypeException::value($value, $type)->getMessage()), + ); + } - if (!$type->isValid($result)) { - throw new InvalidArgumentException(InvalidTypeException::value($result, $type)->getMessage()); + $arrays[] = $value; } - return $result; + return $arrays; } - public function asBoolean(Row $row, FlowContext $context): ?bool + /** + * @return list + */ + public function asBooleans(Rows $rows, FlowContext $context): array { - $result = $this->eval($row, $context); + $column = $this->column($rows, $context); - if ($result === null) { - return null; + if (type_bare($column->type()) instanceof BooleanType) { + /** @var list */ + return $column->physicals(); } - if (!is_scalar($result)) { - throw new InvalidArgumentException(sprintf('Expected type "boolean", got "%s".', get_debug_type($result))); - } + $booleans = []; - return (bool) $result; - } + // @mago-ignore analysis:mixed-assignment + foreach ($column->values() as $i => $value) { + if ($value === null) { + $booleans[] = null; - /** - * @return null|array|bool|float|int|object|string - */ - public function asValue(Row $row): mixed - { - if ($this->function instanceof Reference) { - return $row->has($this->function) ? $row->get($this->function) : null; + continue; + } + + if (!is_scalar($value)) { + throw new EvaluationException( + $i, + new InvalidArgumentException(sprintf('Expected type "boolean", got "%s".', get_debug_type($value))), + ); + } + + $booleans[] = (bool) $value; } - return null; + return $booleans; } /** @@ -121,40 +130,12 @@ public function asValue(Row $row): mixed * * @param class-string $enumClass * - * @return null|T + * @return list */ - public function asEnum(Row $row, FlowContext $context, string $enumClass): ?UnitEnum + public function asEnums(Rows $rows, FlowContext $context, string $enumClass): array { - $result = $this->eval($row, $context); - - if ($result === null) { - return null; - } - - $type = type_instance_of($enumClass); - - if (!$type->isValid($result)) { - throw new InvalidArgumentException(InvalidTypeException::value($result, $type)->getMessage()); - } - - return $result; - } - - public function asFloat(Row $row, FlowContext $context): ?float - { - $result = $this->eval($row, $context); - - if ($result === null) { - return null; - } - - $type = type_float(); - - if (!$type->isValid($result)) { - throw new InvalidArgumentException(InvalidTypeException::value($result, $type)->getMessage()); - } - - return $result; + /** @var list */ + return $this->checked($this->values($rows, $context), type_instance_of($enumClass)); } /** @@ -162,149 +143,207 @@ public function asFloat(Row $row, FlowContext $context): ?float * * @param class-string $class * - * @return null|T + * @return list */ - public function asInstanceOf(Row $row, FlowContext $context, string $class): ?object + public function asInstancesOf(Rows $rows, FlowContext $context, string $class): array { - $result = $this->eval($row, $context); - - if ($result === null) { - return null; - } - - $type = type_instance_of($class); - - if (!$type->isValid($result)) { - throw new InvalidArgumentException(InvalidTypeException::value($result, $type)->getMessage()); - } - - return $result; + /** @var list */ + return $this->checked($this->values($rows, $context), type_instance_of($class)); } /** * $default applies to a NULL input only - a malformed value always throws. * - * @return ($default is null ? int|null : int) + * @return ($default is null ? list : list) */ - public function asInt(Row $row, FlowContext $context, ?int $default = null): ?int + public function asInts(Rows $rows, FlowContext $context, ?int $default = null): array { - $result = $this->eval($row, $context); - - if ($result === null) { - return $default; - } + $column = $this->column($rows, $context); - $type = type_integer(); + /** @var list $ints */ + $ints = type_bare($column->type()) instanceof IntegerType + ? $column->physicals() + : $this->checked($column->values(), type_integer()); - if (!$type->isValid($result)) { - throw new InvalidArgumentException(InvalidTypeException::value($result, $type)->getMessage()); - } - - return $result; + return $default === null ? $ints : array_map(static fn(?int $int): int => $int ?? $default, $ints); } /** * @param class-string $class * - * @return null|array + * @return list> */ - public function asListOfObjects(Row $row, FlowContext $context, string $class): ?array + public function asListsOfObjects(Rows $rows, FlowContext $context, string $class): array { - $result = $this->asArray($row, $context); + $objectType = type_instance_of($class); + $lists = []; - if ($result === null) { - return null; - } + foreach ($this->asArrays($rows, $context) as $i => $array) { + if ($array === null) { + $lists[] = null; - $objectType = type_instance_of($class); - $objects = []; + continue; + } - // @mago-ignore analysis:mixed-assignment - foreach ($result as $item) { - if (!$objectType->isValid($item)) { - throw new InvalidArgumentException(InvalidTypeException::value($item, $objectType)->getMessage()); + $objects = []; + + // @mago-ignore analysis:mixed-assignment + foreach ($array as $item) { + if (!$objectType->isValid($item)) { + throw new EvaluationException( + $i, + new InvalidArgumentException(InvalidTypeException::value($item, $objectType)->getMessage()), + ); + } + + $objects[] = $item; } - $objects[] = $item; + $lists[] = $objects; } - return $objects; + return $lists; } /** * $default applies to a NULL input only - a malformed value always throws. * - * @return ($default is null ? int|float|null : int|float) + * @return ($default is null ? list : list) */ - public function asNumber(Row $row, FlowContext $context, int|float|null $default = null): int|float|null + public function asNumbers(Rows $rows, FlowContext $context, int|float|null $default = null): array { - $result = $this->eval($row, $context); + $column = $this->column($rows, $context); + $type = type_bare($column->type()); - if ($result === null) { - return $default; - } + if ($type instanceof IntegerType || $type instanceof FloatType) { + /** @var list $numbers */ + $numbers = $column->physicals(); - if (!is_numeric($result)) { - throw new InvalidArgumentException(sprintf('Expected type "numeric", got "%s".', get_debug_type($result))); + return $default === null + ? $numbers + : array_map(static fn(int|float|null $number): int|float => $number ?? $default, $numbers); } - if (is_int($result) || is_float($result)) { - return $result; - } + $numbers = []; - // numeric-string: prefer int if the value is integral, otherwise float. - if ((string) (int) $result === $result) { - return (int) $result; - } + // @mago-ignore analysis:mixed-assignment + foreach ($column->values() as $i => $value) { + if ($value === null) { + $numbers[] = $default; - return (float) $result; - } + continue; + } - public function asObject(Row $row, FlowContext $context): ?object - { - $result = $this->eval($row, $context); + if (is_int($value) || is_float($value)) { + $numbers[] = $value; - if ($result === null) { - return null; - } + continue; + } - $type = type_object(); + if (!is_numeric($value)) { + throw new EvaluationException( + $i, + new InvalidArgumentException(sprintf('Expected type "numeric", got "%s".', get_debug_type($value))), + ); + } - if (!$type->isValid($result)) { - throw new InvalidArgumentException(InvalidTypeException::value($result, $type)->getMessage()); + // numeric-string: prefer int if the value is integral, otherwise float. + $numbers[] = (string) (int) $value === $value ? (int) $value : (float) $value; } - return $result; + return $numbers; } /** * $default applies to a NULL input only - a malformed value always throws. * - * @return ($default is null ? string|null : string) + * @return ($default is null ? list : list) */ - public function asString(Row $row, FlowContext $context, ?string $default = null): ?string + public function asStrings(Rows $rows, FlowContext $context, ?string $default = null): array { - $result = $this->eval($row, $context); + $column = $this->column($rows, $context); + + /** @var list $strings */ + $strings = type_bare($column->type()) instanceof StringType + ? $column->physicals() + : $this->checked($column->values(), type_string()); + + return $default === null + ? $strings + : array_map(static fn(?string $string): string => $string ?? $default, $strings); + } - if ($result === null) { - return $default; + /** + * @template T + * + * @param Type ...$types + * + * @return list + */ + public function asTypes(Rows $rows, FlowContext $context, Type ...$types): array + { + $accepted = []; + + // @mago-ignore analysis:mixed-assignment + foreach ($this->values($rows, $context) as $i => $value) { + if ($value === null) { + $accepted[] = null; + + continue; + } + + foreach ($types as $nextType) { + if ($nextType->isValid($value)) { + $accepted[] = $value; + + continue 2; + } + } + + throw new EvaluationException( + $i, + new InvalidArgumentException(sprintf( + 'Expected one of "%s", got "%s".', + implode('", "', array_map(static fn(Type $type): string => $type->toString(), $types)), + get_debug_type($value), + )), + ); } - $type = type_string(); + return $accepted; + } - if (!$type->isValid($result)) { - throw new InvalidArgumentException(InvalidTypeException::value($result, $type)->getMessage()); + /** + * @param list $values + * @param Type $type + * + * @return list + */ + public function checked(array $values, Type $type): array + { + // @mago-ignore analysis:mixed-assignment + foreach ($values as $i => $value) { + if ($value !== null && !$type->isValid($value)) { + throw new EvaluationException( + $i, + new InvalidArgumentException(InvalidTypeException::value($value, $type)->getMessage()), + ); + } } - return $result; + return $values; + } + + public function column(Rows $rows, FlowContext $context): Column + { + return $this->function->eval($rows, $context); } /** - * @return null|array|bool|float|int|object|string + * @return list */ - public function eval(Row $row, FlowContext $context): mixed + public function values(Rows $rows, FlowContext $context): array { - // @mago-ignore analysis:mixed-return-statement - return $this->function->eval($row, $context); + return $this->column($rows, $context)->values(); } } diff --git a/src/core/etl/src/Flow/ETL/Function/Plus.php b/src/core/etl/src/Flow/ETL/Function/Plus.php index ce87007dbe..c6e0b13ccf 100644 --- a/src/core/etl/src/Flow/ETL/Function/Plus.php +++ b/src/core/etl/src/Flow/ETL/Function/Plus.php @@ -4,10 +4,13 @@ namespace Flow\ETL\Function; -use Flow\Calculator\Calculator; +use Exception; +use Flow\ETL\Column\Column; +use Flow\ETL\Exception\EvaluationException; use Flow\ETL\Exception\InvalidArgumentException; use Flow\ETL\FlowContext; -use Flow\ETL\Row; +use Flow\ETL\Function\Evaluation\ResultColumn; +use Flow\ETL\Rows; use Flow\Types\Exception\InvalidTypeException; use Flow\Types\Type; use Flow\Types\Type\Native\FloatType; @@ -17,6 +20,8 @@ use function Flow\Types\DSL\type_bare; use function Flow\Types\DSL\type_float; use function Flow\Types\DSL\type_integer; +use function is_float; +use function is_int; final class Plus implements ScalarFunction { @@ -25,8 +30,11 @@ final class Plus implements ScalarFunction private readonly ScalarFunction $left; private readonly ScalarFunction $right; - public function __construct(ScalarFunction|int|float $left, ScalarFunction|int|float $right) - { + public function __construct( + ScalarFunction|int|float $left, + ScalarFunction|int|float $right, + private readonly bool $exact = false, + ) { $this->left = $left instanceof ScalarFunction ? $left : lit($left); $this->right = $right instanceof ScalarFunction ? $right : lit($right); } @@ -45,7 +53,7 @@ public function children(): array public function withChildren(array $children): static { /** @var list $children */ - return new self($children[0], $children[1]); + return new self($children[0], $children[1], $this->exact); } /** @@ -66,15 +74,39 @@ public function returns(): Type return $left instanceof IntegerType && $right instanceof IntegerType ? type_integer() : type_float(); } - public function eval(Row $row, FlowContext $context): int|float|null + public function eval(Rows $rows, FlowContext $context): Column { - $leftValue = (new Parameter($this->left))->asNumber($row, $context); - $rightValue = (new Parameter($this->right))->asNumber($row, $context); + $lefts = (new Parameter($this->left))->asNumbers($rows, $context); + $rights = (new Parameter($this->right))->asNumbers($rows, $context); + $results = []; + $i = 0; + + try { + foreach ($lefts as $i => $left) { + $right = $rights[$i]; + + if ($left === null || $right === null) { + throw new InvalidArgumentException('Plus function requires non-null values'); + } + + if ($this->exact) { + $results[] = $context->calculator()->add($left, $right); + + continue; + } + + $result = $left + $right; + + if (is_int($left) && is_int($right) && is_float($result)) { + throw new InvalidArgumentException('Plus function integer overflow'); + } - if ($leftValue === null || $rightValue === null) { - throw new InvalidArgumentException('Plus function requires non-null values'); + $results[] = $result; + } + } catch (Exception $e) { + throw EvaluationException::at($i, $e); } - return (new Calculator())->add($leftValue, $rightValue); + return (new ResultColumn($context->backend()))->of($this, $results); } } diff --git a/src/core/etl/src/Flow/ETL/Function/Power.php b/src/core/etl/src/Flow/ETL/Function/Power.php index 37dc3eb450..f7450dbd85 100644 --- a/src/core/etl/src/Flow/ETL/Function/Power.php +++ b/src/core/etl/src/Flow/ETL/Function/Power.php @@ -4,10 +4,13 @@ namespace Flow\ETL\Function; -use Flow\Calculator\Calculator; +use Exception; +use Flow\ETL\Column\Column; +use Flow\ETL\Exception\EvaluationException; use Flow\ETL\Exception\InvalidArgumentException; use Flow\ETL\FlowContext; -use Flow\ETL\Row; +use Flow\ETL\Function\Evaluation\ResultColumn; +use Flow\ETL\Rows; use Flow\Types\Exception\InvalidTypeException; use Flow\Types\Type; use Flow\Types\Type\Native\FloatType; @@ -17,6 +20,8 @@ use function Flow\Types\DSL\type_bare; use function Flow\Types\DSL\type_float; use function Flow\Types\DSL\type_integer; +use function is_float; +use function is_int; final class Power implements ScalarFunction { @@ -25,8 +30,11 @@ final class Power implements ScalarFunction private readonly ScalarFunction $left; private readonly ScalarFunction $right; - public function __construct(ScalarFunction|int|float $left, ScalarFunction|int $right) - { + public function __construct( + ScalarFunction|int|float $left, + ScalarFunction|int $right, + private readonly bool $exact = false, + ) { $this->left = $left instanceof ScalarFunction ? $left : lit($left); $this->right = $right instanceof ScalarFunction ? $right : lit($right); } @@ -45,7 +53,7 @@ public function children(): array public function withChildren(array $children): static { /** @var list $children */ - return new self($children[0], $children[1]); + return new self($children[0], $children[1], $this->exact); } /** @@ -66,15 +74,43 @@ public function returns(): Type return $left instanceof IntegerType && $right instanceof IntegerType ? type_integer() : type_float(); } - public function eval(Row $row, FlowContext $context): float|int|null + public function eval(Rows $rows, FlowContext $context): Column { - $leftValue = (new Parameter($this->left))->asNumber($row, $context); - $rightValue = (new Parameter($this->right))->asInt($row, $context); + $lefts = (new Parameter($this->left))->asNumbers($rows, $context); + $rights = (new Parameter($this->right))->asInts($rows, $context); + $results = []; + $i = 0; + + try { + foreach ($lefts as $i => $left) { + $right = $rights[$i]; + + if ($left === null || $right === null) { + throw new InvalidArgumentException('Power function requires non-null values'); + } + + if ($this->exact) { + $results[] = $context->calculator()->power($left, $right); + + continue; + } + + if (is_int($left) && $right < 0) { + throw new InvalidArgumentException('Power function of an integer requires a non-negative exponent'); + } + + $result = $left ** $right; + + if (is_int($left) && is_float($result)) { + throw new InvalidArgumentException('Power function integer overflow'); + } - if ($leftValue === null || $rightValue === null) { - throw new InvalidArgumentException('Power function requires non-null values'); + $results[] = $result; + } + } catch (Exception $e) { + throw EvaluationException::at($i, $e); } - return (new Calculator())->power($leftValue, $rightValue); + return (new ResultColumn($context->backend()))->of($this, $results); } } diff --git a/src/core/etl/src/Flow/ETL/Function/Prepend.php b/src/core/etl/src/Flow/ETL/Function/Prepend.php index 70e73e72b9..bb456a7a0d 100644 --- a/src/core/etl/src/Flow/ETL/Function/Prepend.php +++ b/src/core/etl/src/Flow/ETL/Function/Prepend.php @@ -4,9 +4,13 @@ namespace Flow\ETL\Function; +use Exception; +use Flow\ETL\Column\Column; +use Flow\ETL\Exception\EvaluationException; use Flow\ETL\Exception\InvalidArgumentException; use Flow\ETL\FlowContext; -use Flow\ETL\Row; +use Flow\ETL\Function\Evaluation\ResultColumn; +use Flow\ETL\Rows; use Flow\Types\Type; use function Flow\ETL\DSL\lit; @@ -51,19 +55,33 @@ public function returns(): Type return type_string(); } - public function eval(Row $row, FlowContext $context): ?string + public function eval(Rows $rows, FlowContext $context): Column { - $value = (new Parameter($this->value))->asString($row, $context); - $prefix = (new Parameter($this->prefix))->asString($row, $context); + $values = (new Parameter($this->value))->asStrings($rows, $context); + $prefixes = (new Parameter($this->prefix))->asStrings($rows, $context); + $results = []; + $i = 0; - if ($value === null) { - throw new InvalidArgumentException('Prepend function requires non-null value'); - } + try { + foreach ($values as $i => $value) { + $prefix = $prefixes[$i]; + + if ($value === null) { + throw new InvalidArgumentException('Prepend function requires non-null value'); + } + + if ($prefix === null) { + $results[] = $value; + + continue; + } - if ($prefix === null) { - return $value; + $results[] = s($value)->prepend($prefix)->toString(); + } + } catch (Exception $e) { + throw EvaluationException::at($i, $e); } - return s($value)->prepend($prefix)->toString(); + return (new ResultColumn($context->backend()))->of($this, $results); } } diff --git a/src/core/etl/src/Flow/ETL/Function/RandomString.php b/src/core/etl/src/Flow/ETL/Function/RandomString.php index 5bc0953d62..00e767d344 100644 --- a/src/core/etl/src/Flow/ETL/Function/RandomString.php +++ b/src/core/etl/src/Flow/ETL/Function/RandomString.php @@ -4,11 +4,15 @@ namespace Flow\ETL\Function; +use Exception; +use Flow\ETL\Column\Column; +use Flow\ETL\Exception\EvaluationException; use Flow\ETL\Exception\InvalidArgumentException; use Flow\ETL\FlowContext; +use Flow\ETL\Function\Evaluation\ResultColumn; use Flow\ETL\NativePHPRandomValueGenerator; use Flow\ETL\RandomValueGenerator; -use Flow\ETL\Row; +use Flow\ETL\Rows; use Flow\Types\Type; use function Flow\ETL\DSL\lit; @@ -57,14 +61,24 @@ public function returns(): Type return type_string(); } - public function eval(Row $row, FlowContext $context): ?string + public function eval(Rows $rows, FlowContext $context): Column { - $length = (new Parameter($this->length))->asInt($row, $context); + $lengths = (new Parameter($this->length))->asInts($rows, $context); + $results = []; + $i = 0; - if ($length === null) { - throw new InvalidArgumentException('RandomString requires non-null length'); + try { + foreach ($lengths as $i => $length) { + if ($length === null) { + throw new InvalidArgumentException('RandomString requires non-null length'); + } + + $results[] = $this->generator->string($length); + } + } catch (Exception $e) { + throw EvaluationException::at($i, $e); } - return $this->generator->string($length); + return (new ResultColumn($context->backend()))->of($this, $results); } } diff --git a/src/core/etl/src/Flow/ETL/Function/Rank.php b/src/core/etl/src/Flow/ETL/Function/Rank.php index 9a38afd7e3..e771241b24 100644 --- a/src/core/etl/src/Flow/ETL/Function/Rank.php +++ b/src/core/etl/src/Flow/ETL/Function/Rank.php @@ -55,20 +55,15 @@ public function rankPartition(Rows $partition): array throw new BaseRuntimeException('Rank window function requires to be ordered by one column'); } - $comparator = new PeerComparator($orderBy); $ranks = []; $rank = 1; - $index = 0; - $previous = null; - foreach ($partition as $row) { - if ($previous !== null && !$comparator->arePeers($previous, $row, $partition->schema())) { + foreach ((new PeerComparator($orderBy))->peersOfPrevious($partition) as $index => $peer) { + if ($index > 0 && !$peer) { $rank = $index + 1; } $ranks[] = $rank; - $previous = $row; - $index++; } return $ranks; diff --git a/src/core/etl/src/Flow/ETL/Function/Regex.php b/src/core/etl/src/Flow/ETL/Function/Regex.php index f606bf7cad..2f0d862d6d 100644 --- a/src/core/etl/src/Flow/ETL/Function/Regex.php +++ b/src/core/etl/src/Flow/ETL/Function/Regex.php @@ -4,9 +4,13 @@ namespace Flow\ETL\Function; +use Exception; +use Flow\ETL\Column\Column; +use Flow\ETL\Exception\EvaluationException; use Flow\ETL\Exception\InvalidArgumentException; use Flow\ETL\FlowContext; -use Flow\ETL\Row; +use Flow\ETL\Function\Evaluation\ResultColumn; +use Flow\ETL\Rows; use Flow\Types\Type; use function Flow\ETL\DSL\lit; @@ -69,34 +73,46 @@ public function returns(): Type return type_optional(type_array()); } - /** - * @return null|array - */ - public function eval(Row $row, FlowContext $context): ?array + public function eval(Rows $rows, FlowContext $context): Column { - $pattern = (new Parameter($this->pattern))->asString($row, $context); - $subject = (new Parameter($this->subject))->asString($row, $context); - $offset = (new Parameter($this->offset))->asInt($row, $context); - - if ($pattern === null) { - throw new InvalidArgumentException('Regex requires non-null pattern'); - } - - if ($subject === null) { - throw new InvalidArgumentException('Regex requires non-null subject'); - } - - if ($offset === null) { - throw new InvalidArgumentException('Regex requires non-null offset'); - } - - $matches = []; - - // preg_match() returns 1 if the pattern matches given subject, 0 if it does not, or false on failure. - if (preg_match($pattern, $subject, $matches, $this->flags, $offset) === 1) { - return $matches; + $patterns = (new Parameter($this->pattern))->asStrings($rows, $context); + $subjects = (new Parameter($this->subject))->asStrings($rows, $context); + $offsets = (new Parameter($this->offset))->asInts($rows, $context); + $results = []; + $i = 0; + + try { + foreach ($patterns as $i => $pattern) { + $subject = $subjects[$i]; + $offset = $offsets[$i]; + + if ($pattern === null) { + throw new InvalidArgumentException('Regex requires non-null pattern'); + } + + if ($subject === null) { + throw new InvalidArgumentException('Regex requires non-null subject'); + } + + if ($offset === null) { + throw new InvalidArgumentException('Regex requires non-null offset'); + } + + $matches = []; + + // preg_match() returns 1 if the pattern matches given subject, 0 if it does not, or false on failure. + if (preg_match($pattern, $subject, $matches, $this->flags, $offset) === 1) { + $results[] = $matches; + + continue; + } + + $results[] = null; + } + } catch (Exception $e) { + throw EvaluationException::at($i, $e); } - return null; + return (new ResultColumn($context->backend()))->of($this, $results); } } diff --git a/src/core/etl/src/Flow/ETL/Function/RegexAll.php b/src/core/etl/src/Flow/ETL/Function/RegexAll.php index 7a217f1d1f..db6bcf6b14 100644 --- a/src/core/etl/src/Flow/ETL/Function/RegexAll.php +++ b/src/core/etl/src/Flow/ETL/Function/RegexAll.php @@ -4,9 +4,13 @@ namespace Flow\ETL\Function; +use Exception; +use Flow\ETL\Column\Column; +use Flow\ETL\Exception\EvaluationException; use Flow\ETL\Exception\InvalidArgumentException; use Flow\ETL\FlowContext; -use Flow\ETL\Row; +use Flow\ETL\Function\Evaluation\ResultColumn; +use Flow\ETL\Rows; use Flow\Types\Type; use function Flow\ETL\DSL\lit; @@ -73,38 +77,52 @@ public function returns(): Type return type_optional(type_array()); } - /** - * @return null|array - */ - public function eval(Row $row, FlowContext $context): ?array + public function eval(Rows $rows, FlowContext $context): Column { - $pattern = (new Parameter($this->pattern))->asString($row, $context); - $subject = (new Parameter($this->subject))->asString($row, $context); - $offset = (new Parameter($this->offset))->asInt($row, $context); + $patterns = (new Parameter($this->pattern))->asStrings($rows, $context); + $subjects = (new Parameter($this->subject))->asStrings($rows, $context); + $offsets = (new Parameter($this->offset))->asInts($rows, $context); + $results = []; + $i = 0; - if ($pattern === null) { - throw new InvalidArgumentException('RegexAll requires non-null pattern'); - } + try { + foreach ($patterns as $i => $pattern) { + $subject = $subjects[$i]; + $offset = $offsets[$i]; - if ($subject === null) { - throw new InvalidArgumentException('RegexAll requires non-null subject'); - } + if ($pattern === null) { + throw new InvalidArgumentException('RegexAll requires non-null pattern'); + } - if ($offset === null) { - throw new InvalidArgumentException('RegexAll requires non-null offset'); - } + if ($subject === null) { + throw new InvalidArgumentException('RegexAll requires non-null subject'); + } - $matches = []; + if ($offset === null) { + throw new InvalidArgumentException('RegexAll requires non-null offset'); + } - // Returns the number of full pattern matches (which might be zero), or false on failure. - if (preg_match_all($pattern, $subject, $matches, $this->flags, $offset) !== false) { - if ($matches === [[]]) { - return null; - } + $matches = []; - return $matches; + // Returns the number of full pattern matches (which might be zero), or false on failure. + if (preg_match_all($pattern, $subject, $matches, $this->flags, $offset) !== false) { + if ($matches === [[]]) { + $results[] = null; + + continue; + } + + $results[] = $matches; + + continue; + } + + $results[] = null; + } + } catch (Exception $e) { + throw EvaluationException::at($i, $e); } - return null; + return (new ResultColumn($context->backend()))->of($this, $results); } } diff --git a/src/core/etl/src/Flow/ETL/Function/RegexMatch.php b/src/core/etl/src/Flow/ETL/Function/RegexMatch.php index 1e2c6d20ff..f59d0e3b4b 100644 --- a/src/core/etl/src/Flow/ETL/Function/RegexMatch.php +++ b/src/core/etl/src/Flow/ETL/Function/RegexMatch.php @@ -4,8 +4,12 @@ namespace Flow\ETL\Function; +use Exception; +use Flow\ETL\Column\Column; +use Flow\ETL\Exception\EvaluationException; use Flow\ETL\FlowContext; -use Flow\ETL\Row; +use Flow\ETL\Function\Evaluation\ResultColumn; +use Flow\ETL\Rows; use Flow\Types\Type; use Flow\Types\Type\Nullability; @@ -68,17 +72,33 @@ public function returns(): Type ); } - public function eval(Row $row, FlowContext $context): ?bool + public function eval(Rows $rows, FlowContext $context): Column { - $pattern = (new Parameter($this->pattern))->asString($row, $context); - $subject = (new Parameter($this->subject))->asString($row, $context); - $flags = (new Parameter($this->flags))->asInt($row, $context); - $offset = (new Parameter($this->offset))->asInt($row, $context); + $patterns = (new Parameter($this->pattern))->asStrings($rows, $context); + $subjects = (new Parameter($this->subject))->asStrings($rows, $context); + $flagsList = (new Parameter($this->flags))->asInts($rows, $context); + $offsets = (new Parameter($this->offset))->asInts($rows, $context); + $results = []; + $i = 0; - if ($pattern === null || $subject === null || $flags === null || $offset === null) { - return null; + try { + foreach ($patterns as $i => $pattern) { + $subject = $subjects[$i]; + $flags = $flagsList[$i]; + $offset = $offsets[$i]; + + if ($pattern === null || $subject === null || $flags === null || $offset === null) { + $results[] = null; + + continue; + } + + $results[] = preg_match(pattern: $pattern, subject: $subject, flags: $flags, offset: $offset) === 1; + } + } catch (Exception $e) { + throw EvaluationException::at($i, $e); } - return preg_match(pattern: $pattern, subject: $subject, flags: $flags, offset: $offset) === 1; + return (new ResultColumn($context->backend()))->of($this, $results); } } diff --git a/src/core/etl/src/Flow/ETL/Function/RegexMatchAll.php b/src/core/etl/src/Flow/ETL/Function/RegexMatchAll.php index 56fee74850..fd085b68f4 100644 --- a/src/core/etl/src/Flow/ETL/Function/RegexMatchAll.php +++ b/src/core/etl/src/Flow/ETL/Function/RegexMatchAll.php @@ -4,8 +4,12 @@ namespace Flow\ETL\Function; +use Exception; +use Flow\ETL\Column\Column; +use Flow\ETL\Exception\EvaluationException; use Flow\ETL\FlowContext; -use Flow\ETL\Row; +use Flow\ETL\Function\Evaluation\ResultColumn; +use Flow\ETL\Rows; use Flow\Types\Type; use Flow\Types\Type\Nullability; @@ -68,17 +72,34 @@ public function returns(): Type ); } - public function eval(Row $row, FlowContext $context): ?bool + public function eval(Rows $rows, FlowContext $context): Column { - $pattern = (new Parameter($this->pattern))->asString($row, $context); - $subject = (new Parameter($this->subject))->asString($row, $context); - $flags = (new Parameter($this->flags))->asInt($row, $context); - $offset = (new Parameter($this->offset))->asInt($row, $context); + $patterns = (new Parameter($this->pattern))->asStrings($rows, $context); + $subjects = (new Parameter($this->subject))->asStrings($rows, $context); + $flagsList = (new Parameter($this->flags))->asInts($rows, $context); + $offsets = (new Parameter($this->offset))->asInts($rows, $context); + $results = []; + $i = 0; - if ($pattern === null || $subject === null || $flags === null || $offset === null) { - return null; + try { + foreach ($patterns as $i => $pattern) { + $subject = $subjects[$i]; + $flags = $flagsList[$i]; + $offset = $offsets[$i]; + + if ($pattern === null || $subject === null || $flags === null || $offset === null) { + $results[] = null; + + continue; + } + + $results[] = + preg_match_all(pattern: $pattern, subject: $subject, flags: $flags, offset: $offset) !== false; + } + } catch (Exception $e) { + throw EvaluationException::at($i, $e); } - return preg_match_all(pattern: $pattern, subject: $subject, flags: $flags, offset: $offset) !== false; + return (new ResultColumn($context->backend()))->of($this, $results); } } diff --git a/src/core/etl/src/Flow/ETL/Function/RegexReplace.php b/src/core/etl/src/Flow/ETL/Function/RegexReplace.php index 299ab0200c..0789f19ffd 100644 --- a/src/core/etl/src/Flow/ETL/Function/RegexReplace.php +++ b/src/core/etl/src/Flow/ETL/Function/RegexReplace.php @@ -4,9 +4,13 @@ namespace Flow\ETL\Function; +use Exception; +use Flow\ETL\Column\Column; +use Flow\ETL\Exception\EvaluationException; use Flow\ETL\Exception\InvalidArgumentException; use Flow\ETL\FlowContext; -use Flow\ETL\Row; +use Flow\ETL\Function\Evaluation\ResultColumn; +use Flow\ETL\Rows; use Flow\Types\Type; use function count; @@ -72,29 +76,45 @@ public function returns(): Type return type_string(); } - public function eval(Row $row, FlowContext $context): ?string + public function eval(Rows $rows, FlowContext $context): Column { - $pattern = (new Parameter($this->pattern))->asString($row, $context); - $replacement = (new Parameter($this->replacement))->asString($row, $context); - $subject = (new Parameter($this->subject))->asString($row, $context); - $limit = $this->limit !== null ? (new Parameter($this->limit))->asInt($row, $context) : -1; - - if ($pattern === null) { - throw new InvalidArgumentException('RegexReplace requires non-null pattern'); - } - - if ($replacement === null) { - throw new InvalidArgumentException('RegexReplace requires non-null replacement'); - } - - if ($subject === null) { - throw new InvalidArgumentException('RegexReplace requires non-null subject'); - } - - if ($limit === null) { - throw new InvalidArgumentException('RegexReplace requires non-null limit'); + $patterns = (new Parameter($this->pattern))->asStrings($rows, $context); + $replacements = (new Parameter($this->replacement))->asStrings($rows, $context); + $subjects = (new Parameter($this->subject))->asStrings($rows, $context); + $limits = (new Parameter($this->limit))->asInts($rows, $context); + $results = []; + $i = 0; + + try { + foreach ($patterns as $i => $pattern) { + $replacement = $replacements[$i]; + $subject = $subjects[$i]; + $limit = $limits[$i]; + + $limit = $this->limit !== null ? $limit : -1; + + if ($pattern === null) { + throw new InvalidArgumentException('RegexReplace requires non-null pattern'); + } + + if ($replacement === null) { + throw new InvalidArgumentException('RegexReplace requires non-null replacement'); + } + + if ($subject === null) { + throw new InvalidArgumentException('RegexReplace requires non-null subject'); + } + + if ($limit === null) { + throw new InvalidArgumentException('RegexReplace requires non-null limit'); + } + + $results[] = preg_replace($pattern, $replacement, $subject, $limit); + } + } catch (Exception $e) { + throw EvaluationException::at($i, $e); } - return preg_replace($pattern, $replacement, $subject, $limit); + return (new ResultColumn($context->backend()))->of($this, $results); } } diff --git a/src/core/etl/src/Flow/ETL/Function/Repeat.php b/src/core/etl/src/Flow/ETL/Function/Repeat.php index 5a551d57a3..068b9dc9b6 100644 --- a/src/core/etl/src/Flow/ETL/Function/Repeat.php +++ b/src/core/etl/src/Flow/ETL/Function/Repeat.php @@ -4,9 +4,13 @@ namespace Flow\ETL\Function; +use Exception; +use Flow\ETL\Column\Column; +use Flow\ETL\Exception\EvaluationException; use Flow\ETL\Exception\InvalidArgumentException; use Flow\ETL\FlowContext; -use Flow\ETL\Row; +use Flow\ETL\Function\Evaluation\ResultColumn; +use Flow\ETL\Rows; use Flow\Types\Type; use function Flow\ETL\DSL\lit; @@ -51,19 +55,31 @@ public function returns(): Type return type_string(); } - public function eval(Row $row, FlowContext $context): string + public function eval(Rows $rows, FlowContext $context): Column { - $value = (new Parameter($this->value))->asString($row, $context); - $times = (new Parameter($this->times))->asInt($row, $context); + $values = (new Parameter($this->value))->asStrings($rows, $context); + $timesList = (new Parameter($this->times))->asInts($rows, $context); + $results = []; + $i = 0; - if ($value === null) { - throw new InvalidArgumentException('Repeat function requires non-null value'); - } + try { + foreach ($values as $i => $value) { + $times = $timesList[$i]; + + if ($value === null) { + throw new InvalidArgumentException('Repeat function requires non-null value'); + } + + if ($times === null || $times <= 0) { + throw new InvalidArgumentException('Repeat function requires non-null, positive times'); + } - if ($times === null || $times <= 0) { - throw new InvalidArgumentException('Repeat function requires non-null, positive times'); + $results[] = s($value)->repeat($times)->toString(); + } + } catch (Exception $e) { + throw EvaluationException::at($i, $e); } - return s($value)->repeat($times)->toString(); + return (new ResultColumn($context->backend()))->of($this, $results); } } diff --git a/src/core/etl/src/Flow/ETL/Function/Reverse.php b/src/core/etl/src/Flow/ETL/Function/Reverse.php index 8f51678087..d7aa4df4e6 100644 --- a/src/core/etl/src/Flow/ETL/Function/Reverse.php +++ b/src/core/etl/src/Flow/ETL/Function/Reverse.php @@ -4,9 +4,13 @@ namespace Flow\ETL\Function; +use Exception; +use Flow\ETL\Column\Column; +use Flow\ETL\Exception\EvaluationException; use Flow\ETL\Exception\InvalidArgumentException; use Flow\ETL\FlowContext; -use Flow\ETL\Row; +use Flow\ETL\Function\Evaluation\ResultColumn; +use Flow\ETL\Rows; use Flow\Types\Type; use function Flow\ETL\DSL\lit; @@ -49,14 +53,24 @@ public function returns(): Type return type_string(); } - public function eval(Row $row, FlowContext $context): ?string + public function eval(Rows $rows, FlowContext $context): Column { - $value = (new Parameter($this->value))->asString($row, $context); + $values = (new Parameter($this->value))->asStrings($rows, $context); + $results = []; + $i = 0; - if ($value === null) { - throw new InvalidArgumentException('Reverse function requires non-null value'); + try { + foreach ($values as $i => $value) { + if ($value === null) { + throw new InvalidArgumentException('Reverse function requires non-null value'); + } + + $results[] = s($value)->reverse()->toString(); + } + } catch (Exception $e) { + throw EvaluationException::at($i, $e); } - return s($value)->reverse()->toString(); + return (new ResultColumn($context->backend()))->of($this, $results); } } diff --git a/src/core/etl/src/Flow/ETL/Function/Round.php b/src/core/etl/src/Flow/ETL/Function/Round.php index 9c3ec2dddb..988d6a77a5 100644 --- a/src/core/etl/src/Flow/ETL/Function/Round.php +++ b/src/core/etl/src/Flow/ETL/Function/Round.php @@ -4,9 +4,13 @@ namespace Flow\ETL\Function; +use Exception; +use Flow\ETL\Column\Column; +use Flow\ETL\Exception\EvaluationException; use Flow\ETL\Exception\InvalidArgumentException; use Flow\ETL\FlowContext; -use Flow\ETL\Row; +use Flow\ETL\Function\Evaluation\ResultColumn; +use Flow\ETL\Rows; use Flow\Types\Type; use function Flow\ETL\DSL\lit; @@ -56,20 +60,33 @@ public function returns(): Type return type_float(); } - public function eval(Row $row, FlowContext $context): float + public function eval(Rows $rows, FlowContext $context): Column { - $value = (new Parameter($this->value))->asNumber($row, $context); - $precision = (new Parameter($this->precision))->asInt($row, $context); - $mode = (new Parameter($this->mode))->asInt($row, $context); + $values = (new Parameter($this->value))->asNumbers($rows, $context); + $precisions = (new Parameter($this->precision))->asInts($rows, $context); + $modes = (new Parameter($this->mode))->asInts($rows, $context); + $results = []; + $i = 0; - if ($value === null || $precision === null || $mode === null) { - throw new InvalidArgumentException('Round function requires non-null values'); - } + try { + foreach ($values as $i => $value) { + $precision = $precisions[$i]; + $mode = $modes[$i]; + + if ($value === null || $precision === null || $mode === null) { + throw new InvalidArgumentException('Round function requires non-null values'); + } + + if ($mode < 1 || $mode > 4) { + $mode = 1; + } - if ($mode < 1 || $mode > 4) { - $mode = 1; + $results[] = round($value, $precision, $mode); + } + } catch (Exception $e) { + throw EvaluationException::at($i, $e); } - return round($value, $precision, $mode); + return (new ResultColumn($context->backend()))->of($this, $results); } } diff --git a/src/core/etl/src/Flow/ETL/Function/Same.php b/src/core/etl/src/Flow/ETL/Function/Same.php index 618bdfbd94..2f0887105b 100644 --- a/src/core/etl/src/Flow/ETL/Function/Same.php +++ b/src/core/etl/src/Flow/ETL/Function/Same.php @@ -4,12 +4,18 @@ namespace Flow\ETL\Function; +use Exception; +use Flow\ETL\Column\Column; +use Flow\ETL\Column\ComparableValues; +use Flow\ETL\Exception\EvaluationException; use Flow\ETL\FlowContext; -use Flow\ETL\Row; +use Flow\ETL\Function\Evaluation\ResultColumn; +use Flow\ETL\Rows; use Flow\Types\Type; use Flow\Types\Type\ValueComparator; use function Flow\ETL\DSL\lit; +use function Flow\Types\DSL\type_bare; use function Flow\Types\DSL\type_boolean; final class Same implements ScalarFunction @@ -52,11 +58,30 @@ public function returns(): Type return type_boolean(); } - public function eval(Row $row, FlowContext $context): bool + public function eval(Rows $rows, FlowContext $context): Column { - // PHP identity, not SQL equality: null === null is a defined, useful answer - return ( - (new Parameter($this->left))->eval($row, $context) === (new Parameter($this->right))->eval($row, $context) - ); + $leftColumn = (new Parameter($this->left))->column($rows, $context); + $rightColumn = (new Parameter($this->right))->column($rows, $context); + $comparable = new ComparableValues(); + [$lefts, $rights] = type_bare($leftColumn->type())::class === type_bare($rightColumn->type())::class + ? [$comparable->equality($leftColumn), $comparable->equality($rightColumn)] + : [$leftColumn->values(), $rightColumn->values()]; + $results = []; + $i = 0; + + try { + // @mago-ignore analysis:mixed-assignment + foreach ($lefts as $i => $left) { + // @mago-ignore analysis:mixed-assignment + $right = $rights[$i]; + + // PHP identity, not SQL equality: null === null is a defined, useful answer + $results[] = $left === $right; + } + } catch (Exception $e) { + throw EvaluationException::at($i, $e); + } + + return (new ResultColumn($context->backend()))->of($this, $results); } } diff --git a/src/core/etl/src/Flow/ETL/Function/Sanitize.php b/src/core/etl/src/Flow/ETL/Function/Sanitize.php index bb1aea0d2e..4164f02493 100644 --- a/src/core/etl/src/Flow/ETL/Function/Sanitize.php +++ b/src/core/etl/src/Flow/ETL/Function/Sanitize.php @@ -4,9 +4,13 @@ namespace Flow\ETL\Function; +use Exception; +use Flow\ETL\Column\Column; +use Flow\ETL\Exception\EvaluationException; use Flow\ETL\Exception\InvalidArgumentException; use Flow\ETL\FlowContext; -use Flow\ETL\Row; +use Flow\ETL\Function\Evaluation\ResultColumn; +use Flow\ETL\Rows; use Flow\Types\Type; use function Flow\ETL\DSL\lit; @@ -58,26 +62,42 @@ public function returns(): Type return type_string(); } - public function eval(Row $row, FlowContext $context): ?string + public function eval(Rows $rows, FlowContext $context): Column { - $val = (new Parameter($this->value))->asString($row, $context); - $placeholder = (new Parameter($this->placeholder))->asString($row, $context); - $skipCharacters = (new Parameter($this->skipCharacters))->asInt($row, $context); - - if ($val === null) { - throw new InvalidArgumentException('Sanitize function requires non-null value'); - } - - if ($placeholder === null) { - throw new InvalidArgumentException('Sanitize function requires non-null placeholder'); - } - - $size = mb_strlen($val); - - if ($skipCharacters !== null && $size > $skipCharacters) { - return mb_substr($val, 0, $skipCharacters) . str_repeat($placeholder, max(0, $size - $skipCharacters)); + $vals = (new Parameter($this->value))->asStrings($rows, $context); + $placeholders = (new Parameter($this->placeholder))->asStrings($rows, $context); + $skipCharactersList = (new Parameter($this->skipCharacters))->asInts($rows, $context); + $results = []; + $i = 0; + + try { + foreach ($vals as $i => $val) { + $placeholder = $placeholders[$i]; + $skipCharacters = $skipCharactersList[$i]; + + if ($val === null) { + throw new InvalidArgumentException('Sanitize function requires non-null value'); + } + + if ($placeholder === null) { + throw new InvalidArgumentException('Sanitize function requires non-null placeholder'); + } + + $size = mb_strlen($val); + + if ($skipCharacters !== null && $size > $skipCharacters) { + $results[] = + mb_substr($val, 0, $skipCharacters) . str_repeat($placeholder, max(0, $size - $skipCharacters)); + + continue; + } + + $results[] = str_repeat($placeholder, $size); + } + } catch (Exception $e) { + throw EvaluationException::at($i, $e); } - return str_repeat($placeholder, $size); + return (new ResultColumn($context->backend()))->of($this, $results); } } diff --git a/src/core/etl/src/Flow/ETL/Function/ScalarFunction.php b/src/core/etl/src/Flow/ETL/Function/ScalarFunction.php index a0f1016369..b5be81bcb7 100644 --- a/src/core/etl/src/Flow/ETL/Function/ScalarFunction.php +++ b/src/core/etl/src/Flow/ETL/Function/ScalarFunction.php @@ -4,13 +4,17 @@ namespace Flow\ETL\Function; +use Flow\ETL\Column\Column; use Flow\ETL\FlowContext; -use Flow\ETL\Row; +use Flow\ETL\Rows; use Flow\Types\Type; interface ScalarFunction extends FunctionTree { - public function eval(Row $row, FlowContext $context): mixed; + /** + * One column of $rows->count() values, of returns()'s type (nullable), same row order. + */ + public function eval(Rows $rows, FlowContext $context): Column; /** * The type of the column this function produces, before any row is read. diff --git a/src/core/etl/src/Flow/ETL/Function/ScalarFunction/ExpandResults.php b/src/core/etl/src/Flow/ETL/Function/ScalarFunction/ExpandResults.php index ad08446328..e1496438f8 100644 --- a/src/core/etl/src/Flow/ETL/Function/ScalarFunction/ExpandResults.php +++ b/src/core/etl/src/Flow/ETL/Function/ScalarFunction/ExpandResults.php @@ -4,14 +4,9 @@ namespace Flow\ETL\Function\ScalarFunction; -use Flow\ETL\FlowContext; use Flow\ETL\Function\ScalarFunction; -use Flow\ETL\Row; -interface ExpandResults extends ScalarFunction -{ - /** - * @return array - */ - public function eval(Row $row, FlowContext $context): array; -} +/** + * eval returns a `list` column; the step explodes it. + */ +interface ExpandResults extends ScalarFunction {} diff --git a/src/core/etl/src/Flow/ETL/Function/ScalarFunction/UnpackResults.php b/src/core/etl/src/Flow/ETL/Function/ScalarFunction/UnpackResults.php index 87544ddbdf..0c2b81f658 100644 --- a/src/core/etl/src/Flow/ETL/Function/ScalarFunction/UnpackResults.php +++ b/src/core/etl/src/Flow/ETL/Function/ScalarFunction/UnpackResults.php @@ -4,14 +4,9 @@ namespace Flow\ETL\Function\ScalarFunction; -use Flow\ETL\FlowContext; use Flow\ETL\Function\ScalarFunction; -use Flow\ETL\Row; -interface UnpackResults extends ScalarFunction -{ - /** - * @return array - */ - public function eval(Row $row, FlowContext $context): array; -} +/** + * eval returns the `StructureType` column `returns()` declares. + */ +interface UnpackResults extends ScalarFunction {} diff --git a/src/core/etl/src/Flow/ETL/Function/ScalarFunctionChain.php b/src/core/etl/src/Flow/ETL/Function/ScalarFunctionChain.php index 2978207475..03c0d25f2e 100644 --- a/src/core/etl/src/Flow/ETL/Function/ScalarFunctionChain.php +++ b/src/core/etl/src/Flow/ETL/Function/ScalarFunctionChain.php @@ -497,9 +497,9 @@ public function lower(): ToLower return new ToLower($this); } - public function minus(ScalarFunction|int|float $ref): Minus + public function minus(ScalarFunction|int|float $ref, bool $exact = false): Minus { - return new Minus($this, $ref); + return new Minus($this, $ref, $exact); } public function mod(ScalarFunction|int $value): Mod @@ -512,9 +512,9 @@ public function modifyDateTime(string|ScalarFunction $modifier): ModifyDateTime return new ModifyDateTime($this, $modifier); } - public function multiply(ScalarFunction|int|float $value): Multiply + public function multiply(ScalarFunction|int|float $value, bool $exact = false): Multiply { - return new Multiply($this, $value); + return new Multiply($this, $value, $exact); } public function notEquals(mixed $value): NotEquals @@ -556,14 +556,14 @@ public function orNot(ScalarFunction $function): Any return new Any($this, new Not($function)); } - public function plus(ScalarFunction|int|float $ref): Plus + public function plus(ScalarFunction|int|float $ref, bool $exact = false): Plus { - return new Plus($this, $ref); + return new Plus($this, $ref, $exact); } - public function power(ScalarFunction|int $value): Power + public function power(ScalarFunction|int $value, bool $exact = false): Power { - return new Power($this, $value); + return new Power($this, $value, $exact); } public function prepend(ScalarFunction|string $prefix): Prepend diff --git a/src/core/etl/src/Flow/ETL/Function/Size.php b/src/core/etl/src/Flow/ETL/Function/Size.php index a6d796b80e..a3e4799d91 100644 --- a/src/core/etl/src/Flow/ETL/Function/Size.php +++ b/src/core/etl/src/Flow/ETL/Function/Size.php @@ -4,8 +4,12 @@ namespace Flow\ETL\Function; +use Exception; +use Flow\ETL\Column\Column; +use Flow\ETL\Exception\EvaluationException; use Flow\ETL\FlowContext; -use Flow\ETL\Row; +use Flow\ETL\Function\Evaluation\ResultColumn; +use Flow\ETL\Rows; use Flow\Types\Type; use function count; @@ -52,18 +56,33 @@ public function returns(): Type return type_optional(type_integer()); } - public function eval(Row $row, FlowContext $context): ?int + public function eval(Rows $rows, FlowContext $context): Column { - $value = (new Parameter($this->value))->eval($row, $context); + $values = (new Parameter($this->value))->values($rows, $context); + $results = []; + $i = 0; - if (is_string($value)) { - return s($value)->length(); - } + try { + // @mago-ignore analysis:mixed-assignment + foreach ($values as $i => $value) { + if (is_string($value)) { + $results[] = s($value)->length(); + + continue; + } + + if (is_countable($value)) { + $results[] = count($value); + + continue; + } - if (is_countable($value)) { - return count($value); + $results[] = null; + } + } catch (Exception $e) { + throw EvaluationException::at($i, $e); } - return null; + return (new ResultColumn($context->backend()))->of($this, $results); } } diff --git a/src/core/etl/src/Flow/ETL/Function/Slug.php b/src/core/etl/src/Flow/ETL/Function/Slug.php index 88faaaf4ee..cffac03608 100644 --- a/src/core/etl/src/Flow/ETL/Function/Slug.php +++ b/src/core/etl/src/Flow/ETL/Function/Slug.php @@ -4,9 +4,13 @@ namespace Flow\ETL\Function; +use Exception; +use Flow\ETL\Column\Column; +use Flow\ETL\Exception\EvaluationException; use Flow\ETL\Exception\InvalidArgumentException; use Flow\ETL\FlowContext; -use Flow\ETL\Row; +use Flow\ETL\Function\Evaluation\ResultColumn; +use Flow\ETL\Rows; use Flow\Types\Type; use Symfony\Component\String\Slugger\AsciiSlugger; @@ -62,19 +66,33 @@ public function returns(): Type return type_string(); } - public function eval(Row $row, FlowContext $context): ?string + public function eval(Rows $rows, FlowContext $context): Column { - $string = (new Parameter($this->string))->asString($row, $context); - $separator = (new Parameter($this->separator))->asString($row, $context, '-'); - $locale = (new Parameter($this->locale))->asString($row, $context); - $symbolsMap = (new Parameter($this->symbolsMap))->asArray($row, $context); + $strings = (new Parameter($this->string))->asStrings($rows, $context); + $separators = (new Parameter($this->separator))->asStrings($rows, $context, '-'); + $locales = (new Parameter($this->locale))->asStrings($rows, $context); + $symbolsMaps = (new Parameter($this->symbolsMap))->asArrays($rows, $context); + $results = []; + $i = 0; - if ($string === null) { - throw new InvalidArgumentException('Slug function requires non-null value'); + try { + foreach ($strings as $i => $string) { + $separator = $separators[$i]; + $locale = $locales[$i]; + $symbolsMap = $symbolsMaps[$i]; + + if ($string === null) { + throw new InvalidArgumentException('Slug function requires non-null value'); + } + + $results[] = (new AsciiSlugger(symbolsMap: $symbolsMap)) + ->slug($string, $separator, $locale) + ->toString(); + } + } catch (Exception $e) { + throw EvaluationException::at($i, $e); } - return (new AsciiSlugger(symbolsMap: $symbolsMap)) - ->slug($string, $separator, $locale) - ->toString(); + return (new ResultColumn($context->backend()))->of($this, $results); } } diff --git a/src/core/etl/src/Flow/ETL/Function/Split.php b/src/core/etl/src/Flow/ETL/Function/Split.php index 1ad0e1aa8f..00a8993816 100644 --- a/src/core/etl/src/Flow/ETL/Function/Split.php +++ b/src/core/etl/src/Flow/ETL/Function/Split.php @@ -4,9 +4,13 @@ namespace Flow\ETL\Function; +use Exception; +use Flow\ETL\Column\Column; +use Flow\ETL\Exception\EvaluationException; use Flow\ETL\Exception\InvalidArgumentException; use Flow\ETL\FlowContext; -use Flow\ETL\Row; +use Flow\ETL\Function\Evaluation\ResultColumn; +use Flow\ETL\Rows; use Flow\Types\Type; use Symfony\Component\String\AbstractString; @@ -59,26 +63,38 @@ public function returns(): Type return type_list(type_string()); } - /** - * @return null|array - */ - public function eval(Row $row, FlowContext $context): ?array + public function eval(Rows $rows, FlowContext $context): Column { - $value = (new Parameter($this->value))->asString($row, $context); - $separator = (new Parameter($this->separator))->asString($row, $context); - $limit = (new Parameter($this->limit))->asInt($row, $context); + $values = (new Parameter($this->value))->asStrings($rows, $context); + $separators = (new Parameter($this->separator))->asStrings($rows, $context); + $limits = (new Parameter($this->limit))->asInts($rows, $context); + $results = []; + $i = 0; - if ($value === null) { - throw new InvalidArgumentException('Split function requires non-null value'); - } + try { + foreach ($values as $i => $value) { + $separator = $separators[$i]; + $limit = $limits[$i]; + + if ($value === null) { + throw new InvalidArgumentException('Split function requires non-null value'); + } + + if ($separator === null || $limit === null || $separator === '') { + throw new InvalidArgumentException( + 'Split function requires non-null separator and limit, separator cannot be empty', + ); + } - if ($separator === null || $limit === null || $separator === '') { - throw new InvalidArgumentException( - 'Split function requires non-null separator and limit, separator cannot be empty', - ); + $results[] = array_map( + static fn(AbstractString $s): string => $s->toString(), + s($value)->split($separator, $limit), + ); + } + } catch (Exception $e) { + throw EvaluationException::at($i, $e); } - // @mago-ignore analysis:less-specific-return-statement - return array_map(static fn(AbstractString $s): string => $s->toString(), s($value)->split($separator, $limit)); + return (new ResultColumn($context->backend()))->of($this, $results); } } diff --git a/src/core/etl/src/Flow/ETL/Function/Sprintf.php b/src/core/etl/src/Flow/ETL/Function/Sprintf.php index d404648804..aa4a4aeae3 100644 --- a/src/core/etl/src/Flow/ETL/Function/Sprintf.php +++ b/src/core/etl/src/Flow/ETL/Function/Sprintf.php @@ -4,9 +4,13 @@ namespace Flow\ETL\Function; +use Exception; +use Flow\ETL\Column\Column; +use Flow\ETL\Exception\EvaluationException; use Flow\ETL\Exception\InvalidArgumentException; use Flow\ETL\FlowContext; -use Flow\ETL\Row; +use Flow\ETL\Function\Evaluation\ResultColumn; +use Flow\ETL\Rows; use Flow\Types\Type; use function array_map; @@ -64,25 +68,36 @@ public function returns(): Type return type_string(); } - public function eval(Row $row, FlowContext $context): ?string + public function eval(Rows $rows, FlowContext $context): Column { - $format = (new Parameter($this->format))->asString($row, $context); - - /** - * @var array $values - */ - $values = array_map(static fn(ScalarFunction $value): mixed => (new Parameter($value))->eval( - $row, + $formats = (new Parameter($this->format))->asStrings($rows, $context); + $arguments = array_map(static fn(ScalarFunction $value): array => (new Parameter($value))->values( + $rows, $context, ), $this->values); - - if ($format === null || in_array(null, $values, true)) { - throw new InvalidArgumentException('Sprintf requires non-null format and values'); + $results = []; + $i = 0; + + try { + foreach ($formats as $i => $format) { + /** + * @var array $values + */ + $values = array_map(static fn(array $argument): mixed => $argument[$i], $arguments); + + if ($format === null || in_array(null, $values, true)) { + throw new InvalidArgumentException('Sprintf requires non-null format and values'); + } + + /** @var array $nonNullValues */ + $nonNullValues = $values; + + $results[] = sprintf($format, ...$nonNullValues); + } + } catch (Exception $e) { + throw EvaluationException::at($i, $e); } - /** @var array $nonNullValues */ - $nonNullValues = $values; - - return sprintf($format, ...$nonNullValues); + return (new ResultColumn($context->backend()))->of($this, $results); } } diff --git a/src/core/etl/src/Flow/ETL/Function/StartsWith.php b/src/core/etl/src/Flow/ETL/Function/StartsWith.php index cc22032bec..b5754a3c1e 100644 --- a/src/core/etl/src/Flow/ETL/Function/StartsWith.php +++ b/src/core/etl/src/Flow/ETL/Function/StartsWith.php @@ -4,8 +4,12 @@ namespace Flow\ETL\Function; +use Exception; +use Flow\ETL\Column\Column; +use Flow\ETL\Exception\EvaluationException; use Flow\ETL\FlowContext; -use Flow\ETL\Row; +use Flow\ETL\Function\Evaluation\ResultColumn; +use Flow\ETL\Rows; use Flow\Types\Type; use Flow\Types\Type\Nullability; @@ -51,15 +55,29 @@ public function returns(): Type return (new Nullability())->any(type_boolean(), $this->haystack->returns(), $this->needle->returns()); } - public function eval(Row $row, FlowContext $context): ?bool + public function eval(Rows $rows, FlowContext $context): Column { - $haystack = (new Parameter($this->haystack))->asString($row, $context); - $needle = (new Parameter($this->needle))->asString($row, $context); + $haystacks = (new Parameter($this->haystack))->asStrings($rows, $context); + $needles = (new Parameter($this->needle))->asStrings($rows, $context); + $results = []; + $i = 0; - if ($haystack === null || $needle === null) { - return null; + try { + foreach ($haystacks as $i => $haystack) { + $needle = $needles[$i]; + + if ($haystack === null || $needle === null) { + $results[] = null; + + continue; + } + + $results[] = str_starts_with($haystack, $needle); + } + } catch (Exception $e) { + throw EvaluationException::at($i, $e); } - return str_starts_with($haystack, $needle); + return (new ResultColumn($context->backend()))->of($this, $results); } } diff --git a/src/core/etl/src/Flow/ETL/Function/StrPad.php b/src/core/etl/src/Flow/ETL/Function/StrPad.php index 4ad985609c..a75e26dbd3 100644 --- a/src/core/etl/src/Flow/ETL/Function/StrPad.php +++ b/src/core/etl/src/Flow/ETL/Function/StrPad.php @@ -4,9 +4,13 @@ namespace Flow\ETL\Function; +use Exception; +use Flow\ETL\Column\Column; +use Flow\ETL\Exception\EvaluationException; use Flow\ETL\Exception\InvalidArgumentException; use Flow\ETL\FlowContext; -use Flow\ETL\Row; +use Flow\ETL\Function\Evaluation\ResultColumn; +use Flow\ETL\Rows; use Flow\Types\Type; use function Flow\ETL\DSL\lit; @@ -59,21 +63,35 @@ public function returns(): Type return type_string(); } - public function eval(Row $row, FlowContext $context): mixed + public function eval(Rows $rows, FlowContext $context): Column { - $value = (new Parameter($this->value))->asString($row, $context); - $length = (new Parameter($this->length))->asInt($row, $context); - $padString = (new Parameter($this->padString))->asString($row, $context); - $type = (new Parameter($this->type))->asInt($row, $context); + $values = (new Parameter($this->value))->asStrings($rows, $context); + $lengths = (new Parameter($this->length))->asInts($rows, $context); + $padStrings = (new Parameter($this->padString))->asStrings($rows, $context); + $types = (new Parameter($this->type))->asInts($rows, $context); + $results = []; + $i = 0; - if ($value === null) { - throw new InvalidArgumentException('StrPad function requires non-null value'); - } + try { + foreach ($values as $i => $value) { + $length = $lengths[$i]; + $padString = $padStrings[$i]; + $type = $types[$i]; + + if ($value === null) { + throw new InvalidArgumentException('StrPad function requires non-null value'); + } + + if ($length === null || $padString === null || $type === null) { + throw new InvalidArgumentException('StrPad function requires non-null length, padString and type'); + } - if ($length === null || $padString === null || $type === null) { - throw new InvalidArgumentException('StrPad function requires non-null length, padString and type'); + $results[] = str_pad($value, $length, $padString, $type); + } + } catch (Exception $e) { + throw EvaluationException::at($i, $e); } - return str_pad($value, $length, $padString, $type); + return (new ResultColumn($context->backend()))->of($this, $results); } } diff --git a/src/core/etl/src/Flow/ETL/Function/StrReplace.php b/src/core/etl/src/Flow/ETL/Function/StrReplace.php index fd4e01e825..0657ec5b4d 100644 --- a/src/core/etl/src/Flow/ETL/Function/StrReplace.php +++ b/src/core/etl/src/Flow/ETL/Function/StrReplace.php @@ -4,9 +4,13 @@ namespace Flow\ETL\Function; +use Exception; +use Flow\ETL\Column\Column; +use Flow\ETL\Exception\EvaluationException; use Flow\ETL\Exception\InvalidArgumentException; use Flow\ETL\FlowContext; -use Flow\ETL\Row; +use Flow\ETL\Function\Evaluation\ResultColumn; +use Flow\ETL\Rows; use Flow\Types\Type; use function Flow\ETL\DSL\lit; @@ -63,23 +67,36 @@ public function returns(): Type return type_string(); } - public function eval(Row $row, FlowContext $context): ?string + public function eval(Rows $rows, FlowContext $context): Column { - $value = (new Parameter($this->value))->asString($row, $context); - $search = (new Parameter($this->search))->as($row, $context, type_string(), type_array()); - $replace = (new Parameter($this->replace))->as($row, $context, type_string(), type_array()); + $values = (new Parameter($this->value))->asStrings($rows, $context); + $searches = (new Parameter($this->search))->asTypes($rows, $context, type_string(), type_array()); + $replaces = (new Parameter($this->replace))->asTypes($rows, $context, type_string(), type_array()); + $results = []; + $i = 0; - if ($value === null) { - throw new InvalidArgumentException('StrReplace function requires non-null value'); - } + try { + foreach ($values as $i => $value) { + $search = $searches[$i]; + $replace = $replaces[$i]; - if ($search === null || $replace === null) { - throw new InvalidArgumentException('StrReplace function requires non-null search and replace'); - } + if ($value === null) { + throw new InvalidArgumentException('StrReplace function requires non-null value'); + } + + if ($search === null || $replace === null) { + throw new InvalidArgumentException('StrReplace function requires non-null search and replace'); + } - $typedSearch = type_union(type_string(), type_list(type_string()))->assert($search); - $typedReplace = type_union(type_string(), type_list(type_string()))->assert($replace); + $typedSearch = type_union(type_string(), type_list(type_string()))->assert($search); + $typedReplace = type_union(type_string(), type_list(type_string()))->assert($replace); + + $results[] = str_replace($typedSearch, $typedReplace, $value); + } + } catch (Exception $e) { + throw EvaluationException::at($i, $e); + } - return str_replace($typedSearch, $typedReplace, $value); + return (new ResultColumn($context->backend()))->of($this, $results); } } diff --git a/src/core/etl/src/Flow/ETL/Function/StringAfter.php b/src/core/etl/src/Flow/ETL/Function/StringAfter.php index 8421cb5027..f1194932fa 100644 --- a/src/core/etl/src/Flow/ETL/Function/StringAfter.php +++ b/src/core/etl/src/Flow/ETL/Function/StringAfter.php @@ -4,9 +4,13 @@ namespace Flow\ETL\Function; +use Exception; +use Flow\ETL\Column\Column; +use Flow\ETL\Exception\EvaluationException; use Flow\ETL\Exception\InvalidArgumentException; use Flow\ETL\FlowContext; -use Flow\ETL\Row; +use Flow\ETL\Function\Evaluation\ResultColumn; +use Flow\ETL\Rows; use Flow\Types\Type; use function Flow\ETL\DSL\lit; @@ -58,21 +62,27 @@ public function returns(): Type return type_string(); } - public function eval(Row $row, FlowContext $context): ?string + public function eval(Rows $rows, FlowContext $context): Column { - $string = (new Parameter($this->string))->asString($row, $context); + $strings = (new Parameter($this->string))->asStrings($rows, $context); + $needles = (new Parameter($this->needle))->asStrings($rows, $context); + $includeNeedles = (new Parameter($this->includeNeedle))->asBooleans($rows, $context); + $results = []; + $i = 0; - if ($string === null) { - throw new InvalidArgumentException('StringAfter function requires non-null value'); - } + try { + foreach ($strings as $i => $string) { + if ($string === null) { + throw new InvalidArgumentException('StringAfter function requires non-null value'); + } - $needle = (new Parameter($this->needle))->asString($row, $context) ?? (new Parameter($this->needle))->asArray( - $row, - $context, - ); - $typedNeedle = type_union(type_string(), type_list(type_string()))->assert($needle); - $includeNeedle = (new Parameter($this->includeNeedle))->asBoolean($row, $context) ?? false; + $typedNeedle = type_union(type_string(), type_list(type_string()))->assert($needles[$i]); + $results[] = u($string)->after($typedNeedle, $includeNeedles[$i] ?? false)->toString(); + } + } catch (Exception $e) { + throw EvaluationException::at($i, $e); + } - return u($string)->after($typedNeedle, $includeNeedle)->toString(); + return (new ResultColumn($context->backend()))->of($this, $results); } } diff --git a/src/core/etl/src/Flow/ETL/Function/StringAfterLast.php b/src/core/etl/src/Flow/ETL/Function/StringAfterLast.php index 8c15e4ebe1..80a23edb77 100644 --- a/src/core/etl/src/Flow/ETL/Function/StringAfterLast.php +++ b/src/core/etl/src/Flow/ETL/Function/StringAfterLast.php @@ -4,9 +4,13 @@ namespace Flow\ETL\Function; +use Exception; +use Flow\ETL\Column\Column; +use Flow\ETL\Exception\EvaluationException; use Flow\ETL\Exception\InvalidArgumentException; use Flow\ETL\FlowContext; -use Flow\ETL\Row; +use Flow\ETL\Function\Evaluation\ResultColumn; +use Flow\ETL\Rows; use Flow\Types\Type; use function Flow\ETL\DSL\lit; @@ -58,21 +62,27 @@ public function returns(): Type return type_string(); } - public function eval(Row $row, FlowContext $context): ?string + public function eval(Rows $rows, FlowContext $context): Column { - $string = (new Parameter($this->string))->asString($row, $context); + $strings = (new Parameter($this->string))->asStrings($rows, $context); + $needles = (new Parameter($this->needle))->asStrings($rows, $context); + $includeNeedles = (new Parameter($this->includeNeedle))->asBooleans($rows, $context); + $results = []; + $i = 0; - if ($string === null) { - throw new InvalidArgumentException('StringAfterLast function requires non-null value'); - } + try { + foreach ($strings as $i => $string) { + if ($string === null) { + throw new InvalidArgumentException('StringAfterLast function requires non-null value'); + } - $needle = (new Parameter($this->needle))->asString($row, $context) ?? (new Parameter($this->needle))->asArray( - $row, - $context, - ); - $typedNeedle = type_union(type_string(), type_list(type_string()))->assert($needle); - $includeNeedle = (new Parameter($this->includeNeedle))->asBoolean($row, $context) ?? false; + $typedNeedle = type_union(type_string(), type_list(type_string()))->assert($needles[$i]); + $results[] = u($string)->afterLast($typedNeedle, $includeNeedles[$i] ?? false)->toString(); + } + } catch (Exception $e) { + throw EvaluationException::at($i, $e); + } - return u($string)->afterLast($typedNeedle, $includeNeedle)->toString(); + return (new ResultColumn($context->backend()))->of($this, $results); } } diff --git a/src/core/etl/src/Flow/ETL/Function/StringAggregate.php b/src/core/etl/src/Flow/ETL/Function/StringAggregate.php index 0eccc85766..92bb789a3d 100644 --- a/src/core/etl/src/Flow/ETL/Function/StringAggregate.php +++ b/src/core/etl/src/Flow/ETL/Function/StringAggregate.php @@ -4,10 +4,12 @@ namespace Flow\ETL\Function; +use Flow\ETL\Exception\InvalidArgumentException; use Flow\ETL\FlowContext; -use Flow\ETL\Row; use Flow\ETL\Row\Reference; use Flow\ETL\Row\SortOrder; +use Flow\ETL\Rows; +use Flow\ETL\Sort\RowOrder; use Flow\Types\Type; use function count; @@ -17,6 +19,7 @@ use function is_string; use function rsort; use function sort; +use function sprintf; final class StringAggregate implements AggregatingFunction { @@ -54,17 +57,27 @@ public function withChildren(array $children): static return new self($children[0], $this->separator, $this->sort); } - public function aggregate(Row $row, FlowContext $context): void + public function aggregate(Rows $rows, array $indices, FlowContext $context): void { - if (!$row->has($this->ref->to())) { + if ($rows->schema()->findDefinition($this->ref->to()) === null) { return; } - $stringValue = $row->get($this->ref->to()); + // @mago-ignore analysis:mixed-assignment + foreach ($rows->column($this->ref->to())->take($indices)->values() as $value) { + if (is_string($value)) { + $this->values[] = $value; + } + } + } - if (is_string($stringValue)) { - $this->values[] = $stringValue; + public function merge(AggregatingFunction $other, FlowContext $context): void + { + if (!$other instanceof self) { + throw new InvalidArgumentException(sprintf('%s cannot merge %s', self::class, $other::class)); } + + $this->values = [...$this->values, ...$other->values]; } public function outputName(): string @@ -97,7 +110,8 @@ public function value(): string $values = $this->values; if ($this->sort) { - $this->sort === SortOrder::ASC ? sort($values) : rsort($values); + $flag = RowOrder::flag(type_string()); + $this->sort === SortOrder::ASC ? sort($values, $flag) : rsort($values, $flag); } return implode($this->separator, $values); diff --git a/src/core/etl/src/Flow/ETL/Function/StringBefore.php b/src/core/etl/src/Flow/ETL/Function/StringBefore.php index f9da268468..26e37e96b4 100644 --- a/src/core/etl/src/Flow/ETL/Function/StringBefore.php +++ b/src/core/etl/src/Flow/ETL/Function/StringBefore.php @@ -4,9 +4,13 @@ namespace Flow\ETL\Function; +use Exception; +use Flow\ETL\Column\Column; +use Flow\ETL\Exception\EvaluationException; use Flow\ETL\Exception\InvalidArgumentException; use Flow\ETL\FlowContext; -use Flow\ETL\Row; +use Flow\ETL\Function\Evaluation\ResultColumn; +use Flow\ETL\Rows; use Flow\Types\Type; use function Flow\ETL\DSL\lit; @@ -58,21 +62,27 @@ public function returns(): Type return type_string(); } - public function eval(Row $row, FlowContext $context): ?string + public function eval(Rows $rows, FlowContext $context): Column { - $string = (new Parameter($this->string))->asString($row, $context); + $strings = (new Parameter($this->string))->asStrings($rows, $context); + $needles = (new Parameter($this->needle))->asStrings($rows, $context); + $includeNeedles = (new Parameter($this->includeNeedle))->asBooleans($rows, $context); + $results = []; + $i = 0; - if ($string === null) { - throw new InvalidArgumentException('StringBefore function requires non-null value'); - } + try { + foreach ($strings as $i => $string) { + if ($string === null) { + throw new InvalidArgumentException('StringBefore function requires non-null value'); + } - $needle = (new Parameter($this->needle))->asString($row, $context) ?? (new Parameter($this->needle))->asArray( - $row, - $context, - ); - $typedNeedle = type_union(type_string(), type_list(type_string()))->assert($needle); - $includeNeedle = (new Parameter($this->includeNeedle))->asBoolean($row, $context) ?? false; + $typedNeedle = type_union(type_string(), type_list(type_string()))->assert($needles[$i]); + $results[] = u($string)->before($typedNeedle, $includeNeedles[$i] ?? false)->toString(); + } + } catch (Exception $e) { + throw EvaluationException::at($i, $e); + } - return u($string)->before($typedNeedle, $includeNeedle)->toString(); + return (new ResultColumn($context->backend()))->of($this, $results); } } diff --git a/src/core/etl/src/Flow/ETL/Function/StringBeforeLast.php b/src/core/etl/src/Flow/ETL/Function/StringBeforeLast.php index df8752ef1f..552dbcdac6 100644 --- a/src/core/etl/src/Flow/ETL/Function/StringBeforeLast.php +++ b/src/core/etl/src/Flow/ETL/Function/StringBeforeLast.php @@ -4,9 +4,13 @@ namespace Flow\ETL\Function; +use Exception; +use Flow\ETL\Column\Column; +use Flow\ETL\Exception\EvaluationException; use Flow\ETL\Exception\InvalidArgumentException; use Flow\ETL\FlowContext; -use Flow\ETL\Row; +use Flow\ETL\Function\Evaluation\ResultColumn; +use Flow\ETL\Rows; use Flow\Types\Type; use function Flow\ETL\DSL\lit; @@ -58,21 +62,27 @@ public function returns(): Type return type_string(); } - public function eval(Row $row, FlowContext $context): ?string + public function eval(Rows $rows, FlowContext $context): Column { - $string = (new Parameter($this->string))->asString($row, $context); + $strings = (new Parameter($this->string))->asStrings($rows, $context); + $needles = (new Parameter($this->needle))->asStrings($rows, $context); + $includeNeedles = (new Parameter($this->includeNeedle))->asBooleans($rows, $context); + $results = []; + $i = 0; - if ($string === null) { - throw new InvalidArgumentException('StringBeforeLast function requires non-null value'); - } + try { + foreach ($strings as $i => $string) { + if ($string === null) { + throw new InvalidArgumentException('StringBeforeLast function requires non-null value'); + } - $needle = (new Parameter($this->needle))->asString($row, $context) ?? (new Parameter($this->needle))->asArray( - $row, - $context, - ); - $typedNeedle = type_union(type_string(), type_list(type_string()))->assert($needle); - $includeNeedle = (new Parameter($this->includeNeedle))->asBoolean($row, $context) ?? false; + $typedNeedle = type_union(type_string(), type_list(type_string()))->assert($needles[$i]); + $results[] = u($string)->beforeLast($typedNeedle, $includeNeedles[$i] ?? false)->toString(); + } + } catch (Exception $e) { + throw EvaluationException::at($i, $e); + } - return u($string)->beforeLast($typedNeedle, $includeNeedle)->toString(); + return (new ResultColumn($context->backend()))->of($this, $results); } } diff --git a/src/core/etl/src/Flow/ETL/Function/StringContainsAny.php b/src/core/etl/src/Flow/ETL/Function/StringContainsAny.php index c4bdb879d3..e0164a5f7a 100644 --- a/src/core/etl/src/Flow/ETL/Function/StringContainsAny.php +++ b/src/core/etl/src/Flow/ETL/Function/StringContainsAny.php @@ -4,9 +4,13 @@ namespace Flow\ETL\Function; +use Exception; +use Flow\ETL\Column\Column; +use Flow\ETL\Exception\EvaluationException; use Flow\ETL\Exception\InvalidArgumentException; use Flow\ETL\FlowContext; -use Flow\ETL\Row; +use Flow\ETL\Function\Evaluation\ResultColumn; +use Flow\ETL\Rows; use Flow\Types\Type; use Flow\Types\Type\Nullability; @@ -58,21 +62,35 @@ public function returns(): Type return (new Nullability())->any(type_boolean(), $this->value->returns(), $this->needles->returns()); } - public function eval(Row $row, FlowContext $context): ?bool + public function eval(Rows $rows, FlowContext $context): Column { - $value = (new Parameter($this->value))->asString($row, $context); - $needles = (new Parameter($this->needles))->asArray($row, $context); + $values = (new Parameter($this->value))->asStrings($rows, $context); + $needlesList = (new Parameter($this->needles))->asArrays($rows, $context); + $results = []; + $i = 0; - if ($value === null || $needles === null) { - return null; - } + try { + foreach ($values as $i => $value) { + $needles = $needlesList[$i]; - if (count($needles) === 0) { - throw new InvalidArgumentException('StringContainsAny function requires a non-empty needles array'); - } + if ($value === null || $needles === null) { + $results[] = null; + + continue; + } - $typedNeedles = type_list(type_string())->assert($needles); + if (count($needles) === 0) { + throw new InvalidArgumentException('StringContainsAny function requires a non-empty needles array'); + } + + $typedNeedles = type_list(type_string())->assert($needles); + + $results[] = s($value)->containsAny($typedNeedles); + } + } catch (Exception $e) { + throw EvaluationException::at($i, $e); + } - return s($value)->containsAny($typedNeedles); + return (new ResultColumn($context->backend()))->of($this, $results); } } diff --git a/src/core/etl/src/Flow/ETL/Function/StringEqualsTo.php b/src/core/etl/src/Flow/ETL/Function/StringEqualsTo.php index c96f8b13d5..360bd7a675 100644 --- a/src/core/etl/src/Flow/ETL/Function/StringEqualsTo.php +++ b/src/core/etl/src/Flow/ETL/Function/StringEqualsTo.php @@ -4,8 +4,12 @@ namespace Flow\ETL\Function; +use Exception; +use Flow\ETL\Column\Column; +use Flow\ETL\Exception\EvaluationException; use Flow\ETL\FlowContext; -use Flow\ETL\Row; +use Flow\ETL\Function\Evaluation\ResultColumn; +use Flow\ETL\Rows; use Flow\Types\Type; use Flow\Types\Type\Nullability; @@ -51,15 +55,29 @@ public function returns(): Type return (new Nullability())->any(type_boolean(), $this->value->returns(), $this->string->returns()); } - public function eval(Row $row, FlowContext $context): ?bool + public function eval(Rows $rows, FlowContext $context): Column { - $value = (new Parameter($this->value))->asString($row, $context); - $string = (new Parameter($this->string))->asString($row, $context); + $values = (new Parameter($this->value))->asStrings($rows, $context); + $strings = (new Parameter($this->string))->asStrings($rows, $context); + $results = []; + $i = 0; - if ($value === null || $string === null) { - return null; + try { + foreach ($values as $i => $value) { + $string = $strings[$i]; + + if ($value === null || $string === null) { + $results[] = null; + + continue; + } + + $results[] = s($value)->equalsTo($string); + } + } catch (Exception $e) { + throw EvaluationException::at($i, $e); } - return s($value)->equalsTo($string); + return (new ResultColumn($context->backend()))->of($this, $results); } } diff --git a/src/core/etl/src/Flow/ETL/Function/StringFold.php b/src/core/etl/src/Flow/ETL/Function/StringFold.php index e3011479bd..c3b22c99b2 100644 --- a/src/core/etl/src/Flow/ETL/Function/StringFold.php +++ b/src/core/etl/src/Flow/ETL/Function/StringFold.php @@ -4,9 +4,13 @@ namespace Flow\ETL\Function; +use Exception; +use Flow\ETL\Column\Column; +use Flow\ETL\Exception\EvaluationException; use Flow\ETL\Exception\InvalidArgumentException; use Flow\ETL\FlowContext; -use Flow\ETL\Row; +use Flow\ETL\Function\Evaluation\ResultColumn; +use Flow\ETL\Rows; use Flow\Types\Type; use function Flow\ETL\DSL\lit; @@ -49,14 +53,24 @@ public function returns(): Type return type_string(); } - public function eval(Row $row, FlowContext $context): ?string + public function eval(Rows $rows, FlowContext $context): Column { - $string = (new Parameter($this->string))->asString($row, $context); + $strings = (new Parameter($this->string))->asStrings($rows, $context); + $results = []; + $i = 0; - if ($string === null) { - throw new InvalidArgumentException('StringFold function requires non-null value'); + try { + foreach ($strings as $i => $string) { + if ($string === null) { + throw new InvalidArgumentException('StringFold function requires non-null value'); + } + + $results[] = u($string)->folded()->toString(); + } + } catch (Exception $e) { + throw EvaluationException::at($i, $e); } - return u($string)->folded()->toString(); + return (new ResultColumn($context->backend()))->of($this, $results); } } diff --git a/src/core/etl/src/Flow/ETL/Function/StringMatch.php b/src/core/etl/src/Flow/ETL/Function/StringMatch.php index 29b057c472..b9085877d5 100644 --- a/src/core/etl/src/Flow/ETL/Function/StringMatch.php +++ b/src/core/etl/src/Flow/ETL/Function/StringMatch.php @@ -4,11 +4,14 @@ namespace Flow\ETL\Function; +use Exception; +use Flow\ETL\Column\Column; +use Flow\ETL\Exception\EvaluationException; use Flow\ETL\Exception\InvalidArgumentException; use Flow\ETL\FlowContext; -use Flow\ETL\Row; +use Flow\ETL\Function\Evaluation\ResultColumn; +use Flow\ETL\Rows; use Flow\Types\Type; -use Throwable; use function count; use function Flow\ETL\DSL\lit; @@ -54,29 +57,38 @@ public function returns(): Type return type_optional(type_array()); } - /** - * @return null|array - */ - public function eval(Row $row, FlowContext $context): ?array + public function eval(Rows $rows, FlowContext $context): Column { - $haystack = (new Parameter($this->haystack))->asString($row, $context); - $pattern = (new Parameter($this->pattern))->asString($row, $context); + $haystacks = (new Parameter($this->haystack))->asStrings($rows, $context); + $patterns = (new Parameter($this->pattern))->asStrings($rows, $context); + $results = []; + $i = 0; - if ($haystack === null) { - throw new InvalidArgumentException('StringMatch function requires non-null haystack'); - } + try { + foreach ($haystacks as $i => $haystack) { + $pattern = $patterns[$i]; - if ($pattern === null) { - throw new InvalidArgumentException('StringMatch function requires non-null pattern'); - } + if ($haystack === null) { + throw new InvalidArgumentException('StringMatch function requires non-null haystack'); + } - try { - /** @var array $result */ - $result = s($haystack)->match($pattern); + if ($pattern === null) { + throw new InvalidArgumentException('StringMatch function requires non-null pattern'); + } - return count($result) > 0 ? $result : null; - } catch (Throwable $e) { - throw new InvalidArgumentException('StringMatch error: ' . $e->getMessage()); + try { + /** @var array $result */ + $result = s($haystack)->match($pattern); + + $results[] = count($result) > 0 ? $result : null; + } catch (Exception $e) { + throw new InvalidArgumentException('StringMatch error: ' . $e->getMessage(), 0, $e); + } + } + } catch (Exception $e) { + throw EvaluationException::at($i, $e); } + + return (new ResultColumn($context->backend()))->of($this, $results); } } diff --git a/src/core/etl/src/Flow/ETL/Function/StringMatchAll.php b/src/core/etl/src/Flow/ETL/Function/StringMatchAll.php index 8114df29a8..6355ad9e35 100644 --- a/src/core/etl/src/Flow/ETL/Function/StringMatchAll.php +++ b/src/core/etl/src/Flow/ETL/Function/StringMatchAll.php @@ -4,11 +4,14 @@ namespace Flow\ETL\Function; +use Exception; +use Flow\ETL\Column\Column; +use Flow\ETL\Exception\EvaluationException; use Flow\ETL\Exception\InvalidArgumentException; use Flow\ETL\FlowContext; -use Flow\ETL\Row; +use Flow\ETL\Function\Evaluation\ResultColumn; +use Flow\ETL\Rows; use Flow\Types\Type; -use Throwable; use function Flow\ETL\DSL\lit; use function Flow\Types\DSL\type_array; @@ -54,32 +57,43 @@ public function returns(): Type return type_array(); } - /** - * @return array> - */ - public function eval(Row $row, FlowContext $context): array + public function eval(Rows $rows, FlowContext $context): Column { - $haystack = (new Parameter($this->haystack))->asString($row, $context); - $pattern = (new Parameter($this->pattern))->asString($row, $context); + $haystacks = (new Parameter($this->haystack))->asStrings($rows, $context); + $patterns = (new Parameter($this->pattern))->asStrings($rows, $context); + $results = []; + $i = 0; - if ($haystack === null) { - throw new InvalidArgumentException('StringMatchAll function requires non-null haystack'); - } + try { + foreach ($haystacks as $i => $haystack) { + $pattern = $patterns[$i]; - if ($pattern === null) { - throw new InvalidArgumentException('StringMatchAll function requires non-null pattern'); - } + if ($haystack === null) { + throw new InvalidArgumentException('StringMatchAll function requires non-null haystack'); + } - try { - $matches = []; + if ($pattern === null) { + throw new InvalidArgumentException('StringMatchAll function requires non-null pattern'); + } - if (preg_match_all($pattern, $haystack, $matches, PREG_SET_ORDER) !== false) { - return $matches; - } + try { + $matches = []; + + if (preg_match_all($pattern, $haystack, $matches, PREG_SET_ORDER) !== false) { + $results[] = $matches; + + continue; + } - return []; - } catch (Throwable $e) { - throw new InvalidArgumentException('StringMatchAll error: ' . $e->getMessage()); + $results[] = []; + } catch (Exception $e) { + throw new InvalidArgumentException('StringMatchAll error: ' . $e->getMessage(), 0, $e); + } + } + } catch (Exception $e) { + throw EvaluationException::at($i, $e); } + + return (new ResultColumn($context->backend()))->of($this, $results); } } diff --git a/src/core/etl/src/Flow/ETL/Function/StringNormalize.php b/src/core/etl/src/Flow/ETL/Function/StringNormalize.php index fd214bd39a..21a7dbaa34 100644 --- a/src/core/etl/src/Flow/ETL/Function/StringNormalize.php +++ b/src/core/etl/src/Flow/ETL/Function/StringNormalize.php @@ -4,9 +4,13 @@ namespace Flow\ETL\Function; +use Exception; +use Flow\ETL\Column\Column; +use Flow\ETL\Exception\EvaluationException; use Flow\ETL\Exception\InvalidArgumentException; use Flow\ETL\FlowContext; -use Flow\ETL\Row; +use Flow\ETL\Function\Evaluation\ResultColumn; +use Flow\ETL\Rows; use Flow\Types\Type; use Normalizer; @@ -52,15 +56,27 @@ public function returns(): Type return type_string(); } - public function eval(Row $row, FlowContext $context): ?string + public function eval(Rows $rows, FlowContext $context): Column { - $value = (new Parameter($this->value))->asString($row, $context); - $form = (new Parameter($this->form))->asInt($row, $context, Normalizer::NFC); + $values = (new Parameter($this->value))->asStrings($rows, $context); + $forms = (new Parameter($this->form))->asInts($rows, $context, Normalizer::NFC); + $results = []; + $i = 0; - if ($value === null) { - throw new InvalidArgumentException('StringNormalize function requires non-null value'); + try { + foreach ($values as $i => $value) { + $form = $forms[$i]; + + if ($value === null) { + throw new InvalidArgumentException('StringNormalize function requires non-null value'); + } + + $results[] = u($value)->normalize($form)->toString(); + } + } catch (Exception $e) { + throw EvaluationException::at($i, $e); } - return u($value)->normalize($form)->toString(); + return (new ResultColumn($context->backend()))->of($this, $results); } } diff --git a/src/core/etl/src/Flow/ETL/Function/StringStyle.php b/src/core/etl/src/Flow/ETL/Function/StringStyle.php index 01faa18b9e..aec1c9d78f 100644 --- a/src/core/etl/src/Flow/ETL/Function/StringStyle.php +++ b/src/core/etl/src/Flow/ETL/Function/StringStyle.php @@ -4,9 +4,13 @@ namespace Flow\ETL\Function; +use Exception; +use Flow\ETL\Column\Column; +use Flow\ETL\Exception\EvaluationException; use Flow\ETL\Exception\InvalidArgumentException; use Flow\ETL\FlowContext; -use Flow\ETL\Row; +use Flow\ETL\Function\Evaluation\ResultColumn; +use Flow\ETL\Rows; use Flow\ETL\String\StringStyles; use Flow\Types\Type; @@ -52,23 +56,40 @@ public function returns(): Type return type_string(); } - public function eval(Row $row, FlowContext $context): ?string + public function eval(Rows $rows, FlowContext $context): Column { - $string = (new Parameter($this->string))->asString($row, $context); - $style = (new Parameter($this->style))->as($row, $context, type_string(), type_enum(StringStyles::class)); + $strings = (new Parameter($this->string))->asStrings($rows, $context); + $styles = (new Parameter($this->style))->asTypes( + $rows, + $context, + type_string(), + type_enum(StringStyles::class), + ); + $results = []; + $i = 0; - if ($string === null) { - throw new InvalidArgumentException('StringStyle function requires non-null value'); - } + try { + foreach ($strings as $i => $string) { + $style = $styles[$i]; - if ($style === null) { - throw new InvalidArgumentException('StringStyle function requires non-null style'); - } + if ($string === null) { + throw new InvalidArgumentException('StringStyle function requires non-null value'); + } + + if ($style === null) { + throw new InvalidArgumentException('StringStyle function requires non-null style'); + } + + if (is_string($style)) { + $style = StringStyles::fromString($style); + } - if (is_string($style)) { - $style = StringStyles::fromString($style); + $results[] = $style->convert($string); + } + } catch (Exception $e) { + throw EvaluationException::at($i, $e); } - return $style->convert($string); + return (new ResultColumn($context->backend()))->of($this, $results); } } diff --git a/src/core/etl/src/Flow/ETL/Function/StringTitle.php b/src/core/etl/src/Flow/ETL/Function/StringTitle.php index 701a4483e9..060100931f 100644 --- a/src/core/etl/src/Flow/ETL/Function/StringTitle.php +++ b/src/core/etl/src/Flow/ETL/Function/StringTitle.php @@ -4,9 +4,13 @@ namespace Flow\ETL\Function; +use Exception; +use Flow\ETL\Column\Column; +use Flow\ETL\Exception\EvaluationException; use Flow\ETL\Exception\InvalidArgumentException; use Flow\ETL\FlowContext; -use Flow\ETL\Row; +use Flow\ETL\Function\Evaluation\ResultColumn; +use Flow\ETL\Rows; use Flow\Types\Type; use function Flow\ETL\DSL\lit; @@ -51,15 +55,29 @@ public function returns(): Type return type_string(); } - public function eval(Row $row, FlowContext $context): ?string + public function eval(Rows $rows, FlowContext $context): Column { - $string = (new Parameter($this->string))->asString($row, $context); - $allWords = (new Parameter($this->allWords))->asBoolean($row, $context) ?? false; + $strings = (new Parameter($this->string))->asStrings($rows, $context); + $allWordsList = (new Parameter($this->allWords))->asBooleans($rows, $context); + $results = []; + $i = 0; - if ($string === null) { - throw new InvalidArgumentException('StringTitle function requires non-null value'); + try { + foreach ($strings as $i => $string) { + $allWords = $allWordsList[$i]; + + $allWords ??= false; + + if ($string === null) { + throw new InvalidArgumentException('StringTitle function requires non-null value'); + } + + $results[] = u($string)->title(allWords: $allWords)->toString(); + } + } catch (Exception $e) { + throw EvaluationException::at($i, $e); } - return u($string)->title(allWords: $allWords)->toString(); + return (new ResultColumn($context->backend()))->of($this, $results); } } diff --git a/src/core/etl/src/Flow/ETL/Function/StringWidth.php b/src/core/etl/src/Flow/ETL/Function/StringWidth.php index 01aed5e651..1e21ef72da 100644 --- a/src/core/etl/src/Flow/ETL/Function/StringWidth.php +++ b/src/core/etl/src/Flow/ETL/Function/StringWidth.php @@ -4,9 +4,13 @@ namespace Flow\ETL\Function; +use Exception; +use Flow\ETL\Column\Column; +use Flow\ETL\Exception\EvaluationException; use Flow\ETL\Exception\InvalidArgumentException; use Flow\ETL\FlowContext; -use Flow\ETL\Row; +use Flow\ETL\Function\Evaluation\ResultColumn; +use Flow\ETL\Rows; use Flow\Types\Type; use function Flow\ETL\DSL\lit; @@ -49,14 +53,24 @@ public function returns(): Type return type_integer(); } - public function eval(Row $row, FlowContext $context): ?int + public function eval(Rows $rows, FlowContext $context): Column { - $value = (new Parameter($this->value))->asString($row, $context); + $values = (new Parameter($this->value))->asStrings($rows, $context); + $results = []; + $i = 0; - if ($value === null) { - throw new InvalidArgumentException('StringWidth function requires non-null value'); + try { + foreach ($values as $i => $value) { + if ($value === null) { + throw new InvalidArgumentException('StringWidth function requires non-null value'); + } + + $results[] = s($value)->width(); + } + } catch (Exception $e) { + throw EvaluationException::at($i, $e); } - return s($value)->width(); + return (new ResultColumn($context->backend()))->of($this, $results); } } diff --git a/src/core/etl/src/Flow/ETL/Function/Structure.php b/src/core/etl/src/Flow/ETL/Function/Structure.php index 0c7f92fa37..c5f2f95c85 100644 --- a/src/core/etl/src/Flow/ETL/Function/Structure.php +++ b/src/core/etl/src/Flow/ETL/Function/Structure.php @@ -4,10 +4,12 @@ namespace Flow\ETL\Function; +use Flow\ETL\Column\Column; use Flow\ETL\Exception\InvalidArgumentException; use Flow\ETL\Exception\InvalidLogicException; use Flow\ETL\FlowContext; -use Flow\ETL\Row; +use Flow\ETL\Function\Evaluation\ResultColumn; +use Flow\ETL\Rows; use Flow\Types\Type; use Flow\Types\Type\Logical\StructureType; @@ -87,17 +89,26 @@ public function returns(): Type return new StructureType($elements); } - /** - * @return non-empty-array - */ - public function eval(Row $row, FlowContext $context): array + public function eval(Rows $rows, FlowContext $context): Column { - $values = []; + $elements = []; foreach ($this->elements as $name => $element) { - $values[$name] = $element->eval($row, $context); + $elements[$name] = (new Parameter($element))->values($rows, $context); + } + + $results = []; + + for ($i = 0, $count = $rows->count(); $i < $count; $i++) { + $values = []; + + foreach ($elements as $name => $element) { + $values[$name] = $element[$i]; + } + + $results[] = $values; } - return $values; + return (new ResultColumn($context->backend()))->of($this, $results); } } diff --git a/src/core/etl/src/Flow/ETL/Function/StructureSelect.php b/src/core/etl/src/Flow/ETL/Function/StructureSelect.php index 29e3833470..f4e6211930 100644 --- a/src/core/etl/src/Flow/ETL/Function/StructureSelect.php +++ b/src/core/etl/src/Flow/ETL/Function/StructureSelect.php @@ -4,17 +4,20 @@ namespace Flow\ETL\Function; +use Flow\ETL\Column\Column; use Flow\ETL\Exception\InvalidLogicException; use Flow\ETL\Exception\SchemaNotDerivableException; use Flow\ETL\FlowContext; -use Flow\ETL\Row; +use Flow\ETL\Function\Evaluation\ResultColumn; use Flow\ETL\Row\Reference; use Flow\ETL\Row\References; use Flow\ETL\Row\UnresolvedReference; +use Flow\ETL\Rows; use Flow\Types\Type; use Flow\Types\Type\ArrayKey; use Flow\Types\Type\Logical\StructureType; +use function array_fill; use function array_key_exists; use function Flow\Types\DSL\structure_element; use function Flow\Types\DSL\type_bare; @@ -88,30 +91,39 @@ public function returns(): Type return type_optional(new StructureType($elements)); } - /** - * @return null|array - */ - public function eval(Row $row, FlowContext $context): ?array + public function eval(Rows $rows, FlowContext $context): Column { - if (!$row->has($this->ref)) { - return null; + if ($rows->schema()->findDefinition($this->ref->base()) === null) { + // @mago-ignore analysis:possibly-invalid-argument + return (new ResultColumn($context->backend()))->of( + $this, + $rows->isEmpty() ? [] : array_fill(0, $rows->count(), null), + ); } - $value = $row->get($this->ref); + $results = []; - if (!is_array($value)) { - return null; - } - $output = []; + // @mago-ignore analysis:mixed-assignment + foreach ($rows->column($this->ref->base())->values() as $value) { + if (!is_array($value)) { + $results[] = null; - foreach ($this->refs as $ref) { - if (array_key_exists($ref->to(), $value)) { - $output[$ref->name()] = $value[$ref->to()]; - } else { - $output[$ref->name()] = null; + continue; } + + $output = []; + + foreach ($this->refs as $ref) { + if (array_key_exists($ref->to(), $value)) { + $output[$ref->name()] = $value[$ref->to()]; + } else { + $output[$ref->name()] = null; + } + } + + $results[] = $output; } - return $output; + return (new ResultColumn($context->backend()))->of($this, $results); } } diff --git a/src/core/etl/src/Flow/ETL/Function/Sum.php b/src/core/etl/src/Flow/ETL/Function/Sum.php index 3737d7da5e..7c488fbf88 100644 --- a/src/core/etl/src/Flow/ETL/Function/Sum.php +++ b/src/core/etl/src/Flow/ETL/Function/Sum.php @@ -8,8 +8,8 @@ use Flow\ETL\Exception\InvalidArgumentException; use Flow\ETL\Exception\RuntimeException; use Flow\ETL\FlowContext; -use Flow\ETL\Row; use Flow\ETL\Row\Reference; +use Flow\ETL\Rows; use Flow\ETL\Window; use Flow\ETL\Window\Accumulator\SumAccumulator; use Flow\ETL\Window\FrameAccumulator; @@ -19,6 +19,7 @@ use function Flow\Types\DSL\type_float; use function Flow\Types\DSL\type_optional; use function is_numeric; +use function sprintf; final class Sum implements AggregatingFunction, FrameAccumulating, WindowFunction { @@ -30,15 +31,12 @@ final class Sum implements AggregatingFunction, FrameAccumulating, WindowFunctio private ?RunningSum $runningSum = null; - private float|int $sum; - public function __construct( private readonly Reference $ref, - private readonly ScalarFunction|bool $exact = false, + private readonly bool $exact = false, private readonly ?Window $window = null, ) { $this->outputName = $ref->hasAlias() ? $ref->name() : $ref->to() . '_sum'; - $this->sum = 0; } /** @@ -46,7 +44,7 @@ public function __construct( */ public function children(): array { - return $this->exact instanceof ScalarFunction ? [$this->ref, $this->exact] : [$this->ref]; + return [$this->ref]; } /** @@ -54,29 +52,41 @@ public function children(): array */ public function withChildren(array $children): static { - /** @var array{0: Reference, 1?: ScalarFunction} $children */ - return new self($children[0], $children[1] ?? $this->exact, $this->window); + /** @var array{0: Reference} $children */ + return new self($children[0], $this->exact, $this->window); } - public function aggregate(Row $row, FlowContext $context): void + public function aggregate(Rows $rows, array $indices, FlowContext $context): void { - if (!$row->has($this->ref)) { + if ($rows->schema()->findDefinition($this->ref->base()) === null) { return; } - try { - $value = $row->get($this->ref); - + // @mago-ignore analysis:mixed-assignment + foreach ($rows->column($this->ref->base())->take($indices)->values() as $value) { if (is_int($value) || is_float($value) || is_string($value) && is_numeric($value)) { $this->runningSum ??= new RunningSum($context->calculator()); - $this->sum = $this->runningSum->add($this->sum, $value, $this->isExact($row, $context)); + $this->runningSum->add($value, $this->exact); $this->aggregated++; } - } catch (InvalidArgumentException $e) { - throw new InvalidArgumentException('Sum error: ' . $e->getMessage(), 0, $e); } } + public function merge(AggregatingFunction $other, FlowContext $context): void + { + if (!$other instanceof self) { + throw new InvalidArgumentException(sprintf('%s cannot merge %s', self::class, $other::class)); + } + + if ($other->aggregated === 0 || $other->runningSum === null) { + return; + } + + $this->runningSum ??= new RunningSum($context->calculator()); + $this->runningSum->merge($other->runningSum, $this->exact); + $this->aggregated += $other->aggregated; + } + public function accumulator(FlowContext $context): FrameAccumulator { return new SumAccumulator($this->ref, $this->exact, $context); @@ -86,8 +96,10 @@ public function apply(WindowContext $window): mixed { $accumulator = $this->accumulator($window->flowContext()); - foreach ($window->frame() as $frameRow) { - $accumulator->accumulate($frameRow); + $frame = $window->frame(); + + for ($index = 0, $count = $frame->count(); $index < $count; $index++) { + $accumulator->accumulate($frame, $index); } return $accumulator->value(); @@ -99,12 +111,11 @@ public function over(Window $window): static } /** - * @return null|list null when $exact is a ScalarFunction - its entries cannot be - * statically enumerated, so spill column pruning must be disabled + * @return list */ - public function references(): ?array + public function references(): array { - return $this->exact instanceof ScalarFunction ? null : [$this->ref]; + return [$this->ref]; } public function outputName(): string @@ -113,7 +124,7 @@ public function outputName(): string } /** - * float, not the argument type - RunningSum::add() promotes to float on int overflow, so integer + * float, not the argument type - RunningSum promotes to float on int overflow, so integer * would be a declaration the accumulator can violate on ordinary data * * @return Type @@ -125,11 +136,11 @@ public function returns(): Type public function value(): ?float { - if ($this->aggregated === 0) { + if ($this->aggregated === 0 || $this->runningSum === null) { return null; } - return (float) $this->sum; + return (float) $this->runningSum->value(); } public function toString(): string @@ -145,13 +156,4 @@ public function window(): Window return $this->window; } - - private function isExact(Row $row, FlowContext $context): bool - { - if (is_bool($this->exact)) { - return $this->exact; - } - - return (new Parameter($this->exact))->asBoolean($row, $context) ?? false; - } } diff --git a/src/core/etl/src/Flow/ETL/Function/ToDate.php b/src/core/etl/src/Flow/ETL/Function/ToDate.php index 948f861789..5566808fd4 100644 --- a/src/core/etl/src/Flow/ETL/Function/ToDate.php +++ b/src/core/etl/src/Flow/ETL/Function/ToDate.php @@ -6,11 +6,14 @@ use DateTime; use DateTimeImmutable; -use DateTimeInterface; use DateTimeZone; +use Exception; +use Flow\ETL\Column\Column; +use Flow\ETL\Exception\EvaluationException; use Flow\ETL\Exception\InvalidArgumentException; use Flow\ETL\FlowContext; -use Flow\ETL\Row; +use Flow\ETL\Function\Evaluation\ResultColumn; +use Flow\ETL\Rows; use Flow\Types\Type; use function Flow\ETL\DSL\lit; @@ -63,36 +66,56 @@ public function returns(): Type return type_date(); } - public function eval(Row $row, FlowContext $context): ?DateTimeInterface + public function eval(Rows $rows, FlowContext $context): Column { - $value = (new Parameter($this->value))->eval($row, $context); - $format = (new Parameter($this->format))->asString($row, $context); - $timeZone = (new Parameter($this->timeZone))->asInstanceOf($row, $context, DateTimeZone::class); + $values = (new Parameter($this->value))->values($rows, $context); + $formats = (new Parameter($this->format))->asStrings($rows, $context); + $timeZones = (new Parameter($this->timeZone))->asInstancesOf($rows, $context, DateTimeZone::class); + $results = []; + $i = 0; - if ($value === null || $format === null || $timeZone === null) { - throw new InvalidArgumentException('ToDate function requires non-null values'); - } + try { + // @mago-ignore analysis:mixed-assignment + foreach ($values as $i => $value) { + $format = $formats[$i]; + $timeZone = $timeZones[$i]; - if (is_object($value)) { - if (is_a($value, DateTimeImmutable::class) || is_a($value, DateTime::class)) { - return $value->setTimezone($timeZone)->setTime(0, 0, 0, 0); - } + if ($value === null || $format === null || $timeZone === null) { + throw new InvalidArgumentException('ToDate function requires non-null values'); + } - throw new InvalidArgumentException('ToDate function requires DateTimeInterface object'); - } + if (is_object($value)) { + if (is_a($value, DateTimeImmutable::class) || is_a($value, DateTime::class)) { + $results[] = $value->setTimezone($timeZone)->setTime(0, 0, 0, 0); - if (is_int($value)) { - $date = DateTimeImmutable::createFromFormat('U', (string) $value, $timeZone); + continue; + } - return $date === false ? null : $date->setTime(0, 0, 0, 0); - } + throw new InvalidArgumentException('ToDate function requires DateTimeInterface object'); + } + + if (is_int($value)) { + $date = DateTimeImmutable::createFromFormat('U', (string) $value, $timeZone); - if (is_string($value)) { - $date = DateTimeImmutable::createFromFormat($format, $value, $timeZone); + $results[] = $date === false ? null : $date->setTime(0, 0, 0, 0); - return $date === false ? null : $date->setTime(0, 0, 0, 0); + continue; + } + + if (is_string($value)) { + $date = DateTimeImmutable::createFromFormat($format, $value, $timeZone); + + $results[] = $date === false ? null : $date->setTime(0, 0, 0, 0); + + continue; + } + + throw new InvalidArgumentException('ToDate function requires int or string value'); + } + } catch (Exception $e) { + throw EvaluationException::at($i, $e); } - throw new InvalidArgumentException('ToDate function requires int or string value'); + return (new ResultColumn($context->backend()))->of($this, $results); } } diff --git a/src/core/etl/src/Flow/ETL/Function/ToDateTime.php b/src/core/etl/src/Flow/ETL/Function/ToDateTime.php index 32b64556f7..df32ee2c08 100644 --- a/src/core/etl/src/Flow/ETL/Function/ToDateTime.php +++ b/src/core/etl/src/Flow/ETL/Function/ToDateTime.php @@ -7,10 +7,14 @@ use DateTimeImmutable; use DateTimeInterface; use DateTimeZone; +use Exception; +use Flow\ETL\Column\Column; +use Flow\ETL\Exception\EvaluationException; use Flow\ETL\Exception\InvalidArgumentException; use Flow\ETL\FlowContext; +use Flow\ETL\Function\Evaluation\ResultColumn; use Flow\ETL\Function\ToDateTime\PatternCoverage; -use Flow\ETL\Row; +use Flow\ETL\Rows; use Flow\Types\Type; use Flow\Types\Type\Logical\DateTimeType; @@ -102,36 +106,56 @@ public function returns(): Type return type_optional($this->type); } - public function eval(Row $row, FlowContext $context): ?DateTimeImmutable + public function eval(Rows $rows, FlowContext $context): Column { - $value = (new Parameter($this->value))->eval($row, $context); - $format = (new Parameter($this->format))->asString($row, $context); - $timeZone = (new Parameter($this->timeZone))->asInstanceOf($row, $context, DateTimeZone::class); + $values = (new Parameter($this->value))->values($rows, $context); + $formats = (new Parameter($this->format))->asStrings($rows, $context); + $timeZones = (new Parameter($this->timeZone))->asInstancesOf($rows, $context, DateTimeZone::class); + $results = []; + $i = 0; - if ($value === null || $format === null || $timeZone === null) { - throw new InvalidArgumentException('ToDateTime function requires non-null values'); - } + try { + // @mago-ignore analysis:mixed-assignment + foreach ($values as $i => $value) { + $format = $formats[$i]; + $timeZone = $timeZones[$i]; - if (is_object($value)) { - if ($value instanceof DateTimeInterface) { - return $this->type->cast($value); - } + if ($value === null || $format === null || $timeZone === null) { + throw new InvalidArgumentException('ToDateTime function requires non-null values'); + } - throw new InvalidArgumentException('ToDateTime function requires DateTimeInterface object'); - } + if (is_object($value)) { + if ($value instanceof DateTimeInterface) { + $results[] = $this->type->cast($value); - if (is_int($value)) { - $dateTime = DateTimeImmutable::createFromFormat('U', (string) $value, $timeZone); + continue; + } - return $dateTime === false ? null : $this->type->cast($dateTime); - } + throw new InvalidArgumentException('ToDateTime function requires DateTimeInterface object'); + } + + if (is_int($value)) { + $dateTime = DateTimeImmutable::createFromFormat('U', (string) $value, $timeZone); - if (is_string($value)) { - $dateTime = DateTimeImmutable::createFromFormat($format, $value, $timeZone); + $results[] = $dateTime === false ? null : $this->type->cast($dateTime); - return $dateTime === false ? null : $this->type->cast($dateTime); + continue; + } + + if (is_string($value)) { + $dateTime = DateTimeImmutable::createFromFormat($format, $value, $timeZone); + + $results[] = $dateTime === false ? null : $this->type->cast($dateTime); + + continue; + } + + throw new InvalidArgumentException('ToDateTime function requires int or string value'); + } + } catch (Exception $e) { + throw EvaluationException::at($i, $e); } - throw new InvalidArgumentException('ToDateTime function requires int or string value'); + return (new ResultColumn($context->backend()))->of($this, $results); } } diff --git a/src/core/etl/src/Flow/ETL/Function/ToLower.php b/src/core/etl/src/Flow/ETL/Function/ToLower.php index 106fafdc23..f2f85d6639 100644 --- a/src/core/etl/src/Flow/ETL/Function/ToLower.php +++ b/src/core/etl/src/Flow/ETL/Function/ToLower.php @@ -4,9 +4,13 @@ namespace Flow\ETL\Function; +use Exception; +use Flow\ETL\Column\Column; +use Flow\ETL\Exception\EvaluationException; use Flow\ETL\Exception\InvalidArgumentException; use Flow\ETL\FlowContext; -use Flow\ETL\Row; +use Flow\ETL\Function\Evaluation\ResultColumn; +use Flow\ETL\Rows; use Flow\Types\Type; use function Flow\ETL\DSL\lit; @@ -49,14 +53,24 @@ public function returns(): Type return type_string(); } - public function eval(Row $row, FlowContext $context): ?string + public function eval(Rows $rows, FlowContext $context): Column { - $value = (new Parameter($this->value))->asString($row, $context); + $values = (new Parameter($this->value))->asStrings($rows, $context); + $results = []; + $i = 0; - if ($value === null) { - throw new InvalidArgumentException('ToLower function requires non-null value'); + try { + foreach ($values as $i => $value) { + if ($value === null) { + throw new InvalidArgumentException('ToLower function requires non-null value'); + } + + $results[] = strtolower($value); + } + } catch (Exception $e) { + throw EvaluationException::at($i, $e); } - return strtolower($value); + return (new ResultColumn($context->backend()))->of($this, $results); } } diff --git a/src/core/etl/src/Flow/ETL/Function/ToTimeZone.php b/src/core/etl/src/Flow/ETL/Function/ToTimeZone.php index 528f326a85..38eb7a8c3c 100644 --- a/src/core/etl/src/Flow/ETL/Function/ToTimeZone.php +++ b/src/core/etl/src/Flow/ETL/Function/ToTimeZone.php @@ -6,13 +6,18 @@ use DateTimeInterface; use DateTimeZone; +use Exception; +use Flow\ETL\Column\Column; +use Flow\ETL\Exception\EvaluationException; use Flow\ETL\Exception\InvalidArgumentException; use Flow\ETL\FlowContext; -use Flow\ETL\Row; +use Flow\ETL\Function\Evaluation\ResultColumn; +use Flow\ETL\Rows; use Flow\Types\Type; use Flow\Types\Type\Logical\DateTimeType; use function Flow\ETL\DSL\lit; +use function Flow\Types\DSL\type_bare; use function Flow\Types\DSL\type_datetime; final class ToTimeZone implements ScalarFunction @@ -57,14 +62,42 @@ public function returns(): Type return $this->type; } - public function eval(Row $row, FlowContext $context): mixed + public function eval(Rows $rows, FlowContext $context): Column { - $dateTime = (new Parameter($this->value))->asInstanceOf($row, $context, DateTimeInterface::class); + $column = (new Parameter($this->value))->column($rows, $context); - if ($dateTime === null) { - throw new InvalidArgumentException('ToTimeZone function requires non-null values'); + if (type_bare($column->type()) instanceof DateTimeType) { + for ($i = 0, $count = $column->count(); $column->nullCount() > 0 && $i < $count; $i++) { + if ($column->isNull($i)) { + throw new EvaluationException( + $i, + new InvalidArgumentException('ToTimeZone function requires non-null values'), + ); + } + } + + return $column->withType($this->type); + } + + $results = []; + $i = 0; + + try { + foreach ((new Parameter($this->value))->asInstancesOf( + $rows, + $context, + DateTimeInterface::class, + ) as $i => $dateTime) { + if ($dateTime === null) { + throw new InvalidArgumentException('ToTimeZone function requires non-null values'); + } + + $results[] = $this->type->cast($dateTime); + } + } catch (Exception $e) { + throw EvaluationException::at($i, $e); } - return $this->type->cast($dateTime); + return (new ResultColumn($context->backend()))->of($this, $results); } } diff --git a/src/core/etl/src/Flow/ETL/Function/ToUpper.php b/src/core/etl/src/Flow/ETL/Function/ToUpper.php index a587676f07..5e3719ce32 100644 --- a/src/core/etl/src/Flow/ETL/Function/ToUpper.php +++ b/src/core/etl/src/Flow/ETL/Function/ToUpper.php @@ -4,9 +4,13 @@ namespace Flow\ETL\Function; +use Exception; +use Flow\ETL\Column\Column; +use Flow\ETL\Exception\EvaluationException; use Flow\ETL\Exception\InvalidArgumentException; use Flow\ETL\FlowContext; -use Flow\ETL\Row; +use Flow\ETL\Function\Evaluation\ResultColumn; +use Flow\ETL\Rows; use Flow\Types\Type; use function Flow\ETL\DSL\lit; @@ -49,14 +53,24 @@ public function returns(): Type return type_string(); } - public function eval(Row $row, FlowContext $context): mixed + public function eval(Rows $rows, FlowContext $context): Column { - $value = (new Parameter($this->value))->asString($row, $context); + $values = (new Parameter($this->value))->asStrings($rows, $context); + $results = []; + $i = 0; - if ($value === null) { - throw new InvalidArgumentException('ToUpper function requires non-null value'); + try { + foreach ($values as $i => $value) { + if ($value === null) { + throw new InvalidArgumentException('ToUpper function requires non-null value'); + } + + $results[] = mb_strtoupper($value); + } + } catch (Exception $e) { + throw EvaluationException::at($i, $e); } - return mb_strtoupper($value); + return (new ResultColumn($context->backend()))->of($this, $results); } } diff --git a/src/core/etl/src/Flow/ETL/Function/Trim.php b/src/core/etl/src/Flow/ETL/Function/Trim.php index 01737f2b69..3df1eee2ba 100644 --- a/src/core/etl/src/Flow/ETL/Function/Trim.php +++ b/src/core/etl/src/Flow/ETL/Function/Trim.php @@ -4,10 +4,14 @@ namespace Flow\ETL\Function; +use Exception; +use Flow\ETL\Column\Column; +use Flow\ETL\Exception\EvaluationException; use Flow\ETL\Exception\InvalidArgumentException; use Flow\ETL\FlowContext; +use Flow\ETL\Function\Evaluation\ResultColumn; use Flow\ETL\Function\Trim\Type; -use Flow\ETL\Row; +use Flow\ETL\Rows; use Flow\Types\Type as FlowType; use function Flow\ETL\DSL\lit; @@ -61,24 +65,37 @@ public function returns(): FlowType return type_string(); } - public function eval(Row $row, FlowContext $context): mixed + public function eval(Rows $rows, FlowContext $context): Column { - $value = (new Parameter($this->value))->asString($row, $context); - $type = (new Parameter($this->type))->asEnum($row, $context, Type::class); - $characters = (new Parameter($this->characters))->asString($row, $context); - - if ($value === null) { - throw new InvalidArgumentException('Trim function requires non-null value'); - } - - if ($type === null || $characters === null) { - throw new InvalidArgumentException('Trim function requires non-null type and characters'); + $values = (new Parameter($this->value))->asStrings($rows, $context); + $types = (new Parameter($this->type))->asEnums($rows, $context, Type::class); + $charactersList = (new Parameter($this->characters))->asStrings($rows, $context); + $results = []; + $i = 0; + + try { + foreach ($values as $i => $value) { + $type = $types[$i]; + $characters = $charactersList[$i]; + + if ($value === null) { + throw new InvalidArgumentException('Trim function requires non-null value'); + } + + if ($type === null || $characters === null) { + throw new InvalidArgumentException('Trim function requires non-null type and characters'); + } + + $results[] = match ($type) { + Type::LEFT => ltrim($value, $characters), + Type::RIGHT => rtrim($value, $characters), + Type::BOTH => trim($value, $characters), + }; + } + } catch (Exception $e) { + throw EvaluationException::at($i, $e); } - return match ($type) { - Type::LEFT => ltrim($value, $characters), - Type::RIGHT => rtrim($value, $characters), - Type::BOTH => trim($value, $characters), - }; + return (new ResultColumn($context->backend()))->of($this, $results); } } diff --git a/src/core/etl/src/Flow/ETL/Function/Truncate.php b/src/core/etl/src/Flow/ETL/Function/Truncate.php index 9658ff6931..cca7ff4337 100644 --- a/src/core/etl/src/Flow/ETL/Function/Truncate.php +++ b/src/core/etl/src/Flow/ETL/Function/Truncate.php @@ -4,9 +4,13 @@ namespace Flow\ETL\Function; +use Exception; +use Flow\ETL\Column\Column; +use Flow\ETL\Exception\EvaluationException; use Flow\ETL\Exception\InvalidArgumentException; use Flow\ETL\FlowContext; -use Flow\ETL\Row; +use Flow\ETL\Function\Evaluation\ResultColumn; +use Flow\ETL\Rows; use Flow\Types\Type; use function Flow\ETL\DSL\lit; @@ -56,20 +60,35 @@ public function returns(): Type return type_string(); } - public function eval(Row $row, FlowContext $context): ?string + public function eval(Rows $rows, FlowContext $context): Column { - $value = (new Parameter($this->value))->asString($row, $context); - $length = (new Parameter($this->length))->asInt($row, $context); - $ellipsis = (new Parameter($this->ellipsis))->asString($row, $context); + $values = (new Parameter($this->value))->asStrings($rows, $context); + $lengths = (new Parameter($this->length))->asInts($rows, $context); + $ellipsisList = (new Parameter($this->ellipsis))->asStrings($rows, $context); + $results = []; + $i = 0; - if ($value === null) { - throw new InvalidArgumentException('Truncate function requires non-null value'); - } + try { + foreach ($values as $i => $value) { + $length = $lengths[$i]; + $ellipsis = $ellipsisList[$i]; + + if ($value === null) { + throw new InvalidArgumentException('Truncate function requires non-null value'); + } + + if ($length === null) { + $results[] = $value; + + continue; + } - if ($length === null) { - return $value; + $results[] = s($value)->truncate($length, $ellipsis ?? '...')->toString(); + } + } catch (Exception $e) { + throw EvaluationException::at($i, $e); } - return s($value)->truncate($length, $ellipsis ?? '...')->toString(); + return (new ResultColumn($context->backend()))->of($this, $results); } } diff --git a/src/core/etl/src/Flow/ETL/Function/Ulid.php b/src/core/etl/src/Flow/ETL/Function/Ulid.php index d661a56a43..7a4a4c1e12 100644 --- a/src/core/etl/src/Flow/ETL/Function/Ulid.php +++ b/src/core/etl/src/Flow/ETL/Function/Ulid.php @@ -4,10 +4,14 @@ namespace Flow\ETL\Function; +use Exception; +use Flow\ETL\Column\Column; +use Flow\ETL\Exception\EvaluationException; use Flow\ETL\Exception\InvalidArgumentException; use Flow\ETL\Exception\RuntimeException; use Flow\ETL\FlowContext; -use Flow\ETL\Row; +use Flow\ETL\Function\Evaluation\ResultColumn; +use Flow\ETL\Rows; use Flow\Types\Type; use InvalidArgumentException as BaseInvalidArgumentException; use Symfony\Component\Uid\Ulid as SymfonyUlid; @@ -67,19 +71,37 @@ public function returns(): Type return type_string(); } - public function eval(Row $row, FlowContext $context): string + public function eval(Rows $rows, FlowContext $context): Column { - $param = $this->ref === null ? null : (new Parameter($this->ref))->asString($row, $context); - - if (null !== $param) { - try { - // A ULID is a string column - the object form cannot be typed as one (L6). - return SymfonyUlid::fromString($param)->toBase32(); - } catch (BaseInvalidArgumentException $e) { - throw new InvalidArgumentException('Ulid requires valid ULID string: ' . $e->getMessage(), 0, $e); + $refs = (new Parameter($this->ref))->asStrings($rows, $context); + $results = []; + $i = 0; + + try { + foreach ($refs as $i => $ref) { + $param = $this->ref === null ? null : $ref; + + if (null !== $param) { + try { + // A ULID is a string column - the object form cannot be typed as one. + $results[] = SymfonyUlid::fromString($param)->toBase32(); + + continue; + } catch (BaseInvalidArgumentException $e) { + throw new InvalidArgumentException( + 'Ulid requires valid ULID string: ' . $e->getMessage(), + 0, + $e, + ); + } + } + + $results[] = (new SymfonyUlid())->toBase32(); } + } catch (Exception $e) { + throw EvaluationException::at($i, $e); } - return (new SymfonyUlid())->toBase32(); + return (new ResultColumn($context->backend()))->of($this, $results); } } diff --git a/src/core/etl/src/Flow/ETL/Function/UnicodeLength.php b/src/core/etl/src/Flow/ETL/Function/UnicodeLength.php index 65a5f7eb23..d1dbfd388e 100644 --- a/src/core/etl/src/Flow/ETL/Function/UnicodeLength.php +++ b/src/core/etl/src/Flow/ETL/Function/UnicodeLength.php @@ -4,9 +4,13 @@ namespace Flow\ETL\Function; +use Exception; +use Flow\ETL\Column\Column; +use Flow\ETL\Exception\EvaluationException; use Flow\ETL\Exception\InvalidArgumentException; use Flow\ETL\FlowContext; -use Flow\ETL\Row; +use Flow\ETL\Function\Evaluation\ResultColumn; +use Flow\ETL\Rows; use Flow\Types\Type; use function Flow\ETL\DSL\lit; @@ -49,14 +53,24 @@ public function returns(): Type return type_integer(); } - public function eval(Row $row, FlowContext $context): ?int + public function eval(Rows $rows, FlowContext $context): Column { - $value = (new Parameter($this->value))->asString($row, $context); + $values = (new Parameter($this->value))->asStrings($rows, $context); + $results = []; + $i = 0; - if ($value === null) { - throw new InvalidArgumentException('UnicodeLength function requires non-null value'); + try { + foreach ($values as $i => $value) { + if ($value === null) { + throw new InvalidArgumentException('UnicodeLength function requires non-null value'); + } + + $results[] = u($value)->length(); + } + } catch (Exception $e) { + throw EvaluationException::at($i, $e); } - return u($value)->length(); + return (new ResultColumn($context->backend()))->of($this, $results); } } diff --git a/src/core/etl/src/Flow/ETL/Function/Uuid.php b/src/core/etl/src/Flow/ETL/Function/Uuid.php index 9330d999c8..6c6ce3391a 100644 --- a/src/core/etl/src/Flow/ETL/Function/Uuid.php +++ b/src/core/etl/src/Flow/ETL/Function/Uuid.php @@ -5,10 +5,14 @@ namespace Flow\ETL\Function; use DateTimeInterface; +use Exception; +use Flow\ETL\Column\Column; +use Flow\ETL\Exception\EvaluationException; use Flow\ETL\Exception\InvalidArgumentException; use Flow\ETL\Exception\RuntimeException; use Flow\ETL\FlowContext; -use Flow\ETL\Row; +use Flow\ETL\Function\Evaluation\ResultColumn; +use Flow\ETL\Rows; use Flow\Types\Type; use Flow\Types\Value\Uuid as FlowUuid; use Ramsey\Uuid\Uuid as RamseyUuid; @@ -86,24 +90,39 @@ public static function uuid7(ScalarFunction|DateTimeInterface $value): self return new self('uuid7', $value); } - public function eval(Row $row, FlowContext $context): FlowUuid + public function eval(Rows $rows, FlowContext $context): Column { + $results = []; + if ($this->uuidVersion === 'uuid4') { - return new FlowUuid($this->generateV4()); + for ($i = 0, $count = $rows->count(); $i < $count; $i++) { + $results[] = new FlowUuid($this->generateV4()); + } + + return (new ResultColumn($context->backend()))->of($this, $results); } - $param = (new Parameter($this->value))->as( - $row, + $params = (new Parameter($this->value))->asTypes( + $rows, $context, type_string(), type_instance_of(DateTimeInterface::class), ); - - if (!$param instanceof DateTimeInterface) { - throw new InvalidArgumentException('Uuid uuid7 function requires a DateTimeInterface value'); + $i = 0; + + try { + foreach ($params as $i => $param) { + if (!$param instanceof DateTimeInterface) { + throw new InvalidArgumentException('Uuid uuid7 function requires a DateTimeInterface value'); + } + + $results[] = new FlowUuid($this->generateV7($param)); + } + } catch (Exception $e) { + throw EvaluationException::at($i, $e); } - return new FlowUuid($this->generateV7($param)); + return (new ResultColumn($context->backend()))->of($this, $results); } private function generateV4(): UuidV4|UuidInterface diff --git a/src/core/etl/src/Flow/ETL/Function/When.php b/src/core/etl/src/Flow/ETL/Function/When.php index c39f1968d3..a6fbe6e64b 100644 --- a/src/core/etl/src/Flow/ETL/Function/When.php +++ b/src/core/etl/src/Flow/ETL/Function/When.php @@ -4,8 +4,11 @@ namespace Flow\ETL\Function; +use Flow\ETL\Column\Column; use Flow\ETL\FlowContext; -use Flow\ETL\Row; +use Flow\ETL\Function\Evaluation\ResultColumn; +use Flow\ETL\Function\Evaluation\Selection; +use Flow\ETL\Rows; use Flow\Types\Exception\InvalidTypeException; use Flow\Types\Type; use Flow\Types\Type\Unifier\NullabilityRule; @@ -75,18 +78,45 @@ public function returns(): Type return $this->else === null ? type_optional($result) : $result; } - public function eval(Row $row, FlowContext $context): mixed + public function eval(Rows $rows, FlowContext $context): Column { - $condition = (new Parameter($this->condition))->asBoolean($row, $context); + $then = []; + $else = []; + + foreach ((new Parameter($this->condition))->asBooleans($rows, $context) as $i => $condition) { + if ($condition) { + $then[] = $i; + } else { + $else[] = $i; + } + } + + $values = []; - if ($condition) { - return (new Parameter($this->then))->eval($row, $context); + if ($then !== []) { + $column = (new Selection($then))->evaluate($this->then, $rows, $context); + + // @mago-ignore analysis:mixed-assignment + foreach ($column->values() as $k => $value) { + $values[$then[$k]] = $value; + } } - if ($this->else !== null) { - return (new Parameter($this->else))->eval($row, $context); + if ($else !== [] && $this->else !== null) { + $column = (new Selection($else))->evaluate($this->else, $rows, $context); + + // @mago-ignore analysis:mixed-assignment + foreach ($column->values() as $k => $value) { + $values[$else[$k]] = $value; + } + } + + $results = []; + + for ($i = 0, $count = $rows->count(); $i < $count; $i++) { + $results[] = $values[$i] ?? null; } - return null; + return (new ResultColumn($context->backend()))->of($this, $results); } } diff --git a/src/core/etl/src/Flow/ETL/Function/Wordwrap.php b/src/core/etl/src/Flow/ETL/Function/Wordwrap.php index 8f3f78317f..f189768ac8 100644 --- a/src/core/etl/src/Flow/ETL/Function/Wordwrap.php +++ b/src/core/etl/src/Flow/ETL/Function/Wordwrap.php @@ -4,9 +4,13 @@ namespace Flow\ETL\Function; +use Exception; +use Flow\ETL\Column\Column; +use Flow\ETL\Exception\EvaluationException; use Flow\ETL\Exception\InvalidArgumentException; use Flow\ETL\FlowContext; -use Flow\ETL\Row; +use Flow\ETL\Function\Evaluation\ResultColumn; +use Flow\ETL\Rows; use Flow\Types\Type; use function Flow\ETL\DSL\lit; @@ -60,25 +64,44 @@ public function returns(): Type return type_string(); } - public function eval(Row $row, FlowContext $context): ?string + public function eval(Rows $rows, FlowContext $context): Column { - $value = (new Parameter($this->value))->asString($row, $context); - $width = type_integer()->assert((new Parameter($this->width))->as($row, $context, type_integer())); - $break = (new Parameter($this->break))->asString($row, $context); - $cut = (new Parameter($this->cut))->asBoolean($row, $context) ?? false; + $values = (new Parameter($this->value))->asStrings($rows, $context); + $breaks = (new Parameter($this->break))->asStrings($rows, $context); + $widths = (new Parameter($this->width))->asTypes($rows, $context, type_integer()); + $cuts = (new Parameter($this->cut))->asBooleans($rows, $context); + $results = []; + $i = 0; - if ($value === null) { - throw new InvalidArgumentException('Wordwrap function requires non-null value'); - } + try { + foreach ($values as $i => $value) { + $break = $breaks[$i]; + $width = $widths[$i]; + $cut = $cuts[$i]; - if ($width <= 0) { - return $value; - } + $width = type_integer()->assert($width); + $cut ??= false; + + if ($value === null) { + throw new InvalidArgumentException('Wordwrap function requires non-null value'); + } + + if ($width <= 0) { + $results[] = $value; + + continue; + } + + if ($break === null) { + $break = "\n"; + } - if ($break === null) { - $break = "\n"; + $results[] = s($value)->wordwrap($width, $break, $cut)->toString(); + } + } catch (Exception $e) { + throw EvaluationException::at($i, $e); } - return s($value)->wordwrap($width, $break, $cut)->toString(); + return (new ResultColumn($context->backend()))->of($this, $results); } } diff --git a/src/core/etl/src/Flow/ETL/Function/XPath.php b/src/core/etl/src/Flow/ETL/Function/XPath.php index ca98988d18..a6dd8a61ed 100644 --- a/src/core/etl/src/Flow/ETL/Function/XPath.php +++ b/src/core/etl/src/Flow/ETL/Function/XPath.php @@ -10,9 +10,13 @@ use DOMNode; use DOMNodeList; use DOMXPath; +use Exception; +use Flow\ETL\Column\Column; +use Flow\ETL\Exception\EvaluationException; use Flow\ETL\Exception\InvalidArgumentException; use Flow\ETL\FlowContext; -use Flow\ETL\Row; +use Flow\ETL\Function\Evaluation\ResultColumn; +use Flow\ETL\Rows; use Flow\Types\Type; use function Flow\ETL\DSL\lit; @@ -58,52 +62,63 @@ public function returns(): Type return type_optional(type_list(type_xml_element())); } - /** - * @return null|list - */ - public function eval(Row $row, FlowContext $context): ?array + public function eval(Rows $rows, FlowContext $context): Column { - $value = (new Parameter($this->value))->asInstanceOf($row, $context, DOMNode::class); - $path = (new Parameter($this->path))->asString($row, $context); + $values = (new Parameter($this->value))->asInstancesOf($rows, $context, DOMNode::class); + $paths = (new Parameter($this->path))->asStrings($rows, $context); + $results = []; + $i = 0; - if ($value === null) { - throw new InvalidArgumentException('XPath requires non-null DOMNode value'); - } + try { + foreach ($values as $i => $value) { + $path = $paths[$i]; - if ($path === null) { - throw new InvalidArgumentException('XPath requires non-null path'); - } + if ($value === null) { + throw new InvalidArgumentException('XPath requires non-null DOMNode value'); + } - if (!$value instanceof DOMDocument) { - $dom = $value->ownerDocument ?? new DOMDocument(); - $importedNode = $dom->importNode($value, true); + if ($path === null) { + throw new InvalidArgumentException('XPath requires non-null path'); + } - if ($importedNode !== false && $importedNode->parentNode === null) { - $dom->appendChild($importedNode); - } + if (!$value instanceof DOMDocument) { + $dom = $value->ownerDocument ?? new DOMDocument(); + $importedNode = $dom->importNode($value, true); - $value = $dom; - } + if ($importedNode !== false && $importedNode->parentNode === null) { + $dom->appendChild($importedNode); + } - $xpath = new DOMXPath($value); - /** @var DOMNodeList|false $result */ - $result = $xpath->query($path); + $value = $dom; + } - if ($result === false || $result->length === 0) { - return null; - } + $xpath = new DOMXPath($value); + /** @var DOMNodeList|false $result */ + $result = $xpath->query($path); - $nodes = []; + if ($result === false || $result->length === 0) { + $results[] = null; - foreach ($result as $node) { - // text(), attribute and comment queries yield nodes the declared list cannot hold. - if (!$node instanceof DOMElement) { - continue; - } + continue; + } + + $nodes = []; - $nodes[] = $node; + foreach ($result as $node) { + // text(), attribute and comment queries yield nodes the declared list cannot hold. + if (!$node instanceof DOMElement) { + continue; + } + + $nodes[] = $node; + } + + $results[] = $nodes === [] ? null : $nodes; + } + } catch (Exception $e) { + throw EvaluationException::at($i, $e); } - return $nodes === [] ? null : $nodes; + return (new ResultColumn($context->backend()))->of($this, $results); } } diff --git a/src/core/etl/src/Flow/ETL/GroupBy.php b/src/core/etl/src/Flow/ETL/GroupBy.php index 9fa6ab4d08..8644710e45 100644 --- a/src/core/etl/src/Flow/ETL/GroupBy.php +++ b/src/core/etl/src/Flow/ETL/GroupBy.php @@ -4,6 +4,7 @@ namespace Flow\ETL; +use Flow\ETL\Bucketing\KeyValues; use Flow\ETL\Exception\InvalidArgumentException; use Flow\ETL\Exception\RuntimeException; use Flow\ETL\Exception\SchemaDefinitionNotFoundException; @@ -15,9 +16,12 @@ use Flow\ETL\Row\Reference; use Flow\ETL\Row\References; +use function array_key_exists; +use function array_key_last; use function array_unique; use function count; use function Flow\ETL\DSL\definition_from_type; +use function serialize; final class GroupBy { @@ -48,7 +52,10 @@ public function aggregate(AggregatingFunction ...$aggregator): void $this->aggregations = new Aggregators(...$aggregator); } - public function aggregatedRow(GroupKey $key, Aggregators $aggregators, Schema $output): Row + /** + * @return array + */ + public function aggregatedValues(GroupKey $key, Aggregators $aggregators, Schema $output): array { $values = []; @@ -64,7 +71,7 @@ public function aggregatedRow(GroupKey $key, Aggregators $aggregators, Schema $o $values[$aggregator->outputName()] = $value === null ? null : $definition->type()->cast($value); } - return new Row($values); + return $values; } /** @@ -117,17 +124,59 @@ public function outputSchema(Schema $input, Aggregators $bound): Schema return new Schema(...$definitions); } - public function keyValues(Row $row, Schema $input): GroupKey + /** + * @return list one per row of $rows + */ + public function keys(Rows $rows, Schema $input): array { - $values = []; + if ($rows->isEmpty()) { + return []; + } - foreach ($this->refs as $ref) { - // absent under a nullable declaration is a legitimate null; absent under NOT NULL is a - // row-shape violation, and Row::get() already names it and lists the available columns. - $values[$ref->name()] = !$row->has($ref) && $input->get($ref)->isNullable() ? null : $row->get($ref); + $present = []; + $absent = []; + + foreach ($this->refs->all() as $ref) { + $absent[] = $rows->schema()->findDefinition($ref->base()) === null; + + if (!$absent[array_key_last($absent)]) { + $present[] = $ref; + } + } + + $keys = []; + $built = []; + + // one key per distinct identity over every ref in order (an absent ref is a null slot), its values read from + // the first row that carries it + foreach ((new KeyValues($present))->of($rows) as $i => $presentValues) { + $identity = []; + $position = 0; + + foreach ($absent as $isAbsent) { + $identity[] = $isAbsent ? null : $presentValues[$position++]; + } + + $string = serialize($identity); + + if (!array_key_exists($string, $built)) { + $values = []; + + foreach ($this->refs->all() as $index => $ref) { + // absent under a nullable declaration is a legitimate null; absent under NOT NULL is a + // row-shape violation, and Rows::column() names it and lists the available columns. + $values[$ref->name()] = $absent[$index] && $input->get($ref)->isNullable() + ? null + : $rows->column($ref->base())->value($i); + } + + $built[$string] = new GroupKey($values, $string); + } + + $keys[] = $built[$string]; } - return new GroupKey($values); + return $keys; } public function pivot(Reference $ref, DeclaredPivotValues $values): void diff --git a/src/core/etl/src/Flow/ETL/GroupBy/AggregatedGroups.php b/src/core/etl/src/Flow/ETL/GroupBy/AggregatedGroups.php index 10161f7178..92e2f5284e 100644 --- a/src/core/etl/src/Flow/ETL/GroupBy/AggregatedGroups.php +++ b/src/core/etl/src/Flow/ETL/GroupBy/AggregatedGroups.php @@ -6,10 +6,13 @@ use Flow\ETL\FlowContext; use Flow\ETL\GroupBy; -use Flow\ETL\Row\RowsBuffer; use Flow\ETL\Rows; use Generator; +use function array_chunk; +use function array_key_exists; +use function count; + final class AggregatedGroups { /** @@ -22,13 +25,44 @@ public function __construct( private readonly GroupByShape $shape, ) {} + /** + * @return array by the key's string form + */ + public function groups(): array + { + return $this->groups; + } + public function accumulate(Rows $batch, FlowContext $context): void { - foreach ($batch as $row) { - $key = $this->groupBy->keyValues($row, $this->shape->input); - $group = $this->groups[(string) $key] ??= new Group($key, $this->shape->aggregators->cloned()); - $group->aggregators->aggregate($row, $context); + /** @var array> $indices */ + $indices = []; + + /** @var array $keys */ + $keys = []; + + foreach ($this->groupBy->keys($batch, $this->shape->input) as $i => $key) { + $index = (string) $key; + $indices[$index][] = $i; + $keys[$index] ??= $key; + } + + foreach ($indices as $index => $rows) { + $group = $this->groups[$index] ??= new Group($keys[$index], $this->shape->aggregators->cloned()); + $group->aggregators->aggregate($batch, $rows, $context); + } + } + + /** + * $group aggregated rows that come before every row this instance aggregated under the same key. + */ + public function absorbEarlier(string $key, Group $group, FlowContext $context): void + { + if (array_key_exists($key, $this->groups)) { + $group->aggregators->merge($this->groups[$key]->aggregators, $context); } + + $this->groups[$key] = $group; } /** @@ -36,20 +70,44 @@ public function accumulate(Rows $batch, FlowContext $context): void * * @return Generator */ - public function flush(int $batchSize): Generator + public function flush(int $batchSize, FlowContext $context): Generator + { + foreach (array_chunk($this->groups, $batchSize) as $groups) { + yield $this->rows($groups, $context); + } + } + + /** + * @param list $groups + */ + public function rows(array $groups, FlowContext $context): Rows { - $buffer = new RowsBuffer($this->shape->output, $batchSize); + $output = $this->shape->output; + $values = []; - foreach ($this->groups as $group) { - $aggregated = $this->groupBy->aggregatedRow($group->key, $group->aggregators, $this->shape->output); + foreach ($output->definitions() as $name => $_) { + $values[$name] = []; + } - if (null !== ($batch = $buffer->add($aggregated))) { - yield $batch; + foreach ($groups as $group) { + // @mago-ignore analysis:mixed-assignment + foreach ($group->key as $name => $value) { + $values[$output->get($name)->entry()->name()][] = $value; + } + + foreach ($group->aggregators as $aggregator) { + $values[$aggregator->outputName()][] = $aggregator->value(); } } - if (null !== ($batch = $buffer->flush())) { - yield $batch; + $columns = []; + + foreach ($output->definitions() as $name => $definition) { + $builder = $context->backend()->builder($definition); + $builder->appendMany($values[$name]); + $columns[$name] = $builder->finish(); } + + return Rows::fromColumns($output, $columns, count($groups)); } } diff --git a/src/core/etl/src/Flow/ETL/GroupBy/Aggregators.php b/src/core/etl/src/Flow/ETL/GroupBy/Aggregators.php index e6c787fcdb..7262fdf484 100644 --- a/src/core/etl/src/Flow/ETL/GroupBy/Aggregators.php +++ b/src/core/etl/src/Flow/ETL/GroupBy/Aggregators.php @@ -11,7 +11,8 @@ use Flow\ETL\Function\AggregatingFunction; use Flow\ETL\Function\ExpandingFunctions; use Flow\ETL\Function\ReferenceResolver; -use Flow\ETL\Row; +use Flow\ETL\Row\Reference; +use Flow\ETL\Rows; use Flow\ETL\Schema; use IteratorAggregate; use Traversable; @@ -35,10 +36,13 @@ public function __construct(AggregatingFunction ...$aggregators) $this->aggregators = $aggregators; } - public function aggregate(Row $row, FlowContext $context): void + /** + * @param list $indices + */ + public function aggregate(Rows $rows, array $indices, FlowContext $context): void { foreach ($this->aggregators as $aggregator) { - $aggregator->aggregate($row, $context); + $aggregator->aggregate($rows, $indices, $context); } } @@ -55,6 +59,16 @@ public function count(): int return count($this->aggregators); } + /** + * Folds $other's partial states into these, pairwise; $other aggregated rows that come after these. + */ + public function merge(self $other, FlowContext $context): void + { + foreach ($this->aggregators as $position => $aggregator) { + $aggregator->merge($other->aggregators[$position], $context); + } + } + public function first(): AggregatingFunction { if ($this->aggregators === []) { @@ -90,7 +104,7 @@ public function resolved(Schema $schema): self } /** - * @return null|list union of references read by all aggregators, or null when any + * @return null|list union of references read by all aggregators, or null when any * aggregator cannot enumerate them (disables spill column pruning) */ public function references(): ?array diff --git a/src/core/etl/src/Flow/ETL/GroupBy/BucketAggregation.php b/src/core/etl/src/Flow/ETL/GroupBy/BucketAggregation.php deleted file mode 100644 index 8240c13744..0000000000 --- a/src/core/etl/src/Flow/ETL/GroupBy/BucketAggregation.php +++ /dev/null @@ -1,75 +0,0 @@ - $batchSize - */ - public function __construct( - private int $batchSize = 1000, - ) { - // @mago-ignore analysis:invalid-operand - // @mago-ignore analysis:impossible-condition,redundant-comparison - if ($this->batchSize < 1) { - throw new InvalidArgumentException('Batch size must be greater than 0, given: ' . $this->batchSize); - } - } - - /** - * Reached only when the plan could not bind, so the shape comes from the first batch that carries rows. - * - * @param Generator $rows - * - * @return Generator - */ - public function aggregate(Generator $rows, FlowContext $context, GroupBy $groupBy): Generator - { - $groups = null; - - foreach ($rows as $batch) { - if ($groups === null) { - if (!$batch->count()) { - continue; - } - - $groups = new AggregatedGroups($groupBy, GroupByShape::of($groupBy, $batch->schema())); - } - - $groups->accumulate($batch, $context); - } - - if ($groups !== null) { - yield from $groups->flush($this->batchSize); - } - } - - /** - * @param Generator $rows - * - * @return Generator - */ - public function aggregateBound( - Generator $rows, - FlowContext $context, - GroupBy $groupBy, - GroupByShape $shape, - ): Generator { - $groups = new AggregatedGroups($groupBy, $shape); - - foreach ($rows as $batch) { - $groups->accumulate($batch, $context); - } - - yield from $groups->flush($this->batchSize); - } -} diff --git a/src/core/etl/src/Flow/ETL/GroupBy/DiscoveredPivotValues.php b/src/core/etl/src/Flow/ETL/GroupBy/DiscoveredPivotValues.php index 995d9857c1..77f7f61c3b 100644 --- a/src/core/etl/src/Flow/ETL/GroupBy/DiscoveredPivotValues.php +++ b/src/core/etl/src/Flow/ETL/GroupBy/DiscoveredPivotValues.php @@ -10,6 +10,7 @@ use Flow\ETL\Extractor\Repeatability; use Flow\ETL\Plan\Trigger; use Flow\ETL\Row\Reference; +use Flow\ETL\Sort\RowOrder; use function array_values; use function count; @@ -37,14 +38,20 @@ public function resolve(DataFrame $source, Reference $pivot): DeclaredPivotValue } $distinct = []; + $type = null; // the frame, not its source: the pivot column may be produced by an earlier step, and an // earlier filter decides which values exist at all foreach ($source->get() as $batch) { - foreach ($batch as $row) { - /** @var mixed $value */ - $value = $row->get($pivot); + if ($batch->isEmpty()) { + continue; + } + + $column = $batch->column($pivot->base()); + $type ??= $column->type(); + // @mago-ignore analysis:mixed-assignment + foreach ($column->values() as $value) { if ($value === null) { continue; } @@ -71,8 +78,8 @@ public function resolve(DataFrame $source, Reference $pivot): DeclaredPivotValue } $values = array_values($distinct); - // sorted so the discovered column order is deterministic across runs - sort($values); + // sorted so the discovered column order is deterministic across runs, in the order sortBy() uses + sort($values, RowOrder::flag($type ?? type_string())); return new DeclaredPivotValues(...$values); } diff --git a/src/core/etl/src/Flow/ETL/GroupBy/GroupBySteps.php b/src/core/etl/src/Flow/ETL/GroupBy/GroupBySteps.php index 8913b23e28..fdeb7a7421 100644 --- a/src/core/etl/src/Flow/ETL/GroupBy/GroupBySteps.php +++ b/src/core/etl/src/Flow/ETL/GroupBy/GroupBySteps.php @@ -7,11 +7,11 @@ use Flow\ETL\Bucketing\Buckets; use Flow\ETL\Bucketing\HashBucketing; use Flow\ETL\Bucketing\NativeHasher; +use Flow\ETL\Bucketing\Storage\SpillingBuckets; use Flow\ETL\Config; use Flow\ETL\Config\Grouping\GroupByAlgorithmBuilder; use Flow\ETL\GroupBy; use Flow\ETL\Processor; -use Flow\ETL\Processor\BucketingProcessor; use Flow\ETL\Processor\GroupByAggregationProcessor; use Flow\ETL\Processor\PivotProcessor; use Flow\ETL\Transformer; @@ -29,7 +29,8 @@ */ public static function of(GroupBy $groupBy, Config $config, ?GroupByAlgorithmBuilder $algorithm = null): array { - $grouping = $algorithm?->build($config->cache->localFilesystemCacheDir) ?? $config->grouping; + $grouping = + $algorithm?->build($config->cache->localFilesystemCacheDir, $config->backend()) ?? $config->grouping; if ($groupBy->isPivot()) { return [new PivotProcessor($groupBy, $grouping->bucketing->batchSize)]; } @@ -47,8 +48,13 @@ public static function of(GroupBy $groupBy, Config $config, ?GroupByAlgorithmBui $steps[] = new PruneEntriesTransformer(...array_values($pruned)); } - $buckets = new Buckets($grouping->bucketing->storage); - $steps[] = new BucketingProcessor( + $buckets = new Buckets(SpillingBuckets::around( + $grouping->bucketing, + $grouping->memoryLimit, + $config->backend(), + )); + $steps[] = new GroupByAggregationProcessor( + $groupBy, new HashBucketing( $groupBy->references(), $grouping->bucketing->bucketsCount, @@ -57,8 +63,9 @@ public static function of(GroupBy $groupBy, Config $config, ?GroupByAlgorithmBui 'group-by', ), $buckets, + $grouping->memoryLimit, + $grouping->bucketing->batchSize, ); - $steps[] = new GroupByAggregationProcessor($groupBy, $buckets, $grouping->bucketing->batchSize); return $steps; } diff --git a/src/core/etl/src/Flow/ETL/GroupBy/GroupKey.php b/src/core/etl/src/Flow/ETL/GroupBy/GroupKey.php index f313fd1362..76015ba396 100644 --- a/src/core/etl/src/Flow/ETL/GroupBy/GroupKey.php +++ b/src/core/etl/src/Flow/ETL/GroupBy/GroupKey.php @@ -9,8 +9,6 @@ use Stringable; use Traversable; -use function serialize; - /** * @implements IteratorAggregate */ @@ -18,14 +16,16 @@ { /** * @param array $values ref name => key value + * @param string $identity what tells two keys apart: the serialized equality forms of the key, in ref order */ public function __construct( private array $values, + private string $identity, ) {} public function __toString(): string { - return serialize($this->values); + return $this->identity; } public function getIterator(): Traversable diff --git a/src/core/etl/src/Flow/ETL/GroupBy/PivotedTable.php b/src/core/etl/src/Flow/ETL/GroupBy/PivotedTable.php index 4ee00dfb0c..4640403fd5 100644 --- a/src/core/etl/src/Flow/ETL/GroupBy/PivotedTable.php +++ b/src/core/etl/src/Flow/ETL/GroupBy/PivotedTable.php @@ -36,22 +36,35 @@ public function __construct( public function accumulate(Rows $batch, FlowContext $context): void { - foreach ($batch as $row) { - $key = $this->groupBy->keyValues($row, $this->shape->input); + $keys = $this->groupBy->keys($batch, $this->shape->input); + + if ($keys === []) { + return; + } + + $pivotValues = $batch->column($this->shape->pivot->column->base())->values(); + + /** @var array>> $indices */ + $indices = []; + + foreach ($keys as $i => $key) { $index = (string) $key; $this->keys[$index] ??= $key; $this->accumulators[$index] ??= []; - $pivotValue = $row->get($this->shape->pivot->column); - - if ($pivotValue === null) { + if ($pivotValues[$i] === null) { continue; } - $column = type_union(type_string(), type_integer())->assert($pivotValue); - $accumulator = $this->accumulators[$index][$column] ??= clone $this->shape->aggregation; - $accumulator->aggregate($row, $context); + $indices[$index][type_union(type_string(), type_integer())->assert($pivotValues[$i])][] = $i; + } + + foreach ($indices as $index => $columns) { + foreach ($columns as $column => $rows) { + $accumulator = $this->accumulators[$index][$column] ??= clone $this->shape->aggregation; + $accumulator->aggregate($batch, $rows, $context); + } } } @@ -88,13 +101,13 @@ public function flush(int $batchSize, FlowContext $context): Generator $buffer[] = $row; if (count($buffer) >= $batchSize) { - yield array_to_rows($buffer, $this->shape->output, $context->hydrator()); + yield array_to_rows($buffer, $this->shape->output, $context->backend()); $buffer = []; } } if ($buffer !== []) { - yield array_to_rows($buffer, $this->shape->output, $context->hydrator()); + yield array_to_rows($buffer, $this->shape->output, $context->backend()); } } } diff --git a/src/core/etl/src/Flow/ETL/Join/Comparison.php b/src/core/etl/src/Flow/ETL/Join/Comparison.php index 268162d645..b297b956d1 100644 --- a/src/core/etl/src/Flow/ETL/Join/Comparison.php +++ b/src/core/etl/src/Flow/ETL/Join/Comparison.php @@ -4,12 +4,17 @@ namespace Flow\ETL\Join; -use Flow\ETL\Row; use Flow\ETL\Row\Reference; +use Flow\ETL\Rows; interface Comparison { - public function compare(Row $left, Row $right): bool; + /** + * $left and $right hold the same number of rows; pair i is (left row i, right row i). + * + * @return list + */ + public function compare(Rows $left, Rows $right): array; /** * @return array diff --git a/src/core/etl/src/Flow/ETL/Join/Comparison/All.php b/src/core/etl/src/Flow/ETL/Join/Comparison/All.php index 05e7224f60..8f5fa34f09 100644 --- a/src/core/etl/src/Flow/ETL/Join/Comparison/All.php +++ b/src/core/etl/src/Flow/ETL/Join/Comparison/All.php @@ -5,8 +5,8 @@ namespace Flow\ETL\Join\Comparison; use Flow\ETL\Join\Comparison; -use Flow\ETL\Row; use Flow\ETL\Row\Reference; +use Flow\ETL\Rows; use function array_merge; use function array_unique; @@ -23,15 +23,25 @@ public function __construct(Comparison $comparison, Comparison ...$comparisons) $this->comparisons = array_merge([$comparison], $comparisons); } - public function compare(Row $left, Row $right): bool + public function compare(Rows $left, Rows $right): array { + $result = null; + foreach ($this->comparisons as $comparison) { - if (!$comparison->compare($left, $right)) { - return false; + $compared = $comparison->compare($left, $right); + + if ($result === null) { + $result = $compared; + + continue; + } + + foreach ($compared as $i => $met) { + $result[$i] = $result[$i] && $met; } } - return true; + return $result ?? []; } /** diff --git a/src/core/etl/src/Flow/ETL/Join/Comparison/Any.php b/src/core/etl/src/Flow/ETL/Join/Comparison/Any.php index 7584490e84..2d43278819 100644 --- a/src/core/etl/src/Flow/ETL/Join/Comparison/Any.php +++ b/src/core/etl/src/Flow/ETL/Join/Comparison/Any.php @@ -5,8 +5,8 @@ namespace Flow\ETL\Join\Comparison; use Flow\ETL\Join\Comparison; -use Flow\ETL\Row; use Flow\ETL\Row\Reference; +use Flow\ETL\Rows; use function array_merge; use function array_unique; @@ -23,15 +23,25 @@ public function __construct(Comparison $comparison, Comparison ...$comparisons) $this->comparisons = array_merge([$comparison], $comparisons); } - public function compare(Row $left, Row $right): bool + public function compare(Rows $left, Rows $right): array { + $result = null; + foreach ($this->comparisons as $comparison) { - if ($comparison->compare($left, $right)) { - return true; + $compared = $comparison->compare($left, $right); + + if ($result === null) { + $result = $compared; + + continue; + } + + foreach ($compared as $i => $met) { + $result[$i] = $result[$i] || $met; } } - return false; + return $result ?? []; } /** diff --git a/src/core/etl/src/Flow/ETL/Join/Comparison/Equal.php b/src/core/etl/src/Flow/ETL/Join/Comparison/Equal.php index 6bce1b61ea..fdc56e3425 100644 --- a/src/core/etl/src/Flow/ETL/Join/Comparison/Equal.php +++ b/src/core/etl/src/Flow/ETL/Join/Comparison/Equal.php @@ -5,9 +5,9 @@ namespace Flow\ETL\Join\Comparison; use Flow\ETL\Join\Comparison; -use Flow\ETL\Row; use Flow\ETL\Row\Reference; use Flow\ETL\Row\UnresolvedReference; +use Flow\ETL\Rows; use Flow\Types\Value\Uuid; use function is_array; @@ -23,41 +23,30 @@ public function __construct( private string|Reference $entryRight, ) {} - public function compare(Row $left, Row $right): bool + public function compare(Rows $left, Rows $right): array { - $leftValue = $left->get($this->entryLeft); - $rightValue = $right->get($this->entryRight); - - // SQL semantics - null never equals anything, including null - if ($leftValue === null || $rightValue === null) { - return false; - } - - if (is_numeric($leftValue) && is_numeric($rightValue)) { - return (float) $leftValue == (float) $rightValue; - } - - if (is_string($leftValue) && is_string($rightValue)) { - return $leftValue === $rightValue; - } - - if (is_bool($leftValue) && is_bool($rightValue)) { - return $leftValue === $rightValue; - } - - if (is_array($leftValue) && is_array($rightValue)) { - return $leftValue === $rightValue; - } - - if (is_object($leftValue) && is_object($rightValue)) { - if ($leftValue instanceof Uuid && $rightValue instanceof Uuid) { - return $leftValue->isEqual($rightValue); - } - - return $leftValue == $rightValue; + $rightValues = $right->column($this->right()[0]->base())->values(); + $result = []; + + // @mago-ignore analysis:mixed-assignment + foreach ($left->column($this->left()[0]->base())->values() as $i => $leftValue) { + // @mago-ignore analysis:mixed-assignment + $rightValue = $rightValues[$i]; + + $result[] = match (true) { + // SQL semantics - null never equals anything, including null + $leftValue === null || $rightValue === null => false, + is_numeric($leftValue) && is_numeric($rightValue) => (float) $leftValue == (float) $rightValue, + is_string($leftValue) && is_string($rightValue) => $leftValue === $rightValue, + is_bool($leftValue) && is_bool($rightValue) => $leftValue === $rightValue, + is_array($leftValue) && is_array($rightValue) => $leftValue === $rightValue, + $leftValue instanceof Uuid && $rightValue instanceof Uuid => $leftValue->isEqual($rightValue), + is_object($leftValue) && is_object($rightValue) => $leftValue == $rightValue, + default => false, + }; } - return false; + return $result; } /** diff --git a/src/core/etl/src/Flow/ETL/Join/Comparison/Identical.php b/src/core/etl/src/Flow/ETL/Join/Comparison/Identical.php index 177105bd1a..576c0e650f 100644 --- a/src/core/etl/src/Flow/ETL/Join/Comparison/Identical.php +++ b/src/core/etl/src/Flow/ETL/Join/Comparison/Identical.php @@ -4,11 +4,13 @@ namespace Flow\ETL\Join\Comparison; +use Flow\ETL\Column\ComparableValues; use Flow\ETL\Join\Comparison; -use Flow\ETL\Row; use Flow\ETL\Row\Reference; use Flow\ETL\Row\UnresolvedReference; +use Flow\ETL\Rows; +use function Flow\Types\DSL\type_bare; use function is_string; final readonly class Identical implements Comparison @@ -18,17 +20,22 @@ public function __construct( private string|Reference $entryRight, ) {} - public function compare(Row $left, Row $right): bool + public function compare(Rows $left, Rows $right): array { - $leftValue = $left->get($this->entryLeft); - $rightValue = $right->get($this->entryRight); - - // SQL semantics - null never equals anything, including null - if ($leftValue === null || $rightValue === null) { - return false; + $leftColumn = $left->column($this->left()[0]->base()); + $rightColumn = $right->column($this->right()[0]->base()); + $comparable = new ComparableValues(); + $sameType = type_bare($leftColumn->type())::class === type_bare($rightColumn->type())::class; + $rightValues = $sameType ? $comparable->equality($rightColumn) : $rightColumn->values(); + $result = []; + + // @mago-ignore analysis:mixed-assignment + foreach ($sameType ? $comparable->equality($leftColumn) : $leftColumn->values() as $i => $leftValue) { + // SQL semantics - null never equals anything, including null + $result[] = $leftValue !== null && $rightValues[$i] !== null && $leftValue === $rightValues[$i]; } - return $leftValue === $rightValue; + return $result; } /** diff --git a/src/core/etl/src/Flow/ETL/Join/Expression.php b/src/core/etl/src/Flow/ETL/Join/Expression.php index f7ac646e64..50d5106543 100644 --- a/src/core/etl/src/Flow/ETL/Join/Expression.php +++ b/src/core/etl/src/Flow/ETL/Join/Expression.php @@ -7,8 +7,8 @@ use Flow\ETL\Exception\RuntimeException; use Flow\ETL\Join\Comparison\All; use Flow\ETL\Join\Comparison\Equal; -use Flow\ETL\Row; use Flow\ETL\Row\Reference; +use Flow\ETL\Rows; use function array_slice; use function gettype; @@ -81,7 +81,10 @@ public function left(): array return $this->comparison->left(); } - public function meet(Row $left, Row $right): bool + /** + * @return list pair i is (left row i, right row i) + */ + public function meet(Rows $left, Rows $right): array { return $this->comparison->compare($left, $right); } diff --git a/src/core/etl/src/Flow/ETL/Join/HashJoin/HashTable.php b/src/core/etl/src/Flow/ETL/Join/HashJoin/HashTable.php index 537973a189..4014dd7786 100644 --- a/src/core/etl/src/Flow/ETL/Join/HashJoin/HashTable.php +++ b/src/core/etl/src/Flow/ETL/Join/HashJoin/HashTable.php @@ -4,12 +4,8 @@ namespace Flow\ETL\Join\HashJoin; -use Flow\ETL\Row; -use Generator; - use function array_key_exists; use function array_keys; -use function count; final class HashTable { @@ -18,10 +14,7 @@ final class HashTable */ private array $buckets = []; - /** - * @var list - */ - private array $rows = []; + private int $count = 0; /** * @var array @@ -32,11 +25,10 @@ public function __construct( private readonly bool $trackUnmatched = false, ) {} - public function add(string $hash, Row $row): void + public function add(string $hash, int $index): void { - $index = count($this->rows); - $this->rows[] = $row; $this->buckets[$hash][] = $index; + $this->count++; if ($this->trackUnmatched) { $this->unmatched[$index] = true; @@ -44,26 +36,16 @@ public function add(string $hash, Row $row): void } /** - * @return array + * @return list the build-side rows under $hash, in the order they were added */ public function candidatesFor(string $hash): array { - if (!array_key_exists($hash, $this->buckets)) { - return []; - } - - $candidates = []; - - foreach ($this->buckets[$hash] as $index) { - $candidates[$index] = $this->rows[$index]; - } - - return $candidates; + return array_key_exists($hash, $this->buckets) ? $this->buckets[$hash] : []; } public function count(): int { - return count($this->rows); + return $this->count; } public function matched(int $index): void @@ -72,12 +54,10 @@ public function matched(int $index): void } /** - * @return Generator + * @return list the build-side rows never matched, in the order they were added */ - public function unmatchedRows(): Generator + public function unmatched(): array { - foreach (array_keys($this->unmatched) as $index) { - yield $this->rows[$index]; - } + return array_keys($this->unmatched); } } diff --git a/src/core/etl/src/Flow/ETL/Join/HashJoin/JoinSide.php b/src/core/etl/src/Flow/ETL/Join/HashJoin/JoinSide.php index df7136da47..07bc92c01c 100644 --- a/src/core/etl/src/Flow/ETL/Join/HashJoin/JoinSide.php +++ b/src/core/etl/src/Flow/ETL/Join/HashJoin/JoinSide.php @@ -4,7 +4,6 @@ namespace Flow\ETL\Join\HashJoin; -use Flow\ETL\Row; use Flow\ETL\Rows; use Flow\ETL\Schema; use Generator; @@ -13,19 +12,19 @@ { /** * @param Generator $rows - * @param null|Row $nullRow pads a row on the opposite side that found no match + * @param null|Rows $nullRow the one-row batch that pads a row on the opposite side that found no match * @param null|Schema $schema the whole side's shape; a bucket pair only streams a slice of it */ private function __construct( public Generator $rows, - public ?Row $nullRow = null, + public ?Rows $nullRow = null, public ?Schema $schema = null, ) {} /** * @param Generator $rows */ - public static function of(Generator $rows, ?Row $nullRow = null, ?Schema $schema = null): self + public static function of(Generator $rows, ?Rows $nullRow = null, ?Schema $schema = null): self { return new self($rows, $nullRow, $schema); } diff --git a/src/core/etl/src/Flow/ETL/Join/HashJoin/Joiner.php b/src/core/etl/src/Flow/ETL/Join/HashJoin/Joiner.php index ffa525b55e..854069e729 100644 --- a/src/core/etl/src/Flow/ETL/Join/HashJoin/Joiner.php +++ b/src/core/etl/src/Flow/ETL/Join/HashJoin/Joiner.php @@ -8,18 +8,26 @@ use Flow\ETL\Bucketing\KeyValues; use Flow\ETL\Bucketing\NativeHasher; use Flow\ETL\Bucketing\SingleBucketHasher; +use Flow\ETL\Column\Backend; use Flow\ETL\Exception\DuplicatedEntriesException; use Flow\ETL\Exception\InvalidArgumentException; use Flow\ETL\Join\Expression; use Flow\ETL\Join\Join; use Flow\ETL\Join\JoinSchema; use Flow\ETL\Join\JoinShape; -use Flow\ETL\Row; -use Flow\ETL\Row\RowsBuffer; +use Flow\ETL\Row\Reference; use Flow\ETL\Rows; use Flow\ETL\Schema; use Generator; +use function array_chunk; +use function array_fill; +use function array_map; +use function array_slice; +use function array_unique; +use function array_values; +use function count; + final class Joiner { private readonly Hasher $hasher; @@ -35,11 +43,12 @@ final class Joiner private readonly KeyValues $rightValues; /** - * @param int<1, max> $batchSize - size of output batches emitted for unmatched right side rows + * @param int<1, max> $batchSize - size of output batches emitted for unmatched build side rows */ public function __construct( private readonly Expression $expression, private readonly Join $type, + private readonly Backend $backend, private readonly int $batchSize = 1000, ) { // @mago-ignore analysis:invalid-operand @@ -75,98 +84,80 @@ public function join(JoinSide $left, JoinSide $right, bool $buildLeft = false): return; } - $nullLeftRow = $left->nullRow; - $nullRightRow = $right->nullRow; - $leftSideSchema = $left->schema; - $rightSideSchema = $right->schema; - - $hashTable = new HashTable(trackUnmatched: $this->type === Join::right); - $rightSchema = $rightSideSchema; - - foreach ($right->rows as $batch) { - $rightSchema ??= $batch->schema(); - $hashes = $this->hasher->hash($this->rightValues->of($batch)); - - foreach ($batch as $i => $row) { - $hashTable->add($hashes[$i], $row); - } - } - - $rightSchema ??= new Schema(); + [$build, $rightSchema] = $this->build($right); + $table = $this->table($build, $this->rightValues, $this->type === Join::right); - if ($this->type === Join::left) { - $nullRightRow ??= (new NullRowBuilder($rightSchema))->row(); - } + // the null row sits at $build->count(), so a gather pads an unmatched left row with it + $probe = $this->type === Join::left ? $this->padded($build, $right->nullRow, $rightSchema) : $build; + $nullIndex = $build->count(); - $leftSchema = $leftSideSchema; + $leftSchema = $left->schema; $outputSchema = null; foreach ($left->rows as $batch) { $leftSchema ??= $batch->schema(); $outputSchema ??= $this->joinSchema->of($this->type, $leftSchema, $rightSchema); - $hashes = $this->hasher->hash($this->leftValues->of($batch)); - $joined = []; + [$candidatesLeft, $candidatesRight] = $this->candidates($table, $this->leftValues->of($batch)); + $met = $this->meet($batch, $candidatesLeft, $build, $candidatesRight); + + $outputLeft = []; + $outputRight = []; + $pair = 0; + $pairs = count($candidatesLeft); - foreach ($batch as $i => $leftRow) { + for ($i = 0, $rows = $batch->count(); $i < $rows; $i++) { $matched = false; - foreach ($hashTable->candidatesFor($hashes[$i]) as $index => $rightRow) { - if (!$this->expression->meet($leftRow, $rightRow)) { + for (; $pair < $pairs && $candidatesLeft[$pair] === $i; $pair++) { + if (!$met[$pair]) { continue; } $matched = true; if ($this->type === Join::left_anti) { - break; + continue; } if ($this->type === Join::right) { - $hashTable->matched($index); + $table->matched($candidatesRight[$pair]); } - $joined[] = $this->merge($leftRow, $rightRow); - } - - if (!$matched && $this->type === Join::left) { - /** @var Row $nullRightRow */ - $joined[] = $this->merge($leftRow, $nullRightRow); + $outputLeft[] = $i; + $outputRight[] = $candidatesRight[$pair]; } - if (!$matched && $this->type === Join::left_anti) { - $joined[] = $leftRow; + if (!$matched && ($this->type === Join::left || $this->type === Join::left_anti)) { + $outputLeft[] = $i; + $outputRight[] = $nullIndex; } } - if ($joined !== []) { - // a later batch can be wider than the side schema the output was derived from, and - // every arm answers for the same declared output - so all of them project, not one - $projected = []; - - foreach ($joined as $joinedRow) { - $projected[] = $joinedRow->project($outputSchema); - } - - // every joined value passed the gate on its own side - conforming checks the joined shape and the - // nulls the join introduced, not the values again - yield Rows::conformed($outputSchema, $projected); + if ($outputLeft === []) { + continue; } + + // a later batch can be wider than the side schema the output was derived from, and every arm answers + // for the same declared output - the projection drops the undeclared columns and pads the nulls the join + // introduced + yield $this->type === Join::left_anti + ? $batch->gather($outputLeft)->project($outputSchema, $this->backend) + : $this->merge($batch->gather($outputLeft), $probe->gather($outputRight))->project( + $outputSchema, + $this->backend, + ); } if ($this->type === Join::right) { $leftSchema ??= new Schema(); - $nullLeftRow ??= (new NullRowBuilder($leftSchema))->row(); + $nullLeft = $left->nullRow ?? (new NullRowBuilder($leftSchema, $this->backend))->rows(); $outputSchema ??= $this->joinSchema->of($this->type, $leftSchema, $rightSchema); - $buffer = new RowsBuffer($outputSchema, $this->batchSize, Rows::conformed(...)); - foreach ($hashTable->unmatchedRows() as $rightRow) { - if (null !== ($batch = $buffer->add($this->merge($nullLeftRow, $rightRow)->project($outputSchema)))) { - yield $batch; - } - } - - if (null !== ($batch = $buffer->flush())) { - yield $batch; + foreach (array_chunk($table->unmatched(), $this->batchSize) as $unmatched) { + yield $this->merge( + $nullLeft->gather(array_fill(0, count($unmatched), 0)), + $build->gather($unmatched), + )->project($outputSchema, $this->backend); } } } @@ -185,100 +176,161 @@ public function schema(Schema $left, Schema $right): Schema } /** - * Mirror of the default execution - roles swap, output rows do not: merge($left, $right) - * argument order is preserved, unmatched-row tracking moves to whichever side sits in the table. - * - * - * @return Generator + * @return array{Rows, Schema} */ - private function joinBuildingLeft(JoinSide $left, JoinSide $right): Generator + private function build(JoinSide $side): array { - $nullLeftRow = $left->nullRow; - $nullRightRow = $right->nullRow; - $leftSideSchema = $left->schema; - $rightSideSchema = $right->schema; + $schema = $side->schema; + $parts = []; - $hashTable = new HashTable(trackUnmatched: $this->type === Join::left || $this->type === Join::left_anti); - $leftSchema = $leftSideSchema; + foreach ($side->rows as $batch) { + $schema ??= $batch->schema(); - foreach ($left->rows as $batch) { - $leftSchema ??= $batch->schema(); - $hashes = $this->hasher->hash($this->leftValues->of($batch)); + if (!$batch->isEmpty()) { + $parts[] = $batch->project($schema, $this->backend); + } + } + + $schema ??= new Schema(); + + return [ + $parts === [] + ? Rows::empty($schema, $this->backend) + : $parts[0]->concat($this->backend, ...array_slice($parts, 1)), + $schema, + ]; + } + + /** + * @param list> $keys + * + * @return array{list, list} + */ + private function candidates(HashTable $table, array $keys): array + { + $left = []; + $right = []; - foreach ($batch as $i => $row) { - $hashTable->add($hashes[$i], $row); + foreach ($this->hasher->hash($keys) as $i => $hash) { + foreach ($table->candidatesFor($hash) as $j) { + $left[] = $i; + $right[] = $j; } } - $leftSchema ??= new Schema(); - $nullLeftRow ??= (new NullRowBuilder($leftSchema))->row(); + return [$left, $right]; + } + + /** + * Mirror of the default execution - roles swap, output rows do not: merge($left, $right) + * argument order is preserved, unmatched-row tracking moves to whichever side sits in the table. + * + * @return Generator + */ + private function joinBuildingLeft(JoinSide $left, JoinSide $right): Generator + { + [$build, $leftSchema] = $this->build($left); + $table = $this->table($build, $this->leftValues, $this->type === Join::left || $this->type === Join::left_anti); + + $probe = $this->type === Join::right ? $this->padded($build, $left->nullRow, $leftSchema) : $build; + $nullIndex = $build->count(); - $rightSchema = $rightSideSchema; + $rightSchema = $right->schema; $outputSchema = null; foreach ($right->rows as $batch) { $rightSchema ??= $batch->schema(); $outputSchema ??= $this->joinSchema->of($this->type, $leftSchema, $rightSchema); - $hashes = $this->hasher->hash($this->rightValues->of($batch)); - $joined = []; + [$candidatesRight, $candidatesLeft] = $this->candidates($table, $this->rightValues->of($batch)); + $met = $this->meet($build, $candidatesLeft, $batch, $candidatesRight); + + $outputLeft = []; + $outputRight = []; + $pair = 0; + $pairs = count($candidatesRight); - foreach ($batch as $i => $rightRow) { + for ($i = 0, $rows = $batch->count(); $i < $rows; $i++) { $matched = false; - foreach ($hashTable->candidatesFor($hashes[$i]) as $index => $leftRow) { - if (!$this->expression->meet($leftRow, $rightRow)) { + for (; $pair < $pairs && $candidatesRight[$pair] === $i; $pair++) { + if (!$met[$pair]) { continue; } $matched = true; if ($this->type === Join::left || $this->type === Join::left_anti) { - $hashTable->matched($index); + $table->matched($candidatesLeft[$pair]); } if ($this->type !== Join::left_anti) { - $joined[] = $this->merge($leftRow, $rightRow); + $outputLeft[] = $candidatesLeft[$pair]; + $outputRight[] = $i; } } if (!$matched && $this->type === Join::right) { - $joined[] = $this->merge($nullLeftRow, $rightRow); + $outputLeft[] = $nullIndex; + $outputRight[] = $i; } } - if ($joined !== []) { - $projected = []; - - foreach ($joined as $joinedRow) { - $projected[] = $joinedRow->project($outputSchema); - } - - yield Rows::conformed($outputSchema, $projected); + if ($outputLeft !== []) { + yield $this->merge($probe->gather($outputLeft), $batch->gather($outputRight))->project( + $outputSchema, + $this->backend, + ); } } $rightSchema ??= new Schema(); if ($this->type === Join::left || $this->type === Join::left_anti) { - $nullRightRow ??= (new NullRowBuilder($rightSchema))->row(); + $nullRight = $right->nullRow ?? (new NullRowBuilder($rightSchema, $this->backend))->rows(); $outputSchema ??= $this->joinSchema->of($this->type, $leftSchema, $rightSchema); - $buffer = new RowsBuffer($outputSchema, $this->batchSize, Rows::conformed(...)); - - foreach ($hashTable->unmatchedRows() as $leftRow) { - $joined = $this->type === Join::left ? $this->merge($leftRow, $nullRightRow) : $leftRow; - if (null !== ($batch = $buffer->add($joined->project($outputSchema)))) { - yield $batch; - } + foreach (array_chunk($table->unmatched(), $this->batchSize) as $unmatched) { + yield $this->type === Join::left + ? $this->merge( + $build->gather($unmatched), + $nullRight->gather(array_fill(0, count($unmatched), 0)), + )->project($outputSchema, $this->backend) + : $build->gather($unmatched)->project($outputSchema, $this->backend); } + } + } - if (null !== ($batch = $buffer->flush())) { - yield $batch; - } + /** + * @param list $leftIndices + * @param list $rightIndices + * + * @return list + */ + private function meet(Rows $left, array $leftIndices, Rows $right, array $rightIndices): array + { + if ($leftIndices === []) { + return []; } + + $names = static fn(array $refs): array => array_values(array_unique(array_map( + static fn(Reference $ref): string => $ref->base(), + $refs, + ))); + + return $this->expression->meet( + $left->project($left->schema()->keep(...$names($this->expression->left())), $this->backend)->gather( + $leftIndices, + ), + $right + ->project($right->schema()->keep(...$names($this->expression->right())), $this->backend) + ->gather($rightIndices), + ); } - private function merge(Row $left, Row $right): Row + /** + * @throws DuplicatedEntriesException + */ + private function merge(Rows $left, Rows $right): Rows { try { return $this->merger->merge($left, $right); @@ -290,4 +342,22 @@ private function merge(Row $left, Row $right): Row ); } } + + private function padded(Rows $build, ?Rows $nullRow, Schema $schema): Rows + { + $null = $nullRow ?? (new NullRowBuilder($schema, $this->backend))->rows(); + + return $build->withSchema($null->schema())->concat($this->backend, $null); + } + + private function table(Rows $build, KeyValues $values, bool $trackUnmatched): HashTable + { + $table = new HashTable(trackUnmatched: $trackUnmatched); + + foreach ($this->hasher->hash($values->of($build)) as $index => $hash) { + $table->add($hash, $index); + } + + return $table; + } } diff --git a/src/core/etl/src/Flow/ETL/Join/HashJoin/NullRowBuilder.php b/src/core/etl/src/Flow/ETL/Join/HashJoin/NullRowBuilder.php index ca54921c03..6046242686 100644 --- a/src/core/etl/src/Flow/ETL/Join/HashJoin/NullRowBuilder.php +++ b/src/core/etl/src/Flow/ETL/Join/HashJoin/NullRowBuilder.php @@ -4,19 +4,28 @@ namespace Flow\ETL\Join\HashJoin; -use Flow\ETL\Row; +use Flow\ETL\Column\Backend; +use Flow\ETL\Rows; use Flow\ETL\Schema; -use function array_fill_keys; - final readonly class NullRowBuilder { - public function __construct( - private Schema $schema, - ) {} + private Rows $rows; + + public function __construct(Schema $schema, Backend $backend) + { + $nullable = $schema->makeNullable(); + $columns = []; + + foreach ($nullable->definitions() as $name => $definition) { + $columns[$name] = $backend->constant($definition, null, 1); + } + + $this->rows = Rows::fromColumns($nullable, $columns, 1); + } - public function row(): Row + public function rows(): Rows { - return new Row(array_fill_keys($this->schema->references()->names(), null)); + return $this->rows; } } diff --git a/src/core/etl/src/Flow/ETL/Join/HashJoin/RowMerger.php b/src/core/etl/src/Flow/ETL/Join/HashJoin/RowMerger.php index 731d290599..f4a0307480 100644 --- a/src/core/etl/src/Flow/ETL/Join/HashJoin/RowMerger.php +++ b/src/core/etl/src/Flow/ETL/Join/HashJoin/RowMerger.php @@ -5,7 +5,8 @@ namespace Flow\ETL\Join\HashJoin; use Flow\ETL\Exception\DuplicatedEntriesException; -use Flow\ETL\Row; +use Flow\ETL\Rows; +use Flow\ETL\Schema; use function array_key_exists; use function array_keys; @@ -55,37 +56,46 @@ public function __construct( } /** + * Pair-aligned: row i of the result is left row i merged with right row i. + * * @throws DuplicatedEntriesException */ - public function merge(Row $left, Row $right): Row + public function merge(Rows $left, Rows $right): Rows { - $leftValues = $left->values(); - $rightValues = $right->values(); + $leftNames = array_keys($left->schema()->definitions()); + $rightNames = array_keys($right->schema()->definitions()); - $planKey = implode("\x00", array_keys($leftValues)) . "\x01" . implode("\x00", array_keys($rightValues)); + $planKey = implode("\x00", $leftNames) . "\x01" . implode("\x00", $rightNames); - [$keepLeft, $keepRight, $renames] = - $this->plans[$planKey] ??= $this->plan(array_keys($leftValues), array_keys($rightValues)); + [$keepLeft, $keepRight, $renames] = $this->plans[$planKey] ??= $this->plan($leftNames, $rightNames); - $values = []; + $definitions = []; + $columns = []; - // @mago-ignore analysis:mixed-assignment - foreach ($leftValues as $name => $value) { + foreach ($left->schema()->definitions() as $name => $definition) { if (array_key_exists($name, $keepLeft)) { - $values[$name] = $value; + $definitions[] = $definition; + $columns[$name] = $left->column($name); } } - // @mago-ignore analysis:mixed-assignment - foreach ($rightValues as $name => $value) { + foreach ($right->schema()->definitions() as $name => $definition) { if (!array_key_exists($name, $keepRight)) { continue; } - $values[$renames[$name] ?? $name] = $value; + if (array_key_exists($name, $renames)) { + $definitions[] = $definition->rename($renames[$name]); + $columns[$renames[$name]] = $right->column($name); + + continue; + } + + $definitions[] = $definition; + $columns[$name] = $right->column($name); } - return new Row($values); + return Rows::fromColumns(new Schema(...$definitions), $columns, $left->count()); } /** diff --git a/src/core/etl/src/Flow/ETL/Join/JoinSchema.php b/src/core/etl/src/Flow/ETL/Join/JoinSchema.php index 74d1013a29..b1da2293f5 100644 --- a/src/core/etl/src/Flow/ETL/Join/JoinSchema.php +++ b/src/core/etl/src/Flow/ETL/Join/JoinSchema.php @@ -4,6 +4,7 @@ namespace Flow\ETL\Join; +use Flow\ETL\Exception\SchemaDefinitionNotUniqueException; use Flow\ETL\Schema; final readonly class JoinSchema @@ -34,11 +35,19 @@ public function of(Join $type, Schema $left, Schema $right): Schema } } - return match ($type) { - Join::left_anti => $leftSide, - Join::left => $leftSide->add(...$rightSide->makeNullable()->definitions()), - Join::right => $leftSide->makeNullable()->add(...$rightSide->definitions()), - Join::inner => $leftSide->add(...$rightSide->definitions()), - }; + try { + return match ($type) { + Join::left_anti => $leftSide, + Join::left => $leftSide->add(...$rightSide->makeNullable()->definitions()), + Join::right => $leftSide->makeNullable()->add(...$rightSide->definitions()), + Join::inner => $leftSide->add(...$rightSide->definitions()), + }; + } catch (SchemaDefinitionNotUniqueException $e) { + throw new SchemaDefinitionNotUniqueException( + $e->getMessage() . '. Please consider using join prefix option', + (int) $e->getCode(), + $e, + ); + } } } diff --git a/src/core/etl/src/Flow/ETL/Join/JoinSteps.php b/src/core/etl/src/Flow/ETL/Join/JoinSteps.php index 8ab7bf631c..9d6e9c0cef 100644 --- a/src/core/etl/src/Flow/ETL/Join/JoinSteps.php +++ b/src/core/etl/src/Flow/ETL/Join/JoinSteps.php @@ -5,6 +5,7 @@ namespace Flow\ETL\Join; use Flow\ETL\Bucketing\Buckets; +use Flow\ETL\Bucketing\Storage\SpillingBuckets; use Flow\ETL\Config; use Flow\ETL\Config\Join\JoinAlgorithmBuilder; use Flow\ETL\Executor\PhysicalPlan; @@ -26,7 +27,7 @@ public static function of( Config $config, ?JoinAlgorithmBuilder $algorithm = null, ): array { - $join = $algorithm?->build($config->cache->localFilesystemCacheDir) ?? $config->join; + $join = $algorithm?->build($config->cache->localFilesystemCacheDir, $config->backend()) ?? $config->join; return [ new HashJoinProcessor( @@ -34,9 +35,10 @@ public static function of( $config->executor(), $on, $type, - new Buckets($join->bucketing->storage), - new Buckets($join->bucketing->storage), + new Buckets(SpillingBuckets::around($join->bucketing, $join->memoryLimit, $config->backend())), + new Buckets(SpillingBuckets::around($join->bucketing, $join->memoryLimit, $config->backend())), $config->randomValueGenerator(), + $join->memoryLimit, $join->bucketing->bucketsCount, $join->bucketing->batchSize, ), diff --git a/src/core/etl/src/Flow/ETL/Join/RowsJoin.php b/src/core/etl/src/Flow/ETL/Join/RowsJoin.php new file mode 100644 index 0000000000..def69f4353 --- /dev/null +++ b/src/core/etl/src/Flow/ETL/Join/RowsJoin.php @@ -0,0 +1,94 @@ +cross($left->schema(), $right->schema()); + $leftIndices = []; + $rightIndices = []; + $rightCount = $right->count(); + + for ($i = 0; $i < $left->count(); $i++) { + for ($j = 0; $j < $rightCount; $j++) { + $leftIndices[] = $i; + $rightIndices[] = $j; + } + } + + return (new RowMerger($joinPrefix)) + ->merge($left->gather($leftIndices), $right->gather($rightIndices)) + ->project($schema, $this->backend); + } + + /** + * @throws InvalidArgumentException + */ + public function inner(Rows $left, Rows $right, Expression $expression): Rows + { + return $this->join($left, $right, $expression, Join::inner); + } + + /** + * @throws InvalidArgumentException + */ + public function left(Rows $left, Rows $right, Expression $expression): Rows + { + return $this->join($left, $right, $expression, Join::left); + } + + /** + * @throws InvalidArgumentException + */ + public function leftAnti(Rows $left, Rows $right, Expression $expression): Rows + { + return $this->join($left, $right, $expression, Join::left_anti); + } + + /** + * @throws InvalidArgumentException + */ + public function right(Rows $left, Rows $right, Expression $expression): Rows + { + return $this->join($left, $right, $expression, Join::right); + } + + /** + * @throws InvalidArgumentException + */ + public function join(Rows $left, Rows $right, Expression $expression, Join $type): Rows + { + $single = static function (Rows $rows): Generator { + yield $rows; + }; + + $joiner = new Joiner($expression, $type, $this->backend); + $joined = []; + + foreach ($joiner->join(JoinSide::of($single($left)), JoinSide::of($single($right))) as $batch) { + $joined[] = $batch; + } + + return $joined === [] + ? Rows::empty($joiner->schema($left->schema(), $right->schema()), $this->backend) + : $joined[0]->concat($this->backend, ...array_slice($joined, 1)); + } +} diff --git a/src/core/etl/src/Flow/ETL/Loader/ArrayLoader.php b/src/core/etl/src/Flow/ETL/Loader/ArrayLoader.php index 7742c3cf41..c9c7e8db6a 100644 --- a/src/core/etl/src/Flow/ETL/Loader/ArrayLoader.php +++ b/src/core/etl/src/Flow/ETL/Loader/ArrayLoader.php @@ -10,8 +10,6 @@ use Flow\ETL\Rows; use Throwable; -use function array_merge; - final class ArrayLoader implements Loader { /** @@ -26,7 +24,9 @@ public function load(Rows $rows, FlowContext $context): void $context->telemetry()->loadingStarted($this); try { - $this->array = array_merge($this->array, $rows->toArray()); + foreach ($rows->toArray() as $row) { + $this->array[] = $row; + } $context->telemetry()->loadingCompleted($this, [TelemetryAttributes::ATTR_LOADING_ROWS => $rows->count()]); } catch (Throwable $e) { diff --git a/src/core/etl/src/Flow/ETL/Loader/FileLoader.php b/src/core/etl/src/Flow/ETL/Loader/File/FileLoader.php similarity index 87% rename from src/core/etl/src/Flow/ETL/Loader/FileLoader.php rename to src/core/etl/src/Flow/ETL/Loader/File/FileLoader.php index a88549e166..e836740b13 100644 --- a/src/core/etl/src/Flow/ETL/Loader/FileLoader.php +++ b/src/core/etl/src/Flow/ETL/Loader/File/FileLoader.php @@ -2,7 +2,7 @@ declare(strict_types=1); -namespace Flow\ETL\Loader; +namespace Flow\ETL\Loader\File; use Flow\ETL\Filesystem\SaveMode; use Flow\Filesystem\Path; diff --git a/src/core/etl/src/Flow/ETL/Loader/File/FileSink.php b/src/core/etl/src/Flow/ETL/Loader/File/FileSink.php new file mode 100644 index 0000000000..47b6eea2b1 --- /dev/null +++ b/src/core/etl/src/Flow/ETL/Loader/File/FileSink.php @@ -0,0 +1,14 @@ + by stream URI + */ + private array $open = []; + + public function __construct( + private readonly Filesystem $filesystem, + private readonly Path $path, + private readonly SaveMode $saveMode, + private readonly PartitionRouter $router, + private readonly FileSinks $sinks, + ) {} + + /** + * Closes every sink - the first close failure rethrown once all were closed - then publishes the files. + * + * @throws Throwable + */ + public function closure(): void + { + $failure = $this->closeSinks(); + + if ($failure !== null) { + throw $failure; + } + + $this->files?->publish(); + $this->files = null; + } + + /** + * Closes every sink, then abandons the files whatever a close threw. + * + * @throws Throwable + */ + public function discard(): void + { + $failure = $this->closeSinks(); + + $this->files?->abandon(); + $this->files = null; + + if ($failure !== null) { + throw $failure; + } + } + + public function write(Rows $rows, FlowContext $context, Loader $loader): void + { + $context->telemetry()->loadingStarted($loader, [ + TelemetryAttributes::ATTR_LOADER_DESTINATION_URI => $this->path->uri(), + ]); + + try { + foreach ($this->router->route($rows) as [$partitions, $group]) { + $files = $this->files ??= new FilesSink($this->filesystem, $this->path, $this->saveMode); + $stream = $files->writeTo($partitions->toArray()); + + ($this->open[$stream->path()->uri()] ??= $this->sinks->open($stream, $context->backend()))->write( + $group, + ); + } + + $context->telemetry()->loadingCompleted($loader, [ + TelemetryAttributes::ATTR_LOADING_ROWS => $rows->count(), + ]); + } catch (Throwable $e) { + $context->telemetry()->loadingFailed($loader, $e); + + throw $e; + } + } + + /** + * Closes every open sink, all of them even when one throws. + * + * @return null|Throwable the first close failure + */ + private function closeSinks(): ?Throwable + { + $failure = null; + + foreach ($this->open as $uri => $sink) { + unset($this->open[$uri]); + + try { + $sink->close(); + } catch (Throwable $e) { + $failure ??= $e; + } + } + + return $failure; + } +} diff --git a/src/core/etl/src/Flow/ETL/Filesystem/FilesSink.php b/src/core/etl/src/Flow/ETL/Loader/File/FilesSink.php similarity index 90% rename from src/core/etl/src/Flow/ETL/Filesystem/FilesSink.php rename to src/core/etl/src/Flow/ETL/Loader/File/FilesSink.php index 8090e628af..4fe2e93af8 100644 --- a/src/core/etl/src/Flow/ETL/Filesystem/FilesSink.php +++ b/src/core/etl/src/Flow/ETL/Loader/File/FilesSink.php @@ -2,15 +2,15 @@ declare(strict_types=1); -namespace Flow\ETL\Filesystem; +namespace Flow\ETL\Loader\File; use Flow\ETL\Exception\RuntimeException; +use Flow\ETL\Filesystem\SaveMode; use Flow\Filesystem\DestinationStream; use Flow\Filesystem\Filesystem; use Flow\Filesystem\Partition; use Flow\Filesystem\Path; use Flow\Filesystem\Stream\VoidStream; -use Generator; use Throwable; use function array_key_exists; @@ -111,29 +111,6 @@ public function publish(): void } } - /** - * @param array $partitions - */ - public function touched(array $partitions = []): bool - { - return array_key_exists( - count($partitions) ? $this->destination->addPartitions(...$partitions)->uri() : $this->destination->uri(), - $this->streams, - ); - } - - /** - * @return \Generator - */ - public function openStreams(): Generator - { - foreach ($this->streams as $stream) { - if ($stream->isOpen()) { - yield $stream; - } - } - } - /** * @param array $partitions */ diff --git a/src/core/etl/src/Flow/ETL/Loader/File/PartitionRouter.php b/src/core/etl/src/Flow/ETL/Loader/File/PartitionRouter.php new file mode 100644 index 0000000000..ed55879263 --- /dev/null +++ b/src/core/etl/src/Flow/ETL/Loader/File/PartitionRouter.php @@ -0,0 +1,92 @@ + the partition columns the files do not carry + */ + private readonly array $dropped; + + public function __construct( + private readonly Partitioning $partitioning, + private readonly Hasher $hasher = new NativeHasher(), + ) { + $this->rowPartitions = new RowPartitions($partitioning->by); + $this->dropped = $partitioning->writeColumns ? [] : array_values($partitioning->by->names()); + } + + /** + * @return list + */ + public function droppedNames(): array + { + return $this->dropped; + } + + /** + * @return Generator + */ + public function route(Rows $rows): Generator + { + if (!$this->partitioning->by->count()) { + yield [new Partitions(), $rows]; + + return; + } + + $schema = $rows->schema(); + $this->assertSomethingIsLeftToWrite($schema); + + $hashes = $this->hasher->hash((new KeyValues($this->partitioning->by->all()))->of($rows)); + + /** @var array}> $groups */ + $groups = []; + + foreach ($hashes as $index => $hash) { + $groups[$hash] ??= [$this->rowPartitions->of($rows, $index), []]; + $groups[$hash][1][] = $index; + } + + $stripped = $this->dropped === [] ? $schema : $schema->gracefulRemove(...$this->partitioning->by->names()); + + foreach ($groups as [$partitions, $indices]) { + $group = $rows->gather($indices); + + yield [$partitions, $this->dropped === [] ? $group : $group->select(...$stripped->references()->names())]; + } + } + + private function assertSomethingIsLeftToWrite(Schema $schema): void + { + if ($this->dropped === [] || $schema->count() > $this->partitioning->by->count()) { + return; + } + + throw new InvalidArgumentException(sprintf('No column left to write, every column is a partition column: "%s". ' + . 'Use partition_by(...)->writeColumns() to write them into the file as well.', implode( + '", "', + $this->partitioning->by->names(), + ))); + } +} diff --git a/src/core/etl/src/Flow/ETL/Loader/PartitionRouter.php b/src/core/etl/src/Flow/ETL/Loader/PartitionRouter.php deleted file mode 100644 index 325ff9361c..0000000000 --- a/src/core/etl/src/Flow/ETL/Loader/PartitionRouter.php +++ /dev/null @@ -1,106 +0,0 @@ - - */ - private readonly array $dropped; - - public function __construct( - private readonly Partitioning $partitioning, - private readonly Hasher $hasher = new NativeHasher(), - ) { - $this->rowPartitions = new RowPartitions($partitioning->by); - $this->dropped = $partitioning->writeColumns ? [] : array_flip($partitioning->by->names()); - } - - /** - * @return array - */ - public function droppedNames(): array - { - return array_keys($this->dropped); - } - - /** - * @return Generator - */ - public function route(Rows $rows): Generator - { - if (!$this->partitioning->by->count()) { - yield [new Partitions(), $rows]; - - return; - } - - $schema = $rows->schema(); - $this->assertSomethingIsLeftToWrite($schema); - - $hashes = $this->hasher->hash((new KeyValues($this->partitioning->by->all()))->of($rows)); - - /** @var array}> $groups */ - $groups = []; - - foreach (array_values($rows->all()) as $index => $row) { - $groups[$hashes[$index]] ??= [$this->rowPartitions->of($row, $schema), []]; - $groups[$hashes[$index]][1][] = $row; - } - - $stripped = $this->dropped === [] ? $schema : $schema->gracefulRemove(...$this->partitioning->by->names()); - - // a group is a subset of a batch that already passed the gate, and stripping removes the same - // columns from its rows and from the schema, so neither needs checking again - foreach ($groups as [$partitions, $rowsOfGroup]) { - if ($this->dropped === []) { - yield [$partitions, Rows::trusted($schema, $rowsOfGroup)]; - - continue; - } - - $strippedRows = []; - - foreach ($rowsOfGroup as $row) { - $strippedRows[] = new Row(array_diff_key($row->values(), $this->dropped)); - } - - yield [$partitions, Rows::trusted($stripped, $strippedRows)]; - } - } - - private function assertSomethingIsLeftToWrite(Schema $schema): void - { - if ($this->dropped === [] || $schema->count() > $this->partitioning->by->count()) { - return; - } - - throw new InvalidArgumentException(sprintf('No column left to write, every column is a partition column: "%s". ' - . 'Use partition_by(...)->writeColumns() to write them into the file as well.', implode( - '", "', - $this->partitioning->by->names(), - ))); - } -} diff --git a/src/core/etl/src/Flow/ETL/Loader/RowPartitions.php b/src/core/etl/src/Flow/ETL/Loader/RowPartitions.php index 73a39e6d4b..c8f827c4bc 100644 --- a/src/core/etl/src/Flow/ETL/Loader/RowPartitions.php +++ b/src/core/etl/src/Flow/ETL/Loader/RowPartitions.php @@ -4,9 +4,8 @@ namespace Flow\ETL\Loader; -use Flow\ETL\Row; use Flow\ETL\Row\References; -use Flow\ETL\Schema; +use Flow\ETL\Rows; use Flow\Filesystem\Partition; use Flow\Filesystem\Partitions; @@ -16,18 +15,19 @@ public function __construct( private References $by, ) {} - public function of(Row $row, Schema $schema): Partitions + public function of(Rows $rows, int $index): Partitions { $partitions = []; foreach ($this->by as $reference) { - $value = $row->get($reference); + // @mago-ignore analysis:mixed-assignment + $value = $rows->column($reference->base())->value($index); $partitions[] = new Partition( $reference->name(), $value === null ? null - : Partition::fromValue($reference->name(), $schema->get($reference)->type(), $value), + : Partition::fromValue($reference->name(), $rows->schema()->get($reference)->type(), $value), ); } diff --git a/src/core/etl/src/Flow/ETL/Memory/ArrayMemory.php b/src/core/etl/src/Flow/ETL/Memory/ArrayMemory.php index ee8229cec7..5370f5375b 100644 --- a/src/core/etl/src/Flow/ETL/Memory/ArrayMemory.php +++ b/src/core/etl/src/Flow/ETL/Memory/ArrayMemory.php @@ -101,7 +101,9 @@ public function save(array $data): void { $this->assertMemoryStructure($data); - $this->memory = array_merge($this->memory, $data); + foreach ($data as $entry) { + $this->memory[] = $entry; + } } /** diff --git a/src/core/etl/src/Flow/ETL/Optimizer/Rule/CombineSortAndLimit.php b/src/core/etl/src/Flow/ETL/Optimizer/Rule/CombineSortAndLimit.php index 63c8530cc3..e0d2999512 100644 --- a/src/core/etl/src/Flow/ETL/Optimizer/Rule/CombineSortAndLimit.php +++ b/src/core/etl/src/Flow/ETL/Optimizer/Rule/CombineSortAndLimit.php @@ -20,6 +20,6 @@ { public function apply(LogicalPlan $plan, FlowContext $context): LogicalPlan { - return $plan->transformUp(new TopNRewrite($context)); + return $plan->transformUp(new TopNRewrite()); } } diff --git a/src/core/etl/src/Flow/ETL/Optimizer/Rule/CountFromStatistics.php b/src/core/etl/src/Flow/ETL/Optimizer/Rule/CountFromStatistics.php index 0122ea91c6..6ab5bce034 100644 --- a/src/core/etl/src/Flow/ETL/Optimizer/Rule/CountFromStatistics.php +++ b/src/core/etl/src/Flow/ETL/Optimizer/Rule/CountFromStatistics.php @@ -30,6 +30,9 @@ public function apply(LogicalPlan $plan, FlowContext $context): LogicalPlan return $rows === null ? $plan - : new LogicalPlan(new Result(new Read(from_rows((new CountingProcessor())->rows($rows))))); + : new LogicalPlan(new Result(new Read(from_rows((new CountingProcessor())->rows( + $rows, + $context->backend(), + ))))); } } diff --git a/src/core/etl/src/Flow/ETL/Optimizer/Rule/PushFilterIntoSource.php b/src/core/etl/src/Flow/ETL/Optimizer/Rule/PushFilterIntoSource.php index 945f176528..3d597af875 100644 --- a/src/core/etl/src/Flow/ETL/Optimizer/Rule/PushFilterIntoSource.php +++ b/src/core/etl/src/Flow/ETL/Optimizer/Rule/PushFilterIntoSource.php @@ -4,7 +4,7 @@ namespace Flow\ETL\Optimizer\Rule; -use Flow\ETL\Extractor\FileExtractor; +use Flow\ETL\Extractor\File\FileExtractor; use Flow\ETL\Filesystem\ScalarFunctionFilter; use Flow\ETL\FlowContext; use Flow\ETL\Function\All; diff --git a/src/core/etl/src/Flow/ETL/Optimizer/TopNRewrite.php b/src/core/etl/src/Flow/ETL/Optimizer/TopNRewrite.php index b276930834..2beb7e7720 100644 --- a/src/core/etl/src/Flow/ETL/Optimizer/TopNRewrite.php +++ b/src/core/etl/src/Flow/ETL/Optimizer/TopNRewrite.php @@ -4,8 +4,6 @@ namespace Flow\ETL\Optimizer; -use Flow\ETL\Config\Sort\ExternalSortConfig; -use Flow\ETL\FlowContext; use Flow\ETL\Plan\Node; use Flow\ETL\Plan\Node\Limit; use Flow\ETL\Plan\Node\Sort; @@ -17,10 +15,6 @@ */ final readonly class TopNRewrite implements Rewrite { - public function __construct( - private FlowContext $context, - ) {} - public function of(Node $node): Node { if (!$node instanceof Limit) { @@ -33,13 +27,7 @@ public function of(Node $node): Node return $node; } - $config = $this->context->config; - $algorithm = $sort->algorithm?->build($config->cache->localFilesystemCacheDir) ?? $config->sort; - - if ($algorithm instanceof ExternalSortConfig && $node->limit > $algorithm->runSize) { - return $node; - } - - return new TopN($sort->children()[0], $sort->refs, $node->limit); + // a TopN past its memory limit falls back to the sort's own spilling path, so any limit is safe to rewrite + return new TopN($sort->children()[0], $sort->refs, $node->limit, $sort->algorithm); } } diff --git a/src/core/etl/src/Flow/ETL/Plan/Explain/Details.php b/src/core/etl/src/Flow/ETL/Plan/Explain/Details.php index d282c27e2d..fb8733ce71 100644 --- a/src/core/etl/src/Flow/ETL/Plan/Explain/Details.php +++ b/src/core/etl/src/Flow/ETL/Plan/Explain/Details.php @@ -4,7 +4,7 @@ namespace Flow\ETL\Plan\Explain; -use Flow\ETL\Extractor\FileExtractor; +use Flow\ETL\Extractor\File\FileExtractor; use Flow\ETL\Join\Comparison\Any; use Flow\ETL\Plan\Materialization; use Flow\ETL\Plan\Node; diff --git a/src/core/etl/src/Flow/ETL/Plan/Explain/StepDetails.php b/src/core/etl/src/Flow/ETL/Plan/Explain/StepDetails.php index 26ad2c6fd6..9fa1e97900 100644 --- a/src/core/etl/src/Flow/ETL/Plan/Explain/StepDetails.php +++ b/src/core/etl/src/Flow/ETL/Plan/Explain/StepDetails.php @@ -4,12 +4,13 @@ namespace Flow\ETL\Plan\Explain; +use Flow\ETL\Bucketing\BucketsStorage; +use Flow\ETL\Bucketing\Storage\SpillingBuckets; use Flow\ETL\Extractor; use Flow\ETL\Loader; use Flow\ETL\Processor; use Flow\ETL\Processor\BatchingByProcessor; use Flow\ETL\Processor\BatchingProcessor; -use Flow\ETL\Processor\BucketingProcessor; use Flow\ETL\Processor\CachingProcessor; use Flow\ETL\Processor\CollectingProcessor; use Flow\ETL\Processor\ConstrainedProcessor; @@ -22,6 +23,7 @@ use Flow\ETL\Processor\RepartitionProcessor; use Flow\ETL\Processor\TopNProcessor; use Flow\ETL\Processor\WindowProcessor; +use Flow\ETL\Row\NullsOrder; use Flow\ETL\Row\Reference; use Flow\ETL\Row\References; use Flow\ETL\Row\SortOrder; @@ -31,6 +33,7 @@ use function array_map; use function implode; +use function strtolower; final readonly class StepDetails { @@ -74,17 +77,19 @@ public function settings(Extractor|Loader|Processor|Transformer $step): array ? ['Join: cross'] : ['Join: cross', 'Prefix: ' . $step->prefix], $step instanceof MergeSortProcessor => [ - 'Sort: ' . $this->sort($step->refs), - 'Spill: ' . $this->details->name($step->spill->storage()), - 'Merge: ' . $step->mergeFanIn . ' ways', - 'Batch: ' . $step->batchSize, + 'Sort: ' . $this->sort($step->sort->refs), + 'Memory: ' . $step->sort->memoryLimit->inMb() . ' MB', + 'Spill: ' . $this->details->name($step->sort->spill->storage()), + 'Merge: ' . $step->sort->mergeFanIn . ' ways', + 'Batch: ' . $step->sort->batchSize, ], $step instanceof MemorySortProcessor => ['Sort: ' . $this->sort($step->refs)], $step instanceof TopNProcessor => ['Top: ' . $step->limit, 'Sort: ' . $this->sort($step->refs)], $step instanceof GroupByAggregationProcessor => [ 'Group by: ' . $this->columns($step->groupBy->refs()), 'Aggregations: ' . $this->aggregations($step), - 'Storage: ' . $this->details->name($step->buckets->storage()), + 'Memory: ' . $step->memoryLimit->inMb() . ' MB', + 'Storage: ' . $this->storageName($step->buckets->storage()), 'Batch: ' . $step->batchSize, ], $step instanceof PivotProcessor => $this->pivot($step), @@ -92,11 +97,8 @@ public function settings(Extractor|Loader|Processor|Transformer $step): array $step instanceof RepartitionProcessor => [ 'By: ' . $this->columns($step->by), 'Hasher: ' . $this->details->name($step->hasher), - 'Storage: ' . $this->details->name($step->buckets->storage()), - ], - $step instanceof BucketingProcessor => [ - 'Strategy: ' . $this->details->name($step->strategy), - 'Storage: ' . $this->details->name($step->buckets->storage()), + 'Memory: ' . $step->memoryLimit->inMb() . ' MB', + 'Storage: ' . $this->storageName($step->buckets->storage()), ], $step instanceof BatchingProcessor => ['Batch: ' . $step->size], $step instanceof BatchingByProcessor => $this->batchingBy($step), @@ -138,7 +140,7 @@ public function hashJoin(HashJoinProcessor $join): array return [ ...$lines, - 'Storage: ' . $this->details->name($join->rightBuckets->storage()), + ...$this->storage($join->rightBuckets->storage()), 'Buckets: ' . $join->bucketsCount, 'Batch: ' . $join->batchSize, ]; @@ -215,11 +217,35 @@ public function columns(References $refs): string return implode(', ', array_map(static fn(Reference $ref): string => $ref->name(), $refs->all())); } + /** + * @return list the configured storage, and the memory limit it stands behind + */ + public function storage(BucketsStorage $storage): array + { + return ( + $storage instanceof SpillingBuckets + ? ['Storage: ' . $this->storageName($storage), 'Memory: ' . $storage->memoryLimit->inMb() . ' MB'] + : ['Storage: ' . $this->storageName($storage)] + ); + } + + public function storageName(BucketsStorage $storage): string + { + return $this->details->name($storage instanceof SpillingBuckets ? $storage->disk : $storage); + } + public function sort(References $refs): string { return implode(', ', array_map( static fn(Reference $ref): string => ( - $ref->name() . ' ' . ($ref->sort() === SortOrder::ASC ? 'asc' : 'desc') + $ref->name() + . ' ' + . ($ref->sort() === SortOrder::ASC ? 'asc' : 'desc') + . ( + $ref->nulls() === NullsOrder::defaultFor($ref->sort()) + ? '' + : ' nulls ' . strtolower($ref->nulls()->name) + ) ), $refs->all(), )); diff --git a/src/core/etl/src/Flow/ETL/Plan/Node/TopN.php b/src/core/etl/src/Flow/ETL/Plan/Node/TopN.php index f54dee43a5..3af8e47b00 100644 --- a/src/core/etl/src/Flow/ETL/Plan/Node/TopN.php +++ b/src/core/etl/src/Flow/ETL/Plan/Node/TopN.php @@ -4,6 +4,7 @@ namespace Flow\ETL\Plan\Node; +use Flow\ETL\Config\Sort\SortAlgorithmBuilder; use Flow\ETL\Exception\InvalidArgumentException; use Flow\ETL\Plan\Materialization; use Flow\ETL\Plan\Node; @@ -25,6 +26,7 @@ public function __construct( private Node $input, public References $refs, public int $limit, + public ?SortAlgorithmBuilder $algorithm = null, ) { if ($this->limit < 1) { throw new InvalidArgumentException('TopN limit must be greater than 0, given: ' . $this->limit); @@ -41,7 +43,9 @@ public function children(): array public function withChildren(array $children): self { - return $children[0] === $this->input ? $this : new self($children[0], $this->refs, $this->limit); + return $children[0] === $this->input + ? $this + : new self($children[0], $this->refs, $this->limit, $this->algorithm); } public function rowCount(): RowCount diff --git a/src/core/etl/src/Flow/ETL/Planner/NodeTranslator.php b/src/core/etl/src/Flow/ETL/Planner/NodeTranslator.php index 354deaf9cb..eaab7e1459 100644 --- a/src/core/etl/src/Flow/ETL/Planner/NodeTranslator.php +++ b/src/core/etl/src/Flow/ETL/Planner/NodeTranslator.php @@ -103,7 +103,11 @@ public static function toSteps(Node $node, FlowContext $context, array $frames): $node instanceof Node\Repartition => RepartitionSteps::of($node->by, $context->config), $node instanceof Node\Select => [new SelectEntriesTransformer(...$node->entries)], $node instanceof Node\Sort => SortSteps::of($node->refs, $context->config, $node->algorithm), - $node instanceof Node\TopN => [new TopNProcessor($node->refs, $node->limit)], + $node instanceof Node\TopN => [new TopNProcessor( + $node->refs, + $node->limit, + SortSteps::external($node->refs, $context->config, $node->algorithm), + )], $node instanceof Node\Transform => [ $node->transformer instanceof Stateful ? $node->transformer->fresh() : $node->transformer, ], diff --git a/src/core/etl/src/Flow/ETL/Processor/BatchingByProcessor.php b/src/core/etl/src/Flow/ETL/Processor/BatchingByProcessor.php index 3900d9a64c..2e04885a23 100644 --- a/src/core/etl/src/Flow/ETL/Processor/BatchingByProcessor.php +++ b/src/core/etl/src/Flow/ETL/Processor/BatchingByProcessor.php @@ -5,17 +5,17 @@ namespace Flow\ETL\Processor; use Flow\ETL\BoundStep; +use Flow\ETL\Column\ComparableValues; use Flow\ETL\Exception\InvalidArgumentException; use Flow\ETL\Extractor\Signal; use Flow\ETL\FlowContext; use Flow\ETL\Processor; -use Flow\ETL\Row; use Flow\ETL\Row\Reference; use Flow\ETL\Rows; use Flow\ETL\Schema; use Generator; -use function count; +use function array_slice; /** * Groups rows into batches by column value. @@ -52,49 +52,69 @@ public function bind(Schema $input): BoundStep */ public function process(Generator $rows, FlowContext $context): Generator { - /** @var array $buffer */ - $buffer = []; + /** @var list $parts */ + $parts = []; + $buffered = 0; $currentValue = null; $hasValue = false; + $comparable = new ComparableValues(); $schema = null; while ($rows->valid()) { - $batch = $rows->current(); - $schema ??= $batch->schema(); + $schema ??= $rows->current()->schema(); + $batch = $rows->current()->matchTo($schema, $context->backend()); + $start = 0; - foreach ($batch as $row) { - $value = $row->get($this->column); + if ($batch->isEmpty()) { + $rows->next(); + continue; + } + + // @mago-ignore analysis:mixed-assignment + foreach ($comparable->equality($batch->column($this->column->base())) as $i => $value) { if (!$hasValue) { + // @mago-ignore analysis:mixed-assignment $currentValue = $value; $hasValue = true; } if ($value !== $currentValue) { - if ($this->minSize === null || count($buffer) >= $this->minSize) { - if ($buffer !== []) { - $signal = yield new Rows($schema, ...$buffer); + $size = $buffered + $i - $start; + + if (($this->minSize === null || $size >= $this->minSize) && $size > 0) { + if ($i > $start) { + $parts[] = $batch->slice($start, $i - $start); + } - if ($signal === Signal::STOP) { - $rows->send(Signal::STOP); + $signal = yield $parts[0]->concat($context->backend(), ...array_slice($parts, 1)); - return; - } + if ($signal === Signal::STOP) { + $rows->send(Signal::STOP); - $buffer = []; + return; } + + $parts = []; + $buffered = 0; + $start = $i; } + + // @mago-ignore analysis:mixed-assignment $currentValue = $value; } + } - $buffer[] = $row; + if ($start < $batch->count()) { + $parts[] = $batch->slice($start, $batch->count() - $start); + $buffered += $batch->count() - $start; } $rows->next(); } - if ($buffer !== []) { - yield new Rows($schema ?? new Schema(), ...$buffer); + if ($parts !== []) { + yield $parts[0]->concat($context->backend(), ...array_slice($parts, 1)); } } } diff --git a/src/core/etl/src/Flow/ETL/Processor/BatchingProcessor.php b/src/core/etl/src/Flow/ETL/Processor/BatchingProcessor.php index e9608210f7..fae858c3b8 100644 --- a/src/core/etl/src/Flow/ETL/Processor/BatchingProcessor.php +++ b/src/core/etl/src/Flow/ETL/Processor/BatchingProcessor.php @@ -9,14 +9,10 @@ use Flow\ETL\Extractor\Signal; use Flow\ETL\FlowContext; use Flow\ETL\Processor; -use Flow\ETL\Row; use Flow\ETL\Rows; use Flow\ETL\Schema; use Generator; -use function array_splice; -use function count; - /** * Re-batches rows into fixed-size batches. */ @@ -48,42 +44,36 @@ public function bind(Schema $input): BoundStep */ public function process(Generator $rows, FlowContext $context): Generator { - /** @var array $buffer */ - $buffer = []; - + $pending = null; $schema = null; - // rows re-batched from batches under one schema already passed the gate under it - only a batch that - // arrives under another schema needs the full gate to conform it to the first one - $uniform = true; - while ($rows->valid()) { $batch = $rows->current(); $schema ??= $batch->schema(); - $uniform = $uniform && $batch->schema()->isSame($schema); - foreach ($batch as $row) { - $buffer[] = $row; + if (!$batch->isEmpty()) { + $pending = $pending === null + ? $batch->matchTo($schema, $context->backend()) + : $pending->concat($context->backend(), $batch->matchTo($schema, $context->backend())); - if (count($buffer) >= $this->size) { - $chunk = array_splice($buffer, 0, $this->size); - $signal = yield $uniform ? Rows::trusted($schema, $chunk) : new Rows($schema, ...$chunk); + while ($pending->count() >= $this->size) { + $signal = yield $pending->slice(0, $this->size); if ($signal === Signal::STOP) { $rows->send(Signal::STOP); return; } + + $pending = $pending->drop($this->size); } } $rows->next(); } - if ($buffer !== []) { - $schema ??= new Schema(); - - yield $uniform ? Rows::trusted($schema, $buffer) : new Rows($schema, ...$buffer); + if ($pending !== null && !$pending->isEmpty()) { + yield $pending; } } } diff --git a/src/core/etl/src/Flow/ETL/Processor/BucketingProcessor.php b/src/core/etl/src/Flow/ETL/Processor/BucketingProcessor.php deleted file mode 100644 index 0771eb56f5..0000000000 --- a/src/core/etl/src/Flow/ETL/Processor/BucketingProcessor.php +++ /dev/null @@ -1,42 +0,0 @@ - $rows - * - * @return Generator - */ - public function process(Generator $rows, FlowContext $context): Generator - { - foreach ($this->strategy->bucketize($rows, $this->buckets->storage()) as $bucket) { - $this->buckets->add($bucket); - - yield new Rows(Bucket::schema(), $bucket->toRow()); - } - } - - public function bind(Schema $input): BoundStep - { - return new BoundStep($this, $input); - } -} diff --git a/src/core/etl/src/Flow/ETL/Processor/CachingProcessor.php b/src/core/etl/src/Flow/ETL/Processor/CachingProcessor.php index af160151f0..5d56387825 100644 --- a/src/core/etl/src/Flow/ETL/Processor/CachingProcessor.php +++ b/src/core/etl/src/Flow/ETL/Processor/CachingProcessor.php @@ -62,7 +62,7 @@ public function process(Generator $rows, FlowContext $context): Generator } } - $cache->set($id, $index->toRows()); + $cache->set($id, $index->toRows($context->backend())); } public function bind(Schema $input): BoundStep diff --git a/src/core/etl/src/Flow/ETL/Processor/CollectingProcessor.php b/src/core/etl/src/Flow/ETL/Processor/CollectingProcessor.php index 4c988fd0d9..697ba81af0 100644 --- a/src/core/etl/src/Flow/ETL/Processor/CollectingProcessor.php +++ b/src/core/etl/src/Flow/ETL/Processor/CollectingProcessor.php @@ -34,9 +34,14 @@ public function process(Generator $rows, FlowContext $context): Generator $collected = null; foreach ($rows as $batch) { - $collected = $collected === null ? $batch : $collected->merge($batch); + // an empty part adds nothing, whatever its schema + $collected = match (true) { + $collected === null, $collected->isEmpty() => $batch, + $batch->isEmpty() => $collected, + default => $collected->concat($context->backend(), $batch), + }; } - yield $collected ?? new Rows($this->declared ?? new Schema()); + yield $collected ?? Rows::empty($this->declared ?? new Schema(), $context->backend()); } } diff --git a/src/core/etl/src/Flow/ETL/Processor/ConstrainedProcessor.php b/src/core/etl/src/Flow/ETL/Processor/ConstrainedProcessor.php index b820ec1bcb..7d72c94316 100644 --- a/src/core/etl/src/Flow/ETL/Processor/ConstrainedProcessor.php +++ b/src/core/etl/src/Flow/ETL/Processor/ConstrainedProcessor.php @@ -54,20 +54,32 @@ public function process(Generator $rows, FlowContext $context): Generator /** @var Rows $batch */ $batch = $rows->current(); - foreach ($batch->all() as $row) { - foreach ($this->constraints as $constraint) { - if (!$constraint->isSatisfiedBy($row, $batch->schema())) { - throw new ConstraintViolationException( - $constraint->toString(), - $constraint->violation($row, $batch->schema()), - $this->rowIndex, - ); - } + $violationIndex = null; + $violated = null; + + // a constraint sees only the rows before the earliest violation found so far - the row-by-row order + // in which an earlier constraint's violation stops the later ones + foreach ($this->constraints as $constraint) { + $index = $constraint->firstViolation( + $violationIndex === null ? $batch : $batch->slice(0, $violationIndex), + ); + + if ($index !== null) { + $violationIndex = $index; + $violated = $constraint; } + } - $this->rowIndex++; + if ($violated !== null && $violationIndex !== null) { + throw new ConstraintViolationException( + $violated->toString(), + $violated->violation($batch, $violationIndex), + $this->rowIndex + $violationIndex, + ); } + $this->rowIndex += $batch->count(); + $signal = yield $batch; if ($signal === Signal::STOP) { diff --git a/src/core/etl/src/Flow/ETL/Processor/CountingProcessor.php b/src/core/etl/src/Flow/ETL/Processor/CountingProcessor.php index 485304d5c5..86cd9d9ef1 100644 --- a/src/core/etl/src/Flow/ETL/Processor/CountingProcessor.php +++ b/src/core/etl/src/Flow/ETL/Processor/CountingProcessor.php @@ -5,15 +5,15 @@ namespace Flow\ETL\Processor; use Flow\ETL\BoundStep; +use Flow\ETL\Column\Backend; use Flow\ETL\FlowContext; use Flow\ETL\Processor; use Flow\ETL\Rows; use Flow\ETL\Schema; use Generator; +use function Flow\ETL\DSL\array_to_rows; use function Flow\ETL\DSL\int_schema; -use function Flow\ETL\DSL\row; -use function Flow\ETL\DSL\rows; use function Flow\ETL\DSL\schema; final readonly class CountingProcessor implements Processor @@ -31,15 +31,15 @@ public function process(Generator $rows, FlowContext $context): Generator $count += $batch->count(); } - yield $this->rows($count); + yield $this->rows($count, $context->backend()); } /** * The one row a count hands out, whether it was counted or known upfront. */ - public function rows(int $count): Rows + public function rows(int $count, Backend $backend): Rows { - return rows($this->schema(), row(['count' => $count])); + return array_to_rows([['count' => $count]], $this->schema(), $backend); } public function schema(): Schema diff --git a/src/core/etl/src/Flow/ETL/Processor/GroupByAggregationProcessor.php b/src/core/etl/src/Flow/ETL/Processor/GroupByAggregationProcessor.php index 77d8b765f1..a0e9e7f68b 100644 --- a/src/core/etl/src/Flow/ETL/Processor/GroupByAggregationProcessor.php +++ b/src/core/etl/src/Flow/ETL/Processor/GroupByAggregationProcessor.php @@ -6,21 +6,32 @@ use Flow\ETL\BoundStep; use Flow\ETL\Bucketing\Buckets; -use Flow\ETL\Bucketing\BucketShape; +use Flow\ETL\Bucketing\HashBucketing; +use Flow\ETL\Dataset\Memory\BoundedRead; +use Flow\ETL\Dataset\Memory\MemoryBudget; +use Flow\ETL\Dataset\Memory\Unit; use Flow\ETL\Exception\InvalidArgumentException; use Flow\ETL\Exception\SchemaDefinitionNotFoundException; use Flow\ETL\FlowContext; use Flow\ETL\GroupBy; -use Flow\ETL\GroupBy\BucketAggregation; +use Flow\ETL\GroupBy\AggregatedGroups; +use Flow\ETL\GroupBy\Group; use Flow\ETL\GroupBy\GroupByShape; use Flow\ETL\GroupBy\GroupKey; use Flow\ETL\Processor; use Flow\ETL\Rows; +use Flow\ETL\Rows\RowsBuilder; use Flow\ETL\Schema; use Generator; +use function array_values; +use function iterator_to_array; +use function serialize; + /** - * Aggregates buckets spilled by BucketingProcessor, one bucket per incoming metadata Row. + * Aggregates in one pass while the process stays under the memory limit. Past it, the rest of the stream is partitioned + * into buckets and each bucket aggregated on its own; the groups aggregated before the limit are merged into the bucket + * their key hashes to. */ final class GroupByAggregationProcessor implements Processor { @@ -35,7 +46,9 @@ final class GroupByAggregationProcessor implements Processor */ public function __construct( public readonly GroupBy $groupBy, + public readonly HashBucketing $bucketing, public readonly Buckets $buckets, + public readonly Unit $memoryLimit, public readonly int $batchSize = 1000, ) { // @mago-ignore analysis:invalid-operand @@ -50,7 +63,7 @@ public function __construct( */ public function bind(Schema $input): BoundStep { - $bound = new self($this->groupBy, $this->buckets, $this->batchSize); + $bound = new self($this->groupBy, $this->bucketing, $this->buckets, $this->memoryLimit, $this->batchSize); $bound->shape = GroupByShape::of($this->groupBy, $input); return new BoundStep($bound, $bound->shape->output); @@ -63,43 +76,110 @@ public function bind(Schema $input): BoundStep */ public function process(Generator $rows, FlowContext $context): Generator { - $aggregation = new BucketAggregation($this->batchSize); - $emitted = false; + $shape = $this->shape; + $groups = null; + $budget = new MemoryBudget($context->backend(), $this->memoryLimit); + $spilling = false; try { - foreach ($rows as $metadata) { - foreach ($metadata as $row) { - /** @var string $bucketId */ - $bucketId = $row->get(BucketShape::id->value); - $bucket = $this->buckets->rows($bucketId); - - $aggregated = $this->shape === null - ? $aggregation->aggregate($bucket, $context, $this->groupBy) - : $aggregation->aggregateBound($bucket, $context, $this->groupBy, $this->shape); - - // re-key batches, yield from would restart keys at 0 for every bucket - foreach ($aggregated as $aggregatedBatch) { - $emitted = true; - - yield $aggregatedBatch; - } + while ($rows->valid()) { + /** @var Rows $batch valid() holds, so current() is a batch */ + $batch = $rows->current(); + $rows->next(); + + if ($batch->isEmpty()) { + continue; } + + $shape ??= GroupByShape::of($this->groupBy, $batch->schema()); + $groups ??= new AggregatedGroups($this->groupBy, $shape); + $groups->accumulate($batch, $context); + + if ($budget->exceeded()) { + $spilling = true; + + break; + } + } + + if ($spilling && $groups !== null && $shape !== null) { + yield from $this->spilled($groups, $shape, $rows, $context); + + return; + } + + if ($groups !== null) { + yield from $groups->flush($this->batchSize, $context); + + return; } // SQL's scalar aggregate: no input and no grouping key is one row of the initial accumulators, // not no row. Only reachable bound - an unbound plan has no schema to type the defaults with. - if (!$emitted && $this->shape !== null && $this->groupBy->isGlobal()) { - yield new Rows( - $this->shape->output, - $this->groupBy->aggregatedRow( - new GroupKey([]), - $this->shape->aggregators->cloned(), - $this->shape->output, + if ($shape !== null && $this->groupBy->isGlobal()) { + yield (new RowsBuilder($shape->output, $context->backend()))->appendRows([ + $this->groupBy->aggregatedValues( + new GroupKey([], serialize([])), + $shape->aggregators->cloned(), + $shape->output, ), - ); + ])->finish(); } } finally { $this->buckets->clear(); } } + + /** + * @param Generator $rest the stream after the batch that passed the limit + * + * @return Generator + */ + public function spilled( + AggregatedGroups $earlier, + GroupByShape $shape, + Generator $rest, + FlowContext $context, + ): Generator { + foreach ($this->bucketing->bucketize( + (new BoundedRead($this->memoryLimit, $context->backend()))->followedBy([], $rest), + $this->buckets->storage(), + ) as $bucket) { + $this->buckets->add($bucket); + } + + /** @var array> $partials */ + $partials = []; + + foreach ($earlier->groups() as $key => $group) { + $partials[$this->bucketing->indexOf(array_values(iterator_to_array($group->key)))][$key] = $group; + } + + foreach ($this->buckets->all() as $bucket) { + $groups = new AggregatedGroups($this->groupBy, $shape); + + foreach ($this->buckets->rows($bucket->id) as $batch) { + $groups->accumulate($batch, $context); + } + + foreach ($partials[$bucket->index] ?? [] as $key => $group) { + $groups->absorbEarlier($key, $group, $context); + } + + unset($partials[$bucket->index]); + + yield from $groups->flush($this->batchSize, $context); + } + + // groups whose bucket received no row after the limit + $remaining = new AggregatedGroups($this->groupBy, $shape); + + foreach ($partials as $groups) { + foreach ($groups as $key => $group) { + $remaining->absorbEarlier($key, $group, $context); + } + } + + yield from $remaining->flush($this->batchSize, $context); + } } diff --git a/src/core/etl/src/Flow/ETL/Processor/HashJoinProcessor.php b/src/core/etl/src/Flow/ETL/Processor/HashJoinProcessor.php index 8453c718f6..4feda5beed 100644 --- a/src/core/etl/src/Flow/ETL/Processor/HashJoinProcessor.php +++ b/src/core/etl/src/Flow/ETL/Processor/HashJoinProcessor.php @@ -11,6 +11,8 @@ use Flow\ETL\Bucketing\NativeHasher; use Flow\ETL\Bucketing\ResidentBucketsStorage; use Flow\ETL\Bucketing\SingleBucketHasher; +use Flow\ETL\Dataset\Memory\BoundedRead; +use Flow\ETL\Dataset\Memory\Unit; use Flow\ETL\Exception\DuplicatedEntriesException; use Flow\ETL\Exception\InvalidArgumentException; use Flow\ETL\Exception\JoinException; @@ -33,8 +35,10 @@ use Flow\ETL\Schema; use Generator; +use function array_diff_key; use function array_intersect_key; use function array_keys; +use function range; final class HashJoinProcessor implements Processor { @@ -58,6 +62,7 @@ public function __construct( public readonly Buckets $leftBuckets, public readonly Buckets $rightBuckets, private readonly RandomValueGenerator $random, + public readonly Unit $memoryLimit, public readonly int $bucketsCount = 64, public readonly int $batchSize = 1000, ) { @@ -86,6 +91,7 @@ public function bind(Schema $input): BoundStep $this->leftBuckets, $this->rightBuckets, $this->random, + $this->memoryLimit, $this->bucketsCount, $this->batchSize, ); @@ -119,7 +125,7 @@ public function process(Generator $rows, FlowContext $context): Generator } } - $joiner = new Joiner($this->expression, $this->type, $this->batchSize); + $joiner = new Joiner($this->expression, $this->type, $context->backend(), $this->batchSize); $equalityKeys = $joiner->keys(); $resident = $this->rightBuckets->storage() instanceof ResidentBucketsStorage; @@ -131,15 +137,31 @@ public function process(Generator $rows, FlowContext $context): Generator $equalityKeys !== null && $this->type !== Join::right ? $equalityKeys->rightRefs() : null, ); - $this->bucketize( - $rightRows, - $this->rightBuckets, - $equalityKeys !== null && !$resident ? $equalityKeys->rightRefs() : null, - 'join-right', - ); + /** @var null|list $held the build side, when all of it fits under the memory limit */ + $held = null; + + if (!$resident) { + $bounded = new BoundedRead($this->memoryLimit, $context->backend()); + [$read, $fits] = $bounded->read($rightRows); + + if ($fits) { + $held = $read; + } else { + $rightRows = $bounded->followedBy($read, $rightRows); + } + } + + if ($held === null) { + $this->bucketize( + $rightRows, + $this->rightBuckets, + $equalityKeys !== null && !$resident ? $equalityKeys->rightRefs() : null, + 'join-right', + ); + } $nullRightRow = $this->type === Join::left - ? (new NullRowBuilder($rightSchema ?? new Schema()))->row() + ? (new NullRowBuilder($rightSchema ?? new Schema(), $context->backend()))->rows() : null; $leftRows = $this->tap( @@ -151,13 +173,20 @@ public function process(Generator $rows, FlowContext $context): Generator : null, ); - if ($resident) { + if ($resident || $held !== null) { $rightBucket = $this->rightBuckets->all()[0] ?? null; $joinedBatches = $joiner->join( JoinSide::of($leftRows, null, $leftSchema), JoinSide::of( - $rightBucket === null ? self::noRows() : $this->rightBuckets->rows($rightBucket->id), + match (true) { + $held !== null => (new BoundedRead($this->memoryLimit, $context->backend()))->followedBy( + $held, + self::noRows(), + ), + $rightBucket === null => self::noRows(), + default => $this->rightBuckets->rows($rightBucket->id), + }, $nullRightRow, $rightSchema, ), @@ -183,7 +212,7 @@ public function process(Generator $rows, FlowContext $context): Generator // only the bucketized path builds it here, the resident path above derives the null-left // row inside the Joiner while streaming $nullLeftRow = $this->type === Join::right - ? (new NullRowBuilder($leftSchema ?? new Schema()))->row() + ? (new NullRowBuilder($leftSchema ?? new Schema(), $context->backend()))->rows() : null; foreach ($this->bucketPairs() as [$leftBucket, $rightBucket]) { @@ -287,21 +316,31 @@ private function tap(Generator $rows, ?Schema &$schema, ?array $dropNullKeyRefs) $schema ??= $batch->schema(); if ($dropNullKeyRefs !== null) { - $kept = []; + if ($batch->isEmpty()) { + continue; + } - foreach ($batch->all() as $row) { - foreach ($dropNullKeyRefs as $ref) { - if ($row->get($ref) === null) { - continue 2; - } + $null = []; + + foreach ($dropNullKeyRefs as $ref) { + $column = $batch->column($ref->base()); + + if ($column->nullCount() === 0) { + continue; } - $kept[] = $row; + for ($i = 0, $count = $column->count(); $i < $count; $i++) { + if ($column->isNull($i)) { + $null[$i] = true; + } + } } - $batch = Rows::trusted($batch->schema(), $kept); + if ($null !== []) { + $batch = $batch->gather(array_keys(array_diff_key(range(0, $batch->count() - 1), $null))); + } - if ($batch->empty()) { + if ($batch->isEmpty()) { continue; } } diff --git a/src/core/etl/src/Flow/ETL/Processor/MemorySortProcessor.php b/src/core/etl/src/Flow/ETL/Processor/MemorySortProcessor.php index ffbb7a23ae..1b84086f19 100644 --- a/src/core/etl/src/Flow/ETL/Processor/MemorySortProcessor.php +++ b/src/core/etl/src/Flow/ETL/Processor/MemorySortProcessor.php @@ -7,12 +7,12 @@ use Flow\ETL\BoundStep; use Flow\ETL\FlowContext; use Flow\ETL\Processor; -use Flow\ETL\Row; use Flow\ETL\Row\References; use Flow\ETL\Rows; use Flow\ETL\Schema; use Generator; +use function array_slice; use function max; /** @@ -28,7 +28,7 @@ public function __construct( public function process(Generator $rows, FlowContext $context): Generator { - /** @var array $buffer */ + /** @var list $buffer */ $buffer = []; $maxSize = 1; @@ -36,20 +36,19 @@ public function process(Generator $rows, FlowContext $context): Generator foreach ($rows as $batch) { $schema ??= $batch->schema(); - if ($batch->empty()) { + if ($batch->isEmpty()) { continue; } $maxSize = max($batch->count(), $maxSize); - - foreach ($batch->all() as $row) { - $buffer[] = $row; - } + $buffer[] = $batch->matchTo($this->declared ?? $schema, $context->backend()); } - yield from (new Rows($this->declared ?? $schema ?? new Schema(), ...$buffer)) - ->sortBy(...$this->refs->all()) - ->chunks($maxSize); + $all = $buffer === [] + ? Rows::empty($this->declared ?? $schema ?? new Schema(), $context->backend()) + : $buffer[0]->concat($context->backend(), ...array_slice($buffer, 1)); + + yield from $all->sortBy(...$this->refs->all())->chunks($maxSize); } public function bind(Schema $input): BoundStep diff --git a/src/core/etl/src/Flow/ETL/Processor/MergeSortProcessor.php b/src/core/etl/src/Flow/ETL/Processor/MergeSortProcessor.php index 13931e45ae..3095129018 100644 --- a/src/core/etl/src/Flow/ETL/Processor/MergeSortProcessor.php +++ b/src/core/etl/src/Flow/ETL/Processor/MergeSortProcessor.php @@ -5,49 +5,17 @@ namespace Flow\ETL\Processor; use Flow\ETL\BoundStep; -use Flow\ETL\Bucketing\Bucket; -use Flow\ETL\Bucketing\BucketRun; -use Flow\ETL\Bucketing\Buckets; -use Flow\ETL\Bucketing\BucketShape; -use Flow\ETL\Exception\InvalidArgumentException; use Flow\ETL\FlowContext; use Flow\ETL\Processor; -use Flow\ETL\RandomValueGenerator; -use Flow\ETL\Row\References; use Flow\ETL\Schema; -use Flow\ETL\Sort\Merge\KWayMerge; +use Flow\ETL\Sort\ExternalSort; use Generator; -use function array_slice; -use function array_splice; -use function count; - -final class MergeSortProcessor implements Processor +final readonly class MergeSortProcessor implements Processor { - /** - * @param int<1, max> $mergeFanIn - * @param int<1, max> $batchSize - */ public function __construct( - public readonly References $refs, - public readonly Buckets $spill, - public readonly Buckets $merge, - private readonly RandomValueGenerator $random, - public readonly int $mergeFanIn = 10, - public readonly int $batchSize = 1000, - ) { - // @mago-ignore analysis:invalid-operand - // @mago-ignore analysis:impossible-condition,redundant-comparison - if ($this->mergeFanIn < 1) { - throw new InvalidArgumentException('Merge fan-in must be greater than 0, given: ' . $this->mergeFanIn); - } - - // @mago-ignore analysis:invalid-operand - // @mago-ignore analysis:impossible-condition,redundant-comparison - if ($this->batchSize < 1) { - throw new InvalidArgumentException('Batch size must be greater than 0, given: ' . $this->batchSize); - } - } + public ExternalSort $sort, + ) {} public function bind(Schema $input): BoundStep { @@ -56,62 +24,6 @@ public function bind(Schema $input): BoundStep public function process(Generator $rows, FlowContext $context): Generator { - /** @var list $runs */ - $runs = []; - - try { - // draining upstream inside the try is what makes a mid-stream failure release the spilled runs - foreach ($rows as $batch) { - foreach ($batch as $row) { - /** @var string $bucketId */ - $bucketId = $row->get(BucketShape::id->value); - $runs[] = new BucketRun($bucketId, $this->spill); - } - } - - $merger = new KWayMerge($this->refs, $this->batchSize); - $mergedIndex = 0; - - while (count($runs) > $this->mergeFanIn) { - $runs[] = $this->reduce(array_slice($runs, 0, $this->mergeFanIn), $merger, $mergedIndex++); - array_splice($runs, 0, $this->mergeFanIn); - } - - yield from $merger->merge($runs); - } finally { - // nested, so a throwing spill clear still leaves the merge storage cleared - and chains as previous - try { - $this->spill->clear(); - } finally { - $this->merge->clear(); - } - } - } - - /** - * @param list $group - * - * @return BucketRun the merged run, bound to the merge storage that wrote it - */ - private function reduce(array $group, KWayMerge $merger, int $index): BucketRun - { - $bucketId = 'sort-merge-' . $this->random->string(16); - - // registered before the spill so clear() covers a partially written merged run - $this->merge->add(new Bucket($bucketId, 0, $index)); - $totalRows = 0; - - foreach ($merger->merge($group) as $batch) { - $this->merge->storage()->append($bucketId, $batch); - $totalRows += $batch->count(); - } - - foreach ($group as $run) { - $run->remove(); - } - - $this->merge->add(new Bucket($bucketId, $totalRows, $index)); - - return new BucketRun($bucketId, $this->merge); + yield from $this->sort->sort($rows, $context); } } diff --git a/src/core/etl/src/Flow/ETL/Processor/RepartitionProcessor.php b/src/core/etl/src/Flow/ETL/Processor/RepartitionProcessor.php index 660eb92d13..9a80ee6351 100644 --- a/src/core/etl/src/Flow/ETL/Processor/RepartitionProcessor.php +++ b/src/core/etl/src/Flow/ETL/Processor/RepartitionProcessor.php @@ -6,11 +6,14 @@ use Flow\ETL\BoundStep; use Flow\ETL\Bucketing\Buckets; -use Flow\ETL\Bucketing\BucketShape; +use Flow\ETL\Bucketing\HashBucketing; use Flow\ETL\Bucketing\Hasher; use Flow\ETL\Bucketing\KeyGrouping; use Flow\ETL\Bucketing\KeyValues; use Flow\ETL\Bucketing\NativeHasher; +use Flow\ETL\Column\Backend; +use Flow\ETL\Dataset\Memory\BoundedRead; +use Flow\ETL\Dataset\Memory\Unit; use Flow\ETL\FlowContext; use Flow\ETL\Processor; use Flow\ETL\Row\References; @@ -18,16 +21,19 @@ use Flow\ETL\Schema; use Generator; -use function Flow\Types\DSL\type_string; +use function array_slice; /** - * Regroups buckets spilled by BucketingProcessor so every row sharing a key arrives in one batch. + * One batch per key. While the process stays under the memory limit the input is held and grouped in one pass; past + * it, the input is partitioned into buckets and each bucket grouped on its own. */ final readonly class RepartitionProcessor implements Processor { public function __construct( public References $by, + public HashBucketing $bucketing, public Buckets $buckets, + public Unit $memoryLimit, public Hasher $hasher = new NativeHasher(), ) {} @@ -38,17 +44,28 @@ public function __construct( */ public function process(Generator $rows, FlowContext $context): Generator { - $grouping = new KeyGrouping(new KeyValues($this->by->all()), $this->hasher); + $grouping = new KeyGrouping(new KeyValues($this->by->all()), $this->hasher, $context->backend()); + $bounded = new BoundedRead($this->memoryLimit, $context->backend()); + [$read, $fits] = $bounded->read($rows); try { - foreach ($rows as $metadata) { - foreach ($metadata as $row) { - $bucketId = type_string()->assert($row->get(BucketShape::id->value)); - - // re-key batches, yield from would restart keys at 0 for every bucket - foreach ($grouping->group($this->buckets->rows($bucketId)) as $group) { - yield $group; - } + if ($fits) { + yield from $grouping->group($this->concatenated($read, $context->backend())); + + return; + } + + foreach ($this->bucketing->bucketize( + $bounded->followedBy($read, $rows), + $this->buckets->storage(), + ) as $bucket) { + $this->buckets->add($bucket); + } + + foreach ($this->buckets->all() as $bucket) { + // re-key batches, yield from would restart keys at 0 for every bucket + foreach ($grouping->group($this->buckets->rows($bucket->id)) as $group) { + yield $group; } } } finally { @@ -60,4 +77,30 @@ public function bind(Schema $input): BoundStep { return new BoundStep($this, $input); } + + /** + * The held batches as one, so each key is gathered once rather than once per batch. + * + * @param list $batches + * + * @return Generator + */ + public function concatenated(array $batches, Backend $backend): Generator + { + $parts = []; + $schema = null; + + foreach ($batches as $batch) { + if ($batch->isEmpty()) { + continue; + } + + $schema ??= $batch->schema(); + $parts[] = $batch->matchTo($schema, $backend); + } + + if ($parts !== []) { + yield $parts[0]->concat($backend, ...array_slice($parts, 1)); + } + } } diff --git a/src/core/etl/src/Flow/ETL/Processor/TopNProcessor.php b/src/core/etl/src/Flow/ETL/Processor/TopNProcessor.php index 92e2a4860f..7c2f3f8e7f 100644 --- a/src/core/etl/src/Flow/ETL/Processor/TopNProcessor.php +++ b/src/core/etl/src/Flow/ETL/Processor/TopNProcessor.php @@ -5,16 +5,16 @@ namespace Flow\ETL\Processor; use Flow\ETL\BoundStep; +use Flow\ETL\Dataset\Memory\MemoryBudget; use Flow\ETL\Exception\InvalidArgumentException; use Flow\ETL\FlowContext; use Flow\ETL\Processor; -use Flow\ETL\Row; use Flow\ETL\Row\References; use Flow\ETL\Rows; use Flow\ETL\Schema; +use Flow\ETL\Sort\ExternalSort; use Generator; -use function count; use function max; /** @@ -30,6 +30,7 @@ public function __construct( public References $refs, public int $limit, + public ?ExternalSort $fallback = null, private ?Schema $declared = null, ) { if ($this->limit < 1) { @@ -39,50 +40,69 @@ public function __construct( public function bind(Schema $input): BoundStep { - return new BoundStep(new self($this->refs, $this->limit, $input), $input); + return new BoundStep(new self($this->refs, $this->limit, $this->fallback, $input), $input); } public function process(Generator $rows, FlowContext $context): Generator { - /** @var array $kept */ - $kept = []; + $kept = null; $maxSize = 1; $schema = null; + $budget = $this->fallback === null ? null : new MemoryBudget($context->backend(), $this->fallback->memoryLimit); - foreach ($rows as $batch) { + while ($rows->valid()) { + $batch = $rows->current(); + $rows->next(); $schema ??= $batch->schema(); - if ($batch->empty()) { + if ($batch->isEmpty()) { continue; } $maxSize = max($batch->count(), $maxSize); + $conformed = $batch->matchTo($this->declared ?? $schema, $context->backend()); + $kept = $kept === null ? $conformed : $kept->concat($context->backend(), $conformed); - foreach ($batch->all() as $row) { - $kept[] = $row; + // trimming only past twice the limit keeps the re-sorts amortised + if ($kept->count() > (2 * $this->limit)) { + $kept = $this->top($kept); } - // trimming only past twice the limit keeps the re-sorts amortised - if (count($kept) > (2 * $this->limit)) { - $kept = $this->top($kept, $schema); + // past the memory limit the kept rows and the rest of the stream go to the spilling sort; a trimmed row + // cannot be in the top $limit, so the result is the same + if ($this->fallback !== null && $budget?->exceeded()) { + yield from $this->fallback->sort($this->followedBy($kept, $rows), $context, $this->limit); + + return; } } - yield from (new Rows($this->declared ?? $schema ?? new Schema(), ...$this->top($kept, $schema)))->chunks( - $maxSize, - ); + if ($kept === null) { + yield from Rows::empty($this->declared ?? $schema ?? new Schema(), $context->backend())->chunks($maxSize); + + return; + } + + yield from $this->top($kept)->chunks($maxSize); } /** - * @param array $rows + * @param Generator $rest * - * @return array + * @return Generator */ - public function top(array $rows, ?Schema $schema): array + public function followedBy(Rows $kept, Generator $rest): Generator + { + yield $kept; + + // continues the half-read stream where it stopped; PHP refuses to yield from a generator that already finished + if ($rest->valid()) { + yield from $rest; + } + } + + public function top(Rows $rows): Rows { - return (new Rows($this->declared ?? $schema ?? new Schema(), ...$rows)) - ->sortBy(...$this->refs->all()) - ->take($this->limit) - ->all(); + return $rows->sortBy(...$this->refs->all())->take($this->limit); } } diff --git a/src/core/etl/src/Flow/ETL/Processor/VoidProcessor.php b/src/core/etl/src/Flow/ETL/Processor/VoidProcessor.php index 33077907ba..1fe4c3cc08 100644 --- a/src/core/etl/src/Flow/ETL/Processor/VoidProcessor.php +++ b/src/core/etl/src/Flow/ETL/Processor/VoidProcessor.php @@ -31,6 +31,6 @@ public function process(Generator $rows, FlowContext $context): Generator } // void() drops rows, not columns - yield new Rows($this->declared ?? new Schema()); + yield Rows::empty($this->declared ?? new Schema(), $context->backend()); } } diff --git a/src/core/etl/src/Flow/ETL/Processor/WindowProcessor.php b/src/core/etl/src/Flow/ETL/Processor/WindowProcessor.php index 7cc8772789..50c2db02c3 100644 --- a/src/core/etl/src/Flow/ETL/Processor/WindowProcessor.php +++ b/src/core/etl/src/Flow/ETL/Processor/WindowProcessor.php @@ -13,7 +13,6 @@ use Flow\ETL\Function\ReferenceResolver; use Flow\ETL\Function\WindowFunction; use Flow\ETL\Processor; -use Flow\ETL\Row; use Flow\ETL\Rows; use Flow\ETL\Schema; use Flow\ETL\Schema\Definition; @@ -22,10 +21,7 @@ use Flow\ETL\Window\WindowFrame; use Generator; -use function array_values; -use function count; use function Flow\ETL\DSL\definition_from_type; -use function Flow\ETL\DSL\rows; /** * Applies window functions over partitioned and ordered data. @@ -56,14 +52,14 @@ public function process(Generator $rows, FlowContext $context): Generator $bound ??= $this->boundTo($batch->schema()); if (!$batch->count()) { - yield new Rows($bound->output); + yield Rows::empty($bound->output, $context->backend()); continue; } // one incoming batch is one partition: RepartitionSteps put every row sharing the // partition key into a single Rows before this processor ever sees it - yield $this->processPartition($bound, $batch->all(), $context); + yield $this->processPartition($bound, $batch, $context); } } @@ -93,12 +89,6 @@ private function boundTo(Schema $input): BoundWindow ); } - /** - * An order reference already failed loudly - sortBy() reaches Rows::sortDescending(), which throws - * on the first row of the first partition. Only the partition path degraded silently, because - * extractPartitionKey() substitutes null for a missing entry. This check makes both refuse - * identically, one batch earlier, with the gate's message. - */ private static function assertWindowReferences(WindowFunction $function, Schema $schema): void { $window = $function->window(); @@ -114,7 +104,7 @@ private static function assertWindowReferences(WindowFunction $function, Schema * Evaluates a frame-only window function for every row of the partition, reusing the previous * result whenever the frame bounds did not move. * - * @return array + * @return list */ private function accumulateValues( FrameAccumulating $function, @@ -122,10 +112,7 @@ private function accumulateValues( Rows $partition, FlowContext $context, ): array { - // Rows guarantees a list internally, but all() is typed array - narrow it here rather than - // widen the shared Rows contract. One copy per partition, not per row. - $rows = array_values($partition->all()); - $count = count($rows); + $count = $partition->count(); $values = []; $accumulator = null; $previousBounds = null; @@ -136,7 +123,7 @@ private function accumulateValues( $bounds = $frame->bounds($index, $partition); if ($bounds === $previousBounds) { - $values[$index] = $value; + $values[] = $value; continue; } @@ -150,39 +137,37 @@ private function accumulateValues( && $bounds[1] > $previousBounds[1] ) { for ($i = $previousBounds[1] + 1; $i <= $bounds[1]; $i++) { - $accumulator->accumulate($rows[$i]); + $accumulator->accumulate($partition, $i); } } else { $accumulator = $function->accumulator($context); for ($i = $bounds[0]; $i <= $bounds[1]; $i++) { - $accumulator->accumulate($rows[$i]); + $accumulator->accumulate($partition, $i); } } - // @mago-ignore analysis:mixed-assignment $value = $accumulator->value(); $previousBounds = $bounds; - $values[$index] = $value; + $values[] = $value; } return $values; } /** - * @param array $rows - never empty; process() answers a zero-row batch from the bound schema + * @param Rows $partition never empty; process() answers a zero-row batch from the bound schema */ - private function processPartition(BoundWindow $bound, array $rows, FlowContext $context): Rows + private function processPartition(BoundWindow $bound, Rows $partition, FlowContext $context): Rows { $resolved = $bound->resolved; $derived = $bound->derived; $window = $resolved->window(); $orderBy = $window->order(); $sortBy = $orderBy === [] ? $window->partitions()->all() : $orderBy; - $partitionRows = rows($bound->input, ...$rows)->sortBy(...$sortBy); + $partitionRows = $partition->matchTo($bound->input, $context->backend())->sortBy(...$sortBy); $frame = $window->frame(); - $processedRows = []; $values = match (true) { $resolved instanceof PartitionRanking => $resolved->rankPartition($partitionRows), @@ -195,17 +180,17 @@ private function processPartition(BoundWindow $bound, array $rows, FlowContext $ default => null, }; - foreach ($partitionRows as $index => $row) { - $value = $values === null - ? $resolved->apply(new WindowContext($row, $index, $partitionRows, $frame, $context)) - : $values[$index]; + if ($values === null) { + $values = []; - $processedRows[] = new Row([ - ...$row->values(), - $derived->entry()->name() => $value === null ? null : $derived->type()->cast($value), - ]); + for ($index = 0, $count = $partitionRows->count(); $index < $count; $index++) { + $values[] = $resolved->apply(new WindowContext($index, $partitionRows, $frame, $context)); + } } - return rows($bound->output, ...$processedRows); + $builder = $context->backend()->builder($derived); + $builder->appendMany($values); + + return $partitionRows->withColumns($bound->output, [$derived->entry()->name() => $builder->finish()]); } } diff --git a/src/core/etl/src/Flow/ETL/Repartition/RepartitionSteps.php b/src/core/etl/src/Flow/ETL/Repartition/RepartitionSteps.php index 1458a6ab3c..3c6ba26992 100644 --- a/src/core/etl/src/Flow/ETL/Repartition/RepartitionSteps.php +++ b/src/core/etl/src/Flow/ETL/Repartition/RepartitionSteps.php @@ -7,10 +7,10 @@ use Flow\ETL\Bucketing\Buckets; use Flow\ETL\Bucketing\HashBucketing; use Flow\ETL\Bucketing\NativeHasher; +use Flow\ETL\Bucketing\Storage\SpillingBuckets; use Flow\ETL\Config; use Flow\ETL\Config\Repartition\RepartitionAlgorithmBuilder; use Flow\ETL\Processor; -use Flow\ETL\Processor\BucketingProcessor; use Flow\ETL\Processor\RepartitionProcessor; use Flow\ETL\Row\References; @@ -24,11 +24,17 @@ */ public static function of(References $by, Config $config, ?RepartitionAlgorithmBuilder $algorithm = null): array { - $repartition = $algorithm?->build($config->cache->localFilesystemCacheDir) ?? $config->repartition; - $buckets = new Buckets($repartition->bucketing->storage); + $repartition = + $algorithm?->build($config->cache->localFilesystemCacheDir, $config->backend()) ?? $config->repartition; + $buckets = new Buckets(SpillingBuckets::around( + $repartition->bucketing, + $repartition->memoryLimit, + $config->backend(), + )); return [ - new BucketingProcessor( + new RepartitionProcessor( + $by, new HashBucketing( $by->all(), $repartition->bucketing->bucketsCount, @@ -37,8 +43,8 @@ public static function of(References $by, Config $config, ?RepartitionAlgorithmB 'repartition', ), $buckets, + $repartition->memoryLimit, ), - new RepartitionProcessor($by, $buckets), ]; } } diff --git a/src/core/etl/src/Flow/ETL/Row.php b/src/core/etl/src/Flow/ETL/Row.php deleted file mode 100644 index 6eaa1e9fd1..0000000000 --- a/src/core/etl/src/Flow/ETL/Row.php +++ /dev/null @@ -1,255 +0,0 @@ - $values storage keyed by name; the Schema defines column order (Rows::schema()->references()) - */ - public function __construct( - private array $values, - ) {} - - /** - * Rewrites the storage to satisfy $schema: columns take the Schema's order and a declared - * nullable column the row omits is padded with null. The row does not know its position in a - * batch - Rows places the violation with SchemaMismatchException. - * - * @throws ColumnMismatchException - */ - public function matchTo(Schema $schema): self - { - return $this->conform($schema, checkValues: true); - } - - /** - * matchTo() without its value check: the order, padding, missing, unknown and null rules all hold, but a non-null - * value is not validated against its type - the caller produced it by casting to, or decoding from, that type. - * - * @throws ColumnMismatchException - */ - public function conformTo(Schema $schema): self - { - return $this->conform($schema, checkValues: false); - } - - /** - * Drops the columns $schema does not declare and carries its order into the row. Unlike - * matchTo() it validates nothing - the caller is changing the shape, not checking it. - */ - public function project(Schema $schema): self - { - $projected = []; - - foreach ($schema->definitions() as $name => $_) { - if (array_key_exists($name, $this->values)) { - $projected[$name] = $this->values[$name]; - } - } - - return new self($projected); - } - - /** - * @throws InvalidArgumentException - * - * @return null|array|bool|float|int|object|string - */ - public function get(string|Reference $reference): mixed - { - $name = $reference instanceof Reference ? $reference->base() : $reference; - - if (!array_key_exists($name, $this->values)) { - $suggestions = (new SimilarNames())->closestTo( - $name, - array_values(array_map( - static fn(int|string $column): string => (string) $column, - array_keys($this->values), - )), - ); - - throw new InvalidArgumentException( - $suggestions === [] - ? "Column \"{$name}\" does not exist." - : "Column \"{$name}\" does not exist. Did you mean one of the following? [\"" - . implode('", "', $suggestions) - . '"]', - ); - } - - // @mago-ignore analysis:mixed-return-statement - return $this->values[$name]; - } - - public function has(string|Reference ...$references): bool - { - foreach ($references as $reference) { - if (!array_key_exists($reference instanceof Reference ? $reference->base() : $reference, $this->values)) { - return false; - } - } - - return true; - } - - public function hash(Schema $schema, Algorithm $algorithm = new NativePHPHash()): string - { - $formatter = new TypedValueFormatter(); - $string = ''; - - foreach ($schema->sort()->definitions() as $definition) { - $name = $definition->entry()->name(); - $string .= $name . $formatter->format($definition->type(), $this->values[$name] ?? null); - } - - return $algorithm->hash($string); - } - - /** - * @return list - */ - public function names(): array - { - return array_map(strval(...), array_keys($this->values)); - } - - /** - * @return array - */ - public function toArray(bool $withKeys = true): array - { - $data = []; - - // @mago-ignore analysis:mixed-assignment - foreach ($this->values as $name => $value) { - if ($value instanceof Json) { - $value = $value->toArray(); - } - - $withKeys ? ($data[$name] = $value) : ($data[] = $value); - } - - return $data; - } - - /** - * @return array - */ - public function values(): array - { - return $this->values; - } - - /** - * @throws ColumnMismatchException - */ - private function conform(Schema $schema, bool $checkValues): self - { - $definitions = $schema->definitions(); - $columns = $checkValues ? $schema->zonedDefinitions() : []; - $changed = false; - - if (count($this->values) === count($definitions) && array_keys($this->values) === array_keys($definitions)) { - foreach ($columns === [] ? $definitions : array_diff_key($definitions, $columns) as $name => $definition) { - if ( - $checkValues - ? !$definition->matches($this->values[$name]) - : $this->values[$name] === null && !$definition->isNullable() - ) { - throw ColumnMismatchException::valueDoesNotMatch($definition, $this->values[$name]); - } - } - - if ($columns === []) { - return $this; - } - - $values = $this->values; - } else { - $matched = []; - $taken = 0; - - foreach ($definitions as $name => $definition) { - if (!array_key_exists($name, $this->values)) { - if (!$definition->isNullable()) { - throw ColumnMismatchException::missingColumn($definition); - } - - $matched[$name] = null; - - continue; - } - - $taken++; - - if ( - $checkValues - ? !$definition->matches($this->values[$name]) - : $this->values[$name] === null && !$definition->isNullable() - ) { - throw ColumnMismatchException::valueDoesNotMatch($definition, $this->values[$name]); - } - - $matched[$name] = $this->values[$name]; - } - - if ($taken !== count($this->values)) { - foreach ($this->values as $name => $_) { - if (!array_key_exists($name, $definitions)) { - throw ColumnMismatchException::unexpectedColumn((string) $name); - } - } - } - - if ($columns === []) { - return new self($matched); - } - - $values = $matched; - $changed = true; - } - - $alignment = new ZoneAlignment(); - - foreach ($columns as $name => $definition) { - if ($alignment->inZone($definition, $values[$name])) { - continue; - } - - if (!$definition->matches($values[$name])) { - throw ColumnMismatchException::valueDoesNotMatch($definition, $values[$name]); - } - - // @mago-ignore analysis:mixed-assignment - $aligned = $alignment->align($definition, $values[$name]); - - if ($aligned !== $values[$name]) { - $values[$name] = $aligned; - $changed = true; - } - } - - return $changed ? new self($values) : $this; - } -} diff --git a/src/core/etl/src/Flow/ETL/Row/AdaptiveRowHydrator.php b/src/core/etl/src/Flow/ETL/Row/AdaptiveRowHydrator.php deleted file mode 100644 index 91017e3e52..0000000000 --- a/src/core/etl/src/Flow/ETL/Row/AdaptiveRowHydrator.php +++ /dev/null @@ -1,33 +0,0 @@ -delegate = NativeRowHydrator::isSupported() ? new NativeRowHydrator() : new PhpRowHydrator(); - } - - public function isNative(): bool - { - return $this->delegate instanceof NativeRowHydrator; - } - - public function dehydrate(Rows $rows): array - { - return $this->delegate->dehydrate($rows); - } - - public function hydrate(array $batch, Schema $schema): Rows - { - return $this->delegate->hydrate($batch, $schema); - } -} diff --git a/src/core/etl/src/Flow/ETL/Row/Comparator.php b/src/core/etl/src/Flow/ETL/Row/Comparator.php deleted file mode 100644 index 7efb5c9d7e..0000000000 --- a/src/core/etl/src/Flow/ETL/Row/Comparator.php +++ /dev/null @@ -1,13 +0,0 @@ -values()) !== array_keys($nextRow->values())) { - return false; - } - - $comparator = new TypedValueComparator(); - - foreach ($schema->definitions() as $definition) { - $name = $definition->entry()->name(); - - if (!$row->has($name) || !$nextRow->has($name)) { - continue; - } - - if (!$comparator->equals($definition->type(), $row->get($name), $nextRow->get($name))) { - return false; - } - } - - return true; - } -} diff --git a/src/core/etl/src/Flow/ETL/Row/Encoder.php b/src/core/etl/src/Flow/ETL/Row/Encoder.php deleted file mode 100644 index 0cab32540f..0000000000 --- a/src/core/etl/src/Flow/ETL/Row/Encoder.php +++ /dev/null @@ -1,25 +0,0 @@ - $batch - * - * @return list - */ - public function encode(array $batch): array; - - /** - * @param list $batch - * - * @return list - */ - public function decode(array $batch): array; -} diff --git a/src/core/etl/src/Flow/ETL/Row/Formatter/ASCIISchemaFormatter.php b/src/core/etl/src/Flow/ETL/Row/Formatter/ASCIISchemaFormatter.php index 23f563f02e..213f406330 100644 --- a/src/core/etl/src/Flow/ETL/Row/Formatter/ASCIISchemaFormatter.php +++ b/src/core/etl/src/Flow/ETL/Row/Formatter/ASCIISchemaFormatter.php @@ -4,9 +4,10 @@ namespace Flow\ETL\Row\Formatter; +use Flow\ETL\Column\AdaptiveBackend; use Flow\ETL\Formatter\AsciiTableFormatter; -use Flow\ETL\Row; use Flow\ETL\Rows; +use Flow\ETL\Rows\RowsBuilder; use Flow\ETL\Schema; use Flow\ETL\Schema\Definition; use Flow\ETL\Schema\SchemaFormatter; @@ -68,7 +69,7 @@ private function table(Schema $schema): Rows $values['metadata'] = json_encode($definition->metadata()->normalize(), JSON_THROW_ON_ERROR); } - $rows[] = new Row($values); + $rows[] = $values; } $columns = [str_schema('name'), str_schema('type'), bool_schema('nullable')]; @@ -77,7 +78,9 @@ private function table(Schema $schema): Rows $columns[] = str_schema('metadata'); } - return new Rows(schema(...$columns), ...$rows); + return (new RowsBuilder(schema(...$columns), new AdaptiveBackend())) + ->appendRows($rows) + ->finish(); } /** diff --git a/src/core/etl/src/Flow/ETL/Row/HydratedBatch.php b/src/core/etl/src/Flow/ETL/Row/HydratedBatch.php deleted file mode 100644 index 0e81ad170e..0000000000 --- a/src/core/etl/src/Flow/ETL/Row/HydratedBatch.php +++ /dev/null @@ -1,84 +0,0 @@ - $batch - * - * @throws SchemaMismatchException - */ - public function of(array $batch, Schema $schema): Rows - { - // Nullability is a Definition flag, never a property of a Type: definition_from_type() - // unwraps OptionalType into nullable: true over a BARE type, so $definition->type() is - // non-nullable even for a nullable column and would refuse every legitimate null. - $prepare = static fn(mixed $value, Definition $definition): mixed => ( - $value === null - ? ($definition->isNullable() ? null : throw new CastingException(null, $definition->type())) - : $definition->type()->cast($value) - ); - - foreach ($batch as $rowValues) { - foreach ($rowValues->metadata as $name => $metadata) { - // PHP casts a numeric array key to int, column names are always strings - $name = (string) $name; - - if ($schema->findDefinition($name) !== null) { - $schema = $schema->setMetadata($name, $metadata); - } - } - } - - $definitions = $schema->definitions(); - $rows = []; - - foreach ($batch as $rowIndex => $rowValues) { - $values = []; - - foreach ($definitions as $definition) { - $name = $definition->entry()->name(); - - if (!array_key_exists($name, $rowValues->values)) { - // never fill: Rows::conformed() runs Row::conformTo(), which pads a declared-nullable - // absence and refuses a NOT-NULL one as a missing column rather than as a null value - continue; - } - - try { - $values[$name] = $prepare($rowValues->values[$name], $definition); - } catch (TypesException) { - // Rows' gate reports a misfit this way, but $prepare runs before the Rows - // constructor, so the batch has to place the violation itself. - throw new SchemaMismatchException($rowIndex, ColumnMismatchException::valueDoesNotMatch( - $definition, - $rowValues->values[$name], - )); - } - } - - $rows[] = new Row($values); - } - - // every non-null value was just cast to its column's type - the one place a value is validated - return Rows::conformed($schema, $rows); - } -} diff --git a/src/core/etl/src/Flow/ETL/Row/Hydrator.php b/src/core/etl/src/Flow/ETL/Row/Hydrator.php deleted file mode 100644 index 9c1861d866..0000000000 --- a/src/core/etl/src/Flow/ETL/Row/Hydrator.php +++ /dev/null @@ -1,24 +0,0 @@ - - */ - public function dehydrate(Rows $rows): array; - - /** - * @param list $batch - * - * @throws SchemaMismatchException - */ - public function hydrate(array $batch, Schema $schema): Rows; -} diff --git a/src/core/etl/src/Flow/ETL/Row/InferredBatch.php b/src/core/etl/src/Flow/ETL/Row/InferredBatch.php deleted file mode 100644 index ee78177a1a..0000000000 --- a/src/core/etl/src/Flow/ETL/Row/InferredBatch.php +++ /dev/null @@ -1,73 +0,0 @@ - $batch - */ - public function of(array $batch): Rows - { - $schema = new Schema(); - - foreach ($batch as $rowValues) { - $definitions = []; - - /** @var mixed $value */ - foreach ($rowValues->values as $name => $value) { - // RawRowValues::$values is declared array, but PHP hands a numeric - // column name back as an INT key - a pivot names its columns by their own values. - // definition_from_type() takes Reference|string, so the cast is not redundant at runtime. - $name = (string) $name; - $definition = definition_from_type($name, $this->typeDetector->detectType($value), $value === null); - - if (array_key_exists($name, $rowValues->metadata)) { - $definition = $definition->setMetadata($rowValues->metadata[$name]); - } - - $definitions[] = $definition; - } - - $schema = $schema->merge(new Schema(...$definitions)); - } - - $rows = []; - - // the fold widens a column typed from one row's value to cover every row's - so a value the - // widened definition no longer accepts follows the schema this method just derived - foreach ($batch as $rowValues) { - $values = []; - - /** @var mixed $value */ - foreach ($rowValues->values as $name => $value) { - // see above - an int key here would miss the column the schema declares as "5" - $name = (string) $name; - $definition = $schema->get($name); - - $values[$name] = $definition->matches($value) ? $value : $definition->type()->cast($value); - } - - $rows[] = new Row($values); - } - - return new Rows($schema, ...$rows); - } -} diff --git a/src/core/etl/src/Flow/ETL/Row/NativeRowHydrator.php b/src/core/etl/src/Flow/ETL/Row/NativeRowHydrator.php deleted file mode 100644 index e4dca59f8d..0000000000 --- a/src/core/etl/src/Flow/ETL/Row/NativeRowHydrator.php +++ /dev/null @@ -1,46 +0,0 @@ -native = new RustRowHydratorNative(); - } - - public static function isSupported(): bool - { - return ( - extension_loaded('flow_php') - && class_exists(RustRowHydratorNative::class, false) - && method_exists(RustRowHydratorNative::class, 'datetimeZones') - ); - } - - public function dehydrate(Rows $rows): array - { - return $this->native->dehydrate($rows); - } - - public function hydrate(array $batch, Schema $schema): Rows - { - return $this->native->hydrate($batch, $schema); - } -} diff --git a/src/core/etl/src/Flow/ETL/Row/NullsOrder.php b/src/core/etl/src/Flow/ETL/Row/NullsOrder.php new file mode 100644 index 0000000000..ea567f0848 --- /dev/null +++ b/src/core/etl/src/Flow/ETL/Row/NullsOrder.php @@ -0,0 +1,19 @@ +schema()->definitions() as $definition) { - $name = $definition->entry()->name(); - - $values[$name] = $row->get($name); - $types[$name] = $definition->type(); - - if (!$definition->metadata()->isEmpty()) { - $metadata[$name] = $definition->metadata(); - } - } - - $batch[] = new TypedRowValues($values, $types, $metadata); - } - - return $batch; - } - - public function hydrate(array $batch, Schema $schema): Rows - { - return (new HydratedBatch())->of($batch, $schema); - } -} diff --git a/src/core/etl/src/Flow/ETL/Row/RawRowValues.php b/src/core/etl/src/Flow/ETL/Row/RawRowValues.php deleted file mode 100644 index 17398caaad..0000000000 --- a/src/core/etl/src/Flow/ETL/Row/RawRowValues.php +++ /dev/null @@ -1,19 +0,0 @@ - $values - * @param array $metadata - */ - public function __construct( - public array $values, - public array $metadata = [], - ) {} -} diff --git a/src/core/etl/src/Flow/ETL/Row/Reference.php b/src/core/etl/src/Flow/ETL/Row/Reference.php index f6a5025e4e..af0097a415 100644 --- a/src/core/etl/src/Flow/ETL/Row/Reference.php +++ b/src/core/etl/src/Flow/ETL/Row/Reference.php @@ -20,6 +20,8 @@ public function is(self $ref): bool; public function name(): string; + public function nulls(): NullsOrder; + public function sort(): SortOrder; public function to(): string; diff --git a/src/core/etl/src/Flow/ETL/Row/ResolvedReference.php b/src/core/etl/src/Flow/ETL/Row/ResolvedReference.php index 5173a4b7ba..913b38c8a0 100644 --- a/src/core/etl/src/Flow/ETL/Row/ResolvedReference.php +++ b/src/core/etl/src/Flow/ETL/Row/ResolvedReference.php @@ -4,13 +4,14 @@ namespace Flow\ETL\Row; +use Flow\ETL\Column\Column; use Flow\ETL\FlowContext; use Flow\ETL\Function\FunctionTree; use Flow\ETL\Function\ListFunctions; use Flow\ETL\Function\ScalarFunction; use Flow\ETL\Function\ScalarFunctionChain; use Flow\ETL\Function\StructureFunctions; -use Flow\ETL\Row; +use Flow\ETL\Rows; use Flow\Types\Type; final class ResolvedReference implements Reference @@ -28,6 +29,7 @@ public function __construct( private readonly Type $type, private readonly ?string $alias = null, private readonly SortOrder $sort = SortOrder::ASC, + private readonly NullsOrder $nulls = NullsOrder::FIRST, ) {} public function __toString(): string @@ -37,12 +39,12 @@ public function __toString(): string public function as(string $alias): self { - return new self($this->entry, $this->type, $alias, $this->sort); + return new self($this->entry, $this->type, $alias, $this->sort, $this->nulls); } - public function asc(): self + public function asc(NullsOrder $nulls = NullsOrder::FIRST): self { - return new self($this->entry, $this->type, $this->alias, SortOrder::ASC); + return new self($this->entry, $this->type, $this->alias, SortOrder::ASC, $nulls); } public function base(): string @@ -66,14 +68,14 @@ public function withChildren(array $children): static return $this; } - public function desc(): self + public function desc(NullsOrder $nulls = NullsOrder::LAST): self { - return new self($this->entry, $this->type, $this->alias, SortOrder::DESC); + return new self($this->entry, $this->type, $this->alias, SortOrder::DESC, $nulls); } - public function eval(Row $row, FlowContext $context): mixed + public function eval(Rows $rows, FlowContext $context): Column { - return $row->get($this->entry); + return $rows->column($this->base()); } public function hasAlias(): bool @@ -104,6 +106,11 @@ public function returns(): Type return $this->type; } + public function nulls(): NullsOrder + { + return $this->nulls; + } + public function sort(): SortOrder { return $this->sort; diff --git a/src/core/etl/src/Flow/ETL/Row/RowRenaming.php b/src/core/etl/src/Flow/ETL/Row/RowRenaming.php deleted file mode 100644 index 466a10d80e..0000000000 --- a/src/core/etl/src/Flow/ETL/Row/RowRenaming.php +++ /dev/null @@ -1,41 +0,0 @@ - $renames current_name => new_name - */ - private function __construct( - private array $renames, - ) {} - - /** - * @param array $renames current_name => new_name - */ - public static function of(array $renames): self - { - return new self($renames); - } - - public function apply(Row $row): Row - { - if ($this->renames === []) { - return $row; - } - - $values = []; - - // @mago-ignore analysis:mixed-assignment - foreach ($row->values() as $name => $value) { - $values[$this->renames[$name] ?? $name] = $value; - } - - return new Row($values); - } -} diff --git a/src/core/etl/src/Flow/ETL/Row/RowsBuffer.php b/src/core/etl/src/Flow/ETL/Row/RowsBuffer.php deleted file mode 100644 index 1e3bdef4f5..0000000000 --- a/src/core/etl/src/Flow/ETL/Row/RowsBuffer.php +++ /dev/null @@ -1,77 +0,0 @@ -): Rows - */ - private readonly Closure $batch; - - /** - * @var list - */ - private array $rows = []; - - /** - * @param int<1, max> $size - * @param null|Closure(Schema, list): Rows $batch builds each released batch - Rows::trusted(...) or - * Rows::conformed(...) for rows that already passed the gate; - * null checks every row through new Rows() - */ - public function __construct( - private readonly Schema $schema, - private readonly int $size, - ?Closure $batch = null, - ) { - // @mago-ignore analysis:invalid-operand - // @mago-ignore analysis:impossible-condition,redundant-comparison - if ($this->size < 1) { - throw new InvalidArgumentException('Buffer size must be greater than 0, given: ' . $this->size); - } - - $this->batch = - $batch ?? - /** - * @param list $rows - */ - static fn(Schema $schema, array $rows): Rows => new Rows($schema, ...$rows); - } - - public function add(Row $row): ?Rows - { - $this->rows[] = $row; - - if (count($this->rows) < $this->size) { - return null; - } - - return $this->flush(); - } - - public function flush(): ?Rows - { - if ($this->rows === []) { - return null; - } - - $batch = ($this->batch)($this->schema, $this->rows); - $this->rows = []; - - return $batch; - } -} diff --git a/src/core/etl/src/Flow/ETL/Row/TypedRowValues.php b/src/core/etl/src/Flow/ETL/Row/TypedRowValues.php deleted file mode 100644 index 88cd6db06b..0000000000 --- a/src/core/etl/src/Flow/ETL/Row/TypedRowValues.php +++ /dev/null @@ -1,22 +0,0 @@ - $values - * @param array> $types - * @param array $metadata - */ - public function __construct( - public array $values, - public array $types, - public array $metadata = [], - ) {} -} diff --git a/src/core/etl/src/Flow/ETL/Row/UnresolvedReference.php b/src/core/etl/src/Flow/ETL/Row/UnresolvedReference.php index 535e77675d..1e14517a89 100644 --- a/src/core/etl/src/Flow/ETL/Row/UnresolvedReference.php +++ b/src/core/etl/src/Flow/ETL/Row/UnresolvedReference.php @@ -4,6 +4,7 @@ namespace Flow\ETL\Row; +use Flow\ETL\Column\Column; use Flow\ETL\Exception\InvalidLogicException; use Flow\ETL\FlowContext; use Flow\ETL\Function\FunctionTree; @@ -11,7 +12,7 @@ use Flow\ETL\Function\ScalarFunction; use Flow\ETL\Function\ScalarFunctionChain; use Flow\ETL\Function\StructureFunctions; -use Flow\ETL\Row; +use Flow\ETL\Rows; use Flow\ETL\Schema\Definition; use Flow\Types\Type; @@ -30,6 +31,7 @@ public function __construct( private readonly string $entry, private readonly ?string $alias = null, private readonly SortOrder $sort = SortOrder::ASC, + private readonly NullsOrder $nulls = NullsOrder::FIRST, ) {} public static function init(string|Reference $ref): Reference @@ -48,12 +50,12 @@ public function __toString(): string public function as(string $alias): self { - return new self($this->entry, $alias, $this->sort); + return new self($this->entry, $alias, $this->sort, $this->nulls); } - public function asc(): self + public function asc(NullsOrder $nulls = NullsOrder::FIRST): self { - return new self($this->entry, $this->alias, SortOrder::ASC); + return new self($this->entry, $this->alias, SortOrder::ASC, $nulls); } public function base(): string @@ -90,14 +92,14 @@ public function returns(): Type throw InvalidLogicException::because(self::UNRESOLVED, 'returns', $this->entry); } - public function desc(): self + public function desc(NullsOrder $nulls = NullsOrder::LAST): self { - return new self($this->entry, $this->alias, SortOrder::DESC); + return new self($this->entry, $this->alias, SortOrder::DESC, $nulls); } - public function eval(Row $row, FlowContext $context): mixed + public function eval(Rows $rows, FlowContext $context): Column { - return $row->get($this->entry); + throw InvalidLogicException::because(self::UNRESOLVED, 'eval', $this->entry); } public function hasAlias(): bool @@ -135,9 +137,15 @@ public function resolve(Definition $definition): ResolvedReference $definition->isNullable() ? type_optional($type) : $type, $this->alias, $this->sort, + $this->nulls, ); } + public function nulls(): NullsOrder + { + return $this->nulls; + } + public function sort(): SortOrder { return $this->sort; diff --git a/src/core/etl/src/Flow/ETL/Rows.php b/src/core/etl/src/Flow/ETL/Rows.php index 3b735c54fc..68f35666d7 100644 --- a/src/core/etl/src/Flow/ETL/Rows.php +++ b/src/core/etl/src/Flow/ETL/Rows.php @@ -4,580 +4,444 @@ namespace Flow\ETL; -use ArrayAccess; -use ArrayIterator; use Countable; +use Flow\ETL\Column\AdaptiveBackend; +use Flow\ETL\Column\Backend; +use Flow\ETL\Column\Column; +use Flow\ETL\Column\ValueColumn; use Flow\ETL\Exception\ColumnMismatchException; use Flow\ETL\Exception\InvalidArgumentException; -use Flow\ETL\Exception\RuntimeException; +use Flow\ETL\Exception\SchemaDefinitionNotFoundException; use Flow\ETL\Exception\SchemaMismatchException; -use Flow\ETL\Hash\Algorithm; -use Flow\ETL\Hash\NativePHPHash; -use Flow\ETL\Join\Expression; -use Flow\ETL\Join\HashJoin\Joiner; -use Flow\ETL\Join\HashJoin\JoinSide; -use Flow\ETL\Join\HashJoin\RowMerger; -use Flow\ETL\Join\Join; -use Flow\ETL\Join\JoinSchema; -use Flow\ETL\Row\Comparator; -use Flow\ETL\Row\Comparator\NativeComparator; use Flow\ETL\Row\Reference; -use Flow\ETL\Row\SortOrder; -use Flow\ETL\Schema\Definition; -use Flow\ETL\Serializer\DomValueCodec; -use Flow\ETL\Sort\ValuesSorter; -use Flow\Types\Exception\InvalidTypeException; +use Flow\ETL\Rows\ColumnRetyping; +use Flow\ETL\Schema\Formatter\InlineSchemaFormatter; +use Flow\ETL\Schema\SimilarNames; +use Flow\ETL\Sort\RowOrder; +use Flow\Floe\FrameDecoder; +use Flow\Floe\FrameEncoder; +use Flow\Types\Value\Json; use Generator; -use Iterator; -use IteratorAggregate; +use function array_diff_key; +use function array_fill; +use function array_flip; use function array_key_exists; -use function array_map; -use function array_reverse; -use function array_slice; +use function array_keys; +use function array_shift; use function array_values; use function count; -use function Flow\Types\DSL\type_integer; use function implode; -use function is_int; -use function iterator_to_array; +use function max; +use function min; use function sprintf; /** - * @implements \ArrayAccess - * @implements \IteratorAggregate + * @type RowsPayload = array{schema: Schema, frame: string} */ -final class Rows implements ArrayAccess, Countable, IteratorAggregate +final class Rows implements Countable { /** - * @var array + * @param array $columns keyed and ordered by $schema */ - private array $rows; + private function __construct( + private Schema $schema, + private array $columns, + private int $count, + ) {} /** - * @throws SchemaMismatchException + * @return RowsPayload */ - public function __construct( - private Schema $schema, - Row ...$rows, - ) { - $this->rows = []; - - foreach ($rows as $row) { - try { - $this->rows[] = $row->matchTo($schema); - } catch (ColumnMismatchException $e) { - throw new SchemaMismatchException(count($this->rows), $e); - } - } + public function __serialize(): array + { + return ['schema' => $this->schema, 'frame' => $this->encodeFrame()]; } /** - * Skips the shape check the constructor performs. The caller vouches that every row already - * satisfies $schema and stores its columns in the Schema's order - which holds when the same - * operation produced the schema and the rows, or when the rows are a subset or a permutation of - * a batch that already passed. - * - * Engine paths only. + * A serialized batch carries no context, so it decodes in the default backend. * - * @param array $rows re-indexed here - first(), last(), chunks() and offsetGet() read by position + * @param RowsPayload $data */ - public static function trusted(Schema $schema, array $rows): self + public function __unserialize(array $data): void { - $instance = new self($schema); - $instance->rows = array_values($rows); + $rows = (new FrameDecoder())->decode($data['frame'], $data['schema'], new AdaptiveBackend()); - return $instance; + $this->schema = $rows->schema; + $this->columns = $rows->columns; + $this->count = $rows->count; } - /** - * The constructor's shape check without its value check. Columns take the Schema's order, a declared-nullable - * absence is padded, and a missing NOT NULL column, an unknown column or a null under NOT NULL is refused - but a - * non-null value is not validated against its type, because the caller produced it by casting to, or decoding - * from, that type. A value is validated once, where it enters the engine. - * - * Engine paths only. - * - * @param array $rows - * - * @throws SchemaMismatchException - */ - public static function conformed(Schema $schema, array $rows): self + public static function empty(Schema $schema, Backend $backend): self { - $instance = new self($schema); + $columns = []; - foreach ($rows as $row) { - try { - $instance->rows[] = $row->conformTo($schema); - } catch (ColumnMismatchException $e) { - throw new SchemaMismatchException(count($instance->rows), $e); - } + foreach ($schema->definitions() as $name => $definition) { + $columns[$name] = $backend->builder($definition)->finish(); } - return $instance; + return new self($schema, $columns, 0); } /** - * @return array{schema: Schema, rows: list>} + * @param array $columns keyed and ordered by $schema, every one of $count rows + * + * @throws InvalidArgumentException + * @throws SchemaMismatchException a NOT NULL column holds a null */ - public function __serialize(): array + public static function fromColumns(Schema $schema, array $columns, int $count): self { - $codec = new DomValueCodec(); - $domColumns = []; + if (array_keys($columns) !== array_keys($schema->definitions())) { + throw new InvalidArgumentException(sprintf( + 'Rows::fromColumns() expects columns [%s], got [%s]', + implode(', ', array_keys($schema->definitions())), + implode(', ', array_keys($columns)), + )); + } - foreach ($this->schema->definitions() as $definition) { - if ($codec->handles($definition->type())) { - $domColumns[$definition->entry()->name()] = $definition->type(); + foreach ($columns as $name => $column) { + if ($column->count() !== $count) { + throw new InvalidArgumentException(sprintf( + 'Column "%s" holds %d rows, the batch %d', + $name, + $column->count(), + $count, + )); } } - $rows = []; - - foreach ($this->rows as $row) { - $values = $row->values(); + $retyping = new ColumnRetyping(); - foreach ($domColumns as $name => $type) { - if (array_key_exists($name, $values)) { - $values[$name] = $codec->encode($type, $values[$name]); - } + foreach ($schema->definitions() as $name => $definition) { + if ($columns[$name] instanceof ValueColumn) { + throw ColumnMismatchException::untypedColumn($definition); } - $rows[] = $values; + $retyping->notNull($definition, $columns[$name]); } - return ['schema' => $this->schema, 'rows' => $rows]; + return new self($schema, $columns, $count); } /** - * @param array{schema: Schema, rows: list>} $data + * @param int<1, max> $size + * + * @return \Generator */ - public function __unserialize(array $data): void + public function chunks(int $size): Generator { - $this->schema = $data['schema']; - - $codec = new DomValueCodec(); - $domColumns = []; - - foreach ($this->schema->definitions() as $definition) { - if ($codec->handles($definition->type())) { - $domColumns[$definition->entry()->name()] = $definition->type(); - } - } - - $rows = []; - - foreach ($data['rows'] as $values) { - foreach ($domColumns as $name => $type) { - if (array_key_exists($name, $values)) { - $values[$name] = $codec->decode($type, $values[$name]); - } - } - - $rows[] = new Row($values); + for ($offset = 0; $offset < $this->count; $offset += $size) { + yield $this->slice($offset, min($size, $this->count - $offset)); } - - $this->rows = (new self($this->schema, ...$rows))->rows; } /** - * @throws SchemaMismatchException + * @throws InvalidArgumentException */ - public function add(Row ...$rows): self + public function column(string $name): Column { - $matched = $this->rows; - - foreach ($rows as $row) { - try { - $matched[] = $row->matchTo($this->schema); - } catch (ColumnMismatchException $e) { - throw new SchemaMismatchException(count($matched), $e); - } + if (array_key_exists($name, $this->columns)) { + return $this->columns[$name]; } - return self::trusted($this->schema, $matched); + $suggestions = (new SimilarNames())->closestTo($name, array_values($this->schema->references()->names())); + + throw new InvalidArgumentException( + $suggestions === [] + ? "Column \"{$name}\" does not exist." + : "Column \"{$name}\" does not exist. Did you mean one of the following? [\"" + . implode('", "', $suggestions) + . '"]', + ); } /** - * @return array + * @return array schema order */ - public function all(): array + public function columns(): array { - return $this->rows; + return $this->columns; } - /** - * @param int<1, max> $size - * - * @return \Generator - */ - public function chunks(int $size): Generator + public function encodeFrame(): string { - foreach (array_chunk($this->rows, $size) as $chunk) { - yield self::trusted($this->schema, $chunk); - } + return (new FrameEncoder())->encode($this); } /** - * Re-checks the batch against a different Schema and adopts it - the door the constructor opens, - * for rows already gathered into a batch. + * The same batch with every column in $backend's storage - itself when $backend already owns every column. * - * @throws SchemaMismatchException + * @throws ColumnMismatchException a null under NOT NULL on a column that is copied */ - public function matchTo(Schema $schema): self - { - return new self($schema, ...$this->rows); - } - - public function count(): int + public function adoptedBy(Backend $backend): self { - return count($this->rows); - } + $columns = []; + $copied = false; - public function diffLeft(self $rows): self - { - $comparator = new NativeComparator(); - $differentRows = []; - - foreach ($this->rows as $row) { - $found = false; + foreach ($this->schema->definitions() as $name => $definition) { + $columns[$name] = $backend->adopt($definition, $this->columns[$name]); + $copied = $copied || $columns[$name] !== $this->columns[$name]; + } - foreach ($rows->rows as $otherRow) { - if ($comparator->equals($row, $otherRow, $this->schema)) { - $found = true; + return $copied ? new self($this->schema, $columns, $this->count) : $this; + } - break; - } + /** + * Every input carries this schema; zero-row inputs add no rows. The result's columns are $backend's. + * + * @throws InvalidArgumentException + */ + public function concat(Backend $backend, self ...$others): self + { + $inputs = $this->count > 0 ? [$this] : []; + $count = $this->count; + + foreach ($others as $other) { + if (!$other->schema->isSame($this->schema)) { + throw InvalidArgumentException::because( + 'Cannot concat Rows with different schemas: [%s] and [%s]', + (new InlineSchemaFormatter())->format($this->schema), + (new InlineSchemaFormatter())->format($other->schema), + ); } - if (!$found) { - $differentRows[] = $row; + if ($other->count > 0) { + $inputs[] = $other; + $count += $other->count; } } - return self::trusted($this->schema, $differentRows); - } + $head = array_shift($inputs); - public function diffRight(self $rows): self - { - $comparator = new NativeComparator(); - $differentRows = []; + if ($head === null) { + return $this->adoptedBy($backend); + } - foreach ($rows->rows as $row) { - $found = false; + $columns = []; - foreach ($this->rows as $otherRow) { - if ($comparator->equals($row, $otherRow, $this->schema)) { - $found = true; + foreach ($this->schema->definitions() as $name => $definition) { + $first = $head->columns[$name]; + $rest = []; + $sameClass = true; - break; - } + foreach ($inputs as $input) { + $part = $input->columns[$name]; + $rest[] = $part; + $sameClass = $sameClass && $part::class === $first::class; } - if (!$found) { - $differentRows[] = $row; - } - } + if ($sameClass) { + $columns[$name] = $backend->adopt($definition, $first->concat(...$rest)); - // the surviving rows come from the right side, so the right side's schema describes them - return self::trusted($rows->schema, $differentRows); - } + continue; + } - public function drop(int $size): self - { - if ($size === 0) { - return $this; - } + $builder = $backend->builder($definition); - return self::trusted($this->schema, array_slice($this->rows, $size)); - } + foreach ([$first, ...$rest] as $part) { + $builder->appendTake($part, range(0, $part->count() - 1)); + } - public function dropRight(int $size): self - { - if ($size === 0) { - return $this; + $columns[$name] = $builder->finish(); } - return self::trusted($this->schema, array_slice($this->rows, 0, -$size)); - } - - public function empty(): bool - { - return $this->count() === 0; + return new self($this->schema, $columns, $count); } - public function first(): Row - { - return $this->rows[0] ?? throw new RuntimeException('First row does not exist in empty collection'); - } - - /** - * @return \Iterator - */ - public function getIterator(): Iterator + public function count(): int { - return new ArrayIterator($this->rows); + return $this->count; } - public function hash(Algorithm $algorithm = new NativePHPHash()): string + public function drop(int $size): self { - $hash = ''; + if ($size < 0) { + throw new InvalidArgumentException('Size must be greater than or equal to 0'); + } - foreach ($this->rows as $row) { - $hash .= $row->hash($this->schema, $algorithm); + if ($size === 0) { + return $this; } - return $algorithm->hash($hash); + return $this->slice(min($size, $this->count), $this->count - min($size, $this->count)); } - /** - * @param int $count - Count of rows to return. Must be >= 0. - * - * @throws InvalidArgumentException When count is negative - * @throws InvalidTypeException When count is not an integer */ - public function head(int $count): self + public function dropRight(int $size): self { - $count = type_integer()->assert($count); - - if ($count < 0) { - throw new InvalidArgumentException('Count must be greater than or equal to 0'); + if ($size < 0) { + throw new InvalidArgumentException('Size must be greater than or equal to 0'); } - if ($count === 0) { - return self::trusted($this->schema, []); + if ($size === 0) { + return $this; } - return self::trusted($this->schema, array_slice($this->rows, 0, $count)); + return $this->slice(0, $this->count - min($size, $this->count)); } - public function joinCross(self $right, string $joinPrefix = 'joined_'): self - { - $schema = (new JoinSchema($joinPrefix))->cross($this->schema, $right->schema); - - /** - * @var array $joined - */ - $joined = []; - - $merger = new RowMerger($joinPrefix); - - foreach ($this->rows as $leftRow) { - foreach ($right->rows as $rightRow) { - try { - $joined[] = $merger->merge($leftRow, $rightRow); - } catch (InvalidArgumentException $e) { - throw new InvalidArgumentException($e->getMessage() . '. Please consider using join prefix option'); + /** + * @param list $indices rows to keep, in the order given + * + * @throws InvalidArgumentException + */ + public function gather(array $indices): self + { + if ($indices !== [] && (min($indices) < 0 || max($indices) >= $this->count)) { + foreach ($indices as $index) { + if ($index < 0 || $index >= $this->count) { + throw new InvalidArgumentException(sprintf( + 'Rows::gather() index %d is outside a batch of %d rows', + $index, + $this->count, + )); } } } - return new self($schema, ...$joined); - } + $columns = []; - /** - * @throws InvalidArgumentException - */ - public function joinInner(self $right, Expression $expression): self - { - return $this->joinUsing($right, $expression, Join::inner); - } - - /** - * @throws InvalidArgumentException - */ - public function joinLeft(self $right, Expression $expression): self - { - return $this->joinUsing($right, $expression, Join::left); - } + foreach ($this->columns as $name => $column) { + $columns[$name] = $column->take($indices); + } - /** - * @throws InvalidArgumentException - */ - public function joinLeftAnti(self $right, Expression $expression): self - { - return $this->joinUsing($right, $expression, Join::left_anti); + return new self($this->schema, $columns, count($indices)); } - /** - * @throws InvalidArgumentException - */ - public function joinRight(self $right, Expression $expression): self + public function isEmpty(): bool { - return $this->joinUsing($right, $expression, Join::right); + return $this->count === 0; } /** - * @throws InvalidArgumentException + * Adopts $schema. A changed column whose type change the types prove is restamped; any other change is checked + * value by value and rebuilt in $backend. + * + * @throws SchemaMismatchException */ - private function joinUsing(self $right, Expression $expression, Join $type): self + public function matchTo(Schema $schema, Backend $backend): self { - $single = static function (self $rows): Generator { - yield $rows; - }; - - $joiner = new Joiner($expression, $type); + if ($schema->isSame($this->schema)) { + return new self($schema, $this->columns, $this->count); + } - /** - * @var array $joined - */ - $joined = []; + foreach ($this->schema->definitions() as $definition) { + $name = $definition->entry()->name(); - foreach ($joiner->join(JoinSide::of($single($this)), JoinSide::of($single($right))) as $batch) { - foreach ($batch as $row) { - $joined[] = $row; + if ($this->count > 0 && $schema->findDefinition($name) === null) { + throw new SchemaMismatchException(0, ColumnMismatchException::unexpectedColumn($name)); } } - return new self($joiner->schema($this->schema, $right->schema), ...$joined); - } + $columns = []; + $changed = []; + $retyping = new ColumnRetyping(); - public function last(): ?Row - { - if (empty($this->rows)) { - return null; - } + foreach ($schema->definitions() as $name => $definition) { + $own = $this->schema->findDefinition($definition->entry()->name()); - return $this->rows[count($this->rows) - 1]; - } + if ($own === null) { + $columns[$name] = $retyping->absent($definition, $this->count, $backend); - public function merge(self $rows): self - { - if ($this->empty()) { - return $rows; - } + continue; + } - if ($rows->empty()) { - return $this; - } + $restamped = $retyping->restamped($this->columns[$name], $own, $definition); - if (!$this->schema->isSame($rows->schema())) { - // names alone cannot show a type-only disagreement - both sides would print identically - $describe = static fn(Schema $schema): string => implode(', ', array_map( - static fn(Definition $definition): string => ( - $definition->entry()->name() - . ': ' - . ($definition->isNullable() ? '?' : '') - . $definition->type()->toString() - ), - $schema->definitions(), - )); + if ($restamped === null) { + $changed[$name] = [$this->columns[$name], $definition]; + } - throw new InvalidArgumentException(sprintf( - 'Cannot merge Rows with different schemas: [%s] and [%s]', - $describe($this->schema), - $describe($rows->schema()), - )); + $columns[$name] = $restamped; } - return self::trusted($this->schema, [...$this->rows, ...$rows->rows]); - } + $retyping->validate($changed, $this->count); - /** - * @param int $offset - * - * @throws InvalidArgumentException - */ - public function offsetExists($offset): bool - { - // @mago-ignore analysis:impossible-condition,redundant-type-comparison - if (!is_int($offset)) { - throw new InvalidArgumentException('Rows accepts only integer offsets'); + foreach ($changed as $name => [$column, $definition]) { + $columns[$name] = $retyping->rebuilt($column, $definition, $backend); } - return isset($this->rows[$offset]); + /** @var array $columns */ + return self::fromColumns($schema, $columns, $this->count); } /** - * @param int $offset + * Drops the columns $schema does not declare and adopts it, converting a kept column whose definition $schema + * changes (in $backend). Contrast select(), which keeps the named columns as they are. Widening the batch with + * columns it lacks is not a projection - that goes through matchTo(). * - * @throws InvalidArgumentException + * @throws SchemaMismatchException */ - public function offsetGet($offset): Row + public function project(Schema $schema, Backend $backend): self { - if ($this->offsetExists($offset)) { - return $this->rows[$offset]; - } + $definitions = []; + $columns = []; - throw new InvalidArgumentException("Row {$offset} does not exists."); - } + foreach ($schema->definitions() as $name => $definition) { + $own = $this->schema->findDefinition($definition->entry()->name()); - public function offsetSet(mixed $offset, mixed $value): void - { - throw new RuntimeException('In order to add new rows use Rows::add(Row $row) : self'); - } + if ($own !== null) { + $definitions[] = $own; + $columns[$name] = $this->columns[$name]; + } + } - /** - * @param int $offset - * - * @throws RuntimeException - */ - public function offsetUnset(mixed $offset): void - { - throw new RuntimeException('In order to remove rows use Rows::remove(int $offset) : self'); + return (new self(new Schema(...$definitions), $columns, $this->count))->matchTo($schema, $backend); } /** - * @param Reference|string $reference - * @param Reference|string ...$references - * - * @throws InvalidArgumentException + * The named columns as they are, in schema order - every column kept unconverted, in whatever backend built it. + * Contrast project(), which adopts a given schema and converts the columns it changes. * - * @return array + * @throws SchemaDefinitionNotFoundException */ - /** - * @return array - */ - /** - * Drops the columns $schema does not declare and adopts it. Widening or retyping the batch is - * not a projection - that goes through matchTo(). - * - * @throws SchemaMismatchException - */ - public function project(Schema $schema): self + public function select(string ...$names): self { - $projected = []; + $keep = array_flip($names); + $definitions = []; + $columns = []; - foreach ($this->rows as $row) { - $projected[] = $row->project($schema); + foreach ($this->schema->definitions() as $name => $definition) { + if (array_key_exists($definition->entry()->name(), $keep)) { + $definitions[] = $definition; + $columns[$name] = $this->columns[$name]; + } } - foreach ($schema->definitions() as $name => $definition) { - $own = $this->schema->findDefinition($name); - - if ($own === null || !$own->isSame($definition)) { - return new self($schema, ...$projected); + if (count($definitions) !== count($keep)) { + foreach ($names as $selected) { + if ($this->schema->findDefinition($selected) === null) { + throw SchemaDefinitionNotFoundException::withAvailable( + $selected, + ...$this->schema->references()->names(), + ); + } } } - // every kept column kept its definition, so each value already passed the gate under it - return self::trusted($schema, $projected); + return new self(new Schema(...$definitions), $columns, $this->count); } + /** + * @throws InvalidArgumentException + * + * @return list + */ public function reduceToArray(string|Reference $reference): array { - $ids = []; - - foreach ($this->rows as $row) { - $ids[] = $row->get($reference); + if ($this->count === 0) { + return []; } - return $ids; + return $this->column($reference instanceof Reference ? $reference->base() : $reference)->values(); } public function remove(int $offset): self { - if (!$this->offsetExists($offset)) { + if ($offset < 0 || $offset >= $this->count) { throw new InvalidArgumentException("Rows does not have {$offset} offset"); } - $rows = iterator_to_array($this->getIterator()); - unset($rows[$offset]); - - return self::trusted($this->schema, array_values($rows)); - } - - public function reverse(): self - { - return self::trusted($this->schema, array_reverse($this->rows)); + return $this->gather(array_values(array_diff_key(range(0, $this->count - 1), [$offset => true]))); } public function schema(): Schema @@ -588,21 +452,24 @@ public function schema(): Schema /** * @throws InvalidArgumentException */ - public function sortAscending(string|Reference $reference): self + public function slice(int $offset, int $length): self { - $values = []; - - foreach ($this->rows as $index => $row) { - $values[$index] = $row->get($reference); + if ($offset < 0 || $length < 0 || ($offset + $length) > $this->count) { + throw new InvalidArgumentException(sprintf( + 'Rows::slice(%d, %d) is outside a batch of %d rows', + $offset, + $length, + $this->count, + )); } - $rows = []; + $columns = []; - foreach (array_keys(ValuesSorter::sort($values, SortOrder::ASC)) as $index) { - $rows[] = $this->rows[$index]; + foreach ($this->columns as $name => $column) { + $columns[$name] = $column->slice($offset, $length); } - return self::trusted($this->schema, $rows); + return new self($this->schema, $columns, $length); } /** @@ -610,108 +477,130 @@ public function sortAscending(string|Reference $reference): self */ public function sortBy(Reference ...$references): self { - $rows = $this; + if ($this->count === 0) { + return $this; + } + + $order = new RowOrder(array_values($references)); + + return $this->gather($order->permutation($order->keys($this), $this->count)); + } - foreach (array_reverse($references) as $ref) { - $rows = $ref->sort() === SortOrder::ASC ? $rows->sortAscending($ref) : $rows->sortDescending($ref); + public function take(int $size): self + { + if ($size < 0) { + throw new InvalidArgumentException('Size must be greater than or equal to 0'); } - return $rows; + return $this->slice(0, min($size, $this->count)); } /** - * @throws InvalidArgumentException + * @return array> */ - public function sortDescending(string|Reference $reference): self + public function toArray(bool $withKeys = true): array { - $values = []; - - foreach ($this->rows as $index => $row) { - $values[$index] = $row->get($reference); - } + // @mago-ignore analysis:possibly-invalid-argument + $array = $this->count === 0 ? [] : array_fill(0, $this->count, []); - $rows = []; + foreach ($this->columns as $name => $column) { + // @mago-ignore analysis:mixed-assignment + foreach ($column->values() as $i => $value) { + if ($value instanceof Json) { + $value = $value->toArray(); + } - foreach (array_keys(ValuesSorter::sort($values, SortOrder::DESC)) as $index) { - $rows[] = $this->rows[$index]; + $withKeys ? ($array[$i][$name] = $value) : ($array[$i][] = $value); + } } - return self::trusted($this->schema, $rows); + return $array; } /** - * @param int $count - Count of rows to return. Must be >= 0. + * @throws InvalidArgumentException * - * @throws InvalidArgumentException When count is negative - * @throws InvalidTypeException When count is not an integer */ - public function tail(int $count): self + * @return array the logical row, schema order + */ + public function values(int $i): array { - $count = type_integer()->assert($count); - - if ($count < 0) { - throw new InvalidArgumentException('Count must be greater than or equal to 0'); + if ($i < 0 || $i >= $this->count) { + throw InvalidArgumentException::because('Row %d does not exist in a batch of %d rows', $i, $this->count); } - if ($count === 0) { - return self::trusted($this->schema, []); - } - - $rowsCount = count($this->rows); + $values = []; - if ($count >= $rowsCount) { - return self::trusted($this->schema, $this->rows); + foreach ($this->columns as $name => $column) { + $values[$name] = $column->value($i); } - return self::trusted($this->schema, array_slice($this->rows, -$count)); - } - - public function take(int $size): self - { - return self::trusted($this->schema, array_slice($this->rows, 0, $size)); - } - - public function takeRight(int $size): self - { - return self::trusted($this->schema, array_reverse(array_slice($this->rows, -$size, $size))); + return $values; } /** - * @return array> + * @param array $columns each of count() rows; a same-named column is replaced in place + * + * @throws InvalidArgumentException */ - public function toArray(bool $withKeys = true): array - { - $array = []; + public function withColumns(Schema $schema, array $columns): self + { + foreach ($columns as $name => $column) { + if ($column->count() !== $this->count) { + throw new InvalidArgumentException(sprintf( + 'Column "%s" holds %d rows, the batch %d', + $name, + $column->count(), + $this->count, + )); + } + } - foreach ($this->rows as $row) { - $array[] = $row->toArray($withKeys); + $merged = []; + + foreach ($schema->definitions() as $name => $_) { + if (array_key_exists($name, $columns)) { + $merged[$name] = $columns[$name]; + } elseif (array_key_exists($name, $this->columns)) { + $merged[$name] = $this->columns[$name]; + } } - return $array; + return self::fromColumns($schema, $merged, $this->count); } - public function unique(Comparator $comparator = new NativeComparator()): self + /** + * Positional restamp: the n-th definition of $schema retypes the n-th column (Column::withType). + * + * @throws ColumnMismatchException + * @throws SchemaMismatchException a null under a NOT NULL definition, at its row + * @throws InvalidArgumentException + */ + public function withSchema(Schema $schema): self { - /** - * @var array $uniqueRows - */ - $uniqueRows = []; - - foreach ($this->rows as $row) { - $alreadyAdded = false; + if ($schema->count() !== $this->schema->count()) { + throw new InvalidArgumentException(sprintf( + 'Rows::withSchema() expects %d definitions, got %d', + $this->schema->count(), + $schema->count(), + )); + } - foreach ($uniqueRows as $uniqueRow) { - if ($comparator->equals($row, $uniqueRow, $this->schema)) { - $alreadyAdded = true; + $columns = array_values($this->columns); + $retyped = []; + $position = 0; + $retyping = new ColumnRetyping(); - break; - } - } + foreach ($schema->definitions() as $name => $definition) { + $column = $columns[$position++]; + $retyping->notNull($definition, $column); - if (!$alreadyAdded) { - $uniqueRows[] = $row; + try { + $retyped[$name] = $column->withType($definition->type()); + } catch (InvalidArgumentException $e) { + throw ColumnMismatchException::retype($definition, $e->getMessage()); } } - return self::trusted($this->schema, array_values($uniqueRows)); + return new self($schema, $retyped, $this->count); } } diff --git a/src/core/etl/src/Flow/ETL/Rows/ColumnRetyping.php b/src/core/etl/src/Flow/ETL/Rows/ColumnRetyping.php new file mode 100644 index 0000000000..93de4105b1 --- /dev/null +++ b/src/core/etl/src/Flow/ETL/Rows/ColumnRetyping.php @@ -0,0 +1,139 @@ + $definition + * + * @throws SchemaMismatchException + */ + public function absent(Definition $definition, int $count, Backend $backend): Column + { + if (!$definition->isNullable()) { + if ($count > 0) { + throw new SchemaMismatchException(0, ColumnMismatchException::missingColumn($definition)); + } + + return $backend->builder($definition)->finish(); + } + + return $backend->constant($definition, null, $count); + } + + /** + * $column under $definition for a declared schema over another frame's batch: restamped when the types prove the + * change, otherwise cast value by value into $backend - a value the definition does not take is cast, not refused. + * Contrast validate(), which refuses it. + * + * @param Definition $own + * @param Definition $definition + * + * @throws SchemaMismatchException a value the cast refuses, at its row + */ + public function cast(Column $column, Definition $own, Definition $definition, Backend $backend): Column + { + return $this->restamped($column, $own, $definition) ?? $this->rebuilt($column, $definition, $backend); + } + + /** + * Refuses the first null of $column when $definition is NOT NULL. + * + * @param Definition $definition + * + * @throws SchemaMismatchException + */ + public function notNull(Definition $definition, Column $column): void + { + if ($definition->isNullable() || $column->nullCount() === 0) { + return; + } + + $index = 0; + + while (!$column->isNull($index)) { + $index++; + } + + throw new SchemaMismatchException($index, ColumnMismatchException::valueDoesNotMatch($definition, null)); + } + + /** + * $column's values built anew under $definition in $backend. + * + * @param Definition $definition + * + * @throws SchemaMismatchException + */ + public function rebuilt(Column $column, Definition $definition, Backend $backend): Column + { + $builder = $backend->builder($definition); + $builder->appendMany($column->values()); + + return $builder->finish(); + } + + /** + * $column as it is when $own is $definition, restamped when the types prove the change and no null breaks a NOT NULL + * definition, null otherwise - the caller then casts or validates it. + * + * @param Definition $own + * @param Definition $definition + */ + public function restamped(Column $column, Definition $own, Definition $definition): ?Column + { + if ($own->isSame($definition)) { + return $column; + } + + if (!$definition->isNullable() && $column->nullCount() > 0) { + return null; + } + + return (new Retype())->proves($column->type(), $definition->type()) + ? $column->withType($definition->type()) + : null; + } + + /** + * Rows::matchTo()'s check of the columns restamped() could not prove: every value must already be one its + * definition takes - nothing is cast. One row-major scan, so the refusal names the lowest bad row and, within it, + * the first column in the given order. Contrast cast(), which converts such values instead. + * + * @param array}> $changed by name, in schema order + * + * @throws SchemaMismatchException + */ + public function validate(array $changed, int $count): void + { + for ($i = 0; $i < $count; $i++) { + foreach ($changed as [$column, $definition]) { + // @mago-ignore analysis:mixed-assignment + $value = $column->value($i); + + if ($value === null ? !$definition->isNullable() : !$definition->matches($value)) { + throw new SchemaMismatchException($i, ColumnMismatchException::valueDoesNotMatch( + $definition, + $value, + )); + } + } + } + } +} diff --git a/src/core/etl/src/Flow/ETL/Rows/RowHashes.php b/src/core/etl/src/Flow/ETL/Rows/RowHashes.php new file mode 100644 index 0000000000..c0a1b8d075 --- /dev/null +++ b/src/core/etl/src/Flow/ETL/Rows/RowHashes.php @@ -0,0 +1,58 @@ + + */ + public function of(Rows $rows, Algorithm $algorithm = new NativePHPHash()): array + { + if ($rows->isEmpty()) { + return []; + } + + $formatter = new TypedValueFormatter(); + $comparable = new ComparableValues(); + // @mago-ignore analysis:possibly-invalid-argument + $strings = array_fill(0, $rows->count(), ''); + + foreach ($rows->schema()->sort()->definitions() as $definition) { + $name = $definition->entry()->name(); + $type = $definition->type(); + $column = $rows->column($name); + + if ($comparable->equalByPhysical($type)) { + // @mago-ignore analysis:mixed-assignment + foreach ($comparable->equality($column) as $i => $value) { + $strings[$i] .= $name . NativeHasher::normalize($value); + } + + continue; + } + + // @mago-ignore analysis:mixed-assignment + foreach ($column->values() as $i => $value) { + $strings[$i] .= $name . $formatter->format($type, $value); + } + } + + return array_map(static fn(string $string): string => $algorithm->hash($string), $strings); + } +} diff --git a/src/core/etl/src/Flow/ETL/Rows/RowsBuilder.php b/src/core/etl/src/Flow/ETL/Rows/RowsBuilder.php new file mode 100644 index 0000000000..0c42e4dcba --- /dev/null +++ b/src/core/etl/src/Flow/ETL/Rows/RowsBuilder.php @@ -0,0 +1,435 @@ + + */ + private array $builders = []; + + private int $count = 0; + + /** + * @var list + */ + private array $names = []; + + /** + * @var list> + */ + private array $definitions = []; + + private ?Rows $source = null; + + /** + * @var list per target position: the source column of the same name, null when the source lacks it + */ + private array $columns = []; + + /** + * @var list per target position: the source column shares the target definition + */ + private array $shared = []; + + /** + * @var array per target position, filled only when every target column is copied from the source + */ + private array $copies = []; + + private ?string $unexpected = null; + + public function __construct( + private readonly Schema $schema, + Backend $backend, + ) { + foreach ($schema->definitions() as $name => $definition) { + $this->builders[$name] = $backend->builder($definition); + $this->names[] = $definition->entry()->name(); + $this->definitions[] = $definition; + } + } + + /** + * @param array $row name-keyed; a key the schema lacks is ignored + * + * @throws SchemaMismatchException + */ + public function append(array $row): self + { + return $this->appendRows([$row]); + } + + /** + * A column sharing its definition with the target is copied physically; any other is refused unless its value + * matches the target definition as it is - views are never cast. + * + * @throws SchemaMismatchException + */ + public function appendFrom(Rows $rows, int $i): self + { + if ($rows !== $this->source) { + $sources = $rows->schema()->definitions(); + $present = 0; + $copies = []; + $this->unexpected = null; + + foreach ($this->names as $position => $name) { + if (!array_key_exists($name, $sources)) { + $this->columns[$position] = null; + $this->shared[$position] = false; + + continue; + } + + $present++; + $column = $rows->column($name); + $this->columns[$position] = $column; + $this->shared[$position] = $sources[$name]->isSame($this->definitions[$position]); + + if ($this->shared[$position]) { + $copies[$position] = $column; + } + } + + if (count($sources) > $present) { + foreach ($sources as $definition) { + if (!array_key_exists($definition->entry()->name(), $this->builders)) { + $this->unexpected = $definition->entry()->name(); + + break; + } + } + } + + $this->copies = $this->unexpected === null && count($copies) === count($this->names) ? $copies : []; + $this->source = $rows; + } + + if ($this->copies !== []) { + foreach ($this->copies as $position => $column) { + $this->builders[$this->names[$position]]->appendFrom($column, $i); + } + + $this->count++; + + return $this; + } + + $values = []; + + foreach ($this->definitions as $position => $definition) { + $column = $this->columns[$position]; + + if ($column === null) { + if (!$definition->isNullable()) { + throw new SchemaMismatchException( + $this->count, + ColumnMismatchException::missingColumn($definition), + ); + } + + $values[$position] = null; + + continue; + } + + if ($this->shared[$position]) { + continue; + } + + // @mago-ignore analysis:mixed-assignment + $value = $column->value($i); + + if ($value === null ? !$definition->isNullable() : !$definition->matches($value)) { + throw new SchemaMismatchException( + $this->count, + ColumnMismatchException::valueDoesNotMatch($definition, $value), + ); + } + + $values[$position] = $value; + } + + if ($this->unexpected !== null) { + throw new SchemaMismatchException( + $this->count, + ColumnMismatchException::unexpectedColumn($this->unexpected), + ); + } + + foreach ($this->names as $position => $name) { + $column = $this->columns[$position]; + + if ($column === null || !$this->shared[$position]) { + $this->builders[$name]->append($values[$position] ?? null); + + continue; + } + + $this->builders[$name]->appendFrom($column, $i); + } + + $this->count++; + + return $this; + } + + /** + * Records projected onto the schema: a key it does not declare is dropped, an int key is kept under its positional + * name (ColumnName) when the schema declares that name. + * + * @param array> $rows + * + * @throws SchemaMismatchException + */ + public function appendProjected(array $rows): self + { + $definitions = $this->schema->definitions(); + $columnName = new ColumnName(); + $maps = []; + + foreach ($rows as $row) { + $map = []; + + // @mago-ignore analysis:mixed-assignment + foreach ($row as $key => $value) { + if (array_key_exists($key, $definitions)) { + $map[$key] = $value; + + continue; + } + + if (is_int($key) && $this->schema->findDefinition($name = $columnName->of($key)) !== null) { + $map[$name] = $value; + } + } + + $maps[] = $map; + } + + return $this->appendRows($maps); + } + + /** + * One record or a list of them. A key the schema does not declare is refused at its row; a numeric-string key + * PHP turned into an int keeps its declared name, any other int key is named by its position (ColumnName). + * + * @param array $data + * + * @throws SchemaMismatchException + */ + public function appendRecords(array $data): self + { + $isRows = true; + + // @mago-ignore analysis:mixed-assignment + foreach ($data as $v) { + if (!is_array($v)) { + $isRows = false; + + break; + } + } + + $definitions = $this->schema->definitions(); + $columnName = new ColumnName(); + $maps = []; + + // @mago-ignore analysis:mixed-assignment + foreach (array_values($isRows ? $data : [$data]) as $index => $row) { + $row = type_array()->assert($row); + + if (array_diff_key($row, $definitions) === []) { + $maps[] = $row; + + continue; + } + + $map = []; + + // @mago-ignore analysis:mixed-assignment + foreach ($row as $key => $value) { + // PHP gives back a numeric-string column name as an int key, which the positional rule + // would rename to eNN. A declared schema naming that column settles which one it is. + $declared = $this->schema->findDefinition((string) $key); + $name = $declared === null ? $columnName->of($key) : (string) $key; + + if ($declared === null && $this->schema->findDefinition($name) === null) { + throw new SchemaMismatchException( + $this->count + $index, + ColumnMismatchException::unexpectedColumn($name), + ); + } + + $map[$name] = $value; + } + + $maps[] = $map; + } + + return $this->appendRows($maps); + } + + /** + * @param list> $rows name-keyed; a declared key a row lacks is null, or a missing column + * under NOT NULL; a key the schema lacks is ignored + * + * @throws SchemaMismatchException + */ + public function appendRows(array $rows): self + { + if ($rows === []) { + return $this; + } + + $refusal = null; + $absence = null; + + foreach ($this->definitions as $position => $definition) { + $name = $this->names[$position]; + /** @var list $values */ + $values = array_column($rows, $name); + $present = null; + + if (count($values) !== count($rows)) { + $values = []; + $absent = null; + + foreach ($rows as $index => $row) { + if (array_key_exists($name, $row)) { + $values[] = $row[$name]; + + continue; + } + + if (!$definition->isNullable()) { + $absent ??= $index; + } + + $values[] = null; + } + + // an absence under NOT NULL is refused below; the values the rows do carry are still checked, because + // a refused value wins over an absence + if ($absent !== null) { + if ($absence === null || ($this->count + $absent) < $absence->rowIndex) { + $absence = new SchemaMismatchException( + $this->count + $absent, + ColumnMismatchException::missingColumn($definition), + ); + } + + $present = array_keys(array_filter($rows, static fn(array $row): bool => array_key_exists( + $name, + $row, + ))); + $values = array_map(static fn(int $index): mixed => $rows[$index][$name], $present); + } + } + + try { + $this->builders[$name]->appendMany($values); + } catch (SchemaMismatchException $e) { + $row = $this->count + ($present === null ? $e->rowIndex : $present[$e->rowIndex]); + + if ($refusal === null || $row < $refusal->rowIndex) { + $refusal = new SchemaMismatchException($row, $e->cause); + } + } + } + + if ($refusal !== null) { + throw $refusal; + } + + if ($absence !== null) { + throw $absence; + } + + $this->count += count($rows); + + return $this; + } + + /** + * @param list $indices + */ + public function appendTake(Rows $rows, array $indices): self + { + foreach ($this->schema->definitions() as $definition) { + $name = $definition->entry()->name(); + $this->builders[$name]->appendTake($rows->column($name), $indices); + } + + $this->count += count($indices); + + return $this; + } + + public function column(string $name): ColumnBuilder + { + return ( + $this->builders[$name] ?? throw new InvalidArgumentException(sprintf( + 'RowsBuilder has no column "%s"', + $name, + )) + ); + } + + public function count(): int + { + foreach ($this->builders as $builder) { + return $builder->count(); + } + + return $this->count; + } + + public function finish(): Rows + { + $columns = []; + $counts = []; + + foreach ($this->builders as $name => $builder) { + $columns[$name] = $builder->finish(); + $counts[$columns[$name]->count()][] = $name; + } + + if (count($counts) > 1) { + $ragged = []; + + foreach ($counts as $count => $names) { + $ragged[] = sprintf('%d rows in [%s]', $count, implode(', ', $names)); + } + + throw new InvalidArgumentException('Ragged columns: ' . implode(', ', $ragged)); + } + + return Rows::fromColumns($this->schema, $columns, $this->count()); + } +} diff --git a/src/core/etl/src/Flow/ETL/Schema.php b/src/core/etl/src/Flow/ETL/Schema.php index a51d95223c..6eecfc34a9 100644 --- a/src/core/etl/src/Flow/ETL/Schema.php +++ b/src/core/etl/src/Flow/ETL/Schema.php @@ -12,8 +12,8 @@ use Flow\ETL\Row\References; use Flow\ETL\Row\UnresolvedReference; use Flow\ETL\Schema\Definition; -use Flow\ETL\Schema\Definition\ZoneAlignment; use Flow\ETL\Schema\Metadata; +use Flow\ETL\Schema\SameSchemas; use Flow\ETL\Schema\SortingStrategy; use Flow\ETL\Schema\SortingStrategy\AlphabeticalStrategy; @@ -37,11 +37,6 @@ */ private array $definitions; - /** - * @var array> - */ - private array $zonedDefinitions; - /** * @param Definition ...$definitions */ @@ -246,6 +241,10 @@ public function insertAt(int $index, Definition ...$definitions): self */ public function isSame(self $schema): bool { + if ($this === $schema || SameSchemas::proved($this, $schema)) { + return true; + } + if (array_keys($this->definitions) !== array_keys($schema->definitions)) { return false; } @@ -256,6 +255,8 @@ public function isSame(self $schema): bool } } + SameSchemas::remember($this, $schema); + return true; } @@ -561,14 +562,6 @@ public function sort(SortingStrategy $strategy = new AlphabeticalStrategy()): se return new self(...$definitions); } - /** - * @return array> - */ - public function zonedDefinitions(): array - { - return $this->zonedDefinitions; - } - private function indexOf(string|Reference $reference): int { $index = array_search(UnresolvedReference::init($reference)->name(), array_keys($this->definitions), true); @@ -642,6 +635,5 @@ private function setDefinitions(Definition ...$definitions): void } $this->definitions = $uniqueDefinitions; - $this->zonedDefinitions = (new ZoneAlignment())->columns($uniqueDefinitions); } } diff --git a/src/core/etl/src/Flow/ETL/Schema/Definition/ZoneAlignment.php b/src/core/etl/src/Flow/ETL/Schema/Definition/ZoneAlignment.php deleted file mode 100644 index a8ec54d639..0000000000 --- a/src/core/etl/src/Flow/ETL/Schema/Definition/ZoneAlignment.php +++ /dev/null @@ -1,79 +0,0 @@ - $definition - */ - public function align(Definition $definition, mixed $value): mixed - { - return match (true) { - $value === null => null, - $this->inZone($definition, $value) => $value, - default => $definition->type()->cast($value), - }; - } - - /** - * @param Type $type - */ - public function carriesDateTime(Type $type): bool - { - return match (true) { - $type instanceof DateTimeType => true, - $type instanceof OptionalType => $this->carriesDateTime($type->base()), - $type instanceof ListType => $this->carriesDateTime($type->element()), - $type instanceof MapType => $this->carriesDateTime($type->value()), - $type instanceof StructureType - => array_filter($type->elements(), fn(StructureElement $element): bool => $this->carriesDateTime($element->type)) - !== [], - default => false, - }; - } - - /** - * @param array> $definitions - * - * @return array> - */ - public function columns(array $definitions): array - { - $zoned = []; - - foreach ($definitions as $name => $definition) { - if ($this->carriesDateTime($definition->type())) { - $zoned[$name] = $definition; - } - } - - return $zoned; - } - - /** - * @param Definition $definition - */ - public function inZone(Definition $definition, mixed $value): bool - { - return ( - $definition instanceof DateTimeDefinition - && $value instanceof DateTimeInterface - && $value->getTimezone()->getName() === $definition->type()->zoneName() - ); - } -} diff --git a/src/core/etl/src/Flow/ETL/Schema/Formatter/InlineSchemaFormatter.php b/src/core/etl/src/Flow/ETL/Schema/Formatter/InlineSchemaFormatter.php new file mode 100644 index 0000000000..b51ea7e28c --- /dev/null +++ b/src/core/etl/src/Flow/ETL/Schema/Formatter/InlineSchemaFormatter.php @@ -0,0 +1,24 @@ +definitions() as $name => $definition) { + $columns[] = $name . ': ' . ($definition->isNullable() ? '?' : '') . $definition->type()->toString(); + } + + return implode(', ', $columns); + } +} diff --git a/src/core/etl/src/Flow/ETL/Schema/Inference/ColumnTypes.php b/src/core/etl/src/Flow/ETL/Schema/Inference/ColumnTypes.php index fee2112843..7f4f51fe62 100644 --- a/src/core/etl/src/Flow/ETL/Schema/Inference/ColumnTypes.php +++ b/src/core/etl/src/Flow/ETL/Schema/Inference/ColumnTypes.php @@ -4,7 +4,6 @@ namespace Flow\ETL\Schema\Inference; -use Flow\ETL\Row\RawRowValues; use Flow\ETL\Schema; use Flow\Types\Type; use Flow\Types\Type\Logical\OptionalType; @@ -92,12 +91,12 @@ public function merge(self $other, bool $unionByName): self } /** - * RawRowValues::$metadata is not read: the fold types columns, metadata is folded by the hydrator per batch. + * @param array $row */ - public function observe(RawRowValues $row): void + public function observe(array $row): void { /** @var mixed $value */ - foreach ($row->values as $name => $value) { + foreach ($row as $name => $value) { $current = array_key_exists($name, $this->types) ? $this->types[$name] : null; if ( diff --git a/src/core/etl/src/Flow/ETL/Schema/Inference/SchemaInferrer.php b/src/core/etl/src/Flow/ETL/Schema/Inference/SchemaInferrer.php index 9bc005ea82..c081916691 100644 --- a/src/core/etl/src/Flow/ETL/Schema/Inference/SchemaInferrer.php +++ b/src/core/etl/src/Flow/ETL/Schema/Inference/SchemaInferrer.php @@ -4,7 +4,6 @@ namespace Flow\ETL\Schema\Inference; -use Flow\ETL\Row\RawRowValues; use Flow\ETL\Schema; use Flow\Types\Type\TypeNarrower; @@ -22,7 +21,7 @@ public function __construct( * a source abandoned mid-way is closed by its producer, not here. * * @param list $names - columns known before any row; [] when the format carries none (JSON) - * @param iterable> $sources - one inner iterable per source, in listing order. + * @param iterable>> $sources - one inner iterable per source, in listing order. * A source that lists no files yields exactly one. * An empty outer iterable yields every name as ?string. */ @@ -62,7 +61,7 @@ public function infer(array $names, iterable $sources): Schema /** * @param list $names - columns known before any row; [] when the format carries none (JSON) - * @param iterable $source - one source's rows, unstarted + * @param iterable> $source - one source's rows, unstarted * @param int<0, max>|-1 $rowBudget - rows to observe, 0 for none, -1 for all of them */ public function sniff(array $names, iterable $source, int $rowBudget): ColumnTypes diff --git a/src/core/etl/src/Flow/ETL/Schema/Inference/SchemaSampler.php b/src/core/etl/src/Flow/ETL/Schema/Inference/SchemaSampler.php index d7224f93c1..8a8e1d3140 100644 --- a/src/core/etl/src/Flow/ETL/Schema/Inference/SchemaSampler.php +++ b/src/core/etl/src/Flow/ETL/Schema/Inference/SchemaSampler.php @@ -4,8 +4,6 @@ namespace Flow\ETL\Schema\Inference; -use Flow\ETL\Row\RawRowValues; - interface SchemaSampler { /** @@ -13,7 +11,7 @@ interface SchemaSampler * The outer iterable is exactly what SchemaInferrer::infer() consumes; a unit that implements * SniffsColumnTypes folds itself into its partial instead of being iterated row by row. * - * @return iterable> + * @return iterable>> */ public function samples(int $rowBudget): iterable; } diff --git a/src/core/etl/src/Flow/ETL/Schema/Inference/SniffsColumnTypes.php b/src/core/etl/src/Flow/ETL/Schema/Inference/SniffsColumnTypes.php index 97b55c0a3d..46a120e4c5 100644 --- a/src/core/etl/src/Flow/ETL/Schema/Inference/SniffsColumnTypes.php +++ b/src/core/etl/src/Flow/ETL/Schema/Inference/SniffsColumnTypes.php @@ -4,12 +4,11 @@ namespace Flow\ETL\Schema\Inference; -use Flow\ETL\Row\RawRowValues; use Flow\Types\Type\TypeNarrower; use Traversable; /** - * @extends Traversable + * @extends Traversable> */ interface SniffsColumnTypes extends Traversable { diff --git a/src/core/etl/src/Flow/ETL/Schema/SameSchemas.php b/src/core/etl/src/Flow/ETL/Schema/SameSchemas.php new file mode 100644 index 0000000000..36266e326d --- /dev/null +++ b/src/core/etl/src/Flow/ETL/Schema/SameSchemas.php @@ -0,0 +1,38 @@ +> + */ + private static ?WeakMap $proved = null; + + public static function proved(Schema $schema, Schema $other): bool + { + return ( + self::$proved !== null + && self::$proved->offsetExists($schema) + && self::$proved[$schema]->offsetExists($other) + ); + } + + public static function remember(Schema $schema, Schema $other): void + { + self::$proved ??= new WeakMap(); + /** @var WeakMap $others */ + $others = self::$proved->offsetExists($schema) ? self::$proved[$schema] : new WeakMap(); + $others[$other] = true; + self::$proved[$schema] = $others; + } +} diff --git a/src/core/etl/src/Flow/ETL/Serializer/DomValueCodec.php b/src/core/etl/src/Flow/ETL/Serializer/DomValueCodec.php deleted file mode 100644 index 281ea1f857..0000000000 --- a/src/core/etl/src/Flow/ETL/Serializer/DomValueCodec.php +++ /dev/null @@ -1,128 +0,0 @@ - $type - */ - public function decode(Type $type, mixed $value): mixed - { - if (!$this->handles($type) || $value === null) { - return $value; - } - - $decoded = base64_decode(type_string()->assert($value), true); - - if ($decoded === false) { - throw new InvalidArgumentException('Given value is not valid base64'); - } - - $xml = @gzuncompress($decoded); - - if ($xml === false) { - throw new InvalidArgumentException('Given value is not valid gzcompressed XML'); - } - - if ($type instanceof HTMLType) { - // @mago-expect analysis:unavailable-method - return HTMLDocument::createFromString($xml, LIBXML_NOERROR); - } - - if ($type instanceof HTMLElementType) { - return ValueDecoder::htmlElementFromString($xml); - } - - $document = new DOMDocument(); - - if (!@$document->loadXML($xml)) { - throw new InvalidArgumentException(sprintf('Given string "%s" is not valid XML', $xml)); - } - - if ($type instanceof XMLType) { - return $document; - } - - $documentElement = $document->documentElement; - - if ($documentElement === null) { - throw new InvalidArgumentException('Given XML does not contain a document element'); - } - - $target = new DOMDocument(); - $imported = $target->importNode($documentElement, true); - - if (!$imported instanceof DOMElement) { - throw new InvalidArgumentException('Imported node is not a DOMElement'); - } - - // an unparented node canonicalizes to "", which would make every restored element compare equal - $target->appendChild($imported); - - return $imported; - } - - /** - * @param Type $type - */ - public function encode(Type $type, mixed $value): mixed - { - if (!$this->handles($type) || $value === null) { - return $value; - } - - $compressed = gzcompress( - $type instanceof HTMLElementType - ? ValueEncoder::htmlElementToString(type_html_element()->assert($value)) - : type_string()->cast($value), - ); - - if ($compressed === false) { - throw new RuntimeException('Failed to gzcompress XML column value.'); - } - - return base64_encode($compressed); - } - - /** - * @param Type $type - */ - public function handles(Type $type): bool - { - return ( - $type instanceof XMLType - || $type instanceof XMLElementType - || $type instanceof HTMLType - || $type instanceof HTMLElementType - ); - } -} diff --git a/src/core/etl/src/Flow/ETL/Sort/ExternalSort.php b/src/core/etl/src/Flow/ETL/Sort/ExternalSort.php new file mode 100644 index 0000000000..211644d330 --- /dev/null +++ b/src/core/etl/src/Flow/ETL/Sort/ExternalSort.php @@ -0,0 +1,203 @@ + $mergeFanIn + * @param int<1, max> $batchSize + */ + public function __construct( + public References $refs, + public Buckets $spill, + public Buckets $merge, + private RandomValueGenerator $random, + public Unit $memoryLimit, + public int $mergeFanIn = 10, + public int $batchSize = 1000, + ) { + // @mago-ignore analysis:invalid-operand + // @mago-ignore analysis:impossible-condition,redundant-comparison + if ($this->mergeFanIn < 1) { + throw new InvalidArgumentException('Merge fan-in must be greater than 0, given: ' . $this->mergeFanIn); + } + + // @mago-ignore analysis:invalid-operand + // @mago-ignore analysis:impossible-condition,redundant-comparison + if ($this->batchSize < 1) { + throw new InvalidArgumentException('Batch size must be greater than 0, given: ' . $this->batchSize); + } + } + + /** + * @param Generator $rows + * @param null|int $limit + * + * @return Generator + */ + public function sort(Generator $rows, FlowContext $context, ?int $limit = null): Generator + { + $budget = new MemoryBudget($context->backend(), $this->memoryLimit); + $runId = $this->random->string(16); + + /** @var list $buffer */ + $buffer = []; + + /** @var list $runs */ + $runs = []; + $schema = null; + + try { + // draining upstream inside the try is what makes a mid-stream failure release the spilled runs + foreach ($rows as $batch) { + if ($batch->isEmpty()) { + continue; + } + + $schema ??= $batch->schema(); + $buffer[] = $batch->matchTo($schema, $context->backend()); + + // the whole process, not this buffer: another spilling step in the pipeline holds memory too + if ($budget->exceeded()) { + $runs[] = $this->spill($this->sorted($buffer, $limit, $context->backend()), $runId, count($runs)); + $buffer = []; + $budget->released(); + } + } + + if ($runs === []) { + if ($buffer !== []) { + yield from $this->sorted($buffer, $limit, $context->backend())->chunks($this->batchSize); + } + + return; + } + + if ($buffer !== []) { + $runs[] = $this->spill($this->sorted($buffer, $limit, $context->backend()), $runId, count($runs)); + } + + $merger = new KWayMerge($this->refs, $context->backend(), $this->batchSize); + $mergedIndex = 0; + + // each pass merges neighbouring runs in place, so equal keys keep the order they arrived in + while (count($runs) > $this->mergeFanIn) { + $runs = array_map( + fn(array $group): BucketRun => $this->reduce($group, $merger, $mergedIndex++, $limit), + array_chunk($runs, $this->mergeFanIn), + ); + } + + yield from $this->firstRows($merger->merge($runs), $limit); + } finally { + // nested, so a throwing spill clear still leaves the merge storage cleared - and chains as previous + try { + $this->spill->clear(); + } finally { + $this->merge->clear(); + } + } + } + + /** + * @param Generator $batches + * @param null|int $limit + * + * @return Generator + */ + public function firstRows(Generator $batches, ?int $limit): Generator + { + $remaining = $limit; + + foreach ($batches as $batch) { + if ($remaining === null) { + yield $batch; + + continue; + } + + yield $batch->slice(0, min($remaining, $batch->count())); + $remaining -= min($remaining, $batch->count()); + + if ($remaining === 0) { + return; + } + } + } + + /** + * @param list $group + * @param null|int $limit + * + * @return BucketRun the merged run, bound to the merge storage that wrote it + */ + public function reduce(array $group, KWayMerge $merger, int $index, ?int $limit): BucketRun + { + $bucketId = 'sort-merge-' . $this->random->string(16); + + // registered before the spill so clear() covers a partially written merged run + $this->merge->add(new Bucket($bucketId, 0, $index)); + $totalRows = 0; + + foreach ($this->firstRows($merger->merge($group), $limit) as $batch) { + $this->merge->storage()->append($bucketId, $batch); + $totalRows += $batch->count(); + } + + foreach ($group as $run) { + $run->remove(); + } + + $this->merge->add(new Bucket($bucketId, $totalRows, $index)); + + return new BucketRun($bucketId, $this->merge); + } + + /** + * @param list $buffer + * @param null|int $limit + */ + public function sorted(array $buffer, ?int $limit, Backend $backend): Rows + { + $sorted = $buffer[0]->concat($backend, ...array_slice($buffer, 1))->sortBy(...$this->refs->all()); + + return $limit === null ? $sorted : $sorted->take($limit); + } + + public function spill(Rows $run, string $runId, int $index): BucketRun + { + $bucketId = sprintf('sort-%s-%d', $runId, $index); + $this->spill->storage()->set($bucketId, $run); + $this->spill->add(new Bucket($bucketId, $run->count(), $index)); + + return new BucketRun($bucketId, $this->spill); + } +} diff --git a/src/core/etl/src/Flow/ETL/Sort/Merge/BucketCursor.php b/src/core/etl/src/Flow/ETL/Sort/Merge/BucketCursor.php index 0bf117cc64..26aee60f75 100644 --- a/src/core/etl/src/Flow/ETL/Sort/Merge/BucketCursor.php +++ b/src/core/etl/src/Flow/ETL/Sort/Merge/BucketCursor.php @@ -4,24 +4,23 @@ namespace Flow\ETL\Sort\Merge; -use Flow\ETL\Row; +use Flow\ETL\Column\Backend; use Flow\ETL\Rows; use Flow\ETL\Schema; +use Flow\ETL\Sort\RowOrder; +use Flow\ETL\Sort\SortKey; use Generator; -use function array_values; -use function count; - final class BucketCursor { - private int $count = 0; + private Rows $batch; private int $index = 0; /** - * @var list + * @var list $batch's sort keys */ - private array $rows = []; + private array $keys = []; private ?Schema $schema = null; @@ -32,22 +31,40 @@ final class BucketCursor */ public function __construct( private readonly Generator $batches, + private readonly RowOrder $order, + private readonly Backend $backend, ) { + $this->batch = Rows::empty(new Schema(), $this->backend); $this->load(); } - public function current(): Row + public function advance(int $index): void { - return $this->rows[$this->index]; - } + $this->index = $index; - public function next(): void - { - if (++$this->index >= $this->count) { + if ($this->index >= $this->batch->count()) { $this->load(); } } + public function batch(): Rows + { + return $this->batch; + } + + public function index(): int + { + return $this->index; + } + + /** + * @return list + */ + public function keys(): array + { + return $this->keys; + } + public function schema(): Schema { return $this->schema ?? new Schema(); @@ -55,7 +72,7 @@ public function schema(): Schema public function valid(): bool { - return $this->index < $this->count; + return $this->index < $this->batch->count(); } /** @@ -78,8 +95,8 @@ private function load(): void if ($batch->count() > 0) { $this->schema ??= $batch->schema(); - $this->rows = array_values($batch->all()); - $this->count = count($this->rows); + $this->batch = $batch; + $this->keys = $this->order->keys($batch); return; } @@ -87,7 +104,7 @@ private function load(): void $this->batches->next(); } - $this->rows = []; - $this->count = 0; + $this->batch = Rows::empty(new Schema(), $this->backend); + $this->keys = []; } } diff --git a/src/core/etl/src/Flow/ETL/Sort/Merge/ComparableBucketRow.php b/src/core/etl/src/Flow/ETL/Sort/Merge/ComparableBucketRow.php deleted file mode 100644 index 562aa27d37..0000000000 --- a/src/core/etl/src/Flow/ETL/Sort/Merge/ComparableBucketRow.php +++ /dev/null @@ -1,19 +0,0 @@ - $values - */ - public function __construct( - public array $values, - public Row $row, - public string $bucketId, - ) {} -} diff --git a/src/core/etl/src/Flow/ETL/Sort/Merge/KWayMerge.php b/src/core/etl/src/Flow/ETL/Sort/Merge/KWayMerge.php index 37b53bffa5..6fea1d724e 100644 --- a/src/core/etl/src/Flow/ETL/Sort/Merge/KWayMerge.php +++ b/src/core/etl/src/Flow/ETL/Sort/Merge/KWayMerge.php @@ -5,13 +5,18 @@ namespace Flow\ETL\Sort\Merge; use Flow\ETL\Bucketing\BucketRun; +use Flow\ETL\Column\Backend; use Flow\ETL\Exception\InvalidArgumentException; use Flow\ETL\Row\References; -use Flow\ETL\Row\RowsBuffer; use Flow\ETL\Rows; use Flow\ETL\Schema; +use Flow\ETL\Sort\RowOrder; use Generator; +use function array_filter; +use function array_slice; +use function array_values; + final readonly class KWayMerge { /** @@ -19,6 +24,7 @@ */ public function __construct( private References $refs, + private Backend $backend, private int $batchSize = 1000, ) { // @mago-ignore analysis:invalid-operand @@ -29,52 +35,89 @@ public function __construct( } /** + * Merges block by block: every loaded row that sorts at or before the smallest last loaded row (the frontier) is + * final, because a run's unloaded rows sort after its loaded ones. Ties sort by run, then by position, so equal + * rows keep their input order. + * * @param list $runs * * @return Generator */ public function merge(array $runs): Generator { - $heap = new RowsMinHeap(...$this->refs->all()); + $order = new RowOrder($this->refs->all()); - /** @var array $cursors */ + /** @var list $cursors */ $cursors = []; $schema = null; - $uniform = true; foreach ($runs as $run) { - $cursor = new BucketCursor($run->rows()); + $cursor = new BucketCursor($run->rows(), $order, $this->backend); if ($cursor->valid()) { $schema ??= $cursor->schema(); - $uniform = $uniform && $cursor->schema()->isSame($schema); - $heap->push($cursor->current(), $run->id); - $cursor->next(); - $cursors[$run->id] = $cursor; + $cursors[] = $cursor; } } - // every run's rows passed the gate when they were read back - merged under the one schema they share, a - // batch needs no second check - $buffer = new RowsBuffer($schema ?? new Schema(), $this->batchSize, $uniform ? Rows::trusted(...) : null); + $schema ??= new Schema(); + $pending = Rows::empty($schema, $this->backend); - while (!$heap->isEmpty()) { - $top = $heap->extract(); + while ($cursors !== []) { + $frontier = 0; - if (null !== ($batch = $buffer->add($top->row))) { - yield $batch; + foreach ($cursors as $position => $cursor) { + if ( + $order->compare( + $cursor->keys(), + $cursor->batch()->count() - 1, + $cursors[$frontier]->keys(), + $cursors[$frontier]->batch()->count() - 1, + ) < 0 + ) { + $frontier = $position; + } } - $cursor = $cursors[$top->bucketId]; + $bound = $cursors[$frontier]->keys(); + $boundIndex = $cursors[$frontier]->batch()->count() - 1; + $slices = []; - if ($cursor->valid()) { - $heap->push($cursor->current(), $top->bucketId); - $cursor->next(); + foreach ($cursors as $position => $cursor) { + $from = $cursor->index(); + $to = $order->firstAfter( + $cursor->keys(), + $from, + $cursor->batch()->count(), + $bound, + $boundIndex, + $position <= $frontier, + ); + + if ($to > $from) { + $slices[] = $cursor + ->batch() + ->slice($from, $to - $from) + ->matchTo($schema, $this->backend); + $cursor->advance($to); + } + } + + $cursors = array_values(array_filter($cursors, static fn(BucketCursor $cursor): bool => $cursor->valid())); + $block = $slices[0]->concat($this->backend, ...array_slice($slices, 1)); + $pending = $pending->concat( + $this->backend, + $block->gather($order->permutation($order->keys($block), $block->count())), + ); + + while ($pending->count() >= $this->batchSize) { + yield $pending->slice(0, $this->batchSize); + $pending = $pending->slice($this->batchSize, $pending->count() - $this->batchSize); } } - if (null !== ($batch = $buffer->flush())) { - yield $batch; + if (!$pending->isEmpty()) { + yield $pending; } } } diff --git a/src/core/etl/src/Flow/ETL/Sort/Merge/RowsMinHeap.php b/src/core/etl/src/Flow/ETL/Sort/Merge/RowsMinHeap.php deleted file mode 100644 index 4f4e3a4675..0000000000 --- a/src/core/etl/src/Flow/ETL/Sort/Merge/RowsMinHeap.php +++ /dev/null @@ -1,83 +0,0 @@ - - */ -final class RowsMinHeap extends SplMinHeap -{ - private readonly References $ref; - - /** - * @var array - */ - private readonly array $orders; - - public function __construct(Reference ...$refs) - { - $this->ref = References::init(...$refs); - - $orders = []; - - foreach ($this->ref as $ref) { - $orders[] = $ref->sort(); - } - - $this->orders = $orders; - } - - public function __debugInfo(): array - { - $clone = clone $this; - $elements = []; - - while (!$clone->isEmpty()) { - $cachedRow = $clone->extract(); - $elements[] = [$cachedRow->bucketId => $cachedRow->row->toArray()]; - } - - return $elements; - } - - public function push(Row $row, string $bucketId): void - { - $values = []; - - foreach ($this->ref as $ref) { - $values[] = $row->get($ref->name()); - } - - parent::insert(new ComparableBucketRow($values, $row, $bucketId)); - } - - /** - * @param ComparableBucketRow $value1 - * @param ComparableBucketRow $value2 - */ - protected function compare($value1, $value2): int - { - $left = []; - $right = []; - - foreach ($this->orders as $index => $order) { - if ($order === SortOrder::DESC) { - $left[] = $value1->values[$index]; - $right[] = $value2->values[$index]; - } else { - $left[] = $value2->values[$index]; - $right[] = $value1->values[$index]; - } - } - - return $left <=> $right; - } -} diff --git a/src/core/etl/src/Flow/ETL/Sort/RowOrder.php b/src/core/etl/src/Flow/ETL/Sort/RowOrder.php new file mode 100644 index 0000000000..c020642c23 --- /dev/null +++ b/src/core/etl/src/Flow/ETL/Sort/RowOrder.php @@ -0,0 +1,191 @@ + $references + */ + public function __construct( + private array $references, + ) {} + + /** + * @param list $left + * @param list $right + * + * @return int negative when row $i of $left sorts before row $j of $right + */ + public function compare(array $left, int $i, array $right, int $j): int + { + foreach ($left as $position => $key) { + $other = $right[$position]; + $comparison = $key->ranks[$i] <=> $other->ranks[$j]; + + if ($comparison === 0 && $key->ranks[$i] === 1) { + // @mago-expect analysis:mixed-argument(2),mixed-operand(2) + $comparison = $key->flag === SORT_STRING + ? strcmp($key->values[$i], $other->values[$j]) <=> 0 + : $key->values[$i] <=> $other->values[$j]; + } + + if ($comparison !== 0) { + return $key->descending ? -$comparison : $comparison; + } + } + + return 0; + } + + /** + * The first row in [$from, $count) of a batch sorted in this order that sorts after row $boundIndex of $bound - + * or at or after it when rows equal to the bound are excluded. + * + * @param list $keys + * @param list $bound + */ + public function firstAfter( + array $keys, + int $from, + int $count, + array $bound, + int $boundIndex, + bool $includeEqual, + ): int { + $low = $from; + $high = $count; + + while ($low < $high) { + $middle = ($low + $high) >> 1; + $comparison = $this->compare($keys, $middle, $bound, $boundIndex); + + if ($comparison < 0 || $comparison === 0 && $includeEqual) { + $low = $middle + 1; + } else { + $high = $middle; + } + } + + return $low; + } + + /** + * @return list one per reference, in reference order + */ + public function keys(Rows $rows): array + { + $comparable = new ComparableValues(); + $keys = []; + + foreach ($this->references as $reference) { + $column = $rows->column($reference->base()); + $type = type_bare($column->type()); + $ordered = $comparable->orderedByPhysical($type); + $flag = self::flag($type); + $descending = $reference->sort() === SortOrder::DESC; + // nulls rank below every value, or above NaN when the reference moves them to the other end + $nullRank = $reference->nulls() === NullsOrder::defaultFor($reference->sort()) ? 0 : 3; + $ranks = []; + $values = []; + + // @mago-ignore analysis:mixed-assignment + foreach ($ordered ? $column->physicals() : $column->values() as $value) { + if ($value === null) { + $ranks[] = $nullRank; + $values[] = $flag === SORT_STRING ? '' : 0; + } elseif (is_float($value) && is_nan($value)) { + $ranks[] = 2; + $values[] = 0; + } else { + $ranks[] = 1; + // objects have no order: every one of them ties + $values[] = is_bool($value) ? (int) $value : (is_object($value) ? 0 : $value); + } + } + + $keys[] = new SortKey($ranks, $values, $flag, $descending); + } + + return $keys; + } + + /** + * The sort() flag of this order for non-null values of $type: strings by bytes, numbers, datetimes, dates, times + * and bools by their physical form. + * + * @param Type $type + */ + public static function flag(Type $type): int + { + $bare = type_bare($type); + + return match (true) { + $bare instanceof StringType => SORT_STRING, + (new ComparableValues())->orderedByPhysical($bare) => SORT_NUMERIC, + default => SORT_REGULAR, + }; + } + + /** + * @param list $keys of a batch of $count rows + * + * @return list the batch's indices in this order + */ + public function permutation(array $keys, int $count): array + { + // @mago-ignore analysis:possibly-invalid-argument + $positions = array_keys(array_fill(0, $count, true)); + + if ($keys === [] || $count < 2) { + return $positions; + } + + $arguments = []; + + foreach ($keys as $key) { + $direction = $key->descending ? SORT_DESC : SORT_ASC; + array_push($arguments, $key->ranks, $direction, SORT_NUMERIC, $key->values, $direction, $key->flag); + } + + // rows equal on every key keep their input order + $arguments[] = &$positions; + array_multisort(...$arguments); + + return $positions; + } +} diff --git a/src/core/etl/src/Flow/ETL/Sort/SortKey.php b/src/core/etl/src/Flow/ETL/Sort/SortKey.php new file mode 100644 index 0000000000..63c44022ce --- /dev/null +++ b/src/core/etl/src/Flow/ETL/Sort/SortKey.php @@ -0,0 +1,23 @@ + $ranks + * @param list $values + */ + public function __construct( + public array $ranks, + public array $values, + public int $flag, + public bool $descending, + ) {} +} diff --git a/src/core/etl/src/Flow/ETL/Sort/SortSteps.php b/src/core/etl/src/Flow/ETL/Sort/SortSteps.php index a9a6b0731c..2b1ff765af 100644 --- a/src/core/etl/src/Flow/ETL/Sort/SortSteps.php +++ b/src/core/etl/src/Flow/ETL/Sort/SortSteps.php @@ -5,12 +5,10 @@ namespace Flow\ETL\Sort; use Flow\ETL\Bucketing\Buckets; -use Flow\ETL\Bucketing\SortedRunBucketing; use Flow\ETL\Config; use Flow\ETL\Config\Sort\MemorySortConfig; use Flow\ETL\Config\Sort\SortAlgorithmBuilder; use Flow\ETL\Processor; -use Flow\ETL\Processor\BucketingProcessor; use Flow\ETL\Processor\MemorySortProcessor; use Flow\ETL\Processor\MergeSortProcessor; use Flow\ETL\Row\References; @@ -25,27 +23,34 @@ */ public static function of(References $refs, Config $config, ?SortAlgorithmBuilder $algorithm = null): array { - $sort = $algorithm?->build($config->cache->localFilesystemCacheDir) ?? $config->sort; + $external = self::external($refs, $config, $algorithm); + + return $external === null ? [new MemorySortProcessor($refs)] : [new MergeSortProcessor($external)]; + } + + /** + * @param null|SortAlgorithmBuilder $algorithm null defers to configuration + */ + public static function external( + References $refs, + Config $config, + ?SortAlgorithmBuilder $algorithm = null, + ): ?ExternalSort { + $sort = $algorithm?->build($config->cache->localFilesystemCacheDir, $config->backend()) ?? $config->sort; if ($sort instanceof MemorySortConfig) { - return [new MemorySortProcessor($refs)]; + return null; } - $random = $config->randomValueGenerator(); - $spill = new Buckets($sort->bucketing->storage); - // without the ??, a merge side built on its own would silently spill to local disk for a MemoryBuckets user - $merge = new Buckets($sort->merge ?? $sort->bucketing->storage); - - return [ - new BucketingProcessor(new SortedRunBucketing($refs->all(), $sort->runSize, $random), $spill), - new MergeSortProcessor( - $refs, - $spill, - $merge, - $random, - $sort->bucketing->bucketsCount, - $sort->bucketing->batchSize, - ), - ]; + return new ExternalSort( + $refs, + new Buckets($sort->bucketing->storage), + // without the ??, a merge side built on its own would silently spill to local disk for a MemoryBuckets user + new Buckets($sort->merge ?? $sort->bucketing->storage), + $config->randomValueGenerator(), + $sort->memoryLimit, + $sort->bucketing->bucketsCount, + $sort->bucketing->batchSize, + ); } } diff --git a/src/core/etl/src/Flow/ETL/Sort/ValuesSorter.php b/src/core/etl/src/Flow/ETL/Sort/ValuesSorter.php deleted file mode 100644 index 26cc8d4be7..0000000000 --- a/src/core/etl/src/Flow/ETL/Sort/ValuesSorter.php +++ /dev/null @@ -1,129 +0,0 @@ - $values - * - * @return array the same key => value pairs, sorted, keys preserved - */ - public static function sort(array $values, SortOrder $order): array - { - $allNumeric = true; - $allStrings = true; - $allDateTimes = true; - - // @mago-ignore analysis:mixed-assignment - foreach ($values as $value) { - if ($allNumeric && !is_numeric($value)) { - $allNumeric = false; - } - - if ($allStrings && !is_string($value)) { - $allStrings = false; - } - - if ($allDateTimes && !$value instanceof DateTimeInterface) { - $allDateTimes = false; - } - - if (!$allNumeric && !$allStrings && !$allDateTimes) { - break; - } - } - - if ($allNumeric) { - $order === SortOrder::ASC ? asort($values, SORT_NUMERIC) : arsort($values, SORT_NUMERIC); - - return $values; - } - - if ($allStrings) { - $order === SortOrder::ASC ? asort($values, SORT_STRING) : arsort($values, SORT_STRING); - - return $values; - } - - if ($allDateTimes) { - $keys = []; - - // @mago-ignore analysis:mixed-assignment - foreach ($values as $index => $value) { - if ($value instanceof DateTimeInterface) { - // integer microsecond timestamp - exact chronological order, no float precision loss - $keys[$index] = ((int) $value->format('U') * 1_000_000) + (int) $value->format('u'); - } - } - - $order === SortOrder::ASC ? asort($keys, SORT_NUMERIC) : arsort($keys, SORT_NUMERIC); - - $sorted = []; - - foreach ($keys as $index => $_) { - $sorted[$index] = $values[$index]; - } - - return $sorted; - } - - uasort($values, static function (mixed $valueA, mixed $valueB) use ($order): int { - $result = self::compare($valueA, $valueB); - - return $order === SortOrder::ASC ? $result : -$result; - }); - - return $values; - } - - private static function compare(mixed $valueA, mixed $valueB): int - { - if (is_numeric($valueA) && is_numeric($valueB)) { - return (float) $valueA <=> (float) $valueB; - } - - if (is_string($valueA) && is_string($valueB)) { - return $valueA <=> $valueB; - } - - if ($valueA instanceof DateTimeInterface && $valueB instanceof DateTimeInterface) { - return $valueA <=> $valueB; - } - - if ($valueA instanceof DateInterval && $valueB instanceof DateInterval) { - // DateInterval objects are not comparable directly, anchor both to the epoch - $epoch = new DateTimeImmutable('@0'); - - return $epoch->add($valueA) <=> $epoch->add($valueB); - } - - if (is_array($valueA) && is_array($valueB)) { - return $valueA <=> $valueB; - } - - return 0; - } -} diff --git a/src/core/etl/src/Flow/ETL/Transformer/AddRowIndexTransformer.php b/src/core/etl/src/Flow/ETL/Transformer/AddRowIndexTransformer.php index bf24502ee5..c85a228264 100644 --- a/src/core/etl/src/Flow/ETL/Transformer/AddRowIndexTransformer.php +++ b/src/core/etl/src/Flow/ETL/Transformer/AddRowIndexTransformer.php @@ -7,13 +7,13 @@ use Flow\ETL\BoundStep; use Flow\ETL\Config\Telemetry\TelemetryAttributes; use Flow\ETL\FlowContext; -use Flow\ETL\Row; use Flow\ETL\Rows; use Flow\ETL\Schema; use Flow\ETL\Transformation\AddRowIndex\StartFrom; use Throwable; use function Flow\ETL\DSL\int_schema; +use function range; final class AddRowIndexTransformer implements Stateful { @@ -41,14 +41,16 @@ public function transform(Rows $rows, FlowContext $context): Rows $context->telemetry()->transformationStarted($this); try { - $indexed = []; + $column = $context->backend()->builder(int_schema($this->indexColumn)); - foreach ($rows->all() as $row) { - $indexed[] = new Row([...$row->values(), $this->indexColumn => $this->index]); - $this->index++; + if ($rows->count() > 0) { + $column->appendMany(range($this->index, $this->index + $rows->count() - 1)); + $this->index += $rows->count(); } - $result = new Rows($rows->schema()->add(int_schema($this->indexColumn)), ...$indexed); + $result = $rows->withColumns($rows->schema()->add(int_schema($this->indexColumn)), [ + $this->indexColumn => $column->finish(), + ]); $context->telemetry()->transformationCompleted($this, [ TelemetryAttributes::ATTR_TRANSFORMATION_INPUT_ROWS => $rows->count(), diff --git a/src/core/etl/src/Flow/ETL/Transformer/CrossJoinRowsTransformer.php b/src/core/etl/src/Flow/ETL/Transformer/CrossJoinRowsTransformer.php index f3a65924eb..70a9b32f6e 100644 --- a/src/core/etl/src/Flow/ETL/Transformer/CrossJoinRowsTransformer.php +++ b/src/core/etl/src/Flow/ETL/Transformer/CrossJoinRowsTransformer.php @@ -10,6 +10,7 @@ use Flow\ETL\Executor\PhysicalPlan; use Flow\ETL\FlowContext; use Flow\ETL\Join\JoinSchema; +use Flow\ETL\Join\RowsJoin; use Flow\ETL\Rows; use Flow\ETL\Schema; use Flow\ETL\Transformer; @@ -35,7 +36,7 @@ public function transform(Rows $rows, FlowContext $context): Rows $context->telemetry()->transformationStarted($this); try { - $result = $rows->joinCross($this->rows(), $this->prefix); + $result = (new RowsJoin($context->backend()))->cross($rows, $this->rows(), $this->prefix); $context->telemetry()->transformationCompleted($this, [ TelemetryAttributes::ATTR_TRANSFORMATION_INPUT_ROWS => $rows->count(), diff --git a/src/core/etl/src/Flow/ETL/Transformer/DerivedColumns.php b/src/core/etl/src/Flow/ETL/Transformer/DerivedColumns.php index fd635caba0..85e014fa4b 100644 --- a/src/core/etl/src/Flow/ETL/Transformer/DerivedColumns.php +++ b/src/core/etl/src/Flow/ETL/Transformer/DerivedColumns.php @@ -4,13 +4,16 @@ namespace Flow\ETL\Transformer; -use Flow\ETL\Exception\ColumnMismatchException; +use Flow\ETL\Column\Backend; +use Flow\ETL\Column\Column; use Flow\ETL\Exception\SchemaMismatchException; -use Flow\ETL\Row; use Flow\ETL\Rows; +use Flow\ETL\Rows\ColumnRetyping; use Flow\ETL\Schema; use Flow\ETL\Schema\Definition; +use function Flow\Types\DSL\type_bare; + final readonly class DerivedColumns { /** @@ -24,27 +27,37 @@ public function declare(Schema $input, Definition $derived): Schema } /** + * The function's column as the stored derived column: NOT NULL enforced here (intermediates are nullable), the + * same type adopted into the configured backend, any other type rebuilt through it. + * * @param Definition $derived * * @throws SchemaMismatchException */ - public function value(Definition $derived, mixed $value, int $rowIndex): mixed + public function stored(Definition $derived, Column $column, Backend $backend): Column { - // @mago-ignore analysis:mixed-assignment - $cast = $value === null ? null : $derived->type()->cast($value); + (new ColumnRetyping())->notNull($derived, $column); - if (!$derived->matches($cast)) { - throw new SchemaMismatchException($rowIndex, ColumnMismatchException::valueDoesNotMatch($derived, $cast)); + if (type_bare($column->type())->normalize() === type_bare($derived->type())->normalize()) { + return $backend->adopt($derived, $column->withType($derived->type())); } - return $cast; + $builder = $backend->builder($derived); + $builder->appendMany($column->values()); + + return $builder->finish(); } - /** - * @param list $rows each carrying a derived column checked by value() - */ - public function rows(Schema $declared, Schema $output, array $rows): Rows - { - return $declared->isSame($output) ? Rows::trusted($output, $rows) : new Rows($output, ...$rows); + public function rows( + Rows $input, + Schema $declared, + Schema $output, + string $name, + Column $derived, + Backend $backend, + ): Rows { + $withDerived = $input->withColumns($declared, [$name => $derived]); + + return $declared->isSame($output) ? $withDerived : $withDerived->matchTo($output, $backend); } } diff --git a/src/core/etl/src/Flow/ETL/Transformer/DropDuplicatesTransformer.php b/src/core/etl/src/Flow/ETL/Transformer/DropDuplicatesTransformer.php index 07f4c4fffb..19d5bb805a 100644 --- a/src/core/etl/src/Flow/ETL/Transformer/DropDuplicatesTransformer.php +++ b/src/core/etl/src/Flow/ETL/Transformer/DropDuplicatesTransformer.php @@ -51,28 +51,32 @@ public function transform(Rows $rows, FlowContext $context): Rows $context->telemetry()->transformationStarted($this); try { - $newRows = []; + $columns = []; - foreach ($rows as $row) { + foreach ($this->entries as $entry) { + $name = $entry instanceof Reference ? $entry->base() : $entry; + + $columns[] = $rows->schema()->findDefinition($name) === null ? [] : $rows->column($name)->values(); + } + + $kept = []; + + for ($i = 0, $count = $rows->count(); $i < $count; $i++) { $values = []; - foreach ($this->entries as $entry) { - try { - $values[] = $row->get($entry); - } catch (InvalidArgumentException) { - $values[] = null; - } + foreach ($columns as $column) { + $values[] = $column[$i] ?? null; } $hash = $this->hashAlgorithm->hash(serialize($values)); if (!$this->deduplication->exists($hash)) { - $newRows[] = $row; + $kept[] = $i; $this->deduplication->add($hash); } } - $result = new Rows($rows->schema(), ...$newRows); + $result = $rows->gather($kept); $context->telemetry()->transformationCompleted($this, [ TelemetryAttributes::ATTR_TRANSFORMATION_INPUT_ROWS => $rows->count(), diff --git a/src/core/etl/src/Flow/ETL/Transformer/DropEntriesTransformer.php b/src/core/etl/src/Flow/ETL/Transformer/DropEntriesTransformer.php index 0b5f892af4..fd270b3ec0 100644 --- a/src/core/etl/src/Flow/ETL/Transformer/DropEntriesTransformer.php +++ b/src/core/etl/src/Flow/ETL/Transformer/DropEntriesTransformer.php @@ -34,7 +34,7 @@ public function transform(Rows $rows, FlowContext $context): Rows try { $schema = $rows->schema()->gracefulRemove(...$this->refs); - $result = $rows->project($schema); + $result = $rows->project($schema, $context->backend()); $context->telemetry()->transformationCompleted($this, [ TelemetryAttributes::ATTR_TRANSFORMATION_INPUT_ROWS => $rows->count(), diff --git a/src/core/etl/src/Flow/ETL/Transformer/DuplicateRowTransformer.php b/src/core/etl/src/Flow/ETL/Transformer/DuplicateRowTransformer.php index 3ecddd0409..5b06777d2d 100644 --- a/src/core/etl/src/Flow/ETL/Transformer/DuplicateRowTransformer.php +++ b/src/core/etl/src/Flow/ETL/Transformer/DuplicateRowTransformer.php @@ -20,10 +20,8 @@ use Flow\Types\Type\Unifier\PromotingUnifier; use Throwable; -use function array_key_exists; use function array_values; use function Flow\ETL\DSL\definition_from_type; -use function Flow\ETL\DSL\rows; use function Flow\Types\DSL\type_equals; final class DuplicateRowTransformer implements Transformer @@ -68,43 +66,44 @@ public function transform(Rows $rows, FlowContext $context): Rows $condition = $this->resolved ?? $this->resolve($rows->schema()); $output = $this->output ?? $this->declare($rows->schema()); $duplicated = []; - $sources = []; - foreach ($rows->all() as $position => $row) { - if ((new Parameter($condition))->asBoolean($row, $context) ?? false) { - $duplicated[] = $row; - $sources[$position] = true; + foreach ((new Parameter($condition))->asBooleans($rows, $context) as $position => $duplicate) { + if ($duplicate ?? false) { + $duplicated[] = $position; } } if ($duplicated !== []) { - // The maps inside ScalarFunctionTransformer are per-row and stateless, so applying + // The maps inside ScalarFunctionTransformer are column-wise and stateless, so applying // each entry once over all duplicated rows is equivalent to applying it per row. - $duplicatedRows = rows($rows->schema(), ...$duplicated); + $copies = $rows->gather($duplicated); foreach ($this->entries as $entry) { - $duplicatedRows = (new ScalarFunctionTransformer($entry->name, $entry->function))->transform( - $duplicatedRows, + $copies = (new ScalarFunctionTransformer($entry->name, $entry->function))->transform( + $copies, $context, ); } - $copies = $duplicatedRows->all(); + $count = $rows->count(); $copy = 0; $interleaved = []; // each copy follows the row it duplicates, so the output does not depend on where a batch ends - foreach ($rows->all() as $position => $row) { - $interleaved[] = $row; + for ($position = 0; $position < $count; $position++) { + $interleaved[] = $position; - if (array_key_exists($position, $sources)) { - $interleaved[] = $copies[$copy++]; + if ($copy < count($duplicated) && $duplicated[$copy] === $position) { + $interleaved[] = $count + $copy++; } } - $rows = new Rows($output, ...$interleaved); + $rows = $rows + ->project($output, $context->backend()) + ->concat($context->backend(), $copies->project($output, $context->backend())) + ->gather($interleaved); } else { - $rows = new Rows($output, ...$rows->all()); + $rows = $rows->project($output, $context->backend()); } $context->telemetry()->transformationCompleted($this, [ diff --git a/src/core/etl/src/Flow/ETL/Transformer/JoinEachRowsTransformer.php b/src/core/etl/src/Flow/ETL/Transformer/JoinEachRowsTransformer.php index f585240fbc..2676a4eef0 100644 --- a/src/core/etl/src/Flow/ETL/Transformer/JoinEachRowsTransformer.php +++ b/src/core/etl/src/Flow/ETL/Transformer/JoinEachRowsTransformer.php @@ -12,6 +12,7 @@ use Flow\ETL\FlowContext; use Flow\ETL\Join\Expression; use Flow\ETL\Join\Join; +use Flow\ETL\Join\RowsJoin; use Flow\ETL\Rows; use Flow\ETL\Schema; use Flow\ETL\Transformer; @@ -65,11 +66,13 @@ public function transform(Rows $rows, FlowContext $context): Rows try { $rightRows = $this->factory->from($rows)->fetch(); + $join = new RowsJoin($context->backend()); + $result = match ($this->type) { - Join::left => $rows->joinLeft($rightRows, $this->expression), - Join::left_anti => $rows->joinLeftAnti($rightRows, $this->expression), - Join::right => $rows->joinRight($rightRows, $this->expression), - default => $rows->joinInner($rightRows, $this->expression), + Join::left => $join->left($rows, $rightRows, $this->expression), + Join::left_anti => $join->leftAnti($rows, $rightRows, $this->expression), + Join::right => $join->right($rows, $rightRows, $this->expression), + default => $join->inner($rows, $rightRows, $this->expression), }; $context->telemetry()->transformationCompleted($this, [ diff --git a/src/core/etl/src/Flow/ETL/Transformer/NestedExpandTransformer.php b/src/core/etl/src/Flow/ETL/Transformer/NestedExpandTransformer.php index 8a9f49819b..008a90be5b 100644 --- a/src/core/etl/src/Flow/ETL/Transformer/NestedExpandTransformer.php +++ b/src/core/etl/src/Flow/ETL/Transformer/NestedExpandTransformer.php @@ -7,14 +7,12 @@ use Flow\ETL\BoundStep; use Flow\ETL\Config\Telemetry\TelemetryAttributes; use Flow\ETL\FlowContext; -use Flow\ETL\Row; use Flow\ETL\Rows; use Flow\ETL\Schema; use Flow\ETL\Schema\Definition; use Flow\ETL\Transformer; use Throwable; -use function count; use function Flow\Types\DSL\type_array; final readonly class NestedExpandTransformer implements Transformer @@ -66,36 +64,48 @@ private function map(Rows $rows, FlowContext $context, Definition $derived): Row { $columns = new DerivedColumns(); $declared = $columns->declare($rows->schema(), $derived); - $name = $derived->entry()->name(); - $mapped = []; - - foreach ($rows->all() as $r) { - // @mago-ignore analysis:mixed-assignment - foreach ($this->expansion->eval($r, $context) as $value) { - $mapped[] = new Row([...$r->values(), $name => $columns->value($derived, $value, count($mapped))]); - } - } - - return $columns->rows($declared, $this->output, $mapped); + [$sources, $values] = $this->expansion->eval($rows, $context); + + $builder = $context->backend()->builder($derived); + $builder->appendMany($values); + + return $columns->rows( + $rows->gather($sources), + $declared, + $this->output, + $derived->entry()->name(), + $builder->finish(), + $context->backend(), + ); } private function unpack(Rows $rows, FlowContext $context, Schema $declared): Rows { $columns = new UnpackedColumns(); - $unpacked = []; + $prefix = $this->entryName . '.'; + [$sources, $payloads] = $this->expansion->eval($rows, $context); + $values = []; - foreach ($rows->all() as $r) { + foreach ($declared->definitions() as $name => $_) { + $values[$prefix . $name] = []; + } + + // @mago-ignore analysis:mixed-assignment + foreach ($payloads as $payload) { // @mago-ignore analysis:mixed-assignment - foreach ($this->expansion->eval($r, $context) as $payload) { - $unpacked[] = new Row($columns->values( - $r->values(), - $this->entryName . '.', - $declared, - type_array()->assert($payload), - )); + foreach ($columns->values($prefix, $declared, type_array()->assert($payload)) as $name => $value) { + $values[$name][] = $value; } } - return new Rows($this->output, ...$unpacked); + $unpacked = []; + + foreach ($values as $name => $columnValues) { + $builder = $context->backend()->builder($this->output->get((string) $name)); + $builder->appendMany($columnValues); + $unpacked[$name] = $builder->finish(); + } + + return $rows->gather($sources)->withColumns($this->output, $unpacked); } } diff --git a/src/core/etl/src/Flow/ETL/Transformer/NestedExpansion.php b/src/core/etl/src/Flow/ETL/Transformer/NestedExpansion.php index 168d2e3f64..3b4ba14326 100644 --- a/src/core/etl/src/Flow/ETL/Transformer/NestedExpansion.php +++ b/src/core/etl/src/Flow/ETL/Transformer/NestedExpansion.php @@ -7,18 +7,23 @@ use Flow\ETL\FlowContext; use Flow\ETL\Function\ExpandingFunctions; use Flow\ETL\Function\FunctionTree; +use Flow\ETL\Function\Parameter; use Flow\ETL\Function\ScalarFunction; use Flow\ETL\Function\ScalarFunction\ExpandResults; -use Flow\ETL\Row; use Flow\ETL\Row\ResolvedReference; +use Flow\ETL\Rows; use Flow\ETL\Schema; +use Flow\ETL\Schema\Definition; use Flow\Types\Type; +use Flow\Types\Type\Logical\OptionalType; use Flow\Types\Type\TypeWidener; use function array_key_exists; use function array_map; use function array_values; use function count; +use function Flow\ETL\DSL\definition_from_type; +use function is_array; use function max; use function spl_object_id; @@ -27,10 +32,15 @@ /** * @param non-empty-array $expands keyed by the synthesized column the root reads * each element from + * @param array> $expandDefinitions the synthesized columns, keyed like $expands + * @param Schema $elementSchema $input with the synthesized columns, built once */ private function __construct( private ScalarFunction $root, private array $expands, + private Schema $input, + private array $expandDefinitions, + private Schema $elementSchema, ) {} public static function of(ScalarFunction $resolved, Schema $input): ?self @@ -54,6 +64,7 @@ public static function of(ScalarFunction $resolved, Schema $input): ?self $pads = count($distinct) > 1; $references = []; $expands = []; + $expandDefinitions = []; $suffix = 0; foreach ($distinct as $id => $expand) { @@ -61,9 +72,12 @@ public static function of(ScalarFunction $resolved, Schema $input): ?self $name = "\0expand:" . $suffix++; } while ($input->findDefinition($name) !== null); - $references[$id] = new ResolvedReference( + $type = $pads ? (new TypeWidener())->nullable($expand->returns()) : $expand->returns(); + $references[$id] = new ResolvedReference($name, $type); + $expandDefinitions[$name] = definition_from_type( $name, - $pads ? (new TypeWidener())->nullable($expand->returns()) : $expand->returns(), + $type instanceof OptionalType ? $type->base() : $type, + nullable: true, ); $expands[$name] = $expand; } @@ -71,7 +85,7 @@ public static function of(ScalarFunction $resolved, Schema $input): ?self /** @var ScalarFunction $root the root is not an expand, so rewrite() rebuilds it with its own class */ $root = self::rewrite($resolved, $references); - return new self($root, $expands); + return new self($root, $expands, $input, $expandDefinitions, $input->add(...array_values($expandDefinitions))); } /** @@ -83,29 +97,56 @@ public function returns(): Type } /** - * @return list + * @return array{list, list} [$sources, $values] */ - public function eval(Row $row, FlowContext $context): array + public function eval(Rows $rows, FlowContext $context): array { $lists = []; foreach ($this->expands as $name => $expand) { - $lists[$name] = array_values($expand->eval($row, $context)); + $lists[$name] = (new Parameter($expand))->values($rows, $context); } - $values = []; - $length = max(array_map(count(...), $lists)); - $element = $row->values(); + $sources = []; + $cells = []; + + foreach ($lists as $name => $_) { + $cells[$name] = []; + } + + for ($index = 0, $count = $rows->count(); $index < $count; $index++) { + $length = 0; + + foreach ($lists as $list) { + $length = max($length, is_array($list[$index]) ? count($list[$index]) : 0); + } - for ($position = 0; $position < $length; $position++) { foreach ($lists as $name => $list) { - $element[$name] = $list[$position] ?? null; + $elements = is_array($list[$index]) ? array_values($list[$index]) : []; + + for ($position = 0; $position < $length; $position++) { + $cells[$name][] = $elements[$position] ?? null; + } + } + + for ($position = 0; $position < $length; $position++) { + $sources[] = $index; } + } + + $elements = $rows->gather($sources); + $schema = $elements->schema()->isSame($this->input) + ? $this->elementSchema + : $elements->schema()->add(...array_values($this->expandDefinitions)); + $columns = []; - $values[] = $this->root->eval(new Row($element), $context); + foreach ($cells as $name => $values) { + $builder = $context->backend()->builder($this->expandDefinitions[$name]); + $builder->appendMany($values); + $columns[$name] = $builder->finish(); } - return $values; + return [$sources, (new Parameter($this->root))->values($elements->withColumns($schema, $columns), $context)]; } /** diff --git a/src/core/etl/src/Flow/ETL/Transformer/PruneEntriesTransformer.php b/src/core/etl/src/Flow/ETL/Transformer/PruneEntriesTransformer.php index 8e2fd241dd..29037ea755 100644 --- a/src/core/etl/src/Flow/ETL/Transformer/PruneEntriesTransformer.php +++ b/src/core/etl/src/Flow/ETL/Transformer/PruneEntriesTransformer.php @@ -14,10 +14,6 @@ use Flow\ETL\Transformer; use Throwable; -use function array_key_exists; -use function Flow\ETL\DSL\row; -use function Flow\ETL\DSL\rows; - /** * Unlike {@see SelectEntriesTransformer}, which throws when a reference is not declared by the * schema, prune keeps what is there and silently skips absent references - pruned rows are spilled @@ -59,22 +55,7 @@ public function transform(Rows $rows, FlowContext $context): Rows } $schema = $rows->schema()->keep(...$present)->reorder(...$present); - $newRows = []; - - foreach ($rows as $row) { - $values = $row->values(); - $pruned = []; - - foreach ($this->refs as $ref) { - if (array_key_exists($ref->name(), $values)) { - $pruned[$ref->name()] = $values[$ref->name()]; - } - } - - $newRows[] = row($pruned); - } - - $result = rows($schema, ...$newRows); + $result = $rows->project($schema, $context->backend()); $context->telemetry()->transformationCompleted($this, [ TelemetryAttributes::ATTR_TRANSFORMATION_INPUT_ROWS => $rows->count(), diff --git a/src/core/etl/src/Flow/ETL/Transformer/RenameEachEntryTransformer.php b/src/core/etl/src/Flow/ETL/Transformer/RenameEachEntryTransformer.php index be7ad8b77f..73c5e2b8bd 100644 --- a/src/core/etl/src/Flow/ETL/Transformer/RenameEachEntryTransformer.php +++ b/src/core/etl/src/Flow/ETL/Transformer/RenameEachEntryTransformer.php @@ -8,15 +8,12 @@ use Flow\ETL\Config\Telemetry\TelemetryAttributes; use Flow\ETL\Exception\InvalidArgumentException; use Flow\ETL\FlowContext; -use Flow\ETL\Row\RowRenaming; use Flow\ETL\Rows; use Flow\ETL\Schema; use Flow\ETL\Transformer; use Flow\ETL\Transformer\Rename\RenameEntryStrategy; use Throwable; -use function array_search; - final readonly class RenameEachEntryTransformer implements Transformer { /** @@ -52,31 +49,15 @@ public function transform(Rows $rows, FlowContext $context): Rows try { $schema = $rows->schema(); - $renames = []; - + // strategies chain, so a later one renames what an earlier one produced foreach ($this->strategies as $strategy) { foreach ($strategy->renames($schema) as $from => $to) { // a numeric entry name arrives as int - PHP casts numeric-string array keys - $from = (string) $from; - - $schema = $schema->rename($from, $to); - - // strategies chain, so a later one renames what an earlier one produced; the map - // has to stay keyed by the row's original name or the projection lands short - $original = array_search($from, $renames, true); - - $renames[false === $original ? $from : $original] = $to; + $schema = $schema->rename((string) $from, $to); } } - $renaming = RowRenaming::of($renames); - $renamed = []; - - foreach ($rows->all() as $row) { - $renamed[] = $renaming->apply($row); - } - - $result = new Rows($schema, ...$renamed); + $result = $rows->withSchema($schema); $context->telemetry()->transformationCompleted($this, [ TelemetryAttributes::ATTR_TRANSFORMATION_INPUT_ROWS => $rows->count(), diff --git a/src/core/etl/src/Flow/ETL/Transformer/RenameEntryTransformer.php b/src/core/etl/src/Flow/ETL/Transformer/RenameEntryTransformer.php index 1b79ed4af6..c9f56b1b0d 100644 --- a/src/core/etl/src/Flow/ETL/Transformer/RenameEntryTransformer.php +++ b/src/core/etl/src/Flow/ETL/Transformer/RenameEntryTransformer.php @@ -7,7 +7,6 @@ use Flow\ETL\BoundStep; use Flow\ETL\Config\Telemetry\TelemetryAttributes; use Flow\ETL\FlowContext; -use Flow\ETL\Row\RowRenaming; use Flow\ETL\Rows; use Flow\ETL\Schema; use Flow\ETL\Transformer; @@ -38,14 +37,7 @@ public function transform(Rows $rows, FlowContext $context): Rows $context->telemetry()->transformationStarted($this); try { - $renaming = RowRenaming::of([$this->from => $this->to]); - $renamed = []; - - foreach ($rows->all() as $row) { - $renamed[] = $renaming->apply($row); - } - - $result = new Rows($rows->schema()->rename($this->from, $this->to), ...$renamed); + $result = $rows->withSchema($rows->schema()->rename($this->from, $this->to)); $context->telemetry()->transformationCompleted($this, [ TelemetryAttributes::ATTR_TRANSFORMATION_INPUT_ROWS => $rows->count(), diff --git a/src/core/etl/src/Flow/ETL/Transformer/ScalarFunctionFilterTransformer.php b/src/core/etl/src/Flow/ETL/Transformer/ScalarFunctionFilterTransformer.php index 0a01576ea9..c7a9b6ef23 100644 --- a/src/core/etl/src/Flow/ETL/Transformer/ScalarFunctionFilterTransformer.php +++ b/src/core/etl/src/Flow/ETL/Transformer/ScalarFunctionFilterTransformer.php @@ -16,6 +16,7 @@ use Flow\ETL\Transformer; use Throwable; +use function array_keys; use function Flow\Types\DSL\type_bare; use function Flow\Types\DSL\type_boolean; use function Flow\Types\DSL\type_equals; @@ -42,16 +43,7 @@ public function transform(Rows $rows, FlowContext $context): Rows try { $function = $this->resolved ?? $this->resolve($rows->schema()); - $kept = []; - - foreach ($rows->all() as $r) { - // @mago-ignore analysis:mixed-operand - if ((bool) $function->eval($r, $context)) { - $kept[] = $r; - } - } - - $result = Rows::trusted($rows->schema(), $kept); + $result = $rows->gather(array_keys($function->eval($rows, $context)->physicals(), true, true)); $context->telemetry()->transformationCompleted($this, [ TelemetryAttributes::ATTR_TRANSFORMATION_INPUT_ROWS => $rows->count(), diff --git a/src/core/etl/src/Flow/ETL/Transformer/ScalarFunctionTransformer.php b/src/core/etl/src/Flow/ETL/Transformer/ScalarFunctionTransformer.php index 717d7be21d..a182cd3116 100644 --- a/src/core/etl/src/Flow/ETL/Transformer/ScalarFunctionTransformer.php +++ b/src/core/etl/src/Flow/ETL/Transformer/ScalarFunctionTransformer.php @@ -15,7 +15,6 @@ use Flow\ETL\Function\ScalarFunction; use Flow\ETL\Function\ScalarFunction\ExpandResults; use Flow\ETL\Function\ScalarFunction\UnpackResults; -use Flow\ETL\Row; use Flow\ETL\Rows; use Flow\ETL\Schema; use Flow\ETL\Schema\Definition; @@ -23,7 +22,6 @@ use Flow\Types\Type\Logical\StructureType; use Throwable; -use function count; use function Flow\ETL\DSL\definition_from_type; use function Flow\Types\DSL\type_array; use function sprintf; @@ -143,29 +141,42 @@ private function map(Rows $rows, FlowContext $context): Rows $derived = $this->derived ?? $this->derived($function); $declared = $columns->declare($rows->schema(), $derived); $output = $this->output ?? $declared; - $name = $derived->entry()->name(); - $mapped = []; + $column = $function->eval($rows, $context); - foreach ($rows->all() as $r) { - if ($function instanceof ExpandResults) { + if ($function instanceof ExpandResults) { + $sources = []; + $values = []; + + // @mago-ignore analysis:mixed-assignment + foreach ($column->values() as $index => $list) { // @mago-ignore analysis:mixed-assignment - foreach (type_array()->assert($function->eval($r, $context)) as $val) { - $mapped[] = new Row([ - ...$r->values(), - $name => $columns->value($derived, $val, count($mapped)), - ]); + foreach (type_array()->assert($list) as $value) { + $sources[] = $index; + $values[] = $value; } - - continue; } - $mapped[] = new Row([ - ...$r->values(), - $name => $columns->value($derived, $function->eval($r, $context), count($mapped)), - ]); + $builder = $context->backend()->builder($derived); + $builder->appendMany($values); + + return $columns->rows( + $rows->gather($sources), + $declared, + $output, + $derived->entry()->name(), + $builder->finish(), + $context->backend(), + ); } - return $columns->rows($declared, $output, $mapped); + return $columns->rows( + $rows, + $declared, + $output, + $derived->entry()->name(), + $columns->stored($derived, $column, $context->backend()), + $context->backend(), + ); } /** @@ -201,17 +212,29 @@ private function unpack(Rows $rows, FlowContext $context): Rows $declared = $this->unpacked($function); $columns = new UnpackedColumns(); $output = $this->output ?? $columns->of($rows->schema(), $this->entryName() . '.', $declared); + $prefix = $this->entryName() . '.'; + $values = []; + + foreach ($declared->definitions() as $name => $_) { + $values[$prefix . $name] = []; + } + + // @mago-ignore analysis:mixed-assignment + foreach ($function->eval($rows, $context)->values() as $payload) { + // @mago-ignore analysis:mixed-assignment + foreach ($columns->values($prefix, $declared, type_array()->assert($payload)) as $name => $value) { + $values[$name][] = $value; + } + } + $unpacked = []; - foreach ($rows->all() as $r) { - $unpacked[] = new Row($columns->values( - $r->values(), - $this->entryName() . '.', - $declared, - $function->eval($r, $context), - )); + foreach ($values as $name => $columnValues) { + $builder = $context->backend()->builder($output->get((string) $name)); + $builder->appendMany($columnValues); + $unpacked[$name] = $builder->finish(); } - return new Rows($output, ...$unpacked); + return $rows->withColumns($output, $unpacked); } } diff --git a/src/core/etl/src/Flow/ETL/Transformer/SelectEntriesTransformer.php b/src/core/etl/src/Flow/ETL/Transformer/SelectEntriesTransformer.php index eaadb6a5b1..582693946d 100644 --- a/src/core/etl/src/Flow/ETL/Transformer/SelectEntriesTransformer.php +++ b/src/core/etl/src/Flow/ETL/Transformer/SelectEntriesTransformer.php @@ -34,7 +34,7 @@ public function transform(Rows $rows, FlowContext $context): Rows try { $schema = $rows->schema()->keep(...$this->refs)->reorder(...$this->refs); - $result = $rows->project($schema); + $result = $rows->project($schema, $context->backend()); $context->telemetry()->transformationCompleted($this, [ TelemetryAttributes::ATTR_TRANSFORMATION_INPUT_ROWS => $rows->count(), diff --git a/src/core/etl/src/Flow/ETL/Transformer/SerializeTransformer.php b/src/core/etl/src/Flow/ETL/Transformer/SerializeTransformer.php index aa69a0adc7..b5090081b1 100644 --- a/src/core/etl/src/Flow/ETL/Transformer/SerializeTransformer.php +++ b/src/core/etl/src/Flow/ETL/Transformer/SerializeTransformer.php @@ -15,8 +15,6 @@ use Throwable; use function Flow\ETL\DSL\ref; -use function Flow\ETL\DSL\row; -use function Flow\ETL\DSL\rows; use function Flow\ETL\DSL\schema; use function Flow\ETL\DSL\str_schema; use function Flow\Serializer\DSL\serialize_to_string; @@ -62,17 +60,15 @@ public function transform(Rows $rows, FlowContext $context): Rows : $inputSchema->replace($target->name(), $column) ); - $serialized = []; + $serialized = $context->backend()->builder($column); - foreach ($rows->all() as $row) { - $payload = serialize_to_string($serializer, rows($inputSchema, $row)); - - $serialized[] = $this->standalone - ? row([$target->name() => $payload]) - : row([...$row->values(), $target->name() => $payload]); + for ($i = 0, $count = $rows->count(); $i < $count; $i++) { + $serialized->append(serialize_to_string($serializer, $rows->slice($i, 1))); } - $result = new Rows($outputSchema, ...$serialized); + $result = $this->standalone + ? Rows::fromColumns($outputSchema, [$target->name() => $serialized->finish()], $rows->count()) + : $rows->withColumns($outputSchema, [$target->name() => $serialized->finish()]); $context->telemetry()->transformationCompleted($this, [ TelemetryAttributes::ATTR_TRANSFORMATION_INPUT_ROWS => $rows->count(), diff --git a/src/core/etl/src/Flow/ETL/Transformer/SerializedPayloadDecoder.php b/src/core/etl/src/Flow/ETL/Transformer/SerializedPayloadDecoder.php index cae2b7f040..f1fce1e025 100644 --- a/src/core/etl/src/Flow/ETL/Transformer/SerializedPayloadDecoder.php +++ b/src/core/etl/src/Flow/ETL/Transformer/SerializedPayloadDecoder.php @@ -4,80 +4,73 @@ namespace Flow\ETL\Transformer; -use Flow\ETL\Row; use Flow\ETL\Row\Reference; +use Flow\ETL\Rows; +use Flow\ETL\Schema; use Flow\Serializer\Exception\SerializationException; use Flow\Serializer\Serializer; -use function array_key_exists; use function Flow\Serializer\DSL\unserialize_from_string; use function is_string; -/** - * A payload that does not decode still has to produce the declared shape, so every branch that gives - * up answers with the declared columns as nulls - the rule from_json follows. - */ final readonly class SerializedPayloadDecoder { /** - * @param array $declared output column name => column name in the payload - * @param array $nulls + * @param list $names the payload columns to read */ private function __construct( private Reference $source, private Serializer $serializer, - private array $declared, - private array $nulls, + private array $names, + private string $prefix, + private Rows $nothing, ) {} /** - * @param array $declared output column name => column name in the payload + * @param list $names the payload columns to read, each landing under $prefix . name */ - public static function of(Reference $source, Serializer $serializer, array $declared): self + public static function of(Reference $source, Serializer $serializer, array $names, string $prefix = ''): self { - $nulls = []; - - foreach ($declared as $name => $_) { - $nulls[$name] = null; - } - - return new self($source, $serializer, $declared, $nulls); + return new self($source, $serializer, $names, $prefix, Rows::fromColumns(new Schema(), [], 1)); } - /** - * @return array - */ - public function decode(Row $row): array + public function decode(Rows $rows, int $index): Rows { - if (!$row->has($this->source->name())) { - return $this->nulls; + if ($rows->schema()->findDefinition($this->source->name()) === null) { + return $this->nothing; } - $serialized = $row->get($this->source->name()); + // @mago-ignore analysis:mixed-assignment + $serialized = $rows->column($this->source->name())->value($index); if (!is_string($serialized)) { - return $this->nulls; + return $this->nothing; } try { $decoded = unserialize_from_string($this->serializer, $serialized); } catch (SerializationException) { - return $this->nulls; + return $this->nothing; } if ($decoded->count() !== 1) { - return $this->nulls; + return $this->nothing; } - $payload = $decoded->first()->values(); - $values = $this->nulls; + $definitions = []; + $columns = []; + + foreach ($this->names as $name) { + $definition = $decoded->schema()->findDefinition($name); - foreach ($this->declared as $name => $payloadName) { - if (array_key_exists($payloadName, $payload)) { - $values[$name] = $payload[$payloadName]; + if ($definition === null) { + continue; } + + $definitions[] = $definition->rename($this->prefix . $name); + $columns[$this->prefix . $name] = $decoded->column($name); } - return $values; + return Rows::fromColumns(new Schema(...$definitions), $columns, 1); } } diff --git a/src/core/etl/src/Flow/ETL/Transformer/UnpackedColumns.php b/src/core/etl/src/Flow/ETL/Transformer/UnpackedColumns.php index 1dd1da23df..7486640ef5 100644 --- a/src/core/etl/src/Flow/ETL/Transformer/UnpackedColumns.php +++ b/src/core/etl/src/Flow/ETL/Transformer/UnpackedColumns.php @@ -28,13 +28,14 @@ public function of(Schema $base, string $prefix, Schema $declared): Schema } /** - * @param array $values * @param array $payload * - * @return array + * @return array the cast values, keyed "$prefix$column" */ - public function values(array $values, string $prefix, Schema $declared, array $payload): array + public function values(string $prefix, Schema $declared, array $payload): array { + $values = []; + foreach ($declared->definitions() as $name => $definition) { // @mago-ignore analysis:mixed-assignment $value = $payload[$name] ?? null; diff --git a/src/core/etl/src/Flow/ETL/Transformer/UnserializeTransformer.php b/src/core/etl/src/Flow/ETL/Transformer/UnserializeTransformer.php index ca0c6f962a..3f6849fb3c 100644 --- a/src/core/etl/src/Flow/ETL/Transformer/UnserializeTransformer.php +++ b/src/core/etl/src/Flow/ETL/Transformer/UnserializeTransformer.php @@ -7,9 +7,9 @@ use Flow\ETL\BoundStep; use Flow\ETL\Config\Telemetry\TelemetryAttributes; use Flow\ETL\FlowContext; -use Flow\ETL\Row; use Flow\ETL\Row\Reference; use Flow\ETL\Rows; +use Flow\ETL\Rows\RowsBuilder; use Flow\ETL\Schema; use Flow\ETL\Transformer; use Flow\Serializer\Base64Serializer; @@ -50,24 +50,24 @@ public function transform(Rows $rows, FlowContext $context): Rows // a payload that fails to decode still has to produce the declared shape, so every declared // column is nullable - the same rule from_json follows $outputSchema = $this->bind($rows->schema())->output; - $declared = []; + $prefix = $this->merge ? $this->mergePrefix : ''; + $names = []; + $outputNames = []; foreach ($this->target->definitions() as $definition) { - $payloadName = $definition->entry()->name(); - $declared[$this->merge ? $this->mergePrefix . $payloadName : $payloadName] = $payloadName; + $names[] = $definition->entry()->name(); + $outputNames[] = $prefix . $definition->entry()->name(); } - $decoder = SerializedPayloadDecoder::of($source, $serializer, $declared); - $unserialized = []; + $decoder = SerializedPayloadDecoder::of($source, $serializer, $names, $prefix); + $builder = new RowsBuilder($outputSchema->keep(...$outputNames), $context->backend()); - foreach ($rows->all() as $row) { - $unserialized[] = new Row([ - ...($this->merge ? $row->values() : []), - ...$decoder->decode($row), - ]); + for ($i = 0, $count = $rows->count(); $i < $count; $i++) { + $builder->appendFrom($decoder->decode($rows, $i), 0); } - $result = new Rows($outputSchema, ...$unserialized); + $decoded = $builder->finish(); + $result = $this->merge ? $rows->withColumns($outputSchema, $decoded->columns()) : $decoded; $context->telemetry()->transformationCompleted($this, [ TelemetryAttributes::ATTR_TRANSFORMATION_INPUT_ROWS => $rows->count(), diff --git a/src/core/etl/src/Flow/ETL/Transformer/UntilTransformer.php b/src/core/etl/src/Flow/ETL/Transformer/UntilTransformer.php index 6b60597ad6..ee06fb5e51 100644 --- a/src/core/etl/src/Flow/ETL/Transformer/UntilTransformer.php +++ b/src/core/etl/src/Flow/ETL/Transformer/UntilTransformer.php @@ -17,7 +17,6 @@ use Flow\ETL\Transformer; use Throwable; -use function count; use function Flow\Types\DSL\type_bare; use function Flow\Types\DSL\type_boolean; use function Flow\Types\DSL\type_equals; @@ -51,27 +50,25 @@ public function transform(Rows $rows, FlowContext $context): Rows TelemetryAttributes::ATTR_TRANSFORMATION_OUTPUT_ROWS => 0, ]); - throw new LimitReachedException(0, new Rows($rows->schema())); + throw new LimitReachedException(0, Rows::empty($rows->schema(), $context->backend())); } // the unbound path has no plan to hold the resolved predicate, so it is memoised here - // until() stops the whole stream, not one batch $this->resolved ??= $this->resolve($rows->schema()); - $nextRows = []; - foreach ($rows as $row) { - if (!$this->resolved->eval($row, $context)) { + // @mago-ignore analysis:mixed-assignment + foreach ($this->resolved->eval($rows, $context)->physicals() as $i => $value) { + if (!$value) { $this->limitReached = true; $context->telemetry()->transformationCompleted($this, [ TelemetryAttributes::ATTR_TRANSFORMATION_INPUT_ROWS => $rows->count(), - TelemetryAttributes::ATTR_TRANSFORMATION_OUTPUT_ROWS => count($nextRows), + TelemetryAttributes::ATTR_TRANSFORMATION_OUTPUT_ROWS => $i, ]); - throw new LimitReachedException(0, new Rows($rows->schema(), ...$nextRows)); + throw new LimitReachedException(0, $rows->slice(0, $i)); } - - $nextRows[] = $row; } $context->telemetry()->transformationCompleted($this, [ diff --git a/src/core/etl/src/Flow/ETL/Window/Accumulator/AverageAccumulator.php b/src/core/etl/src/Flow/ETL/Window/Accumulator/AverageAccumulator.php index ba820c526a..6c8390d5bf 100644 --- a/src/core/etl/src/Flow/ETL/Window/Accumulator/AverageAccumulator.php +++ b/src/core/etl/src/Flow/ETL/Window/Accumulator/AverageAccumulator.php @@ -6,10 +6,11 @@ use Flow\Calculator\Calculator; use Flow\Calculator\Rounding; +use Flow\Calculator\RunningSum; use Flow\ETL\Exception\InvalidArgumentException; use Flow\ETL\FlowContext; -use Flow\ETL\Row; use Flow\ETL\Row\Reference; +use Flow\ETL\Rows; use Flow\ETL\Window\FrameAccumulator; use function is_numeric; @@ -20,26 +21,28 @@ final class AverageAccumulator implements FrameAccumulator private int $count = 0; - private float|int $sum = 0; + private readonly RunningSum $runningSum; public function __construct( private readonly Reference $ref, private readonly int $scale, private readonly Rounding $rounding, + private readonly bool $exact, FlowContext $context, ) { $this->calculator = $context->calculator(); + $this->runningSum = new RunningSum($this->calculator); } - public function accumulate(Row $row): void + public function accumulate(Rows $rows, int $index): void { try { /** @var mixed $value */ - $value = $row->get($this->ref); + $value = $rows->column($this->ref->base())->value($index); if (is_numeric($value)) { // @mago-ignore analysis:possibly-invalid-argument - $this->sum = $this->calculator->add($this->sum, $value); + $this->runningSum->add($value, $this->exact); $this->count++; } } catch (InvalidArgumentException $e) { @@ -53,6 +56,6 @@ public function value(): float|int|null return null; } - return $this->calculator->divide($this->sum, $this->count, $this->scale, $this->rounding); + return $this->calculator->divide($this->runningSum->value(), $this->count, $this->scale, $this->rounding); } } diff --git a/src/core/etl/src/Flow/ETL/Window/Accumulator/CountAccumulator.php b/src/core/etl/src/Flow/ETL/Window/Accumulator/CountAccumulator.php index 3930ce0733..a64aa8944e 100644 --- a/src/core/etl/src/Flow/ETL/Window/Accumulator/CountAccumulator.php +++ b/src/core/etl/src/Flow/ETL/Window/Accumulator/CountAccumulator.php @@ -6,8 +6,8 @@ use Flow\ETL\Exception\InvalidArgumentException; use Flow\ETL\FlowContext; -use Flow\ETL\Row; use Flow\ETL\Row\Reference; +use Flow\ETL\Rows; use Flow\ETL\Window\FrameAccumulator; final class CountAccumulator implements FrameAccumulator @@ -19,7 +19,7 @@ public function __construct( FlowContext $context, ) {} - public function accumulate(Row $row): void + public function accumulate(Rows $rows, int $index): void { if ($this->ref === null) { $this->count++; @@ -28,7 +28,7 @@ public function accumulate(Row $row): void } try { - if ($row->get($this->ref) !== null) { + if ($rows->column($this->ref->base())->value($index) !== null) { $this->count++; } } catch (InvalidArgumentException $e) { diff --git a/src/core/etl/src/Flow/ETL/Window/Accumulator/SumAccumulator.php b/src/core/etl/src/Flow/ETL/Window/Accumulator/SumAccumulator.php index 9cffa9afeb..6987b73523 100644 --- a/src/core/etl/src/Flow/ETL/Window/Accumulator/SumAccumulator.php +++ b/src/core/etl/src/Flow/ETL/Window/Accumulator/SumAccumulator.php @@ -7,45 +7,35 @@ use Flow\Calculator\RunningSum; use Flow\ETL\Exception\InvalidArgumentException; use Flow\ETL\FlowContext; -use Flow\ETL\Function\Parameter; -use Flow\ETL\Function\ScalarFunction; -use Flow\ETL\Row; use Flow\ETL\Row\Reference; +use Flow\ETL\Rows; use Flow\ETL\Window\FrameAccumulator; use function is_numeric; final class SumAccumulator implements FrameAccumulator { - /** - * Constant for a bool $exact, null when it is a ScalarFunction and has to be resolved per row. - */ - private readonly ?bool $constantExact; - private readonly RunningSum $runningSum; - private float|int|null $sum = null; + private bool $summed = false; public function __construct( private readonly Reference $ref, - private readonly ScalarFunction|bool $exact, - private readonly FlowContext $context, + private readonly bool $exact, + FlowContext $context, ) { - $this->constantExact = is_bool($exact) ? $exact : null; $this->runningSum = new RunningSum($context->calculator()); } - public function accumulate(Row $row): void + public function accumulate(Rows $rows, int $index): void { try { - $value = $row->get($this->ref); + // @mago-ignore analysis:mixed-assignment + $value = $rows->column($this->ref->base())->value($index); if (is_int($value) || is_float($value) || is_string($value) && is_numeric($value)) { - $this->sum = $this->runningSum->add( - $this->sum ?? 0, - $value, - $this->constantExact ?? (new Parameter($this->exact))->asBoolean($row, $this->context) ?? false, - ); + $this->runningSum->add($value, $this->exact); + $this->summed = true; } } catch (InvalidArgumentException $e) { throw new InvalidArgumentException('Sum window function error: ' . $e->getMessage(), 0, $e); @@ -54,6 +44,6 @@ public function accumulate(Row $row): void public function value(): float|int|null { - return $this->sum; + return $this->summed ? $this->runningSum->value() : null; } } diff --git a/src/core/etl/src/Flow/ETL/Window/FrameAccumulator.php b/src/core/etl/src/Flow/ETL/Window/FrameAccumulator.php index db059a7a9b..f9f24bb8fc 100644 --- a/src/core/etl/src/Flow/ETL/Window/FrameAccumulator.php +++ b/src/core/etl/src/Flow/ETL/Window/FrameAccumulator.php @@ -4,11 +4,11 @@ namespace Flow\ETL\Window; -use Flow\ETL\Row; +use Flow\ETL\Rows; interface FrameAccumulator { - public function accumulate(Row $row): void; + public function accumulate(Rows $rows, int $index): void; /** * Must be idempotent and non-destructive - WindowProcessor reads it after every growth step of an diff --git a/src/core/etl/src/Flow/ETL/Window/PeerComparator.php b/src/core/etl/src/Flow/ETL/Window/PeerComparator.php index 6c31187f90..b24a061c7b 100644 --- a/src/core/etl/src/Flow/ETL/Window/PeerComparator.php +++ b/src/core/etl/src/Flow/ETL/Window/PeerComparator.php @@ -4,10 +4,14 @@ namespace Flow\ETL\Window; -use Flow\ETL\Row; +use Flow\ETL\Column\ComparableValues; use Flow\ETL\Row\Reference; use Flow\ETL\Row\TypedValueComparator; -use Flow\ETL\Schema; +use Flow\ETL\Rows; +use Flow\Types\Type; + +use function is_float; +use function is_nan; /** * Two rows are peers when they carry equal values in every ORDER BY column. Compared with @@ -22,16 +26,69 @@ public function __construct( private array $orderBy, ) {} - public function arePeers(Row $left, Row $right, Schema $schema): bool + /** + * NaN is equal to itself here: the sort order puts it after every value. + * + * @param Type $type + */ + public function equal(Type $type, mixed $left, mixed $right): bool + { + if (is_float($left) && is_float($right) && is_nan($left) && is_nan($right)) { + return true; + } + + return (new TypedValueComparator())->equals($type, $left, $right); + } + + /** + * @return list per row: a peer of the row before it (row 0 never is) + */ + public function peersOfPrevious(Rows $partition): array { - $comparator = new TypedValueComparator(); + $count = $partition->count(); + $peers = []; + + for ($i = 0; $i < $count; $i++) { + $peers[] = $i > 0; + } + + $comparable = new ComparableValues(); foreach ($this->orderBy as $ref) { - if (!$comparator->equals($schema->get($ref)->type(), $left->get($ref), $right->get($ref))) { - return false; + $type = $partition->schema()->get($ref)->type(); + $column = $partition->column($ref->base()); + + if ($comparable->equalByPhysical($type)) { + // === on physicals is value equality; NaN is the one physical that is not === to itself + $physicals = $comparable->equality($column); + + for ($i = 1; $i < $count; $i++) { + // @mago-ignore analysis:mixed-assignment + $previous = $physicals[$i - 1]; + // @mago-ignore analysis:mixed-assignment + $current = $physicals[$i]; + + if ( + $peers[$i] + && $previous !== $current + && !(is_float($previous) && is_float($current) && is_nan($previous) && is_nan($current)) + ) { + $peers[$i] = false; + } + } + + continue; + } + + $values = $column->values(); + + for ($i = 1; $i < $count; $i++) { + if ($peers[$i] && !$this->equal($type, $values[$i - 1], $values[$i])) { + $peers[$i] = false; + } } } - return true; + return $peers; } } diff --git a/src/core/etl/src/Flow/ETL/Window/PeerFrame.php b/src/core/etl/src/Flow/ETL/Window/PeerFrame.php index 69493164a4..e7dc75226e 100644 --- a/src/core/etl/src/Flow/ETL/Window/PeerFrame.php +++ b/src/core/etl/src/Flow/ETL/Window/PeerFrame.php @@ -5,43 +5,58 @@ namespace Flow\ETL\Window; use Flow\ETL\Row\Reference; -use Flow\ETL\Row\TypedValueComparator; use Flow\ETL\Rows; +use WeakMap; + +use function array_reverse; final readonly class PeerFrame implements WindowFrame { + /** + * @var WeakMap> per partition: the last row of each row's peer group, derived once + */ + private WeakMap $ends; + /** * @param array $orderBy */ public function __construct( private array $orderBy, - ) {} + ) { + $this->ends = new WeakMap(); + } public function bounds(int $index, Rows $partition): array { - $lastIndex = $partition->count() - 1; - - if ($index > $lastIndex) { + if ($index > ($partition->count() - 1)) { return [1, 0]; } - $row = $partition[$index]; - $end = $index; - $schema = $partition->schema(); - $comparator = new TypedValueComparator(); + return [0, $this->ends($partition)[$index]]; + } + + /** + * @return list for every row the index of the last row of its peer group + */ + public function ends(Rows $partition): array + { + if (!$this->ends->offsetExists($partition)) { + $peers = (new PeerComparator($this->orderBy))->peersOfPrevious($partition); + $ends = []; + $end = $partition->count() - 1; - while ($end < $lastIndex) { - $next = $partition[$end + 1]; + // walking back, a row that is not a peer of the one before it closes the previous group + for ($i = $end; $i >= 0; $i--) { + $ends[] = $end; - foreach ($this->orderBy as $ref) { - if (!$comparator->equals($schema->get($ref)->type(), $row->get($ref), $next->get($ref))) { - return [0, $end]; + if ($i > 0 && !$peers[$i]) { + $end = $i - 1; } } - $end++; + $this->ends[$partition] = array_reverse($ends); } - return [0, $end]; + return $this->ends[$partition]; } } diff --git a/src/core/etl/src/Flow/ETL/Window/WindowContext.php b/src/core/etl/src/Flow/ETL/Window/WindowContext.php index 9f962e1c45..d70fbe6cd7 100644 --- a/src/core/etl/src/Flow/ETL/Window/WindowContext.php +++ b/src/core/etl/src/Flow/ETL/Window/WindowContext.php @@ -5,18 +5,13 @@ namespace Flow\ETL\Window; use Flow\ETL\FlowContext; -use Flow\ETL\Row; use Flow\ETL\Rows; -use function array_slice; -use function Flow\ETL\DSL\rows; - final class WindowContext { private ?Rows $frame = null; public function __construct( - private readonly Row $row, private readonly int $index, private readonly Rows $partition, private readonly WindowFrame $windowFrame, @@ -37,8 +32,8 @@ public function frame(): Rows [$start, $end] = $this->windowFrame->bounds($this->index, $this->partition); $this->frame = $start > $end - ? rows($this->partition->schema()) - : rows($this->partition->schema(), ...array_slice($this->partition->all(), $start, $end - $start + 1)); + ? Rows::empty($this->partition->schema(), $this->flowContext->backend()) + : $this->partition->slice($start, $end - $start + 1); } return $this->frame; @@ -53,9 +48,4 @@ public function partition(): Rows { return $this->partition; } - - public function row(): Row - { - return $this->row; - } } diff --git a/src/core/etl/src/Flow/Floe/Codec.php b/src/core/etl/src/Flow/Floe/Codec.php index 22198f50c2..4ea4fea019 100644 --- a/src/core/etl/src/Flow/Floe/Codec.php +++ b/src/core/etl/src/Flow/Floe/Codec.php @@ -5,8 +5,8 @@ namespace Flow\Floe; /** - * Compression codec contract for ROW frame bodies. Only the no-op codec (id 0x00) - * is supported; the codec id is stamped into the header flags byte. + * Compression codec contract, applied per buffer of a BATCH frame (behind an i64 uncompressed-length prefix, -1 = + * stored raw). Only the no-op codec (id 0x00) is supported; the codec id is stamped into the header flags byte. */ interface Codec { diff --git a/src/core/etl/src/Flow/Floe/ColumnBlueprint.php b/src/core/etl/src/Flow/Floe/ColumnBlueprint.php deleted file mode 100644 index 16464ec222..0000000000 --- a/src/core/etl/src/Flow/Floe/ColumnBlueprint.php +++ /dev/null @@ -1,19 +0,0 @@ - $definition - */ - public function __construct( - public string $name, - public Definition $definition, - public Decoding\ValueDecoder $decoder, - ) {} -} diff --git a/src/core/etl/src/Flow/Floe/DSL/functions.php b/src/core/etl/src/Flow/Floe/DSL/functions.php index 70343179bc..e607c94ff5 100644 --- a/src/core/etl/src/Flow/Floe/DSL/functions.php +++ b/src/core/etl/src/Flow/Floe/DSL/functions.php @@ -7,13 +7,13 @@ use Flow\Documentation\Attribute\DocumentationDSL; use Flow\Documentation\Attribute\Module; use Flow\Documentation\Attribute\Type as DSLType; +use Flow\ETL\Column\Backend; use Flow\ETL\Schema\Metadata; use Flow\Filesystem\Filesystem; use Flow\Filesystem\Local\NativeLocalFilesystem; use Flow\Filesystem\Path; use Flow\Floe\Codec; use Flow\Floe\Codec\NoopCodec; -use Flow\Floe\FloeEngine; use Flow\Floe\FloeExtractor; use Flow\Floe\FloeLoader; use Flow\Floe\FloeMerger; @@ -32,10 +32,9 @@ function from_floe( string|Path $path, Codec $codec = new NoopCodec(), int $chunk_size = 65536, - FloeEngine $engine = FloeEngine::adaptive, Filesystem $filesystem = new NativeLocalFilesystem(), ): FloeExtractor { - return new FloeExtractor(is_string($path) ? path_real($path) : $path, $codec, $chunk_size, $engine, $filesystem); + return new FloeExtractor(is_string($path) ? path_real($path) : $path, $codec, $chunk_size, $filesystem); } /** @@ -46,10 +45,9 @@ function to_floe( string|Path $path, ?Metadata $metadata = null, Options $options = new Options(), - FloeEngine $engine = FloeEngine::adaptive, Filesystem $filesystem = new NativeLocalFilesystem(), ): FloeLoader { - return new FloeLoader(is_string($path) ? path_real($path) : $path, $metadata, $options, $engine, $filesystem); + return new FloeLoader(is_string($path) ? path_real($path) : $path, $metadata, $options, $filesystem); } #[DocumentationDSL(module: Module::FLOE, type: DSLType::HELPER)] @@ -68,11 +66,12 @@ function floe_options(int $buffer_size = 65536, Codec $codec = new NoopCodec()): function merge_floe( array $sources, string|Path $dest, + Backend $backend, bool $compact = false, ?Metadata $metadata = null, Filesystem $filesystem = new NativeLocalFilesystem(), ): void { - (new FloeMerger($filesystem))->merge( + (new FloeMerger($filesystem, $backend))->merge( array_map(static fn(string|Path $source): Path => is_string($source) ? path_real($source) : $source, $sources), is_string($dest) ? path($dest) : $dest, $compact, diff --git a/src/core/etl/src/Flow/Floe/Decoding/BooleanDecoder.php b/src/core/etl/src/Flow/Floe/Decoding/BooleanDecoder.php deleted file mode 100644 index 885ff07310..0000000000 --- a/src/core/etl/src/Flow/Floe/Decoding/BooleanDecoder.php +++ /dev/null @@ -1,13 +0,0 @@ -setTimezone($this->columnZone ?? $this->timeZones->get($timezoneName)); - } -} diff --git a/src/core/etl/src/Flow/Floe/Decoding/EnumDecoder.php b/src/core/etl/src/Flow/Floe/Decoding/EnumDecoder.php deleted file mode 100644 index 3763a976b8..0000000000 --- a/src/core/etl/src/Flow/Floe/Decoding/EnumDecoder.php +++ /dev/null @@ -1,39 +0,0 @@ -y = $fields['y']; - // @mago-ignore analysis:invalid-property-write - $value->m = $fields['m']; - // @mago-ignore analysis:invalid-property-write - $value->d = $fields['d']; - // @mago-ignore analysis:invalid-property-write - $value->h = $fields['h']; - // @mago-ignore analysis:invalid-property-write - $value->i = $fields['i']; - // @mago-ignore analysis:invalid-property-write - $value->s = $fields['s']; - // @mago-ignore analysis:invalid-property-write - $value->f = $fraction; - // @mago-ignore analysis:invalid-property-write - $value->invert = $invert; - - return $value; - } -} diff --git a/src/core/etl/src/Flow/Floe/Decoding/JsonDecoder.php b/src/core/etl/src/Flow/Floe/Decoding/JsonDecoder.php deleted file mode 100644 index 73582d409d..0000000000 --- a/src/core/etl/src/Flow/Floe/Decoding/JsonDecoder.php +++ /dev/null @@ -1,49 +0,0 @@ -newInstanceWithoutConstructor(); - $json->value = $value; - $json->isObject = $isObject; - - return $json; - }, - null, - Json::class, - ); - - $this->create = $create; - } - - public function decode(string $data, int &$position): Json - { - $length = unpack('V', $data, $position)[1]; - $json = substr($data, $position + 4, $length); - $position += 4 + $length; - - return ($this->create)($json, $data[$position++] === "\x01"); - } -} diff --git a/src/core/etl/src/Flow/Floe/Decoding/ListDecoder.php b/src/core/etl/src/Flow/Floe/Decoding/ListDecoder.php deleted file mode 100644 index 7bb9f1d9ec..0000000000 --- a/src/core/etl/src/Flow/Floe/Decoding/ListDecoder.php +++ /dev/null @@ -1,30 +0,0 @@ - - */ - public function decode(string $data, int &$position): array - { - $count = unpack('V', $data, $position)[1]; - $position += 4; - $values = []; - - for ($i = 0; $i < $count; $i++) { - $values[] = $this->element->decode($data, $position); - } - - return $values; - } -} diff --git a/src/core/etl/src/Flow/Floe/Decoding/MapDecoder.php b/src/core/etl/src/Flow/Floe/Decoding/MapDecoder.php deleted file mode 100644 index 4213c1852e..0000000000 --- a/src/core/etl/src/Flow/Floe/Decoding/MapDecoder.php +++ /dev/null @@ -1,33 +0,0 @@ - - */ - public function decode(string $data, int &$position): array - { - $count = unpack('V', $data, $position)[1]; - $position += 4; - $values = []; - - for ($i = 0; $i < $count; $i++) { - /** @var array-key $key */ - $key = $this->key->decode($data, $position); - $values[$key] = $this->value->decode($data, $position); - } - - return $values; - } -} diff --git a/src/core/etl/src/Flow/Floe/Decoding/NullDecoder.php b/src/core/etl/src/Flow/Floe/Decoding/NullDecoder.php deleted file mode 100644 index 99ffeef51a..0000000000 --- a/src/core/etl/src/Flow/Floe/Decoding/NullDecoder.php +++ /dev/null @@ -1,13 +0,0 @@ -base->decode($data, $position); - } -} diff --git a/src/core/etl/src/Flow/Floe/Decoding/PackedListDecoder.php b/src/core/etl/src/Flow/Floe/Decoding/PackedListDecoder.php deleted file mode 100644 index 5596dc5f05..0000000000 --- a/src/core/etl/src/Flow/Floe/Decoding/PackedListDecoder.php +++ /dev/null @@ -1,37 +0,0 @@ - - */ - public function decode(string $data, int &$position): array - { - $count = unpack('V', $data, $position)[1]; - $position += 4; - - if ($count === 0) { - return []; - } - - $values = array_values(unpack($this->format . '*', substr($data, $position, $count * 8))); - $position += $count * 8; - - return $values; - } -} diff --git a/src/core/etl/src/Flow/Floe/Decoding/StringDecoder.php b/src/core/etl/src/Flow/Floe/Decoding/StringDecoder.php deleted file mode 100644 index b9bba92ae4..0000000000 --- a/src/core/etl/src/Flow/Floe/Decoding/StringDecoder.php +++ /dev/null @@ -1,20 +0,0 @@ - $elements structure element name => decoder - */ - public function __construct( - private readonly array $elements, - ) {} - - /** - * @return array - */ - public function decode(string $data, int &$position): array - { - $structure = []; - - foreach ($this->elements as $name => $element) { - $flag = ord($data[$position++]); - - if ($flag === Format::VALUE_PRESENT) { - $structure[$name] = $element->decode($data, $position); - } elseif ($flag === Format::VALUE_NULL) { - $structure[$name] = null; - } elseif ($flag !== Format::VALUE_ABSENT) { - throw new FloeException(sprintf('Floe found unknown structure element flag 0x%02X', $flag)); - } - } - - return $structure; - } -} diff --git a/src/core/etl/src/Flow/Floe/Decoding/TimeZoneDecoder.php b/src/core/etl/src/Flow/Floe/Decoding/TimeZoneDecoder.php deleted file mode 100644 index 3c6e1aa1a4..0000000000 --- a/src/core/etl/src/Flow/Floe/Decoding/TimeZoneDecoder.php +++ /dev/null @@ -1,26 +0,0 @@ -timeZones->get($name); - } -} diff --git a/src/core/etl/src/Flow/Floe/Decoding/UuidDecoder.php b/src/core/etl/src/Flow/Floe/Decoding/UuidDecoder.php deleted file mode 100644 index 2268b9dd5b..0000000000 --- a/src/core/etl/src/Flow/Floe/Decoding/UuidDecoder.php +++ /dev/null @@ -1,48 +0,0 @@ -newInstanceWithoutConstructor(); - $uuid->value = $value; - - return $uuid; - }, - null, - Uuid::class, - ); - - $this->create = $create; - } - - public function decode(string $data, int &$position): Uuid - { - $length = unpack('V', $data, $position)[1]; - $value = ($this->create)(substr($data, $position + 4, $length)); - $position += 4 + $length; - - return $value; - } -} diff --git a/src/core/etl/src/Flow/Floe/Decoding/ValueDecoder.php b/src/core/etl/src/Flow/Floe/Decoding/ValueDecoder.php deleted file mode 100644 index 33016d4dd8..0000000000 --- a/src/core/etl/src/Flow/Floe/Decoding/ValueDecoder.php +++ /dev/null @@ -1,13 +0,0 @@ - - */ -final class BooleanEncoder implements ValueEncoder -{ - public function encode(mixed $value): string - { - return $value ? "\x01" : "\x00"; - } -} diff --git a/src/core/etl/src/Flow/Floe/Encoding/DateTimeEncoder.php b/src/core/etl/src/Flow/Floe/Encoding/DateTimeEncoder.php deleted file mode 100644 index 122073cda6..0000000000 --- a/src/core/etl/src/Flow/Floe/Encoding/DateTimeEncoder.php +++ /dev/null @@ -1,41 +0,0 @@ - - */ -final class DateTimeEncoder implements ValueEncoder -{ - public function encode(mixed $value): string - { - // The class is not stored: a datetime column always hydrates to DateTimeImmutable, so a - // flag distinguishing the two would describe nothing a reader can observe. Custom - // subclasses are still refused - their extra state would be silently dropped. - if ($value::class !== DateTimeImmutable::class && $value::class !== DateTime::class) { - throw new FloeException(sprintf( - 'Floe supports only DateTime and DateTimeImmutable, got %s - convert custom datetime instances before writing', - $value::class, - )); - } - - $timezone = $value->getTimezone()->getName(); - - return ( - pack('P', $value->getTimestamp()) - . pack('V', (int) $value->format('u')) - . pack('V', strlen($timezone)) - . $timezone - ); - } -} diff --git a/src/core/etl/src/Flow/Floe/Encoding/EnumEncoder.php b/src/core/etl/src/Flow/Floe/Encoding/EnumEncoder.php deleted file mode 100644 index 09b9ea1a2a..0000000000 --- a/src/core/etl/src/Flow/Floe/Encoding/EnumEncoder.php +++ /dev/null @@ -1,21 +0,0 @@ - - */ -final class EnumEncoder implements ValueEncoder -{ - public function encode(mixed $value): string - { - $class = $value::class; - - return pack('V', strlen($class)) . $class . pack('V', strlen($value->name)) . $value->name; - } -} diff --git a/src/core/etl/src/Flow/Floe/Encoding/Float64Encoder.php b/src/core/etl/src/Flow/Floe/Encoding/Float64Encoder.php deleted file mode 100644 index 23c68c9683..0000000000 --- a/src/core/etl/src/Flow/Floe/Encoding/Float64Encoder.php +++ /dev/null @@ -1,18 +0,0 @@ - - */ -final class Float64Encoder implements ValueEncoder -{ - public function encode(mixed $value): string - { - return pack('e', $value); - } -} diff --git a/src/core/etl/src/Flow/Floe/Encoding/HtmlDocumentEncoder.php b/src/core/etl/src/Flow/Floe/Encoding/HtmlDocumentEncoder.php deleted file mode 100644 index cb5e0d44c2..0000000000 --- a/src/core/etl/src/Flow/Floe/Encoding/HtmlDocumentEncoder.php +++ /dev/null @@ -1,19 +0,0 @@ - - */ -final class HtmlDocumentEncoder implements ValueEncoder -{ - public function encode(mixed $value): string - { - // @mago-ignore analysis:unavailable-method - return ValueEncoderFactory::lengthPrefixed($value->saveHtml()); - } -} diff --git a/src/core/etl/src/Flow/Floe/Encoding/HtmlElementEncoder.php b/src/core/etl/src/Flow/Floe/Encoding/HtmlElementEncoder.php deleted file mode 100644 index e7b997d12f..0000000000 --- a/src/core/etl/src/Flow/Floe/Encoding/HtmlElementEncoder.php +++ /dev/null @@ -1,18 +0,0 @@ - - */ -final class HtmlElementEncoder implements ValueEncoder -{ - public function encode(mixed $value): string - { - return ValueEncoderFactory::lengthPrefixed(ValueEncoderFactory::htmlElementToString($value)); - } -} diff --git a/src/core/etl/src/Flow/Floe/Encoding/Int64Encoder.php b/src/core/etl/src/Flow/Floe/Encoding/Int64Encoder.php deleted file mode 100644 index 8855c53b50..0000000000 --- a/src/core/etl/src/Flow/Floe/Encoding/Int64Encoder.php +++ /dev/null @@ -1,18 +0,0 @@ - - */ -final class Int64Encoder implements ValueEncoder -{ - public function encode(mixed $value): string - { - return pack('P', $value); - } -} diff --git a/src/core/etl/src/Flow/Floe/Encoding/IntervalEncoder.php b/src/core/etl/src/Flow/Floe/Encoding/IntervalEncoder.php deleted file mode 100644 index d30eec1dba..0000000000 --- a/src/core/etl/src/Flow/Floe/Encoding/IntervalEncoder.php +++ /dev/null @@ -1,23 +0,0 @@ - - */ -final class IntervalEncoder implements ValueEncoder -{ - public function encode(mixed $value): string - { - return ( - chr($value->invert) - . pack('VVVVVV', $value->y, $value->m, $value->d, $value->h, $value->i, $value->s) - . pack('e', $value->f) - ); - } -} diff --git a/src/core/etl/src/Flow/Floe/Encoding/JsonEncoder.php b/src/core/etl/src/Flow/Floe/Encoding/JsonEncoder.php deleted file mode 100644 index f864064292..0000000000 --- a/src/core/etl/src/Flow/Floe/Encoding/JsonEncoder.php +++ /dev/null @@ -1,21 +0,0 @@ - - */ -final class JsonEncoder implements ValueEncoder -{ - public function encode(mixed $value): string - { - $json = $value->toString(); - - return pack('V', strlen($json)) . $json . ($value->isObject() ? "\x01" : "\x00"); - } -} diff --git a/src/core/etl/src/Flow/Floe/Encoding/ListEncoder.php b/src/core/etl/src/Flow/Floe/Encoding/ListEncoder.php deleted file mode 100644 index 4b62d1cec4..0000000000 --- a/src/core/etl/src/Flow/Floe/Encoding/ListEncoder.php +++ /dev/null @@ -1,30 +0,0 @@ -> - */ -final class ListEncoder implements ValueEncoder -{ - public function __construct( - private readonly ValueEncoder $element, - ) {} - - public function encode(mixed $value): string - { - $buffer = pack('V', count($value)); - - // @mago-ignore analysis:mixed-assignment - foreach ($value as $item) { - $buffer .= $this->element->encode($item); - } - - return $buffer; - } -} diff --git a/src/core/etl/src/Flow/Floe/Encoding/MapEncoder.php b/src/core/etl/src/Flow/Floe/Encoding/MapEncoder.php deleted file mode 100644 index 8d5efeacfd..0000000000 --- a/src/core/etl/src/Flow/Floe/Encoding/MapEncoder.php +++ /dev/null @@ -1,31 +0,0 @@ -> - */ -final class MapEncoder implements ValueEncoder -{ - public function __construct( - private readonly ValueEncoder $key, - private readonly ValueEncoder $value, - ) {} - - public function encode(mixed $value): string - { - $buffer = pack('V', count($value)); - - // @mago-ignore analysis:mixed-assignment - foreach ($value as $key => $item) { - $buffer .= $this->key->encode($key) . $this->value->encode($item); - } - - return $buffer; - } -} diff --git a/src/core/etl/src/Flow/Floe/Encoding/NullEncoder.php b/src/core/etl/src/Flow/Floe/Encoding/NullEncoder.php deleted file mode 100644 index bf0ff74273..0000000000 --- a/src/core/etl/src/Flow/Floe/Encoding/NullEncoder.php +++ /dev/null @@ -1,16 +0,0 @@ - - */ -final class NullEncoder implements ValueEncoder -{ - public function encode(mixed $value): string - { - return ''; - } -} diff --git a/src/core/etl/src/Flow/Floe/Encoding/OptionalEncoder.php b/src/core/etl/src/Flow/Floe/Encoding/OptionalEncoder.php deleted file mode 100644 index efc3fceae6..0000000000 --- a/src/core/etl/src/Flow/Floe/Encoding/OptionalEncoder.php +++ /dev/null @@ -1,22 +0,0 @@ - - */ -final class OptionalEncoder implements ValueEncoder -{ - public function __construct( - private readonly ValueEncoder $base, - ) {} - - public function encode(mixed $value): string - { - return $value === null ? Format::VALUE_NULL_BYTE : Format::VALUE_PRESENT_BYTE . $this->base->encode($value); - } -} diff --git a/src/core/etl/src/Flow/Floe/Encoding/PackedListEncoder.php b/src/core/etl/src/Flow/Floe/Encoding/PackedListEncoder.php deleted file mode 100644 index 47a5e74161..0000000000 --- a/src/core/etl/src/Flow/Floe/Encoding/PackedListEncoder.php +++ /dev/null @@ -1,29 +0,0 @@ -> - */ -final class PackedListEncoder implements ValueEncoder -{ - /** - * @param 'e'|'P' $format - */ - public function __construct( - private readonly string $format, - ) {} - - public function encode(mixed $value): string - { - return count($value) === 0 ? pack('V', 0) : pack('V', count($value)) . pack($this->format . '*', ...$value); - } -} diff --git a/src/core/etl/src/Flow/Floe/Encoding/StringEncoder.php b/src/core/etl/src/Flow/Floe/Encoding/StringEncoder.php deleted file mode 100644 index 4669e6ddf0..0000000000 --- a/src/core/etl/src/Flow/Floe/Encoding/StringEncoder.php +++ /dev/null @@ -1,19 +0,0 @@ - - */ -final class StringEncoder implements ValueEncoder -{ - public function encode(mixed $value): string - { - return pack('V', strlen($value)) . $value; - } -} diff --git a/src/core/etl/src/Flow/Floe/Encoding/StringKeyEncoder.php b/src/core/etl/src/Flow/Floe/Encoding/StringKeyEncoder.php deleted file mode 100644 index 9cf0395ef2..0000000000 --- a/src/core/etl/src/Flow/Floe/Encoding/StringKeyEncoder.php +++ /dev/null @@ -1,22 +0,0 @@ - - */ -final class StringKeyEncoder implements ValueEncoder -{ - public function encode(mixed $value): string - { - return pack('V', strlen((string) $value)) . $value; - } -} diff --git a/src/core/etl/src/Flow/Floe/Encoding/StructureEncoder.php b/src/core/etl/src/Flow/Floe/Encoding/StructureEncoder.php deleted file mode 100644 index dff4e44673..0000000000 --- a/src/core/etl/src/Flow/Floe/Encoding/StructureEncoder.php +++ /dev/null @@ -1,39 +0,0 @@ -> - */ -final class StructureEncoder implements ValueEncoder -{ - /** - * @param array> $elements structure element name => encoder - */ - public function __construct( - private readonly array $elements, - ) {} - - public function encode(mixed $value): string - { - $buffer = ''; - - foreach ($this->elements as $name => $element) { - if (!array_key_exists($name, $value)) { - $buffer .= Format::VALUE_ABSENT_BYTE; - } elseif ($value[$name] === null) { - $buffer .= Format::VALUE_NULL_BYTE; - } else { - $buffer .= Format::VALUE_PRESENT_BYTE . $element->encode($value[$name]); - } - } - - return $buffer; - } -} diff --git a/src/core/etl/src/Flow/Floe/Encoding/TimeZoneEncoder.php b/src/core/etl/src/Flow/Floe/Encoding/TimeZoneEncoder.php deleted file mode 100644 index 589fef9d07..0000000000 --- a/src/core/etl/src/Flow/Floe/Encoding/TimeZoneEncoder.php +++ /dev/null @@ -1,21 +0,0 @@ - - */ -final class TimeZoneEncoder implements ValueEncoder -{ - public function encode(mixed $value): string - { - $name = $value->getName(); - - return pack('V', strlen($name)) . $name; - } -} diff --git a/src/core/etl/src/Flow/Floe/Encoding/UuidEncoder.php b/src/core/etl/src/Flow/Floe/Encoding/UuidEncoder.php deleted file mode 100644 index 7eb4e73381..0000000000 --- a/src/core/etl/src/Flow/Floe/Encoding/UuidEncoder.php +++ /dev/null @@ -1,18 +0,0 @@ - - */ -final class UuidEncoder implements ValueEncoder -{ - public function encode(mixed $value): string - { - return ValueEncoderFactory::lengthPrefixed($value->toString()); - } -} diff --git a/src/core/etl/src/Flow/Floe/Encoding/ValueEncoder.php b/src/core/etl/src/Flow/Floe/Encoding/ValueEncoder.php deleted file mode 100644 index 07365b0111..0000000000 --- a/src/core/etl/src/Flow/Floe/Encoding/ValueEncoder.php +++ /dev/null @@ -1,19 +0,0 @@ - - */ -final class XmlDocumentEncoder implements ValueEncoder -{ - public function encode(mixed $value): string - { - return ValueEncoderFactory::lengthPrefixed(ValueEncoderFactory::xmlDocumentToString($value)); - } -} diff --git a/src/core/etl/src/Flow/Floe/Encoding/XmlElementEncoder.php b/src/core/etl/src/Flow/Floe/Encoding/XmlElementEncoder.php deleted file mode 100644 index 32d4b7f2be..0000000000 --- a/src/core/etl/src/Flow/Floe/Encoding/XmlElementEncoder.php +++ /dev/null @@ -1,18 +0,0 @@ - - */ -final class XmlElementEncoder implements ValueEncoder -{ - public function encode(mixed $value): string - { - return ValueEncoderFactory::lengthPrefixed(ValueEncoderFactory::xmlElementToString($value)); - } -} diff --git a/src/core/etl/src/Flow/Floe/Exception/FrameTooLarge.php b/src/core/etl/src/Flow/Floe/Exception/FrameTooLarge.php new file mode 100644 index 0000000000..3354119107 --- /dev/null +++ b/src/core/etl/src/Flow/Floe/Exception/FrameTooLarge.php @@ -0,0 +1,7 @@ + - */ -interface FloeEncoder extends Encoder -{ - /** - * `$hydrator->hydrate($this->decode($bodies), $schema)`, in one native pass where the hydrator is native too. - * - * @param list $bodies - * - * @throws FloeException - * @throws SchemaMismatchException - */ - public function decodeRows(array $bodies, Schema $schema, Hydrator $hydrator): Rows; - - /** - * `Format::rowFrames($this->encode($hydrator->dehydrate($rows)))`, in one native pass where the hydrator is native - * too - complete ROW frames for a writer whose codec leaves bodies as they are. - * - * @throws FloeException - * @throws SchemaMismatchException - */ - public function encodeFrames(Rows $rows, Hydrator $hydrator): string; -} diff --git a/src/core/etl/src/Flow/Floe/FloeEngine.php b/src/core/etl/src/Flow/Floe/FloeEngine.php deleted file mode 100644 index 15e45f0a41..0000000000 --- a/src/core/etl/src/Flow/Floe/FloeEngine.php +++ /dev/null @@ -1,25 +0,0 @@ - NativeFloeEncoder::isSupported() - ? new NativeFloeEncoder($schema) - : new PhpFloeEncoder($schema), - self::native => new NativeFloeEncoder($schema), - self::php => new PhpFloeEncoder($schema), - }; - } -} diff --git a/src/core/etl/src/Flow/Floe/FloeExtractor.php b/src/core/etl/src/Flow/Floe/FloeExtractor.php index 27b8b2a9df..b612fe975b 100644 --- a/src/core/etl/src/Flow/Floe/FloeExtractor.php +++ b/src/core/etl/src/Flow/Floe/FloeExtractor.php @@ -4,21 +4,23 @@ namespace Flow\Floe; -use Flow\ETL\Exception\InferredSchemaException; +use Flow\ETL\Column\PhpBackend; use Flow\ETL\Exception\InvalidArgumentException; use Flow\ETL\Extractor; use Flow\ETL\Extractor\BatchableExtractor; use Flow\ETL\Extractor\Batches; -use Flow\ETL\Extractor\FileExtractor; -use Flow\ETL\Extractor\FileReading; -use Flow\ETL\Extractor\MetadataColumnsExtractor; +use Flow\ETL\Extractor\File\DerivedSchema; +use Flow\ETL\Extractor\File\FileExtractor; +use Flow\ETL\Extractor\File\FileReading; +use Flow\ETL\Extractor\File\FileReadLoop; +use Flow\ETL\Extractor\File\MetadataColumnsExtractor; +use Flow\ETL\Extractor\File\ReadWindow; use Flow\ETL\Extractor\RewindableExtractor; use Flow\ETL\Extractor\Signal; use Flow\ETL\Extractor\Statistics; use Flow\ETL\FlowContext; -use Flow\ETL\Row\Hydrator; +use Flow\ETL\Rows; use Flow\ETL\Schema; -use Flow\ETL\Schema\Validator\StrictValidator; use Flow\Filesystem\Filesystem; use Flow\Filesystem\Local\NativeLocalFilesystem; use Flow\Filesystem\Path; @@ -28,6 +30,7 @@ use Generator; use function iterator_count; +use function iterator_to_array; use function sprintf; final class FloeExtractor implements @@ -37,8 +40,6 @@ final class FloeExtractor implements MetadataColumnsExtractor, RewindableExtractor { - private ?Schema $schema = null; - use Batches; use FileReading; @@ -54,7 +55,6 @@ public function __construct( private readonly Path $path, private readonly Codec $codec = new NoopCodec(), private readonly int $chunkSize = 65536, - private readonly FloeEngine $engine = FloeEngine::adaptive, Filesystem $filesystem = new NativeLocalFilesystem(), ) { if (!$filesystem->supports($path)) { @@ -76,77 +76,27 @@ public function isRepeatable(): bool } /** - * @return \Generator + * @return Generator */ public function extract(FlowContext $context, ?int $limit = null, Filter $pathFilter = new OnlyFiles()): Generator { - $fileOffset = $this->offset ?? 0; - $yielded = 0; - $promisedSchema = $this->schema === null ? null : $this->schema(); - // undeclared, every file is read under the first file's schema, or the union of all of them - $expected = $this->schema === null ? $this->derivedSchema($this->files(), $this->unionByName) : null; - $target = $promisedSchema ?? $this->schema(); - $fileColumns = $this->fileColumns($this->filesystem, $this->path); - foreach ($this->files($context->hydrator(), $pathFilter) as $file) { - // finally, not a close() per exit: the offset-skip continue, the STOP/limit return - // below and an abandoned generator all have to release the handle (b73) - try { - $fileRows = $file->reader->totalRows(); - - if ($fileOffset >= $fileRows) { - $fileOffset -= $fileRows; - - continue; - } - - if ($expected !== null && !$this->unionByName) { - $validation = (new StrictValidator())->validate($expected, $file->schema()); - - if (!$validation->isValid()) { - throw InferredSchemaException::filesDiverge( - $file->source()->uri(), - $this->derivedFrom, - $validation, - ); - } - } - - // R6: over the FILE's schema, never over schema()'s output - $fileSchema = $fileColumns->declare($this->schema ?? $file->schema()); - $constants = $fileColumns->forFile($file->source(), $fileSchema); - // a declared schema is always matched: the rows below are only trusted against the footer - $matchTo = $promisedSchema ?? (!$fileSchema->isSame($target) ? $target : null); - $remaining = $limit === null ? null : $limit - $yielded; - - foreach ($file->reader->rows($this->batchSize(), $fileOffset, $remaining) as $rows) { - // R7: the stamp stays post-hydration - FloeStreamReader::rows() yields hydrated Rows and - // must not learn about paths - but the constants are the shared ones, already typed - $rows = $constants->fillRows($rows, $fileSchema); - - if ($matchTo !== null) { - $rows = $rows->matchTo($matchTo); - } - - $yielded += $rows->count(); - - $signal = yield $rows; - - if ($signal === Signal::STOP) { - return; - } - - if ($limit !== null && $yielded >= $limit) { - return; - } - } - - $fileOffset = 0; - } finally { - $file->close(); - } - } + yield from (new FileReadLoop($fileColumns, $this->schema()))->read( + iterator_to_array($this->sourceFiles($this->filesystem, $this->path, $pathFilter), false), + new FloeFileBatches( + $this->filesystem, + $this->codec, + $this->chunkSize, + $fileColumns, + $this->unionByName + ? null + : new DerivedSchema($this->derivedSchema($this->files(), $this->unionByName), $this->derivedFrom), + ), + $this->batchSize(), + $context->backend(), + new ReadWindow($this->offset ?? 0, $limit), + ); } /** @@ -155,9 +105,10 @@ public function extract(FlowContext $context, ?int $limit = null, Filter $pathFi */ public function schema(): Schema { - return $this->fileColumns($this->filesystem, $this->path)->declare( - $this->schema ?? $this->derivedSchema($this->files(), $this->unionByName), - ); + return $this->fileColumns($this->filesystem, $this->path)->declare($this->derivedSchema( + $this->files(), + $this->unionByName, + )); } public function statistics(): Statistics @@ -179,7 +130,7 @@ public function unionByName(bool $union = true): self public function partitionSchema(): Schema { - return $this->fileColumns($this->filesystem, $this->path)->partitions($this->schema ?? new Schema()); + return $this->fileColumns($this->filesystem, $this->path)->partitions(new Schema()); } public function source(): Path @@ -205,26 +156,25 @@ public function withOffset(int $offset): self */ private function declare(): Statistics { - $footers = $this->schema === null - ? $this->derivedFooters($this->files(), $this->unionByName) - : $this->footers($this->files(), $this->unionByName); - - return $footers->of(iterator_count($this->sourceFiles($this->filesystem, $this->path)), $this->offset ?? 0); + return $this->derivedFooters($this->files(), $this->unionByName)->of( + iterator_count($this->sourceFiles($this->filesystem, $this->path)), + $this->offset ?? 0, + ); } /** * @return Generator */ - private function files(?Hydrator $hydrator = null, Filter $pathFilter = new OnlyFiles()): Generator + private function files(): Generator { - foreach ($this->sourceFiles($this->filesystem, $this->path, $pathFilter) as $source) { + foreach ($this->sourceFiles($this->filesystem, $this->path) as $source) { yield new FloeSourceFile( + // only the footers are read here, no column is decoded: any backend would do (new FloeReader( $this->filesystem, + new PhpBackend(), $this->codec, $this->chunkSize, - hydrator: $hydrator, - engine: $this->engine, ))->read($source->path), $source, ); @@ -233,8 +183,8 @@ private function files(?Hydrator $hydrator = null, Filter $pathFilter = new Only public function withSchema(Schema $schema): static { - $this->schema = $schema; - - return $this; + throw new InvalidArgumentException( + 'Floe is a self-describing format and does not accept a schema; declaring one is not supported yet.', + ); } } diff --git a/src/core/etl/src/Flow/Floe/FloeFileBatches.php b/src/core/etl/src/Flow/Floe/FloeFileBatches.php new file mode 100644 index 0000000000..0f6d2954c0 --- /dev/null +++ b/src/core/etl/src/Flow/Floe/FloeFileBatches.php @@ -0,0 +1,77 @@ + $batchSize + * + * @throws InferredSchemaException + * + * @return Generator + */ + public function batches( + SourceFile $source, + Schema $body, + int $batchSize, + Backend $backend, + ReadWindow $window, + ): Generator { + $file = new FloeSourceFile( + (new FloeReader($this->filesystem, $backend, $this->codec, $this->chunkSize))->read($source->path), + $source, + ); + + try { + $fileRows = $file->reader->totalRows(); + + if ($window->offset >= $fileRows) { + return $fileRows; + } + + $this->derived?->refuseDivergence($file->source()->uri(), $file->schema()); + + $fileBody = $this->fileColumns->withoutTail($file->schema()); + + foreach ($file->reader->rows($batchSize, $window->offset, $window->limit) as $rows) { + if (!$fileBody->isSame($file->schema())) { + $rows = $rows->project($fileBody, $backend); + } + + yield $fileBody->isSame($body) ? $rows : $rows->matchTo($body, $backend); + } + + return $window->offset; + } finally { + $file->close(); + } + } +} diff --git a/src/core/etl/src/Flow/Floe/FloeFileSink.php b/src/core/etl/src/Flow/Floe/FloeFileSink.php new file mode 100644 index 0000000000..01ece5a9b4 --- /dev/null +++ b/src/core/etl/src/Flow/Floe/FloeFileSink.php @@ -0,0 +1,31 @@ +writer?->close(); + } + + public function write(Rows $rows): void + { + ($this->writer ??= $this->sinks->writer($this->stream, $rows->schema(), $this->backend))->write($rows); + } +} diff --git a/src/core/etl/src/Flow/Floe/FloeFileSinks.php b/src/core/etl/src/Flow/Floe/FloeFileSinks.php new file mode 100644 index 0000000000..ba45f8c16e --- /dev/null +++ b/src/core/etl/src/Flow/Floe/FloeFileSinks.php @@ -0,0 +1,47 @@ +schema = $declared; + } + + public function open(DestinationStream $stream, Backend $backend): FileSink + { + return new FloeFileSink($this, $stream, $backend); + } + + /** + * The writer of one stream under the run's schema - the declared one, or the first written batch's. + */ + public function writer(DestinationStream $stream, Schema $written, Backend $backend): FloeWriter + { + $writer = new FloeWriter($this->filesystem, $this->schema ??= $written, $backend, $this->options); + $writer->createForStream($stream, $this->metadata); + + return $writer; + } +} diff --git a/src/core/etl/src/Flow/Floe/FloeLoader.php b/src/core/etl/src/Flow/Floe/FloeLoader.php index 3878e5a930..69c9465903 100644 --- a/src/core/etl/src/Flow/Floe/FloeLoader.php +++ b/src/core/etl/src/Flow/Floe/FloeLoader.php @@ -4,32 +4,30 @@ namespace Flow\Floe; -use Flow\ETL\Config\Telemetry\TelemetryAttributes; use Flow\ETL\Exception\InvalidArgumentException; -use Flow\ETL\Filesystem\FilesSink; use Flow\ETL\Filesystem\SaveMode; use Flow\ETL\FlowContext; use Flow\ETL\Loader; use Flow\ETL\Loader\Closure; use Flow\ETL\Loader\Discardable; -use Flow\ETL\Loader\FileLoader; +use Flow\ETL\Loader\File\FileLoader; +use Flow\ETL\Loader\File\FileWriteFrame; +use Flow\ETL\Loader\File\PartitionRouter; use Flow\ETL\Loader\Partitioning; use Flow\ETL\Loader\PartitioningLoader; -use Flow\ETL\Loader\PartitionRouter; use Flow\ETL\Rows; use Flow\ETL\Schema; use Flow\ETL\Schema\Metadata; -use Flow\Filesystem\DestinationStream; use Flow\Filesystem\Filesystem; use Flow\Filesystem\Local\NativeLocalFilesystem; use Flow\Filesystem\Path; -use Flow\Floe\Exception\FloeException; -use Throwable; use function sprintf; final class FloeLoader implements Closure, Discardable, FileLoader, Loader, PartitioningLoader { + private ?FileWriteFrame $frame = null; + private PartitionRouter $router; public function partitionBy(Partitioning $partitioning): static @@ -43,20 +41,12 @@ public function partitionBy(Partitioning $partitioning): static private SaveMode $saveMode = SaveMode::ExceptionIfExists; - private ?FilesSink $files = null; - - /** @var array */ - private array $writers = []; - private ?Schema $schema = null; - private ?Schema $inferredSchema = null; - public function __construct( private readonly Path $path, private readonly ?Metadata $metadata = null, private readonly Options $options = new Options(), - private readonly FloeEngine $engine = FloeEngine::adaptive, Filesystem $filesystem = new NativeLocalFilesystem(), ) { if (!$filesystem->supports($path)) { @@ -89,20 +79,14 @@ public function withSchema(Schema $schema): static public function closure(FlowContext $context): void { - $this->closeWriters(); - - $this->files?->publish(); - $this->files = null; + $this->frame?->closure(); + $this->frame = null; } public function discard(FlowContext $context): void { - try { - $this->closeWriters(); - } finally { - $this->files?->abandon(); - $this->files = null; - } + $this->frame?->discard(); + $this->frame = null; } public function destination(): Path @@ -112,70 +96,20 @@ public function destination(): Path public function load(Rows $rows, FlowContext $context): void { - $context->telemetry()->loadingStarted($this, [ - TelemetryAttributes::ATTR_LOADER_DESTINATION_URI => $this->path->uri(), - ]); - - try { - if ($this->schema === null && $this->inferredSchema === null) { - $this->inferredSchema = $rows->schema(); - } - - foreach ($this->router->route($rows) as [$partitions, $group]) { - $stream = ($this->files ??= new FilesSink($this->filesystem, $this->path, $this->saveMode))->writeTo( - $partitions->toArray(), - ); - - ($this->writers[$stream->path()->uri()] ??= $this->openWriter($stream, $context))->write($group); - } - - $context->telemetry()->loadingCompleted($this, [TelemetryAttributes::ATTR_LOADING_ROWS => $rows->count()]); - } catch (Throwable $e) { - $context->telemetry()->loadingFailed($this, $e); - - throw $e; - } - } - - private function closeWriters(): void - { - $failure = null; - - foreach ($this->writers as $uri => $writer) { - unset($this->writers[$uri]); - - try { - $writer->close(); - } catch (Throwable $closeFailure) { - $failure ??= $closeFailure; - } - } - - if ($failure !== null) { - throw $failure; - } - } - - private function openWriter(DestinationStream $stream, FlowContext $context): FloeWriter - { - $declared = - $this->schema - ?? $this->inferredSchema - ?? throw new FloeException('Floe loader has no schema to write with'); - - // partitionBy() keeps the partition columns in the path rather than the body, so the file - // schema must not declare them either. Every other declared column stays - a batch that does - // not carry one is the writer's to refuse. - - $writer = new FloeWriter( + // partitionBy() keeps the partition columns in the path rather than the body, so the file schema must not + // declare them either. Every other declared column stays - a batch that does not carry one is the writer's + // to refuse. + ($this->frame ??= new FileWriteFrame( $this->filesystem, - $declared->gracefulRemove(...$this->router->droppedNames()), - $this->options, - hydrator: $context->hydrator(), - engine: $this->engine, - ); - $writer->createForStream($stream, $this->metadata); - - return $writer; + $this->path, + $this->saveMode, + $this->router, + new FloeFileSinks( + $this->filesystem, + $this->schema?->gracefulRemove(...$this->router->droppedNames()), + $this->metadata, + $this->options, + ), + ))->write($rows, $context, $this); } } diff --git a/src/core/etl/src/Flow/Floe/FloeMerger.php b/src/core/etl/src/Flow/Floe/FloeMerger.php index 29b46851b5..cdac513cce 100644 --- a/src/core/etl/src/Flow/Floe/FloeMerger.php +++ b/src/core/etl/src/Flow/Floe/FloeMerger.php @@ -4,7 +4,7 @@ namespace Flow\Floe; -use Flow\ETL\Row\Hydrator; +use Flow\ETL\Column\Backend; use Flow\ETL\Schema; use Flow\ETL\Schema\Metadata; use Flow\ETL\Schema\Validator\EvolvingValidator; @@ -22,7 +22,7 @@ public function __construct( private Filesystem $filesystem, - private ?Hydrator $hydrator = null, + private Backend $backend, private Codec $codec = new NoopCodec(), ) { Format::validateCodecId($this->codec->id()); @@ -45,9 +45,6 @@ public function merge(array $sources, Path $dest, bool $compact = false, ?Metada $metadata ??= Metadata::empty(); - // a merged file carries exactly one schema; raw splicing is only valid when every - // source already shares it - differing (but compatible) schemas are re-encoded to - // the union, exactly as compaction does if ($compact || !$this->sourcesShareSchema($reconciled['layouts'], $reconciled['merged'])) { $this->mergeCompact($sources, $reconciled['layouts'], $reconciled['merged'], $dest, $metadata); @@ -147,12 +144,12 @@ private function mergeCompact(array $sources, array $layouts, ?Schema $merged, P $writer = new FloeWriter( $this->filesystem, $merged ?? new Schema(), + $this->backend, new Options(codec: $this->codec), - hydrator: $this->hydrator, ); $writer->create($dest, $mergedMetadata->merge($metadata)); - $reader = new FloeReader($this->filesystem, $this->codec, hydrator: $this->hydrator); + $reader = new FloeReader($this->filesystem, $this->backend, $this->codec); foreach ($sources as $source) { $streamReader = $reader->read($source); diff --git a/src/core/etl/src/Flow/Floe/FloeReader.php b/src/core/etl/src/Flow/Floe/FloeReader.php index 0378ea72a5..0ac998b76b 100644 --- a/src/core/etl/src/Flow/Floe/FloeReader.php +++ b/src/core/etl/src/Flow/Floe/FloeReader.php @@ -4,7 +4,7 @@ namespace Flow\Floe; -use Flow\ETL\Row\Hydrator; +use Flow\ETL\Column\Backend; use Flow\Filesystem\Filesystem; use Flow\Filesystem\Path; use Flow\Floe\Codec\NoopCodec; @@ -12,15 +12,11 @@ final readonly class FloeReader { - /** - * @param null|Hydrator $hydrator null uses the adaptive hydrator - */ public function __construct( private Filesystem $filesystem, + private Backend $backend, private Codec $codec = new NoopCodec(), private int $chunkSize = 65536, - private ?Hydrator $hydrator = null, - private FloeEngine $engine = FloeEngine::adaptive, ) {} /** @@ -28,12 +24,6 @@ public function __construct( */ public function read(Path $path): FloeStreamReader { - return new FloeStreamReader( - $this->filesystem->readFrom($path), - $this->codec, - $this->chunkSize, - $this->hydrator, - $this->engine, - ); + return new FloeStreamReader($this->filesystem->readFrom($path), $this->codec, $this->chunkSize, $this->backend); } } diff --git a/src/core/etl/src/Flow/Floe/FloeSerializer.php b/src/core/etl/src/Flow/Floe/FloeSerializer.php index 3b7c8d0e90..f20f0c7ac4 100644 --- a/src/core/etl/src/Flow/Floe/FloeSerializer.php +++ b/src/core/etl/src/Flow/Floe/FloeSerializer.php @@ -4,17 +4,18 @@ namespace Flow\Floe; -use Flow\ETL\Row\Hydrator; +use Flow\ETL\Column\Backend; +use Flow\ETL\Exception\RuntimeException; use Flow\ETL\Rows; +use Flow\ETL\Rows\RowsBuilder; use Flow\Filesystem\DestinationStream; use Flow\Filesystem\SourceStream; use Flow\Floe\Codec\NoopCodec; -use Flow\Floe\Exception\ExtensionException; use Flow\Floe\Exception\FloeException; use Flow\Serializer\Exception\SerializationException; use Flow\Serializer\Serializer; -use function count; +use function array_slice; use function sprintf; final class FloeSerializer implements Serializer @@ -23,11 +24,10 @@ final class FloeSerializer implements Serializer /** * @param int<1, max> $batchSize - * @param null|Hydrator $hydrator null uses the adaptive hydrator */ public function __construct( + private readonly Backend $backend, private readonly int $batchSize = 1000, - private readonly ?Hydrator $hydrator = null, ) { // @mago-ignore analysis:impossible-condition,redundant-comparison if ($this->batchSize < 1) { @@ -39,11 +39,11 @@ public function serialize(Rows $rows, DestinationStream $destination): void { try { // validation stays on until an upstream mechanism guarantees Rows match their schema - $writer = new FloeStreamWriter($rows->schema(), new Options(), hydrator: $this->hydrator); + $writer = new FloeStreamWriter($rows->schema(), $this->backend, new Options()); $writer->create($destination); $writer->write($rows); $writer->close(); - } catch (FloeException|ExtensionException $e) { + } catch (FloeException|RuntimeException $e) { throw new SerializationException($e->getMessage(), 0, $e); } } @@ -51,29 +51,30 @@ public function serialize(Rows $rows, DestinationStream $destination): void public function unserialize(SourceStream $source): Rows { try { - $reader = new FloeStreamReader($source, new NoopCodec(), self::CHUNK_SIZE, $this->hydrator); + $reader = new FloeStreamReader($source, new NoopCodec(), self::CHUNK_SIZE, $this->backend); $footer = $reader->footer(); - $rows = []; + $batches = []; + $decoded = 0; foreach ($reader->rows($this->batchSize) as $batch) { - foreach ($batch->all() as $row) { - $rows[] = $row; - } + $batches[] = $batch; + $decoded += $batch->count(); } - if (count($rows) !== $footer->statistics->rows) { + if ($decoded !== $footer->statistics->rows) { throw new FloeException(sprintf( 'Floe payload is corrupted, decoded %d of %d rows', - count($rows), + $decoded, $footer->statistics->rows, )); } - // every row came out of a reader batch already conformed to this schema - return Rows::trusted($reader->schema(), $rows); - } catch (FloeException|ExtensionException $e) { + return $batches === [] + ? (new RowsBuilder($reader->schema(), $this->backend))->finish() + : $batches[0]->concat($this->backend, ...array_slice($batches, 1)); + } catch (FloeException|RuntimeException $e) { throw new SerializationException($e->getMessage(), 0, $e); } finally { $source->close(); diff --git a/src/core/etl/src/Flow/Floe/FloeSourceFile.php b/src/core/etl/src/Flow/Floe/FloeSourceFile.php index ec5cf384e5..b12184a258 100644 --- a/src/core/etl/src/Flow/Floe/FloeSourceFile.php +++ b/src/core/etl/src/Flow/Floe/FloeSourceFile.php @@ -5,8 +5,8 @@ namespace Flow\Floe; use Flow\ETL\Cardinality; -use Flow\ETL\Extractor\SelfDescribingFile; -use Flow\ETL\Extractor\SourceFile; +use Flow\ETL\Extractor\File\SelfDescribingFile; +use Flow\ETL\Extractor\File\SourceFile; use Flow\ETL\Extractor\Statistics; use Flow\ETL\Schema; diff --git a/src/core/etl/src/Flow/Floe/FloeStreamReader.php b/src/core/etl/src/Flow/Floe/FloeStreamReader.php index 18bfe2bdee..f83bb56bb9 100644 --- a/src/core/etl/src/Flow/Floe/FloeStreamReader.php +++ b/src/core/etl/src/Flow/Floe/FloeStreamReader.php @@ -4,21 +4,17 @@ namespace Flow\Floe; -use Flow\ETL\Row\AdaptiveRowHydrator; -use Flow\ETL\Row\Hydrator; +use Flow\ETL\Column\Backend; +use Flow\ETL\Exception\RuntimeException; use Flow\ETL\Rows; use Flow\ETL\Schema; use Flow\ETL\Schema\Metadata; use Flow\Filesystem\SourceStream; -use Flow\Floe\Codec\NoopCodec; -use Flow\Floe\Exception\ExtensionException; use Flow\Floe\Exception\FloeException; use Flow\Serializer\Exception\SerializationException; use Generator; -use function count; use function max; -use function min; use function ord; use function sprintf; use function strlen; @@ -27,28 +23,23 @@ final class FloeStreamReader { - private ?FloeEncoder $encoder = null; - private ?Footer $footer = null; private ?Schema $schema = null; - private readonly Hydrator $hydrator; + private readonly FrameDecoder $frameDecoder; /** - * @param null|Hydrator $hydrator null uses the adaptive hydrator - * * @throws FloeException */ public function __construct( private readonly SourceStream $source, private readonly Codec $codec, private readonly int $chunkSize, - ?Hydrator $hydrator = null, - private readonly FloeEngine $engine = FloeEngine::adaptive, + private readonly Backend $backend, ) { Format::validateCodecId($this->codec->id()); - $this->hydrator = $hydrator ?? new AdaptiveRowHydrator(); + $this->frameDecoder = new FrameDecoder($this->codec); } public function close(): void @@ -56,11 +47,6 @@ public function close(): void $this->source->close(); } - private function encoder(Schema $schema): FloeEncoder - { - return $this->encoder ??= $this->engine->encoder($schema); - } - /** * @throws FloeException */ @@ -80,7 +66,8 @@ public function metadata(): Metadata /** * Hot path: frames are walked in-buffer. offset skips whole leading sections * using the footer, then the remaining rows inside the start section; limit - * stops the read after that many rows. + * stops the read after that many rows. Batches are frame-sized: batchSize is + * an upper bound, a batch never spans two BATCH frames. * * @param int<1, max> $batchSize * @@ -173,6 +160,8 @@ public function tail(int $count, int $batchSize = 1000): Generator * The single strict frame-walk shared by rows() and rowsFromOffset(); they * differ only in how it is seeded - start position and skip. The * file carries exactly one schema, so decode state comes from the footer. + * Every BATCH frame yields at most batchSize rows per batch; a frame wholly + * inside skip is passed over without decoding. * * @param \Generator $chunks * @param int<1, max> $batchSize @@ -193,10 +182,6 @@ private function walk( $fill = FrameReader::chunkFiller($buffer, $position, $chunks); $yielded = 0; - /** @var list $pending */ - $pending = []; - $flushThreshold = $limit !== null && $limit < $batchSize ? $limit : $batchSize; - $transforms = !$this->codec instanceof NoopCodec; try { while (true) { @@ -218,29 +203,48 @@ private function walk( $frameEnd = $position + $frameLength; - if ($frameType === Format::FRAME_ROW) { - if ($skip > 0) { - $skip--; - } else { - $body = substr($buffer, $position, $frameLength); - $pending[] = $transforms ? $this->codec->decode($body) : $body; + if ($frameType === Format::FRAME_BATCH) { + if ($frameLength < 4) { + throw new FloeException('Floe BATCH frame is truncated, its directory is incomplete'); } - $position = $frameEnd; + /** @var int $rowCount */ + $rowCount = unpack('V', $buffer, $position)[1]; + + if ($skip >= $rowCount) { + $skip -= $rowCount; + } else { + $rows = $this->frameDecoder->decode( + substr($buffer, $position, $frameLength), + $schema, + $this->backend, + ); + + if ($skip > 0) { + $rows = $rows->slice($skip, $rows->count() - $skip); + $skip = 0; + } - if (count($pending) === $flushThreshold) { - yield $this->decode($schema, $pending); - $yielded += $flushThreshold; - $pending = []; + if ($limit !== null && $rows->count() > ($limit - $yielded)) { + $rows = $rows->slice(0, $limit - $yielded); + } - if ($limit !== null) { - if ($yielded === $limit) { - return; + if ($rows->count() <= $batchSize) { + yield $rows; + } else { + foreach ($rows->chunks($batchSize) as $chunk) { + yield $chunk; } + } - $flushThreshold = min($batchSize, $limit - $yielded); + $yielded += $rows->count(); + + if ($limit !== null && $yielded === $limit) { + return; } } + + $position = $frameEnd; } elseif ($frameType === Format::FRAME_FOOTER) { $position = $frameEnd; } else { @@ -252,27 +256,11 @@ private function walk( $position = 0; } } - - if ($pending !== []) { - yield $this->decode($schema, $pending); - } - } catch (SerializationException|ExtensionException $e) { + } catch (SerializationException|RuntimeException $e) { throw new FloeException($e->getMessage(), 0, $e); } } - /** - * A batch carries the file schema even when the hydrator folded per-value metadata into a schema of its own. - * - * @param list $pending - */ - private function decode(Schema $schema, array $pending): Rows - { - $rows = $this->encoder($schema)->decodeRows($pending, $schema, $this->hydrator); - - return $rows->schema() === $schema ? $rows : Rows::trusted($schema, $rows->all()); - } - /** * @throws FloeException */ diff --git a/src/core/etl/src/Flow/Floe/FloeStreamWriter.php b/src/core/etl/src/Flow/Floe/FloeStreamWriter.php index b82b5975a3..609094fe8a 100644 --- a/src/core/etl/src/Flow/Floe/FloeStreamWriter.php +++ b/src/core/etl/src/Flow/Floe/FloeStreamWriter.php @@ -5,18 +5,21 @@ namespace Flow\Floe; use Composer\InstalledVersions; -use Flow\ETL\Row\AdaptiveRowHydrator; -use Flow\ETL\Row\Hydrator; +use Flow\ETL\Column\Backend; +use Flow\ETL\Exception\OffsetOverflow; use Flow\ETL\Rows; use Flow\ETL\Schema; use Flow\ETL\Schema\Metadata; use Flow\ETL\Schema\Validator\EvolvingValidator; use Flow\Filesystem\DestinationStream; -use Flow\Floe\Codec\NoopCodec; use Flow\Floe\Exception\FloeException; +use Flow\Floe\Exception\FrameTooLarge; use Flow\Floe\Exception\IncompatibleSchemaException; use function array_key_exists; +use function array_shift; +use function array_unshift; +use function intdiv; use function sprintf; final class FloeStreamWriter @@ -51,26 +54,23 @@ final class FloeStreamWriter private Schema $sessionSchema; - private ?FloeEncoder $sessionEncoder = null; - private ?FrameWriter $frameWriter = null; private int $totalRows = 0; - private readonly Hydrator $hydrator; + private readonly FrameEncoder $frameEncoder; /** * @throws FloeException */ public function __construct( Schema $schema, + private readonly Backend $backend, private readonly Options $options = new Options(), - ?Hydrator $hydrator = null, - private readonly FloeEngine $engine = FloeEngine::adaptive, ) { Format::validateCodecId($this->options->codec->id()); $this->sessionSchema = $schema; - $this->hydrator = $hydrator ?? new AdaptiveRowHydrator(); + $this->frameEncoder = new FrameEncoder($this->options->codec); } /** @@ -154,26 +154,46 @@ public function write(Rows $rows): void return; } - $this->openSession(); - $this->assertBatchFitsSession($rows->schema()); - - $matched = $rows->schema()->isSame($this->sessionSchema) ? $rows : $rows->matchTo($this->sessionSchema); - - if ($this->options->codec instanceof NoopCodec) { - $frames = $this->sessionEncoder()->encodeFrames($matched, $this->hydrator); - - $this->startSectionWhenDue(); - $this->frameWriter()->raw($frames); - $this->sectionRowCount += $matched->count(); - $this->totalRows += $matched->count(); + $matched = $rows; - return; + if (!$rows->schema()->isSame($this->sessionSchema)) { + $this->assertBatchFitsSession($rows->schema()); + $matched = $rows->matchTo($this->sessionSchema, $this->backend); } - $typed = $this->hydrator->dehydrate($matched); + $bodies = []; + $parts = [[$matched, 0]]; + + while ($parts !== []) { + [$part, $start] = array_shift($parts); + + try { + $bodies[] = [$this->frameEncoder->encode($part), $part->count()]; + } catch (OffsetOverflow|FrameTooLarge $e) { + if ($part->count() === 1) { + $message = sprintf('%s (row %d)', $e->getMessage(), $start); + + throw $e instanceof OffsetOverflow + ? new OffsetOverflow($message, 0, $e) + : new FrameTooLarge($message, 0, $e); + } + + $half = intdiv($part->count(), 2); + array_unshift( + $parts, + [$part->slice(0, $half), $start], + [$part->slice($half, $part->count() - $half), $start + $half], + ); + } + } $this->startSectionWhenDue(); - $this->emitBatch($typed); + + foreach ($bodies as [$body, $count]) { + $this->frameWriter()->frame(Format::FRAME_BATCH, $body); + $this->sectionRowCount += $count; + $this->totalRows += $count; + } } public static function writerVersion(): string @@ -183,18 +203,6 @@ public static function writerVersion(): string : 'unknown'; } - /** - * @throws FloeException - */ - private function openSession(): void - { - if ($this->sessionEncoder !== null) { - return; - } - - $this->sessionEncoder = $this->engine->encoder($this->sessionSchema); - } - /** * @throws IncompatibleSchemaException */ @@ -203,7 +211,7 @@ private function assertBatchFitsSession(Schema $batch): void $definitions = $this->sessionSchema->definitions(); // unconditional, and before the validator: EvolvingValidator admits a nullable extra column, - // which the session encoder would then drop - silent column loss is not unlockable here + // which the session schema cannot carry foreach ($batch->definitions() as $name => $_) { if (!array_key_exists($name, $definitions)) { throw new IncompatibleSchemaException(sprintf(self::BATCH_MISMATCH, sprintf('new column "%s"', $name))); @@ -217,20 +225,6 @@ private function assertBatchFitsSession(Schema $batch): void } } - /** - * @param list<\Flow\ETL\Row\TypedRowValues> $typed - * - * @throws FloeException - */ - private function emitBatch(array $typed): void - { - foreach ($this->sessionEncoder()->encode($typed) as $encoded) { - $this->frameWriter()->row($this->options->codec->encode($encoded)); - $this->sectionRowCount++; - $this->totalRows++; - } - } - private function closeSection(): void { if ($this->sectionOpen) { @@ -282,14 +276,6 @@ private function startSection(): void $this->sectionRowCount = 0; } - /** - * @throws FloeException - */ - private function sessionEncoder(): FloeEncoder - { - return $this->sessionEncoder ?? throw new FloeException('Floe writer has no active session encoder'); - } - /** * @throws FloeException */ diff --git a/src/core/etl/src/Flow/Floe/FloeWriter.php b/src/core/etl/src/Flow/Floe/FloeWriter.php index dd9df37b69..894d36ead1 100644 --- a/src/core/etl/src/Flow/Floe/FloeWriter.php +++ b/src/core/etl/src/Flow/Floe/FloeWriter.php @@ -4,7 +4,8 @@ namespace Flow\Floe; -use Flow\ETL\Row\Hydrator; +use Flow\ETL\Column\Backend; +use Flow\ETL\Loader\File\FileSink; use Flow\ETL\Rows; use Flow\ETL\Schema; use Flow\ETL\Schema\Metadata; @@ -15,24 +16,22 @@ use function sprintf; -final class FloeWriter +final class FloeWriter implements FileSink { private readonly FloeStreamWriter $inner; /** * @param Schema $schema fixes the session schema for the writer's life - * @param null|Hydrator $hydrator null uses the adaptive hydrator * * @throws FloeException */ public function __construct( private readonly Filesystem $filesystem, Schema $schema, + Backend $backend, private readonly Options $options = new Options(), - ?Hydrator $hydrator = null, - FloeEngine $engine = FloeEngine::adaptive, ) { - $this->inner = new FloeStreamWriter($schema, $this->options, $hydrator, $engine); + $this->inner = new FloeStreamWriter($schema, $backend, $this->options); } /** diff --git a/src/core/etl/src/Flow/Floe/Format.php b/src/core/etl/src/Flow/Floe/Format.php index bf4c32517f..814cb97f91 100644 --- a/src/core/etl/src/Flow/Floe/Format.php +++ b/src/core/etl/src/Flow/Floe/Format.php @@ -4,13 +4,9 @@ namespace Flow\Floe; -use Flow\ETL\Schema\Metadata; use Flow\Floe\Exception\FloeException; -use JsonException; use function chr; -use function json_decode; -use function json_encode; use function ord; use function pack; use function sprintf; @@ -19,8 +15,6 @@ use function substr; use function unpack; -use const JSON_THROW_ON_ERROR; - final class Format { public const string MAGIC = 'FLOE'; @@ -33,84 +27,15 @@ final class Format public const int FRAME_HEADER_LENGTH = 5; - public const int FRAME_ROW = 0x02; + public const int FRAME_BATCH = 0x05; public const int FRAME_FOOTER = 0x06; - public const int VALUE_NULL = 0x00; - - public const int VALUE_PRESENT = 0x01; - - public const int VALUE_NULL_WITH_META = 0x02; - - public const int VALUE_ABSENT = 0x03; - - public const int VALUE_PRESENT_WITH_META = 0x04; - - public const string VALUE_NULL_BYTE = "\x00"; - - public const string VALUE_PRESENT_BYTE = "\x01"; - - public const string VALUE_NULL_WITH_META_BYTE = "\x02"; - - public const string VALUE_ABSENT_BYTE = "\x03"; - - public const string VALUE_PRESENT_WITH_META_BYTE = "\x04"; - public static function frame(int $type, string $body): string { return chr($type) . pack('V', strlen($body)) . $body; } - /** - * @param list $bodies - */ - public static function rowFrames(array $bodies): string - { - $frames = ''; - - foreach ($bodies as $body) { - $frames .= self::frame(self::FRAME_ROW, $body); - } - - return $frames; - } - - /** - * @throws FloeException - */ - public static function metadataBytes(Metadata $metadata): string - { - try { - $json = json_encode($metadata->normalize(), JSON_THROW_ON_ERROR); - } catch (JsonException $e) { - throw new FloeException('Floe failed to encode per-value metadata: ' . $e->getMessage(), 0, $e); - } - - return pack('V', strlen($json)) . $json; - } - - /** - * @throws FloeException - */ - public static function readMetadata(string $body, int &$position): Metadata - { - $length = unpack('V', substr($body, $position, 4))[1]; - $position += 4; - - $json = substr($body, $position, $length); - $position += $length; - - try { - /** @var array|bool|float|int|string> $map */ - $map = json_decode($json, true, 512, JSON_THROW_ON_ERROR); - } catch (JsonException $e) { - throw new FloeException('Floe failed to decode per-value metadata: ' . $e->getMessage(), 0, $e); - } - - return Metadata::fromArray($map); - } - public static function header(int $flags): string { return self::MAGIC . chr(self::VERSION) . chr($flags); diff --git a/src/core/etl/src/Flow/Floe/FrameDecoder.php b/src/core/etl/src/Flow/Floe/FrameDecoder.php new file mode 100644 index 0000000000..ef2bb237b5 --- /dev/null +++ b/src/core/etl/src/Flow/Floe/FrameDecoder.php @@ -0,0 +1,200 @@ + $rowCount, 2 => $nodeCount, 3 => $bufferCount] = $counts; + $nodeCounts = []; + $bufferCounts = []; + + foreach ($schema->definitions() as $name => $definition) { + $nodeCounts[$name] = $this->layout->nodeCount($definition->type()); + $bufferCounts[$name] = $this->layout->bufferCount($definition->type()); + } + + $expectedNodes = array_sum($nodeCounts); + $expectedBuffers = array_sum($bufferCounts); + + if ($nodeCount !== $expectedNodes) { + throw new FloeException(sprintf( + 'Floe BATCH frame holds %d nodes, the schema describes %d', + $nodeCount, + $expectedNodes, + )); + } + + if ($bufferCount !== $expectedBuffers) { + throw new FloeException(sprintf( + 'Floe BATCH frame holds %d buffers, the schema describes %d', + $bufferCount, + $expectedBuffers, + )); + } + + $areaStart = intdiv(12 + (8 * ($nodeCount + $bufferCount)) + 7, 8) * 8; + + if (strlen($body) < $areaStart) { + throw new FloeException('Floe BATCH frame is truncated, its directory is incomplete'); + } + + /** @var list $words */ + $words = ($nodeCount + $bufferCount) === 0 + ? [] + : array_values((array) unpack('V' . (2 * ($nodeCount + $bufferCount)), $body, 12)); + $area = substr($body, $areaStart); + $areaLength = strlen($area); + $node = 0; + $extent = 2 * $nodeCount; + $buffer = 0; + $columns = []; + + foreach ($schema->definitions() as $name => $definition) { + $type = $definition->type(); + $length = $words[2 * $node]; + $nullCount = $words[(2 * $node) + 1]; + + if ($length !== $rowCount) { + throw new FloeException(sprintf( + 'Floe BATCH frame column "%s" holds %d rows, the frame %d', + $name, + $length, + $rowCount, + )); + } + + if ($nullCount > $length) { + throw new FloeException(sprintf( + 'Floe BATCH frame column "%s" has %d nulls in %d rows', + $name, + $nullCount, + $length, + )); + } + + $buffers = []; + + for ($i = $bufferCounts[$name]; $i > 0; $i--, $buffer++, $extent += 2) { + $offset = $words[$extent]; + $stored = $words[$extent + 1]; + + if ($stored === 0) { + $buffers[] = ''; + + continue; + } + + if (($offset + $stored) > $areaLength) { + throw new FloeException(sprintf( + 'Floe BATCH frame buffer %d lies outside the %d-byte buffer area', + $buffer, + $areaLength, + )); + } + + if ($stored < 8) { + throw new FloeException(sprintf( + 'Floe BATCH frame buffer %d is shorter than its 8-byte length prefix', + $buffer, + )); + } + + /** @var array{1: int} $prefix */ + $prefix = unpack('P', $area, $offset); + $bytes = substr($area, $offset + 8, $stored - 8); + + if ($prefix[1] === -1) { + $buffers[] = $bytes; + + continue; + } + + $decoded = $this->codec->decode($bytes); + + if (strlen($decoded) !== $prefix[1]) { + throw new FloeException(sprintf( + 'Floe BATCH frame buffer %d decodes to %d bytes, its prefix declares %d', + $buffer, + strlen($decoded), + $prefix[1], + )); + } + + $buffers[] = $decoded; + } + + try { + // a flat column's only node is its top node, already checked above; the backend checks its buffers + if ($nodeCounts[$name] === 1) { + $node++; + $columns[$name] = $backend->decode($definition, $buffers, $rowCount, $nullCount); + + continue; + } + + foreach ($this->layout->nodes($type, $length, $nullCount, $buffers) as [ + $derivedLength, + $derivedNulls, + ]) { + if ($words[2 * $node] !== $derivedLength || $words[(2 * $node) + 1] !== $derivedNulls) { + throw new FloeException(sprintf( + 'Floe BATCH frame column "%s" declares nodes that disagree with its buffers', + $name, + )); + } + + $node++; + } + + $columns[$name] = $backend->decode($definition, $buffers, $rowCount, $nullCount); + } catch (InvalidArgumentException $e) { + throw new FloeException( + sprintf('Floe BATCH frame column "%s" is malformed: %s', $name, $e->getMessage()), + 0, + $e, + ); + } + } + + try { + return Rows::fromColumns($schema, $columns, $rowCount); + } catch (SchemaMismatchException $e) { + throw new FloeException(sprintf('Floe BATCH frame is malformed: %s', $e->getMessage()), 0, $e); + } + } +} diff --git a/src/core/etl/src/Flow/Floe/FrameEncoder.php b/src/core/etl/src/Flow/Floe/FrameEncoder.php new file mode 100644 index 0000000000..d9c0415c0f --- /dev/null +++ b/src/core/etl/src/Flow/Floe/FrameEncoder.php @@ -0,0 +1,80 @@ +schema()->definitions() as $definition) { + $column = $rows->column($definition->entry()->name()); + $buffers = $column->encode(); + + $columnNodes = $this->layout->nodeCount($definition->type()) === 1 + ? [[$rows->count(), $column->nullCount()]] + : $this->layout->nodes($definition->type(), $rows->count(), $column->nullCount(), $buffers); + + foreach ($columnNodes as [$length, $nulls]) { + $nodes .= pack('VV', $length, $nulls); + $nodeCount++; + } + + foreach ($buffers as $buffer) { + $bufferCount++; + + if ($buffer === '') { + $extents .= pack('VV', 0, 0); + + continue; + } + + $encoded = $this->codec->encode($buffer); + $stored = strlen($encoded) < strlen($buffer) + ? pack('P', strlen($buffer)) . $encoded + : pack('P', -1) . $buffer; + $extents .= pack('VV', strlen($area), strlen($stored)); + $area .= $stored . str_repeat("\0", (8 - (strlen($stored) % 8)) % 8); + } + } + + $directory = pack('VVV', $rows->count(), $nodeCount, $bufferCount) . $nodes . $extents; + $body = $directory . str_repeat("\0", (8 - (strlen($directory) % 8)) % 8) . $area; + + if (strlen($body) > 0xFFFFFFFF) { + throw new FrameTooLarge(sprintf( + 'Floe BATCH frame body of %d bytes exceeds the 4 GiB frame length', + strlen($body), + )); + } + + return $body; + } +} diff --git a/src/core/etl/src/Flow/Floe/FrameWriter.php b/src/core/etl/src/Flow/Floe/FrameWriter.php index b47824cc54..ae21d7ce2a 100644 --- a/src/core/etl/src/Flow/Floe/FrameWriter.php +++ b/src/core/etl/src/Flow/Floe/FrameWriter.php @@ -6,8 +6,6 @@ use Flow\Filesystem\DestinationStream; -use function chr; -use function pack; use function strlen; final class FrameWriter @@ -31,9 +29,9 @@ public function header(): void $this->position += Format::HEADER_LENGTH; } - public function row(string $body): void + public function frame(int $type, string $body): void { - $this->buffer .= chr(Format::FRAME_ROW) . pack('V', strlen($body)) . $body; + $this->buffer .= Format::frame($type, $body); $this->position += Format::FRAME_HEADER_LENGTH + strlen($body); $this->flushIfFull(); } diff --git a/src/core/etl/src/Flow/Floe/NativeFloeEncoder.php b/src/core/etl/src/Flow/Floe/NativeFloeEncoder.php deleted file mode 100644 index 2de0bd41da..0000000000 --- a/src/core/etl/src/Flow/Floe/NativeFloeEncoder.php +++ /dev/null @@ -1,117 +0,0 @@ -native = new RustFloeEncoderNative(); - } - - public static function isSupported(): bool - { - // an extension older than this library lacks decodeRows()/encodeFrames()/datetimeZones() - it falls back to the PHP engine instead - return ( - extension_loaded('flow_php') - && class_exists(RustFloeEncoderNative::class, false) - && method_exists(RustFloeEncoderNative::class, 'decodeRows') - && method_exists(RustFloeEncoderNative::class, 'encodeFrames') - && method_exists(RustFloeEncoderNative::class, 'datetimeZones') - ); - } - - public function decode(array $batch): array - { - try { - return $this->native->decode($batch, $this->schemaBody()); - } catch (ExtensionException $e) { - throw new FloeException($e->getMessage(), 0, $e); - } - } - - public function decodeRows(array $bodies, Schema $schema, Hydrator $hydrator): Rows - { - if (!self::isNativeHydrator($hydrator)) { - return $hydrator->hydrate($this->decode($bodies), $schema); - } - - try { - return $this->native->decodeRows($bodies, $this->schemaBody(), $schema); - } catch (ExtensionException $e) { - throw new FloeException($e->getMessage(), 0, $e); - } - } - - public function encodeFrames(Rows $rows, Hydrator $hydrator): string - { - if (!self::isNativeHydrator($hydrator)) { - return Format::rowFrames($this->encode($hydrator->dehydrate($rows))); - } - - try { - return $this->native->encodeFrames($rows, $this->schemaBody(), $this->schema); - } catch (ExtensionException $e) { - throw new FloeException($e->getMessage(), 0, $e); - } - } - - public function encode(array $batch): array - { - try { - return $this->native->encode($batch, $this->schemaBody(), $this->schema); - } catch (ExtensionException $e) { - throw new FloeException($e->getMessage(), 0, $e); - } - } - - private static function isNativeHydrator(Hydrator $hydrator): bool - { - return ( - $hydrator instanceof NativeRowHydrator - || $hydrator instanceof AdaptiveRowHydrator - && $hydrator->isNative() - ); - } - - private function schemaBody(): string - { - if ($this->schemaBody !== null) { - return $this->schemaBody; - } - - try { - return $this->schemaBody = json_encode($this->schema->normalize(), JSON_THROW_ON_ERROR); - } catch (JsonException $e) { - throw new FloeException('Floe failed to encode schema as JSON: ' . $e->getMessage(), 0, $e); - } - } -} diff --git a/src/core/etl/src/Flow/Floe/PhpFloeEncoder.php b/src/core/etl/src/Flow/Floe/PhpFloeEncoder.php deleted file mode 100644 index 94bf44aed0..0000000000 --- a/src/core/etl/src/Flow/Floe/PhpFloeEncoder.php +++ /dev/null @@ -1,176 +0,0 @@ - - */ - private ?array $decodePlan = null; - - /** - * @var null|array per-column value encoder, keyed by column name - */ - private ?array $encoders = null; - - private ?string $schemaBody = null; - - private readonly SchemaDecoder $schemaDecoder; - - public function __construct( - private readonly Schema $schema, - ) { - $this->schemaDecoder = new SchemaDecoder(new ValueDecoder()); - } - - public function decode(array $batch): array - { - $decodePlan = $this->decodePlan ??= $this->schemaDecoder->decode($this->schemaBody()); - - $decoded = []; - - foreach ($batch as $body) { - $position = 0; - $values = []; - $metadata = []; - - foreach ($decodePlan as $column) { - $flag = ord($body[$position++]); - - if ($flag === Format::VALUE_PRESENT) { - $values[$column->name] = $column->decoder->decode($body, $position); - } elseif ($flag === Format::VALUE_NULL) { - $values[$column->name] = null; - } elseif ($flag === Format::VALUE_PRESENT_WITH_META) { - $metadata[$column->name] = Format::readMetadata($body, $position); - $values[$column->name] = $column->decoder->decode($body, $position); - } elseif ($flag === Format::VALUE_NULL_WITH_META) { - $metadata[$column->name] = Format::readMetadata($body, $position); - $values[$column->name] = null; - } else { - throw new FloeException(sprintf('Floe found unknown value flag 0x%02X', $flag)); - } - } - - if ($position !== strlen($body)) { - throw new FloeException('Floe row frame length does not match its content'); - } - - $decoded[] = new RawRowValues($values, $metadata); - } - - return $decoded; - } - - public function decodeRows(array $bodies, Schema $schema, Hydrator $hydrator): Rows - { - return $hydrator->hydrate($this->decode($bodies), $schema); - } - - public function encodeFrames(Rows $rows, Hydrator $hydrator): string - { - return Format::rowFrames($this->encode($hydrator->dehydrate($rows))); - } - - public function encode(array $batch): array - { - $encoders = $this->encoders ??= $this->buildEncoders(); - - $bodies = []; - - foreach ($batch as $rowIndex => $rowValues) { - $body = ''; - - foreach ($this->schema->definitions() as $name => $definition) { - if (!array_key_exists($name, $rowValues->values)) { - throw new FloeException(sprintf( - 'Floe found a row that does not carry the declared column "%s"', - $name, - )); - } - - // @mago-ignore analysis:mixed-assignment - $value = $rowValues->values[$name]; - $metadata = $rowValues->metadata[$name] ?? Metadata::empty(); - $columnMetadata = $definition->metadata(); - $diverges = $metadata->isEmpty() && $columnMetadata->isEmpty() - ? false - : !$metadata->isEqual($columnMetadata); - - if ($value === null) { - if (!$definition->isNullable()) { - throw new SchemaMismatchException($rowIndex, ColumnMismatchException::valueDoesNotMatch( - $definition, - null, - )); - } - - $body .= $diverges - ? Format::VALUE_NULL_WITH_META_BYTE . Format::metadataBytes($metadata) - : Format::VALUE_NULL_BYTE; - - continue; - } - - $body .= $diverges - ? Format::VALUE_PRESENT_WITH_META_BYTE - . Format::metadataBytes($metadata) - . $encoders[$name]->encode($value) - : Format::VALUE_PRESENT_BYTE . $encoders[$name]->encode($value); - } - - $bodies[] = $body; - } - - return $bodies; - } - - /** - * @return array - */ - private function buildEncoders(): array - { - $valueEncoder = new ValueEncoder(); - $encoders = []; - - foreach ($this->schema->definitions() as $name => $definition) { - $encoders[$name] = $valueEncoder->encoderFor($definition); - } - - return $encoders; - } - - private function schemaBody(): string - { - if ($this->schemaBody !== null) { - return $this->schemaBody; - } - - try { - return $this->schemaBody = json_encode($this->schema->normalize(), JSON_THROW_ON_ERROR); - } catch (JsonException $e) { - throw new FloeException('Floe failed to encode schema as JSON: ' . $e->getMessage(), 0, $e); - } - } -} diff --git a/src/core/etl/src/Flow/Floe/SchemaDecoder.php b/src/core/etl/src/Flow/Floe/SchemaDecoder.php deleted file mode 100644 index bc36b4b69b..0000000000 --- a/src/core/etl/src/Flow/Floe/SchemaDecoder.php +++ /dev/null @@ -1,66 +0,0 @@ - - */ - public function decode(string $schemaJson): array - { - try { - // @mago-ignore analysis:mixed-assignment - $definitions = json_decode($schemaJson, true, 512, JSON_THROW_ON_ERROR); - } catch (JsonException $e) { - throw new FloeException('Floe failed to decode schema JSON: ' . $e->getMessage(), 0, $e); - } - - try { - $definitions = type_list(type_array())->assert($definitions); - } catch (InvalidTypeException $e) { - throw new FloeException( - 'Floe expected schema JSON to be a list of definitions: ' . $e->getMessage(), - 0, - $e, - ); - } - - $plan = []; - - /** @var array{ref: string, type: array, nullable?: bool, metadata?: array|bool|float|int|string>} $normalized */ - foreach ($definitions as $normalized) { - $definition = definition_from_array([ - 'ref' => $normalized['ref'], - 'type' => $normalized['type'], - 'nullable' => $normalized['nullable'] ?? false, - 'metadata' => $normalized['metadata'] ?? [], - ]); - - $plan[] = new ColumnBlueprint( - $normalized['ref'], - $definition, - $this->valueDecoder->decoderFor($definition), - ); - } - - return $plan; - } -} diff --git a/src/core/etl/src/Flow/Floe/ValueDecoder.php b/src/core/etl/src/Flow/Floe/ValueDecoder.php deleted file mode 100644 index ba3deecb9f..0000000000 --- a/src/core/etl/src/Flow/Floe/ValueDecoder.php +++ /dev/null @@ -1,295 +0,0 @@ -timeZones = new TimeZones(); - $this->dateTimeDecoder = new DateTimeDecoder($this->timeZones); - $this->uuidDecoder = new UuidDecoder(); - $this->jsonDecoder = new JsonDecoder(); - } - - /** - * @throws FloeException - */ - public function decoderFor(Definition $definition): Decoding\ValueDecoder - { - return match ($definition::class) { - IntegerDefinition::class, - FloatDefinition::class, - BooleanDefinition::class, - StringDefinition::class, - DateTimeDefinition::class, - DateDefinition::class, - TimeDefinition::class, - UuidDefinition::class, - TimeZoneDefinition::class, - JsonDefinition::class, - EnumDefinition::class, - XMLDefinition::class, - XMLElementDefinition::class, - HTMLDefinition::class, - HTMLElementDefinition::class, - ListDefinition::class, - MapDefinition::class, - StructureDefinition::class, - NullDefinition::class, - => $this->elementDecoderFor($definition->type()), - default => throw new FloeException(sprintf( - 'Floe does not support columns of type "%s"', - $definition->type()->toString(), - )), - }; - } - - /** - * Types reachable only inside containers - list elements, map keys and values, - * structure elements - where no Definition exists. - * - * @param Type $type - * - * @throws FloeException - */ - private function elementDecoderFor(Type $type): Decoding\ValueDecoder - { - return match ($type::class) { - IntegerType::class, PositiveIntegerType::class => new Int64Decoder(), - FloatType::class => new Float64Decoder(), - BooleanType::class => new BooleanDecoder(), - StringType::class, - NonEmptyStringType::class, - NumericStringType::class, - ClassStringType::class, - => new StringDecoder(), - TimeZoneType::class => new TimeZoneDecoder($this->timeZones), - DateTimeType::class => $this->zonedDateTimeDecoder($type), - DateType::class => $this->dateTimeDecoder, - TimeType::class => new IntervalDecoder(), - UuidType::class => $this->uuidDecoder, - JsonType::class => $this->jsonDecoder, - EnumType::class => new EnumDecoder(), - XMLType::class => new XmlDocumentDecoder(), - XMLElementType::class => new XmlElementDecoder(), - HTMLType::class => new HtmlDocumentDecoder(), - HTMLElementType::class => new HtmlElementDecoder(), - ListType::class => $this->listDecoder($type), - MapType::class => $this->mapDecoder($type), - StructureType::class => $this->structureDecoder($type), - OptionalType::class => $this->optionalDecoder($type), - NullType::class => new NullDecoder(), - default => throw new FloeException(sprintf('Floe does not support values of type "%s"', $type->toString())), - }; - } - - public static function htmlDocumentFromString(string $html): object - { - if (!class_exists('\Dom\HTMLDocument')) { - throw new FloeException('Floe cannot restore HTML values, \Dom\HTMLDocument requires PHP 8.4+'); - } - - // @mago-expect analysis:unavailable-method - return HTMLDocument::createFromString($html, LIBXML_NOERROR); - } - - public static function htmlElementFromString(string $html): object - { - if (!class_exists('\Dom\HTMLDocument')) { - throw new FloeException('Floe cannot restore HTML values, \Dom\HTMLDocument requires PHP 8.4+'); - } - - // @mago-expect analysis:unavailable-method - $element = HTMLDocument::createFromString( - '' . $html . '', - LIBXML_NOERROR, - )->body?->firstElementChild; - - if ($element === null) { - throw new FloeException(sprintf('Floe failed to restore HTMLElement from "%s"', $html)); - } - - return $element; - } - - public static function xmlDocumentFromString(string $xml): DOMDocument - { - $document = new DOMDocument(); - - if (!@$document->loadXML($xml)) { - throw new FloeException(sprintf('Floe failed to restore DOMDocument from "%s"', $xml)); - } - - return $document; - } - - public static function xmlElementFromString(string $xml): DOMElement - { - $element = self::xmlDocumentFromString($xml)->documentElement; - - if ($element === null) { - throw new FloeException(sprintf('Floe failed to restore DOMElement from "%s"', $xml)); - } - - return $element; - } - - /** - * @param Type $type - */ - private function listDecoder(Type $type): Decoding\ValueDecoder - { - /** @var ListType> $type */ - $element = $type->element(); - - if ($element instanceof IntegerType) { - return new PackedListDecoder('P'); - } - - if ($element instanceof FloatType) { - return new PackedListDecoder('e'); - } - - return new ListDecoder($this->elementDecoderFor($element)); - } - - /** - * @param Type $type - */ - private function mapDecoder(Type $type): Decoding\ValueDecoder - { - /** @var MapType> $type */ - $key = $type->key(); - - $keyDecoder = match (true) { - $key instanceof IntegerType => new Int64Decoder(), - $key instanceof StringType => new StringDecoder(), - default => throw new FloeException(sprintf( - 'Floe does not support map keys of type "%s"', - $key->toString(), - )), - }; - - return new MapDecoder($keyDecoder, $this->elementDecoderFor($type->value())); - } - - /** - * @param Type $type - */ - private function optionalDecoder(Type $type): Decoding\ValueDecoder - { - /** @var OptionalType $type */ - return new OptionalDecoder($this->elementDecoderFor($type->base())); - } - - /** - * @param Type $type - */ - private function structureDecoder(Type $type): Decoding\ValueDecoder - { - /** @var StructureType> $type */ - $elements = []; - - foreach ($type->elements() as $element) { - $elements[$element->name] = $this->elementDecoderFor($element->type); - } - - return new StructureDecoder($elements); - } - - /** - * @param Type $type - */ - private function zonedDateTimeDecoder(Type $type): DateTimeDecoder - { - return new DateTimeDecoder($this->timeZones, type_instance_of(DateTimeType::class)->assert($type)->zone()); - } -} diff --git a/src/core/etl/src/Flow/Floe/ValueEncoder.php b/src/core/etl/src/Flow/Floe/ValueEncoder.php deleted file mode 100644 index 60841ea2f0..0000000000 --- a/src/core/etl/src/Flow/Floe/ValueEncoder.php +++ /dev/null @@ -1,265 +0,0 @@ -dateTimeEncoder = new DateTimeEncoder(); - $this->timeZoneEncoder = new TimeZoneEncoder(); - $this->uuidEncoder = new UuidEncoder(); - $this->jsonEncoder = new JsonEncoder(); - } - - /** - * @throws FloeException - */ - public function encoderFor(Definition $definition): Encoding\ValueEncoder - { - return match ($definition::class) { - IntegerDefinition::class, - FloatDefinition::class, - BooleanDefinition::class, - StringDefinition::class, - DateTimeDefinition::class, - DateDefinition::class, - TimeDefinition::class, - UuidDefinition::class, - TimeZoneDefinition::class, - JsonDefinition::class, - EnumDefinition::class, - XMLDefinition::class, - XMLElementDefinition::class, - HTMLDefinition::class, - HTMLElementDefinition::class, - ListDefinition::class, - MapDefinition::class, - StructureDefinition::class, - NullDefinition::class, - => $this->elementEncoderFor($definition->type()), - default => throw new FloeException(sprintf( - 'Floe does not support columns of type "%s"', - $definition->type()->toString(), - )), - }; - } - - /** - * Types reachable only inside containers - list elements, map keys and values, - * structure elements - where no Definition exists. - * - * @param Type $type - * - * @throws FloeException - */ - private function elementEncoderFor(Type $type): Encoding\ValueEncoder - { - return match ($type::class) { - IntegerType::class, PositiveIntegerType::class => new Int64Encoder(), - FloatType::class => new Float64Encoder(), - BooleanType::class => new BooleanEncoder(), - StringType::class, - NonEmptyStringType::class, - NumericStringType::class, - ClassStringType::class, - => new StringEncoder(), - TimeZoneType::class => $this->timeZoneEncoder, - DateTimeType::class, DateType::class => $this->dateTimeEncoder, - TimeType::class => new IntervalEncoder(), - UuidType::class => $this->uuidEncoder, - JsonType::class => $this->jsonEncoder, - EnumType::class => new EnumEncoder(), - XMLType::class => new XmlDocumentEncoder(), - XMLElementType::class => new XmlElementEncoder(), - HTMLType::class => new HtmlDocumentEncoder(), - HTMLElementType::class => new HtmlElementEncoder(), - ListType::class => $this->listEncoder($type), - MapType::class => $this->mapEncoder($type), - StructureType::class => $this->structureEncoder($type), - OptionalType::class => new OptionalEncoder($this->elementEncoderFor($type->base())), - NullType::class => new NullEncoder(), - default => throw new FloeException(sprintf('Floe does not support values of type "%s"', $type->toString())), - }; - } - - public static function lengthPrefixed(string $value): string - { - return pack('V', strlen($value)) . $value; - } - - public static function htmlElementToString(object $value): string - { - /** @var \Dom\HTMLElement $value */ - // @mago-ignore analysis:non-existent-method - // @mago-ignore analysis:mixed-assignment - $html = $value->ownerDocument?->saveHtml($value); - - if (!is_string($html)) { - throw new FloeException('Floe failed to convert HTMLElement to HTML string'); - } - - return $html; - } - - public static function xmlDocumentToString(DOMDocument $value): string - { - $xml = $value->saveXML(); - - if ($xml === false) { - throw new FloeException('Floe failed to convert DOMDocument to XML string'); - } - - return $xml; - } - - public static function xmlElementToString(DOMElement $value): string - { - $xml = $value->ownerDocument?->saveXML($value); - - if ($xml === null || $xml === false) { - throw new FloeException('Floe failed to convert DOMElement to XML string'); - } - - return $xml; - } - - /** - * @param Type $type - */ - private function listEncoder(Type $type): Encoding\ValueEncoder - { - /** @var ListType> $type */ - $element = $type->element(); - - if ($element instanceof IntegerType) { - return new PackedListEncoder('P'); - } - - if ($element instanceof FloatType) { - return new PackedListEncoder('e'); - } - - return new ListEncoder($this->elementEncoderFor($element)); - } - - /** - * @param Type $type - */ - private function mapEncoder(Type $type): Encoding\ValueEncoder - { - /** @var MapType> $type */ - $key = $type->key(); - - $keyEncoder = match (true) { - $key instanceof IntegerType => new Int64Encoder(), - $key instanceof StringType => new StringKeyEncoder(), - default => throw new FloeException(sprintf( - 'Floe does not support map keys of type "%s"', - $key->toString(), - )), - }; - - return new MapEncoder($keyEncoder, $this->elementEncoderFor($type->value())); - } - - /** - * @param Type $type - */ - private function structureEncoder(Type $type): Encoding\ValueEncoder - { - /** @var StructureType> $type */ - $elements = []; - - foreach ($type->elements() as $element) { - $elements[$element->name] = $this->elementEncoderFor($element->type); - } - - return new StructureEncoder($elements); - } -} diff --git a/src/core/etl/tests/Flow/Calculator/Tests/Unit/RunningSumTest.php b/src/core/etl/tests/Flow/Calculator/Tests/Unit/RunningSumTest.php index 4264103144..0f02883651 100644 --- a/src/core/etl/tests/Flow/Calculator/Tests/Unit/RunningSumTest.php +++ b/src/core/etl/tests/Flow/Calculator/Tests/Unit/RunningSumTest.php @@ -8,23 +8,34 @@ use PHPUnit\Framework\Attributes\TestWith; use PHPUnit\Framework\TestCase; +use function is_nan; use function is_numeric; +use const INF; + final class RunningSumTest extends TestCase { #[TestWith([0, 1, 1])] #[TestWith([10, 5, 15])] - public function test_int_operands_stay_int(int $sum, int $value, int $output): void + public function test_int_operands_stay_int(int $first, int $second, int $output): void { - static::assertSame($output, (new RunningSum())->add($sum, $value, false)); + $sum = new RunningSum(); + $sum->add($first, false); + $sum->add($second, false); + + static::assertSame($output, $sum->value()); } #[TestWith([1.5, 1.5, 3.0])] #[TestWith([0, 2.5, 2.5])] #[TestWith([2.5, 0, 2.5])] - public function test_float_operand_yields_float(float|int $sum, float|int $value, float $output): void + public function test_float_operand_yields_float(float|int $first, float|int $second, float $output): void { - static::assertSame($output, (new RunningSum())->add($sum, $value, false)); + $sum = new RunningSum(); + $sum->add($first, false); + $sum->add($second, false); + + static::assertSame($output, $sum->value()); } #[TestWith(['10', 10.0])] @@ -33,21 +44,87 @@ public function test_numeric_string_operand_yields_float(string $value, float $o { assert(is_numeric($value), 'String parameter $value must be numeric'); - static::assertSame($output, (new RunningSum())->add(0, $value, false)); + $sum = new RunningSum(); + $sum->add($value, false); + + static::assertSame($output, $sum->value()); } - public function test_exact_mode_avoids_floating_point_drift(): void + public function test_compensation_keeps_ten_tenths_at_one(): void { - static::assertSame(0.3, (new RunningSum())->add(0.1, 0.2, true)); + $sum = new RunningSum(); + + for ($i = 0; $i < 10; $i++) { + $sum->add(0.1, false); + } + + // naive IEEE addition gives 0.9999999999999999 + static::assertSame(1.0, $sum->value()); } - public function test_inexact_mode_keeps_native_addition(): void + public function test_compensation_keeps_a_small_value_next_to_large_ones(): void { - static::assertSame(0.1 + 0.2, (new RunningSum())->add(0.1, 0.2, false)); + $sum = new RunningSum(); + $sum->add(1e16, false); + $sum->add(1.0, false); + $sum->add(-1e16, false); + + // naive IEEE addition gives 0.0 + static::assertSame(1.0, $sum->value()); + } + + public function test_an_infinite_sum_stays_infinite(): void + { + $sum = new RunningSum(); + $sum->add(INF, false); + $sum->add(1.5, false); + + static::assertSame(INF, $sum->value()); + static::assertFalse(is_nan($sum->value())); + } + + public function test_exact_mode_avoids_floating_point_drift(): void + { + $sum = new RunningSum(); + $sum->add(0.1, true); + $sum->add(0.2, true); + + static::assertSame(0.3, $sum->value()); } public function test_exact_mode_keeps_int_operands_int(): void { - static::assertSame(3, (new RunningSum())->add(1, 2, true)); + $sum = new RunningSum(); + $sum->add(1, true); + $sum->add(2, true); + + static::assertSame(3, $sum->value()); + } + + public function test_merge_carries_the_other_compensation(): void + { + $left = new RunningSum(); + $right = new RunningSum(); + + for ($i = 0; $i < 5; $i++) { + $left->add(0.1, false); + $right->add(0.1, false); + } + + $left->merge($right, false); + + static::assertSame(1.0, $left->value()); + } + + public function test_exact_merge_is_decimal(): void + { + $left = new RunningSum(); + $right = new RunningSum(); + $left->add(0.1, false); + $right->add(0.2, false); + + $left->merge($right, true); + + static::assertSame(0.3, $left->value()); } } diff --git a/src/core/etl/tests/Flow/ETL/Tests/Comparator/ColumnComparator.php b/src/core/etl/tests/Flow/ETL/Tests/Comparator/ColumnComparator.php new file mode 100644 index 0000000000..bae3aff649 --- /dev/null +++ b/src/core/etl/tests/Flow/ETL/Tests/Comparator/ColumnComparator.php @@ -0,0 +1,63 @@ +type()->toString() !== $actual->type()->toString()) { + throw new ComparisonFailure( + $expected, + $actual, + $expected->type()->toString(), + $actual->type()->toString(), + 'Failed asserting that two columns have the same type.', + ); + } + + $expectedPhysicals = $expected->physicals(); + $actualPhysicals = $actual->physicals(); + + try { + $this + ->factory() + ->getComparatorFor($expectedPhysicals, $actualPhysicals) + ->assertEquals($expectedPhysicals, $actualPhysicals, $delta, $canonicalize, $ignoreCase); + } catch (ComparisonFailure) { + throw new ComparisonFailure( + $expected, + $actual, + $exporter->export($expectedPhysicals), + $exporter->export($actualPhysicals), + 'Failed asserting that two columns are equal.', + ); + } + } +} diff --git a/src/core/etl/tests/Flow/ETL/Tests/Comparator/ComparatorExtension.php b/src/core/etl/tests/Flow/ETL/Tests/Comparator/ComparatorExtension.php new file mode 100644 index 0000000000..2c7e88d599 --- /dev/null +++ b/src/core/etl/tests/Flow/ETL/Tests/Comparator/ComparatorExtension.php @@ -0,0 +1,20 @@ +register(new RowsComparator()); + Factory::getInstance()->register(new ColumnComparator()); + } +} diff --git a/src/core/etl/tests/Flow/ETL/Tests/Comparator/RowsComparator.php b/src/core/etl/tests/Flow/ETL/Tests/Comparator/RowsComparator.php new file mode 100644 index 0000000000..717780e149 --- /dev/null +++ b/src/core/etl/tests/Flow/ETL/Tests/Comparator/RowsComparator.php @@ -0,0 +1,60 @@ +schema()->isSame($actual->schema())) { + throw new ComparisonFailure( + $expected, + $actual, + $exporter->export($expected->schema()->normalize()), + $exporter->export($actual->schema()->normalize()), + 'Failed asserting that two Rows have the same schema.', + ); + } + + $expectedArray = $expected->toArray(); + $actualArray = $actual->toArray(); + + try { + $this + ->factory() + ->getComparatorFor($expectedArray, $actualArray) + ->assertEquals($expectedArray, $actualArray, $delta, $canonicalize, $ignoreCase); + } catch (ComparisonFailure) { + throw new ComparisonFailure( + $expected, + $actual, + $exporter->export($expectedArray), + $exporter->export($actualArray), + 'Failed asserting that two Rows are equal.', + ); + } + } +} diff --git a/src/core/etl/tests/Flow/ETL/Tests/Context/BucketsStorageContext.php b/src/core/etl/tests/Flow/ETL/Tests/Context/BucketsStorageContext.php index 4edc301b79..ae08ac871d 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Context/BucketsStorageContext.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Context/BucketsStorageContext.php @@ -4,7 +4,6 @@ namespace Flow\ETL\Tests\Context; -use Flow\ETL\Row; use Flow\ETL\Rows; use Generator; @@ -15,14 +14,14 @@ final class BucketsStorageContext * * @param Generator $batches * - * @return list + * @return list> */ public static function rows(Generator $batches): array { $rows = []; foreach ($batches as $batch) { - foreach ($batch as $row) { + foreach ($batch->toArray() as $row) { $rows[] = $row; } } diff --git a/src/core/etl/tests/Flow/ETL/Tests/Context/ExtractedRows.php b/src/core/etl/tests/Flow/ETL/Tests/Context/ExtractedRows.php index 3b3fc220d6..836b22c590 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Context/ExtractedRows.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Context/ExtractedRows.php @@ -26,12 +26,39 @@ public static function of( ?int $limit = null, Filter $pathFilter = new OnlyFiles(), ): Rows { - $extracted = rows(schema()); + $context ??= flow_context(); + $extracted = null; - foreach (FlowTestCase::extracted($extractor, $context ?? flow_context(), $limit, $pathFilter) as $batch) { - $extracted = $extracted->merge($batch); + foreach (FlowTestCase::extracted($extractor, $context, $limit, $pathFilter) as $batch) { + $extracted = $extracted === null ? $batch : $extracted->concat($context->backend(), $batch); } - return $extracted; + return $extracted ?? rows(schema()); + } + + /** + * Two reads of one extractor, their batches pulled in turn - each read's rows. + * + * @return array{Rows, Rows} + */ + public static function interleaved(Extractor $extractor, ?FlowContext $context = null): array + { + $context ??= flow_context(); + $reads = [$extractor->extract($context), $extractor->extract($context)]; + /** @var array{0: null|Rows, 1: null|Rows} $extracted */ + $extracted = [null, null]; + + while ($reads[0]->valid() || $reads[1]->valid()) { + foreach ($reads as $i => $read) { + if ($read->valid()) { + $extracted[$i] = $extracted[$i] === null + ? $read->current() + : $extracted[$i]->concat($context->backend(), $read->current()); + $read->next(); + } + } + } + + return [$extracted[0] ?? rows(schema()), $extracted[1] ?? rows(schema())]; } } diff --git a/src/core/etl/tests/Flow/ETL/Tests/Context/FileColumnsContext.php b/src/core/etl/tests/Flow/ETL/Tests/Context/FileColumnsContext.php index 084e4d7fac..d752ef1aaf 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Context/FileColumnsContext.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Context/FileColumnsContext.php @@ -4,9 +4,9 @@ namespace Flow\ETL\Tests\Context; -use Flow\ETL\Extractor\FileColumns; -use Flow\ETL\Extractor\PartitionColumns; -use Flow\ETL\Extractor\PartitionTypes; +use Flow\ETL\Extractor\File\FileColumns; +use Flow\ETL\Extractor\File\PartitionColumns; +use Flow\ETL\Extractor\File\PartitionTypes; use function Flow\Filesystem\DSL\memory_filesystem; diff --git a/src/core/etl/tests/Flow/ETL/Tests/Context/FunctionContext.php b/src/core/etl/tests/Flow/ETL/Tests/Context/FunctionContext.php new file mode 100644 index 0000000000..184c377f95 --- /dev/null +++ b/src/core/etl/tests/Flow/ETL/Tests/Context/FunctionContext.php @@ -0,0 +1,49 @@ +context = $context ?? flow_context(); + } + + /** + * @param list> $rows + */ + public function column(ScalarFunction $function, array $rows, Schema $schema): Column + { + return (new ReferenceResolver()) + ->resolve($function, $schema) + ->eval(array_to_rows($rows, $schema, $this->context->backend()), $this->context); + } + + /** + * Resolves $function against $schema (steps do the same), evaluates one row, returns its value as a parent + * function reads it. + * + * @param array $row + */ + public function eval(ScalarFunction $function, array $row, Schema $schema): mixed + { + return (new Parameter((new ReferenceResolver())->resolve($function, $schema)))->values( + array_to_rows([$row], $schema, $this->context->backend()), + $this->context, + )[0]; + } +} diff --git a/src/core/etl/tests/Flow/ETL/Tests/Context/GroupByContext.php b/src/core/etl/tests/Flow/ETL/Tests/Context/GroupByContext.php index 9551e7ac2d..0550c307a2 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Context/GroupByContext.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Context/GroupByContext.php @@ -4,16 +4,10 @@ namespace Flow\ETL\Tests\Context; -use Flow\ETL\Bucketing\Bucket; -use Flow\ETL\Bucketing\Buckets; -use Flow\ETL\Bucketing\HashBucketing; -use Flow\ETL\Bucketing\NativeHasher; use Flow\ETL\FlowContext; use Flow\ETL\GroupBy; use Flow\ETL\GroupBy\GroupBySteps; -use Flow\ETL\NativePHPRandomValueGenerator; use Flow\ETL\Processor; -use Flow\ETL\Row\Reference; use Flow\ETL\Rows; use Flow\ETL\Transformer; use Generator; @@ -49,7 +43,7 @@ public static function aggregated(GroupBy $groupBy, FlowContext $context, Rows . $result = null; foreach (self::aggregate($groupBy, $context, ...$batches) as $batch) { - $result = $result === null ? $batch : $result->merge($batch); + $result = $result === null ? $batch : $result->concat($context->backend(), $batch); } return $result ?? rows(schema()); @@ -63,28 +57,6 @@ public static function batches(Rows ...$batches): Generator yield from $batches; } - /** - * Spills $batches into $buckets the way BucketingProcessor does, returning one metadata Rows per - * bucket - the input GroupByAggregationProcessor::process() consumes. - * - * @param list $refs - * - * @return list - */ - public static function bucketMetadata(Buckets $buckets, array $refs, Rows ...$batches): array - { - $strategy = new HashBucketing($refs, 4, new NativeHasher(), new NativePHPRandomValueGenerator(), 'group-by'); - - $metadata = []; - - foreach ($strategy->bucketize(self::batches(...$batches), $buckets->storage()) as $bucket) { - $buckets->add($bucket); - $metadata[] = rows(Bucket::schema(), $bucket->toRow()); - } - - return $metadata; - } - /** * @param Generator $upstream * diff --git a/src/core/etl/tests/Flow/ETL/Tests/Context/GroupedRows.php b/src/core/etl/tests/Flow/ETL/Tests/Context/GroupedRows.php new file mode 100644 index 0000000000..62619a8af6 --- /dev/null +++ b/src/core/etl/tests/Flow/ETL/Tests/Context/GroupedRows.php @@ -0,0 +1,95 @@ + $batches + * + * @return array> output rows by the serialized value of $key, sorted + */ + public static function byKey(iterable $batches, string $key): array + { + $rows = []; + + foreach ($batches as $batch) { + foreach ($batch->toArray() as $row) { + $rows[serialize($row[$key])] = $row; + } + } + + ksort($rows); + + return $rows; + } + + /** + * @return Generator two batches: a 10, b 15, then a 20 + */ + public static function categories(): Generator + { + yield array_to_rows( + [['category' => 'a', 'amount' => 10], ['category' => 'b', 'amount' => 15]], + schema(str_schema('category'), int_schema('amount')), + ); + yield array_to_rows( + [['category' => 'a', 'amount' => 20]], + schema(str_schema('category'), int_schema('amount')), + ); + } + + /** + * @param iterable $batches + * + * @return list every batch serialized, sorted - the batches as a set, whatever order they came in + */ + public static function batchesAsSet(iterable $batches): array + { + $serialized = []; + + foreach ($batches as $batch) { + $serialized[] = serialize($batch->toArray()); + } + + sort($serialized); + + return $serialized; + } + + /** + * @param iterable $batches + * + * @return array amount_sum per category + */ + public static function sums(iterable $batches): array + { + $sums = []; + + foreach ($batches as $batch) { + foreach ($batch->toArray() as $row) { + $sums[$row['category']] = $row['amount_sum']; + } + } + + ksort($sums); + + return $sums; + } +} diff --git a/src/core/etl/tests/Flow/ETL/Tests/Context/IdRows.php b/src/core/etl/tests/Flow/ETL/Tests/Context/IdRows.php new file mode 100644 index 0000000000..328478f98f --- /dev/null +++ b/src/core/etl/tests/Flow/ETL/Tests/Context/IdRows.php @@ -0,0 +1,50 @@ + ...$batches + * + * @return Generator + */ + public static function batches(array ...$batches): Generator + { + foreach ($batches as $batch) { + yield array_to_rows(array_map(static fn(int $id): array => [ + 'id' => $id, + ], $batch), schema(int_schema('id'))); + } + } + + /** + * @param Generator $batches + * + * @return list + */ + public static function ids(Generator $batches): array + { + $ids = []; + + foreach ($batches as $batch) { + $ids = array_merge($ids, $batch->reduceToArray('id')); + } + + return $ids; + } +} diff --git a/src/core/etl/tests/Flow/ETL/Tests/Context/KeyGroupingContext.php b/src/core/etl/tests/Flow/ETL/Tests/Context/KeyGroupingContext.php index 7791908c1b..3e8a95080c 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Context/KeyGroupingContext.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Context/KeyGroupingContext.php @@ -7,6 +7,7 @@ use Flow\ETL\Bucketing\KeyGrouping; use Flow\ETL\Bucketing\KeyValues; use Flow\ETL\Bucketing\NativeHasher; +use Flow\ETL\Column\AdaptiveBackend; use Flow\ETL\Row\Reference; use Flow\ETL\Rows; use Generator; @@ -26,7 +27,7 @@ final class KeyGroupingContext public static function groups(array $by, Generator $input): array { return iterator_to_array( - (new KeyGrouping(new KeyValues($by), new NativeHasher()))->group($input), + (new KeyGrouping(new KeyValues($by), new NativeHasher(), new AdaptiveBackend()))->group($input), preserve_keys: false, ); } diff --git a/src/core/etl/tests/Flow/ETL/Tests/Context/MemoryTelemetryContext.php b/src/core/etl/tests/Flow/ETL/Tests/Context/MemoryTelemetryContext.php index b56ceb02e6..ca5ffc2a2a 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Context/MemoryTelemetryContext.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Context/MemoryTelemetryContext.php @@ -67,6 +67,7 @@ public function __construct( $this->config = $configBuilder->build(); $this->flowContext = flow_context($this->config); $this->telemetryContext = new TelemetryContext( + $this->config->backend(), $this->telemetry->logger('flow-php'), $this->telemetry->tracer('flow-php'), $this->telemetry->meter('flow-php'), diff --git a/src/core/etl/tests/Flow/ETL/Tests/Context/PartitionRoutingContext.php b/src/core/etl/tests/Flow/ETL/Tests/Context/PartitionRoutingContext.php index 7eda680892..7e1f925c66 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Context/PartitionRoutingContext.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Context/PartitionRoutingContext.php @@ -4,8 +4,8 @@ namespace Flow\ETL\Tests\Context; +use Flow\ETL\Loader\File\PartitionRouter; use Flow\ETL\Loader\Partitioning; -use Flow\ETL\Loader\PartitionRouter; use Flow\ETL\Rows; use Flow\Filesystem\Partitions; diff --git a/src/core/etl/tests/Flow/ETL/Tests/Context/RankingContext.php b/src/core/etl/tests/Flow/ETL/Tests/Context/RankingContext.php index f134ea4f4b..88f8f3b553 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Context/RankingContext.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Context/RankingContext.php @@ -6,9 +6,8 @@ use Flow\ETL\Rows; +use function Flow\ETL\DSL\array_to_rows; use function Flow\ETL\DSL\int_schema; -use function Flow\ETL\DSL\row; -use function Flow\ETL\DSL\rows; use function Flow\ETL\DSL\schema; /** @@ -22,25 +21,29 @@ final class RankingContext { public static function salariesAscending(): Rows { - return rows( + return array_to_rows( + [ + ['id' => 4, 'salary' => 2000], + ['id' => 5, 'salary' => 4000], + ['id' => 1, 'salary' => 6000], + ['id' => 2, 'salary' => 6000], + ['id' => 3, 'salary' => 6000], + ], schema(int_schema('id'), int_schema('salary')), - row(['id' => 4, 'salary' => 2000]), - row(['id' => 5, 'salary' => 4000]), - row(['id' => 1, 'salary' => 6000]), - row(['id' => 2, 'salary' => 6000]), - row(['id' => 3, 'salary' => 6000]), ); } public static function salariesDescending(): Rows { - return rows( + return array_to_rows( + [ + ['id' => 1, 'salary' => 6000], + ['id' => 2, 'salary' => 6000], + ['id' => 3, 'salary' => 6000], + ['id' => 5, 'salary' => 4000], + ['id' => 4, 'salary' => 2000], + ], schema(int_schema('id'), int_schema('salary')), - row(['id' => 1, 'salary' => 6000]), - row(['id' => 2, 'salary' => 6000]), - row(['id' => 3, 'salary' => 6000]), - row(['id' => 5, 'salary' => 4000]), - row(['id' => 4, 'salary' => 2000]), ); } } diff --git a/src/core/etl/tests/Flow/ETL/Tests/Context/RowsSerializationContext.php b/src/core/etl/tests/Flow/ETL/Tests/Context/RowsSerializationContext.php new file mode 100644 index 0000000000..fde8d461a2 --- /dev/null +++ b/src/core/etl/tests/Flow/ETL/Tests/Context/RowsSerializationContext.php @@ -0,0 +1,31 @@ +assert(unserialize(serialize($rows))); + } + + /** + * @param array{schema: \Flow\ETL\Schema, frame: string} $payload + */ + public static function unserialize(array $payload): Rows + { + $rows = (new ReflectionClass(Rows::class))->newInstanceWithoutConstructor(); + $rows->__unserialize($payload); + + return $rows; + } +} diff --git a/src/core/etl/tests/Flow/ETL/Tests/Context/ScalarFunctionFixtures.php b/src/core/etl/tests/Flow/ETL/Tests/Context/ScalarFunctionFixtures.php index 5d68ba484c..60cae32343 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Context/ScalarFunctionFixtures.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Context/ScalarFunctionFixtures.php @@ -4,7 +4,10 @@ namespace Flow\ETL\Tests\Context; +use DateTimeImmutable; +use DOMDocument; use Flow\ETL\Adapter\Excel\Function\IsValidExcelSheetName; +use Flow\ETL\Column\Backend; use Flow\ETL\Exception\RequiredPHPVersionException; use Flow\ETL\Exception\SchemaNotDerivableException; use Flow\ETL\Function; @@ -15,11 +18,13 @@ use Flow\ETL\Function\ScalarFunction; use Flow\ETL\Row\ResolvedReference; use Flow\ETL\Row\UnresolvedReference; +use Flow\ETL\Rows; use Flow\ETL\Schema; use Flow\ETL\String\StringStyles; use Flow\ETL\Tests\Fixtures\Enum\BackedStringEnum; use Flow\Types\Type; +use function Flow\ETL\DSL\array_to_rows; use function Flow\ETL\DSL\bool_schema; use function Flow\ETL\DSL\datetime_schema; use function Flow\ETL\DSL\enum_schema; @@ -133,6 +138,57 @@ enum_schema('enum', BackedStringEnum::class, true), ); } + /** + * Three rows under schema(): two of values, the last all null. `html` stays null - a document needs PHP 8.4. + */ + public static function rows(Backend $backend): Rows + { + $document = new DOMDocument(); + $document->loadXML('12'); + + return array_to_rows( + [ + [ + 'string' => 'a', + 'integer' => 1, + 'float' => 1.5, + 'boolean' => true, + 'datetime' => new DateTimeImmutable('2026-01-02 03:04:05 UTC'), + 'json' => '{"a":1}', + 'xml' => '1', + 'xml_element' => $document->getElementsByTagName('a')->item(0), + 'html' => null, + 'enum' => BackedStringEnum::one, + 'list' => ['a', 'b'], + 'list_of_lists' => [['a'], ['b']], + 'list_of_structs' => [['field' => 'x']], + 'map' => ['k' => 1], + 'structure' => ['field' => 1], + ], + [ + 'string' => 'b', + 'integer' => 2, + 'float' => -0.5, + 'boolean' => false, + 'datetime' => new DateTimeImmutable('2026-01-03 00:00:00 UTC'), + 'json' => '[1,2]', + 'xml' => '2', + 'xml_element' => $document->getElementsByTagName('b')->item(0), + 'html' => null, + 'enum' => BackedStringEnum::two, + 'list' => [], + 'list_of_lists' => [], + 'list_of_structs' => [], + 'map' => [], + 'structure' => ['field' => 2], + ], + [], + ], + self::schema(), + $backend, + ); + } + /** * @param class-string $class * @@ -263,7 +319,7 @@ public static function all(): array ], Function\CallUserFunc::class => [ 'factory' => static fn(): ScalarFunction => new Function\CallUserFunc( - lit('strtoupper'), + lit('strval'), type_string(), [ref('string')], ), diff --git a/src/core/etl/tests/Flow/ETL/Tests/Context/ScoredRows.php b/src/core/etl/tests/Flow/ETL/Tests/Context/ScoredRows.php index 5ec471442b..117cc04509 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Context/ScoredRows.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Context/ScoredRows.php @@ -8,9 +8,8 @@ use Generator; use function array_merge; +use function Flow\ETL\DSL\array_to_rows; use function Flow\ETL\DSL\int_schema; -use function Flow\ETL\DSL\row; -use function Flow\ETL\DSL\rows; use function Flow\ETL\DSL\schema; use function Flow\ETL\DSL\str_schema; @@ -30,10 +29,10 @@ public static function batches(array ...$batches): Generator $rows = []; foreach ($batch as [$score, $name]) { - $rows[] = row(['score' => $score, 'name' => $name]); + $rows[] = ['score' => $score, 'name' => $name]; } - yield rows(schema(int_schema('score'), str_schema('name')), ...$rows); + yield array_to_rows($rows, schema(int_schema('score'), str_schema('name'))); } } diff --git a/src/core/etl/tests/Flow/ETL/Tests/Context/SelfDescribingFilesContext.php b/src/core/etl/tests/Flow/ETL/Tests/Context/SelfDescribingFilesContext.php index ca0b384d7f..16e54ddf43 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Context/SelfDescribingFilesContext.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Context/SelfDescribingFilesContext.php @@ -4,8 +4,8 @@ namespace Flow\ETL\Tests\Context; -use Flow\ETL\Extractor\SelfDescribingFile; -use Flow\ETL\Extractor\SourceFile; +use Flow\ETL\Extractor\File\SelfDescribingFile; +use Flow\ETL\Extractor\File\SourceFile; use Flow\ETL\Schema; use Flow\ETL\Tests\Double\FakeSelfDescribingFile; use Generator; diff --git a/src/core/etl/tests/Flow/ETL/Tests/Context/TextWriterBatches.php b/src/core/etl/tests/Flow/ETL/Tests/Context/TextWriterBatches.php new file mode 100644 index 0000000000..9812349708 --- /dev/null +++ b/src/core/etl/tests/Flow/ETL/Tests/Context/TextWriterBatches.php @@ -0,0 +1,264 @@ + + */ + public static function names(): array + { + return [ + 'scalars with nulls', + 'floats', + 'not finite floats', + 'a not finite float inside a list', + 'strings that need quoting or escaping', + 'text that is not UTF-8', + 'temporal kinds', + 'a named zone across a transition', + 'uuid, enum and time zone', + 'a json column', + 'markup columns', + 'nested kinds rendered natively', + 'maps by key', + 'empty containers', + 'a list of json documents', + 'nested markup leaves', + 'awkward column names', + 'no rows', + ]; + } + + /** + * @return array{Schema, list>} + */ + public static function of(string $name): array + { + return match ($name) { + 'scalars with nulls' => [ + schema( + int_schema('id', nullable: true), + str_schema('name', nullable: true), + bool_schema('active', nullable: true), + null_schema('nothing'), + ), + [ + ['id' => 1, 'name' => 'Norbert', 'active' => true, 'nothing' => null], + ['id' => null, 'name' => null, 'active' => null, 'nothing' => null], + ['id' => -2, 'name' => '', 'active' => false, 'nothing' => null], + ], + ], + 'floats' => [ + schema(float_schema('f', nullable: true)), + [ + ['f' => 1.0], + ['f' => -0.0], + ['f' => 1.0e25], + ['f' => 1.0e-7], + ['f' => PHP_FLOAT_MAX], + ['f' => PHP_FLOAT_MIN], + ['f' => 5.0e-324], + ['f' => 4.9e-324], + ['f' => 0.1 + 0.2], + ['f' => null], + ], + ], + 'not finite floats' => [ + schema(int_schema('id'), float_schema('f')), + [['id' => 1, 'f' => NAN], ['id' => 2, 'f' => INF], ['id' => 3, 'f' => -INF]], + ], + 'a not finite float inside a list' => [ + schema(int_schema('id'), list_schema('l', type_list(type_float()))), + [['id' => 1, 'l' => [1.5]], ['id' => 2, 'l' => [2.0, NAN]]], + ], + 'strings that need quoting or escaping' => [ + schema(str_schema('s')), + [ + ['s' => 'a,b'], + ['s' => 'say "hi"'], + ['s' => "two\nlines"], + ['s' => 'back\slash"q'], + ['s' => ' lead'], + ['s' => "tab\t"], + ['s' => 'a;b|c'], + ['s' => "it's"], + ['s' => 'path/to'], + ['s' => "control \x01 and \x7f"], + ['s' => 'zażółć 😀'], + ], + ], + 'text that is not UTF-8' => [ + schema(int_schema('id'), str_schema('s')), + [['id' => 1, 's' => 'fine'], ['id' => 2, 's' => "a\xffb"]], + ], + 'temporal kinds' => [ + schema( + datetime_schema('at', nullable: true), + datetime_schema('offset', zone: '+02:30'), + date_schema('on', nullable: true), + time_schema('took', nullable: true), + ), + [ + [ + 'at' => new DateTimeImmutable('2026-01-02 03:04:05.123456 UTC'), + 'offset' => new DateTimeImmutable('2026-01-02 03:04:05 UTC'), + 'on' => new DateTimeImmutable('2026-01-02'), + 'took' => new DateInterval('PT1H'), + ], + [ + 'at' => null, + 'offset' => new DateTimeImmutable('1969-12-31 23:59:59.5 UTC'), + 'on' => null, + 'took' => null, + ], + ], + ], + 'a named zone across a transition' => [ + schema(datetime_schema('at', zone: 'Europe/Warsaw')), + [ + ['at' => new DateTimeImmutable('@1774745999')], + ['at' => new DateTimeImmutable('@1774746000')], + ['at' => new DateTimeImmutable('@1792890000')], + ], + ], + 'uuid, enum and time zone' => [ + schema( + uuid_schema('id', nullable: true), + enum_schema('backed', BackedStringEnum::class), + enum_schema('basic', BasicEnum::class, nullable: true), + time_zone_schema('zone', nullable: true), + ), + [ + [ + 'id' => new Uuid('f47ac10b-58cc-4372-a567-0e02b2c3d479'), + 'backed' => BackedStringEnum::one, + 'basic' => BasicEnum::two, + 'zone' => new DateTimeZone('Europe/Warsaw'), + ], + ['id' => null, 'backed' => BackedStringEnum::two, 'basic' => null, 'zone' => null], + ], + ], + 'a json column' => [ + schema(int_schema('id'), json_schema('j', nullable: true)), + [ + ['id' => 1, 'j' => '{"a": 1, "b": [1.0, "x/y"]}'], + ['id' => 2, 'j' => '{}'], + ['id' => 3, 'j' => '[]'], + ['id' => 4, 'j' => null], + ], + ], + 'markup columns' => [ + schema(int_schema('id'), xml_schema('x', nullable: true), xml_element_schema('e')), + [ + ['id' => 1, 'x' => '1', 'e' => ''], + ['id' => 2, 'x' => null, 'e' => ''], + ], + ], + 'nested kinds rendered natively' => [ + schema( + list_schema('l', type_list(type_datetime()), nullable: true), + map_schema('m', type_map(type_string(), type_float())), + structure_schema('s', type_structure([ + 'a' => type_integer(), + 'b' => structure_element('b', type_date(), true), + ])), + list_schema('deep', type_list(type_list(type_optional(type_string())))), + ), + [ + [ + 'l' => [new DateTimeImmutable('2026-01-02 03:04:05 UTC')], + 'm' => ['a b' => 1.0, 'c/d' => 0.1 + 0.2], + 's' => ['a' => 1, 'b' => new DateTimeImmutable('2026-01-02')], + 'deep' => [['x', null], []], + ], + ['l' => null, 'm' => ['k' => -0.0], 's' => ['a' => 2], 'deep' => []], + ], + ], + 'maps by key' => [ + schema(map_schema('m', type_map(type_integer(), type_string()))), + [['m' => [0 => 'a', 1 => 'b']], ['m' => [5 => 'a', 7 => 'b']], ['m' => [-1 => 'z']]], + ], + 'empty containers' => [ + schema( + map_schema('m', type_map(type_string(), type_integer())), + list_schema('l', type_list(type_integer())), + ), + [['m' => [], 'l' => []]], + ], + 'a list of json documents' => [ + schema(int_schema('id'), list_schema('l', type_list(type_json()))), + [['id' => 1, 'l' => ['{"a": {}}', '[]']], ['id' => 2, 'l' => []]], + ], + 'nested markup leaves' => [ + schema( + int_schema('id'), + list_schema('l', type_list(type_xml())), + structure_schema('s', type_structure(['x' => type_xml_element()])), + ), + [['id' => 1, 'l' => ['1'], 's' => ['x' => '']]], + ], + 'awkward column names' => [ + schema( + int_schema('first name'), + int_schema('a"b'), + int_schema('x/y'), + int_schema('ż'), + int_schema('0'), + int_schema('7'), + ), + [['first name' => 1, 'a"b' => 2, 'x/y' => 3, 'ż' => 4, '0' => 5, '7' => 6]], + ], + 'no rows' => [schema(int_schema('id'), str_schema('name')), []], + default => throw new InvalidArgumentException('There is no text writer batch named "' . $name . '"'), + }; + } +} diff --git a/src/core/etl/tests/Flow/ETL/Tests/Context/WindowFrameContext.php b/src/core/etl/tests/Flow/ETL/Tests/Context/WindowFrameContext.php index 1fddebc210..35cb55e87f 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Context/WindowFrameContext.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Context/WindowFrameContext.php @@ -22,9 +22,8 @@ public static function results(DataFrame $frame): array $results = []; foreach ($frame->get() as $batch) { - foreach ($batch as $row) { - $value = $row->get('result'); - + // @mago-ignore analysis:mixed-assignment + foreach ($batch->column('result')->values() as $value) { $results[] = is_int($value) || is_float($value) ? $value : null; } } diff --git a/src/core/etl/tests/Flow/ETL/Tests/Context/WindowProcessorContext.php b/src/core/etl/tests/Flow/ETL/Tests/Context/WindowProcessorContext.php index 42ade2eeaa..b8a97fffc5 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Context/WindowProcessorContext.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Context/WindowProcessorContext.php @@ -48,8 +48,8 @@ public static function values(string|Definition $entry, WindowFunction $function $values = []; foreach (self::batches($entry, $function, $rows) as $batch) { - foreach ($batch as $row) { - $values[] = $row->get($name); + foreach ($batch->toArray() as $row) { + $values[] = $row[$name]; } } diff --git a/src/core/etl/tests/Flow/ETL/Tests/Double/AcceptingBackend.php b/src/core/etl/tests/Flow/ETL/Tests/Double/AcceptingBackend.php new file mode 100644 index 0000000000..e457c4bef1 --- /dev/null +++ b/src/core/etl/tests/Flow/ETL/Tests/Double/AcceptingBackend.php @@ -0,0 +1,42 @@ +constant($definition, $value, $count); + } + + public function decode(Definition $definition, array $buffers, int $count, int $nullCount): Column + { + return (new PhpBackend())->decode($definition, $buffers, $count, $nullCount); + } + + public function adopt(Definition $definition, Column $column): Column + { + return $column; + } + + public function allocatedBytes(): int + { + return 0; + } +} diff --git a/src/core/etl/tests/Flow/ETL/Tests/Double/AcceptingColumnBuilder.php b/src/core/etl/tests/Flow/ETL/Tests/Double/AcceptingColumnBuilder.php new file mode 100644 index 0000000000..1232b7e067 --- /dev/null +++ b/src/core/etl/tests/Flow/ETL/Tests/Double/AcceptingColumnBuilder.php @@ -0,0 +1,53 @@ +count++; + } + + public function appendFrom(Column $column, int $i): void + { + $this->count++; + } + + public function appendMany(array $values): void + { + $this->count += count($values); + } + + public function appendPhysicals(array $physicals, ?int $nullCount = null): void + { + $this->count += count($physicals); + } + + public function appendTake(Column $column, array $indices): void + { + $this->count += count($indices); + } + + public function count(): int + { + return $this->count; + } + + public function finish(): Column + { + return new ConstantColumn(type_null(), new NullPhysical(), null, $this->count); + } +} diff --git a/src/core/etl/tests/Flow/ETL/Tests/Double/AddStampToStringEntryTransformer.php b/src/core/etl/tests/Flow/ETL/Tests/Double/AddStampToStringEntryTransformer.php index 590373f8d9..a5d481fde4 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Double/AddStampToStringEntryTransformer.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Double/AddStampToStringEntryTransformer.php @@ -10,7 +10,7 @@ use Flow\ETL\Schema; use Flow\ETL\Transformer; -use function Flow\ETL\DSL\row; +use function Flow\ETL\DSL\array_to_rows; use function Flow\Types\DSL\type_string; use function sprintf; @@ -36,18 +36,18 @@ public function transform(Rows $rows, FlowContext $context): Rows { $stamped = []; - foreach ($rows->all() as $row) { - $stamped[] = row([ - ...$row->values(), + for ($i = 0; $i < $rows->count(); $i++) { + $stamped[] = [ + ...$rows->values($i), $this->entryName => sprintf( '%s%s%s', - type_string()->assert($row->get($this->entryName)), + type_string()->assert($rows->column($this->entryName)->value($i)), $this->divider, $this->stamp, ), - ]); + ]; } - return new Rows($rows->schema(), ...$stamped); + return array_to_rows($stamped, $rows->schema()); } } diff --git a/src/core/etl/tests/Flow/ETL/Tests/Double/ConstantComparison.php b/src/core/etl/tests/Flow/ETL/Tests/Double/ConstantComparison.php new file mode 100644 index 0000000000..1c4853906a --- /dev/null +++ b/src/core/etl/tests/Flow/ETL/Tests/Double/ConstantComparison.php @@ -0,0 +1,36 @@ +isEmpty() ? [] : array_fill(0, $left->count(), $this->result); + } + + public function left(): array + { + return []; + } + + public function right(): array + { + return []; + } +} diff --git a/src/core/etl/tests/Flow/ETL/Tests/Double/CountingDOMDocument.php b/src/core/etl/tests/Flow/ETL/Tests/Double/CountingDOMDocument.php new file mode 100644 index 0000000000..2b4a0ab78d --- /dev/null +++ b/src/core/etl/tests/Flow/ETL/Tests/Double/CountingDOMDocument.php @@ -0,0 +1,35 @@ + $calls + */ + public function __construct( + public readonly ArrayObject $calls, + ) { + parent::__construct(); + $this->loadXML(''); + } + + public function C14N( + bool $exclusive = false, + bool $withComments = false, + ?array $xpath = null, + ?array $nsPrefixes = null, + ): string|false { + $this->calls->append('C14N'); + + return parent::C14N($exclusive, $withComments, $xpath, $nsPrefixes); + } +} diff --git a/src/core/etl/tests/Flow/ETL/Tests/Double/CountingExtractor.php b/src/core/etl/tests/Flow/ETL/Tests/Double/CountingExtractor.php index 08010bdf56..f8eb6274e3 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Double/CountingExtractor.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Double/CountingExtractor.php @@ -4,6 +4,7 @@ namespace Flow\ETL\Tests\Double; +use Flow\ETL\Column\AdaptiveBackend; use Flow\ETL\Extractor; use Flow\ETL\Extractor\BatchableExtractor; use Flow\ETL\Extractor\Batches; @@ -16,7 +17,7 @@ use Generator; use function array_values; -use function count; +use function Flow\ETL\DSL\rows; /** * Counts how many times extract() was pulled, so a test can prove the plan bind read no row, and how @@ -54,30 +55,33 @@ public function extract(FlowContext $context, ?int $limit = null): Generator $this->extractCalls++; $this->contexts[] = $context; - $buffer = []; + $buffer = rows($this->schema); foreach ($this->batches as $rows) { - foreach ($rows->all() as $row) { - $buffer[] = $row; + for ($i = 0; $i < $rows->count(); $i++) { + $buffer = $buffer->concat( + new AdaptiveBackend(), + $rows->slice($i, 1)->matchTo($this->schema, new AdaptiveBackend()), + ); - if (count($buffer) === $this->batchSize()) { + if ($buffer->count() === $this->batchSize()) { $this->batchesYielded++; - $signal = yield Rows::trusted($this->schema, $buffer); + $signal = yield $buffer; if ($signal === Signal::STOP) { return; } - $buffer = []; + $buffer = rows($this->schema); } } } - if ($buffer !== []) { + if (!$buffer->isEmpty()) { $this->batchesYielded++; - yield Rows::trusted($this->schema, $buffer); + yield $buffer; } } diff --git a/src/core/etl/tests/Flow/ETL/Tests/Double/CountingFrameAccumulating.php b/src/core/etl/tests/Flow/ETL/Tests/Double/CountingFrameAccumulating.php index b04f222955..9d87e452ab 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Double/CountingFrameAccumulating.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Double/CountingFrameAccumulating.php @@ -54,8 +54,10 @@ public function apply(WindowContext $window): mixed { $accumulator = $this->accumulator($window->flowContext()); - foreach ($window->frame() as $frameRow) { - $accumulator->accumulate($frameRow); + $frame = $window->frame(); + + for ($index = 0, $count = $frame->count(); $index < $count; $index++) { + $accumulator->accumulate($frame, $index); } return $accumulator->value(); diff --git a/src/core/etl/tests/Flow/ETL/Tests/Double/CountingFrameAccumulator.php b/src/core/etl/tests/Flow/ETL/Tests/Double/CountingFrameAccumulator.php index 54794a2fc6..947c5e9fe2 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Double/CountingFrameAccumulator.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Double/CountingFrameAccumulator.php @@ -4,8 +4,8 @@ namespace Flow\ETL\Tests\Double; -use Flow\ETL\Row; use Flow\ETL\Row\Reference; +use Flow\ETL\Rows; use Flow\ETL\Window\FrameAccumulator; final class CountingFrameAccumulator implements FrameAccumulator @@ -17,11 +17,12 @@ public function __construct( private readonly CountingFrameAccumulating $spy, ) {} - public function accumulate(Row $row): void + public function accumulate(Rows $rows, int $index): void { $this->spy->accumulateCalls++; - $value = $row->get($this->ref); + // @mago-ignore analysis:mixed-assignment + $value = $rows->column($this->ref->base())->value($index); if (is_int($value) || is_float($value)) { $this->sum += $value; diff --git a/src/core/etl/tests/Flow/ETL/Tests/Double/CountingReturnsFunction.php b/src/core/etl/tests/Flow/ETL/Tests/Double/CountingReturnsFunction.php index 33ef8d468d..8585e606ba 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Double/CountingReturnsFunction.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Double/CountingReturnsFunction.php @@ -4,11 +4,12 @@ namespace Flow\ETL\Tests\Double; +use Flow\ETL\Column\Column; use Flow\ETL\FlowContext; use Flow\ETL\Function\FunctionTree; use Flow\ETL\Function\ScalarFunction; use Flow\ETL\Function\ScalarFunctionChain; -use Flow\ETL\Row; +use Flow\ETL\Rows; use Flow\Types\Type; /** @@ -46,9 +47,9 @@ public function withChildren(array $children): static return new self($children[0], $this->type); } - public function eval(Row $row, FlowContext $context): mixed + public function eval(Rows $rows, FlowContext $context): Column { - return $this->value->eval($row, $context); + return $this->value->eval($rows, $context); } /** diff --git a/src/core/etl/tests/Flow/ETL/Tests/Double/DeclaringExtractor.php b/src/core/etl/tests/Flow/ETL/Tests/Double/DeclaringExtractor.php index 7a70182d65..365e063f2c 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Double/DeclaringExtractor.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Double/DeclaringExtractor.php @@ -4,6 +4,7 @@ namespace Flow\ETL\Tests\Double; +use Flow\ETL\Column\AdaptiveBackend; use Flow\ETL\Extractor; use Flow\ETL\Extractor\Statistics; use Flow\ETL\FlowContext; @@ -19,7 +20,7 @@ public function __construct( public function extract(FlowContext $context, ?int $limit = null): Generator { - yield new Rows($this->schema()); + yield Rows::empty($this->schema(), new AdaptiveBackend()); } public function schema(): Schema diff --git a/src/core/etl/tests/Flow/ETL/Tests/Double/FailingOnValuesFunction.php b/src/core/etl/tests/Flow/ETL/Tests/Double/FailingOnValuesFunction.php new file mode 100644 index 0000000000..23c3065139 --- /dev/null +++ b/src/core/etl/tests/Flow/ETL/Tests/Double/FailingOnValuesFunction.php @@ -0,0 +1,108 @@ + + */ + public array $seen = []; + + /** + * @param list $failing + */ + public function __construct( + public readonly ScalarFunction $operand, + public readonly array $failing = [], + public readonly string $failure = self::WITH_ROW, + public readonly ?self $owner = null, + ) {} + + /** + * @return list + */ + public function children(): array + { + return [$this->operand]; + } + + public function counters(): self + { + return $this->owner ?? $this; + } + + public function eval(Rows $rows, FlowContext $context): Column + { + $this->counters()->evals++; + $this->counters()->seen[] = $rows; + $values = (new Parameter($this->operand))->asInts($rows, $context); + + foreach ($values as $i => $value) { + if (!in_array($value, $this->failing, true)) { + continue; + } + + $message = sprintf('fails on %d', $value); + + match ($this->failure) { + self::WITHOUT_ROW => throw new RuntimeException($message), + self::ERROR => throw new Error($message), + default => throw new EvaluationException($i, new RuntimeException($message)), + }; + } + + return (new ResultColumn($context->backend()))->of($this, $values); + } + + /** + * @return Type + */ + public function returns(): Type + { + return type_optional($this->operand->returns()); + } + + /** + * @param list $children + */ + public function withChildren(array $children): static + { + /** @var list $children */ + return new self($children[0], $this->failing, $this->failure, $this->counters()); + } +} diff --git a/src/core/etl/tests/Flow/ETL/Tests/Double/FakeExtractor.php b/src/core/etl/tests/Flow/ETL/Tests/Double/FakeExtractor.php index d6e616ac4e..52e2008edc 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Double/FakeExtractor.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Double/FakeExtractor.php @@ -16,6 +16,7 @@ use Generator; use Ramsey\Uuid\Uuid; +use function Flow\ETL\DSL\array_to_rows; use function Flow\ETL\DSL\bool_schema; use function Flow\ETL\DSL\datetime_schema; use function Flow\ETL\DSL\enum_schema; @@ -26,8 +27,6 @@ use function Flow\ETL\DSL\list_schema; use function Flow\ETL\DSL\map_schema; use function Flow\ETL\DSL\null_schema; -use function Flow\ETL\DSL\row; -use function Flow\ETL\DSL\rows; use function Flow\ETL\DSL\schema; use function Flow\ETL\DSL\string_schema; use function Flow\ETL\DSL\structure_schema; @@ -91,7 +90,28 @@ public function extract(FlowContext $context, ?int $limit = null): Generator for ($i = 0; $i < $this->total; $i++) { $id = $i; - yield rows( + yield array_to_rows( + [[ + 'int' => $id, + 'float' => type_float()->cast(generate_random_int(100, 100000) / 100), + 'bool' => random_int(0, 1) === 1, + 'datetime' => new DateTimeImmutable('now'), + 'null' => null, + 'uuid' => new FlowUuid(Uuid::uuid4()), + 'json' => type_json()->cast(['id' => $id, 'status' => 'NEW']), + 'list' => [1, 2, 3], + 'list_of_datetimes' => [new DateTimeImmutable(), new DateTimeImmutable(), new DateTimeImmutable()], + 'map' => ['NEW', 'PENDING'], + 'struct' => [ + 'street' => 'street_' . $id, + 'city' => 'city_' . $id, + 'zip' => 'zip_' . $id, + 'country' => 'country_' . $id, + 'location' => ['lat' => 1.5, 'lon' => 1.5], + ], + 'enum' => BackedStringEnum::three, + 'xml' => type_xml()->cast('node-' . $id . ''), + ]], schema( int_schema('int'), float_schema('float'), @@ -116,27 +136,6 @@ public function extract(FlowContext $context, ?int $limit = null): Generator enum_schema('enum', BackedStringEnum::class), xml_schema('xml'), ), - row([ - 'int' => $id, - 'float' => type_float()->cast(generate_random_int(100, 100000) / 100), - 'bool' => random_int(0, 1) === 1, - 'datetime' => new DateTimeImmutable('now'), - 'null' => null, - 'uuid' => new FlowUuid(Uuid::uuid4()), - 'json' => type_json()->cast(['id' => $id, 'status' => 'NEW']), - 'list' => [1, 2, 3], - 'list_of_datetimes' => [new DateTimeImmutable(), new DateTimeImmutable(), new DateTimeImmutable()], - 'map' => ['NEW', 'PENDING'], - 'struct' => [ - 'street' => 'street_' . $id, - 'city' => 'city_' . $id, - 'zip' => 'zip_' . $id, - 'country' => 'country_' . $id, - 'location' => ['lat' => 1.5, 'lon' => 1.5], - ], - 'enum' => BackedStringEnum::three, - 'xml' => type_xml()->cast('node-' . $id . ''), - ]), ); } } diff --git a/src/core/etl/tests/Flow/ETL/Tests/Double/FakeRandomOrdersExtractor.php b/src/core/etl/tests/Flow/ETL/Tests/Double/FakeRandomOrdersExtractor.php index fc44c791f7..7ef3e86fdd 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Double/FakeRandomOrdersExtractor.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Double/FakeRandomOrdersExtractor.php @@ -79,7 +79,7 @@ public function extract(FlowContext $context, ?int $limit = null): Generator $schema = self::schema(); foreach ($this->rawData() as $row) { - yield array_to_rows($row, $schema, $context->hydrator()); + yield array_to_rows($row, $schema, $context->backend()); } } diff --git a/src/core/etl/tests/Flow/ETL/Tests/Double/FakeSchemaSampler.php b/src/core/etl/tests/Flow/ETL/Tests/Double/FakeSchemaSampler.php index 7ef84fa86a..e6e89ea073 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Double/FakeSchemaSampler.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Double/FakeSchemaSampler.php @@ -4,7 +4,6 @@ namespace Flow\ETL\Tests\Double; -use Flow\ETL\Row\RawRowValues; use Flow\ETL\Schema\Inference\SchemaSampler; final class FakeSchemaSampler implements SchemaSampler @@ -15,14 +14,14 @@ final class FakeSchemaSampler implements SchemaSampler public array $askedBudgets = []; /** - * @param list> $units + * @param list>> $units */ public function __construct( private readonly array $units, ) {} /** - * @return list> + * @return list>> */ public function samples(int $rowBudget): iterable { diff --git a/src/core/etl/tests/Flow/ETL/Tests/Double/FakeSelfDescribingFile.php b/src/core/etl/tests/Flow/ETL/Tests/Double/FakeSelfDescribingFile.php index bcdd55f5eb..15460fe9ab 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Double/FakeSelfDescribingFile.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Double/FakeSelfDescribingFile.php @@ -5,8 +5,8 @@ namespace Flow\ETL\Tests\Double; use Flow\ETL\Exception\RuntimeException; -use Flow\ETL\Extractor\SelfDescribingFile; -use Flow\ETL\Extractor\SourceFile; +use Flow\ETL\Extractor\File\SelfDescribingFile; +use Flow\ETL\Extractor\File\SourceFile; use Flow\ETL\Extractor\Statistics; use Flow\ETL\Schema; diff --git a/src/core/etl/tests/Flow/ETL/Tests/Double/FakeStaticOrdersExtractor.php b/src/core/etl/tests/Flow/ETL/Tests/Double/FakeStaticOrdersExtractor.php index 5446922670..f2f6ef2be9 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Double/FakeStaticOrdersExtractor.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Double/FakeStaticOrdersExtractor.php @@ -5,12 +5,12 @@ namespace Flow\ETL\Tests\Double; use DateTimeImmutable; +use Flow\ETL\Column\AdaptiveBackend; +use Flow\ETL\Column\Backend; use Flow\ETL\Extractor; use Flow\ETL\Extractor\Signal; use Flow\ETL\Extractor\Statistics; use Flow\ETL\FlowContext; -use Flow\ETL\Row\AdaptiveRowHydrator; -use Flow\ETL\Row\Hydrator; use Flow\ETL\Rows; use Flow\ETL\Schema; use Generator; @@ -78,7 +78,7 @@ public function extract(FlowContext $context, ?int $limit = null): Generator $schema = self::schema(); foreach ($this->rawData() as $row) { - yield array_to_rows($row, $schema, $context->hydrator()); + yield array_to_rows($row, $schema, $context->backend()); } } @@ -138,13 +138,13 @@ public function rawData(): Generator } } - public function toRows(Hydrator $hydrator = new AdaptiveRowHydrator()): Rows + public function toRows(Backend $backend = new AdaptiveBackend()): Rows { - $rows = rows(schema()); $schema = self::schema(); + $rows = rows($schema, $backend); foreach ($this->rawData() as $row) { - $rows = $rows->merge(array_to_rows($row, $schema, $hydrator)); + $rows = $rows->concat($backend, array_to_rows($row, $schema, $backend)); } return $rows; diff --git a/src/core/etl/tests/Flow/ETL/Tests/Double/FileReadingExtractor.php b/src/core/etl/tests/Flow/ETL/Tests/Double/FileReadingExtractor.php index cdd0e75ee4..68332ed393 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Double/FileReadingExtractor.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Double/FileReadingExtractor.php @@ -4,9 +4,9 @@ namespace Flow\ETL\Tests\Double; -use Flow\ETL\Extractor\FileReading; -use Flow\ETL\Extractor\SelfDescribingFile; -use Flow\ETL\Extractor\SourceFile; +use Flow\ETL\Extractor\File\FileReading; +use Flow\ETL\Extractor\File\SelfDescribingFile; +use Flow\ETL\Extractor\File\SourceFile; use Flow\ETL\Schema; use Flow\Filesystem\Filesystem; use Flow\Filesystem\Path; diff --git a/src/core/etl/tests/Flow/ETL/Tests/Double/ForeignColumnStub.php b/src/core/etl/tests/Flow/ETL/Tests/Double/ForeignColumnStub.php new file mode 100644 index 0000000000..f6c284c11e --- /dev/null +++ b/src/core/etl/tests/Flow/ETL/Tests/Double/ForeignColumnStub.php @@ -0,0 +1,83 @@ +column->at($i); + } + + public function concat(Column ...$others): Column + { + return $this->column->concat(...$others); + } + + public function count(): int + { + return $this->column->count(); + } + + public function encode(): array + { + return $this->column->encode(); + } + + public function isNull(int $i): bool + { + return $this->column->isNull($i); + } + + public function nullCount(): int + { + return $this->column->nullCount(); + } + + public function physicals(): array + { + return $this->column->physicals(); + } + + public function slice(int $offset, int $length): Column + { + return new self($this->column->slice($offset, $length)); + } + + public function take(array $indices): Column + { + return $this->column->take($indices); + } + + public function type(): Type + { + return $this->column->type(); + } + + public function value(int $i): mixed + { + return $this->column->value($i); + } + + public function values(): array + { + return $this->column->values(); + } + + public function withType(Type $type): Column + { + return $this->column->withType($type); + } +} diff --git a/src/core/etl/tests/Flow/ETL/Tests/Double/HoldingExtractor.php b/src/core/etl/tests/Flow/ETL/Tests/Double/HoldingExtractor.php new file mode 100644 index 0000000000..cf39c9d388 --- /dev/null +++ b/src/core/etl/tests/Flow/ETL/Tests/Double/HoldingExtractor.php @@ -0,0 +1,57 @@ + $batches + */ + public function __construct( + public readonly ReportedMemoryBackend $backend, + public readonly int $bytesPerBatch, + public readonly array $batches, + ) {} + + public function extract(FlowContext $context, ?int $limit = null): Generator + { + foreach ($this->batches as $batch) { + $this->backend->reported += $this->bytesPerBatch; + + $signal = yield $batch; + + if ($signal === Signal::STOP) { + return; + } + } + } + + public function schema(): Schema + { + return $this->batches[0]->schema(); + } + + public function statistics(): Statistics + { + return new Statistics(); + } + + public function withSchema(Schema $schema): static + { + return $this; + } +} diff --git a/src/core/etl/tests/Flow/ETL/Tests/Double/InMemoryFileBatches.php b/src/core/etl/tests/Flow/ETL/Tests/Double/InMemoryFileBatches.php new file mode 100644 index 0000000000..9fe072eeac --- /dev/null +++ b/src/core/etl/tests/Flow/ETL/Tests/Double/InMemoryFileBatches.php @@ -0,0 +1,56 @@ +> + */ + public array $windows = []; + + /** + * @param array> $ids by URI + */ + public function __construct( + private readonly array $ids, + ) {} + + public function batches( + SourceFile $source, + Schema $body, + int $batchSize, + Backend $backend, + ReadWindow $window, + ): Generator { + $this->windows[$source->uri()][] = $window; + $ids = $this->ids[$source->uri()] ?? []; + $read = array_slice($ids, min($window->offset, count($ids)), $window->limit); + + foreach ($read === [] ? [] : array_chunk($read, $batchSize) as $chunk) { + yield array_to_rows(array_map(static fn(int $id): array => ['id' => $id], $chunk), $body, $backend); + } + + return min($window->offset, count($ids)); + } +} diff --git a/src/core/etl/tests/Flow/ETL/Tests/Double/InMemoryUnskippableFileBatches.php b/src/core/etl/tests/Flow/ETL/Tests/Double/InMemoryUnskippableFileBatches.php new file mode 100644 index 0000000000..3ce2e2b674 --- /dev/null +++ b/src/core/etl/tests/Flow/ETL/Tests/Double/InMemoryUnskippableFileBatches.php @@ -0,0 +1,53 @@ +> + */ + public array $windows = []; + + /** + * @param array> $ids by URI + */ + public function __construct( + private readonly array $ids, + ) {} + + public function batches( + SourceFile $source, + Schema $body, + int $batchSize, + Backend $backend, + ReadWindow $window, + ): Generator { + $this->windows[$source->uri()][] = $window; + $ids = $this->ids[$source->uri()] ?? []; + $read = $ids; + + foreach ($read === [] ? [] : array_chunk($read, $batchSize) as $chunk) { + yield array_to_rows(array_map(static fn(int $id): array => ['id' => $id], $chunk), $body, $backend); + } + + return 0; + } +} diff --git a/src/core/etl/tests/Flow/ETL/Tests/Double/NonStructureUnpackStub.php b/src/core/etl/tests/Flow/ETL/Tests/Double/NonStructureUnpackStub.php index 3c0d828219..87c6e18e29 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Double/NonStructureUnpackStub.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Double/NonStructureUnpackStub.php @@ -4,14 +4,17 @@ namespace Flow\ETL\Tests\Double; +use Flow\ETL\Column\Column; +use Flow\ETL\Column\ValueColumn; use Flow\ETL\FlowContext; use Flow\ETL\Function\FunctionTree; use Flow\ETL\Function\ScalarFunction; use Flow\ETL\Function\ScalarFunction\UnpackResults; use Flow\ETL\Function\ScalarFunctionChain; -use Flow\ETL\Row; +use Flow\ETL\Rows; use Flow\Types\Type; +use function array_fill; use function Flow\Types\DSL\type_string; /** @@ -37,12 +40,10 @@ public function withChildren(array $children): static return $this; } - /** - * @return array - */ - public function eval(Row $row, FlowContext $context): array + public function eval(Rows $rows, FlowContext $context): Column { - return []; + // @mago-ignore analysis:possibly-invalid-argument + return new ValueColumn(array_fill(0, $rows->count(), [])); } /** diff --git a/src/core/etl/tests/Flow/ETL/Tests/Double/OneRowBatchesExtractor.php b/src/core/etl/tests/Flow/ETL/Tests/Double/OneRowBatchesExtractor.php index de0daac4e3..68343b27cc 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Double/OneRowBatchesExtractor.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Double/OneRowBatchesExtractor.php @@ -29,8 +29,8 @@ public function __construct( public function extract(FlowContext $context, ?int $limit = null): Generator { - foreach ($this->rows->all() as $row) { - $signal = yield Rows::trusted($this->rows->schema(), [$row]); + for ($i = 0; $i < $this->rows->count(); $i++) { + $signal = yield $this->rows->slice($i, 1); if ($signal === Signal::STOP) { return; diff --git a/src/core/etl/tests/Flow/ETL/Tests/Double/RecordingFileExtractor.php b/src/core/etl/tests/Flow/ETL/Tests/Double/RecordingFileExtractor.php index 52cffddc9c..5e9cbe6011 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Double/RecordingFileExtractor.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Double/RecordingFileExtractor.php @@ -4,7 +4,7 @@ namespace Flow\ETL\Tests\Double; -use Flow\ETL\Extractor\FileExtractor; +use Flow\ETL\Extractor\File\FileExtractor; use Flow\ETL\Extractor\Statistics; use Flow\ETL\FlowContext; use Flow\ETL\Rows; diff --git a/src/core/etl/tests/Flow/ETL/Tests/Double/RecordingSources.php b/src/core/etl/tests/Flow/ETL/Tests/Double/RecordingSources.php index b1959ca661..82db2e8081 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Double/RecordingSources.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Double/RecordingSources.php @@ -4,7 +4,6 @@ namespace Flow\ETL\Tests\Double; -use Flow\ETL\Row\RawRowValues; use Generator; use function array_keys; @@ -24,14 +23,14 @@ final class RecordingSources public array $rowsRead = []; /** - * @param list> $sources + * @param list>> $sources */ public function __construct( private readonly array $sources, ) {} /** - * @return Generator + * @return Generator> */ public function source(int $index): Generator { @@ -46,7 +45,7 @@ public function source(int $index): Generator } /** - * @return Generator> + * @return Generator>> */ public function sources(): Generator { diff --git a/src/core/etl/tests/Flow/ETL/Tests/Double/RepeatableExtractor.php b/src/core/etl/tests/Flow/ETL/Tests/Double/RepeatableExtractor.php index fc488b488b..4c2d60a258 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Double/RepeatableExtractor.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Double/RepeatableExtractor.php @@ -4,6 +4,7 @@ namespace Flow\ETL\Tests\Double; +use Flow\ETL\Column\AdaptiveBackend; use Flow\ETL\Extractor; use Flow\ETL\Extractor\OverridingExtractor; use Flow\ETL\Extractor\RewindableExtractor; @@ -35,7 +36,7 @@ public function __construct( public function extract(FlowContext $context, ?int $limit = null): Generator { - yield new Rows($this->schema()); + yield Rows::empty($this->schema(), new AdaptiveBackend()); } /** diff --git a/src/core/etl/tests/Flow/ETL/Tests/Double/ReportedMemoryBackend.php b/src/core/etl/tests/Flow/ETL/Tests/Double/ReportedMemoryBackend.php new file mode 100644 index 0000000000..091526bc00 --- /dev/null +++ b/src/core/etl/tests/Flow/ETL/Tests/Double/ReportedMemoryBackend.php @@ -0,0 +1,48 @@ +php->adopt($definition, $column); + } + + public function allocatedBytes(): int + { + return $this->reported; + } + + public function builder(Definition $definition): ColumnBuilder + { + return $this->php->builder($definition); + } + + public function constant(Definition $definition, mixed $value, int $count): Column + { + return $this->php->constant($definition, $value, $count); + } + + public function decode(Definition $definition, array $buffers, int $count, int $nullCount): Column + { + return $this->php->decode($definition, $buffers, $count, $nullCount); + } +} diff --git a/src/core/etl/tests/Flow/ETL/Tests/Double/ReportsResolvedOperand.php b/src/core/etl/tests/Flow/ETL/Tests/Double/ReportsResolvedOperand.php index 4a59319a9c..59e03bbd83 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Double/ReportsResolvedOperand.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Double/ReportsResolvedOperand.php @@ -4,13 +4,16 @@ namespace Flow\ETL\Tests\Double; +use Flow\ETL\Column\Column; use Flow\ETL\FlowContext; +use Flow\ETL\Function\Evaluation\ResultColumn; use Flow\ETL\Function\FunctionTree; use Flow\ETL\Function\ScalarFunction; use Flow\ETL\Function\ScalarFunctionChain; -use Flow\ETL\Row; +use Flow\ETL\Rows; use Flow\Types\Type; +use function array_fill; use function Flow\Types\DSL\type_boolean; /** @@ -41,9 +44,14 @@ public function withChildren(array $children): static return new self($children[0]); } - public function eval(Row $row, FlowContext $context): bool + public function eval(Rows $rows, FlowContext $context): Column { - return $this->operand->resolved(); + // @mago-ignore analysis:possibly-invalid-argument + return (new ResultColumn($context->backend()))->of($this, array_fill( + 0, + $rows->count(), + $this->operand->resolved(), + )); } /** diff --git a/src/core/etl/tests/Flow/ETL/Tests/Double/ReturnsSpyFunction.php b/src/core/etl/tests/Flow/ETL/Tests/Double/ReturnsSpyFunction.php new file mode 100644 index 0000000000..8e0738b54a --- /dev/null +++ b/src/core/etl/tests/Flow/ETL/Tests/Double/ReturnsSpyFunction.php @@ -0,0 +1,72 @@ + + */ + public function children(): array + { + return []; + } + + public function eval(Rows $rows, FlowContext $context): Column + { + // @mago-ignore analysis:possibly-invalid-argument + return (new ResultColumn($context->backend()))->of( + $this, + $rows->isEmpty() ? [] : array_fill(0, $rows->count(), 1), + ); + } + + /** + * @return Type + */ + public function returns(): Type + { + $this->derivations++; + + if (!$this->derivable) { + throw SchemaNotDerivableException::function('counting', 'no declared type'); + } + + return type_integer(); + } + + /** + * @param list $children + */ + public function withChildren(array $children): static + { + return $this; + } +} diff --git a/src/core/etl/tests/Flow/ETL/Tests/Double/SpyBackend.php b/src/core/etl/tests/Flow/ETL/Tests/Double/SpyBackend.php new file mode 100644 index 0000000000..692a45fbb3 --- /dev/null +++ b/src/core/etl/tests/Flow/ETL/Tests/Double/SpyBackend.php @@ -0,0 +1,83 @@ +php = new PhpBackend(); + } + + public function adopt(Definition $definition, Column $column): Column + { + $this->adopts++; + + return $this->php->adopt($definition, $column); + } + + public function adopts(): int + { + return $this->adopts; + } + + public function allocatedBytes(): int + { + return $this->allocatedBytes; + } + + public function builder(Definition $definition): ColumnBuilder + { + $this->builders++; + + return $this->php->builder($definition); + } + + public function builders(): int + { + return $this->builders; + } + + public function decodes(): int + { + return $this->decodes; + } + + public function constant(Definition $definition, mixed $value, int $count): Column + { + $this->constants++; + + return $this->php->constant($definition, $value, $count); + } + + public function constants(): int + { + return $this->constants; + } + + public function decode(Definition $definition, array $buffers, int $count, int $nullCount): Column + { + $this->decodes++; + + return $this->php->decode($definition, $buffers, $count, $nullCount); + } +} diff --git a/src/core/etl/tests/Flow/ETL/Tests/Double/SpyBucketsStorage.php b/src/core/etl/tests/Flow/ETL/Tests/Double/SpyBucketsStorage.php index 4a9e864b85..69b62466cf 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Double/SpyBucketsStorage.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Double/SpyBucketsStorage.php @@ -27,6 +27,11 @@ final class SpyBucketsStorage implements BucketsStorage */ private array $readBucketIds = []; + /** + * @var array rows of every set() call, by bucket + */ + public array $setRowCounts = []; + public function __construct( private readonly BucketsStorage $inner, ) {} @@ -77,6 +82,7 @@ public function remove(string $bucketId): void public function set(string $bucketId, Rows $rows): void { + $this->setRowCounts[$bucketId] = $rows->count(); $this->liveBucketIds[$bucketId] = true; $this->inner->set($bucketId, $rows); } diff --git a/src/core/etl/tests/Flow/ETL/Tests/Double/SpyColumn.php b/src/core/etl/tests/Flow/ETL/Tests/Double/SpyColumn.php new file mode 100644 index 0000000000..34f75089d9 --- /dev/null +++ b/src/core/etl/tests/Flow/ETL/Tests/Double/SpyColumn.php @@ -0,0 +1,109 @@ + $calls one method name per call + */ + public function __construct( + public Column $inner, + public ArrayObject $calls = new ArrayObject(), + ) {} + + public function at(int $i): mixed + { + $this->calls->append('at'); + + return $this->inner->at($i); + } + + public function concat(Column ...$others): Column + { + $this->calls->append('concat'); + + return new self( + $this->inner->concat(...array_map(static fn(Column $other): Column => $other instanceof self + ? $other->inner + : $other, $others)), + $this->calls, + ); + } + + public function count(): int + { + return $this->inner->count(); + } + + public function encode(): array + { + return $this->inner->encode(); + } + + public function isNull(int $i): bool + { + return $this->inner->isNull($i); + } + + public function nullCount(): int + { + return $this->inner->nullCount(); + } + + public function physicals(): array + { + $this->calls->append('physicals'); + + return $this->inner->physicals(); + } + + public function slice(int $offset, int $length): Column + { + return new self($this->inner->slice($offset, $length), $this->calls); + } + + public function take(array $indices): Column + { + return new self($this->inner->take($indices), $this->calls); + } + + /** + * @return Type + */ + public function type(): Type + { + return $this->inner->type(); + } + + public function value(int $i): mixed + { + $this->calls->append('value'); + + return $this->inner->value($i); + } + + public function values(): array + { + $this->calls->append('values'); + + return $this->inner->values(); + } + + public function withType(Type $type): Column + { + return new self($this->inner->withType($type), $this->calls); + } +} diff --git a/src/core/etl/tests/Flow/ETL/Tests/Double/SpyFileSinks.php b/src/core/etl/tests/Flow/ETL/Tests/Double/SpyFileSinks.php new file mode 100644 index 0000000000..0ca4f9fd49 --- /dev/null +++ b/src/core/etl/tests/Flow/ETL/Tests/Double/SpyFileSinks.php @@ -0,0 +1,68 @@ + + */ + public array $opened = []; + + /** + * @var list + */ + public array $closed = []; + + /** + * @param bool $failing every write throws + * @param bool $failingClose every close() throws "close of failed" once it recorded the close + */ + public function __construct( + public readonly bool $failing = false, + public readonly bool $failingClose = false, + ) {} + + public function open(DestinationStream $stream, Backend $backend): FileSink + { + $this->opened[] = $stream->path()->uri(); + return new readonly class($stream, $this) implements FileSink { + public function __construct( + private DestinationStream $stream, + private SpyFileSinks $spy, + ) {} + + public function close(): void + { + $this->stream->append('closed;'); + $this->spy->closed[] = $this->stream->path()->uri(); + + if ($this->spy->failingClose) { + throw new RuntimeException('close of ' . $this->stream->path()->uri() . ' failed'); + } + } + + public function write(Rows $rows): void + { + if ($this->spy->failing) { + throw new RuntimeException('sink refused the batch'); + } + + $this->stream->append('rows:' . $rows->count() . ';'); + } + }; + } +} diff --git a/src/core/etl/tests/Flow/ETL/Tests/Double/SpySerializer.php b/src/core/etl/tests/Flow/ETL/Tests/Double/SpySerializer.php index b485dd23db..e1aafae1ed 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Double/SpySerializer.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Double/SpySerializer.php @@ -4,6 +4,7 @@ namespace Flow\ETL\Tests\Double; +use Flow\ETL\Column\AdaptiveBackend; use Flow\ETL\Rows; use Flow\Filesystem\DestinationStream; use Flow\Filesystem\SourceStream; @@ -26,7 +27,7 @@ final class SpySerializer implements Serializer public array $unserialized = []; public function __construct( - private readonly Serializer $inner = new FloeSerializer(), + private readonly Serializer $inner = new FloeSerializer(new AdaptiveBackend()), ) {} public function serialize(Rows $rows, DestinationStream $destination): void diff --git a/src/core/etl/tests/Flow/ETL/Tests/Double/SpySniffingSample.php b/src/core/etl/tests/Flow/ETL/Tests/Double/SpySniffingSample.php index f63085dfce..b869bed1f7 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Double/SpySniffingSample.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Double/SpySniffingSample.php @@ -5,7 +5,6 @@ namespace Flow\ETL\Tests\Double; use ArrayIterator; -use Flow\ETL\Row\RawRowValues; use Flow\ETL\Schema\Inference\ColumnTypes; use Flow\ETL\Schema\Inference\SchemaInference; use Flow\ETL\Schema\Inference\SniffsColumnTypes; @@ -17,7 +16,7 @@ * A sample that sniffs itself: records what SchemaInferrer hands it and answers with a canned fold. Iterating it * fails - infer() must never fall back to row-by-row observe() for a unit that sniffs itself. * - * @implements IteratorAggregate + * @implements IteratorAggregate> */ final class SpySniffingSample implements IteratorAggregate, SniffsColumnTypes { @@ -31,7 +30,7 @@ public function __construct( ) {} /** - * @return ArrayIterator + * @return ArrayIterator> */ public function getIterator(): ArrayIterator { diff --git a/src/core/etl/tests/Flow/ETL/Tests/Double/StopIgnoringExtractor.php b/src/core/etl/tests/Flow/ETL/Tests/Double/StopIgnoringExtractor.php index 93bfa17393..7bbb15aa7e 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Double/StopIgnoringExtractor.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Double/StopIgnoringExtractor.php @@ -14,8 +14,6 @@ use Flow\ETL\Schema; use Generator; -use function array_chunk; - /** * Breaks exactly one clause of the batch contract: it batches at its size, but never reads what is * sent into it, so Signal::STOP does not end it. @@ -30,9 +28,7 @@ public function __construct( public function extract(FlowContext $context, ?int $limit = null): Generator { - foreach (array_chunk($this->rows->all(), $this->batchSize()) as $chunk) { - yield Rows::trusted($this->rows->schema(), $chunk); - } + yield from $this->rows->chunks($this->batchSize()); } public function isRepeatable(): bool diff --git a/src/core/etl/tests/Flow/ETL/Tests/Double/ThrowWhenRowMatches.php b/src/core/etl/tests/Flow/ETL/Tests/Double/ThrowWhenRowMatches.php index d8093317f4..b04f38fc49 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Double/ThrowWhenRowMatches.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Double/ThrowWhenRowMatches.php @@ -21,8 +21,8 @@ public function __construct( public function transform(Rows $rows, FlowContext $context): Rows { - foreach ($rows as $row) { - if ($row->get($this->column) === $this->value) { + foreach ($rows->toArray() as $row) { + if ($row[$this->column] === $this->value) { throw $this->throwable; } } diff --git a/src/core/etl/tests/Flow/ETL/Tests/Double/ThrowingAfterFirstBatchExtractor.php b/src/core/etl/tests/Flow/ETL/Tests/Double/ThrowingAfterFirstBatchExtractor.php index b600a182b0..4223d92e8b 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Double/ThrowingAfterFirstBatchExtractor.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Double/ThrowingAfterFirstBatchExtractor.php @@ -11,16 +11,15 @@ use Generator; use RuntimeException; +use function Flow\ETL\DSL\array_to_rows; use function Flow\ETL\DSL\int_schema; -use function Flow\ETL\DSL\row; -use function Flow\ETL\DSL\rows; use function Flow\ETL\DSL\schema; final class ThrowingAfterFirstBatchExtractor implements Extractor { public function extract(FlowContext $context, ?int $limit = null): Generator { - yield rows($this->schema(), row(['id' => 1])); + yield array_to_rows([['id' => 1]], $this->schema()); throw new RuntimeException('source failed after its first batch'); } diff --git a/src/core/etl/tests/Flow/ETL/Tests/Double/UnenumerableSum.php b/src/core/etl/tests/Flow/ETL/Tests/Double/UnenumerableSum.php new file mode 100644 index 0000000000..2714b3a3a7 --- /dev/null +++ b/src/core/etl/tests/Flow/ETL/Tests/Double/UnenumerableSum.php @@ -0,0 +1,114 @@ + + */ + public readonly array $operands; + + public function __construct( + public readonly Reference $ref, + ScalarFunction ...$operands, + ) { + $this->operands = array_values($operands); + } + + public function aggregate(Rows $rows, array $indices, FlowContext $context): void + { + if ($rows->schema()->findDefinition($this->ref->base()) === null) { + return; + } + + // @mago-ignore analysis:mixed-assignment + foreach ($rows->column($this->ref->base())->take($indices)->values() as $value) { + if (is_int($value) || is_float($value)) { + $this->total = ($this->total ?? 0.0) + $value; + } + } + } + + /** + * @return list + */ + public function children(): array + { + return [$this->ref, ...$this->operands]; + } + + public function merge(AggregatingFunction $other, FlowContext $context): void + { + if (!$other instanceof self) { + throw new InvalidArgumentException(sprintf('%s cannot merge %s', self::class, $other::class)); + } + + if ($other->total !== null) { + $this->total = ($this->total ?? 0.0) + $other->total; + } + } + + public function outputName(): string + { + return $this->ref->to() . '_sum'; + } + + public function references(): ?array + { + return null; + } + + /** + * @return Type + */ + public function returns(): Type + { + return type_optional(type_float()); + } + + public function value(): ?float + { + return $this->total; + } + + /** + * @param list $children + */ + public function withChildren(array $children): static + { + /** @var list $operands */ + $operands = array_slice($children, 1); + + return new self(type_instance_of(Reference::class)->assert($children[0]), ...$operands); + } +} diff --git a/src/core/etl/tests/Flow/ETL/Tests/Double/WindowFunctionOverOperand.php b/src/core/etl/tests/Flow/ETL/Tests/Double/WindowFunctionOverOperand.php new file mode 100644 index 0000000000..6ce2168395 --- /dev/null +++ b/src/core/etl/tests/Flow/ETL/Tests/Double/WindowFunctionOverOperand.php @@ -0,0 +1,74 @@ +operand->eval($window->frame(), $window->flowContext())->value(0); + } + + /** + * @return list + */ + public function children(): array + { + return [$this->operand]; + } + + public function over(Window $window): static + { + return new self($this->operand, $window); + } + + /** + * @return Type + */ + public function returns(): Type + { + return type_optional($this->operand->returns()); + } + + public function toString(): string + { + return 'operand()'; + } + + public function window(): Window + { + return $this->window ?? throw new RuntimeException('operand() requires an OVER clause.'); + } + + /** + * @param list $children + */ + public function withChildren(array $children): static + { + /** @var list $children */ + return new self($children[0], $this->window); + } +} diff --git a/src/core/etl/tests/Flow/ETL/Tests/Fixtures/Join/AlwaysMeets.php b/src/core/etl/tests/Flow/ETL/Tests/Fixtures/Join/AlwaysMeets.php index 972611841a..c238b8e0a6 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Fixtures/Join/AlwaysMeets.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Fixtures/Join/AlwaysMeets.php @@ -5,13 +5,19 @@ namespace Flow\ETL\Tests\Fixtures\Join; use Flow\ETL\Join\Comparison; -use Flow\ETL\Row; +use Flow\ETL\Rows; final readonly class AlwaysMeets implements Comparison { - public function compare(Row $left, Row $right): bool + public function compare(Rows $left, Rows $right): array { - return true; + $met = []; + + for ($i = 0, $count = $left->count(); $i < $count; $i++) { + $met[] = true; + } + + return $met; } public function left(): array diff --git a/src/core/etl/tests/Flow/ETL/Tests/Fixtures/with-dataframe.php b/src/core/etl/tests/Flow/ETL/Tests/Fixtures/with-dataframe.php index b8e3660c37..9ba4c9f09f 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Fixtures/with-dataframe.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Fixtures/with-dataframe.php @@ -2,15 +2,16 @@ declare(strict_types=1); +use function Flow\ETL\DSL\array_to_rows; use function Flow\ETL\DSL\data_frame; -use function Flow\ETL\DSL\row; -use function Flow\ETL\DSL\rows; use function Flow\ETL\DSL\schema; use function Flow\ETL\DSL\str_schema; -return data_frame()->process(rows( +return data_frame()->process(array_to_rows( + [ + ['code' => 'PL', 'name' => 'Poland'], + ['code' => 'US', 'name' => 'United States'], + ['code' => 'GB', 'name' => 'Great Britain'], + ], schema(str_schema('code'), str_schema('name')), - row(['code' => 'PL', 'name' => 'Poland']), - row(['code' => 'US', 'name' => 'United States']), - row(['code' => 'GB', 'name' => 'Great Britain']), )); diff --git a/src/core/etl/tests/Flow/ETL/Tests/FlowIntegrationTestCase.php b/src/core/etl/tests/Flow/ETL/Tests/FlowIntegrationTestCase.php index ba0845d0ae..174cfbbe2f 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/FlowIntegrationTestCase.php +++ b/src/core/etl/tests/Flow/ETL/Tests/FlowIntegrationTestCase.php @@ -4,6 +4,7 @@ namespace Flow\ETL\Tests; +use Flow\ETL\Column\AdaptiveBackend; use Flow\ETL\Config\Cache\CacheConfig; use Flow\Filesystem\Filesystem; use Flow\Filesystem\FilesystemTable; @@ -62,7 +63,7 @@ protected function setUp(): void $this->fs = new NativeLocalFilesystem(); $this->fstab = new FilesystemTable($this->fs, new StdOutFilesystem()); - $this->serializer = new FloeSerializer(); + $this->serializer = new FloeSerializer(new AdaptiveBackend()); $this->cleanupCacheDir($this->cacheDir); mkdir($this->cacheDir->path(), recursive: true); diff --git a/src/core/etl/tests/Flow/ETL/Tests/FlowTestCase.php b/src/core/etl/tests/Flow/ETL/Tests/FlowTestCase.php index a34a0cbea6..a62327205e 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/FlowTestCase.php +++ b/src/core/etl/tests/Flow/ETL/Tests/FlowTestCase.php @@ -4,9 +4,10 @@ namespace Flow\ETL\Tests; +use Flow\ETL\Column\PhpBackend; use Flow\ETL\Extractor; use Flow\ETL\Extractor\BatchableExtractor; -use Flow\ETL\Extractor\FileExtractor; +use Flow\ETL\Extractor\File\FileExtractor; use Flow\ETL\Extractor\RewindableExtractor; use Flow\ETL\Extractor\Signal; use Flow\ETL\FlowContext; @@ -45,7 +46,7 @@ final public static function assertExtractorHonoursBatchContract( foreach ($extractor->extract(flow_context()) as $batch) { static::assertLessThanOrEqual($extractor->batchSize(), $batch->count(), $message); - $collected = $collected->merge($batch); + $collected = $collected->isEmpty() ? $batch : $collected->concat(new PhpBackend(), $batch); } static::assertEquals($expected->toArray(), $collected->toArray(), $message); @@ -75,11 +76,11 @@ final public static function assertExtractorHonoursBatchContract( $second = rows(schema()); foreach ($extractor->extract(flow_context()) as $batch) { - $first = $first->merge($batch); + $first = $first->isEmpty() ? $batch : $first->concat(new PhpBackend(), $batch); } foreach ($extractor->extract(flow_context()) as $batch) { - $second = $second->merge($batch); + $second = $second->isEmpty() ? $batch : $second->concat(new PhpBackend(), $batch); } static::assertEquals($first->toArray(), $second->toArray(), $message); @@ -123,7 +124,9 @@ final public static function assertExtractedRowsAsArrayEquals( $extractedRows = rows(schema()); foreach (self::extracted($extractor, $flowContext, $limit, $pathFilter) as $nextRows) { - $extractedRows = $extractedRows->merge($nextRows); + $extractedRows = $extractedRows->isEmpty() + ? $nextRows + : $extractedRows->concat(new PhpBackend(), $nextRows); } static::assertEquals($expectedArray, $extractedRows->toArray(), $message); @@ -165,7 +168,9 @@ final public static function assertExtractedRowsEquals( $extractedRows = rows(schema()); foreach (self::extracted($extractor, $flowContext, $limit, $pathFilter) as $nextRows) { - $extractedRows = $extractedRows->merge($nextRows); + $extractedRows = $extractedRows->isEmpty() + ? $nextRows + : $extractedRows->concat(new PhpBackend(), $nextRows); } static::assertEquals($expectedRows, $extractedRows, $message); diff --git a/src/core/etl/tests/Flow/ETL/Tests/Integration/Bucketing/SpillingPathsTest.php b/src/core/etl/tests/Flow/ETL/Tests/Integration/Bucketing/SpillingPathsTest.php new file mode 100644 index 0000000000..d00d68ce73 --- /dev/null +++ b/src/core/etl/tests/Flow/ETL/Tests/Integration/Bucketing/SpillingPathsTest.php @@ -0,0 +1,113 @@ + $id, 'g' => $id % 4]; + } + + yield 'group by' => [ + static fn(SpyBucketsStorage $storage, Unit $limit): ConfigBuilder => config_builder()->groupBy( + hash_group_by()->storage($storage)->memoryLimit($limit)->batchSize(2)->bucketsCount(4), + ), + static fn(ConfigBuilder $config): DataFrame => data_frame($config->build()) + ->read(from_array($rows)) + ->batchSize(3) + ->groupBy('g') + ->aggregate(count(ref('id'))), + 4, + ]; + + yield 'repartition' => [ + static fn(SpyBucketsStorage $storage, Unit $limit): ConfigBuilder => config_builder()->repartition( + hash_repartition()->storage($storage)->memoryLimit($limit)->batchSize(2)->bucketsCount(4), + ), + static fn(ConfigBuilder $config): DataFrame => data_frame($config->build()) + ->read(from_array($rows)) + ->batchSize(3) + ->repartition(ref('g')), + 40, + ]; + + yield 'join' => [ + static fn(SpyBucketsStorage $storage, Unit $limit): ConfigBuilder => config_builder()->join( + hash_join()->storage($storage)->memoryLimit($limit)->batchSize(2)->bucketsCount(4), + ), + static fn(ConfigBuilder $config): DataFrame => data_frame($config->build()) + ->read(from_array($rows)) + ->batchSize(3) + ->join( + data_frame()->read(from_array([['g' => 0], ['g' => 1], ['g' => 2], ['g' => 3]]))->batchSize(1), + join_on(['g' => 'g'], 'r_'), + ), + 40, + ]; + } + + /** + * @param callable(SpyBucketsStorage, Unit): ConfigBuilder $config + * @param callable(ConfigBuilder): DataFrame $pipeline + */ + #[DataProvider('paths')] + public function test_under_the_limit_no_frame_is_written(callable $config, callable $pipeline, int $rows): void + { + $storage = new SpyBucketsStorage(new MemoryBuckets()); + + static::assertCount($rows, $pipeline($config($storage, Unit::fromGb(64)))->fetch()->toArray()); + static::assertSame([], $storage->appendedRows()); + } + + /** + * @param callable(SpyBucketsStorage, Unit): ConfigBuilder $config + * @param callable(ConfigBuilder): DataFrame $pipeline + */ + #[DataProvider('paths')] + public function test_past_the_limit_every_frame_holds_a_full_batch( + callable $config, + callable $pipeline, + int $rows, + ): void { + $storage = new SpyBucketsStorage(new MemoryBuckets()); + + static::assertCount($rows, $pipeline($config($storage, Unit::fromBytes(1)))->fetch()->toArray()); + + $frames = array_merge(...array_values($storage->appendedRows())); + + static::assertNotSame([], $frames); + static::assertSame( + array_map(static fn(): int => 2, $frames), + array_map(static fn(Rows $frame): int => $frame->count(), $frames), + ); + } +} diff --git a/src/core/etl/tests/Flow/ETL/Tests/Integration/Bucketing/Storage/FilesystemBucketsTest.php b/src/core/etl/tests/Flow/ETL/Tests/Integration/Bucketing/Storage/FilesystemBucketsTest.php index f2a28f159f..7534693611 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Integration/Bucketing/Storage/FilesystemBucketsTest.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Integration/Bucketing/Storage/FilesystemBucketsTest.php @@ -6,17 +6,20 @@ use DateTimeImmutable; use Flow\ETL\Bucketing\Storage\FilesystemBuckets; -use Flow\ETL\Row; +use Flow\ETL\Column\AdaptiveBackend; +use Flow\ETL\Column\PhpBackend; +use Flow\ETL\Rows; use Flow\ETL\Tests\Context\BucketsStorageContext; +use Flow\ETL\Tests\Double\SpyBackend; use Flow\ETL\Tests\FlowIntegrationTestCase; use Flow\Floe\Exception\IncompatibleSchemaException; use Flow\Floe\FloeWriter; use function array_map; +use function Flow\ETL\DSL\array_to_rows; use function Flow\ETL\DSL\datetime_schema; use function Flow\ETL\DSL\float_schema; use function Flow\ETL\DSL\int_schema; -use function Flow\ETL\DSL\row; use function Flow\ETL\DSL\rows; use function Flow\ETL\DSL\schema; use function Flow\ETL\DSL\str_schema; @@ -26,6 +29,7 @@ use function Flow\Types\DSL\type_integer; use function Flow\Types\DSL\type_string; use function Flow\Types\DSL\type_structure; +use function iterator_to_array; final class FilesystemBucketsTest extends FlowIntegrationTestCase { @@ -34,12 +38,12 @@ public function test_append_after_get_does_not_truncate_the_bucket(): void $cacheDir = path(__DIR__ . '/var/buckets_append_reopen'); $this->fs()->rm($cacheDir); - $storage = new FilesystemBuckets($this->fs(), cacheDir: $cacheDir); - $storage->append('bucket', rows(schema(int_schema('id')), row(['id' => 1]))); + $storage = new FilesystemBuckets($this->fs(), cacheDir: $cacheDir, backend: new PhpBackend()); + $storage->append('bucket', array_to_rows([['id' => 1]], schema(int_schema('id')))); static::assertCount(1, BucketsStorageContext::rows($storage->get('bucket'))); - $storage->append('bucket', rows(schema(int_schema('id')), row(['id' => 2]))); + $storage->append('bucket', array_to_rows([['id' => 2]], schema(int_schema('id')))); static::assertCount(2, BucketsStorageContext::rows($storage->get('bucket'))); @@ -51,15 +55,15 @@ public function test_append_with_a_column_absent_from_the_bucket_schema_fails(): $cacheDir = path(__DIR__ . '/var/buckets_append_new_column'); $this->fs()->rm($cacheDir); - $storage = new FilesystemBuckets($this->fs(), cacheDir: $cacheDir); - $storage->append('bucket', rows(schema(int_schema('id')), row(['id' => 1]))); + $storage = new FilesystemBuckets($this->fs(), cacheDir: $cacheDir, backend: new PhpBackend()); + $storage->append('bucket', array_to_rows([['id' => 1]], schema(int_schema('id')))); $this->expectException(IncompatibleSchemaException::class); $this->expectExceptionMessageMatches('/new column "name"/'); - $storage->append('bucket', rows( + $storage->append('bucket', array_to_rows( + [['id' => 2, 'name' => 'John']], schema(int_schema('id'), str_schema('name')), - row(['id' => 2, 'name' => 'John']), )); } @@ -68,13 +72,13 @@ public function test_append_widening_a_column_type_fails(): void $cacheDir = path(__DIR__ . '/var/buckets_append_widen'); $this->fs()->rm($cacheDir); - $storage = new FilesystemBuckets($this->fs(), cacheDir: $cacheDir); - $storage->append('bucket', rows(schema(int_schema('amount')), row(['amount' => 1]))); + $storage = new FilesystemBuckets($this->fs(), cacheDir: $cacheDir, backend: new PhpBackend()); + $storage->append('bucket', array_to_rows([['amount' => 1]], schema(int_schema('amount')))); $this->expectException(IncompatibleSchemaException::class); $this->expectExceptionMessageMatches('/expected: amount, given: amount/'); - $storage->append('bucket', rows(schema(float_schema('amount')), row(['amount' => 1.5]))); + $storage->append('bucket', array_to_rows([['amount' => 1.5]], schema(float_schema('amount')))); } public function test_appends_accumulate_into_one_bucket_in_order(): void @@ -82,15 +86,12 @@ public function test_appends_accumulate_into_one_bucket_in_order(): void $cacheDir = path(__DIR__ . '/var/buckets_append'); $this->fs()->rm($cacheDir); - $storage = new FilesystemBuckets($this->fs(), cacheDir: $cacheDir, batchSize: 2); - $storage->append('bucket', rows(schema(int_schema('id')), row(['id' => 1]), row(['id' => 2]))); - $storage->append('bucket', rows(schema(int_schema('id')), row(['id' => 3]))); - $storage->append('bucket', rows(schema(int_schema('id')), row(['id' => 4]), row(['id' => 5]))); + $storage = new FilesystemBuckets($this->fs(), cacheDir: $cacheDir, backend: new PhpBackend(), batchSize: 2); + $storage->append('bucket', array_to_rows([['id' => 1], ['id' => 2]], schema(int_schema('id')))); + $storage->append('bucket', array_to_rows([['id' => 3]], schema(int_schema('id')))); + $storage->append('bucket', array_to_rows([['id' => 4], ['id' => 5]], schema(int_schema('id')))); - static::assertSame( - [1, 2, 3, 4, 5], - array_map(static fn(Row $r): mixed => $r->get('id'), BucketsStorageContext::rows($storage->get('bucket'))), - ); + static::assertSame([1, 2, 3, 4, 5], array_column(BucketsStorageContext::rows($storage->get('bucket')), 'id')); $this->fs()->rm($cacheDir); } @@ -100,19 +101,36 @@ public function test_custom_batch_size_round_trips_all_rows(): void $cacheDir = path(__DIR__ . '/var/buckets_batch_size'); $this->fs()->rm($cacheDir); - $storage = new FilesystemBuckets($this->fs(), cacheDir: $cacheDir, batchSize: 2); - $storage->append('bucket', rows( - schema(int_schema('id')), - row(['id' => 1]), - row(['id' => 2]), - row(['id' => 3]), - )); + $storage = new FilesystemBuckets($this->fs(), cacheDir: $cacheDir, backend: new PhpBackend(), batchSize: 2); + $storage->append('bucket', array_to_rows([['id' => 1], ['id' => 2], ['id' => 3]], schema(int_schema('id')))); static::assertCount(3, BucketsStorageContext::rows($storage->get('bucket'))); $this->fs()->rm($cacheDir); } + public function test_small_appends_read_back_coalesced_up_to_the_batch_size(): void + { + $cacheDir = path(__DIR__ . '/var/buckets_coalesced'); + $this->fs()->rm($cacheDir); + + $storage = new FilesystemBuckets($this->fs(), cacheDir: $cacheDir, backend: new PhpBackend(), batchSize: 3); + + foreach ([1, 2, 3, 4, 5] as $id) { + $storage->append('bucket', array_to_rows([['id' => $id]], schema(int_schema('id')))); + } + + static::assertSame( + [[1, 2, 3], [4, 5]], + array_map( + static fn(Rows $batch) => $batch->reduceToArray('id'), + iterator_to_array($storage->get('bucket'), false), + ), + ); + + $this->fs()->rm($cacheDir); + } + public function test_get_missing_bucket_yields_nothing(): void { $cacheDir = path(__DIR__ . '/var/buckets_missing'); @@ -120,7 +138,11 @@ public function test_get_missing_bucket_yields_nothing(): void static::assertSame( [], - BucketsStorageContext::rows((new FilesystemBuckets($this->fs(), cacheDir: $cacheDir))->get('nope')), + BucketsStorageContext::rows((new FilesystemBuckets( + $this->fs(), + cacheDir: $cacheDir, + backend: new PhpBackend(), + ))->get('nope')), ); } @@ -129,8 +151,8 @@ public function test_remove_closes_an_open_append_session(): void $cacheDir = path(__DIR__ . '/var/buckets_append_remove'); $this->fs()->rm($cacheDir); - $storage = new FilesystemBuckets($this->fs(), cacheDir: $cacheDir); - $storage->append('bucket', rows(schema(int_schema('id')), row(['id' => 1]))); + $storage = new FilesystemBuckets($this->fs(), cacheDir: $cacheDir, backend: new PhpBackend()); + $storage->append('bucket', array_to_rows([['id' => 1]], schema(int_schema('id')))); $storage->remove('bucket'); static::assertSame([], BucketsStorageContext::rows($storage->get('bucket'))); @@ -143,8 +165,8 @@ public function test_remove_deletes_the_bucket(): void $cacheDir = path(__DIR__ . '/var/buckets_remove'); $this->fs()->rm($cacheDir); - $storage = new FilesystemBuckets($this->fs(), cacheDir: $cacheDir); - $storage->append('bucket', rows(schema(int_schema('id')), row(['id' => 1]))); + $storage = new FilesystemBuckets($this->fs(), cacheDir: $cacheDir, backend: new PhpBackend()); + $storage->append('bucket', array_to_rows([['id' => 1]], schema(int_schema('id')))); $storage->remove('bucket'); static::assertSame([], BucketsStorageContext::rows($storage->get('bucket'))); @@ -157,17 +179,17 @@ public function test_round_trips_rows_across_the_read_batch_boundary(): void $cacheDir = path(__DIR__ . '/var/buckets_batch'); $this->fs()->rm($cacheDir); - $storage = new FilesystemBuckets($this->fs(), cacheDir: $cacheDir); + $storage = new FilesystemBuckets($this->fs(), cacheDir: $cacheDir, backend: new PhpBackend()); $input = []; for ($i = 0; $i < 1500; $i++) { - $input[] = row(['id' => $i]); + $input[] = ['id' => $i]; } - $storage->append('bucket', rows(schema(int_schema('id')), ...$input)); + $storage->append('bucket', array_to_rows($input, schema(int_schema('id')))); - static::assertEquals($input, BucketsStorageContext::rows($storage->get('bucket'))); + static::assertSame($input, BucketsStorageContext::rows($storage->get('bucket'))); $this->fs()->rm($cacheDir); } @@ -177,14 +199,11 @@ public function test_set_replaces_a_bucket_written_with_append(): void $cacheDir = path(__DIR__ . '/var/buckets_append_set'); $this->fs()->rm($cacheDir); - $storage = new FilesystemBuckets($this->fs(), cacheDir: $cacheDir); - $storage->append('bucket', rows(schema(int_schema('id')), row(['id' => 1]), row(['id' => 2]))); - $storage->set('bucket', rows(schema(int_schema('id')), row(['id' => 3]))); + $storage = new FilesystemBuckets($this->fs(), cacheDir: $cacheDir, backend: new PhpBackend()); + $storage->append('bucket', array_to_rows([['id' => 1], ['id' => 2]], schema(int_schema('id')))); + $storage->set('bucket', array_to_rows([['id' => 3]], schema(int_schema('id')))); - static::assertSame( - [3], - array_map(static fn(Row $r): mixed => $r->get('id'), BucketsStorageContext::rows($storage->get('bucket'))), - ); + static::assertSame([3], array_column(BucketsStorageContext::rows($storage->get('bucket')), 'id')); $this->fs()->rm($cacheDir); } @@ -194,7 +213,7 @@ public function test_set_with_empty_rows_creates_a_readable_empty_bucket(): void $cacheDir = path(__DIR__ . '/var/buckets_empty'); $this->fs()->rm($cacheDir); - $storage = new FilesystemBuckets($this->fs(), cacheDir: $cacheDir); + $storage = new FilesystemBuckets($this->fs(), cacheDir: $cacheDir, backend: new PhpBackend()); $storage->set('bucket', rows(schema())); static::assertSame([], BucketsStorageContext::rows($storage->get('bucket'))); @@ -207,17 +226,11 @@ public function test_round_trips_schema_changing_rows(): void $cacheDir = path(__DIR__ . '/var/buckets_heterogeneous'); $this->fs()->rm($cacheDir); - $storage = new FilesystemBuckets($this->fs(), cacheDir: $cacheDir); + $storage = new FilesystemBuckets($this->fs(), cacheDir: $cacheDir, backend: new PhpBackend()); - $input = [ - row(['id' => 1]), - row(['id' => 2, 'name' => 'John']), - row(['name' => 'Jane']), - ]; - - $storage->append('bucket', rows( + $storage->append('bucket', array_to_rows( + [['id' => 1], ['id' => 2, 'name' => 'John'], ['name' => 'Jane']], schema(int_schema('id', nullable: true), str_schema('name', nullable: true)), - ...$input, )); // one write session = one schema, and every row now carries every column the schema declares - @@ -228,7 +241,7 @@ public function test_round_trips_schema_changing_rows(): void ['id' => 2, 'name' => 'John'], ['id' => null, 'name' => 'Jane'], ], - array_map(static fn(Row $r): array => $r->toArray(), BucketsStorageContext::rows($storage->get('bucket'))), + BucketsStorageContext::rows($storage->get('bucket')), ); $this->fs()->rm($cacheDir); @@ -239,8 +252,9 @@ public function test_append_with_reordered_structure_keys_matches_and_keeps_ever $cacheDir = path(__DIR__ . '/var/buckets_append_struct_order'); $this->fs()->rm($cacheDir); - $storage = new FilesystemBuckets($this->fs(), cacheDir: $cacheDir); - $storage->append('bucket', rows( + $storage = new FilesystemBuckets($this->fs(), cacheDir: $cacheDir, backend: new PhpBackend()); + $storage->append('bucket', array_to_rows( + [['id' => 1, 's' => ['a' => 1, 'b' => 'x']]], schema( int_schema('id'), structure_schema('s', type_structure([ @@ -248,9 +262,9 @@ public function test_append_with_reordered_structure_keys_matches_and_keeps_ever 'b' => type_string(), ])), ), - row(['id' => 1, 's' => ['a' => 1, 'b' => 'x']]), )); - $storage->append('bucket', rows( + $storage->append('bucket', array_to_rows( + [['id' => 2, 's' => ['b' => 'y', 'a' => 2]]], schema( int_schema('id'), structure_schema('s', type_structure([ @@ -258,14 +272,13 @@ public function test_append_with_reordered_structure_keys_matches_and_keeps_ever 'a' => type_integer(), ])), ), - row(['id' => 2, 's' => ['b' => 'y', 'a' => 2]]), )); $read = BucketsStorageContext::rows($storage->get('bucket')); static::assertCount(2, $read); - static::assertSame(['a' => 1, 'b' => 'x'], $read[0]->get('s')); - static::assertSame(['a' => 2, 'b' => 'y'], $read[1]->get('s')); + static::assertSame(['a' => 1, 'b' => 'x'], $read[0]['s']); + static::assertSame(['a' => 2, 'b' => 'y'], $read[1]['s']); $this->fs()->rm($cacheDir); } @@ -275,16 +288,22 @@ public function test_append_with_reordered_columns_matches_and_keeps_every_row() $cacheDir = path(__DIR__ . '/var/buckets_append_col_order'); $this->fs()->rm($cacheDir); - $storage = new FilesystemBuckets($this->fs(), cacheDir: $cacheDir); - $storage->append('bucket', rows(schema(int_schema('id'), str_schema('name')), row(['id' => 1, 'name' => 'x']))); - $storage->append('bucket', rows(schema(str_schema('name'), int_schema('id')), row(['name' => 'y', 'id' => 2]))); + $storage = new FilesystemBuckets($this->fs(), cacheDir: $cacheDir, backend: new PhpBackend()); + $storage->append('bucket', array_to_rows( + [['id' => 1, 'name' => 'x']], + schema(int_schema('id'), str_schema('name')), + )); + $storage->append('bucket', array_to_rows( + [['name' => 'y', 'id' => 2]], + schema(str_schema('name'), int_schema('id')), + )); $read = BucketsStorageContext::rows($storage->get('bucket')); static::assertCount(2, $read); static::assertSame([['id' => 1, 'name' => 'x'], ['id' => 2, 'name' => 'y']], [ - $read[0]->toArray(), - $read[1]->toArray(), + $read[0], + $read[1], ]); $this->fs()->rm($cacheDir); @@ -296,7 +315,11 @@ public function test_append_resume_with_a_genuinely_different_column_set_still_t $writer = new FloeWriter( $this->fs(), - ($first = rows(schema(int_schema('id'), str_schema('name')), row(['id' => 1, 'name' => 'x'])))->schema(), + ($first = array_to_rows( + [['id' => 1, 'name' => 'x']], + schema(int_schema('id'), str_schema('name')), + ))->schema(), + new AdaptiveBackend(), ); $writer->create($path); $writer->write($first); @@ -304,7 +327,8 @@ public function test_append_resume_with_a_genuinely_different_column_set_still_t $writer = new FloeWriter( $this->fs(), - rows(schema(int_schema('id'), str_schema('city')), row(['id' => 2, 'city' => 'y']))->schema(), + array_to_rows([['id' => 2, 'city' => 'y']], schema(int_schema('id'), str_schema('city')))->schema(), + new AdaptiveBackend(), ); $this->expectException(IncompatibleSchemaException::class); @@ -317,21 +341,35 @@ public function test_datetime_values_read_back_as_admitted(): void $cacheDir = path(__DIR__ . '/var/buckets_datetime_zone'); $this->fs()->rm($cacheDir); - $rows = rows( - schema(datetime_schema('at', zone: 'Europe/Warsaw')), - row(['at' => new DateTimeImmutable('2026-01-02 03:04:05+05:00')]), - ); - $admitted = type_datetime()->assert($rows->first()->get('at'))->format('c'); + $rows = array_to_rows([[ + 'at' => new DateTimeImmutable('2026-01-02 03:04:05+05:00'), + ]], schema(datetime_schema('at', zone: 'Europe/Warsaw'))); + $admitted = type_datetime()->assert($rows->column('at')->value(0))->format('c'); - $storage = new FilesystemBuckets($this->fs(), cacheDir: $cacheDir); + $storage = new FilesystemBuckets($this->fs(), cacheDir: $cacheDir, backend: new PhpBackend()); $storage->set('b', $rows); static::assertSame('2026-01-01T23:04:05+01:00', $admitted); static::assertSame( $admitted, - type_datetime()->assert(BucketsStorageContext::rows($storage->get('b'))[0]->get('at'))->format('c'), + type_datetime()->assert(BucketsStorageContext::rows($storage->get('b'))[0]['at'])->format('c'), ); $this->fs()->rm($cacheDir); } + + public function test_buckets_decode_with_the_given_backend(): void + { + $cacheDir = path(__DIR__ . '/var/buckets_backend'); + $this->fs()->rm($cacheDir); + $backend = new SpyBackend(); + $storage = new FilesystemBuckets($this->fs(), cacheDir: $cacheDir, backend: $backend); + $storage->append('bucket', array_to_rows([['id' => 1]], schema(int_schema('id')))); + + BucketsStorageContext::rows($storage->get('bucket')); + + static::assertGreaterThan(0, $backend->decodes()); + + $this->fs()->rm($cacheDir); + } } diff --git a/src/core/etl/tests/Flow/ETL/Tests/Integration/Bucketing/Storage/PSRCacheBucketsTest.php b/src/core/etl/tests/Flow/ETL/Tests/Integration/Bucketing/Storage/PSRCacheBucketsTest.php index 06a477f1ff..5c7f0db6f9 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Integration/Bucketing/Storage/PSRCacheBucketsTest.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Integration/Bucketing/Storage/PSRCacheBucketsTest.php @@ -6,17 +6,16 @@ use Exception; use Flow\ETL\Bucketing\Storage\PSRCacheBuckets; -use Flow\ETL\Row; +use Flow\ETL\Column\AdaptiveBackend; use Flow\ETL\Tests\Context\BucketsStorageContext; use Flow\ETL\Tests\FlowIntegrationTestCase; +use Flow\Floe\FloeSerializer; use Override; use Symfony\Component\Cache\Adapter\RedisAdapter; use Symfony\Component\Cache\Psr16Cache; -use function array_map; +use function Flow\ETL\DSL\array_to_rows; use function Flow\ETL\DSL\int_schema; -use function Flow\ETL\DSL\row; -use function Flow\ETL\DSL\rows; use function Flow\ETL\DSL\schema; use function getenv; @@ -47,14 +46,15 @@ public function test_round_trips_rows_across_chunks_and_removes_the_bucket(): vo 'timeout' => 5, ]))); - $storage = new PSRCacheBuckets($cache, prefix: 'flow:buckets:test:round_trip'); - $storage->append('bucket', rows(schema(int_schema('id')), row(['id' => 1]), row(['id' => 2]))); - $storage->append('bucket', rows(schema(int_schema('id')), row(['id' => 3]))); - - static::assertSame( - [1, 2, 3], - array_map(static fn(Row $r): mixed => $r->get('id'), BucketsStorageContext::rows($storage->get('bucket'))), + $storage = new PSRCacheBuckets( + $cache, + new FloeSerializer(new AdaptiveBackend()), + prefix: 'flow:buckets:test:round_trip', ); + $storage->append('bucket', array_to_rows([['id' => 1], ['id' => 2]], schema(int_schema('id')))); + $storage->append('bucket', array_to_rows([['id' => 3]], schema(int_schema('id')))); + + static::assertSame([1, 2, 3], array_column(BucketsStorageContext::rows($storage->get('bucket')), 'id')); $storage->remove('bucket'); diff --git a/src/core/etl/tests/Flow/ETL/Tests/Integration/Cache/CacheTestCase.php b/src/core/etl/tests/Flow/ETL/Tests/Integration/Cache/CacheTestCase.php index 5a6f440e1a..abdc23e6ec 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Integration/Cache/CacheTestCase.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Integration/Cache/CacheTestCase.php @@ -6,14 +6,14 @@ use Flow\ETL\Cache; use Flow\ETL\Cache\CacheIndex; +use Flow\ETL\Column\PhpBackend; use Flow\ETL\Exception\KeyNotInCacheException; use Flow\ETL\Rows; use Flow\ETL\Tests\FlowIntegrationTestCase; use Override; +use function Flow\ETL\DSL\array_to_rows; use function Flow\ETL\DSL\int_schema; -use function Flow\ETL\DSL\row; -use function Flow\ETL\DSL\rows; use function Flow\ETL\DSL\schema; use function Flow\ETL\DSL\str_schema; @@ -45,7 +45,7 @@ public function test_caching_index(): void $index->add('chunk-1'); $index->add('chunk-2'); - $cache->set('index', $index->toRows()); + $cache->set('index', $index->toRows(new PhpBackend())); static::assertTrue($cache->has('index')); @@ -61,7 +61,10 @@ public function test_caching_rows(): void static::assertFalse($cache->has('rows')); - $cache->set('rows', $rows = rows(schema(str_schema('name')), row(['name' => 'John']), row(['name' => 'Jane']))); + $cache->set( + 'rows', + $rows = array_to_rows([['name' => 'John'], ['name' => 'Jane']], schema(str_schema('name'))), + ); static::assertTrue($cache->has('rows')); @@ -74,7 +77,7 @@ public function test_caching_schema(): void $cache->set( 'rows', - $rows = rows(schema(int_schema('id'), str_schema('name')), row(['id' => 1, 'name' => 'John'])), + $rows = array_to_rows([['id' => 1, 'name' => 'John']], schema(int_schema('id'), str_schema('name'))), ); static::assertEquals($rows->schema(), $cache->schema('rows')); @@ -91,9 +94,9 @@ public function test_clearing_cache(): void { $cache = $this->cache(); - $cache->set('index', (new CacheIndex('index'))->toRows()); - $cache->set('row', rows(schema(str_schema('name')), row(['name' => 'John']))); - $cache->set('rows', rows(schema(str_schema('name')), row(['name' => 'John']), row(['name' => 'Jane']))); + $cache->set('index', (new CacheIndex('index'))->toRows(new PhpBackend())); + $cache->set('row', array_to_rows([['name' => 'John']], schema(str_schema('name')))); + $cache->set('rows', array_to_rows([['name' => 'John'], ['name' => 'Jane']], schema(str_schema('name')))); $cache->clear(); @@ -106,7 +109,7 @@ public function test_clearing_cache_removes_schemas(): void { $cache = $this->cache(); - $cache->set('rows', rows(schema(str_schema('name')), row(['name' => 'John']))); + $cache->set('rows', array_to_rows([['name' => 'John']], schema(str_schema('name')))); $cache->clear(); @@ -137,9 +140,9 @@ public function test_removing_from_cache(): void { $cache = $this->cache(); - $cache->set('index', (new CacheIndex('index'))->toRows()); - $cache->set('row', rows(schema(str_schema('name')), row(['name' => 'John']))); - $cache->set('rows', rows(schema(str_schema('name')), row(['name' => 'John']), row(['name' => 'Jane']))); + $cache->set('index', (new CacheIndex('index'))->toRows(new PhpBackend())); + $cache->set('row', array_to_rows([['name' => 'John']], schema(str_schema('name')))); + $cache->set('rows', array_to_rows([['name' => 'John'], ['name' => 'Jane']], schema(str_schema('name')))); $cache->delete('row'); @@ -152,8 +155,8 @@ public function test_removing_from_cache_removes_its_schema(): void { $cache = $this->cache(); - $cache->set('row', rows(schema(str_schema('name')), row(['name' => 'John']))); - $cache->set('rows', $rows = rows(schema(int_schema('id')), row(['id' => 1]))); + $cache->set('row', array_to_rows([['name' => 'John']], schema(str_schema('name')))); + $cache->set('rows', $rows = array_to_rows([['id' => 1]], schema(int_schema('id')))); $cache->delete('row'); diff --git a/src/core/etl/tests/Flow/ETL/Tests/Integration/Cache/FilesystemCacheTest.php b/src/core/etl/tests/Flow/ETL/Tests/Integration/Cache/FilesystemCacheTest.php index 50c4a80e98..ecaa3f02a5 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Integration/Cache/FilesystemCacheTest.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Integration/Cache/FilesystemCacheTest.php @@ -6,15 +6,16 @@ use Flow\ETL\Cache; use Flow\ETL\Cache\Implementation\FilesystemCache; +use Flow\ETL\Column\AdaptiveBackend; use Flow\ETL\Exception\KeyNotInCacheException; use Flow\ETL\Tests\Double\SpySerializer; +use Flow\Floe\FloeSerializer; use Flow\Types\Exception\InvalidArgumentException; use function file_get_contents; use function file_put_contents; +use function Flow\ETL\DSL\array_to_rows; use function Flow\ETL\DSL\int_schema; -use function Flow\ETL\DSL\row; -use function Flow\ETL\DSL\rows; use function Flow\ETL\DSL\schema; use function Flow\ETL\DSL\structure_schema; use function Flow\Filesystem\DSL\path; @@ -31,7 +32,7 @@ final class FilesystemCacheTest extends CacheTestCase public function test_torn_entry_is_treated_as_a_cache_miss(): void { $cache = $this->cache(); - $cache->set('torn', rows(schema(int_schema('id')), row(['id' => 1]))); + $cache->set('torn', array_to_rows([['id' => 1]], schema(int_schema('id')))); // simulate a crash mid-write / bit rot: overwrite the closed cache file with garbage $files = glob(__DIR__ . '/var/filesystem-cache/*/*/*/*/torn') ?: []; @@ -46,7 +47,7 @@ public function test_torn_entry_is_treated_as_a_cache_miss(): void public function test_corrupted_frame_with_intact_footer_is_treated_as_a_cache_miss(): void { $cache = $this->cache(); - $cache->set('corrupt', rows(schema(int_schema('id')), row(['id' => 1]))); + $cache->set('corrupt', array_to_rows([['id' => 1]], schema(int_schema('id')))); $files = glob(__DIR__ . '/var/filesystem-cache/*/*/*/*/corrupt') ?: []; static::assertNotEmpty($files); @@ -65,10 +66,10 @@ public function test_corrupted_frame_with_intact_footer_is_treated_as_a_cache_mi public function test_custom_serializer_is_used_for_set_and_get(): void { $spy = new SpySerializer(); - $cache = new FilesystemCache($this->fs(), path(__DIR__ . '/var/filesystem-cache-spy'), $spy); + $cache = new FilesystemCache($this->fs(), $spy, path(__DIR__ . '/var/filesystem-cache-spy')); $cache->clear(); - $cache->set('spy', $rows = rows(schema(int_schema('id')), row(['id' => 1]), row(['id' => 2]))); + $cache->set('spy', $rows = array_to_rows([['id' => 1], ['id' => 2]], schema(int_schema('id')))); static::assertEquals($rows, $cache->get('spy')); static::assertCount(1, $spy->serialized); @@ -80,7 +81,7 @@ public function test_custom_serializer_is_used_for_set_and_get(): void public function test_schema_of_an_entry_without_a_stored_schema_is_a_cache_miss(): void { $cache = $this->cache(); - $cache->set('orphan', rows(schema(int_schema('id')), row(['id' => 1]))); + $cache->set('orphan', array_to_rows([['id' => 1]], schema(int_schema('id')))); $files = glob(__DIR__ . '/var/filesystem-cache/*/*/*/*/orphan.schema') ?: []; static::assertNotEmpty($files); @@ -94,7 +95,7 @@ public function test_schema_of_an_entry_without_a_stored_schema_is_a_cache_miss( public function test_torn_schema_is_treated_as_a_cache_miss(): void { $cache = $this->cache(); - $cache->set('torn-schema', rows(schema(int_schema('id')), row(['id' => 1]))); + $cache->set('torn-schema', array_to_rows([['id' => 1]], schema(int_schema('id')))); $files = glob(__DIR__ . '/var/filesystem-cache/*/*/*/*/torn-schema.schema') ?: []; static::assertNotEmpty($files); @@ -112,10 +113,9 @@ public function test_legacy_structure_schema_payload_is_rejected_loudly(): void // this single test pins the contract for all three $cache = $this->cache(); - $cache->set('legacy', rows( - schema(structure_schema('s', type_structure(['a' => type_integer()]))), - row(['s' => ['a' => 1]]), - )); + $cache->set('legacy', array_to_rows([['s' => [ + 'a' => 1, + ]]], schema(structure_schema('s', type_structure(['a' => type_integer()]))))); $files = glob(__DIR__ . '/var/filesystem-cache/*/*/*/*/legacy.schema') ?: []; static::assertNotEmpty($files); @@ -139,6 +139,10 @@ public function test_legacy_structure_schema_payload_is_rejected_loudly(): void protected function cache(): Cache { - return new FilesystemCache($this->fs(), path(__DIR__ . '/var/filesystem-cache')); + return new FilesystemCache( + $this->fs(), + new FloeSerializer(new AdaptiveBackend()), + path(__DIR__ . '/var/filesystem-cache'), + ); } } diff --git a/src/core/etl/tests/Flow/ETL/Tests/Integration/Cache/PSRSimpleFilesystemCacheTest.php b/src/core/etl/tests/Flow/ETL/Tests/Integration/Cache/PSRSimpleFilesystemCacheTest.php index 228abdff07..a21025a34b 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Integration/Cache/PSRSimpleFilesystemCacheTest.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Integration/Cache/PSRSimpleFilesystemCacheTest.php @@ -6,13 +6,14 @@ use Flow\ETL\Cache; use Flow\ETL\Cache\Implementation\PSRSimpleCache; +use Flow\ETL\Column\AdaptiveBackend; use Flow\ETL\Exception\KeyNotInCacheException; +use Flow\Floe\FloeSerializer; use Symfony\Component\Cache\Adapter\FilesystemAdapter; use Symfony\Component\Cache\Psr16Cache; +use function Flow\ETL\DSL\array_to_rows; use function Flow\ETL\DSL\int_schema; -use function Flow\ETL\DSL\row; -use function Flow\ETL\DSL\rows; use function Flow\ETL\DSL\schema; final class PSRSimpleFilesystemCacheTest extends CacheTestCase @@ -20,8 +21,8 @@ final class PSRSimpleFilesystemCacheTest extends CacheTestCase public function test_torn_entry_is_treated_as_a_cache_miss(): void { $psr = new Psr16Cache(new FilesystemAdapter(directory: __DIR__ . '/var/psr-simple-file-cache')); - $cache = new PSRSimpleCache($psr); - $cache->set('torn', rows(schema(int_schema('id')), row(['id' => 1]))); + $cache = new PSRSimpleCache($psr, new FloeSerializer(new AdaptiveBackend())); + $cache->set('torn', array_to_rows([['id' => 1]], schema(int_schema('id')))); $psr->set('torn', 'not a valid floe payload'); @@ -33,8 +34,8 @@ public function test_torn_entry_is_treated_as_a_cache_miss(): void public function test_schema_of_an_entry_without_a_stored_schema_is_a_cache_miss(): void { $psr = new Psr16Cache(new FilesystemAdapter(directory: __DIR__ . '/var/psr-simple-file-cache')); - $cache = new PSRSimpleCache($psr); - $cache->set('orphan', rows(schema(int_schema('id')), row(['id' => 1]))); + $cache = new PSRSimpleCache($psr, new FloeSerializer(new AdaptiveBackend())); + $cache->set('orphan', array_to_rows([['id' => 1]], schema(int_schema('id')))); $psr->delete('orphan.schema'); @@ -46,8 +47,8 @@ public function test_schema_of_an_entry_without_a_stored_schema_is_a_cache_miss( public function test_torn_schema_is_treated_as_a_cache_miss(): void { $psr = new Psr16Cache(new FilesystemAdapter(directory: __DIR__ . '/var/psr-simple-file-cache')); - $cache = new PSRSimpleCache($psr); - $cache->set('torn-schema', rows(schema(int_schema('id')), row(['id' => 1]))); + $cache = new PSRSimpleCache($psr, new FloeSerializer(new AdaptiveBackend())); + $cache->set('torn-schema', array_to_rows([['id' => 1]], schema(int_schema('id')))); $psr->set('torn-schema.schema', 'not a valid schema payload'); @@ -58,8 +59,9 @@ public function test_torn_schema_is_treated_as_a_cache_miss(): void protected function cache(): Cache { - return new PSRSimpleCache(new Psr16Cache( - new FilesystemAdapter(directory: __DIR__ . '/var/psr-simple-file-cache'), - )); + return new PSRSimpleCache( + new Psr16Cache(new FilesystemAdapter(directory: __DIR__ . '/var/psr-simple-file-cache')), + new FloeSerializer(new AdaptiveBackend()), + ); } } diff --git a/src/core/etl/tests/Flow/ETL/Tests/Integration/Cache/PSRSimpleRedisCacheTest.php b/src/core/etl/tests/Flow/ETL/Tests/Integration/Cache/PSRSimpleRedisCacheTest.php index 11a56a2904..b713d85a41 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Integration/Cache/PSRSimpleRedisCacheTest.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Integration/Cache/PSRSimpleRedisCacheTest.php @@ -7,6 +7,8 @@ use Exception; use Flow\ETL\Cache; use Flow\ETL\Cache\Implementation\PSRSimpleCache; +use Flow\ETL\Column\AdaptiveBackend; +use Flow\Floe\FloeSerializer; use Override; use Symfony\Component\Cache\Adapter\RedisAdapter; use Symfony\Component\Cache\Psr16Cache; @@ -44,10 +46,13 @@ protected function tearDown(): void protected function cache(): Cache { - return new PSRSimpleCache(new Psr16Cache(new RedisAdapter(RedisAdapter::createConnection($this->dsn(), [ - 'retry_interval' => 2, - 'timeout' => 5, - ])))); + return new PSRSimpleCache( + new Psr16Cache(new RedisAdapter(RedisAdapter::createConnection($this->dsn(), [ + 'retry_interval' => 2, + 'timeout' => 5, + ]))), + new FloeSerializer(new AdaptiveBackend()), + ); } protected function dsn(): string diff --git a/src/core/etl/tests/Flow/ETL/Tests/Integration/Config/Bucketing/BucketingConfigBuilderTest.php b/src/core/etl/tests/Flow/ETL/Tests/Integration/Config/Bucketing/BucketingConfigBuilderTest.php index 3eb3e44377..0ea0253f72 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Integration/Config/Bucketing/BucketingConfigBuilderTest.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Integration/Config/Bucketing/BucketingConfigBuilderTest.php @@ -4,12 +4,12 @@ namespace Flow\ETL\Tests\Integration\Config\Bucketing; +use Flow\ETL\Column\PhpBackend; use Flow\ETL\Config\Bucketing\BucketingConfigBuilder; use Flow\ETL\Tests\FlowIntegrationTestCase; +use function Flow\ETL\DSL\array_to_rows; use function Flow\ETL\DSL\int_schema; -use function Flow\ETL\DSL\row; -use function Flow\ETL\DSL\rows; use function Flow\ETL\DSL\schema; use function Flow\Filesystem\DSL\path; use function iterator_to_array; @@ -21,8 +21,8 @@ public function test_build_uses_the_given_spill_root_for_the_default_storage(): $spillRoot = $this->cacheDir->suffix('/spill-root'); (new BucketingConfigBuilder('/flow-php-sort/', 64)) - ->build($spillRoot) - ->storage->set('bucket-1', rows(schema(int_schema('id')), row(['id' => 1]))); + ->build($spillRoot, new PhpBackend()) + ->storage->set('bucket-1', array_to_rows([['id' => 1]], schema(int_schema('id')))); static::assertNotSame( [], diff --git a/src/core/etl/tests/Flow/ETL/Tests/Integration/Config/MemoryLimitTest.php b/src/core/etl/tests/Flow/ETL/Tests/Integration/Config/MemoryLimitTest.php new file mode 100644 index 0000000000..951446e45b --- /dev/null +++ b/src/core/etl/tests/Flow/ETL/Tests/Integration/Config/MemoryLimitTest.php @@ -0,0 +1,41 @@ +percentage(70)->inBytes(), MemoryLimit::default()->inBytes()); + } + + public function test_default_is_one_gib_when_the_memory_limit_is_unlimited(): void + { + ini_set('memory_limit', '-1'); + + static::assertSame(Unit::fromGb(1)->inBytes(), MemoryLimit::default()->inBytes()); + } + + public function test_the_environment_wins_over_the_memory_limit(): void + { + ini_set('memory_limit', '1G'); + putenv(MemoryLimit::ENV . '=64M'); + + try { + static::assertSame(Unit::fromMb(64)->inBytes(), MemoryLimit::default()->inBytes()); + } finally { + putenv(MemoryLimit::ENV); + } + } +} diff --git a/src/core/etl/tests/Flow/ETL/Tests/Integration/DataFrame/AlgorithmOverrideTest.php b/src/core/etl/tests/Flow/ETL/Tests/Integration/DataFrame/AlgorithmOverrideTest.php index 0902ace357..f82bfdd829 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Integration/DataFrame/AlgorithmOverrideTest.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Integration/DataFrame/AlgorithmOverrideTest.php @@ -8,6 +8,7 @@ use Flow\ETL\Cache\Implementation\InMemoryCache; use Flow\ETL\DataFrame; use Flow\ETL\DataFrameFactory; +use Flow\ETL\Dataset\Memory\Unit; use Flow\ETL\Join\Join; use Flow\ETL\Rows; use Flow\ETL\Tests\Double\RecordingBucketsStorage; @@ -15,6 +16,7 @@ use Flow\Filesystem\Path\Filter\KeepAll; use function array_column; +use function Flow\ETL\DSL\array_to_rows; use function Flow\ETL\DSL\config_builder; use function Flow\ETL\DSL\count as count_agg; use function Flow\ETL\DSL\df; @@ -26,8 +28,6 @@ use function Flow\ETL\DSL\int_schema; use function Flow\ETL\DSL\join_on; use function Flow\ETL\DSL\ref; -use function Flow\ETL\DSL\row; -use function Flow\ETL\DSL\rows; use function Flow\ETL\DSL\schema; use function Flow\ETL\DSL\str_schema; use function Flow\ETL\DSL\sum; @@ -64,9 +64,16 @@ public function test_aggregate_override_reaches_the_aggregation(): void $configured = new RecordingBucketsStorage(new MemoryBuckets()); $output = []; - df(config_builder()->groupBy(hash_group_by()->storage($configured))->build()) + df( + config_builder() + ->groupBy(hash_group_by()->storage($configured)->memoryLimit(Unit::fromBytes(1))->batchSize(1)) + ->build(), + ) ->read(from_array([['g' => 'a'], ['g' => 'b'], ['g' => 'a']])) - ->aggregate([count_agg(ref('g'))], hash_group_by()->storage($pinned)) + ->batchSize(1) + ->aggregate([count_agg(ref( + 'g', + ))], hash_group_by()->storage($pinned)->memoryLimit(Unit::fromBytes(1))->batchSize(1)) ->write(to_array($output)) ->run(); @@ -80,9 +87,14 @@ public function test_group_by_override_reaches_the_aggregation(): void $configured = new RecordingBucketsStorage(new MemoryBuckets()); $output = []; - df(config_builder()->groupBy(hash_group_by()->storage($configured))->build()) + df( + config_builder() + ->groupBy(hash_group_by()->storage($configured)->memoryLimit(Unit::fromBytes(1))->batchSize(1)) + ->build(), + ) ->read(from_array([['g' => 'a'], ['g' => 'b'], ['g' => 'a']])) - ->groupBy([ref('g')], hash_group_by()->storage($pinned)) + ->batchSize(1) + ->groupBy([ref('g')], hash_group_by()->storage($pinned)->memoryLimit(Unit::fromBytes(1))->batchSize(1)) ->aggregate(count_agg(ref('g'))) ->write(to_array($output)) ->run(); @@ -116,9 +128,9 @@ public function test_join_each_is_not_governed_by_the_join_algorithm(): void new class implements DataFrameFactory { public function from(Rows $rows): DataFrame { - return df()->process(rows( + return df()->process(array_to_rows( + [['id' => 1, 'n' => 'a']], schema(int_schema('id'), str_schema('n')), - row(['id' => 1, 'n' => 'a']), )); } }, @@ -137,13 +149,17 @@ public function test_join_override_reaches_the_hash_join(): void $configured = new RecordingBucketsStorage(new MemoryBuckets()); $output = []; - df(config_builder()->join(hash_join()->storage($configured))->build()) + df( + config_builder() + ->join(hash_join()->storage($configured)->memoryLimit(Unit::fromBytes(1))->batchSize(1)) + ->build(), + ) ->read(from_array([['id' => 1], ['id' => 2]])) ->join( df()->read(from_array([['id' => 1, 'n' => 'a']])), join_on(['id' => 'id'], 'joined_'), Join::left, - hash_join()->storage($pinned), + hash_join()->storage($pinned)->memoryLimit(Unit::fromBytes(1))->batchSize(1), ) ->write(to_array($output)) ->run(); @@ -160,7 +176,7 @@ public function test_override_spill_root_comes_from_the_cache_config(): void $output = []; df(config_builder()->sort(external_sort()->storage(new MemoryBuckets()))->build()) ->read(from_array([['id' => 3], ['id' => 1], ['id' => 2]])) - ->sortBy([ref('id')], external_sort()->runSize(1)->bucketsCount(2)) + ->sortBy([ref('id')], external_sort()->memoryLimit(Unit::fromBytes(1))->batchSize(1)->bucketsCount(2)) ->write(to_array($output)) ->run(); @@ -180,9 +196,13 @@ public function test_sort_by_pins_the_algorithm_for_this_operation(): void $configured = new RecordingBucketsStorage(new MemoryBuckets()); $output = []; - df(config_builder()->sort(external_sort()->storage($configured)->runSize(1))->build()) + df( + config_builder() + ->sort(external_sort()->storage($configured)->memoryLimit(Unit::fromBytes(1))->batchSize(1)) + ->build(), + ) ->read(from_array([['id' => 3], ['id' => 1], ['id' => 2]])) - ->sortBy([ref('id')], external_sort()->storage($pinned)->runSize(1)) + ->sortBy([ref('id')], external_sort()->storage($pinned)->memoryLimit(Unit::fromBytes(1))->batchSize(1)) ->write(to_array($output)) ->run(); @@ -196,7 +216,11 @@ public function test_sort_by_without_an_algorithm_uses_the_configured_one(): voi $configured = new RecordingBucketsStorage(new MemoryBuckets()); $output = []; - df(config_builder()->sort(external_sort()->storage($configured)->runSize(1))->build()) + df( + config_builder() + ->sort(external_sort()->storage($configured)->memoryLimit(Unit::fromBytes(1))->batchSize(1)) + ->build(), + ) ->read(from_array([['id' => 3], ['id' => 1], ['id' => 2]])) ->sortBy([ref('id')]) ->write(to_array($output)) diff --git a/src/core/etl/tests/Flow/ETL/Tests/Integration/DataFrame/CacheTest.php b/src/core/etl/tests/Flow/ETL/Tests/Integration/DataFrame/CacheTest.php index 96db55cdaa..5c356efd94 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Integration/DataFrame/CacheTest.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Integration/DataFrame/CacheTest.php @@ -7,6 +7,7 @@ use Flow\ETL\Cache\CacheIndex; use Flow\ETL\Cache\Implementation\FilesystemCache; use Flow\ETL\Cache\Implementation\InMemoryCache; +use Flow\ETL\Column\AdaptiveBackend; use Flow\ETL\Extractor; use Flow\ETL\Extractor\Statistics; use Flow\ETL\FlowContext; @@ -15,6 +16,7 @@ use Flow\ETL\Tests\Double\FakeExtractor; use Flow\ETL\Tests\Double\SpySerializer; use Flow\ETL\Tests\FlowIntegrationTestCase; +use Flow\Floe\FloeSerializer; use Flow\Telemetry\Context\MemoryContextStorage; use Flow\Telemetry\Logger\LoggerProvider; use Flow\Telemetry\Meter\MeterProvider; @@ -120,9 +122,13 @@ public function test_cache_end_to_end_with_custom_serializer(): void { $input = array_map(static fn(int $i) => ['id' => $i], range(1, 25)); - $defaultCache = new FilesystemCache($this->fs(), path(__DIR__ . '/var/cache-mode-default')); + $defaultCache = new FilesystemCache( + $this->fs(), + new FloeSerializer(new AdaptiveBackend()), + path(__DIR__ . '/var/cache-mode-default'), + ); $spy = new SpySerializer(); - $customCache = new FilesystemCache($this->fs(), path(__DIR__ . '/var/cache-mode-custom'), $spy); + $customCache = new FilesystemCache($this->fs(), $spy, path(__DIR__ . '/var/cache-mode-custom')); df(config_builder()->cache($defaultCache))->read(from_array($input))->batchSize(10)->cache('parity')->run(); df(config_builder()->cache($customCache))->read(from_array($input))->batchSize(10)->cache('parity')->run(); diff --git a/src/core/etl/tests/Flow/ETL/Tests/Integration/DataFrame/ConfigBuilderTest.php b/src/core/etl/tests/Flow/ETL/Tests/Integration/DataFrame/ConfigBuilderTest.php index 0d7e55e024..6e2c7e135c 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Integration/DataFrame/ConfigBuilderTest.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Integration/DataFrame/ConfigBuilderTest.php @@ -15,20 +15,20 @@ use Flow\ETL\Optimizer; use Flow\ETL\Optimizer\Rule\PushLimitIntoSource; use Flow\ETL\Planner; -use Flow\ETL\Row\AdaptiveRowHydrator; -use Flow\ETL\Row\PhpRowHydrator; +use Flow\ETL\Tests\Double\SpyBackend; use Flow\ETL\Tests\Double\SpySerializer; use Flow\ETL\Tests\FlowIntegrationTestCase; use Override; use function Flow\ETL\DSL\analyze; +use function Flow\ETL\DSL\array_to_rows; use function Flow\ETL\DSL\config_builder; use function Flow\ETL\DSL\external_sort; use function Flow\ETL\DSL\int_schema; use function Flow\ETL\DSL\memory_sort; -use function Flow\ETL\DSL\row; -use function Flow\ETL\DSL\rows; use function Flow\ETL\DSL\schema; +use function Flow\Serializer\DSL\serialize_to_string; +use function Flow\Serializer\DSL\unserialize_from_string; use function str_replace; final class ConfigBuilderTest extends FlowIntegrationTestCase @@ -56,7 +56,7 @@ public function test_config_serializer_reaches_the_default_cache(): void $config->cache->cache->set( 'key', - $rows = rows(schema(int_schema('id')), row(['id' => 1]), row(['id' => 2]), row(['id' => 3])), + $rows = array_to_rows([['id' => 1], ['id' => 2], ['id' => 3]], schema(int_schema('id'))), ); static::assertEquals($rows, $config->cache->cache->get('key')); @@ -149,11 +149,6 @@ public function test_default_external_sort_builds_filesystem_storage(): void static::assertInstanceOf(FilesystemBuckets::class, $config->sort->bucketing->storage); } - public function test_default_hydrator_is_the_adaptive_hydrator(): void - { - static::assertInstanceOf(AdaptiveRowHydrator::class, config_builder()->build()->hydrator()); - } - public function test_default_sorting_algorithm_is_external_sort(): void { static::assertInstanceOf(ExternalSortConfig::class, config_builder()->build()->sort); @@ -164,13 +159,6 @@ public function test_memory_sort_algorithm_override(): void static::assertInstanceOf(MemorySortConfig::class, config_builder()->sort(memory_sort())->build()->sort); } - public function test_hydrator_override_wins_over_the_default(): void - { - $hydrator = new PhpRowHydrator(); - - static::assertSame($hydrator, config_builder()->hydrator($hydrator)->build()->hydrator()); - } - public function test_external_sort_storage_override(): void { $storage = new MemoryBuckets(); @@ -211,4 +199,18 @@ public function test_executor_can_be_overridden(): void static::assertSame($executor, config_builder()->executor($executor)->build()->executor()); } + + public function test_the_default_serializer_follows_a_backend_set_after_a_build(): void + { + $builder = config_builder(); + $builder->build(); + $backend = new SpyBackend(); + $serializer = $builder->backend($backend)->build()->serializer(); + + unserialize_from_string($serializer, serialize_to_string($serializer, array_to_rows([[ + 'id' => 1, + ]], schema(int_schema('id'))))); + + static::assertSame(1, $backend->decodes()); + } } diff --git a/src/core/etl/tests/Flow/ETL/Tests/Integration/DataFrame/ConfiguredBackendTest.php b/src/core/etl/tests/Flow/ETL/Tests/Integration/DataFrame/ConfiguredBackendTest.php new file mode 100644 index 0000000000..7a0cddebfe --- /dev/null +++ b/src/core/etl/tests/Flow/ETL/Tests/Integration/DataFrame/ConfiguredBackendTest.php @@ -0,0 +1,68 @@ +backend(new AdaptiveBackend())) + ->read(from_rows( + array_to_rows([['id' => 1], ['id' => 2]], schema(int_schema('id')), new PhpBackend()), + array_to_rows([['id' => 3]], schema(int_schema('id')), new PhpBackend()), + )) + ->batchSize(3) + ->fetch(); + + static::assertSame([1, 2, 3], $rows->column('id')->values()); + static::assertSame( + extension_loaded('flow_php') ? 'Flow\ETL\Column\RustColumn' : 'Flow\ETL\Column\Php\ScalarColumn', + $rows->column('id')::class, + ); + } + + public function test_a_custom_extractor_yielding_php_rows_is_read_into_the_configured_backend(): void + { + $rows = data_frame(config_builder()->backend(new AdaptiveBackend())) + ->read( + new CountingExtractor( + schema(int_schema('id')), + array_to_rows([['id' => 1]], schema(int_schema('id')), new PhpBackend()), + ), + ) + ->fetch(); + + static::assertSame( + extension_loaded('flow_php') ? 'Flow\ETL\Column\RustColumn' : 'Flow\ETL\Column\Php\ScalarColumn', + $rows->column('id')::class, + ); + } + + #[RequiresPhpExtension('flow_php')] + public function test_native_rows_are_read_into_php_columns_under_a_php_backend(): void + { + $rows = data_frame(config_builder()->backend(new PhpBackend())) + ->read(from_rows(array_to_rows([['id' => 1]], schema(int_schema('id')), new RustBackend()))) + ->fetch(); + + static::assertSame([1], $rows->column('id')->values()); + static::assertSame('Flow\ETL\Column\Php\ScalarColumn', $rows->column('id')::class); + } +} diff --git a/src/core/etl/tests/Flow/ETL/Tests/Integration/DataFrame/DisplayTest.php b/src/core/etl/tests/Flow/ETL/Tests/Integration/DataFrame/DisplayTest.php index e362c2e8d8..1b861afc56 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Integration/DataFrame/DisplayTest.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Integration/DataFrame/DisplayTest.php @@ -19,6 +19,7 @@ use Generator; use function array_map; +use function Flow\ETL\DSL\array_to_rows; use function Flow\ETL\DSL\bool_schema; use function Flow\ETL\DSL\datetime_schema; use function Flow\ETL\DSL\df; @@ -31,8 +32,6 @@ use function Flow\ETL\DSL\list_schema; use function Flow\ETL\DSL\map_schema; use function Flow\ETL\DSL\ref; -use function Flow\ETL\DSL\row; -use function Flow\ETL\DSL\rows; use function Flow\ETL\DSL\schema; use function Flow\ETL\DSL\str_schema; use function Flow\ETL\DSL\string_schema; @@ -73,7 +72,24 @@ public function schema(): Schema public function extract(FlowContext $context, ?int $limit = null): Generator { for ($i = 0; $i < 20; $i++) { - yield rows( + yield array_to_rows( + [[ + 'id' => 1234, + 'price' => 123.45, + '100' => 100, + 'deleted' => false, + 'created-at' => new DateTimeImmutable('2020-07-13 15:00'), + 'phase' => null, + 'array' => type_json()->cast([ + ['id' => 1, 'status' => 'NEW'], + ['id' => 2, 'status' => 'PENDING'], + ]), + 'list' => [1, 2, 3], + 'map' => ['NEW', 'PENDING'], + 'items' => ['item-id' => '1', 'name' => 'one'], + 'enum' => BackedStringEnum::three, + 'xml' => type_xml()->cast('testbar'), + ]], schema( int_schema('id'), float_schema('price'), @@ -91,24 +107,6 @@ public function extract(FlowContext $context, ?int $limit = null): Generator enum_schema('enum', BackedStringEnum::class), xml_schema('xml'), ), - // PHP casts the numeric column name "100" to an int array key - row([ - 'id' => 1234, - 'price' => 123.45, - '100' => 100, - 'deleted' => false, - 'created-at' => new DateTimeImmutable('2020-07-13 15:00'), - 'phase' => null, - 'array' => type_json()->cast([ - ['id' => 1, 'status' => 'NEW'], - ['id' => 2, 'status' => 'PENDING'], - ]), - 'list' => [1, 2, 3], - 'map' => ['NEW', 'PENDING'], - 'items' => ['item-id' => '1', 'name' => 'one'], - 'enum' => BackedStringEnum::three, - 'xml' => type_xml()->cast('testbar'), - ]), ); } } @@ -154,7 +152,15 @@ public function schema(): Schema public function extract(FlowContext $context, ?int $limit = null): Generator { for ($i = 0; $i < 5; $i++) { - yield rows( + yield array_to_rows( + [[ + 'id' => 1234, + 'price' => 123.45, + '100' => 100, + 'deleted' => false, + 'created-at' => new DateTimeImmutable('2020-07-13 15:00'), + 'group' => 'A', + ]], schema( int_schema('id'), float_schema('price'), @@ -163,20 +169,19 @@ public function extract(FlowContext $context, ?int $limit = null): Generator datetime_schema('created-at'), string_schema('group'), ), - // PHP casts the numeric column name "100" to an int array key - row([ + ); + } + + for ($i = 0; $i < 5; $i++) { + yield array_to_rows( + [[ 'id' => 1234, 'price' => 123.45, '100' => 100, 'deleted' => false, 'created-at' => new DateTimeImmutable('2020-07-13 15:00'), - 'group' => 'A', - ]), - ); - } - - for ($i = 0; $i < 5; $i++) { - yield rows( + 'group' => 'B', + ]], schema( int_schema('id'), float_schema('price'), @@ -185,15 +190,6 @@ public function extract(FlowContext $context, ?int $limit = null): Generator datetime_schema('created-at'), string_schema('group'), ), - // PHP casts the numeric column name "100" to an int array key - row([ - 'id' => 1234, - 'price' => 123.45, - '100' => 100, - 'deleted' => false, - 'created-at' => new DateTimeImmutable('2020-07-13 15:00'), - 'group' => 'B', - ]), ); } } @@ -288,21 +284,25 @@ public function test_print_rows(): void ob_start(); df() ->read(from_rows( - rows( + array_to_rows( + [ + ['id' => 1, 'country' => 'PL', 'age' => 20], + ['id' => 2, 'country' => 'PL', 'age' => 20], + ['id' => 3, 'country' => 'PL', 'age' => 25], + ], schema(int_schema('id'), str_schema('country'), int_schema('age')), - row(['id' => 1, 'country' => 'PL', 'age' => 20]), - row(['id' => 2, 'country' => 'PL', 'age' => 20]), - row(['id' => 3, 'country' => 'PL', 'age' => 25]), ), - rows( + array_to_rows( + [ + ['id' => 1, 'country' => 'PL', 'age' => 20, 'salary' => 5000], + ['id' => 1, 'country' => 'PL', 'age' => 20, 'salary' => null], + ], schema( int_schema('id'), str_schema('country'), int_schema('age'), int_schema('salary', nullable: true), ), - row(['id' => 1, 'country' => 'PL', 'age' => 20, 'salary' => 5000]), - row(['id' => 1, 'country' => 'PL', 'age' => 20, 'salary' => null]), ), )) ->printRows(); @@ -342,21 +342,25 @@ public function test_print_schema(): void ob_start(); df() ->read(from_rows( - rows( + array_to_rows( + [ + ['id' => 1, 'country' => 'PL', 'age' => 20], + ['id' => 2, 'country' => 'PL', 'age' => 20], + ['id' => 3, 'country' => 'PL', 'age' => 25], + ], schema(int_schema('id'), str_schema('country'), int_schema('age')), - row(['id' => 1, 'country' => 'PL', 'age' => 20]), - row(['id' => 2, 'country' => 'PL', 'age' => 20]), - row(['id' => 3, 'country' => 'PL', 'age' => 25]), ), - rows( + array_to_rows( + [ + ['id' => 1, 'country' => 'PL', 'age' => 20, 'salary' => 5000], + ['id' => 1, 'country' => 'PL', 'age' => 20, 'salary' => null], + ], schema( int_schema('id'), str_schema('country'), int_schema('age'), int_schema('salary', nullable: true), ), - row(['id' => 1, 'country' => 'PL', 'age' => 20, 'salary' => 5000]), - row(['id' => 1, 'country' => 'PL', 'age' => 20, 'salary' => null]), ), )) ->printSchema(); diff --git a/src/core/etl/tests/Flow/ETL/Tests/Integration/DataFrame/GroupByTest.php b/src/core/etl/tests/Flow/ETL/Tests/Integration/DataFrame/GroupByTest.php index e125ce1977..ec56dd24e6 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Integration/DataFrame/GroupByTest.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Integration/DataFrame/GroupByTest.php @@ -11,6 +11,7 @@ use Flow\ETL\Tests\FlowIntegrationTestCase; use Flow\Types\Value\Uuid; +use function Flow\ETL\DSL\array_to_rows; use function Flow\ETL\DSL\average; use function Flow\ETL\DSL\collect; use function Flow\ETL\DSL\collect_unique; @@ -32,8 +33,6 @@ use function Flow\ETL\DSL\pivot_values; use function Flow\ETL\DSL\rank; use function Flow\ETL\DSL\ref; -use function Flow\ETL\DSL\row; -use function Flow\ETL\DSL\rows; use function Flow\ETL\DSL\schema; use function Flow\ETL\DSL\str_schema; use function Flow\ETL\DSL\string_agg; @@ -82,16 +81,18 @@ public function test_a_bare_reference_groups_like_a_one_element_array(): void public function test_group_by_array(): void { $rows = df() - ->read(from_rows(rows( + ->read(from_rows(array_to_rows( + [ + ['id' => 1, 'score' => 20, 'array' => type_json()->cast(['a', 'b', 'c', 'd'])], + ['id' => 2, 'score' => 20, 'array' => type_json()->cast(['a', 'b', 'c', 'd'])], + ['id' => 3, 'score' => 25, 'array' => type_json()->cast(['a', 'b', 'c'])], + ['id' => 4, 'score' => 30, 'array' => type_json()->cast(['a', 'b', 'c'])], + ['id' => 5, 'score' => 40, 'array' => type_json()->cast(['a', 'b'])], + ['id' => 6, 'score' => 40, 'array' => type_json()->cast(['a', 'b'])], + ['id' => 7, 'score' => 45, 'array' => type_json()->cast(['a', 'b'])], + ['id' => 9, 'score' => 50, 'array' => type_json()->cast(['a'])], + ], schema(int_schema('id'), int_schema('score'), json_schema('array')), - row(['id' => 1, 'score' => 20, 'array' => type_json()->cast(['a', 'b', 'c', 'd'])]), - row(['id' => 2, 'score' => 20, 'array' => type_json()->cast(['a', 'b', 'c', 'd'])]), - row(['id' => 3, 'score' => 25, 'array' => type_json()->cast(['a', 'b', 'c'])]), - row(['id' => 4, 'score' => 30, 'array' => type_json()->cast(['a', 'b', 'c'])]), - row(['id' => 5, 'score' => 40, 'array' => type_json()->cast(['a', 'b'])]), - row(['id' => 6, 'score' => 40, 'array' => type_json()->cast(['a', 'b'])]), - row(['id' => 7, 'score' => 45, 'array' => type_json()->cast(['a', 'b'])]), - row(['id' => 9, 'score' => 50, 'array' => type_json()->cast(['a'])]), ))) ->groupBy(['array']) ->aggregate(sum('score'), average('score')) @@ -115,16 +116,18 @@ public function test_group_by_array(): void public function test_group_by_date_time(): void { $rows = df() - ->read(from_rows(rows( + ->read(from_rows(array_to_rows( + [ + ['id' => 1, 'score' => 20, 'date' => type_datetime()->cast('2024-01-01 10:00:00')], + ['id' => 2, 'score' => 20, 'date' => type_datetime()->cast('2024-01-01 10:00:00')], + ['id' => 3, 'score' => 25, 'date' => type_datetime()->cast('2024-01-02 10:00:00')], + ['id' => 4, 'score' => 30, 'date' => type_datetime()->cast('2024-01-02 10:00:00')], + ['id' => 5, 'score' => 40, 'date' => type_datetime()->cast('2024-01-03 10:00:00')], + ['id' => 6, 'score' => 40, 'date' => type_datetime()->cast('2024-01-03 10:00:00')], + ['id' => 7, 'score' => 45, 'date' => type_datetime()->cast('2024-01-03 10:00:00')], + ['id' => 9, 'score' => 50, 'date' => type_datetime()->cast('2024-01-04 10:00:00')], + ], schema(int_schema('id'), int_schema('score'), datetime_schema('date')), - row(['id' => 1, 'score' => 20, 'date' => type_datetime()->cast('2024-01-01 10:00:00')]), - row(['id' => 2, 'score' => 20, 'date' => type_datetime()->cast('2024-01-01 10:00:00')]), - row(['id' => 3, 'score' => 25, 'date' => type_datetime()->cast('2024-01-02 10:00:00')]), - row(['id' => 4, 'score' => 30, 'date' => type_datetime()->cast('2024-01-02 10:00:00')]), - row(['id' => 5, 'score' => 40, 'date' => type_datetime()->cast('2024-01-03 10:00:00')]), - row(['id' => 6, 'score' => 40, 'date' => type_datetime()->cast('2024-01-03 10:00:00')]), - row(['id' => 7, 'score' => 45, 'date' => type_datetime()->cast('2024-01-03 10:00:00')]), - row(['id' => 9, 'score' => 50, 'date' => type_datetime()->cast('2024-01-04 10:00:00')]), ))) ->groupBy(['date']) ->aggregate(sum('score'), average('score')) @@ -156,16 +159,18 @@ public function test_group_by_multiple_columns_and_batch_size(): void $loader->expects(self::exactly(4))->method('load'); $rows = df() - ->read(from_rows(rows( + ->read(from_rows(array_to_rows( + [ + ['id' => 1, 'country' => 'PL', 'age' => 20, 'gender' => 'male'], + ['id' => 2, 'country' => 'PL', 'age' => 20, 'gender' => 'male'], + ['id' => 3, 'country' => 'PL', 'age' => 25, 'gender' => 'male'], + ['id' => 4, 'country' => 'PL', 'age' => 30, 'gender' => 'female'], + ['id' => 5, 'country' => 'US', 'age' => 40, 'gender' => 'female'], + ['id' => 6, 'country' => 'US', 'age' => 40, 'gender' => 'male'], + ['id' => 7, 'country' => 'US', 'age' => 45, 'gender' => 'female'], + ['id' => 9, 'country' => 'US', 'age' => 50, 'gender' => 'male'], + ], schema(int_schema('id'), str_schema('country'), int_schema('age'), str_schema('gender')), - row(['id' => 1, 'country' => 'PL', 'age' => 20, 'gender' => 'male']), - row(['id' => 2, 'country' => 'PL', 'age' => 20, 'gender' => 'male']), - row(['id' => 3, 'country' => 'PL', 'age' => 25, 'gender' => 'male']), - row(['id' => 4, 'country' => 'PL', 'age' => 30, 'gender' => 'female']), - row(['id' => 5, 'country' => 'US', 'age' => 40, 'gender' => 'female']), - row(['id' => 6, 'country' => 'US', 'age' => 40, 'gender' => 'male']), - row(['id' => 7, 'country' => 'US', 'age' => 45, 'gender' => 'female']), - row(['id' => 9, 'country' => 'US', 'age' => 50, 'gender' => 'male']), ))) ->groupBy(['country', 'gender']) ->aggregate(average(ref('age'))) @@ -188,16 +193,18 @@ public function test_group_by_multiple_columns_and_batch_size(): void public function test_group_by_multiples_columns_with_avg_aggregation(): void { $rows = df() - ->read(from_rows(rows( + ->read(from_rows(array_to_rows( + [ + ['id' => 1, 'country' => 'PL', 'age' => 20, 'gender' => 'male'], + ['id' => 2, 'country' => 'PL', 'age' => 20, 'gender' => 'male'], + ['id' => 3, 'country' => 'PL', 'age' => 25, 'gender' => 'male'], + ['id' => 4, 'country' => 'PL', 'age' => 30, 'gender' => 'female'], + ['id' => 5, 'country' => 'US', 'age' => 40, 'gender' => 'female'], + ['id' => 6, 'country' => 'US', 'age' => 40, 'gender' => 'male'], + ['id' => 7, 'country' => 'US', 'age' => 45, 'gender' => 'female'], + ['id' => 9, 'country' => 'US', 'age' => 50, 'gender' => 'male'], + ], schema(int_schema('id'), str_schema('country'), int_schema('age'), str_schema('gender')), - row(['id' => 1, 'country' => 'PL', 'age' => 20, 'gender' => 'male']), - row(['id' => 2, 'country' => 'PL', 'age' => 20, 'gender' => 'male']), - row(['id' => 3, 'country' => 'PL', 'age' => 25, 'gender' => 'male']), - row(['id' => 4, 'country' => 'PL', 'age' => 30, 'gender' => 'female']), - row(['id' => 5, 'country' => 'US', 'age' => 40, 'gender' => 'female']), - row(['id' => 6, 'country' => 'US', 'age' => 40, 'gender' => 'male']), - row(['id' => 7, 'country' => 'US', 'age' => 45, 'gender' => 'female']), - row(['id' => 9, 'country' => 'US', 'age' => 50, 'gender' => 'male']), ))) ->groupBy(['country', 'gender']) ->aggregate(average(ref('age'))) @@ -221,21 +228,23 @@ public function test_group_by_multiples_columns_with_avg_aggregation(): void public function test_group_by_multiples_columns_with_avg_aggregation_with_null(): void { $rows = df() - ->read(from_rows(rows( + ->read(from_rows(array_to_rows( + [ + ['id' => 1, 'country' => 'PL', 'age' => 20, 'gender' => 'male'], + ['id' => 2, 'country' => 'PL', 'age' => 20, 'gender' => 'male'], + ['id' => 3, 'country' => 'PL', 'age' => 25, 'gender' => 'male'], + ['id' => 4, 'country' => 'PL', 'age' => 30, 'gender' => 'female'], + ['id' => 5, 'country' => 'US', 'age' => 40, 'gender' => 'female'], + ['id' => 6, 'country' => 'US', 'age' => 40, 'gender' => 'male'], + ['id' => 7, 'country' => 'US', 'age' => 45, 'gender' => null], + ['id' => 9, 'country' => 'US', 'age' => 50, 'gender' => 'male'], + ], schema( int_schema('id'), str_schema('country'), int_schema('age'), str_schema('gender', nullable: true), ), - row(['id' => 1, 'country' => 'PL', 'age' => 20, 'gender' => 'male']), - row(['id' => 2, 'country' => 'PL', 'age' => 20, 'gender' => 'male']), - row(['id' => 3, 'country' => 'PL', 'age' => 25, 'gender' => 'male']), - row(['id' => 4, 'country' => 'PL', 'age' => 30, 'gender' => 'female']), - row(['id' => 5, 'country' => 'US', 'age' => 40, 'gender' => 'female']), - row(['id' => 6, 'country' => 'US', 'age' => 40, 'gender' => 'male']), - row(['id' => 7, 'country' => 'US', 'age' => 45, 'gender' => null]), - row(['id' => 9, 'country' => 'US', 'age' => 50, 'gender' => 'male']), ))) ->groupBy(['country', 'gender']) ->aggregate(average(ref('age'))) @@ -243,11 +252,11 @@ public function test_group_by_multiples_columns_with_avg_aggregation_with_null() static::assertSame( [ + ['country' => 'US', 'gender' => null, 'age_avg' => 45.0], ['country' => 'PL', 'gender' => 'female', 'age_avg' => 30.0], ['country' => 'US', 'gender' => 'female', 'age_avg' => 40.0], ['country' => 'PL', 'gender' => 'male', 'age_avg' => 21.67], ['country' => 'US', 'gender' => 'male', 'age_avg' => 45.0], - ['country' => 'US', 'gender' => null, 'age_avg' => 45.0], ], $rows->sortBy(ref('gender'), ref('country'))->toArray(), ); @@ -260,16 +269,18 @@ public function test_group_by_multiples_columns_with_avg_aggregation_with_null() public function test_group_by_single_column(): void { $rows = df() - ->read(from_rows(rows( + ->read(from_rows(array_to_rows( + [ + ['id' => 1, 'country' => 'PL', 'age' => 20], + ['id' => 2, 'country' => 'PL', 'age' => 20], + ['id' => 3, 'country' => 'PL', 'age' => 25], + ['id' => 4, 'country' => 'PL', 'age' => 30], + ['id' => 5, 'country' => 'US', 'age' => 40], + ['id' => 6, 'country' => 'US', 'age' => 40], + ['id' => 7, 'country' => 'US', 'age' => 45], + ['id' => 9, 'country' => 'US', 'age' => 50], + ], schema(int_schema('id'), str_schema('country'), int_schema('age')), - row(['id' => 1, 'country' => 'PL', 'age' => 20]), - row(['id' => 2, 'country' => 'PL', 'age' => 20]), - row(['id' => 3, 'country' => 'PL', 'age' => 25]), - row(['id' => 4, 'country' => 'PL', 'age' => 30]), - row(['id' => 5, 'country' => 'US', 'age' => 40]), - row(['id' => 6, 'country' => 'US', 'age' => 40]), - row(['id' => 7, 'country' => 'US', 'age' => 45]), - row(['id' => 9, 'country' => 'US', 'age' => 50]), ))) ->groupBy(['country']) ->aggregate(sum(ref('age'))) @@ -287,16 +298,18 @@ public function test_group_by_single_column(): void public function test_group_by_single_column_with_an_alias(): void { $rows = df() - ->read(from_rows(rows( + ->read(from_rows(array_to_rows( + [ + ['id' => 1, 'country' => 'PL', 'age' => 20], + ['id' => 2, 'country' => 'PL', 'age' => 20], + ['id' => 3, 'country' => 'PL', 'age' => 25], + ['id' => 4, 'country' => 'PL', 'age' => 30], + ['id' => 5, 'country' => 'US', 'age' => 40], + ['id' => 6, 'country' => 'US', 'age' => 40], + ['id' => 7, 'country' => 'US', 'age' => 45], + ['id' => 9, 'country' => 'US', 'age' => 50], + ], schema(int_schema('id'), str_schema('country'), int_schema('age')), - row(['id' => 1, 'country' => 'PL', 'age' => 20]), - row(['id' => 2, 'country' => 'PL', 'age' => 20]), - row(['id' => 3, 'country' => 'PL', 'age' => 25]), - row(['id' => 4, 'country' => 'PL', 'age' => 30]), - row(['id' => 5, 'country' => 'US', 'age' => 40]), - row(['id' => 6, 'country' => 'US', 'age' => 40]), - row(['id' => 7, 'country' => 'US', 'age' => 45]), - row(['id' => 9, 'country' => 'US', 'age' => 50]), ))) ->groupBy(['country']) ->aggregate(sum(ref('age')->as('total_age'))) @@ -314,16 +327,18 @@ public function test_group_by_single_column_with_an_alias(): void public function test_group_by_single_column_with_avg_aggregation(): void { $rows = df() - ->read(from_rows(rows( + ->read(from_rows(array_to_rows( + [ + ['id' => 1, 'country' => 'PL', 'age' => 20], + ['id' => 2, 'country' => 'PL', 'age' => 20], + ['id' => 3, 'country' => 'PL', 'age' => 25], + ['id' => 4, 'country' => 'PL', 'age' => 30], + ['id' => 5, 'country' => 'US', 'age' => 40], + ['id' => 6, 'country' => 'US', 'age' => 40], + ['id' => 7, 'country' => 'US', 'age' => 45], + ['id' => 9, 'country' => 'US', 'age' => 50], + ], schema(int_schema('id'), str_schema('country'), int_schema('age')), - row(['id' => 1, 'country' => 'PL', 'age' => 20]), - row(['id' => 2, 'country' => 'PL', 'age' => 20]), - row(['id' => 3, 'country' => 'PL', 'age' => 25]), - row(['id' => 4, 'country' => 'PL', 'age' => 30]), - row(['id' => 5, 'country' => 'US', 'age' => 40]), - row(['id' => 6, 'country' => 'US', 'age' => 40]), - row(['id' => 7, 'country' => 'US', 'age' => 45]), - row(['id' => 9, 'country' => 'US', 'age' => 50]), ))) ->groupBy(['country']) ->aggregate(average(ref('age'))) @@ -380,16 +395,18 @@ public function test_group_by_twice(): void public function test_group_by_uuid(): void { $rows = df() - ->read(from_rows(rows( + ->read(from_rows(array_to_rows( + [ + ['id' => 1, 'score' => 20, 'uuid' => type_uuid()->cast('b97a23ab-ba84-4d8f-9d9a-abd32cc58110')], + ['id' => 2, 'score' => 20, 'uuid' => type_uuid()->cast('b97a23ab-ba84-4d8f-9d9a-abd32cc58110')], + ['id' => 3, 'score' => 25, 'uuid' => type_uuid()->cast('28fc1a5f-25eb-40e2-88b8-7a0cdc5d18ae')], + ['id' => 4, 'score' => 30, 'uuid' => type_uuid()->cast('28fc1a5f-25eb-40e2-88b8-7a0cdc5d18ae')], + ['id' => 5, 'score' => 40, 'uuid' => type_uuid()->cast('5085fabf-15f7-4467-9076-61547afbbdc9')], + ['id' => 6, 'score' => 40, 'uuid' => type_uuid()->cast('5085fabf-15f7-4467-9076-61547afbbdc9')], + ['id' => 7, 'score' => 45, 'uuid' => type_uuid()->cast('5085fabf-15f7-4467-9076-61547afbbdc9')], + ['id' => 9, 'score' => 50, 'uuid' => type_uuid()->cast('c7c22b40-45ad-46d1-a47b-0d1dd389ae41')], + ], schema(int_schema('id'), int_schema('score'), uuid_schema('uuid')), - row(['id' => 1, 'score' => 20, 'uuid' => type_uuid()->cast('b97a23ab-ba84-4d8f-9d9a-abd32cc58110')]), - row(['id' => 2, 'score' => 20, 'uuid' => type_uuid()->cast('b97a23ab-ba84-4d8f-9d9a-abd32cc58110')]), - row(['id' => 3, 'score' => 25, 'uuid' => type_uuid()->cast('28fc1a5f-25eb-40e2-88b8-7a0cdc5d18ae')]), - row(['id' => 4, 'score' => 30, 'uuid' => type_uuid()->cast('28fc1a5f-25eb-40e2-88b8-7a0cdc5d18ae')]), - row(['id' => 5, 'score' => 40, 'uuid' => type_uuid()->cast('5085fabf-15f7-4467-9076-61547afbbdc9')]), - row(['id' => 6, 'score' => 40, 'uuid' => type_uuid()->cast('5085fabf-15f7-4467-9076-61547afbbdc9')]), - row(['id' => 7, 'score' => 45, 'uuid' => type_uuid()->cast('5085fabf-15f7-4467-9076-61547afbbdc9')]), - row(['id' => 9, 'score' => 50, 'uuid' => type_uuid()->cast('c7c22b40-45ad-46d1-a47b-0d1dd389ae41')]), ))) ->groupBy(['uuid']) ->aggregate(sum('score'), average('score')) @@ -668,23 +685,25 @@ public function test_a_global_aggregate_emptied_by_a_filter_emits_one_row_of_typ public function test_standalone_avg_aggregation(): void { $rows = df() - ->read(from_rows(rows( + ->read(from_rows(array_to_rows( + [ + ['id' => 1, 'country' => 'PL', 'age' => 20], + ['id' => 2, 'country' => 'PL', 'age' => 20], + ['id' => 3, 'country' => 'PL', 'age' => 25], + ['id' => 4, 'country' => 'PL', 'age' => 30], + ['id' => 5, 'country' => 'US', 'age' => 40], + ['id' => 6, 'country' => 'US', 'age' => 40], + ['id' => 7, 'country' => 'US', 'age' => 45], + ['id' => 9, 'country' => 'US', 'age' => 50], + ], schema(int_schema('id'), str_schema('country'), int_schema('age')), - row(['id' => 1, 'country' => 'PL', 'age' => 20]), - row(['id' => 2, 'country' => 'PL', 'age' => 20]), - row(['id' => 3, 'country' => 'PL', 'age' => 25]), - row(['id' => 4, 'country' => 'PL', 'age' => 30]), - row(['id' => 5, 'country' => 'US', 'age' => 40]), - row(['id' => 6, 'country' => 'US', 'age' => 40]), - row(['id' => 7, 'country' => 'US', 'age' => 45]), - row(['id' => 9, 'country' => 'US', 'age' => 50]), ))) ->aggregate([average(ref('age'))]) ->rename('age_avg', 'average_age') ->fetch(); static::assertEquals( - rows(schema(float_schema('average_age', nullable: true)), row(['average_age' => 33.75])), + array_to_rows([['average_age' => 33.75]], schema(float_schema('average_age', nullable: true))), $rows, ); } @@ -692,16 +711,18 @@ public function test_standalone_avg_aggregation(): void public function test_standalone_avg_and_max_aggregation(): void { df() - ->read(from_rows(rows( + ->read(from_rows(array_to_rows( + [ + ['id' => 1, 'country' => 'PL', 'age' => 20], + ['id' => 2, 'country' => 'PL', 'age' => 20], + ['id' => 3, 'country' => 'PL', 'age' => 25], + ['id' => 4, 'country' => 'PL', 'age' => 30], + ['id' => 5, 'country' => 'US', 'age' => 40], + ['id' => 6, 'country' => 'US', 'age' => 40], + ['id' => 7, 'country' => 'US', 'age' => 45], + ['id' => 9, 'country' => 'US', 'age' => 50], + ], schema(int_schema('id'), str_schema('country'), int_schema('age')), - row(['id' => 1, 'country' => 'PL', 'age' => 20]), - row(['id' => 2, 'country' => 'PL', 'age' => 20]), - row(['id' => 3, 'country' => 'PL', 'age' => 25]), - row(['id' => 4, 'country' => 'PL', 'age' => 30]), - row(['id' => 5, 'country' => 'US', 'age' => 40]), - row(['id' => 6, 'country' => 'US', 'age' => 40]), - row(['id' => 7, 'country' => 'US', 'age' => 45]), - row(['id' => 9, 'country' => 'US', 'age' => 50]), ))) ->aggregate([average(ref('age')), max(ref('age'))]) ->forEach(function (Rows $rows): void { @@ -723,11 +744,13 @@ public function test_standalone_avg_and_max_aggregation(): void public function test_the_output_definition_does_not_vary_between_groups(): void { $rows = df() - ->read(from_rows(rows( + ->read(from_rows(array_to_rows( + [ + ['group' => 'a', 'value' => 10.0], + ['group' => 'a', 'value' => 20.0], + ['group' => 'b', 'value' => 0.5], + ], schema(str_schema('group'), float_schema('value')), - row(['group' => 'a', 'value' => 10.0]), - row(['group' => 'a', 'value' => 20.0]), - row(['group' => 'b', 'value' => 0.5]), ))) ->groupBy(['group']) ->aggregate(min(ref('value'))) diff --git a/src/core/etl/tests/Flow/ETL/Tests/Integration/DataFrame/JoinCrossTest.php b/src/core/etl/tests/Flow/ETL/Tests/Integration/DataFrame/JoinCrossTest.php index dd1bc7b85b..78d2389ba7 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Integration/DataFrame/JoinCrossTest.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Integration/DataFrame/JoinCrossTest.php @@ -7,13 +7,12 @@ use Flow\ETL\Loader; use Flow\ETL\Tests\FlowTestCase; +use function Flow\ETL\DSL\array_to_rows; use function Flow\ETL\DSL\bool_schema; use function Flow\ETL\DSL\data_frame; use function Flow\ETL\DSL\df; use function Flow\ETL\DSL\from_rows; use function Flow\ETL\DSL\int_schema; -use function Flow\ETL\DSL\row; -use function Flow\ETL\DSL\rows; use function Flow\ETL\DSL\schema; use function Flow\ETL\DSL\str_schema; @@ -25,18 +24,19 @@ public function test_cross_join(): void $loader->expects(self::exactly(2))->method('load'); $rows = df() - ->from(from_rows(rows( + ->from(from_rows(array_to_rows( + [ + ['id' => 1, 'country' => 'PL'], + ['id' => 2, 'country' => 'PL'], + ['id' => 3, 'country' => 'PL'], + ['id' => 4, 'country' => 'PL'], + ], schema(int_schema('id'), str_schema('country')), - row(['id' => 1, 'country' => 'PL']), - row(['id' => 2, 'country' => 'PL']), - row(['id' => 3, 'country' => 'PL']), - row(['id' => 4, 'country' => 'PL']), ))) ->batchSize(2) - ->crossJoin(data_frame()->process(rows( + ->crossJoin(data_frame()->process(array_to_rows( + [['num' => 1, 'active' => true], ['num' => 2, 'active' => false]], schema(int_schema('num'), bool_schema('active')), - row(['num' => 1, 'active' => true]), - row(['num' => 2, 'active' => false]), ))) ->write($loader) ->fetch(); diff --git a/src/core/etl/tests/Flow/ETL/Tests/Integration/DataFrame/JoinEachTest.php b/src/core/etl/tests/Flow/ETL/Tests/Integration/DataFrame/JoinEachTest.php index a2af8a1219..4b6c734cda 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Integration/DataFrame/JoinEachTest.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Integration/DataFrame/JoinEachTest.php @@ -11,13 +11,12 @@ use Flow\ETL\Rows; use Flow\ETL\Tests\FlowTestCase; +use function Flow\ETL\DSL\array_to_rows; use function Flow\ETL\DSL\data_frame; use function Flow\ETL\DSL\df; use function Flow\ETL\DSL\from_rows; use function Flow\ETL\DSL\int_schema; use function Flow\ETL\DSL\join_on; -use function Flow\ETL\DSL\row; -use function Flow\ETL\DSL\rows; use function Flow\ETL\DSL\schema; use function Flow\ETL\DSL\str_schema; @@ -29,26 +28,27 @@ public function test_join_each(): void $loader->expects(self::exactly(2))->method('load'); $rows = df() - ->read(from_rows(rows( + ->read(from_rows(array_to_rows( + [ + ['id' => 1, 'country' => 'PL'], + ['id' => 2, 'country' => 'PL'], + ['id' => 3, 'country' => 'PL'], + ['id' => 4, 'country' => 'PL'], + ['id' => 5, 'country' => 'US'], + ['id' => 6, 'country' => 'US'], + ['id' => 7, 'country' => 'US'], + ['id' => 9, 'country' => 'US'], + ], schema(int_schema('id'), str_schema('country')), - row(['id' => 1, 'country' => 'PL']), - row(['id' => 2, 'country' => 'PL']), - row(['id' => 3, 'country' => 'PL']), - row(['id' => 4, 'country' => 'PL']), - row(['id' => 5, 'country' => 'US']), - row(['id' => 6, 'country' => 'US']), - row(['id' => 7, 'country' => 'US']), - row(['id' => 9, 'country' => 'US']), ))) ->batchSize(4) ->joinEach( new class implements DataFrameFactory { public function from(Rows $rows): DataFrame { - return data_frame()->process(rows( + return data_frame()->process(array_to_rows( + [['code' => 'PL', 'name' => 'Poland'], ['code' => 'US', 'name' => 'United States']], schema(str_schema('code'), str_schema('name')), - row(['code' => 'PL', 'name' => 'Poland']), - row(['code' => 'US', 'name' => 'United States']), )); } }, @@ -78,26 +78,30 @@ public function test_join_each_without_prefix(): void $loader->expects(self::exactly(2))->method('load'); $rows = df() - ->read(from_rows(rows( + ->read(from_rows(array_to_rows( + [ + ['id' => 1, 'country_code' => 'PL'], + ['id' => 2, 'country_code' => 'PL'], + ['id' => 3, 'country_code' => 'PL'], + ['id' => 4, 'country_code' => 'PL'], + ['id' => 5, 'country_code' => 'US'], + ['id' => 6, 'country_code' => 'US'], + ['id' => 7, 'country_code' => 'US'], + ['id' => 9, 'country_code' => 'US'], + ], schema(int_schema('id'), str_schema('country_code')), - row(['id' => 1, 'country_code' => 'PL']), - row(['id' => 2, 'country_code' => 'PL']), - row(['id' => 3, 'country_code' => 'PL']), - row(['id' => 4, 'country_code' => 'PL']), - row(['id' => 5, 'country_code' => 'US']), - row(['id' => 6, 'country_code' => 'US']), - row(['id' => 7, 'country_code' => 'US']), - row(['id' => 9, 'country_code' => 'US']), ))) ->batchSize(4) ->joinEach( new class implements DataFrameFactory { public function from(Rows $rows): DataFrame { - return data_frame()->process(rows( + return data_frame()->process(array_to_rows( + [ + ['country_code' => 'PL', 'name' => 'Poland'], + ['country_code' => 'US', 'name' => 'United States'], + ], schema(str_schema('country_code'), str_schema('name')), - row(['country_code' => 'PL', 'name' => 'Poland']), - row(['country_code' => 'US', 'name' => 'United States']), )); } }, diff --git a/src/core/etl/tests/Flow/ETL/Tests/Integration/DataFrame/JoinEmptySideTest.php b/src/core/etl/tests/Flow/ETL/Tests/Integration/DataFrame/JoinEmptySideTest.php index 3306190b8f..1f398bc59c 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Integration/DataFrame/JoinEmptySideTest.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Integration/DataFrame/JoinEmptySideTest.php @@ -8,12 +8,12 @@ use Flow\ETL\Rows; use Flow\ETL\Tests\FlowTestCase; +use function Flow\ETL\DSL\array_to_rows; use function Flow\ETL\DSL\df; use function Flow\ETL\DSL\from_rows; use function Flow\ETL\DSL\hash_join; use function Flow\ETL\DSL\int_schema; use function Flow\ETL\DSL\join_on; -use function Flow\ETL\DSL\row; use function Flow\ETL\DSL\rows; use function Flow\ETL\DSL\schema; use function Flow\ETL\DSL\str_schema; @@ -23,7 +23,7 @@ final class JoinEmptySideTest extends FlowTestCase public function test_cross_join_against_an_empty_side_yields_no_rows_under_the_cross_schema(): void { $joined = df() - ->read(from_rows(rows(schema(int_schema('id')), row(['id' => 1])))) + ->read(from_rows(array_to_rows([['id' => 1]], schema(int_schema('id'))))) ->crossJoin(df()->process(rows(schema(str_schema('code'))))) ->fetch(); @@ -57,7 +57,7 @@ public function test_right_join_against_an_empty_left_side_keeps_the_left_column public function leftJoinedAgainstNothing(int $buckets): Rows { return df() - ->read(from_rows(rows(schema(int_schema('id')), row(['id' => 1]), row(['id' => 2])))) + ->read(from_rows(array_to_rows([['id' => 1], ['id' => 2]], schema(int_schema('id'))))) ->join( df()->process(rows(schema(int_schema('id'), str_schema('r')))), join_on(['id' => 'id']), @@ -75,7 +75,7 @@ public function rightJoinedAgainstNothing(int $buckets): Rows return df() ->read(from_rows(rows(schema(int_schema('id'), str_schema('l'))))) ->join( - df()->process(rows(schema(int_schema('id')), row(['id' => 1]))), + df()->process(array_to_rows([['id' => 1]], schema(int_schema('id')))), join_on(['id' => 'id']), Join::right, hash_join()->bucketsCount($buckets), diff --git a/src/core/etl/tests/Flow/ETL/Tests/Integration/DataFrame/JoinTest.php b/src/core/etl/tests/Flow/ETL/Tests/Integration/DataFrame/JoinTest.php index 21fd98233e..6bc19c79c0 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Integration/DataFrame/JoinTest.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Integration/DataFrame/JoinTest.php @@ -14,16 +14,16 @@ use Flow\ETL\Tests\FlowIntegrationTestCase; use Flow\ETL\Tests\Mother\RowsMother; +use function Flow\ETL\DSL\array_to_rows; use function Flow\ETL\DSL\config_builder; use function Flow\ETL\DSL\data_frame; +use function Flow\ETL\DSL\date_schema; use function Flow\ETL\DSL\datetime_schema; use function Flow\ETL\DSL\df; use function Flow\ETL\DSL\from_rows; use function Flow\ETL\DSL\hash_join; use function Flow\ETL\DSL\int_schema; use function Flow\ETL\DSL\join_on; -use function Flow\ETL\DSL\row; -use function Flow\ETL\DSL\rows; use function Flow\ETL\DSL\schema; use function Flow\ETL\DSL\str_schema; use function usort; @@ -36,21 +36,25 @@ public function test_join_inner(): void $loader->expects(self::exactly(2))->method('load'); $rows = df() - ->from(from_rows(rows( + ->from(from_rows(array_to_rows( + [ + ['id' => 1, 'country' => 'PL'], + ['id' => 2, 'country' => 'US'], + ['id' => 3, 'country' => 'FR'], + ['id' => 4, 'country' => 'UK'], + ['id' => 5, 'country' => 'GB'], + ], schema(int_schema('id'), str_schema('country')), - row(['id' => 1, 'country' => 'PL']), - row(['id' => 2, 'country' => 'US']), - row(['id' => 3, 'country' => 'FR']), - row(['id' => 4, 'country' => 'UK']), - row(['id' => 5, 'country' => 'GB']), ))) ->join( - data_frame()->process(rows( + data_frame()->process(array_to_rows( + [ + ['code' => 'PL', 'name' => 'Poland'], + ['code' => 'US', 'name' => 'United States'], + ['code' => 'FR', 'name' => 'France'], + ['code' => 'CN', 'name' => 'Canada'], + ], schema(str_schema('code'), str_schema('name')), - row(['code' => 'PL', 'name' => 'Poland']), - row(['code' => 'US', 'name' => 'United States']), - row(['code' => 'FR', 'name' => 'France']), - row(['code' => 'CN', 'name' => 'Canada']), )), join_on(['country' => 'code'], 'joined_'), Join::inner, @@ -93,19 +97,23 @@ public function test_a_limit_after_a_resident_join_closes_the_loaders_before_it( public function test_join_inner_with_on_disk_buckets_cache(): void { $rows = df(config_builder()->join(hash_join()->bucketsCount(4))) - ->from(from_rows(rows( + ->from(from_rows(array_to_rows( + [ + ['id' => 1, 'country' => 'PL'], + ['id' => 2, 'country' => 'US'], + ['id' => 3, 'country' => 'FR'], + ['id' => 4, 'country' => 'UK'], + ], schema(int_schema('id'), str_schema('country')), - row(['id' => 1, 'country' => 'PL']), - row(['id' => 2, 'country' => 'US']), - row(['id' => 3, 'country' => 'FR']), - row(['id' => 4, 'country' => 'UK']), ))) ->join( - data_frame()->process(rows( + data_frame()->process(array_to_rows( + [ + ['code' => 'PL', 'name' => 'Poland'], + ['code' => 'US', 'name' => 'United States'], + ['code' => 'FR', 'name' => 'France'], + ], schema(str_schema('code'), str_schema('name')), - row(['code' => 'PL', 'name' => 'Poland']), - row(['code' => 'US', 'name' => 'United States']), - row(['code' => 'FR', 'name' => 'France']), )), join_on(['country' => 'code'], 'joined_'), Join::inner, @@ -128,19 +136,23 @@ public function test_join_inner_with_on_disk_buckets_cache(): void public function test_join_inner_with_on_disk_buckets_preserves_a_nullable_column_present_and_null(): void { $rows = df(config_builder()->join(hash_join()->bucketsCount(2))) - ->from(from_rows(rows( + ->from(from_rows(array_to_rows( + [ + ['id' => 1, 'country' => 'PL', 'note' => 'has-note'], + ['id' => 2, 'country' => 'US', 'note' => null], + ['id' => 3, 'country' => 'FR', 'note' => 'third'], + ['id' => 4, 'country' => 'PL', 'note' => null], + ], schema(int_schema('id'), str_schema('country'), str_schema('note', nullable: true)), - row(['id' => 1, 'country' => 'PL', 'note' => 'has-note']), - row(['id' => 2, 'country' => 'US', 'note' => null]), - row(['id' => 3, 'country' => 'FR', 'note' => 'third']), - row(['id' => 4, 'country' => 'PL', 'note' => null]), ))) ->join( - data_frame()->process(rows( + data_frame()->process(array_to_rows( + [ + ['code' => 'PL', 'name' => 'Poland'], + ['code' => 'US', 'name' => 'United States'], + ['code' => 'FR', 'name' => 'France'], + ], schema(str_schema('code'), str_schema('name')), - row(['code' => 'PL', 'name' => 'Poland']), - row(['code' => 'US', 'name' => 'United States']), - row(['code' => 'FR', 'name' => 'France']), )), join_on(['country' => 'code'], 'joined_'), Join::inner, @@ -167,22 +179,23 @@ public function test_join_left(): void $loader->expects(self::exactly(2))->method('load'); $rows = df() - ->from(from_rows(rows( + ->from(from_rows(array_to_rows( + [ + ['id' => 1, 'country' => 'PL'], + ['id' => 2, 'country' => 'PL'], + ['id' => 3, 'country' => 'PL'], + ['id' => 4, 'country' => 'PL'], + ['id' => 5, 'country' => 'US'], + ['id' => 6, 'country' => 'US'], + ['id' => 7, 'country' => 'US'], + ['id' => 9, 'country' => 'US'], + ], schema(int_schema('id'), str_schema('country')), - row(['id' => 1, 'country' => 'PL']), - row(['id' => 2, 'country' => 'PL']), - row(['id' => 3, 'country' => 'PL']), - row(['id' => 4, 'country' => 'PL']), - row(['id' => 5, 'country' => 'US']), - row(['id' => 6, 'country' => 'US']), - row(['id' => 7, 'country' => 'US']), - row(['id' => 9, 'country' => 'US']), ))) ->join( - data_frame()->process(rows( + data_frame()->process(array_to_rows( + [['code' => 'PL', 'name' => 'Poland'], ['code' => 'US', 'name' => 'United States']], schema(str_schema('code'), str_schema('name')), - row(['code' => 'PL', 'name' => 'Poland']), - row(['code' => 'US', 'name' => 'United States']), )), Expression::on(['country' => 'code'], 'joined_'), ) @@ -211,21 +224,25 @@ public function test_join_left_anti(): void $loader->expects(self::exactly(1))->method('load'); $rows = df() - ->from(from_rows(rows( + ->from(from_rows(array_to_rows( + [ + ['id' => 1, 'country' => 'PL'], + ['id' => 2, 'country' => 'US'], + ['id' => 3, 'country' => 'FR'], + ['id' => 5, 'country' => 'GB'], + ['id' => 7, 'country' => 'CN'], + ], schema(int_schema('id'), str_schema('country')), - row(['id' => 1, 'country' => 'PL']), - row(['id' => 2, 'country' => 'US']), - row(['id' => 3, 'country' => 'FR']), - row(['id' => 5, 'country' => 'GB']), - row(['id' => 7, 'country' => 'CN']), ))) ->join( - data_frame()->process(rows( + data_frame()->process(array_to_rows( + [ + ['code' => 'PL', 'name' => 'Poland'], + ['code' => 'US', 'name' => 'United States'], + ['code' => 'FR', 'name' => 'France'], + ['code' => 'CA', 'name' => 'Canada'], + ], schema(str_schema('code'), str_schema('name')), - row(['code' => 'PL', 'name' => 'Poland']), - row(['code' => 'US', 'name' => 'United States']), - row(['code' => 'FR', 'name' => 'France']), - row(['code' => 'CA', 'name' => 'Canada']), )), Expression::on(['country' => 'code'], 'joined_'), Join::left_anti, @@ -249,22 +266,26 @@ public function test_join_left_on_date_time_entry(): void $loader->expects(self::exactly(2))->method('load'); $rows = df() - ->from(from_rows(rows( + ->from(from_rows(array_to_rows( + [ + ['id' => 1, 'date' => new DateTimeImmutable('2024-01-01 00:00:00')], + ['id' => 2, 'date' => new DateTimeImmutable('2024-01-01 00:00:00')], + ['id' => 3, 'date' => new DateTimeImmutable('2024-01-02 00:00:00')], + ['id' => 4, 'date' => new DateTimeImmutable('2024-01-03 00:00:00')], + ['id' => 5, 'date' => new DateTimeImmutable('2024-01-04 00:00:00')], + ['id' => 6, 'date' => new DateTimeImmutable('2024-01-04 00:00:00')], + ['id' => 7, 'date' => new DateTimeImmutable('2024-01-05 00:00:00')], + ['id' => 9, 'date' => new DateTimeImmutable('2024-01-05 00:00:00')], + ], schema(int_schema('id'), datetime_schema('date')), - row(['id' => 1, 'date' => new DateTimeImmutable('2024-01-01 00:00:00')]), - row(['id' => 2, 'date' => new DateTimeImmutable('2024-01-01 00:00:00')]), - row(['id' => 3, 'date' => new DateTimeImmutable('2024-01-02 00:00:00')]), - row(['id' => 4, 'date' => new DateTimeImmutable('2024-01-03 00:00:00')]), - row(['id' => 5, 'date' => new DateTimeImmutable('2024-01-04 00:00:00')]), - row(['id' => 6, 'date' => new DateTimeImmutable('2024-01-04 00:00:00')]), - row(['id' => 7, 'date' => new DateTimeImmutable('2024-01-05 00:00:00')]), - row(['id' => 9, 'date' => new DateTimeImmutable('2024-01-05 00:00:00')]), ))) ->join( - data_frame()->process(rows( + data_frame()->process(array_to_rows( + [ + ['date' => new DateTimeImmutable('2024-01-01 00:00:00'), 'events' => 1], + ['date' => new DateTimeImmutable('2024-01-05 00:00:00'), 'events' => 5], + ], schema(datetime_schema('date'), int_schema('events')), - row(['date' => new DateTimeImmutable('2024-01-01 00:00:00'), 'events' => 1]), - row(['date' => new DateTimeImmutable('2024-01-05 00:00:00'), 'events' => 5]), )), Expression::on(['date' => 'date'], 'joined_'), Join::left, @@ -334,22 +355,23 @@ public function test_join_left_with_empty_prefix(): void $loader->expects(self::exactly(2))->method('load'); $rows = df() - ->from(from_rows(rows( + ->from(from_rows(array_to_rows( + [ + ['id' => 1, 'country_code' => 'PL'], + ['id' => 2, 'country_code' => 'PL'], + ['id' => 3, 'country_code' => 'PL'], + ['id' => 4, 'country_code' => 'PL'], + ['id' => 5, 'country_code' => 'US'], + ['id' => 6, 'country_code' => 'US'], + ['id' => 7, 'country_code' => 'US'], + ['id' => 9, 'country_code' => 'US'], + ], schema(int_schema('id'), str_schema('country_code')), - row(['id' => 1, 'country_code' => 'PL']), - row(['id' => 2, 'country_code' => 'PL']), - row(['id' => 3, 'country_code' => 'PL']), - row(['id' => 4, 'country_code' => 'PL']), - row(['id' => 5, 'country_code' => 'US']), - row(['id' => 6, 'country_code' => 'US']), - row(['id' => 7, 'country_code' => 'US']), - row(['id' => 9, 'country_code' => 'US']), ))) ->join( - data_frame()->process(rows( + data_frame()->process(array_to_rows( + [['country_code' => 'PL', 'name' => 'Poland'], ['country_code' => 'US', 'name' => 'United States']], schema(str_schema('country_code'), str_schema('name')), - row(['country_code' => 'PL', 'name' => 'Poland']), - row(['country_code' => 'US', 'name' => 'United States']), )), join_on(['country_code' => 'country_code']), ) @@ -378,22 +400,23 @@ public function test_join_on_same_column_name(): void $loader->expects(self::exactly(2))->method('load'); $rows = df() - ->from(from_rows(rows( + ->from(from_rows(array_to_rows( + [ + ['id' => 1, 'code' => 'PL'], + ['id' => 2, 'code' => 'PL'], + ['id' => 3, 'code' => 'PL'], + ['id' => 4, 'code' => 'PL'], + ['id' => 5, 'code' => 'US'], + ['id' => 6, 'code' => 'US'], + ['id' => 7, 'code' => 'US'], + ['id' => 9, 'code' => 'US'], + ], schema(int_schema('id'), str_schema('code')), - row(['id' => 1, 'code' => 'PL']), - row(['id' => 2, 'code' => 'PL']), - row(['id' => 3, 'code' => 'PL']), - row(['id' => 4, 'code' => 'PL']), - row(['id' => 5, 'code' => 'US']), - row(['id' => 6, 'code' => 'US']), - row(['id' => 7, 'code' => 'US']), - row(['id' => 9, 'code' => 'US']), ))) ->join( - data_frame()->process(rows( + data_frame()->process(array_to_rows( + [['code' => 'PL', 'name' => 'Poland'], ['code' => 'US', 'name' => 'United States']], schema(str_schema('code'), str_schema('name')), - row(['code' => 'PL', 'name' => 'Poland']), - row(['code' => 'US', 'name' => 'United States']), )), Expression::on(['code' => 'code'], 'joined_'), ) @@ -422,21 +445,25 @@ public function test_join_right(): void $loader->expects(self::exactly(2))->method('load'); $rows = df() - ->from(from_rows(rows( + ->from(from_rows(array_to_rows( + [ + ['id' => 1, 'country' => 'PL'], + ['id' => 2, 'country' => 'US'], + ['id' => 3, 'country' => 'FR'], + ['id' => 4, 'country' => 'UK'], + ['id' => 5, 'country' => 'GB'], + ], schema(int_schema('id'), str_schema('country')), - row(['id' => 1, 'country' => 'PL']), - row(['id' => 2, 'country' => 'US']), - row(['id' => 3, 'country' => 'FR']), - row(['id' => 4, 'country' => 'UK']), - row(['id' => 5, 'country' => 'GB']), ))) ->join( - data_frame()->process(rows( + data_frame()->process(array_to_rows( + [ + ['code' => 'PL', 'name' => 'Poland'], + ['code' => 'US', 'name' => 'United States'], + ['code' => 'FR', 'name' => 'France'], + ['code' => 'CA', 'name' => 'Canada'], + ], schema(str_schema('code'), str_schema('name')), - row(['code' => 'PL', 'name' => 'Poland']), - row(['code' => 'US', 'name' => 'United States']), - row(['code' => 'FR', 'name' => 'France']), - row(['code' => 'CA', 'name' => 'Canada']), )), Expression::on(['country' => 'code'], 'joined_'), Join::right, @@ -462,21 +489,25 @@ public function test_join_right_without_prefix(): void $loader->expects(self::exactly(2))->method('load'); $rows = df() - ->from(from_rows(rows( + ->from(from_rows(array_to_rows( + [ + ['id' => 1, 'country_code' => 'PL'], + ['id' => 2, 'country_code' => 'US'], + ['id' => 3, 'country_code' => 'FR'], + ['id' => 4, 'country_code' => 'UK'], + ['id' => 5, 'country_code' => 'GB'], + ], schema(int_schema('id'), str_schema('country_code')), - row(['id' => 1, 'country_code' => 'PL']), - row(['id' => 2, 'country_code' => 'US']), - row(['id' => 3, 'country_code' => 'FR']), - row(['id' => 4, 'country_code' => 'UK']), - row(['id' => 5, 'country_code' => 'GB']), ))) ->join( - data_frame()->process(rows( + data_frame()->process(array_to_rows( + [ + ['country_code' => 'PL', 'name' => 'Poland'], + ['country_code' => 'US', 'name' => 'United States'], + ['country_code' => 'FR', 'name' => 'France'], + ['country_code' => 'CA', 'name' => 'Canada'], + ], schema(str_schema('country_code'), str_schema('name')), - row(['country_code' => 'PL', 'name' => 'Poland']), - row(['country_code' => 'US', 'name' => 'United States']), - row(['country_code' => 'FR', 'name' => 'France']), - row(['country_code' => 'CA', 'name' => 'Canada']), )), Expression::on(['country_code' => 'country_code']), Join::right, @@ -495,4 +526,20 @@ public function test_join_right_without_prefix(): void $rows->toArray(), ); } + + public function test_a_date_key_meets_an_equal_datetime_key(): void + { + $rows = df() + ->read(from_rows(array_to_rows([['k' => new DateTimeImmutable('2026-01-02')]], schema(date_schema('k'))))) + ->join( + df()->read(from_rows(array_to_rows([[ + 'r' => new DateTimeImmutable('2026-01-02 00:00:00 UTC'), + ]], schema(datetime_schema('r'))))), + join_on(['k' => 'r']), + Join::inner, + ) + ->fetch(); + + static::assertCount(1, $rows); + } } diff --git a/src/core/etl/tests/Flow/ETL/Tests/Integration/DataFrame/LimitTest.php b/src/core/etl/tests/Flow/ETL/Tests/Integration/DataFrame/LimitTest.php index 4d3a548d51..9e9e1003c1 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Integration/DataFrame/LimitTest.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Integration/DataFrame/LimitTest.php @@ -23,6 +23,7 @@ use function array_column; use function array_map; use function array_slice; +use function Flow\ETL\DSL\array_to_rows; use function Flow\ETL\DSL\df; use function Flow\ETL\DSL\from_array; use function Flow\ETL\DSL\from_data_frame; @@ -32,7 +33,6 @@ use function Flow\ETL\DSL\list_schema; use function Flow\ETL\DSL\lit; use function Flow\ETL\DSL\ref; -use function Flow\ETL\DSL\row; use function Flow\ETL\DSL\rows; use function Flow\ETL\DSL\schema; use function Flow\Types\DSL\type_integer; @@ -103,7 +103,7 @@ public function schema(): Schema public function extract(FlowContext $context, ?int $limit = null): Generator { for ($i = 0; $i < 20; $i++) { - yield rows(schema(integer_schema('id')), row(['id' => $i])); + yield array_to_rows([['id' => $i]], schema(integer_schema('id'))); } } @@ -138,7 +138,7 @@ public function schema(): Schema public function extract(FlowContext $context, ?int $limit = null): Generator { for ($i = 0; $i < 1000; $i++) { - yield rows(schema(integer_schema('id')), row(['id' => $i + 1]), row(['id' => $i + 2])); + yield array_to_rows([['id' => $i + 1], ['id' => $i + 2]], schema(integer_schema('id'))); } } @@ -199,22 +199,16 @@ public function schema(): Schema public function extract(FlowContext $context, ?int $limit = null): Generator { for ($i = 0; $i < 1000; $i++) { - yield rows( - schema(list_schema( - 'ids', - type_list(type_structure([ - 'id' => type_integer(), - 'more_ids' => type_list(type_map(type_string(), type_integer())), - ])), - )), - row([ - 'ids' => [ - ['id' => $i + 1, 'more_ids' => [['more_id' => $i + 4], ['more_id' => $i + 7]]], - ['id' => $i + 2, 'more_ids' => [['more_id' => $i + 5], ['more_id' => $i + 8]]], - ['id' => $i + 3, 'more_ids' => [['more_id' => $i + 6], ['more_id' => $i + 9]]], - ], - ]), - ); + yield array_to_rows([[ + 'ids' => [ + ['id' => $i + 1, 'more_ids' => [['more_id' => $i + 4], ['more_id' => $i + 7]]], + ['id' => $i + 2, 'more_ids' => [['more_id' => $i + 5], ['more_id' => $i + 8]]], + ['id' => $i + 3, 'more_ids' => [['more_id' => $i + 6], ['more_id' => $i + 9]]], + ], + ]], schema(list_schema('ids', type_list(type_structure([ + 'id' => type_integer(), + 'more_ids' => type_list(type_map(type_string(), type_integer())), + ]))))); } } @@ -261,7 +255,7 @@ public function schema(): Schema public function extract(FlowContext $context, ?int $limit = null): Generator { for ($i = 0; $i < 1000; $i++) { - yield rows(schema(integer_schema('id')), row(['id' => $i + 1]), row(['id' => $i + 2])); + yield array_to_rows([['id' => $i + 1], ['id' => $i + 2]], schema(integer_schema('id'))); } } @@ -299,7 +293,7 @@ public function schema(): Schema public function extract(FlowContext $context, ?int $limit = null): Generator { for ($i = 0; $i < 100; $i++) { - yield rows(schema(integer_schema('id')), row(['id' => $i + 1]), row(['id' => $i + 2])); + yield array_to_rows([['id' => $i + 1], ['id' => $i + 2]], schema(integer_schema('id'))); } } @@ -337,7 +331,7 @@ public function schema(): Schema public function extract(FlowContext $context, ?int $limit = null): Generator { for ($i = 0; $i < 5; $i++) { - yield rows(schema(integer_schema('id')), row(['id' => $i])); + yield array_to_rows([['id' => $i]], schema(integer_schema('id'))); } } @@ -443,7 +437,7 @@ public function test_a_limit_over_a_read_frame_reaches_that_frames_source(bool $ { $source = new RecordingFileExtractor( schema(int_schema('id')), - rows(schema(int_schema('id')), ...array_map(static fn(int $id) => row(['id' => $id]), range(1, 6))), + array_to_rows(array_map(static fn(int $id) => ['id' => $id], range(1, 6)), schema(int_schema('id'))), ); $nested = from_data_frame(df()->read($source)); diff --git a/src/core/etl/tests/Flow/ETL/Tests/Integration/DataFrame/MathTest.php b/src/core/etl/tests/Flow/ETL/Tests/Integration/DataFrame/MathTest.php index c345d249d8..d2dbf89cca 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Integration/DataFrame/MathTest.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Integration/DataFrame/MathTest.php @@ -4,15 +4,16 @@ namespace Flow\ETL\Tests\Integration\DataFrame; +use Flow\ETL\Column\PhpBackend; use Flow\ETL\Rows; use Flow\ETL\Tests\FlowTestCase; +use function Flow\ETL\DSL\array_to_rows; use function Flow\ETL\DSL\df; use function Flow\ETL\DSL\float_schema; use function Flow\ETL\DSL\from_rows; use function Flow\ETL\DSL\integer_schema; use function Flow\ETL\DSL\ref; -use function Flow\ETL\DSL\row; use function Flow\ETL\DSL\rows; use function Flow\ETL\DSL\schema; use function Flow\ETL\DSL\sum; @@ -23,22 +24,24 @@ public function test_aggregations_on_floats(): void { $rows = rows(schema()); df() - ->read(from_rows(rows( + ->read(from_rows(array_to_rows( + [ + ['id' => 1, 'price' => 29.39, 'quantity' => 2, 'weight' => 1.5], + ['id' => 2, 'price' => 19.3, 'quantity' => 1, 'weight' => 0.5], + ['id' => 3, 'price' => 39.1, 'quantity' => 3, 'weight' => 2.0], + ['id' => 4, 'price' => 49.9, 'quantity' => 4, 'weight' => 2.1284], + ['id' => 5, 'price' => 15.0, 'quantity' => 1, 'weight' => 0.3], + ['id' => 6, 'price' => 30.0, 'quantity' => 2, 'weight' => 1.0], + ['id' => 7, 'price' => 25.0, 'quantity' => 3, 'weight' => 1.8], + ['id' => 8, 'price' => 20.0, 'quantity' => 1, 'weight' => 0.8], + ['id' => 9, 'price' => 35.0, 'quantity' => 4, 'weight' => 2.2], + ['id' => 10, 'price' => 45.0, 'quantity' => 5, 'weight' => 3.0], + ], schema(integer_schema('id'), float_schema('price'), integer_schema('quantity'), float_schema('weight')), - row(['id' => 1, 'price' => 29.39, 'quantity' => 2, 'weight' => 1.5]), - row(['id' => 2, 'price' => 19.3, 'quantity' => 1, 'weight' => 0.5]), - row(['id' => 3, 'price' => 39.1, 'quantity' => 3, 'weight' => 2.0]), - row(['id' => 4, 'price' => 49.9, 'quantity' => 4, 'weight' => 2.1284]), - row(['id' => 5, 'price' => 15.0, 'quantity' => 1, 'weight' => 0.3]), - row(['id' => 6, 'price' => 30.0, 'quantity' => 2, 'weight' => 1.0]), - row(['id' => 7, 'price' => 25.0, 'quantity' => 3, 'weight' => 1.8]), - row(['id' => 8, 'price' => 20.0, 'quantity' => 1, 'weight' => 0.8]), - row(['id' => 9, 'price' => 35.0, 'quantity' => 4, 'weight' => 2.2]), - row(['id' => 10, 'price' => 45.0, 'quantity' => 5, 'weight' => 3.0]), ))) ->aggregate([sum(ref('price')), sum(ref('weight'))]) ->forEach(static function (Rows $r) use (&$rows): void { - $rows = $rows->merge($r); + $rows = $rows->isEmpty() ? $r : $rows->concat(new PhpBackend(), $r); }); static::assertSame( @@ -53,23 +56,25 @@ public function test_mathematical_operations_on_floats(): void { $rows = rows(schema()); df() - ->read(from_rows(rows( + ->read(from_rows(array_to_rows( + [ + ['id' => 1, 'price' => 29.39, 'quantity' => 2, 'weight' => 1.5], + ['id' => 2, 'price' => 19.3, 'quantity' => 1, 'weight' => 0.5], + ['id' => 3, 'price' => 39.1, 'quantity' => 3, 'weight' => 2.0], + ['id' => 4, 'price' => 49.9, 'quantity' => 4, 'weight' => 2.1284], + ['id' => 5, 'price' => 15.0, 'quantity' => 1, 'weight' => 0.3], + ['id' => 6, 'price' => 30.0, 'quantity' => 2, 'weight' => 1.0], + ['id' => 7, 'price' => 25.0, 'quantity' => 3, 'weight' => 1.8], + ['id' => 8, 'price' => 20.0, 'quantity' => 1, 'weight' => 0.8], + ['id' => 9, 'price' => 35.0, 'quantity' => 4, 'weight' => 2.2], + ['id' => 10, 'price' => 45.0, 'quantity' => 5, 'weight' => 3.0], + ], schema(integer_schema('id'), float_schema('price'), integer_schema('quantity'), float_schema('weight')), - row(['id' => 1, 'price' => 29.39, 'quantity' => 2, 'weight' => 1.5]), - row(['id' => 2, 'price' => 19.3, 'quantity' => 1, 'weight' => 0.5]), - row(['id' => 3, 'price' => 39.1, 'quantity' => 3, 'weight' => 2.0]), - row(['id' => 4, 'price' => 49.9, 'quantity' => 4, 'weight' => 2.1284]), - row(['id' => 5, 'price' => 15.0, 'quantity' => 1, 'weight' => 0.3]), - row(['id' => 6, 'price' => 30.0, 'quantity' => 2, 'weight' => 1.0]), - row(['id' => 7, 'price' => 25.0, 'quantity' => 3, 'weight' => 1.8]), - row(['id' => 8, 'price' => 20.0, 'quantity' => 1, 'weight' => 0.8]), - row(['id' => 9, 'price' => 35.0, 'quantity' => 4, 'weight' => 2.2]), - row(['id' => 10, 'price' => 45.0, 'quantity' => 5, 'weight' => 3.0]), ))) - ->withEntry('discount', ref('price')->multiply(-0.1)) - ->withEntry('total_weight', ref('weight')->multiply(ref('quantity'))) + ->withEntry('discount', ref('price')->multiply(-0.1, exact: true)) + ->withEntry('total_weight', ref('weight')->multiply(ref('quantity'), exact: true)) ->forEach(static function (Rows $r) use (&$rows): void { - $rows = $rows->merge($r); + $rows = $rows->isEmpty() ? $r : $rows->concat(new PhpBackend(), $r); }); static::assertEquals( diff --git a/src/core/etl/tests/Flow/ETL/Tests/Integration/DataFrame/OffsetTest.php b/src/core/etl/tests/Flow/ETL/Tests/Integration/DataFrame/OffsetTest.php index e9fb7349ff..914325adce 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Integration/DataFrame/OffsetTest.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Integration/DataFrame/OffsetTest.php @@ -15,6 +15,7 @@ use Generator; use function array_map; +use function Flow\ETL\DSL\array_to_rows; use function Flow\ETL\DSL\df; use function Flow\ETL\DSL\from_array; use function Flow\ETL\DSL\from_rows; @@ -22,7 +23,6 @@ use function Flow\ETL\DSL\integer_schema; use function Flow\ETL\DSL\list_schema; use function Flow\ETL\DSL\ref; -use function Flow\ETL\DSL\row; use function Flow\ETL\DSL\rows; use function Flow\ETL\DSL\schema; use function Flow\Types\DSL\type_integer; @@ -139,7 +139,7 @@ public function schema(): Schema public function extract(FlowContext $context, ?int $limit = null): Generator { for ($i = 0; $i < 10; $i++) { - yield rows(schema(integer_schema('id')), row(['id' => $i + 1])); + yield array_to_rows([['id' => $i + 1]], schema(integer_schema('id'))); } } @@ -187,7 +187,7 @@ public function schema(): Schema public function extract(FlowContext $context, ?int $limit = null): Generator { for ($i = 0; $i < 5; $i++) { - yield rows(schema(integer_schema('id')), row(['id' => $i + 1])); + yield array_to_rows([['id' => $i + 1]], schema(integer_schema('id'))); } } @@ -237,21 +237,15 @@ public function schema(): Schema public function extract(FlowContext $context, ?int $limit = null): Generator { for ($i = 0; $i < 100; $i++) { - yield rows( - schema(list_schema( - 'ids', - type_list(type_structure([ - 'id' => type_integer(), - ])), - )), - row([ - 'ids' => [ - ['id' => $i + 1], - ['id' => $i + 2], - ['id' => $i + 3], - ], - ]), - ); + yield array_to_rows([[ + 'ids' => [ + ['id' => $i + 1], + ['id' => $i + 2], + ['id' => $i + 3], + ], + ]], schema(list_schema('ids', type_list(type_structure([ + 'id' => type_integer(), + ]))))); } } @@ -319,7 +313,7 @@ public function schema(): Schema public function extract(FlowContext $context, ?int $limit = null): Generator { for ($i = 0; $i < 10; $i++) { - yield rows(schema(integer_schema('id')), row(['id' => $i + 1])); + yield array_to_rows([['id' => $i + 1]], schema(integer_schema('id'))); } } @@ -392,11 +386,11 @@ public function test_pagination_scenario(): void $data = array_map(static fn(int $id): array => ['id' => $id, 'name' => 'Item ' . $id], range(1, 100)); $page1 = df()->read(from_array($data))->offset(0)->limit(10)->fetch(); static::assertCount(10, $page1); - static::assertSame(1, $page1->first()->get('id')); - static::assertSame(10, $page1->all()[9]->get('id')); + static::assertSame(1, $page1->column('id')->value(0)); + static::assertSame(10, $page1->column('id')->value(9)); $page3 = df()->read(from_array($data))->offset(20)->limit(10)->fetch(); static::assertCount(10, $page3); - static::assertSame(21, $page3->first()->get('id')); - static::assertSame(30, $page3->all()[9]->get('id')); + static::assertSame(21, $page3->column('id')->value(0)); + static::assertSame(30, $page3->column('id')->value(9)); } } diff --git a/src/core/etl/tests/Flow/ETL/Tests/Integration/DataFrame/PartitioningTest.php b/src/core/etl/tests/Flow/ETL/Tests/Integration/DataFrame/PartitioningTest.php index a563a6cdf9..421acac53b 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Integration/DataFrame/PartitioningTest.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Integration/DataFrame/PartitioningTest.php @@ -19,6 +19,7 @@ use function file_exists; use function Flow\ETL\Adapter\Text\from_text; use function Flow\ETL\Adapter\Text\to_text; +use function Flow\ETL\DSL\array_to_rows; use function Flow\ETL\DSL\collect; use function Flow\ETL\DSL\df; use function Flow\ETL\DSL\files; @@ -31,8 +32,6 @@ use function Flow\ETL\DSL\partition_by; use function Flow\ETL\DSL\partition_types; use function Flow\ETL\DSL\ref; -use function Flow\ETL\DSL\row; -use function Flow\ETL\DSL\rows; use function Flow\ETL\DSL\schema; use function Flow\ETL\DSL\str_schema; use function Flow\Types\DSL\type_integer; @@ -164,7 +163,7 @@ public function test_overwrite_save_mode_not_dropping_old_partitions(): void [ 'path' => 'file://' - . ltrim(str_replace('\\', '/', __DIR__), '/') + . str_replace('\\', '/', __DIR__) . '/Fixtures/Partitioning/overwrite/date=2024-04-01/file.txt', 'partitions' => ['date' => '2024-04-01'], 'date' => '2024-04-01', @@ -172,7 +171,7 @@ public function test_overwrite_save_mode_not_dropping_old_partitions(): void [ 'path' => 'file://' - . ltrim(str_replace('\\', '/', __DIR__), '/') + . str_replace('\\', '/', __DIR__) . '/Fixtures/Partitioning/overwrite/date=2024-04-02/file.txt', 'partitions' => ['date' => '2024-04-02'], 'date' => '2024-04-02', @@ -180,7 +179,7 @@ public function test_overwrite_save_mode_not_dropping_old_partitions(): void [ 'path' => 'file://' - . ltrim(str_replace('\\', '/', __DIR__), '/') + . str_replace('\\', '/', __DIR__) . '/Fixtures/Partitioning/overwrite/date=2024-04-03/file.txt', 'partitions' => ['date' => '2024-04-03'], 'date' => '2024-04-03', @@ -188,7 +187,7 @@ public function test_overwrite_save_mode_not_dropping_old_partitions(): void [ 'path' => 'file://' - . ltrim(str_replace('\\', '/', __DIR__), '/') + . str_replace('\\', '/', __DIR__) . '/Fixtures/Partitioning/overwrite/date=2024-04-04/file.txt', 'partitions' => ['date' => '2024-04-04'], 'date' => '2024-04-04', @@ -215,13 +214,15 @@ public function test_overwrite_save_mode_not_dropping_old_partitions(): void public function test_repartition_groups_every_row_sharing_a_key_into_one_batch(): void { $batches = df() - ->read(from_rows(rows( + ->read(from_rows(array_to_rows( + [ + ['id' => 1, 'country' => 'PL', 'age' => 20], + ['id' => 5, 'country' => 'US', 'age' => 40], + ['id' => 2, 'country' => 'PL', 'age' => 20], + ['id' => 6, 'country' => 'US', 'age' => 40], + ['id' => 3, 'country' => 'PL', 'age' => 25], + ], schema(int_schema('id'), str_schema('country'), int_schema('age')), - row(['id' => 1, 'country' => 'PL', 'age' => 20]), - row(['id' => 5, 'country' => 'US', 'age' => 40]), - row(['id' => 2, 'country' => 'PL', 'age' => 20]), - row(['id' => 6, 'country' => 'US', 'age' => 40]), - row(['id' => 3, 'country' => 'PL', 'age' => 25]), ))) ->repartition(ref('country')) ->get(); @@ -369,6 +370,6 @@ public function test_pruning_single_partition(): void ->fetch(); static::assertCount(1, $rows); - static::assertSame(2023, $rows->first()->get('year')); + static::assertSame(2023, $rows->column('year')->value(0)); } } diff --git a/src/core/etl/tests/Flow/ETL/Tests/Integration/DataFrame/PlanBindTest.php b/src/core/etl/tests/Flow/ETL/Tests/Integration/DataFrame/PlanBindTest.php index 79dd806e45..40b8745285 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Integration/DataFrame/PlanBindTest.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Integration/DataFrame/PlanBindTest.php @@ -19,6 +19,7 @@ use Throwable; use function Flow\ETL\DSL\array_get; +use function Flow\ETL\DSL\array_to_rows; use function Flow\ETL\DSL\datetime_schema; use function Flow\ETL\DSL\df; use function Flow\ETL\DSL\discover_pivot_values; @@ -32,7 +33,6 @@ use function Flow\ETL\DSL\rank; use function Flow\ETL\DSL\ref; use function Flow\ETL\DSL\rename_replace; -use function Flow\ETL\DSL\row; use function Flow\ETL\DSL\rows; use function Flow\ETL\DSL\schema; use function Flow\ETL\DSL\str_schema; @@ -50,8 +50,8 @@ final class PlanBindTest extends FlowTestCase ]; /** - * One plan per bind class of the step table. The two exclusions the plan names - a refused bind - * (joinEach) and the BucketingProcessor metadata hop - are covered by their own tests. + * One plan per bind class of the step table. The exclusion the plan names - a refused bind (joinEach) - is + * covered by its own tests. * * @return Generator */ @@ -330,11 +330,10 @@ public function test_discovered_pivot_values_scan_the_source_once_at_build(): vo { $extractor = new CountingExtractor( $input = schema(str_schema('product'), str_schema('country'), int_schema('amount')), - rows( - $input, - row(['product' => 'Banana', 'country' => 'USA', 'amount' => 1000]), - row(['product' => 'Banana', 'country' => 'China', 'amount' => 400]), - ), + array_to_rows([ + ['product' => 'Banana', 'country' => 'USA', 'amount' => 1000], + ['product' => 'Banana', 'country' => 'China', 'amount' => 400], + ], $input), ); $schema = df() diff --git a/src/core/etl/tests/Flow/ETL/Tests/Integration/DataFrame/RenameTest.php b/src/core/etl/tests/Flow/ETL/Tests/Integration/DataFrame/RenameTest.php index 0d5f4465f2..452c86520d 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Integration/DataFrame/RenameTest.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Integration/DataFrame/RenameTest.php @@ -8,6 +8,7 @@ use Flow\ETL\String\StringStyles; use Flow\ETL\Tests\FlowIntegrationTestCase; +use function Flow\ETL\DSL\array_to_rows; use function Flow\ETL\DSL\bool_schema; use function Flow\ETL\DSL\df; use function Flow\ETL\DSL\from_rows; @@ -18,8 +19,6 @@ use function Flow\ETL\DSL\rename_map; use function Flow\ETL\DSL\rename_replace; use function Flow\ETL\DSL\rename_style; -use function Flow\ETL\DSL\row; -use function Flow\ETL\DSL\rows; use function Flow\ETL\DSL\schema; use function Flow\ETL\DSL\str_schema; use function Flow\Types\DSL\type_json; @@ -30,21 +29,25 @@ final class RenameTest extends FlowIntegrationTestCase public function test_rename(): void { $rows = df() - ->read(from_rows(rows( + ->read(from_rows(array_to_rows( + [ + ['id' => 1, 'name' => 'foo', 'active' => true], + ['id' => 2, 'name' => null, 'active' => false], + ['id' => 2, 'name' => 'bar', 'active' => false], + ], schema(int_schema('id'), str_schema('name', nullable: true), bool_schema('active')), - row(['id' => 1, 'name' => 'foo', 'active' => true]), - row(['id' => 2, 'name' => null, 'active' => false]), - row(['id' => 2, 'name' => 'bar', 'active' => false]), ))) ->rename('name', 'new_name') ->fetch(); static::assertEquals( - rows( + array_to_rows( + [ + ['id' => 1, 'new_name' => 'foo', 'active' => true], + ['id' => 2, 'new_name' => null, 'active' => false], + ['id' => 2, 'new_name' => 'bar', 'active' => false], + ], schema(int_schema('id'), str_schema('new_name', nullable: true), bool_schema('active')), - row(['id' => 1, 'new_name' => 'foo', 'active' => true]), - row(['id' => 2, 'new_name' => null, 'active' => false]), - row(['id' => 2, 'new_name' => 'bar', 'active' => false]), ), $rows, ); @@ -53,17 +56,17 @@ public function test_rename(): void public function test_rename_each_with_numeric_entry_names(): void { $rows = df() - ->read(from_rows(rows( + ->read(from_rows(array_to_rows( + [['broker' => 'acme', '2023' => 1, '2024' => 2]], schema(str_schema('broker'), int_schema('2023'), int_schema('2024')), - row(['broker' => 'acme', '2023' => 1, '2024' => 2]), ))) ->renameEach(rename_map(['2023' => 'y_2023', '2024' => 'y_2024'])) ->fetch(); static::assertEquals( - rows( + array_to_rows( + [['broker' => 'acme', 'y_2023' => 1, 'y_2024' => 2]], schema(str_schema('broker'), int_schema('y_2023'), int_schema('y_2024')), - row(['broker' => 'acme', 'y_2023' => 1, 'y_2024' => 2]), ), $rows, ); @@ -71,11 +74,10 @@ public function test_rename_each_with_numeric_entry_names(): void public function test_rename_all(): void { - $rows = rows( - schema(json_schema('array')), - row(['array' => type_json()->cast(['id' => 1, 'name' => 'name', 'active' => true])]), - row(['array' => type_json()->cast(['id' => 2, 'name' => 'name', 'active' => false])]), - ); + $rows = array_to_rows([ + ['array' => type_json()->cast(['id' => 1, 'name' => 'name', 'active' => true])], + ['array' => type_json()->cast(['id' => 2, 'name' => 'name', 'active' => false])], + ], schema(json_schema('array'))); $ds = df() ->read(from_rows($rows)) @@ -98,10 +100,9 @@ public function test_rename_all(): void public function test_rename_all_lower_case(): void { - $rows = rows( + $rows = array_to_rows( + [['ID' => 1, 'NAME' => 'name', 'ACTIVE' => true], ['ID' => 2, 'NAME' => 'name', 'ACTIVE' => false]], schema(int_schema('ID'), str_schema('NAME'), bool_schema('ACTIVE')), - row(['ID' => 1, 'NAME' => 'name', 'ACTIVE' => true]), - row(['ID' => 2, 'NAME' => 'name', 'ACTIVE' => false]), ); $ds = df()->read(from_rows($rows))->renameEach(rename_style(StringStyles::LOWER))->getEachAsArray(); @@ -117,11 +118,10 @@ public function test_rename_all_lower_case(): void public function test_rename_all_lower_case_i18n(): void { - $rows = rows( - schema(int_schema('ILOŚĆ PRZEDMIOTÓW')), - row(['ILOŚĆ PRZEDMIOTÓW' => 0]), - row(['ILOŚĆ PRZEDMIOTÓW' => 10]), - ); + $rows = array_to_rows([ + ['ILOŚĆ PRZEDMIOTÓW' => 0], + ['ILOŚĆ PRZEDMIOTÓW' => 10], + ], schema(int_schema('ILOŚĆ PRZEDMIOTÓW'))); $ds = df()->read(from_rows($rows))->renameEach(rename_style(StringStyles::LOWER))->getEachAsArray(); @@ -136,11 +136,10 @@ public function test_rename_all_lower_case_i18n(): void public function test_rename_all_multiple(): void { - $rows = rows( - schema(json_schema('array')), - row(['array' => type_json()->cast(['id' => 1, 'name' => 'name', 'isActive' => true])]), - row(['array' => type_json()->cast(['id' => 2, 'name' => 'name', 'isActive' => false])]), - ); + $rows = array_to_rows([ + ['array' => type_json()->cast(['id' => 1, 'name' => 'name', 'isActive' => true])], + ['array' => type_json()->cast(['id' => 2, 'name' => 'name', 'isActive' => false])], + ], schema(json_schema('array'))); $ds = df() ->read(from_rows($rows)) @@ -163,10 +162,9 @@ public function test_rename_all_multiple(): void public function test_rename_all_to_ascii(): void { - $rows = rows( + $rows = array_to_rows( + [['ÓSMY' => 8], ['DZIEWIĄTY' => 9]], schema(int_schema('ÓSMY', nullable: true), int_schema('DZIEWIĄTY', nullable: true)), - row(['ÓSMY' => 8]), - row(['DZIEWIĄTY' => 9]), ); $ds = df()->read(from_rows($rows))->renameEach(rename_style(StringStyles::ASCII))->getEachAsArray(); @@ -182,7 +180,7 @@ public function test_rename_all_to_ascii(): void public function test_rename_all_to_camel(): void { - $rows = rows(schema(int_schema('ósmy i dziewiąty')), row(['ósmy i dziewiąty' => 89])); + $rows = array_to_rows([['ósmy i dziewiąty' => 89]], schema(int_schema('ósmy i dziewiąty'))); $ds = df()->read(from_rows($rows))->renameEach(rename_style(StringStyles::CAMEL))->getEachAsArray(); @@ -196,7 +194,7 @@ public function test_rename_all_to_camel(): void public function test_rename_all_to_slug(): void { - $rows = rows(schema(int_schema('ÓSMY I DZIEWIĄTY')), row(['ÓSMY I DZIEWIĄTY' => 89])); + $rows = array_to_rows([['ÓSMY I DZIEWIĄTY' => 89]], schema(int_schema('ÓSMY I DZIEWIĄTY'))); $ds = df() ->read(from_rows($rows)) @@ -214,10 +212,12 @@ public function test_rename_all_to_slug(): void public function test_rename_all_to_snake_case(): void { - $rows = rows( + $rows = array_to_rows( + [ + ['id' => 1, 'UserName' => 'name', 'isActive' => true], + ['id' => 2, 'UserName' => 'name', 'isActive' => false], + ], schema(int_schema('id'), str_schema('UserName'), bool_schema('isActive')), - row(['id' => 1, 'UserName' => 'name', 'isActive' => true]), - row(['id' => 2, 'UserName' => 'name', 'isActive' => false]), ); $ds = df() @@ -237,7 +237,7 @@ public function test_rename_all_to_snake_case(): void public function test_rename_all_to_title(): void { - $rows = rows(schema(int_schema('ósmy i dziewiąty')), row(['ósmy i dziewiąty' => 89])); + $rows = array_to_rows([['ósmy i dziewiąty' => 89]], schema(int_schema('ósmy i dziewiąty'))); $ds = df()->read(from_rows($rows))->renameEach(rename_style(StringStyles::TITLE))->getEachAsArray(); @@ -251,10 +251,9 @@ public function test_rename_all_to_title(): void public function test_rename_all_upper_case(): void { - $rows = rows( + $rows = array_to_rows( + [['id' => 1, 'name' => 'name', 'active' => true], ['id' => 2, 'name' => 'name', 'active' => false]], schema(int_schema('id'), str_schema('name'), bool_schema('active')), - row(['id' => 1, 'name' => 'name', 'active' => true]), - row(['id' => 2, 'name' => 'name', 'active' => false]), ); $ds = df()->read(from_rows($rows))->renameEach(rename_style(StringStyles::UPPER))->getEachAsArray(); @@ -270,10 +269,9 @@ public function test_rename_all_upper_case(): void public function test_rename_all_upper_case_first(): void { - $rows = rows( + $rows = array_to_rows( + [['id' => 1, 'name' => 'name', 'active' => true], ['id' => 2, 'name' => 'name', 'active' => false]], schema(int_schema('id'), str_schema('name'), bool_schema('active')), - row(['id' => 1, 'name' => 'name', 'active' => true]), - row(['id' => 2, 'name' => 'name', 'active' => false]), ); $ds = df()->read(from_rows($rows))->renameEach(rename_style(StringStyles::UCFIRST))->getEachAsArray(); @@ -289,10 +287,9 @@ public function test_rename_all_upper_case_first(): void public function test_rename_all_upper_case_word(): void { - $rows = rows( + $rows = array_to_rows( + [['id' => 1, 'name' => 'name', 'active' => true], ['id' => 2, 'name' => 'name', 'active' => false]], schema(int_schema('id'), str_schema('name'), bool_schema('active')), - row(['id' => 1, 'name' => 'name', 'active' => true]), - row(['id' => 2, 'name' => 'name', 'active' => false]), ); $ds = df()->read(from_rows($rows))->renameEach(rename_style(StringStyles::UCWORDS))->getEachAsArray(); @@ -308,10 +305,9 @@ public function test_rename_all_upper_case_word(): void public function test_rename_all_upper_case_word_i18n(): void { - $rows = rows( + $rows = array_to_rows( + [['ósmy' => 8], ['dziewiąty' => 9]], schema(int_schema('ósmy', nullable: true), int_schema('dziewiąty', nullable: true)), - row(['ósmy' => 8]), - row(['dziewiąty' => 9]), ); $ds = df()->read(from_rows($rows))->renameEach(rename_style(StringStyles::UCWORDS))->getEachAsArray(); @@ -328,12 +324,15 @@ public function test_rename_all_upper_case_word_i18n(): void public function test_rename_each_with_empty_map(): void { $rows = df() - ->read(from_rows(rows(schema(int_schema('id'), str_schema('name')), row(['id' => 1, 'name' => 'foo'])))) + ->read(from_rows(array_to_rows( + [['id' => 1, 'name' => 'foo']], + schema(int_schema('id'), str_schema('name')), + ))) ->renameEach(rename_map([])) ->fetch(); static::assertEquals( - rows(schema(int_schema('id'), str_schema('name')), row(['id' => 1, 'name' => 'foo'])), + array_to_rows([['id' => 1, 'name' => 'foo']], schema(int_schema('id'), str_schema('name'))), $rows, ); } @@ -341,10 +340,12 @@ public function test_rename_each_with_empty_map(): void public function test_rename_each_with_map_chained_with_other_operations(): void { $ds = df() - ->read(from_rows(rows( + ->read(from_rows(array_to_rows( + [ + ['user_id' => 1, 'user_name' => 'John', 'is_active' => true], + ['user_id' => 2, 'user_name' => 'Jane', 'is_active' => false], + ], schema(int_schema('user_id'), str_schema('user_name'), bool_schema('is_active')), - row(['user_id' => 1, 'user_name' => 'John', 'is_active' => true]), - row(['user_id' => 2, 'user_name' => 'Jane', 'is_active' => false]), ))) ->renameEach(rename_map(['user_id' => 'id', 'user_name' => 'name'])) ->filter(ref('is_active')->equals(lit(true))) @@ -362,10 +363,12 @@ public function test_rename_each_with_map_chained_with_other_operations(): void public function test_rename_each_with_map_multiple_entries(): void { $rows = df() - ->read(from_rows(rows( + ->read(from_rows(array_to_rows( + [ + ['id' => 1, 'first_name' => 'John', 'last_name' => 'Doe'], + ['id' => 2, 'first_name' => 'Jane', 'last_name' => 'Smith'], + ], schema(int_schema('id'), str_schema('first_name'), str_schema('last_name')), - row(['id' => 1, 'first_name' => 'John', 'last_name' => 'Doe']), - row(['id' => 2, 'first_name' => 'Jane', 'last_name' => 'Smith']), ))) ->renameEach(rename_map([ 'first_name' => 'name', @@ -374,10 +377,12 @@ public function test_rename_each_with_map_multiple_entries(): void ->fetch(); static::assertEquals( - rows( + array_to_rows( + [ + ['id' => 1, 'name' => 'John', 'surname' => 'Doe'], + ['id' => 2, 'name' => 'Jane', 'surname' => 'Smith'], + ], schema(int_schema('id'), str_schema('name'), str_schema('surname')), - row(['id' => 1, 'name' => 'John', 'surname' => 'Doe']), - row(['id' => 2, 'name' => 'Jane', 'surname' => 'Smith']), ), $rows, ); @@ -388,7 +393,9 @@ public function test_rename_each_with_map_preserves_metadata(): void $metadata = Metadata::fromArray(['description' => 'test metadata']); $rows = df() - ->read(from_rows(rows(schema(str_schema('old_name', metadata: $metadata)), row(['old_name' => 'value'])))) + ->read(from_rows(array_to_rows([[ + 'old_name' => 'value', + ]], schema(str_schema('old_name', metadata: $metadata))))) ->renameEach(rename_map(['old_name' => 'new_name'])) ->fetch(); @@ -397,15 +404,13 @@ public function test_rename_each_with_map_preserves_metadata(): void public function test_rename_each_with_multiple_strategies(): void { - $rows = rows( + $rows = array_to_rows( + [['ÓSMY' => 8], ['DZIEWIĄTY' => 9], ['ÓSMY I DZIEWIĄTY' => 89]], schema( int_schema('ÓSMY', nullable: true), int_schema('DZIEWIĄTY', nullable: true), int_schema('ÓSMY I DZIEWIĄTY', nullable: true), ), - row(['ÓSMY' => 8]), - row(['DZIEWIĄTY' => 9]), - row(['ÓSMY I DZIEWIĄTY' => 89]), ); $ds = df() @@ -432,7 +437,9 @@ public function test_rename_preserves_metadata(): void $metadata = Metadata::fromArray(['description' => 'test metadata']); $rows = df() - ->read(from_rows(rows(schema(str_schema('old_name', metadata: $metadata)), row(['old_name' => 'value'])))) + ->read(from_rows(array_to_rows([[ + 'old_name' => 'value', + ]], schema(str_schema('old_name', metadata: $metadata))))) ->rename('old_name', 'new_name') ->fetch(); diff --git a/src/core/etl/tests/Flow/ETL/Tests/Integration/DataFrame/SchemaTest.php b/src/core/etl/tests/Flow/ETL/Tests/Integration/DataFrame/SchemaTest.php index 91a6e0f69f..2602009804 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Integration/DataFrame/SchemaTest.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Integration/DataFrame/SchemaTest.php @@ -139,7 +139,7 @@ public function test_getting_schema(): void range(1, 100), ), schema(int_schema('id'), str_schema('name'), bool_schema('active')), - flow_context(config())->hydrator(), + flow_context(config())->backend(), ); static::assertEquals( @@ -165,7 +165,7 @@ public function test_limit_does_not_narrow_the_schema(): void range(1, 100), ), schema(int_schema('id'), str_schema('name'), bool_schema('active'), str_schema('union', nullable: true)), - flow_context(config())->hydrator(), + flow_context(config())->backend(), ); static::assertEquals( diff --git a/src/core/etl/tests/Flow/ETL/Tests/Integration/DataFrame/SinglePlanTest.php b/src/core/etl/tests/Flow/ETL/Tests/Integration/DataFrame/SinglePlanTest.php index e67ca3cf52..e2f12340e5 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Integration/DataFrame/SinglePlanTest.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Integration/DataFrame/SinglePlanTest.php @@ -17,6 +17,7 @@ use RuntimeException; use function array_filter; +use function Flow\ETL\DSL\array_to_rows; use function Flow\ETL\DSL\config_builder; use function Flow\ETL\DSL\df; use function Flow\ETL\DSL\from_all; @@ -26,8 +27,6 @@ use function Flow\ETL\DSL\join_on; use function Flow\ETL\DSL\lit; use function Flow\ETL\DSL\ref; -use function Flow\ETL\DSL\row; -use function Flow\ETL\DSL\rows; use function Flow\ETL\DSL\schema; use function Flow\ETL\DSL\to_memory; @@ -89,7 +88,7 @@ public function test_a_joined_frame_is_optimized_with_the_outer_frames_optimizer { $source = new RecordingFileExtractor( schema(int_schema('id')), - rows(schema(int_schema('id')), row(['id' => 1]), row(['id' => 2])), + array_to_rows([['id' => 1], ['id' => 2]], schema(int_schema('id'))), ); $right = df(config_builder()->optimizer(new Optimizer()))->read($source)->limit(1); diff --git a/src/core/etl/tests/Flow/ETL/Tests/Integration/DataFrame/SinkRootBlockingOperationsTest.php b/src/core/etl/tests/Flow/ETL/Tests/Integration/DataFrame/SinkRootBlockingOperationsTest.php index 42c2a85056..39a3f134f3 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Integration/DataFrame/SinkRootBlockingOperationsTest.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Integration/DataFrame/SinkRootBlockingOperationsTest.php @@ -15,6 +15,7 @@ use RuntimeException; use function array_column; +use function Flow\ETL\DSL\array_to_rows; use function Flow\ETL\DSL\average; use function Flow\ETL\DSL\constraint_unique; use function Flow\ETL\DSL\data_frame; @@ -26,9 +27,7 @@ use function Flow\ETL\DSL\lit; use function Flow\ETL\DSL\pivot_values; use function Flow\ETL\DSL\ref; -use function Flow\ETL\DSL\row; use function Flow\ETL\DSL\row_number; -use function Flow\ETL\DSL\rows; use function Flow\ETL\DSL\schema; use function Flow\ETL\DSL\skip_rows_handler; use function Flow\ETL\DSL\str_schema; @@ -174,7 +173,7 @@ public function test_drop_duplicates_inside_a_transformation_deduplicates_global public function test_group_by_aggregate_inside_a_transformation_merges_groups_across_batches(): void { - // Groups are merged across batches, giving the 2 rows the outer frame gives. + // Groups are merged across batches, giving the 2 rows the outer frame gives - in one batch, aggregated in memory. $spy = new SpyLoader(); $sumVByGroup = new CallbackTransformation(static function (DataFrame $df): DataFrame { return $df->groupBy([ref('g')])->aggregate(sum(ref('v'))); @@ -185,21 +184,20 @@ public function test_group_by_aggregate_inside_a_transformation_merges_groups_ac ->write(to_transformation($sumVByGroup, $spy)) ->run(); - static::assertSame([1, 1], $spy->loadedRowCounts()); + static::assertSame([2], $spy->loadedRowCounts()); static::assertSame([['g' => 'a', 'v_sum' => 6.0], ['g' => 'b', 'v_sum' => 60.0]], $spy->loadedRowsToArray()); } public function test_join_inside_a_transformation_matches_every_row(): void { - // REGRESSION GUARD - asserts CORRECT behaviour. HashJoinProcessor buffers the RIGHT side, which is a separate - // complete frame, and buckets the left side. Under one stream the buckets span the stream, so the output is - // grouped by join key in 2 chunks of 3 - byte for byte what the same join gives on the outer frame. + // REGRESSION GUARD - asserts CORRECT behaviour. HashJoinProcessor reads the RIGHT side, a separate complete + // frame, first; it fits under the memory limit, so the left stream joins batch by batch in its own order - + // byte for byte what the same join gives on the outer frame. $spy = new SpyLoader(); $joinNames = new CallbackTransformation(static fn(DataFrame $df): DataFrame => $df->join( - data_frame()->process(rows( + data_frame()->process(array_to_rows( + [['code' => 'a', 'n' => 'Alpha'], ['code' => 'b', 'n' => 'Bravo']], schema(str_schema('code'), str_schema('n')), - row(['code' => 'a', 'n' => 'Alpha']), - row(['code' => 'b', 'n' => 'Bravo']), )), join_on(['g' => 'code'], 'j_'), Join::inner, @@ -210,14 +208,14 @@ public function test_join_inside_a_transformation_matches_every_row(): void ->write(to_transformation($joinNames, $spy)) ->run(); - static::assertSame([3, 3], $spy->loadedRowCounts()); + static::assertSame([2, 2, 2], $spy->loadedRowCounts()); static::assertSame( [ ['g' => 'a', 'v' => 1, 'j_code' => 'a', 'j_n' => 'Alpha'], - ['g' => 'a', 'v' => 2, 'j_code' => 'a', 'j_n' => 'Alpha'], - ['g' => 'a', 'v' => 3, 'j_code' => 'a', 'j_n' => 'Alpha'], ['g' => 'b', 'v' => 10, 'j_code' => 'b', 'j_n' => 'Bravo'], + ['g' => 'a', 'v' => 2, 'j_code' => 'a', 'j_n' => 'Alpha'], ['g' => 'b', 'v' => 20, 'j_code' => 'b', 'j_n' => 'Bravo'], + ['g' => 'a', 'v' => 3, 'j_code' => 'a', 'j_n' => 'Alpha'], ['g' => 'b', 'v' => 30, 'j_code' => 'b', 'j_n' => 'Bravo'], ], $spy->loadedRowsToArray(), diff --git a/src/core/etl/tests/Flow/ETL/Tests/Integration/DataFrame/SinkRootTest.php b/src/core/etl/tests/Flow/ETL/Tests/Integration/DataFrame/SinkRootTest.php index eca4b01a2c..6c3b61c2a4 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Integration/DataFrame/SinkRootTest.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Integration/DataFrame/SinkRootTest.php @@ -45,6 +45,7 @@ use function file_get_contents; use function Flow\ETL\Adapter\Text\from_text; use function Flow\ETL\DSL\add_row_index; +use function Flow\ETL\DSL\array_to_rows; use function Flow\ETL\DSL\batch_size; use function Flow\ETL\DSL\df; use function Flow\ETL\DSL\drop; @@ -58,8 +59,6 @@ use function Flow\ETL\DSL\mask_columns; use function Flow\ETL\DSL\partition_types; use function Flow\ETL\DSL\ref; -use function Flow\ETL\DSL\row; -use function Flow\ETL\DSL\rows; use function Flow\ETL\DSL\schema; use function Flow\ETL\DSL\select; use function Flow\ETL\DSL\to_branch; @@ -409,7 +408,7 @@ public function test_a_later_limit_does_not_narrow_an_earlier_sink(): void $spy = new SpyLoader(); $extractor = new RecordingFileExtractor( schema(int_schema('id')), - rows(schema(int_schema('id')), row(['id' => 1]), row(['id' => 2]), row(['id' => 3])), + array_to_rows([['id' => 1], ['id' => 2], ['id' => 3]], schema(int_schema('id'))), ); $rows = df()->read($extractor)->write($spy)->limit(2)->fetch(); diff --git a/src/core/etl/tests/Flow/ETL/Tests/Integration/DataFrame/SortTest.php b/src/core/etl/tests/Flow/ETL/Tests/Integration/DataFrame/SortTest.php index d441952df3..18d60f8ec9 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Integration/DataFrame/SortTest.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Integration/DataFrame/SortTest.php @@ -4,6 +4,7 @@ namespace Flow\ETL\Tests\Integration\DataFrame; +use Flow\ETL\Dataset\Memory\Unit; use Flow\ETL\Tests\Double\FakeExtractor; use Flow\ETL\Tests\FlowIntegrationTestCase; @@ -47,15 +48,15 @@ public function test_a_bare_reference_honours_its_sort_order(): void public function test_etl_sort_by_external_sort(): void { - $config = config_builder()->sort(external_sort()->runSize(100)); + $config = config_builder()->sort(external_sort()->memoryLimit(Unit::fromBytes(1))); $rows = df($config->build()) - ->read(new FakeExtractor(2500)) - ->batchSize(50) + ->read(new FakeExtractor(100)) + ->batchSize(20) ->sortBy([ref('int')]) ->fetch(); - static::assertSame(range(0, 2499), $rows->reduceToArray('int')); + static::assertSame(range(0, 99), $rows->reduceToArray('int')); } public function test_etl_sort_by_in_memory(): void diff --git a/src/core/etl/tests/Flow/ETL/Tests/Integration/Dataset/SharedMemoryLimitTest.php b/src/core/etl/tests/Flow/ETL/Tests/Integration/Dataset/SharedMemoryLimitTest.php new file mode 100644 index 0000000000..f203cd725b --- /dev/null +++ b/src/core/etl/tests/Flow/ETL/Tests/Integration/Dataset/SharedMemoryLimitTest.php @@ -0,0 +1,62 @@ +backend($backend) + ->join(hash_join()->storage($joinStorage)->memoryLimit($limit)) + ->sort(external_sort()->storage($sortStorage)->memoryLimit($limit)) + ->build(), + ) + ->read(new HoldingExtractor($backend, 200 * $megabyte, [ + array_to_rows([['id' => 2]], $schema), + array_to_rows([['id' => 3]], $schema), + array_to_rows([['id' => 1]], $schema), + ])) + ->join(data_frame()->read(new HoldingExtractor($backend, 50 * $megabyte, [ + array_to_rows([['id' => 1], ['id' => 2]], $schema), + array_to_rows([['id' => 3]], $schema), + ])), join_on(['id' => 'id'], 'r_')) + ->sortBy(ref('id')) + ->fetch(); + + static::assertSame([1, 2, 3], $sorted->reduceToArray('id')); + static::assertSame([], $joinStorage->appendedRows(), 'the join build fits and stays in memory'); + static::assertNotSame([], $sortStorage->appended, 'the sort must spill: the process passed the limit'); + } +} diff --git a/src/core/etl/tests/Flow/ETL/Tests/Integration/Extractor/CacheExtractorTest.php b/src/core/etl/tests/Flow/ETL/Tests/Integration/Extractor/CacheExtractorTest.php index d4b0ce9658..32cf5f4fc6 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Integration/Extractor/CacheExtractorTest.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Integration/Extractor/CacheExtractorTest.php @@ -8,12 +8,15 @@ use Flow\ETL\Cache\Implementation\FilesystemCache; use Flow\ETL\Cache\Implementation\InMemoryCache; use Flow\ETL\Cardinality; +use Flow\ETL\Column\AdaptiveBackend; +use Flow\ETL\Column\PhpBackend; use Flow\ETL\Extractor\Signal; use Flow\ETL\Rows; use Flow\ETL\Tests\Double\CountingCache; use Flow\ETL\Tests\Double\CountingExtractor; use Flow\ETL\Tests\FlowIntegrationTestCase; use Flow\ETL\Tests\Mother\RowsMother; +use Flow\Floe\FloeSerializer; use function array_map; use function array_merge; @@ -114,20 +117,20 @@ public function test_extracting_rows_from_cache(): void $cache->set('rows_01', array_to_rows( [['id' => 1], ['id' => 2]], schema(int_schema('id')), - flow_context(config())->hydrator(), + flow_context(config())->backend(), )); $cache->set('rows_02', array_to_rows( [['id' => 3], ['id' => 4]], schema(int_schema('id')), - flow_context(config())->hydrator(), + flow_context(config())->backend(), )); $cache->set('rows_03', array_to_rows( [['id' => 5]], schema(int_schema('id')), - flow_context(config())->hydrator(), + flow_context(config())->backend(), )); - $cache->set('key', $index->toRows()); + $cache->set('key', $index->toRows(new PhpBackend())); $extractor = from_cache($cacheKey); @@ -152,20 +155,20 @@ public function test_extracting_rows_from_cache_with_clearing_cache_afterwards() $cache->set('rows_01', array_to_rows( [['id' => 1], ['id' => 2]], schema(int_schema('id')), - flow_context(config())->hydrator(), + flow_context(config())->backend(), )); $cache->set('rows_02', array_to_rows( [['id' => 3], ['id' => 4]], schema(int_schema('id')), - flow_context(config())->hydrator(), + flow_context(config())->backend(), )); $cache->set('rows_03', array_to_rows( [['id' => 5]], schema(int_schema('id')), - flow_context(config())->hydrator(), + flow_context(config())->backend(), )); - $cache->set('key', $index->toRows()); + $cache->set('key', $index->toRows(new PhpBackend())); $extractor = from_cache($cacheKey)->withClearOnFinish(true); @@ -180,7 +183,11 @@ public function test_extracting_rows_from_cache_with_clearing_cache_afterwards() public function test_extracting_rows_from_filesystem_cache(): void { - $cache = new FilesystemCache($this->fs(), path(__DIR__ . '/var/cache-extractor-streaming')); + $cache = new FilesystemCache( + $this->fs(), + new FloeSerializer(new AdaptiveBackend()), + path(__DIR__ . '/var/cache-extractor-streaming'), + ); $cache->clear(); $index = new CacheIndex($cacheKey = 'key'); @@ -195,9 +202,9 @@ public function test_extracting_rows_from_filesystem_cache(): void ['id' => 5], ], schema(int_schema('id')), - flow_context(config())->hydrator(), + flow_context(config())->backend(), )); - $cache->set('key', $index->toRows()); + $cache->set('key', $index->toRows(new PhpBackend())); $extractor = from_cache($cacheKey); @@ -214,7 +221,11 @@ public function test_extracting_rows_from_filesystem_cache(): void public function test_stop_signal_stops_extraction_and_skips_clearing(): void { - $cache = new FilesystemCache($this->fs(), path(__DIR__ . '/var/cache-extractor-streaming-stop')); + $cache = new FilesystemCache( + $this->fs(), + new FloeSerializer(new AdaptiveBackend()), + path(__DIR__ . '/var/cache-extractor-streaming-stop'), + ); $cache->clear(); $index = new CacheIndex($cacheKey = 'key'); @@ -229,9 +240,9 @@ public function test_stop_signal_stops_extraction_and_skips_clearing(): void ['id' => 5], ], schema(int_schema('id')), - flow_context(config())->hydrator(), + flow_context(config())->backend(), )); - $cache->set('key', $index->toRows()); + $cache->set('key', $index->toRows(new PhpBackend())); $generator = from_cache($cacheKey) ->withClearOnFinish(true) diff --git a/src/core/etl/tests/Flow/ETL/Tests/Integration/Extractor/FileSourceFilesystemTest.php b/src/core/etl/tests/Flow/ETL/Tests/Integration/Extractor/FileSourceFilesystemTest.php index a255d916da..1a00c0a944 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Integration/Extractor/FileSourceFilesystemTest.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Integration/Extractor/FileSourceFilesystemTest.php @@ -248,9 +248,9 @@ public function test_reads_through_the_given_filesystem(): void static::assertSame([1, 2], array_column($rows, 'id')); static::assertSame( - 3, + 4, $counting->readFromCalls, - 'header() + the sample pass + the read, all through the filesystem it was given', + 'header() + the sample pass + the divergence check + the read, all through the filesystem it was given', ); } diff --git a/src/core/etl/tests/Flow/ETL/Tests/Integration/Extractor/PathPartitionsExtractorTest.php b/src/core/etl/tests/Flow/ETL/Tests/Integration/Extractor/PathPartitionsExtractorTest.php index 138eac2586..f048299c3a 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Integration/Extractor/PathPartitionsExtractorTest.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Integration/Extractor/PathPartitionsExtractorTest.php @@ -5,6 +5,7 @@ namespace Flow\ETL\Tests\Integration\Extractor; use Flow\ETL\Cardinality; +use Flow\ETL\Column\PhpBackend; use Flow\ETL\Extractor\PathPartitionsExtractor; use Flow\ETL\Rows; use Flow\ETL\Tests\Context\ExtractedRows; @@ -89,7 +90,7 @@ public function test_extracting_data_from_path_partitions(): void foreach ($extractedData as $nextRows) { static::assertInstanceOf(Rows::class, $nextRows); - $rows = $rows->merge($nextRows); + $rows = $rows->isEmpty() ? $nextRows : $rows->concat(new PhpBackend(), $nextRows); } static::assertSame(7, $rows->count()); @@ -102,7 +103,7 @@ public function test_extracting_data_from_path_partitions(): void [ 'path' => 'file://' - . ltrim(str_replace('\\', '/', __DIR__), '/') + . str_replace('\\', '/', __DIR__) . '/Fixtures/multi_partitioned/year=2022/month=12/day=30/file.txt', 'partitions' => ['year' => '2022', 'month' => '12', 'day' => '30'], 'day' => '30', @@ -112,7 +113,7 @@ public function test_extracting_data_from_path_partitions(): void [ 'path' => 'file://' - . ltrim(str_replace('\\', '/', __DIR__), '/') + . str_replace('\\', '/', __DIR__) . '/Fixtures/multi_partitioned/year=2022/month=12/day=31/file.txt', 'partitions' => ['year' => '2022', 'month' => '12', 'day' => '31'], 'day' => '31', @@ -122,7 +123,7 @@ public function test_extracting_data_from_path_partitions(): void [ 'path' => 'file://' - . ltrim(str_replace('\\', '/', __DIR__), '/') + . str_replace('\\', '/', __DIR__) . '/Fixtures/multi_partitioned/year=2023/month=1/day=1/file.txt', 'partitions' => ['year' => '2023', 'month' => '1', 'day' => '1'], 'day' => '1', @@ -132,7 +133,7 @@ public function test_extracting_data_from_path_partitions(): void [ 'path' => 'file://' - . ltrim(str_replace('\\', '/', __DIR__), '/') + . str_replace('\\', '/', __DIR__) . '/Fixtures/multi_partitioned/year=2023/month=1/day=2/file.txt', 'partitions' => ['year' => '2023', 'month' => '1', 'day' => '2'], 'day' => '2', @@ -142,7 +143,7 @@ public function test_extracting_data_from_path_partitions(): void [ 'path' => 'file://' - . ltrim(str_replace('\\', '/', __DIR__), '/') + . str_replace('\\', '/', __DIR__) . '/Fixtures/multi_partitioned/year=2023/month=1/day=3/file.txt', 'partitions' => ['year' => '2023', 'month' => '1', 'day' => '3'], 'day' => '3', @@ -152,7 +153,7 @@ public function test_extracting_data_from_path_partitions(): void [ 'path' => 'file://' - . ltrim(str_replace('\\', '/', __DIR__), '/') + . str_replace('\\', '/', __DIR__) . '/Fixtures/multi_partitioned/year=2023/month=1/day=4/file.txt', 'partitions' => ['year' => '2023', 'month' => '1', 'day' => '4'], 'day' => '4', @@ -162,7 +163,7 @@ public function test_extracting_data_from_path_partitions(): void [ 'path' => 'file://' - . ltrim(str_replace('\\', '/', __DIR__), '/') + . str_replace('\\', '/', __DIR__) . '/Fixtures/multi_partitioned/year=2023/month=1/day=5/file.txt', 'partitions' => ['year' => '2023', 'month' => '1', 'day' => '5'], 'day' => '5', @@ -194,6 +195,44 @@ public function test_batches_at_the_configured_batch_size(): void static::assertSame([2, 2, 2, 1], $sizes); } + public function test_every_row_of_a_batch_carries_its_own_files_partition_value(): void + { + $batches = iterator_to_array( + from_path_partitions('memory://dir/**/*.txt', MemoryFiles::with([ + 'memory://dir/year=2023/a.txt' => 'a', + 'memory://dir/year=2024/b.txt' => 'b', + 'memory://dir/year=2025/c.txt' => 'c', + ])) + ->withBatchSize(3) + ->extract(flow_context()), + false, + ); + + static::assertCount(1, $batches); + static::assertSame(['2023', '2024', '2025'], $batches[0]->column('year')->values()); + static::assertSame( + [['year' => '2023'], ['year' => '2024'], ['year' => '2025']], + $batches[0]->column('partitions')->values(), + ); + } + + public function test_a_limit_at_the_batch_boundary_keeps_each_rows_partition_value(): void + { + $batches = iterator_to_array( + from_path_partitions('memory://dir/**/*.txt', MemoryFiles::with([ + 'memory://dir/year=2023/a.txt' => 'a', + 'memory://dir/year=2024/b.txt' => 'b', + 'memory://dir/year=2025/c.txt' => 'c', + ])) + ->withBatchSize(2) + ->extract(flow_context(), limit: 2), + false, + ); + + static::assertCount(1, $batches); + static::assertSame(['2023', '2024'], $batches[0]->column('year')->values()); + } + public function test_path_partitions_extractor_honours_the_batch_contract(): void { self::assertExtractorHonoursBatchContract( diff --git a/src/core/etl/tests/Flow/ETL/Tests/Integration/Filesystem/ConfigIsolationTest.php b/src/core/etl/tests/Flow/ETL/Tests/Integration/Filesystem/ConfigIsolationTest.php index cc4799a6fa..456b369f49 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Integration/Filesystem/ConfigIsolationTest.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Integration/Filesystem/ConfigIsolationTest.php @@ -37,7 +37,7 @@ public function test_a_failed_run_does_not_leave_a_stream_registered_on_a_shared ->batchSize(10) ->write(to_floe($destination, options: floe_options(buffer_size: 64))) ->write(new InlineLoader(static function (Rows $rows, FlowContext $context): void { - if ($rows->first()->get('id') === 101) { + if ($rows->column('id')->value(0) === 101) { throw new RuntimeException('aborted mid run'); } })) @@ -91,8 +91,8 @@ public function test_a_dead_run_does_not_bleed_into_the_next_on_the_same_sink(): $ids = []; foreach (data_frame($config)->read(from_floe($destination))->get() as $batch) { - foreach ($batch as $row) { - $ids[] = $row->get('id'); + foreach ($batch->toArray() as $row) { + $ids[] = $row['id']; } } diff --git a/src/core/etl/tests/Flow/ETL/Tests/Integration/Filesystem/FilesSink/FilesSinkTest.php b/src/core/etl/tests/Flow/ETL/Tests/Integration/Filesystem/FilesSink/FilesSinkTest.php deleted file mode 100644 index 99042d3479..0000000000 --- a/src/core/etl/tests/Flow/ETL/Tests/Integration/Filesystem/FilesSink/FilesSinkTest.php +++ /dev/null @@ -1,270 +0,0 @@ -cleanFiles(); - } - - public static function save_modes(): Generator - { - yield 'overwrite' => [SaveMode::Overwrite]; - yield 'exception if exists' => [SaveMode::ExceptionIfExists]; - yield 'append' => [SaveMode::Append]; - } - - #[DataProvider('save_modes')] - public function test_abandon_after_a_publish_that_failed_part_way_removes_only_what_was_not_published(SaveMode $saveMode): void - { - $this->setupFiles([__FUNCTION__ => []]); - - $files = new FilesSink( - new FailingCloseFilesystem($this->fs, healthyStreams: 1), - $this->getPath(__FUNCTION__ . '/file.txt'), - $saveMode, - ); - $files->writeTo([partition('p', 'a')])->append('a'); - $b = $files->writeTo([partition('p', 'b')]); - $b->append('b'); - - try { - $files->publish(); - static::fail('publish() was expected to throw'); - } catch (FilesystemRuntimeException $failure) { - static::assertSame('Closing "' . $b->path()->uri() . '" failed', $failure->getMessage()); - } - - $files->abandon(); - - static::assertSame('a', file_get_contents($this->getPath(__FUNCTION__ . '/p=a/file.txt')->path())); - static::assertFileDoesNotExist($this->getPath(__FUNCTION__ . '/p=b/file.txt')->path()); - static::assertFileDoesNotExist($b->path()->path()); - } - - public function test_abandon_removes_every_file_even_when_a_stream_fails_to_close(): void - { - $this->setupFiles([__FUNCTION__ => []]); - - $files = new FilesSink( - new FailingCloseFilesystem($this->fs, failingStreams: 2), - $this->getPath(__FUNCTION__ . '/file.txt'), - SaveMode::Overwrite, - ); - $a = $files->writeTo([partition('p', 'a')]); - $a->append('a'); - $b = $files->writeTo([partition('p', 'b')]); - $b->append('b'); - - try { - $files->abandon(); - static::fail('abandon() was expected to throw'); - } catch (FilesystemRuntimeException $failure) { - static::assertSame('Closing "' . $a->path()->uri() . '" failed', $failure->getMessage()); - } - - static::assertFileDoesNotExist($a->path()->path()); - static::assertFileDoesNotExist($b->path()->path()); - } - - public function test_abandon_leaves_the_destination_untouched(): void - { - $this->setupFiles([ - __FUNCTION__ => [ - 'file.txt' => 'existing content', - ], - ]); - - $file = $this->getPath(__FUNCTION__ . '/file.txt'); - $files = new FilesSink($this->fs, $file, SaveMode::Overwrite); - $stream = $files->writeTo(); - $stream->append('half written content'); - - $files->abandon(); - - static::assertSame('existing content', file_get_contents($file->path())); - // list('/*') cannot see a dot-prefixed tmp file, so the leftover is asserted by its own path - static::assertFileDoesNotExist($stream->path()->path()); - } - - public function test_abandon_leaves_a_destination_it_did_not_create(): void - { - $this->setupFiles([ - __FUNCTION__ => [ - 'file.txt' => 'existing content', - ], - ]); - - $file = $this->getPath(__FUNCTION__ . '/file.txt'); - $files = new FilesSink($this->fs, $file, SaveMode::Ignore); - $files->writeTo()->append('ignored content'); - - $files->abandon(); - - static::assertFileExists($file->path()); - static::assertSame('existing content', file_get_contents($file->path())); - } - - public function test_abandon_removes_the_file_it_created_under_append(): void - { - $this->setupFiles([ - __FUNCTION__ => [ - 'file.txt' => 'existing content', - ], - ]); - - $file = $this->getPath(__FUNCTION__ . '/file.txt'); - $files = new FilesSink($this->fs, $file, SaveMode::Append); - $files->writeTo()->append('half written content'); - - $files->abandon(); - - static::assertCount( - 1, - $files = iterator_to_array($this->fs()->list(path($file->parentDirectory()->path() . '/*'))), - ); - static::assertSame('file.txt', $files[0]->path->basename()); - static::assertSame('existing content', file_get_contents($file->path())); - } - - public function test_abandon_removes_the_file_it_created_under_exception_if_exists(): void - { - $this->setupFiles([__FUNCTION__ => []]); - - $file = $this->getPath(__FUNCTION__ . '/file.txt'); - $files = new FilesSink($this->fs, $file, SaveMode::ExceptionIfExists); - $files->writeTo()->append('half written content'); - - $files->abandon(); - - static::assertFileDoesNotExist($file->path()); - } - - public function test_abandon_removes_the_temporary_file(): void - { - $this->setupFiles([__FUNCTION__ => []]); - - $file = $this->getPath(__FUNCTION__ . '/file.txt'); - $files = new FilesSink($this->fs, $file, SaveMode::Overwrite); - $stream = $files->writeTo(); - $stream->append('half written content'); - - static::assertFileExists($stream->path()->path()); - - $files->abandon(); - - static::assertFileDoesNotExist($stream->path()->path()); - static::assertFileDoesNotExist($file->path()); - } - - public function test_abandoning_a_published_set_does_nothing(): void - { - $this->setupFiles([__FUNCTION__ => []]); - - $file = $this->getPath(__FUNCTION__ . '/file.txt'); - $files = new FilesSink($this->fs, $file, SaveMode::Overwrite); - $files->writeTo()->append('some content'); - $files->publish(); - - $files->abandon(); - - static::assertSame('some content', file_get_contents($file->path())); - } - - public function test_touched_reports_a_file_this_run_has_written_to(): void - { - $this->setupFiles([__FUNCTION__ => []]); - - $files = $this->files($this->getPath(__FUNCTION__ . '/file.txt')); - - static::assertFalse($files->touched()); - - $files->writeTo(); - - static::assertTrue($files->touched()); - - $files->publish(); - - static::assertFalse($files->touched()); - } - - public function test_open_streams_lists_only_streams_that_are_still_open(): void - { - $this->setupFiles([__FUNCTION__ => []]); - - $files = $this->files($this->getPath(__FUNCTION__ . '/file.txt')); - $stream = $files->writeTo(); - - static::assertSame([$stream], iterator_to_array($files->openStreams(), false)); - - $stream->close(); - - static::assertSame([], iterator_to_array($files->openStreams(), false)); - } - - public function test_open_two_write_streams_to_stdout(): void - { - $this->expectExceptionMessage('Only one stream can be open at the same time for php://stdout'); - - // one instance: StdOutFilesystem holds its open targets per instance - $stdout = stdout_filesystem(); - - $this->files(path('stdout://a.stdout'), $stdout)->writeTo(); - $this->files(path('stdout://b.stdout'), $stdout)->writeTo(); - } - - public function test_scan(): void - { - $this->setupFiles([ - __FUNCTION__ => [ - 'file1.txt' => 'file1 content', - 'file2.txt' => 'file2 content', - 'folder' => [ - 'file3.txt' => 'file3 content', - 'file4.txt' => 'file4 content', - ], - ], - ]); - - static::assertCount(4, iterator_to_array( - (new FileListing($this->fs))->list($this->getPath(__FUNCTION__ . '/**/*.txt'), new KeepAll()), - false, - )); - } - - public function test_write_to_stdout(): void - { - $files = $this->files(path('stdout://a.stdout'), stdout_filesystem()); - $files->writeTo(); - - static::assertTrue($files->touched()); - } - - protected function saveMode(): SaveMode - { - return exception_if_exists(); - } -} diff --git a/src/core/etl/tests/Flow/ETL/Tests/Integration/Filesystem/FilesSink/NotPartitioned/AppendModeTest.php b/src/core/etl/tests/Flow/ETL/Tests/Integration/Filesystem/FilesSink/NotPartitioned/AppendModeTest.php deleted file mode 100644 index 6d6c5f584d..0000000000 --- a/src/core/etl/tests/Flow/ETL/Tests/Integration/Filesystem/FilesSink/NotPartitioned/AppendModeTest.php +++ /dev/null @@ -1,70 +0,0 @@ -cleanFiles(); - } - - public function test_open_stream_for_existing_file(): void - { - $this->setupFiles([ - __FUNCTION__ => [ - 'existing-file.txt' => 'some content', - ], - ]); - $file = $this->getPath(__FUNCTION__ . '/existing-file.txt'); - static::assertFileExists($file->path()); - - $files = $this->files($file); - $files->writeTo()->append('new content'); - $files->publish(); - - $files = iterator_to_array($this->fs()->list(path($file->parentDirectory()->path() . '/*'))); - - static::assertCount(2, $files); - - foreach ($files as $streamFile) { - static::assertStringStartsWith('existing-file', $streamFile->path->basename()); - static::assertStringEndsWith('.txt', $streamFile->path->basename()); - } - } - - public function test_open_stream_for_non_existing_file(): void - { - $this->setupFiles([ - __FUNCTION__ => [], - ]); - $file = $this->getPath(__FUNCTION__ . '/non-existing-file.txt'); - static::assertFileDoesNotExist($file->path()); - - $files = $this->files($file); - $files->writeTo()->append('new content'); - $files->publish(); - - $files = iterator_to_array($this->fs()->list(path($file->parentDirectory()->path() . '/*'))); - - static::assertCount(1, $files); - static::assertSame('non-existing-file.txt', $files[0]->path->basename()); - } - - protected function saveMode(): SaveMode - { - return append(); - } -} diff --git a/src/core/etl/tests/Flow/ETL/Tests/Integration/Filesystem/FilesSink/NotPartitioned/ExceptionIfExistsModeTest.php b/src/core/etl/tests/Flow/ETL/Tests/Integration/Filesystem/FilesSink/NotPartitioned/ExceptionIfExistsModeTest.php deleted file mode 100644 index f626629148..0000000000 --- a/src/core/etl/tests/Flow/ETL/Tests/Integration/Filesystem/FilesSink/NotPartitioned/ExceptionIfExistsModeTest.php +++ /dev/null @@ -1,57 +0,0 @@ -cleanFiles(); - } - - public function test_open_stream_for_existing_file(): void - { - $this->expectExceptionMessageMatches( - '/Destination path (.*) already exists, please change path to different or set different SaveMode/', - ); - - $this->setupFiles([ - __FUNCTION__ => [ - 'existing-file.txt' => 'some content', - ], - ]); - - $this->files($this->getPath(__FUNCTION__ . '/existing-file.txt'))->writeTo(); - } - - public function test_open_stream_for_non_existing_file(): void - { - $this->setupFiles([ - __FUNCTION__ => [], - ]); - $file = $this->getPath(__FUNCTION__ . '/non-existing-file.txt'); - - $files = $this->files($file); - $files->writeTo()->append('some content'); - $files->publish(); - - static::assertFileExists($file->path()); - static::assertSame('some content', file_get_contents($file->path())); - } - - protected function saveMode(): SaveMode - { - return exception_if_exists(); - } -} diff --git a/src/core/etl/tests/Flow/ETL/Tests/Integration/Filesystem/FilesSink/NotPartitioned/IgnoreModeTest.php b/src/core/etl/tests/Flow/ETL/Tests/Integration/Filesystem/FilesSink/NotPartitioned/IgnoreModeTest.php deleted file mode 100644 index cb990e50d2..0000000000 --- a/src/core/etl/tests/Flow/ETL/Tests/Integration/Filesystem/FilesSink/NotPartitioned/IgnoreModeTest.php +++ /dev/null @@ -1,59 +0,0 @@ -cleanFiles(); - } - - public function test_open_stream_for_existing_file(): void - { - $this->setupFiles([ - __FUNCTION__ => [ - 'existing-file.txt' => 'some content', - ], - ]); - $path = $this->getPath(__FUNCTION__ . '/existing-file.txt'); - - $files = $this->files($path); - $files->writeTo()->append('different content'); - $files->publish(); - - static::assertFileExists($path->path()); - static::assertSame('some content', file_get_contents($path->path())); - } - - public function test_open_stream_for_non_existing_file(): void - { - $this->setupFiles([ - __FUNCTION__ => [], - ]); - $path = $this->getPath(__FUNCTION__ . '/non-existing-file.txt'); - - $files = $this->files($path); - $files->writeTo()->append('some content'); - $files->publish(); - - static::assertFileExists($path->path()); - static::assertSame('some content', file_get_contents($path->path())); - } - - protected function saveMode(): SaveMode - { - return ignore(); - } -} diff --git a/src/core/etl/tests/Flow/ETL/Tests/Integration/Filesystem/FilesSink/NotPartitioned/OverwriteModeTest.php b/src/core/etl/tests/Flow/ETL/Tests/Integration/Filesystem/FilesSink/NotPartitioned/OverwriteModeTest.php deleted file mode 100644 index 685f65de04..0000000000 --- a/src/core/etl/tests/Flow/ETL/Tests/Integration/Filesystem/FilesSink/NotPartitioned/OverwriteModeTest.php +++ /dev/null @@ -1,70 +0,0 @@ -cleanFiles(); - } - - public function test_open_stream_for_existing_file(): void - { - $this->setupFiles([ - __FUNCTION__ => [ - 'existing-file.txt' => 'some content', - ], - ]); - - $path = $this->getPath(__FUNCTION__ . '/existing-file.txt'); - $files = $this->files($path); - - $fileStream = $files->writeTo(); - static::assertStringContainsString(FilesSink::FLOW_TMP_FILE_PREFIX, $fileStream->path()->path()); - $fileStream->append('some other content'); - static::assertSame('some content', file_get_contents($path->path())); - - $files->publish(); - - static::assertSame('some other content', file_get_contents($path->path())); - - static::assertCount( - 1, - $files = iterator_to_array($this->fs()->list(path($path->parentDirectory()->path() . '/*'))), - ); - static::assertSame('existing-file.txt', $files[0]->path->basename()); - } - - public function test_open_stream_for_non_existing_file(): void - { - $this->setupFiles([__FUNCTION__ => []]); - $path = $this->getPath(__FUNCTION__ . '/non-existing-file.txt'); - - $files = $this->files($path); - $files->writeTo()->append('some content'); - $files->publish(); - - static::assertFileExists($path->path()); - static::assertSame('some content', file_get_contents($path->path())); - } - - protected function saveMode(): SaveMode - { - return overwrite(); - } -} diff --git a/src/core/etl/tests/Flow/ETL/Tests/Integration/Filesystem/FilesSink/Partitioned/AppendModeTest.php b/src/core/etl/tests/Flow/ETL/Tests/Integration/Filesystem/FilesSink/Partitioned/AppendModeTest.php deleted file mode 100644 index 54f4cfc6a4..0000000000 --- a/src/core/etl/tests/Flow/ETL/Tests/Integration/Filesystem/FilesSink/Partitioned/AppendModeTest.php +++ /dev/null @@ -1,96 +0,0 @@ -cleanFiles(); - } - - public function test_open_stream_for_existing_partition_with_existing_file(): void - { - $this->setupFiles([ - __FUNCTION__ => [ - 'partition=value' => [ - 'file.txt' => 'file content', - ], - ], - ]); - - $file = $this->getPath(__FUNCTION__ . '/file.txt'); - $files = $this->files($file); - - $files->writeTo([new Partition('partition', 'value')])->append('appended content'); - $files->publish(); - - $files = iterator_to_array($this->fs()->list(path($file->parentDirectory()->path() . '/**/*.txt'))); - - static::assertCount(2, $files); - - foreach ($files as $streamFile) { - static::assertStringStartsWith('file', $streamFile->path->basename()); - static::assertStringEndsWith('.txt', $streamFile->path->basename()); - } - } - - public function test_open_stream_for_existing_partition_without_existing_file(): void - { - $this->setupFiles([ - __FUNCTION__ => [ - 'partition=value' => [], - ], - ]); - $file = $this->getPath(__FUNCTION__ . '/file.txt'); - $files = $this->files($file); - - $files->writeTo([new Partition('partition', 'value')])->append('appended content'); - $files->publish(); - - $files = iterator_to_array($this->fs()->list(path($file->parentDirectory()->path() . '/**/*.txt'))); - - static::assertCount(1, $files); - - static::assertSame('file.txt', $files[0]->path->basename()); - static::assertSame('appended content', file_get_contents($files[0]->path->path())); - } - - public function test_open_stream_for_non_existing_partition(): void - { - $this->setupFiles([ - __FUNCTION__ => [], - ]); - $file = $this->getPath(__FUNCTION__ . '/file.txt'); - $files = $this->files($file); - - $files->writeTo([new Partition('partition', 'value')])->append('appended content'); - $files->publish(); - - $files = iterator_to_array($this->fs()->list(path($file->parentDirectory()->path() . '/partition=value/*'))); - - static::assertCount(1, $files); - - static::assertSame('file.txt', $files[0]->path->basename()); - static::assertSame('appended content', file_get_contents($files[0]->path->path())); - } - - protected function saveMode(): SaveMode - { - return append(); - } -} diff --git a/src/core/etl/tests/Flow/ETL/Tests/Integration/Filesystem/FilesSink/Partitioned/ExceptionIfExistsModeTest.php b/src/core/etl/tests/Flow/ETL/Tests/Integration/Filesystem/FilesSink/Partitioned/ExceptionIfExistsModeTest.php deleted file mode 100644 index 7884b49d00..0000000000 --- a/src/core/etl/tests/Flow/ETL/Tests/Integration/Filesystem/FilesSink/Partitioned/ExceptionIfExistsModeTest.php +++ /dev/null @@ -1,87 +0,0 @@ -cleanFiles(); - } - - public function test_open_stream_for_existing_partition_with_existing_file(): void - { - $this->expectExceptionMessageMatches( - '/Destination path (.*) already exists, please change path to different or set different SaveMode/', - ); - - $this->setupFiles([ - __FUNCTION__ => [ - 'partition=value' => [ - 'file.txt' => 'file content', - ], - ], - ]); - - $this->files($this->getPath(__FUNCTION__ . '/file.txt'))->writeTo([new Partition('partition', 'value')]); - } - - public function test_open_stream_for_existing_partition_without_existing_file(): void - { - $this->setupFiles([ - __FUNCTION__ => [ - 'partition=value' => [], - ], - ]); - $file = $this->getPath(__FUNCTION__ . '/file.txt'); - $files = $this->files($file); - - $files->writeTo([new Partition('partition', 'value')])->append('file content'); - $files->publish(); - - $files = iterator_to_array($this->fs()->list(path($file->parentDirectory()->path() . '/**/*.txt'))); - - static::assertCount(1, $files); - - static::assertSame('file.txt', $files[0]->path->basename()); - static::assertSame('file content', file_get_contents($files[0]->path->path())); - } - - public function test_open_stream_for_non_existing_partition(): void - { - $this->setupFiles([ - __FUNCTION__ => [], - ]); - $file = $this->getPath(__FUNCTION__ . '/file.txt'); - $files = $this->files($file); - - $files->writeTo([new Partition('partition', 'value')])->append('file content'); - $files->publish(); - - $files = iterator_to_array($this->fs()->list(path($file->parentDirectory()->path() . '/partition=value/*'))); - - static::assertCount(1, $files); - - static::assertSame('file.txt', $files[0]->path->basename()); - static::assertSame('file content', file_get_contents($files[0]->path->path())); - } - - protected function saveMode(): SaveMode - { - return exception_if_exists(); - } -} diff --git a/src/core/etl/tests/Flow/ETL/Tests/Integration/Filesystem/FilesSink/Partitioned/IgnoreModeTest.php b/src/core/etl/tests/Flow/ETL/Tests/Integration/Filesystem/FilesSink/Partitioned/IgnoreModeTest.php deleted file mode 100644 index eb473ee071..0000000000 --- a/src/core/etl/tests/Flow/ETL/Tests/Integration/Filesystem/FilesSink/Partitioned/IgnoreModeTest.php +++ /dev/null @@ -1,93 +0,0 @@ -cleanFiles(); - } - - public function test_open_stream_for_existing_partition_with_existing_file(): void - { - $this->setupFiles([ - __FUNCTION__ => [ - 'partition=value' => [ - 'file.txt' => 'file content', - ], - ], - ]); - $file = $this->getPath(__FUNCTION__ . '/file.txt'); - $files = $this->files($file); - - $files->writeTo([new Partition('partition', 'value')])->append('new content'); - $files->publish(); - - $files = iterator_to_array($this->fs()->list(path($file->parentDirectory()->path() . '/**/*.txt'))); - - static::assertCount(1, $files); - - static::assertStringStartsWith('file.txt', $files[0]->path->basename()); - static::assertSame('file content', file_get_contents($files[0]->path->path())); - } - - public function test_open_stream_for_existing_partition_without_existing_file(): void - { - $this->setupFiles([ - __FUNCTION__ => [ - 'partition=value' => [], - ], - ]); - $file = $this->getPath(__FUNCTION__ . '/file.txt'); - $files = $this->files($file); - - $files->writeTo([new Partition('partition', 'value')])->append('appended content'); - $files->publish(); - - $files = iterator_to_array($this->fs()->list(path($file->parentDirectory()->path() . '/**/*.txt'))); - - static::assertCount(1, $files); - - static::assertSame('file.txt', $files[0]->path->basename()); - static::assertSame('appended content', file_get_contents($files[0]->path->path())); - } - - public function test_open_stream_for_non_existing_partition(): void - { - $this->setupFiles([ - __FUNCTION__ => [], - ]); - $file = $this->getPath(__FUNCTION__ . '/file.txt'); - $files = $this->files($file); - - $files->writeTo([new Partition('partition', 'value')])->append('appended content'); - $files->publish(); - - $files = iterator_to_array($this->fs()->list(path($file->parentDirectory()->path() . '/partition=value/*'))); - - static::assertCount(1, $files); - - static::assertSame('file.txt', $files[0]->path->basename()); - static::assertSame('appended content', file_get_contents($files[0]->path->path())); - } - - protected function saveMode(): SaveMode - { - return ignore(); - } -} diff --git a/src/core/etl/tests/Flow/ETL/Tests/Integration/Filesystem/FilesSink/Partitioned/OverwriteModeTest.php b/src/core/etl/tests/Flow/ETL/Tests/Integration/Filesystem/FilesSink/Partitioned/OverwriteModeTest.php deleted file mode 100644 index 2200702510..0000000000 --- a/src/core/etl/tests/Flow/ETL/Tests/Integration/Filesystem/FilesSink/Partitioned/OverwriteModeTest.php +++ /dev/null @@ -1,118 +0,0 @@ -cleanFiles(); - } - - public function test_open_stream_for_existing_partition_with_existing_file(): void - { - $this->setupFiles([ - __FUNCTION__ => [ - 'partition=value' => [ - 'file.txt' => 'file content', - ], - ], - ]); - $file = $this->getPath(__FUNCTION__ . '/file.txt'); - $files = $this->files($file); - - $files->writeTo([new Partition('partition', 'value')])->append('new content'); - $files->publish(); - - $files = iterator_to_array($this->fs()->list(path($file->parentDirectory()->path() . '/**/*.txt'))); - - static::assertCount(1, $files); - - static::assertStringStartsWith('file.txt', $files[0]->path->basename()); - static::assertSame('new content', file_get_contents($files[0]->path->path())); - } - - public function test_open_stream_for_existing_partition_without_existing_file(): void - { - $this->setupFiles([ - __FUNCTION__ => [ - 'partition=value' => [], - ], - ]); - $file = $this->getPath(__FUNCTION__ . '/file.txt'); - $files = $this->files($file); - - $files->writeTo([new Partition('partition', 'value')])->append('new content'); - $files->publish(); - - $files = iterator_to_array($this->fs()->list(path($file->parentDirectory()->path() . '/**/*.txt'))); - - static::assertCount(1, $files); - - static::assertSame('file.txt', $files[0]->path->basename()); - static::assertSame('new content', file_get_contents($files[0]->path->path())); - } - - public function test_open_stream_for_non_existing_partition(): void - { - $this->setupFiles([ - __FUNCTION__ => [], - ]); - $file = $this->getPath(__FUNCTION__ . '/file.txt'); - $files = $this->files($file); - - $files->writeTo([new Partition('partition', 'value')])->append('new content'); - $files->publish(); - - $files = iterator_to_array($this->fs()->list(path($file->parentDirectory()->path() . '/partition=value/*'))); - - static::assertCount(1, $files); - - static::assertSame('file.txt', $files[0]->path->basename()); - static::assertSame('new content', file_get_contents($files[0]->path->path())); - } - - public function test_open_stream_for_non_existing_partition_with_custom_schema(): void - { - $this->setupFiles([__FUNCTION__ => []]); - - $fs = new FakeNativeLocalFilesystem(); - - $file = path( - $fs->mount()->protocol . '://' . $this->filesDirectory() . DIRECTORY_SEPARATOR . __FUNCTION__ . '/file.txt', - ); - - $files = new FilesSink($fs, $file, overwrite()); - - $files->writeTo([new Partition('partition', 'value')])->append('new content'); - $files->publish(); - - $files = iterator_to_array($this->fs()->list(path($file->parentDirectory()->path() . '/partition=value/*'))); - - static::assertCount(1, $files); - - static::assertSame('file.txt', $files[0]->path->basename()); - static::assertSame('new content', file_get_contents($files[0]->path->path())); - } - - protected function saveMode(): SaveMode - { - return overwrite(); - } -} diff --git a/src/core/etl/tests/Flow/ETL/Tests/Integration/Function/ArrayKeysStyleConvertTest.php b/src/core/etl/tests/Flow/ETL/Tests/Integration/Function/ArrayKeysStyleConvertTest.php index 53fa8734b7..105a6b540b 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Integration/Function/ArrayKeysStyleConvertTest.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Integration/Function/ArrayKeysStyleConvertTest.php @@ -6,6 +6,7 @@ use Flow\ETL\Exception\InvalidArgumentException; use Flow\ETL\Memory\ArrayMemory; +use Flow\ETL\Tests\Context\FunctionContext; use Flow\ETL\Tests\FlowTestCase; use function Flow\ETL\DSL\array_keys_style_convert; @@ -14,7 +15,8 @@ use function Flow\ETL\DSL\flow_context; use function Flow\ETL\DSL\from_array; use function Flow\ETL\DSL\ref; -use function Flow\ETL\DSL\row; +use function Flow\ETL\DSL\schema; +use function Flow\ETL\DSL\str_schema; use function Flow\ETL\DSL\to_memory; final class ArrayKeysStyleConvertTest extends FlowTestCase @@ -43,6 +45,30 @@ public function test_array_keys_style_convert_in_strict_mode(): void $this->expectExceptionMessage('Expected type "array", got "string".'); $context = flow_context(config()); - array_keys_style_convert(ref('string'), 'camel')->eval(row(['string' => 'test']), $context); + (new FunctionContext($context))->eval( + array_keys_style_convert(ref('string'), 'camel'), + [ + 'string' => 'test', + ], + schema(str_schema('string')), + ); + } + + public function test_a_nested_convert_stores_the_converted_shape(): void + { + data_frame() + ->read(from_array([ + ['id' => 1, 'array' => ['itemId' => 1, 'itemVariants' => [['variantName' => 'a']]]], + ])) + ->withEntry('array', array_keys_style_convert(ref('array'), 'snake')) + ->write(to_memory($memory = new ArrayMemory())) + ->run(); + + static::assertSame( + [ + ['id' => 1, 'array' => ['item_id' => 1, 'item_variants' => [['variant_name' => 'a']]]], + ], + $memory->dump(), + ); } } diff --git a/src/core/etl/tests/Flow/ETL/Tests/Integration/Function/CastTest.php b/src/core/etl/tests/Flow/ETL/Tests/Integration/Function/CastTest.php index b8b22f3a55..43e6a6c284 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Integration/Function/CastTest.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Integration/Function/CastTest.php @@ -73,7 +73,8 @@ public function test_cast_to_array(): void ->fetch(); // Cast declares type_array (a json column) and the bind enforces that declaration. - $json = $rows->first()->get('b'); + // @mago-ignore analysis:mixed-assignment + $json = $rows->column('b')->value(0); static::assertInstanceOf(Json::class, $json); static::assertSame([1, 2, 3], $json->toArray()); @@ -81,15 +82,17 @@ public function test_cast_to_array(): void public function test_cast_non_deterministic_values(): void { - $row = df() - ->read(from_array([ - ['array' => []], - ])) - ->withEntry('list_int', ref('array')->cast(type_optional(type_list(type_integer())))) - ->drop('array') - ->fetch() - ->first(); - - static::assertSame([], $row->get('list_int')); + static::assertSame( + [], + df() + ->read(from_array([ + ['array' => []], + ])) + ->withEntry('list_int', ref('array')->cast(type_optional(type_list(type_integer())))) + ->drop('array') + ->fetch() + ->column('list_int') + ->value(0), + ); } } diff --git a/src/core/etl/tests/Flow/ETL/Tests/Integration/Function/DOMElementAttributeValueTest.php b/src/core/etl/tests/Flow/ETL/Tests/Integration/Function/DOMElementAttributeValueTest.php index c369fa98cc..f84c06239c 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Integration/Function/DOMElementAttributeValueTest.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Integration/Function/DOMElementAttributeValueTest.php @@ -6,11 +6,10 @@ use Flow\ETL\Tests\FlowTestCase; +use function Flow\ETL\DSL\array_to_rows; use function Flow\ETL\DSL\df; use function Flow\ETL\DSL\from_rows; use function Flow\ETL\DSL\ref; -use function Flow\ETL\DSL\row; -use function Flow\ETL\DSL\rows; use function Flow\ETL\DSL\schema; use function Flow\ETL\DSL\xml_element_schema; use function Flow\ETL\DSL\xml_schema; @@ -22,10 +21,9 @@ final class DOMElementAttributeValueTest extends FlowTestCase public function test_dom_element_attribute_value(): void { $rows = df() - ->read(from_rows(rows( - schema(xml_element_schema('node')), - row(['node' => type_xml_element()->cast('User Name 01')]), - ))) + ->read(from_rows(array_to_rows([[ + 'node' => type_xml_element()->cast('User Name 01'), + ]], schema(xml_element_schema('node'))))) ->withEntry('user_id', ref('node')->domElementAttributeValue('id')) ->drop('node') ->fetch(); @@ -41,10 +39,9 @@ public function test_dom_element_attribute_value(): void public function test_dom_element_attribute_value_from_dom_document(): void { $rows = df() - ->read(from_rows(rows( - schema(xml_schema('node')), - row(['node' => type_xml()->cast('User Name 01')]), - ))) + ->read(from_rows(array_to_rows([[ + 'node' => type_xml()->cast('User Name 01'), + ]], schema(xml_schema('node'))))) ->withEntry('user_id', ref('node')->domElementAttributeValue('id')) ->drop('node') ->fetch(); @@ -60,10 +57,9 @@ public function test_dom_element_attribute_value_from_dom_document(): void public function test_dom_element_attribute_value_on_xpath_result(): void { $rows = df() - ->read(from_rows(rows( - schema(xml_schema('node')), - row(['node' => type_xml()->cast('User Name 01')]), - ))) + ->read(from_rows(array_to_rows([[ + 'node' => type_xml()->cast('User Name 01'), + ]], schema(xml_schema('node'))))) ->withEntry('user_id', ref('node')->xpath('name')->domElementAttributeValue('id')) ->drop('node') ->fetch(); diff --git a/src/core/etl/tests/Flow/ETL/Tests/Integration/Function/DOMElementNextSiblingTest.php b/src/core/etl/tests/Flow/ETL/Tests/Integration/Function/DOMElementNextSiblingTest.php index 31a4830840..2d9cb560fc 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Integration/Function/DOMElementNextSiblingTest.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Integration/Function/DOMElementNextSiblingTest.php @@ -5,19 +5,20 @@ namespace Flow\ETL\Tests\Integration\Function; use DOMDocument; +use DOMElement; use Flow\ETL\Tests\FlowTestCase; use PHPUnit\Framework\Attributes\RequiresPhp; +use function Flow\ETL\DSL\array_to_rows; use function Flow\ETL\DSL\df; use function Flow\ETL\DSL\from_rows; use function Flow\ETL\DSL\html_element_schema; use function Flow\ETL\DSL\optional; use function Flow\ETL\DSL\ref; -use function Flow\ETL\DSL\row; -use function Flow\ETL\DSL\rows; use function Flow\ETL\DSL\schema; -use function Flow\ETL\DSL\xml_element_schema; +use function Flow\ETL\DSL\xml_schema; use function Flow\Types\DSL\type_html_element; +use function Flow\Types\DSL\type_instance_of; final class DOMElementNextSiblingTest extends FlowTestCase { @@ -25,14 +26,11 @@ final class DOMElementNextSiblingTest extends FlowTestCase public function test_dom_element_sibling_text_value(): void { $rows = df() - ->read(from_rows(rows( - schema(html_element_schema('html_element')), - row([ - 'html_element' => type_html_element()->cast( - '

User Name

01
', - ), - ]), - ))) + ->read(from_rows(array_to_rows([[ + 'html_element' => type_html_element()->cast( + '

User Name

01
', + ), + ]], schema(html_element_schema('html_element'))))) ->withEntry('user_details', ref('html_element')->htmlQuerySelector('section')) ->withEntry('user_name', ref('user_details')->htmlQuerySelector('h1')->domElementValue()) ->withEntry('user_id', optional(ref('user_details')->domElementNextSibling()->domElementValue())) @@ -54,14 +52,11 @@ public function test_dom_element_sibling_text_value(): void public function test_dom_element_sibling_text_value_when_only_element_is_allowed(): void { $rows = df() - ->read(from_rows(rows( - schema(html_element_schema('html_element')), - row([ - 'html_element' => type_html_element()->cast( - '

User Name

01
', - ), - ]), - ))) + ->read(from_rows(array_to_rows([[ + 'html_element' => type_html_element()->cast( + '

User Name

01
', + ), + ]], schema(html_element_schema('html_element'))))) ->withEntry('user_details', ref('html_element')->htmlQuerySelector('section')) ->withEntry('user_name', ref('user_details')->htmlQuerySelector('h1')->domElementValue()) ->withEntry('user_id', optional(ref('user_details')->domElementNextSibling()->domElementValue())) @@ -84,13 +79,14 @@ public function test_xml_sibling_element_value(): void $dom = new DOMDocument(); $dom->loadXML('User Name01'); + // a batch stores an element's own markup, so siblings are reached from the document it belongs to $rows = df() - ->read(from_rows(rows( - schema(xml_element_schema('xml_element')), - row(['xml_element' => $dom->getElementsByTagName('name')->item(0)]), - ))) - ->withEntry('user_name', ref('xml_element')->domElementValue()) - ->withEntry('user_id', ref('xml_element')->domElementNextSibling()->domElementValue()) + ->read(from_rows(array_to_rows([['xml' => $dom]], schema(xml_schema('xml'))))) + ->withEntry('user_name', ref('xml')->xpath('/user/name')->arrayGet('0')->domElementValue()) + ->withEntry( + 'user_id', + ref('xml')->xpath('/user/name')->arrayGet('0')->domElementNextSibling()->domElementValue(), + ) ->select('user_name', 'user_id') ->fetch(); @@ -104,4 +100,20 @@ public function test_xml_sibling_element_value(): void $rows->toArray(), ); } + + public function test_storing_the_element_of_an_xpath_chain_keeps_its_markup(): void + { + $dom = new DOMDocument(); + $dom->loadXML('User Name01'); + + $rows = df() + ->read(from_rows(array_to_rows([['xml' => $dom]], schema(xml_schema('xml'))))) + ->withEntry('sibling', ref('xml')->xpath('/user/name')->arrayGet('0')->domElementNextSibling()) + ->select('sibling') + ->fetch(); + + $sibling = type_instance_of(DOMElement::class)->assert($rows->column('sibling')->value(0)); + + static::assertSame('01', $sibling->ownerDocument?->saveXML($sibling)); + } } diff --git a/src/core/etl/tests/Flow/ETL/Tests/Integration/Function/DOMElementParentTest.php b/src/core/etl/tests/Flow/ETL/Tests/Integration/Function/DOMElementParentTest.php index 49f514a1d6..7b7d485e04 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Integration/Function/DOMElementParentTest.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Integration/Function/DOMElementParentTest.php @@ -7,12 +7,11 @@ use Flow\ETL\Tests\FlowTestCase; use PHPUnit\Framework\Attributes\RequiresPhp; +use function Flow\ETL\DSL\array_to_rows; use function Flow\ETL\DSL\df; use function Flow\ETL\DSL\from_rows; use function Flow\ETL\DSL\html_element_schema; use function Flow\ETL\DSL\ref; -use function Flow\ETL\DSL\row; -use function Flow\ETL\DSL\rows; use function Flow\ETL\DSL\schema; use function Flow\Types\DSL\type_html_element; @@ -22,14 +21,11 @@ final class DOMElementParentTest extends FlowTestCase public function test_dom_element_value_from_dom_document(): void { $rows = df() - ->read(from_rows(rows( - schema(html_element_schema('html_element')), - row([ - 'html_element' => type_html_element()->cast( - '

User Name

01
', - ), - ]), - ))) + ->read(from_rows(array_to_rows([[ + 'html_element' => type_html_element()->cast( + '

User Name

01
', + ), + ]], schema(html_element_schema('html_element'))))) ->withEntry('user_details', ref('html_element')->htmlQuerySelector('section')) ->withEntry('user_name', ref('user_details')->htmlQuerySelector('h1')->domElementValue()) ->withEntry( diff --git a/src/core/etl/tests/Flow/ETL/Tests/Integration/Function/DOMElementPreviousSiblingTest.php b/src/core/etl/tests/Flow/ETL/Tests/Integration/Function/DOMElementPreviousSiblingTest.php index 5489385f66..ff56b6555c 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Integration/Function/DOMElementPreviousSiblingTest.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Integration/Function/DOMElementPreviousSiblingTest.php @@ -8,15 +8,14 @@ use Flow\ETL\Tests\FlowTestCase; use PHPUnit\Framework\Attributes\RequiresPhp; +use function Flow\ETL\DSL\array_to_rows; use function Flow\ETL\DSL\df; use function Flow\ETL\DSL\from_rows; use function Flow\ETL\DSL\html_element_schema; use function Flow\ETL\DSL\optional; use function Flow\ETL\DSL\ref; -use function Flow\ETL\DSL\row; -use function Flow\ETL\DSL\rows; use function Flow\ETL\DSL\schema; -use function Flow\ETL\DSL\xml_element_schema; +use function Flow\ETL\DSL\xml_schema; use function Flow\Types\DSL\type_html_element; final class DOMElementPreviousSiblingTest extends FlowTestCase @@ -25,14 +24,11 @@ final class DOMElementPreviousSiblingTest extends FlowTestCase public function test_dom_element_sibling_text_value(): void { $rows = df() - ->read(from_rows(rows( - schema(html_element_schema('html_element')), - row([ - 'html_element' => type_html_element()->cast( - '
01

User Name

', - ), - ]), - ))) + ->read(from_rows(array_to_rows([[ + 'html_element' => type_html_element()->cast( + '
01

User Name

', + ), + ]], schema(html_element_schema('html_element'))))) ->withEntry('user_details', ref('html_element')->htmlQuerySelector('section')) ->withEntry('user_name', ref('user_details')->htmlQuerySelector('h1')->domElementValue()) ->withEntry('user_id', optional(ref('user_details')->domElementPreviousSibling()->domElementValue())) @@ -54,14 +50,11 @@ public function test_dom_element_sibling_text_value(): void public function test_dom_element_sibling_text_value_when_only_element_is_allowed(): void { $rows = df() - ->read(from_rows(rows( - schema(html_element_schema('html_element')), - row([ - 'html_element' => type_html_element()->cast( - '
01

User Name

', - ), - ]), - ))) + ->read(from_rows(array_to_rows([[ + 'html_element' => type_html_element()->cast( + '
01

User Name

', + ), + ]], schema(html_element_schema('html_element'))))) ->withEntry('user_details', ref('html_element')->htmlQuerySelector('section')) ->withEntry('user_name', ref('user_details')->htmlQuerySelector('h1')->domElementValue()) ->withEntry('user_id', optional(ref('user_details')->domElementPreviousSibling()->domElementValue())) @@ -84,13 +77,14 @@ public function test_xml_sibling_element_value(): void $dom = new DOMDocument(); $dom->loadXML('User Name01'); + // a batch stores an element's own markup, so siblings are reached from the document it belongs to $rows = df() - ->read(from_rows(rows( - schema(xml_element_schema('xml_element')), - row(['xml_element' => $dom->getElementsByTagName('number')->item(0)]), - ))) - ->withEntry('user_id', ref('xml_element')->domElementValue()) - ->withEntry('user_name', ref('xml_element')->domElementPreviousSibling()->domElementValue()) + ->read(from_rows(array_to_rows([['xml' => $dom]], schema(xml_schema('xml'))))) + ->withEntry('user_id', ref('xml')->xpath('/user/number')->arrayGet('0')->domElementValue()) + ->withEntry( + 'user_name', + ref('xml')->xpath('/user/number')->arrayGet('0')->domElementPreviousSibling()->domElementValue(), + ) ->select('user_name', 'user_id') ->fetch(); diff --git a/src/core/etl/tests/Flow/ETL/Tests/Integration/Function/DOMElementValueTest.php b/src/core/etl/tests/Flow/ETL/Tests/Integration/Function/DOMElementValueTest.php index 9181b147a2..789bd65d75 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Integration/Function/DOMElementValueTest.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Integration/Function/DOMElementValueTest.php @@ -7,11 +7,10 @@ use DOMDocument; use Flow\ETL\Tests\FlowTestCase; +use function Flow\ETL\DSL\array_to_rows; use function Flow\ETL\DSL\df; use function Flow\ETL\DSL\from_rows; use function Flow\ETL\DSL\ref; -use function Flow\ETL\DSL\row; -use function Flow\ETL\DSL\rows; use function Flow\ETL\DSL\schema; use function Flow\ETL\DSL\xml_element_schema; use function Flow\ETL\DSL\xml_schema; @@ -27,7 +26,7 @@ public function test_dom_element_cast_as_string(): void $document->loadXml('User Name 01'); $rows = df() - ->read(from_rows(rows(schema(xml_schema('html_raw')), row(['html_raw' => $document])))) + ->read(from_rows(array_to_rows([['html_raw' => $document]], schema(xml_schema('html_raw'))))) ->withEntry('html', ref('html_raw')->cast(type_string())) ->drop('html_raw') ->fetch(); @@ -43,10 +42,9 @@ public function test_dom_element_cast_as_string(): void public function test_dom_element_value(): void { $rows = df() - ->read(from_rows(rows( - schema(xml_element_schema('node')), - row(['node' => type_xml_element()->cast('User Name 01')]), - ))) + ->read(from_rows(array_to_rows([[ + 'node' => type_xml_element()->cast('User Name 01'), + ]], schema(xml_element_schema('node'))))) ->withEntry('user_name', ref('node')->domElementValue()) ->drop('node') ->fetch(); @@ -62,10 +60,9 @@ public function test_dom_element_value(): void public function test_dom_element_value_from_dom_document(): void { $rows = df() - ->read(from_rows(rows( - schema(xml_schema('node')), - row(['node' => type_xml()->cast('User Name 01')]), - ))) + ->read(from_rows(array_to_rows([[ + 'node' => type_xml()->cast('User Name 01'), + ]], schema(xml_schema('node'))))) ->withEntry('user_name', ref('node')->domElementValue()) ->drop('node') ->fetch(); @@ -84,7 +81,7 @@ public function test_dom_element_value_on_dom_document(): void $document->loadXml('User Name 01'); $rows = df() - ->read(from_rows(rows(schema(xml_schema('html_raw')), row(['html_raw' => $document])))) + ->read(from_rows(array_to_rows([['html_raw' => $document]], schema(xml_schema('html_raw'))))) ->withEntry('html', ref('html_raw')->domElementValue()) ->drop('html_raw') ->fetch(); @@ -100,10 +97,9 @@ public function test_dom_element_value_on_dom_document(): void public function test_dom_element_value_on_xpath_result(): void { $rows = df() - ->read(from_rows(rows( - schema(xml_schema('node')), - row(['node' => type_xml()->cast('User Name 01')]), - ))) + ->read(from_rows(array_to_rows([[ + 'node' => type_xml()->cast('User Name 01'), + ]], schema(xml_schema('node'))))) ->withEntry('user_name', ref('node')->xpath('name')->domElementValue()) ->drop('node') ->fetch(); diff --git a/src/core/etl/tests/Flow/ETL/Tests/Integration/Function/EnumNameTest.php b/src/core/etl/tests/Flow/ETL/Tests/Integration/Function/EnumNameTest.php index c8b19104ed..08510199fb 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Integration/Function/EnumNameTest.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Integration/Function/EnumNameTest.php @@ -8,14 +8,13 @@ use Flow\ETL\Tests\Fixtures\Enum\BackedIntEnum; use Flow\ETL\Tests\FlowTestCase; +use function Flow\ETL\DSL\array_to_rows; use function Flow\ETL\DSL\data_frame; use function Flow\ETL\DSL\enum_name; use function Flow\ETL\DSL\enum_schema; use function Flow\ETL\DSL\from_rows; use function Flow\ETL\DSL\optional; use function Flow\ETL\DSL\ref; -use function Flow\ETL\DSL\row; -use function Flow\ETL\DSL\rows; use function Flow\ETL\DSL\schema; use function Flow\ETL\DSL\to_memory; use function Flow\Types\DSL\type_equals; @@ -28,10 +27,9 @@ public function test_enum_name_produces_string_entry_for_int_backed_enum(): void static::assertTrue(type_equals( type_string(), data_frame() - ->read(from_rows(rows( - schema(enum_schema('e', BackedIntEnum::class)), - row(['e' => BackedIntEnum::one]), - ))) + ->read(from_rows(array_to_rows([[ + 'e' => BackedIntEnum::one, + ]], schema(enum_schema('e', BackedIntEnum::class))))) ->withEntry('code', enum_name(ref('e'))) ->schema() ->get('code') @@ -42,11 +40,10 @@ public function test_enum_name_produces_string_entry_for_int_backed_enum(): void public function test_enum_name_writes_null_for_null_enum_in_permissive_mode(): void { data_frame() - ->read(from_rows(rows( - schema(enum_schema('e', BackedIntEnum::class, nullable: true)), - row(['e' => BackedIntEnum::one]), - row(['e' => null]), - ))) + ->read(from_rows(array_to_rows([ + ['e' => BackedIntEnum::one], + ['e' => null], + ], schema(enum_schema('e', BackedIntEnum::class, nullable: true))))) ->withEntry('code', optional(enum_name(ref('e')))) ->select('code') ->write(to_memory($memory = new ArrayMemory())) @@ -58,11 +55,10 @@ public function test_enum_name_writes_null_for_null_enum_in_permissive_mode(): v public function test_enum_name_writes_case_names(): void { data_frame() - ->read(from_rows(rows( - schema(enum_schema('e', BackedIntEnum::class)), - row(['e' => BackedIntEnum::one]), - row(['e' => BackedIntEnum::two]), - ))) + ->read(from_rows(array_to_rows([ + ['e' => BackedIntEnum::one], + ['e' => BackedIntEnum::two], + ], schema(enum_schema('e', BackedIntEnum::class))))) ->withEntry('code', enum_name(ref('e'))) ->select('code') ->write(to_memory($memory = new ArrayMemory())) diff --git a/src/core/etl/tests/Flow/ETL/Tests/Integration/Function/EnumValueTest.php b/src/core/etl/tests/Flow/ETL/Tests/Integration/Function/EnumValueTest.php index 5069f9b152..8c00152044 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Integration/Function/EnumValueTest.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Integration/Function/EnumValueTest.php @@ -9,14 +9,13 @@ use Flow\ETL\Tests\Fixtures\Enum\BackedStringEnum; use Flow\ETL\Tests\FlowTestCase; +use function Flow\ETL\DSL\array_to_rows; use function Flow\ETL\DSL\data_frame; use function Flow\ETL\DSL\enum_schema; use function Flow\ETL\DSL\enum_value; use function Flow\ETL\DSL\from_rows; use function Flow\ETL\DSL\optional; use function Flow\ETL\DSL\ref; -use function Flow\ETL\DSL\row; -use function Flow\ETL\DSL\rows; use function Flow\ETL\DSL\schema; use function Flow\ETL\DSL\to_memory; use function Flow\Types\DSL\type_equals; @@ -30,10 +29,9 @@ public function test_enum_value_produces_integer_entry_for_int_backed_enum(): vo static::assertTrue(type_equals( type_integer(), data_frame() - ->read(from_rows(rows( - schema(enum_schema('e', BackedIntEnum::class)), - row(['e' => BackedIntEnum::one]), - ))) + ->read(from_rows(array_to_rows([[ + 'e' => BackedIntEnum::one, + ]], schema(enum_schema('e', BackedIntEnum::class))))) ->withEntry('code', enum_value(ref('e'))) ->schema() ->get('code') @@ -46,10 +44,9 @@ public function test_enum_value_produces_string_entry_for_string_backed_enum(): static::assertTrue(type_equals( type_string(), data_frame() - ->read(from_rows(rows( - schema(enum_schema('e', BackedStringEnum::class)), - row(['e' => BackedStringEnum::one]), - ))) + ->read(from_rows(array_to_rows([[ + 'e' => BackedStringEnum::one, + ]], schema(enum_schema('e', BackedStringEnum::class))))) ->withEntry('code', enum_value(ref('e'))) ->schema() ->get('code') @@ -60,11 +57,10 @@ public function test_enum_value_produces_string_entry_for_string_backed_enum(): public function test_enum_value_writes_null_for_null_enum_in_permissive_mode(): void { data_frame() - ->read(from_rows(rows( - schema(enum_schema('e', BackedIntEnum::class, nullable: true)), - row(['e' => BackedIntEnum::one]), - row(['e' => null]), - ))) + ->read(from_rows(array_to_rows([ + ['e' => BackedIntEnum::one], + ['e' => null], + ], schema(enum_schema('e', BackedIntEnum::class, nullable: true))))) ->withEntry('code', optional(enum_value(ref('e')))) ->select('code') ->write(to_memory($memory = new ArrayMemory())) @@ -76,11 +72,10 @@ public function test_enum_value_writes_null_for_null_enum_in_permissive_mode(): public function test_enum_value_writes_backing_values(): void { data_frame() - ->read(from_rows(rows( - schema(enum_schema('e', BackedIntEnum::class)), - row(['e' => BackedIntEnum::one]), - row(['e' => BackedIntEnum::two]), - ))) + ->read(from_rows(array_to_rows([ + ['e' => BackedIntEnum::one], + ['e' => BackedIntEnum::two], + ], schema(enum_schema('e', BackedIntEnum::class))))) ->withEntry('code', enum_value(ref('e'))) ->select('code') ->write(to_memory($memory = new ArrayMemory())) diff --git a/src/core/etl/tests/Flow/ETL/Tests/Integration/Function/HTMLQuerySelectorAllTest.php b/src/core/etl/tests/Flow/ETL/Tests/Integration/Function/HTMLQuerySelectorAllTest.php index 609146818c..c436fe7d1b 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Integration/Function/HTMLQuerySelectorAllTest.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Integration/Function/HTMLQuerySelectorAllTest.php @@ -8,12 +8,11 @@ use PHPUnit\Framework\Attributes\RequiresPhp; use PHPUnit\Framework\TestCase; +use function Flow\ETL\DSL\array_to_rows; use function Flow\ETL\DSL\df; use function Flow\ETL\DSL\from_rows; use function Flow\ETL\DSL\html_schema; use function Flow\ETL\DSL\ref; -use function Flow\ETL\DSL\row; -use function Flow\ETL\DSL\rows; use function Flow\ETL\DSL\schema; use const LIBXML_HTML_NOIMPLIED; @@ -36,7 +35,7 @@ public function test_invalid_query_all_on_html_document(): void ], ], df() - ->read(from_rows(rows(schema(html_schema('html_raw')), row(['html_raw' => $html])))) + ->read(from_rows(array_to_rows([['html_raw' => $html]], schema(html_schema('html_raw'))))) ->withEntry('html', ref('html_raw')->htmlQuerySelectorAll('body div p')) ->drop('html_raw') ->fetch() @@ -63,7 +62,7 @@ public function test_valid_query_all_on_html_document(): void ], ], df() - ->read(from_rows(rows(schema(html_schema('html_raw')), row(['html_raw' => $html])))) + ->read(from_rows(array_to_rows([['html_raw' => $html]], schema(html_schema('html_raw'))))) ->withEntry('html', ref('html_raw')->htmlQuerySelectorAll('body div span')) ->drop('html_raw') ->fetch() diff --git a/src/core/etl/tests/Flow/ETL/Tests/Integration/Function/HTMLQuerySelectorTest.php b/src/core/etl/tests/Flow/ETL/Tests/Integration/Function/HTMLQuerySelectorTest.php index 005156d6b2..c33e3ef190 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Integration/Function/HTMLQuerySelectorTest.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Integration/Function/HTMLQuerySelectorTest.php @@ -8,12 +8,11 @@ use PHPUnit\Framework\Attributes\RequiresPhp; use PHPUnit\Framework\TestCase; +use function Flow\ETL\DSL\array_to_rows; use function Flow\ETL\DSL\df; use function Flow\ETL\DSL\from_rows; use function Flow\ETL\DSL\html_schema; use function Flow\ETL\DSL\ref; -use function Flow\ETL\DSL\row; -use function Flow\ETL\DSL\rows; use function Flow\ETL\DSL\schema; // @mago-ignore analysis:unavailable-method @@ -33,7 +32,7 @@ public function test_invalid_query_on_html_document(): void ], ], df() - ->read(from_rows(rows(schema(html_schema('html_raw')), row(['html_raw' => $html])))) + ->read(from_rows(array_to_rows([['html_raw' => $html]], schema(html_schema('html_raw'))))) ->withEntry('html_element', ref('html_raw')->htmlQuerySelector('body div p')) ->drop('html_raw') ->fetch() @@ -71,7 +70,7 @@ public function test_valid_query_on_html_document(): void ], ], df() - ->read(from_rows(rows(schema(html_schema('html_raw')), row(['html_raw' => $html])))) + ->read(from_rows(array_to_rows([['html_raw' => $html]], schema(html_schema('html_raw'))))) ->withEntry('containers', ref('html_raw')->htmlQuerySelectorAll('body div')->expand()) ->withEntry('span_element', ref('containers')->htmlQuerySelector('body span')->domElementValue()) ->withEntry('p_element', ref('containers')->htmlQuerySelector('p')->domElementValue()) diff --git a/src/core/etl/tests/Flow/ETL/Tests/Integration/Function/MatchCasesTest.php b/src/core/etl/tests/Flow/ETL/Tests/Integration/Function/MatchCasesTest.php index d4d825af8b..c209b0698a 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Integration/Function/MatchCasesTest.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Integration/Function/MatchCasesTest.php @@ -6,14 +6,13 @@ use Flow\ETL\Tests\FlowTestCase; +use function Flow\ETL\DSL\array_to_rows; use function Flow\ETL\DSL\df; use function Flow\ETL\DSL\from_rows; use function Flow\ETL\DSL\lit; use function Flow\ETL\DSL\match_cases; use function Flow\ETL\DSL\match_condition; use function Flow\ETL\DSL\ref; -use function Flow\ETL\DSL\row; -use function Flow\ETL\DSL\rows; use function Flow\ETL\DSL\schema; use function Flow\ETL\DSL\string_schema; use function Flow\Types\DSL\type_boolean; @@ -23,14 +22,13 @@ final class MatchCasesTest extends FlowTestCase { public function test_case_match(): void { - $rows = rows( - schema(string_schema('string')), - row(['string' => 'string-with-dashes']), - row(['string' => '123']), - row(['string' => '14%']), - row(['string' => '+14']), - row(['string' => '']), - ); + $rows = array_to_rows([ + ['string' => 'string-with-dashes'], + ['string' => '123'], + ['string' => '14%'], + ['string' => '+14'], + ['string' => ''], + ], schema(string_schema('string'))); $output = df() ->read(from_rows($rows)) diff --git a/src/core/etl/tests/Flow/ETL/Tests/Integration/Function/OnEachTest.php b/src/core/etl/tests/Flow/ETL/Tests/Integration/Function/OnEachTest.php index c97c622a8a..fa8e4d1cb0 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Integration/Function/OnEachTest.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Integration/Function/OnEachTest.php @@ -28,7 +28,8 @@ public function test_on_each_function(): void static::assertEquals( [ ['array' => ['a' => '1', 'b' => '2', 'c' => '3', 'd' => '4', 'e' => '5']], - ['array' => ['f' => '1', 'g' => '2.3', 'h' => '3', 'i' => '4', 'j' => null]], + // an optional and nullable structure element given null reads back absent (ruling 4) + ['array' => ['f' => '1', 'g' => '2.3', 'h' => '3', 'i' => '4']], ], $results, ); diff --git a/src/core/etl/tests/Flow/ETL/Tests/Integration/Function/UntypedIntermediateTest.php b/src/core/etl/tests/Flow/ETL/Tests/Integration/Function/UntypedIntermediateTest.php new file mode 100644 index 0000000000..06a62b3c7e --- /dev/null +++ b/src/core/etl/tests/Flow/ETL/Tests/Integration/Function/UntypedIntermediateTest.php @@ -0,0 +1,58 @@ +) still hands its values to a parent + * that can. + * + * @return Generator + */ + public static function trees(): Generator + { + yield 'upper of an array_get' => [ref('j')->jsonDecode()->arrayGet('name')->upper(), 'string', 'ANN']; + yield 'is null of an array_get' => [ref('j')->jsonDecode()->arrayGet('name')->isNull(), 'boolean', false]; + yield 'size of an array_get' => [ref('j')->jsonDecode()->arrayGet('tags')->size(), 'integer', 2]; + yield 'cast of an array_get' => [ref('j')->jsonDecode()->arrayGet('n')->cast(type_string()), 'string', '3']; + yield 'size of array_keys' => [ref('j')->jsonDecode()->arrayKeys()->size(), 'integer', 3]; + } + + #[DataProvider('trees')] + public function test_a_parent_of_an_untyped_node_runs(ScalarFunction $tree, string $type, mixed $value): void + { + $rows = data_frame() + ->read(from_array([['j' => '{"name":"ann","tags":["a","b"],"n":3}']], schema(json_schema('j')))) + ->withEntry('x', $tree) + ->fetch(); + + static::assertSame($type, $rows->schema()->get('x')->type()->toString()); + static::assertSame($value, $rows->values(0)['x']); + } + + public function test_storing_an_untyped_node_is_refused_at_bind(): void + { + $this->expectException(SchemaNotDerivableException::class); + + data_frame() + ->read(from_array([['j' => '{"name":"ann"}']], schema(json_schema('j')))) + ->withEntry('x', ref('j')->jsonDecode()->arrayGet('name')) + ->fetch(); + } +} diff --git a/src/core/etl/tests/Flow/ETL/Tests/Integration/Loader/File/FilesSink/FilesSinkTest.php b/src/core/etl/tests/Flow/ETL/Tests/Integration/Loader/File/FilesSink/FilesSinkTest.php new file mode 100644 index 0000000000..fbf336f780 --- /dev/null +++ b/src/core/etl/tests/Flow/ETL/Tests/Integration/Loader/File/FilesSink/FilesSinkTest.php @@ -0,0 +1,236 @@ +cleanFiles(); + } + + public static function save_modes(): Generator + { + yield 'overwrite' => [SaveMode::Overwrite]; + yield 'exception if exists' => [SaveMode::ExceptionIfExists]; + yield 'append' => [SaveMode::Append]; + } + + #[DataProvider('save_modes')] + public function test_abandon_after_a_publish_that_failed_part_way_removes_only_what_was_not_published(SaveMode $saveMode): void + { + $this->setupFiles([__FUNCTION__ => []]); + + $files = new FilesSink( + new FailingCloseFilesystem($this->fs, healthyStreams: 1), + $this->getPath(__FUNCTION__ . '/file.txt'), + $saveMode, + ); + $files->writeTo([partition('p', 'a')])->append('a'); + $b = $files->writeTo([partition('p', 'b')]); + $b->append('b'); + + try { + $files->publish(); + static::fail('publish() was expected to throw'); + } catch (FilesystemRuntimeException $failure) { + static::assertSame('Closing "' . $b->path()->uri() . '" failed', $failure->getMessage()); + } + + $files->abandon(); + + static::assertSame('a', file_get_contents($this->getPath(__FUNCTION__ . '/p=a/file.txt')->path())); + static::assertFileDoesNotExist($this->getPath(__FUNCTION__ . '/p=b/file.txt')->path()); + static::assertFileDoesNotExist($b->path()->path()); + } + + public function test_abandon_removes_every_file_even_when_a_stream_fails_to_close(): void + { + $this->setupFiles([__FUNCTION__ => []]); + + $files = new FilesSink( + new FailingCloseFilesystem($this->fs, failingStreams: 2), + $this->getPath(__FUNCTION__ . '/file.txt'), + SaveMode::Overwrite, + ); + $a = $files->writeTo([partition('p', 'a')]); + $a->append('a'); + $b = $files->writeTo([partition('p', 'b')]); + $b->append('b'); + + try { + $files->abandon(); + static::fail('abandon() was expected to throw'); + } catch (FilesystemRuntimeException $failure) { + static::assertSame('Closing "' . $a->path()->uri() . '" failed', $failure->getMessage()); + } + + static::assertFileDoesNotExist($a->path()->path()); + static::assertFileDoesNotExist($b->path()->path()); + } + + public function test_abandon_leaves_the_destination_untouched(): void + { + $this->setupFiles([ + __FUNCTION__ => [ + 'file.txt' => 'existing content', + ], + ]); + + $file = $this->getPath(__FUNCTION__ . '/file.txt'); + $files = new FilesSink($this->fs, $file, SaveMode::Overwrite); + $stream = $files->writeTo(); + $stream->append('half written content'); + + $files->abandon(); + + static::assertSame('existing content', file_get_contents($file->path())); + // list('/*') cannot see a dot-prefixed tmp file, so the leftover is asserted by its own path + static::assertFileDoesNotExist($stream->path()->path()); + } + + public function test_abandon_leaves_a_destination_it_did_not_create(): void + { + $this->setupFiles([ + __FUNCTION__ => [ + 'file.txt' => 'existing content', + ], + ]); + + $file = $this->getPath(__FUNCTION__ . '/file.txt'); + $files = new FilesSink($this->fs, $file, SaveMode::Ignore); + $files->writeTo()->append('ignored content'); + + $files->abandon(); + + static::assertFileExists($file->path()); + static::assertSame('existing content', file_get_contents($file->path())); + } + + public function test_abandon_removes_the_file_it_created_under_append(): void + { + $this->setupFiles([ + __FUNCTION__ => [ + 'file.txt' => 'existing content', + ], + ]); + + $file = $this->getPath(__FUNCTION__ . '/file.txt'); + $files = new FilesSink($this->fs, $file, SaveMode::Append); + $files->writeTo()->append('half written content'); + + $files->abandon(); + + static::assertCount( + 1, + $files = iterator_to_array($this->fs()->list(path($file->parentDirectory()->path() . '/*'))), + ); + static::assertSame('file.txt', $files[0]->path->basename()); + static::assertSame('existing content', file_get_contents($file->path())); + } + + public function test_abandon_removes_the_file_it_created_under_exception_if_exists(): void + { + $this->setupFiles([__FUNCTION__ => []]); + + $file = $this->getPath(__FUNCTION__ . '/file.txt'); + $files = new FilesSink($this->fs, $file, SaveMode::ExceptionIfExists); + $files->writeTo()->append('half written content'); + + $files->abandon(); + + static::assertFileDoesNotExist($file->path()); + } + + public function test_abandon_removes_the_temporary_file(): void + { + $this->setupFiles([__FUNCTION__ => []]); + + $file = $this->getPath(__FUNCTION__ . '/file.txt'); + $files = new FilesSink($this->fs, $file, SaveMode::Overwrite); + $stream = $files->writeTo(); + $stream->append('half written content'); + + static::assertFileExists($stream->path()->path()); + + $files->abandon(); + + static::assertFileDoesNotExist($stream->path()->path()); + static::assertFileDoesNotExist($file->path()); + } + + public function test_abandoning_a_published_set_does_nothing(): void + { + $this->setupFiles([__FUNCTION__ => []]); + + $file = $this->getPath(__FUNCTION__ . '/file.txt'); + $files = new FilesSink($this->fs, $file, SaveMode::Overwrite); + $files->writeTo()->append('some content'); + $files->publish(); + + $files->abandon(); + + static::assertSame('some content', file_get_contents($file->path())); + } + + public function test_open_two_write_streams_to_stdout(): void + { + $this->expectExceptionMessage('Only one stream can be open at the same time for php://stdout'); + + // one instance: StdOutFilesystem holds its open targets per instance + $stdout = stdout_filesystem(); + + $this->files(path('stdout://a.stdout'), $stdout)->writeTo(); + $this->files(path('stdout://b.stdout'), $stdout)->writeTo(); + } + + public function test_scan(): void + { + $this->setupFiles([ + __FUNCTION__ => [ + 'file1.txt' => 'file1 content', + 'file2.txt' => 'file2 content', + 'folder' => [ + 'file3.txt' => 'file3 content', + 'file4.txt' => 'file4 content', + ], + ], + ]); + + static::assertCount(4, iterator_to_array( + (new FileListing($this->fs))->list($this->getPath(__FUNCTION__ . '/**/*.txt'), new KeepAll()), + false, + )); + } + + public function test_write_to_stdout(): void + { + static::assertTrue($this->files(path('stdout://a.stdout'), stdout_filesystem())->writeTo()->isOpen()); + } + + protected function saveMode(): SaveMode + { + return exception_if_exists(); + } +} diff --git a/src/core/etl/tests/Flow/ETL/Tests/Integration/Filesystem/FilesSink/FilesSinkTestCase.php b/src/core/etl/tests/Flow/ETL/Tests/Integration/Loader/File/FilesSink/FilesSinkTestCase.php similarity index 85% rename from src/core/etl/tests/Flow/ETL/Tests/Integration/Filesystem/FilesSink/FilesSinkTestCase.php rename to src/core/etl/tests/Flow/ETL/Tests/Integration/Loader/File/FilesSink/FilesSinkTestCase.php index 41da826ed0..593e84e9bc 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Integration/Filesystem/FilesSink/FilesSinkTestCase.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Integration/Loader/File/FilesSink/FilesSinkTestCase.php @@ -2,10 +2,10 @@ declare(strict_types=1); -namespace Flow\ETL\Tests\Integration\Filesystem\FilesSink; +namespace Flow\ETL\Tests\Integration\Loader\File\FilesSink; -use Flow\ETL\Filesystem\FilesSink; use Flow\ETL\Filesystem\SaveMode; +use Flow\ETL\Loader\File\FilesSink; use Flow\ETL\Tests\FlowIntegrationTestCase; use Flow\Filesystem\Filesystem; use Flow\Filesystem\Path; diff --git a/src/core/etl/tests/Flow/ETL/Tests/Integration/Loader/File/FilesSink/NotPartitioned/AppendModeTest.php b/src/core/etl/tests/Flow/ETL/Tests/Integration/Loader/File/FilesSink/NotPartitioned/AppendModeTest.php new file mode 100644 index 0000000000..2aed26b855 --- /dev/null +++ b/src/core/etl/tests/Flow/ETL/Tests/Integration/Loader/File/FilesSink/NotPartitioned/AppendModeTest.php @@ -0,0 +1,70 @@ +cleanFiles(); + } + + public function test_open_stream_for_existing_file(): void + { + $this->setupFiles([ + __FUNCTION__ => [ + 'existing-file.txt' => 'some content', + ], + ]); + $file = $this->getPath(__FUNCTION__ . '/existing-file.txt'); + static::assertFileExists($file->path()); + + $files = $this->files($file); + $files->writeTo()->append('new content'); + $files->publish(); + + $files = iterator_to_array($this->fs()->list(path($file->parentDirectory()->path() . '/*'))); + + static::assertCount(2, $files); + + foreach ($files as $streamFile) { + static::assertStringStartsWith('existing-file', $streamFile->path->basename()); + static::assertStringEndsWith('.txt', $streamFile->path->basename()); + } + } + + public function test_open_stream_for_non_existing_file(): void + { + $this->setupFiles([ + __FUNCTION__ => [], + ]); + $file = $this->getPath(__FUNCTION__ . '/non-existing-file.txt'); + static::assertFileDoesNotExist($file->path()); + + $files = $this->files($file); + $files->writeTo()->append('new content'); + $files->publish(); + + $files = iterator_to_array($this->fs()->list(path($file->parentDirectory()->path() . '/*'))); + + static::assertCount(1, $files); + static::assertSame('non-existing-file.txt', $files[0]->path->basename()); + } + + protected function saveMode(): SaveMode + { + return append(); + } +} diff --git a/src/core/etl/tests/Flow/ETL/Tests/Integration/Loader/File/FilesSink/NotPartitioned/ExceptionIfExistsModeTest.php b/src/core/etl/tests/Flow/ETL/Tests/Integration/Loader/File/FilesSink/NotPartitioned/ExceptionIfExistsModeTest.php new file mode 100644 index 0000000000..9efcfa5472 --- /dev/null +++ b/src/core/etl/tests/Flow/ETL/Tests/Integration/Loader/File/FilesSink/NotPartitioned/ExceptionIfExistsModeTest.php @@ -0,0 +1,57 @@ +cleanFiles(); + } + + public function test_open_stream_for_existing_file(): void + { + $this->expectExceptionMessageMatches( + '/Destination path (.*) already exists, please change path to different or set different SaveMode/', + ); + + $this->setupFiles([ + __FUNCTION__ => [ + 'existing-file.txt' => 'some content', + ], + ]); + + $this->files($this->getPath(__FUNCTION__ . '/existing-file.txt'))->writeTo(); + } + + public function test_open_stream_for_non_existing_file(): void + { + $this->setupFiles([ + __FUNCTION__ => [], + ]); + $file = $this->getPath(__FUNCTION__ . '/non-existing-file.txt'); + + $files = $this->files($file); + $files->writeTo()->append('some content'); + $files->publish(); + + static::assertFileExists($file->path()); + static::assertSame('some content', file_get_contents($file->path())); + } + + protected function saveMode(): SaveMode + { + return exception_if_exists(); + } +} diff --git a/src/core/etl/tests/Flow/ETL/Tests/Integration/Loader/File/FilesSink/NotPartitioned/IgnoreModeTest.php b/src/core/etl/tests/Flow/ETL/Tests/Integration/Loader/File/FilesSink/NotPartitioned/IgnoreModeTest.php new file mode 100644 index 0000000000..94b22da14b --- /dev/null +++ b/src/core/etl/tests/Flow/ETL/Tests/Integration/Loader/File/FilesSink/NotPartitioned/IgnoreModeTest.php @@ -0,0 +1,59 @@ +cleanFiles(); + } + + public function test_open_stream_for_existing_file(): void + { + $this->setupFiles([ + __FUNCTION__ => [ + 'existing-file.txt' => 'some content', + ], + ]); + $path = $this->getPath(__FUNCTION__ . '/existing-file.txt'); + + $files = $this->files($path); + $files->writeTo()->append('different content'); + $files->publish(); + + static::assertFileExists($path->path()); + static::assertSame('some content', file_get_contents($path->path())); + } + + public function test_open_stream_for_non_existing_file(): void + { + $this->setupFiles([ + __FUNCTION__ => [], + ]); + $path = $this->getPath(__FUNCTION__ . '/non-existing-file.txt'); + + $files = $this->files($path); + $files->writeTo()->append('some content'); + $files->publish(); + + static::assertFileExists($path->path()); + static::assertSame('some content', file_get_contents($path->path())); + } + + protected function saveMode(): SaveMode + { + return ignore(); + } +} diff --git a/src/core/etl/tests/Flow/ETL/Tests/Integration/Loader/File/FilesSink/NotPartitioned/OverwriteModeTest.php b/src/core/etl/tests/Flow/ETL/Tests/Integration/Loader/File/FilesSink/NotPartitioned/OverwriteModeTest.php new file mode 100644 index 0000000000..c1afb09c31 --- /dev/null +++ b/src/core/etl/tests/Flow/ETL/Tests/Integration/Loader/File/FilesSink/NotPartitioned/OverwriteModeTest.php @@ -0,0 +1,70 @@ +cleanFiles(); + } + + public function test_open_stream_for_existing_file(): void + { + $this->setupFiles([ + __FUNCTION__ => [ + 'existing-file.txt' => 'some content', + ], + ]); + + $path = $this->getPath(__FUNCTION__ . '/existing-file.txt'); + $files = $this->files($path); + + $fileStream = $files->writeTo(); + static::assertStringContainsString(FilesSink::FLOW_TMP_FILE_PREFIX, $fileStream->path()->path()); + $fileStream->append('some other content'); + static::assertSame('some content', file_get_contents($path->path())); + + $files->publish(); + + static::assertSame('some other content', file_get_contents($path->path())); + + static::assertCount( + 1, + $files = iterator_to_array($this->fs()->list(path($path->parentDirectory()->path() . '/*'))), + ); + static::assertSame('existing-file.txt', $files[0]->path->basename()); + } + + public function test_open_stream_for_non_existing_file(): void + { + $this->setupFiles([__FUNCTION__ => []]); + $path = $this->getPath(__FUNCTION__ . '/non-existing-file.txt'); + + $files = $this->files($path); + $files->writeTo()->append('some content'); + $files->publish(); + + static::assertFileExists($path->path()); + static::assertSame('some content', file_get_contents($path->path())); + } + + protected function saveMode(): SaveMode + { + return overwrite(); + } +} diff --git a/src/core/etl/tests/Flow/ETL/Tests/Integration/Loader/File/FilesSink/Partitioned/AppendModeTest.php b/src/core/etl/tests/Flow/ETL/Tests/Integration/Loader/File/FilesSink/Partitioned/AppendModeTest.php new file mode 100644 index 0000000000..f39cf2c6c7 --- /dev/null +++ b/src/core/etl/tests/Flow/ETL/Tests/Integration/Loader/File/FilesSink/Partitioned/AppendModeTest.php @@ -0,0 +1,96 @@ +cleanFiles(); + } + + public function test_open_stream_for_existing_partition_with_existing_file(): void + { + $this->setupFiles([ + __FUNCTION__ => [ + 'partition=value' => [ + 'file.txt' => 'file content', + ], + ], + ]); + + $file = $this->getPath(__FUNCTION__ . '/file.txt'); + $files = $this->files($file); + + $files->writeTo([new Partition('partition', 'value')])->append('appended content'); + $files->publish(); + + $files = iterator_to_array($this->fs()->list(path($file->parentDirectory()->path() . '/**/*.txt'))); + + static::assertCount(2, $files); + + foreach ($files as $streamFile) { + static::assertStringStartsWith('file', $streamFile->path->basename()); + static::assertStringEndsWith('.txt', $streamFile->path->basename()); + } + } + + public function test_open_stream_for_existing_partition_without_existing_file(): void + { + $this->setupFiles([ + __FUNCTION__ => [ + 'partition=value' => [], + ], + ]); + $file = $this->getPath(__FUNCTION__ . '/file.txt'); + $files = $this->files($file); + + $files->writeTo([new Partition('partition', 'value')])->append('appended content'); + $files->publish(); + + $files = iterator_to_array($this->fs()->list(path($file->parentDirectory()->path() . '/**/*.txt'))); + + static::assertCount(1, $files); + + static::assertSame('file.txt', $files[0]->path->basename()); + static::assertSame('appended content', file_get_contents($files[0]->path->path())); + } + + public function test_open_stream_for_non_existing_partition(): void + { + $this->setupFiles([ + __FUNCTION__ => [], + ]); + $file = $this->getPath(__FUNCTION__ . '/file.txt'); + $files = $this->files($file); + + $files->writeTo([new Partition('partition', 'value')])->append('appended content'); + $files->publish(); + + $files = iterator_to_array($this->fs()->list(path($file->parentDirectory()->path() . '/partition=value/*'))); + + static::assertCount(1, $files); + + static::assertSame('file.txt', $files[0]->path->basename()); + static::assertSame('appended content', file_get_contents($files[0]->path->path())); + } + + protected function saveMode(): SaveMode + { + return append(); + } +} diff --git a/src/core/etl/tests/Flow/ETL/Tests/Integration/Loader/File/FilesSink/Partitioned/ExceptionIfExistsModeTest.php b/src/core/etl/tests/Flow/ETL/Tests/Integration/Loader/File/FilesSink/Partitioned/ExceptionIfExistsModeTest.php new file mode 100644 index 0000000000..7835809f2b --- /dev/null +++ b/src/core/etl/tests/Flow/ETL/Tests/Integration/Loader/File/FilesSink/Partitioned/ExceptionIfExistsModeTest.php @@ -0,0 +1,87 @@ +cleanFiles(); + } + + public function test_open_stream_for_existing_partition_with_existing_file(): void + { + $this->expectExceptionMessageMatches( + '/Destination path (.*) already exists, please change path to different or set different SaveMode/', + ); + + $this->setupFiles([ + __FUNCTION__ => [ + 'partition=value' => [ + 'file.txt' => 'file content', + ], + ], + ]); + + $this->files($this->getPath(__FUNCTION__ . '/file.txt'))->writeTo([new Partition('partition', 'value')]); + } + + public function test_open_stream_for_existing_partition_without_existing_file(): void + { + $this->setupFiles([ + __FUNCTION__ => [ + 'partition=value' => [], + ], + ]); + $file = $this->getPath(__FUNCTION__ . '/file.txt'); + $files = $this->files($file); + + $files->writeTo([new Partition('partition', 'value')])->append('file content'); + $files->publish(); + + $files = iterator_to_array($this->fs()->list(path($file->parentDirectory()->path() . '/**/*.txt'))); + + static::assertCount(1, $files); + + static::assertSame('file.txt', $files[0]->path->basename()); + static::assertSame('file content', file_get_contents($files[0]->path->path())); + } + + public function test_open_stream_for_non_existing_partition(): void + { + $this->setupFiles([ + __FUNCTION__ => [], + ]); + $file = $this->getPath(__FUNCTION__ . '/file.txt'); + $files = $this->files($file); + + $files->writeTo([new Partition('partition', 'value')])->append('file content'); + $files->publish(); + + $files = iterator_to_array($this->fs()->list(path($file->parentDirectory()->path() . '/partition=value/*'))); + + static::assertCount(1, $files); + + static::assertSame('file.txt', $files[0]->path->basename()); + static::assertSame('file content', file_get_contents($files[0]->path->path())); + } + + protected function saveMode(): SaveMode + { + return exception_if_exists(); + } +} diff --git a/src/core/etl/tests/Flow/ETL/Tests/Integration/Loader/File/FilesSink/Partitioned/IgnoreModeTest.php b/src/core/etl/tests/Flow/ETL/Tests/Integration/Loader/File/FilesSink/Partitioned/IgnoreModeTest.php new file mode 100644 index 0000000000..60a772eea7 --- /dev/null +++ b/src/core/etl/tests/Flow/ETL/Tests/Integration/Loader/File/FilesSink/Partitioned/IgnoreModeTest.php @@ -0,0 +1,93 @@ +cleanFiles(); + } + + public function test_open_stream_for_existing_partition_with_existing_file(): void + { + $this->setupFiles([ + __FUNCTION__ => [ + 'partition=value' => [ + 'file.txt' => 'file content', + ], + ], + ]); + $file = $this->getPath(__FUNCTION__ . '/file.txt'); + $files = $this->files($file); + + $files->writeTo([new Partition('partition', 'value')])->append('new content'); + $files->publish(); + + $files = iterator_to_array($this->fs()->list(path($file->parentDirectory()->path() . '/**/*.txt'))); + + static::assertCount(1, $files); + + static::assertStringStartsWith('file.txt', $files[0]->path->basename()); + static::assertSame('file content', file_get_contents($files[0]->path->path())); + } + + public function test_open_stream_for_existing_partition_without_existing_file(): void + { + $this->setupFiles([ + __FUNCTION__ => [ + 'partition=value' => [], + ], + ]); + $file = $this->getPath(__FUNCTION__ . '/file.txt'); + $files = $this->files($file); + + $files->writeTo([new Partition('partition', 'value')])->append('appended content'); + $files->publish(); + + $files = iterator_to_array($this->fs()->list(path($file->parentDirectory()->path() . '/**/*.txt'))); + + static::assertCount(1, $files); + + static::assertSame('file.txt', $files[0]->path->basename()); + static::assertSame('appended content', file_get_contents($files[0]->path->path())); + } + + public function test_open_stream_for_non_existing_partition(): void + { + $this->setupFiles([ + __FUNCTION__ => [], + ]); + $file = $this->getPath(__FUNCTION__ . '/file.txt'); + $files = $this->files($file); + + $files->writeTo([new Partition('partition', 'value')])->append('appended content'); + $files->publish(); + + $files = iterator_to_array($this->fs()->list(path($file->parentDirectory()->path() . '/partition=value/*'))); + + static::assertCount(1, $files); + + static::assertSame('file.txt', $files[0]->path->basename()); + static::assertSame('appended content', file_get_contents($files[0]->path->path())); + } + + protected function saveMode(): SaveMode + { + return ignore(); + } +} diff --git a/src/core/etl/tests/Flow/ETL/Tests/Integration/Loader/File/FilesSink/Partitioned/OverwriteModeTest.php b/src/core/etl/tests/Flow/ETL/Tests/Integration/Loader/File/FilesSink/Partitioned/OverwriteModeTest.php new file mode 100644 index 0000000000..ac58c721bb --- /dev/null +++ b/src/core/etl/tests/Flow/ETL/Tests/Integration/Loader/File/FilesSink/Partitioned/OverwriteModeTest.php @@ -0,0 +1,118 @@ +cleanFiles(); + } + + public function test_open_stream_for_existing_partition_with_existing_file(): void + { + $this->setupFiles([ + __FUNCTION__ => [ + 'partition=value' => [ + 'file.txt' => 'file content', + ], + ], + ]); + $file = $this->getPath(__FUNCTION__ . '/file.txt'); + $files = $this->files($file); + + $files->writeTo([new Partition('partition', 'value')])->append('new content'); + $files->publish(); + + $files = iterator_to_array($this->fs()->list(path($file->parentDirectory()->path() . '/**/*.txt'))); + + static::assertCount(1, $files); + + static::assertStringStartsWith('file.txt', $files[0]->path->basename()); + static::assertSame('new content', file_get_contents($files[0]->path->path())); + } + + public function test_open_stream_for_existing_partition_without_existing_file(): void + { + $this->setupFiles([ + __FUNCTION__ => [ + 'partition=value' => [], + ], + ]); + $file = $this->getPath(__FUNCTION__ . '/file.txt'); + $files = $this->files($file); + + $files->writeTo([new Partition('partition', 'value')])->append('new content'); + $files->publish(); + + $files = iterator_to_array($this->fs()->list(path($file->parentDirectory()->path() . '/**/*.txt'))); + + static::assertCount(1, $files); + + static::assertSame('file.txt', $files[0]->path->basename()); + static::assertSame('new content', file_get_contents($files[0]->path->path())); + } + + public function test_open_stream_for_non_existing_partition(): void + { + $this->setupFiles([ + __FUNCTION__ => [], + ]); + $file = $this->getPath(__FUNCTION__ . '/file.txt'); + $files = $this->files($file); + + $files->writeTo([new Partition('partition', 'value')])->append('new content'); + $files->publish(); + + $files = iterator_to_array($this->fs()->list(path($file->parentDirectory()->path() . '/partition=value/*'))); + + static::assertCount(1, $files); + + static::assertSame('file.txt', $files[0]->path->basename()); + static::assertSame('new content', file_get_contents($files[0]->path->path())); + } + + public function test_open_stream_for_non_existing_partition_with_custom_schema(): void + { + $this->setupFiles([__FUNCTION__ => []]); + + $fs = new FakeNativeLocalFilesystem(); + + $file = path( + $fs->mount()->protocol . '://' . $this->filesDirectory() . DIRECTORY_SEPARATOR . __FUNCTION__ . '/file.txt', + ); + + $files = new FilesSink($fs, $file, overwrite()); + + $files->writeTo([new Partition('partition', 'value')])->append('new content'); + $files->publish(); + + $files = iterator_to_array($this->fs()->list(path($file->parentDirectory()->path() . '/partition=value/*'))); + + static::assertCount(1, $files); + + static::assertSame('file.txt', $files[0]->path->basename()); + static::assertSame('new content', file_get_contents($files[0]->path->path())); + } + + protected function saveMode(): SaveMode + { + return overwrite(); + } +} diff --git a/src/core/etl/tests/Flow/ETL/Tests/Integration/Filesystem/FilesSink/Partitioned/OverwriteMultipleFilesTest.php b/src/core/etl/tests/Flow/ETL/Tests/Integration/Loader/File/FilesSink/Partitioned/OverwriteMultipleFilesTest.php similarity index 96% rename from src/core/etl/tests/Flow/ETL/Tests/Integration/Filesystem/FilesSink/Partitioned/OverwriteMultipleFilesTest.php rename to src/core/etl/tests/Flow/ETL/Tests/Integration/Loader/File/FilesSink/Partitioned/OverwriteMultipleFilesTest.php index 37e1c3792f..cc276e58f9 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Integration/Filesystem/FilesSink/Partitioned/OverwriteMultipleFilesTest.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Integration/Loader/File/FilesSink/Partitioned/OverwriteMultipleFilesTest.php @@ -2,10 +2,10 @@ declare(strict_types=1); -namespace Flow\ETL\Tests\Integration\Filesystem\FilesSink\Partitioned; +namespace Flow\ETL\Tests\Integration\Loader\File\FilesSink\Partitioned; use Flow\ETL\Filesystem\SaveMode; -use Flow\ETL\Tests\Integration\Filesystem\FilesSink\FilesSinkTestCase; +use Flow\ETL\Tests\Integration\Loader\File\FilesSink\FilesSinkTestCase; use Flow\Filesystem\Partition; use Override; diff --git a/src/core/etl/tests/Flow/ETL/Tests/Integration/Filesystem/FilesSink/Partitioned/PartitionPlaceholdersTest.php b/src/core/etl/tests/Flow/ETL/Tests/Integration/Loader/File/FilesSink/Partitioned/PartitionPlaceholdersTest.php similarity index 97% rename from src/core/etl/tests/Flow/ETL/Tests/Integration/Filesystem/FilesSink/Partitioned/PartitionPlaceholdersTest.php rename to src/core/etl/tests/Flow/ETL/Tests/Integration/Loader/File/FilesSink/Partitioned/PartitionPlaceholdersTest.php index 83aa851060..87d1cd1705 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Integration/Filesystem/FilesSink/Partitioned/PartitionPlaceholdersTest.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Integration/Loader/File/FilesSink/Partitioned/PartitionPlaceholdersTest.php @@ -2,12 +2,12 @@ declare(strict_types=1); -namespace Flow\ETL\Tests\Integration\Filesystem\FilesSink\Partitioned; +namespace Flow\ETL\Tests\Integration\Loader\File\FilesSink\Partitioned; use Flow\ETL\Exception\RuntimeException; -use Flow\ETL\Filesystem\FilesSink; use Flow\ETL\Filesystem\SaveMode; -use Flow\ETL\Tests\Integration\Filesystem\FilesSink\FilesSinkTestCase; +use Flow\ETL\Loader\File\FilesSink; +use Flow\ETL\Tests\Integration\Loader\File\FilesSink\FilesSinkTestCase; use Flow\Filesystem\FileListing; use Flow\Filesystem\Partition; use Flow\Filesystem\Path\Filter\OnlyFiles; diff --git a/src/core/etl/tests/Flow/ETL/Tests/Integration/Filesystem/FilesSink/tmp/.gitignore b/src/core/etl/tests/Flow/ETL/Tests/Integration/Loader/File/FilesSink/tmp/.gitignore similarity index 100% rename from src/core/etl/tests/Flow/ETL/Tests/Integration/Filesystem/FilesSink/tmp/.gitignore rename to src/core/etl/tests/Flow/ETL/Tests/Integration/Loader/File/FilesSink/tmp/.gitignore diff --git a/src/core/etl/tests/Flow/ETL/Tests/Integration/Pipeline/PipelineTest.php b/src/core/etl/tests/Flow/ETL/Tests/Integration/Pipeline/PipelineTest.php index a4620c18ae..4165e08787 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Integration/Pipeline/PipelineTest.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Integration/Pipeline/PipelineTest.php @@ -10,14 +10,13 @@ use Flow\ETL\Tests\Double\StaticDataFrameFactory; use Flow\ETL\Tests\FlowTestCase; +use function Flow\ETL\DSL\array_to_rows; use function Flow\ETL\DSL\df; use function Flow\ETL\DSL\from_array; use function Flow\ETL\DSL\from_data_frame; use function Flow\ETL\DSL\from_rows; use function Flow\ETL\DSL\int_schema; use function Flow\ETL\DSL\join_on; -use function Flow\ETL\DSL\row; -use function Flow\ETL\DSL\rows; use function Flow\ETL\DSL\schema; final class PipelineTest extends FlowTestCase @@ -47,8 +46,8 @@ public function test_a_cycle_closed_on_a_later_batch_is_refused_when_run(): void { $frame = df() ->read(from_rows( - rows(schema(int_schema('id')), row(['id' => 1])), - rows(schema(int_schema('id')), row(['id' => 2])), + array_to_rows([['id' => 1]], schema(int_schema('id'))), + array_to_rows([['id' => 2]], schema(int_schema('id'))), )) ->transform($transformer = new ReadsBackOnSecondBatch()); diff --git a/src/core/etl/tests/Flow/ETL/Tests/Integration/Sort/ExternalSort/ExternalSortLifecycleTest.php b/src/core/etl/tests/Flow/ETL/Tests/Integration/Sort/ExternalSort/ExternalSortLifecycleTest.php index b81a63e105..10d76afb30 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Integration/Sort/ExternalSort/ExternalSortLifecycleTest.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Integration/Sort/ExternalSort/ExternalSortLifecycleTest.php @@ -4,6 +4,7 @@ namespace Flow\ETL\Tests\Integration\Sort\ExternalSort; +use Flow\ETL\Dataset\Memory\Unit; use Flow\ETL\Exception\RuntimeException; use Flow\ETL\Tests\Double\ThrowWhenRowMatches; use Flow\ETL\Tests\FlowIntegrationTestCase; @@ -22,8 +23,9 @@ final class ExternalSortLifecycleTest extends FlowIntegrationTestCase public function test_a_clean_run_leaves_no_spilled_runs(): void { $output = []; - df(config_builder()->sort(external_sort()->runSize(1)->bucketsCount(2))->build()) + df(config_builder()->sort(external_sort()->memoryLimit(Unit::fromBytes(1))->bucketsCount(2))->build()) ->read(from_array($this->descendingIds())) + ->batchSize(1) ->sortBy([ref('id')]) ->write(to_array($output)) ->run(); @@ -37,8 +39,9 @@ public function test_upstream_failure_leaves_no_spilled_runs(): void $output = []; try { - df(config_builder()->sort(external_sort()->runSize(1)->bucketsCount(2))->build()) + df(config_builder()->sort(external_sort()->memoryLimit(Unit::fromBytes(1))->bucketsCount(2))->build()) ->read(from_array($this->descendingIds())) + ->batchSize(1) ->with(new ThrowWhenRowMatches('id', 5, new RuntimeException('upstream failed mid-bucketing'))) ->sortBy([ref('id')]) ->write(to_array($output)) diff --git a/src/core/etl/tests/Flow/ETL/Tests/Integration/Sort/ExternalSort/ExternalSortMergeTest.php b/src/core/etl/tests/Flow/ETL/Tests/Integration/Sort/ExternalSort/ExternalSortMergeTest.php index 2cc5a1422f..3027564133 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Integration/Sort/ExternalSort/ExternalSortMergeTest.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Integration/Sort/ExternalSort/ExternalSortMergeTest.php @@ -5,7 +5,9 @@ namespace Flow\ETL\Tests\Integration\Sort\ExternalSort; use Flow\ETL\Bucketing\Storage\MemoryBuckets; +use Flow\ETL\Dataset\Memory\Unit; use Flow\ETL\Tests\Double\RecordingBucketsStorage; +use Flow\ETL\Tests\Double\SpyBackend; use Flow\ETL\Tests\FlowIntegrationTestCase; use Flow\Filesystem\Path\Filter\KeepAll; @@ -28,18 +30,40 @@ public function test_external_sort_over_three_merge_generations_loses_no_rows(): { $input = []; - for ($id = 100; $id > 0; $id--) { + for ($id = 16; $id > 0; $id--) { $input[] = ['id' => $id]; } $output = []; - df(config_builder()->sort(external_sort()->runSize(1)->bucketsCount(2))->build()) + df(config_builder()->sort(external_sort()->memoryLimit(Unit::fromBytes(1))->bucketsCount(2))->build()) ->read(from_array($input)) + ->batchSize(2) ->sortBy([ref('id')]) ->write(to_array($output)) ->run(); - static::assertSame(range(1, 100), array_column($output, 'id')); + static::assertSame(range(1, 16), array_column($output, 'id')); + } + + public function test_spilled_runs_are_read_back_through_the_configured_backend(): void + { + $backend = new SpyBackend(); + $output = []; + + df( + config_builder() + ->backend($backend) + ->sort(external_sort()->memoryLimit(Unit::fromBytes(1))) + ->build(), + ) + ->read(from_array([['id' => 3], ['id' => 1], ['id' => 2]])) + ->batchSize(1) + ->sortBy(ref('id')) + ->write(to_array($output)) + ->run(); + + static::assertSame([1, 2, 3], array_column($output, 'id')); + static::assertGreaterThan(0, $backend->decodes()); } public function test_memory_storage_spills_nothing_to_disk(): void @@ -51,8 +75,13 @@ public function test_memory_storage_spills_nothing_to_disk(): void } $output = []; - df(config_builder()->sort(external_sort()->storage(new MemoryBuckets())->runSize(1)->bucketsCount(2))->build()) + df( + config_builder() + ->sort(external_sort()->storage(new MemoryBuckets())->memoryLimit(Unit::fromBytes(1))->bucketsCount(2)) + ->build(), + ) ->read(from_array($input)) + ->batchSize(1) ->sortBy([ref('id')]) ->write(to_array($output)) ->run(); @@ -80,10 +109,17 @@ public function test_merge_storage_receives_only_merged_runs(): void $output = []; df( config_builder() - ->sort(external_sort()->storage($spill)->mergeStorage($merge)->runSize(1)->bucketsCount(2)) + ->sort( + external_sort() + ->storage($spill) + ->mergeStorage($merge) + ->memoryLimit(Unit::fromBytes(1)) + ->bucketsCount(2), + ) ->build(), ) ->read(from_array($input)) + ->batchSize(1) ->sortBy([ref('id')]) ->write(to_array($output)) ->run(); diff --git a/src/core/etl/tests/Flow/ETL/Tests/Integration/Sort/ExternalSort/ExternalSortRunSizeTest.php b/src/core/etl/tests/Flow/ETL/Tests/Integration/Sort/ExternalSort/ExternalSortRunSizeTest.php new file mode 100644 index 0000000000..d7b61039cb --- /dev/null +++ b/src/core/etl/tests/Flow/ETL/Tests/Integration/Sort/ExternalSort/ExternalSortRunSizeTest.php @@ -0,0 +1,68 @@ +fs, path($this->cacheDir->path() . '/sort-runs'), new PhpBackend()), + ); + + // one ~1 MB row a batch against ~4.5 MB: a run holds a few batches - once spilled to disk, its memory is free + $input = (static function (): Generator { + for ($id = 40; $id > 0; $id--) { + yield array_to_rows( + [['id' => $id, 'payload' => str_repeat(chr(65 + ($id % 26)), 1_000_000)]], + schema(int_schema('id'), str_schema('payload')), + ); + } + })(); + + $sorted = iterator_to_array( + ExternalSortMother::with(refs('id'), $storage, Unit::fromBytes(memory_get_usage(false) + 4_500_000))->sort( + $input, + flow_context(config()), + ), + false, + ); + + $runs = array_values($storage->setRowCounts); + + static::assertSame(40, array_sum(array_map(static fn($batch): int => $batch->count(), $sorted))); + static::assertGreaterThanOrEqual(2, count($runs)); + + foreach (array_slice($runs, 0, -1) as $rows) { + static::assertGreaterThan(1, $rows, 'a run after a spill must hold more than one batch'); + } + } +} diff --git a/src/core/etl/tests/Flow/ETL/Tests/Mother/ColumnMother.php b/src/core/etl/tests/Flow/ETL/Tests/Mother/ColumnMother.php new file mode 100644 index 0000000000..2cd18d6541 --- /dev/null +++ b/src/core/etl/tests/Flow/ETL/Tests/Mother/ColumnMother.php @@ -0,0 +1,24 @@ + $definition + * @param list $values + */ + public static function of(Definition $definition, array $values): Column + { + $builder = (new PhpBackend())->builder($definition); + $builder->appendMany($values); + + return $builder->finish(); + } +} diff --git a/src/core/etl/tests/Flow/ETL/Tests/Mother/DateIntervalMother.php b/src/core/etl/tests/Flow/ETL/Tests/Mother/DateIntervalMother.php new file mode 100644 index 0000000000..e7e7997f2a --- /dev/null +++ b/src/core/etl/tests/Flow/ETL/Tests/Mother/DateIntervalMother.php @@ -0,0 +1,30 @@ +f = $fraction; + // @mago-ignore analysis:invalid-property-write + $interval->invert = $invert; + + return $interval; + } + + public static function seconds(int $seconds, float $fraction, int $invert): DateInterval + { + $interval = self::of('PT0S', $fraction, $invert); + // @mago-ignore analysis:invalid-property-write + $interval->s = $seconds; + + return $interval; + } +} diff --git a/src/core/etl/tests/Flow/ETL/Tests/Mother/ExternalSortMother.php b/src/core/etl/tests/Flow/ETL/Tests/Mother/ExternalSortMother.php new file mode 100644 index 0000000000..aed13be662 --- /dev/null +++ b/src/core/etl/tests/Flow/ETL/Tests/Mother/ExternalSortMother.php @@ -0,0 +1,67 @@ + $batchSize + */ + public static function inMemory( + References $refs, + BucketsStorage $storage = new MemoryBuckets(), + int $batchSize = 1000, + ): ExternalSort { + return self::with($refs, $storage, Unit::fromGb(1), batchSize: $batchSize); + } + + /** + * A one byte limit - every non-empty batch is spilled as its own sorted run. + * + * @param int<1, max> $mergeFanIn + * @param int<1, max> $batchSize + */ + public static function spilling( + References $refs, + BucketsStorage $storage = new MemoryBuckets(), + int $mergeFanIn = 10, + int $batchSize = 1000, + ): ExternalSort { + return self::with($refs, $storage, Unit::fromBytes(1), $mergeFanIn, $batchSize); + } + + /** + * @param int<1, max> $mergeFanIn + * @param int<1, max> $batchSize + */ + public static function with( + References $refs, + BucketsStorage $storage, + Unit $memoryLimit, + int $mergeFanIn = 10, + int $batchSize = 1000, + ?BucketsStorage $mergeStorage = null, + ): ExternalSort { + return new ExternalSort( + $refs, + new Buckets($storage), + new Buckets($mergeStorage ?? $storage), + new NativePHPRandomValueGenerator(), + $memoryLimit, + $mergeFanIn, + $batchSize, + ); + } +} diff --git a/src/core/etl/tests/Flow/ETL/Tests/Mother/GroupByAggregationProcessorMother.php b/src/core/etl/tests/Flow/ETL/Tests/Mother/GroupByAggregationProcessorMother.php new file mode 100644 index 0000000000..92298a5897 --- /dev/null +++ b/src/core/etl/tests/Flow/ETL/Tests/Mother/GroupByAggregationProcessorMother.php @@ -0,0 +1,54 @@ + $bucketsCount + * @param int<1, max> $batchSize + */ + public static function with( + GroupBy $groupBy, + BucketsStorage $storage, + Unit $memoryLimit, + int $bucketsCount = 4, + int $batchSize = 1000, + ): GroupByAggregationProcessor { + return new GroupByAggregationProcessor( + $groupBy, + new HashBucketing( + $groupBy->references(), + $bucketsCount, + new NativeHasher(), + new NativePHPRandomValueGenerator(), + 'group-by', + ), + new Buckets($storage), + $memoryLimit, + $batchSize, + ); + } +} diff --git a/src/core/etl/tests/Flow/ETL/Tests/Mother/HashJoinProcessorMother.php b/src/core/etl/tests/Flow/ETL/Tests/Mother/HashJoinProcessorMother.php index 8859c806c8..c3697f60ca 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Mother/HashJoinProcessorMother.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Mother/HashJoinProcessorMother.php @@ -7,6 +7,7 @@ use Flow\ETL\Bucketing\Buckets; use Flow\ETL\Bucketing\BucketsStorage; use Flow\ETL\Bucketing\Storage\MemoryBuckets; +use Flow\ETL\Dataset\Memory\Unit; use Flow\ETL\Executor; use Flow\ETL\Executor\PhysicalPlan; use Flow\ETL\Join\Expression; @@ -18,7 +19,7 @@ final class HashJoinProcessorMother { /** - * Non-resident in-memory storage - exercises the grace hash join path without disk I/O. + * Non-resident in-memory storage and a one-byte limit - exercises the grace hash join path without disk I/O. * * @param int<1, max> $bucketsCount * @param int<1, max> $batchSize @@ -38,6 +39,7 @@ public static function grace( $storage ?? new SpyBucketsStorage(new MemoryBuckets()), $bucketsCount, $batchSize, + Unit::fromBytes(1), ); } @@ -66,6 +68,7 @@ public static function with( BucketsStorage $storage, int $bucketsCount = 64, int $batchSize = 1000, + ?Unit $memoryLimit = null, ): HashJoinProcessor { return new HashJoinProcessor( $right, @@ -75,6 +78,7 @@ public static function with( new Buckets($storage), new Buckets($storage), new NativePHPRandomValueGenerator(), + $memoryLimit ?? Unit::fromGb(1), $bucketsCount, $batchSize, ); diff --git a/src/core/etl/tests/Flow/ETL/Tests/Mother/ListColumnsMother.php b/src/core/etl/tests/Flow/ETL/Tests/Mother/ListColumnsMother.php index 5ee05ab780..158beba2a9 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Mother/ListColumnsMother.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Mother/ListColumnsMother.php @@ -4,12 +4,12 @@ namespace Flow\ETL\Tests\Mother; -use Flow\ETL\Row; +use Flow\ETL\Rows; use Flow\ETL\Schema; +use function Flow\ETL\DSL\array_to_rows; use function Flow\ETL\DSL\int_schema; use function Flow\ETL\DSL\list_schema; -use function Flow\ETL\DSL\row; use function Flow\ETL\DSL\schema; use function Flow\ETL\DSL\str_schema; use function Flow\Types\DSL\type_boolean; @@ -33,16 +33,16 @@ public static function schema(): Schema /** * @param array $override */ - public static function row(array $override = []): Row + public static function rows(array $override = []): Rows { - return row([ + return array_to_rows([[ 'id' => 'a', 'tags' => ['x', 'y'], 'nums' => [1, 2, 3], 'lists' => [['q', 'r'], ['s']], 'flags' => [true, false], ...$override, - ]); + ]], self::schema()->makeNullable()); } public static function tagsSchema(): Schema diff --git a/src/core/etl/tests/Flow/ETL/Tests/Mother/PartitionedSourceMother.php b/src/core/etl/tests/Flow/ETL/Tests/Mother/PartitionedSourceMother.php index 0c9a9470be..a6d27a9281 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Mother/PartitionedSourceMother.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Mother/PartitionedSourceMother.php @@ -8,9 +8,8 @@ use Flow\ETL\Tests\Double\RecordingFileExtractor; use Flow\Filesystem\FileStatus; +use function Flow\ETL\DSL\array_to_rows; use function Flow\ETL\DSL\int_schema; -use function Flow\ETL\DSL\row; -use function Flow\ETL\DSL\rows; use function Flow\ETL\DSL\schema; use function Flow\ETL\DSL\str_schema; use function Flow\Filesystem\DSL\path; @@ -30,10 +29,9 @@ public static function yearMonth(): RecordingFileExtractor { return (new RecordingFileExtractor( schema(int_schema('year'), str_schema('month'), str_schema('value')), - rows( + array_to_rows( + [['year' => 2023, 'month' => '07', 'value' => 'a'], ['year' => 2024, 'month' => '08', 'value' => 'b']], schema(int_schema('year'), str_schema('month'), str_schema('value')), - row(['year' => 2023, 'month' => '07', 'value' => 'a']), - row(['year' => 2024, 'month' => '08', 'value' => 'b']), ), ))->withPartitionSchema(self::partitions()); } diff --git a/src/core/etl/tests/Flow/ETL/Tests/Mother/RepartitionProcessorMother.php b/src/core/etl/tests/Flow/ETL/Tests/Mother/RepartitionProcessorMother.php new file mode 100644 index 0000000000..7215e5f352 --- /dev/null +++ b/src/core/etl/tests/Flow/ETL/Tests/Mother/RepartitionProcessorMother.php @@ -0,0 +1,49 @@ + $bucketsCount + */ + public static function with( + References $by, + BucketsStorage $storage, + Unit $memoryLimit, + int $bucketsCount = 4, + ): RepartitionProcessor { + return new RepartitionProcessor( + $by, + new HashBucketing( + $by->all(), + $bucketsCount, + new NativeHasher(), + new NativePHPRandomValueGenerator(), + 'repartition', + ), + new Buckets($storage), + $memoryLimit, + ); + } +} diff --git a/src/core/etl/tests/Flow/ETL/Tests/Mother/RowsMother.php b/src/core/etl/tests/Flow/ETL/Tests/Mother/RowsMother.php index f3fa16b5cb..4b69b1be0f 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Mother/RowsMother.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Mother/RowsMother.php @@ -6,11 +6,15 @@ use Flow\ETL\Rows; +use function Flow\ETL\DSL\array_to_rows; use function Flow\ETL\DSL\int_schema; -use function Flow\ETL\DSL\row; -use function Flow\ETL\DSL\rows; use function Flow\ETL\DSL\schema; use function Flow\ETL\DSL\str_schema; +use function Flow\ETL\DSL\structure_schema; +use function Flow\Types\DSL\type_boolean; +use function Flow\Types\DSL\type_integer; +use function Flow\Types\DSL\type_string; +use function Flow\Types\DSL\type_structure; final class RowsMother { @@ -22,9 +26,9 @@ final class RowsMother public static function descendingIdBatches(): array { return [ - rows(schema(int_schema('id')), row(['id' => 5]), row(['id' => 4])), - rows(schema(int_schema('id')), row(['id' => 3]), row(['id' => 2])), - rows(schema(int_schema('id')), row(['id' => 1]), row(['id' => 0])), + array_to_rows([['id' => 5], ['id' => 4]], schema(int_schema('id'))), + array_to_rows([['id' => 3], ['id' => 2]], schema(int_schema('id'))), + array_to_rows([['id' => 1], ['id' => 0]], schema(int_schema('id'))), ]; } @@ -36,9 +40,9 @@ public static function descendingIdBatches(): array public static function interleavedGroupBatches(): array { return [ - rows(schema(str_schema('g'), int_schema('v')), row(['g' => 'a', 'v' => 1]), row(['g' => 'b', 'v' => 10])), - rows(schema(str_schema('g'), int_schema('v')), row(['g' => 'a', 'v' => 2]), row(['g' => 'b', 'v' => 20])), - rows(schema(str_schema('g'), int_schema('v')), row(['g' => 'a', 'v' => 3]), row(['g' => 'b', 'v' => 30])), + array_to_rows([['g' => 'a', 'v' => 1], ['g' => 'b', 'v' => 10]], schema(str_schema('g'), int_schema('v'))), + array_to_rows([['g' => 'a', 'v' => 2], ['g' => 'b', 'v' => 20]], schema(str_schema('g'), int_schema('v'))), + array_to_rows([['g' => 'a', 'v' => 3], ['g' => 'b', 'v' => 30]], schema(str_schema('g'), int_schema('v'))), ]; } @@ -51,9 +55,9 @@ public static function interleavedGroupBatches(): array public static function sortedGroupBatches(): array { return [ - rows(schema(str_schema('g'), int_schema('v')), row(['g' => 'a', 'v' => 1]), row(['g' => 'a', 'v' => 2])), - rows(schema(str_schema('g'), int_schema('v')), row(['g' => 'a', 'v' => 3]), row(['g' => 'b', 'v' => 10])), - rows(schema(str_schema('g'), int_schema('v')), row(['g' => 'b', 'v' => 20]), row(['g' => 'b', 'v' => 30])), + array_to_rows([['g' => 'a', 'v' => 1], ['g' => 'a', 'v' => 2]], schema(str_schema('g'), int_schema('v'))), + array_to_rows([['g' => 'a', 'v' => 3], ['g' => 'b', 'v' => 10]], schema(str_schema('g'), int_schema('v'))), + array_to_rows([['g' => 'b', 'v' => 20], ['g' => 'b', 'v' => 30]], schema(str_schema('g'), int_schema('v'))), ]; } @@ -62,9 +66,24 @@ public static function sequentialIds(int $count): Rows $sequence = []; for ($id = 1; $id <= $count; $id++) { - $sequence[] = row(['id' => $id]); + $sequence[] = ['id' => $id]; } - return rows(schema(int_schema('id')), ...$sequence); + return array_to_rows($sequence, schema(int_schema('id'))); + } + + /** + * One row whose "array_entry" structure nests a structure: {id, status, enabled, array: {foo}}. + */ + public static function arrayEntry(): Rows + { + return array_to_rows([[ + 'array_entry' => ['id' => 1, 'status' => 'PENDING', 'enabled' => true, 'array' => ['foo' => 'bar']], + ]], schema(structure_schema('array_entry', type_structure([ + 'id' => type_integer(), + 'status' => type_string(), + 'enabled' => type_boolean(), + 'array' => type_structure(['foo' => type_string()]), + ])))); } } diff --git a/src/core/etl/tests/Flow/ETL/Tests/Mother/SinkFeedMother.php b/src/core/etl/tests/Flow/ETL/Tests/Mother/SinkFeedMother.php index 5ce17b6999..6989c9a899 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Mother/SinkFeedMother.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Mother/SinkFeedMother.php @@ -17,16 +17,15 @@ use Flow\ETL\Rows; use Flow\ETL\Transformer; +use function Flow\ETL\DSL\array_to_rows; use function Flow\ETL\DSL\int_schema; -use function Flow\ETL\DSL\row; -use function Flow\ETL\DSL\rows; use function Flow\ETL\DSL\schema; final class SinkFeedMother { public static function batch(int $id = 1): Rows { - return rows(schema(int_schema('id')), row(['id' => $id])); + return array_to_rows([['id' => $id]], schema(int_schema('id'))); } public static function feed(): FeedExtractor diff --git a/src/core/etl/tests/Flow/ETL/Tests/Mother/SortDatasetMother.php b/src/core/etl/tests/Flow/ETL/Tests/Mother/SortDatasetMother.php new file mode 100644 index 0000000000..57253a9d95 --- /dev/null +++ b/src/core/etl/tests/Flow/ETL/Tests/Mother/SortDatasetMother.php @@ -0,0 +1,107 @@ +, + * runs: list>>, + * storedBatchSize: int<1, max>, + * mergeBatchSize: int<1, max> + * } + */ +final class SortDatasetMother +{ + /** + * Rows over small domains (many ties) with nulls, NaN, -0.0, numeric-looking strings and datetimes, split into + * 2-8 consecutive runs, sorted by 1-3 distinct random columns in random directions and null placements. "id" is unique and never a key. + * + * @return SortDatasetShape + */ + public static function random(int $seed): array + { + mt_srand($seed); + + $columns = ['i', 'f', 's', 'at', 'b']; + $floats = [null, NAN, -0.0, 0.0, 1.5, -2.25, 10.0]; + $strings = [null, '10', '9', '100', 'a', 'B', '', '09']; + $instants = [ + null, + '2026-01-01 00:00:00', + '2026-01-01 00:00:00.5', + '1969-12-31 23:59:59', + '2030-06-15 12:00:00', + ]; + $utc = new DateTimeZone('UTC'); + $rows = []; + + for ($id = 0, $count = mt_rand(0, 60); $id < $count; $id++) { + $instant = $instants[mt_rand(0, 4)]; + $rows[] = [ + 'id' => $id, + 'i' => mt_rand(0, 5) === 0 ? null : mt_rand(-3, 3), + 'f' => $floats[mt_rand(0, 6)], + 's' => $strings[mt_rand(0, 7)], + 'at' => $instant === null ? null : new DateTimeImmutable($instant, $utc), + 'b' => mt_rand(0, 3) === 0 ? null : mt_rand(0, 1) === 1, + ]; + } + + $references = []; + + for ($k = 0, $keyCount = mt_rand(1, 3); $k < $keyCount; $k++) { + $reference = ref(array_splice($columns, mt_rand(0, count($columns) - 1), 1)[0]); + $nulls = mt_rand(0, 1) === 1 ? NullsOrder::FIRST : NullsOrder::LAST; + $references[] = mt_rand(0, 1) === 1 ? $reference->asc($nulls) : $reference->desc($nulls); + } + + $runs = []; + $runCount = mt_rand(2, 8); + $offset = 0; + + for ($r = 0; $r < $runCount; $r++) { + $size = $r === ($runCount - 1) ? $count - $offset : mt_rand(0, $count - $offset); + $runs[] = array_slice($rows, $offset, $size); + $offset += $size; + } + + return [ + 'schema' => schema( + int_schema('id'), + int_schema('i', nullable: true), + float_schema('f', nullable: true), + str_schema('s', nullable: true), + datetime_schema('at', nullable: true), + bool_schema('b', nullable: true), + ), + 'references' => $references, + 'runs' => $runs, + 'storedBatchSize' => mt_rand(1, 7), + 'mergeBatchSize' => mt_rand(1, 10), + ]; + } +} diff --git a/src/core/etl/tests/Flow/ETL/Tests/Mother/WindowContextMother.php b/src/core/etl/tests/Flow/ETL/Tests/Mother/WindowContextMother.php index d4b4c87b34..76ee36a471 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Mother/WindowContextMother.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Mother/WindowContextMother.php @@ -5,7 +5,6 @@ namespace Flow\ETL\Tests\Mother; use Flow\ETL\FlowContext; -use Flow\ETL\Row; use Flow\ETL\Rows; use Flow\ETL\Window\WholePartitionFrame; use Flow\ETL\Window\WindowContext; @@ -21,22 +20,6 @@ public static function atIndex( ?WindowFrame $frame = null, ?FlowContext $context = null, ): WindowContext { - return self::forRow($partition[$index], $partition, $index, $frame, $context); - } - - public static function forRow( - Row $row, - Rows $partition, - int $index = 0, - ?WindowFrame $frame = null, - ?FlowContext $context = null, - ): WindowContext { - return new WindowContext( - $row, - $index, - $partition, - $frame ?? new WholePartitionFrame(), - $context ?? flow_context(), - ); + return new WindowContext($index, $partition, $frame ?? new WholePartitionFrame(), $context ?? flow_context()); } } diff --git a/src/core/etl/tests/Flow/ETL/Tests/Mother/XmlMother.php b/src/core/etl/tests/Flow/ETL/Tests/Mother/XmlMother.php new file mode 100644 index 0000000000..b29f09fd53 --- /dev/null +++ b/src/core/etl/tests/Flow/ETL/Tests/Mother/XmlMother.php @@ -0,0 +1,18 @@ +loadXML($xml); + + return $document; + } +} diff --git a/src/core/etl/tests/Flow/ETL/Tests/Mother/XmlNodeMother.php b/src/core/etl/tests/Flow/ETL/Tests/Mother/XmlNodeMother.php new file mode 100644 index 0000000000..1937e4528d --- /dev/null +++ b/src/core/etl/tests/Flow/ETL/Tests/Mother/XmlNodeMother.php @@ -0,0 +1,50 @@ + + */ + public static function edges(): array + { + return [ + 'ascii' => ['document' => 'x', 'node' => 'x'], + 'non-ascii text' => [ + 'document' => '
zażółć gęślą jaźń ☃ 😀
', + 'node' => 'zażółć gęślą jaźń ☃ 😀', + ], + 'non-ascii attribute' => ['document' => '', 'node' => ''], + 'non-ascii name' => [ + 'document' => '<żółć ść="1"/>', + 'node' => '<żółć ść="1"/>', + ], + 'non-ascii comment, pi, cdata' => [ + 'document' => '', + 'node' => '', + ], + 'entities' => [ + 'document' => '<p> & © 😀 ', + 'node' => '<p> & © 😀 ', + ], + 'inherited namespace' => [ + 'document' => 'ż', + 'node' => 'ż', + ], + 'latin1 source' => [ + 'document' => "\xE9", + 'node' => 'é', + ], + 'whitespace and empty' => [ + 'document' => "\n \t ", + 'node' => "\n \t ", + ], + ]; + } +} diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Bucketing/BucketRunTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Bucketing/BucketRunTest.php index 46493fe4c6..c13663bbf5 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Unit/Bucketing/BucketRunTest.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Bucketing/BucketRunTest.php @@ -11,10 +11,8 @@ use Flow\ETL\Tests\FlowTestCase; use Flow\ETL\Tests\Mother\BucketMother; -use function array_map; +use function Flow\ETL\DSL\array_to_rows; use function Flow\ETL\DSL\int_schema; -use function Flow\ETL\DSL\row; -use function Flow\ETL\DSL\rows; use function Flow\ETL\DSL\schema; final class BucketRunTest extends FlowTestCase @@ -24,9 +22,9 @@ public function test_remove_removes_through_its_own_buckets(): void $a = new Buckets($storageA = new MemoryBuckets()); $b = new Buckets($storageB = new MemoryBuckets()); - $storageA->append('shared-id', rows(schema(int_schema('id')), row(['id' => 1]))); + $storageA->append('shared-id', array_to_rows([['id' => 1]], schema(int_schema('id')))); $a->add(BucketMother::withTotalRows('shared-id', 1)); - $storageB->append('shared-id', rows(schema(int_schema('id')), row(['id' => 2]))); + $storageB->append('shared-id', array_to_rows([['id' => 2]], schema(int_schema('id')))); $b->add(BucketMother::withTotalRows('shared-id', 1)); (new BucketRun('shared-id', $a))->remove(); @@ -40,20 +38,16 @@ public function test_rows_are_read_through_the_buckets_that_wrote_them(): void $a = new Buckets($storageA = new MemoryBuckets()); $b = new Buckets($storageB = new MemoryBuckets()); - $storageA->append('shared-id', rows(schema(int_schema('id')), row(['id' => 1]))); - $storageB->append('shared-id', rows(schema(int_schema('id')), row(['id' => 2]))); + $storageA->append('shared-id', array_to_rows([['id' => 1]], schema(int_schema('id')))); + $storageB->append('shared-id', array_to_rows([['id' => 2]], schema(int_schema('id')))); static::assertSame( [1], - array_map(static fn($r): mixed => $r->get( - 'id', - ), BucketsStorageContext::rows((new BucketRun('shared-id', $a))->rows())), + array_column(BucketsStorageContext::rows((new BucketRun('shared-id', $a))->rows()), 'id'), ); static::assertSame( [2], - array_map(static fn($r): mixed => $r->get( - 'id', - ), BucketsStorageContext::rows((new BucketRun('shared-id', $b))->rows())), + array_column(BucketsStorageContext::rows((new BucketRun('shared-id', $b))->rows()), 'id'), ); } } diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Bucketing/BucketShapeTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Bucketing/BucketShapeTest.php deleted file mode 100644 index 478d51d9bf..0000000000 --- a/src/core/etl/tests/Flow/ETL/Tests/Unit/Bucketing/BucketShapeTest.php +++ /dev/null @@ -1,17 +0,0 @@ -value); - static::assertSame('_bucket_total_rows', BucketShape::totalRows->value); - } -} diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Bucketing/BucketTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Bucketing/BucketTest.php index 40ee74dd20..70d30eb219 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Unit/Bucketing/BucketTest.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Bucketing/BucketTest.php @@ -5,7 +5,6 @@ namespace Flow\ETL\Tests\Unit\Bucketing; use Flow\ETL\Bucketing\Bucket; -use Flow\ETL\Bucketing\BucketShape; use Flow\ETL\Tests\FlowTestCase; final class BucketTest extends FlowTestCase @@ -18,13 +17,4 @@ public function test_exposes_identity_total_rows_and_index(): void static::assertSame(7, $bucket->totalRows); static::assertSame(3, $bucket->index); } - - public function test_to_row_carries_id_and_total_rows(): void - { - $row = (new Bucket('bucket-1', 4, 0))->toRow(); - - static::assertSame('bucket-1', $row->get(BucketShape::id->value)); - static::assertSame(4, $row->get(BucketShape::totalRows->value)); - static::assertCount(2, $row->values()); - } } diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Bucketing/BucketsTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Bucketing/BucketsTest.php index af27129993..bf0f792a8d 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Unit/Bucketing/BucketsTest.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Bucketing/BucketsTest.php @@ -9,9 +9,8 @@ use Flow\ETL\Tests\FlowTestCase; use Flow\ETL\Tests\Mother\BucketMother; +use function Flow\ETL\DSL\array_to_rows; use function Flow\ETL\DSL\int_schema; -use function Flow\ETL\DSL\row; -use function Flow\ETL\DSL\rows; use function Flow\ETL\DSL\schema; use function iterator_to_array; @@ -29,8 +28,8 @@ public function test_all_returns_every_registered_bucket(): void public function test_clear_empties_manifest_and_storage(): void { $storage = new MemoryBuckets(); - $storage->append('a', rows(schema(int_schema('id')), row(['id' => 1]))); - $storage->append('b', rows(schema(int_schema('id')), row(['id' => 2]))); + $storage->append('a', array_to_rows([['id' => 1]], schema(int_schema('id')))); + $storage->append('b', array_to_rows([['id' => 2]], schema(int_schema('id')))); $buckets = new Buckets($storage); $buckets->add(BucketMother::withTotalRows('a', 1)); @@ -46,7 +45,7 @@ public function test_clear_empties_manifest_and_storage(): void public function test_remove_drops_manifest_entry_and_storage(): void { $storage = new MemoryBuckets(); - $storage->append('a', rows(schema(int_schema('id')), row(['id' => 1]))); + $storage->append('a', array_to_rows([['id' => 1]], schema(int_schema('id')))); $buckets = new Buckets($storage); $buckets->add(BucketMother::withTotalRows('a', 1)); @@ -60,8 +59,8 @@ public function test_remove_drops_manifest_entry_and_storage(): void public function test_rows_yields_storage_batches(): void { $storage = new MemoryBuckets(); - $storage->append('a', rows(schema(int_schema('id')), row(['id' => 1]), row(['id' => 2]))); - $storage->append('a', rows(schema(int_schema('id')), row(['id' => 3]))); + $storage->append('a', array_to_rows([['id' => 1], ['id' => 2]], schema(int_schema('id')))); + $storage->append('a', array_to_rows([['id' => 3]], schema(int_schema('id')))); $batches = iterator_to_array((new Buckets($storage))->rows('a'), false); diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Bucketing/HashBucketingTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Bucketing/HashBucketingTest.php index 70b133bc89..187311fd59 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Unit/Bucketing/HashBucketingTest.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Bucketing/HashBucketingTest.php @@ -14,10 +14,9 @@ use Flow\ETL\Tests\Double\CountingHasher; use Flow\ETL\Tests\FlowTestCase; +use function Flow\ETL\DSL\array_to_rows; use function Flow\ETL\DSL\int_schema; use function Flow\ETL\DSL\ref; -use function Flow\ETL\DSL\row; -use function Flow\ETL\DSL\rows; use function Flow\ETL\DSL\schema; use function iterator_to_array; use function sort; @@ -35,7 +34,7 @@ public function test_bucket_ids_carry_namespace_and_run_id(): void ); $generator = (static function () { - yield rows(schema(int_schema('id')), row(['id' => 1])); + yield array_to_rows([['id' => 1]], schema(int_schema('id'))); })(); foreach ($strategy->bucketize($generator, new MemoryBuckets()) as $bucket) { @@ -49,8 +48,8 @@ public function test_buckets_are_yielded_after_the_whole_input_is_spilled(): voi $storage = new MemoryBuckets(); $generator = (static function () { - yield rows(schema(int_schema('id')), row(['id' => 1]), row(['id' => 2])); - yield rows(schema(int_schema('id')), row(['id' => 3]), row(['id' => 4])); + yield array_to_rows([['id' => 1], ['id' => 2]], schema(int_schema('id'))); + yield array_to_rows([['id' => 3], ['id' => 4]], schema(int_schema('id'))); })(); $totalRows = 0; @@ -73,7 +72,7 @@ public function test_custom_hasher_overrides_assignment(): void ); $generator = (static function () { - yield rows(schema(int_schema('id')), row(['id' => 1]), row(['id' => 2]), row(['id' => 3])); + yield array_to_rows([['id' => 1], ['id' => 2], ['id' => 3]], schema(int_schema('id'))); })(); $buckets = iterator_to_array($strategy->bucketize($generator, new MemoryBuckets())); @@ -89,11 +88,11 @@ public function test_distributes_rows_within_buckets_count(): void $rows = []; for ($i = 0; $i < 100; $i++) { - $rows[] = row(['id' => $i]); + $rows[] = ['id' => $i]; } $generator = (static function () use ($rows) { - yield rows(schema(int_schema('id')), ...$rows); + yield array_to_rows($rows, schema(int_schema('id'))); })(); $total = 0; @@ -113,11 +112,11 @@ public function test_index_is_the_partition_number(): void $rows = []; for ($i = 0; $i < 100; $i++) { - $rows[] = row(['id' => $i]); + $rows[] = ['id' => $i]; } $generator = (static function () use ($rows) { - yield rows(schema(int_schema('id')), ...$rows); + yield array_to_rows($rows, schema(int_schema('id'))); })(); $indexes = []; @@ -137,8 +136,8 @@ public function test_hashes_each_row_exactly_once(): void $strategy = new HashBucketing([ref('id')], 4, $spy, new NativePHPRandomValueGenerator()); $generator = (static function () { - yield rows(schema(int_schema('id')), row(['id' => 1]), row(['id' => 2])); - yield rows(schema(int_schema('id')), row(['id' => 3])); + yield array_to_rows([['id' => 1], ['id' => 2]], schema(int_schema('id'))); + yield array_to_rows([['id' => 3]], schema(int_schema('id'))); })(); iterator_to_array($strategy->bucketize($generator, new MemoryBuckets())); @@ -152,8 +151,8 @@ public function test_same_key_lands_in_the_same_bucket(): void $storage = new MemoryBuckets(); $generator = (static function () { - yield rows(schema(int_schema('id')), row(['id' => 1]), row(['id' => 1])); - yield rows(schema(int_schema('id')), row(['id' => 1])); + yield array_to_rows([['id' => 1], ['id' => 1]], schema(int_schema('id'))); + yield array_to_rows([['id' => 1]], schema(int_schema('id'))); })(); $buckets = iterator_to_array($strategy->bucketize($generator, $storage)); @@ -169,10 +168,9 @@ public function test_missing_key_column_is_bucketed_as_null_when_the_schema_allo $storage = new MemoryBuckets(); $generator = (static function () { - yield rows( + yield array_to_rows( + [['id' => null], ['other' => 1]], schema(int_schema('id', nullable: true), int_schema('other', nullable: true)), - row(['id' => null]), - row(['other' => 1]), ); })(); @@ -187,7 +185,7 @@ public function test_rows_missing_the_key_column_throw_by_default(): void $strategy = new HashBucketing([ref('id')], 4, new NativeHasher(), new NativePHPRandomValueGenerator()); $generator = (static function () { - yield rows(schema(int_schema('other')), row(['other' => 1])); + yield array_to_rows([['other' => 1]], schema(int_schema('other'))); })(); $this->expectException(InvalidArgumentException::class); @@ -203,4 +201,27 @@ public function test_throws_when_buckets_count_below_one(): void // @mago-ignore analysis:invalid-argument new HashBucketing([ref('id')], 0, new NativeHasher(), new NativePHPRandomValueGenerator()); } + + public function test_rows_of_one_key_across_batches_stay_in_batch_order(): void + { + $strategy = new HashBucketing([ref('k')], 1, new NativeHasher(), new NativePHPRandomValueGenerator()); + $storage = new MemoryBuckets(); + + $generator = (static function () { + yield array_to_rows( + [['k' => 1, 'v' => 1], ['k' => 2, 'v' => 2], ['k' => 1, 'v' => 3]], + schema(int_schema('k'), int_schema('v')), + ); + yield array_to_rows([['k' => 1, 'v' => 4]], schema(int_schema('k'), int_schema('v'))); + })(); + + $buckets = iterator_to_array($strategy->bucketize($generator, $storage)); + $values = []; + + foreach ($storage->get($buckets[0]->id) as $batch) { + $values = [...$values, ...$batch->column('v')->values()]; + } + + static::assertSame([1, 2, 3, 4], $values); + } } diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Bucketing/KeyGroupingTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Bucketing/KeyGroupingTest.php index 20c7223554..7c085f3c80 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Unit/Bucketing/KeyGroupingTest.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Bucketing/KeyGroupingTest.php @@ -9,12 +9,12 @@ use Flow\ETL\Tests\FlowTestCase; use Generator; +use function Flow\ETL\DSL\array_to_rows; use function Flow\ETL\DSL\datetime_schema; use function Flow\ETL\DSL\float_schema; use function Flow\ETL\DSL\int_schema; use function Flow\ETL\DSL\list_schema; use function Flow\ETL\DSL\ref; -use function Flow\ETL\DSL\row; use function Flow\ETL\DSL\rows; use function Flow\ETL\DSL\schema; use function Flow\ETL\DSL\str_schema; @@ -32,8 +32,8 @@ public function test_a_key_spanning_several_input_batches_lands_in_one_group(): KeyGroupingContext::groupedValuesOf( [ref('k')], (static function () use ($schema): Generator { - yield rows($schema, row(['k' => 'a', 'v' => 1]), row(['k' => 'b', 'v' => 2])); - yield rows($schema, row(['k' => 'a', 'v' => 3])); + yield array_to_rows([['k' => 'a', 'v' => 1], ['k' => 'b', 'v' => 2]], $schema); + yield array_to_rows([['k' => 'a', 'v' => 3]], $schema); })(), 'v', ), @@ -50,12 +50,11 @@ public function test_a_list_column_is_a_usable_key(): void KeyGroupingContext::groupedValuesOf( [ref('tags')], (static function () use ($schema): Generator { - yield rows( - $schema, - row(['tags' => [1, 2], 'v' => 1]), - row(['tags' => [3], 'v' => 2]), - row(['tags' => [1, 2], 'v' => 3]), - ); + yield array_to_rows([ + ['tags' => [1, 2], 'v' => 1], + ['tags' => [3], 'v' => 2], + ['tags' => [1, 2], 'v' => 3], + ], $schema); })(), 'v', ), @@ -73,13 +72,12 @@ public function test_datetime_keys_keep_full_precision(): void KeyGroupingContext::groupedValuesOf( [ref('at')], (static function () use ($schema): Generator { - yield rows( - $schema, - row(['at' => new DateTimeImmutable('2024-01-01 09:00:00'), 'v' => 1]), - row(['at' => new DateTimeImmutable('2024-01-01 21:00:00'), 'v' => 2]), - row(['at' => new DateTimeImmutable('2024-01-01 09:00:00'), 'v' => 3]), - row(['at' => new DateTimeImmutable('2024-01-01 21:00:00'), 'v' => 4]), - ); + yield array_to_rows([ + ['at' => new DateTimeImmutable('2024-01-01 09:00:00'), 'v' => 1], + ['at' => new DateTimeImmutable('2024-01-01 21:00:00'), 'v' => 2], + ['at' => new DateTimeImmutable('2024-01-01 09:00:00'), 'v' => 3], + ['at' => new DateTimeImmutable('2024-01-01 21:00:00'), 'v' => 4], + ], $schema); })(), 'v', ), @@ -96,7 +94,7 @@ public function test_empty_batches_are_skipped(): void [ref('k')], (static function () use ($schema): Generator { yield rows($schema); - yield rows($schema, row(['k' => 'a', 'v' => 1])); + yield array_to_rows([['k' => 'a', 'v' => 1]], $schema); })(), 'v', ), @@ -127,7 +125,7 @@ public function test_numerically_equal_keys_of_different_types_collapse(): void KeyGroupingContext::groupedValuesOf( [ref('k')], (static function () use ($schema): Generator { - yield rows($schema, row(['k' => 1.0, 'v' => 1]), row(['k' => 1.0, 'v' => 2])); + yield array_to_rows([['k' => 1.0, 'v' => 1], ['k' => 1.0, 'v' => 2]], $schema); })(), 'v', ), @@ -143,12 +141,11 @@ public function test_null_key_values_group_together(): void KeyGroupingContext::groupedValuesOf( [ref('k')], (static function () use ($schema): Generator { - yield rows( - $schema, - row(['k' => null, 'v' => 1]), - row(['k' => 'a', 'v' => 2]), - row(['k' => null, 'v' => 3]), - ); + yield array_to_rows([ + ['k' => null, 'v' => 1], + ['k' => 'a', 'v' => 2], + ['k' => null, 'v' => 3], + ], $schema); })(), 'v', ), @@ -164,7 +161,7 @@ public function test_one_key_yields_one_group(): void KeyGroupingContext::groupedValuesOf( [ref('k')], (static function () use ($schema): Generator { - yield rows($schema, row(['k' => 'a', 'v' => 1]), row(['k' => 'a', 'v' => 2])); + yield array_to_rows([['k' => 'a', 'v' => 1], ['k' => 'a', 'v' => 2]], $schema); })(), 'v', ), @@ -180,7 +177,7 @@ public function test_the_group_keeps_the_schema_of_the_first_non_empty_batch(): KeyGroupingContext::groups( [ref('k')], (static function () use ($schema): Generator { - yield rows($schema, row(['k' => 'a', 'v' => 1])); + yield array_to_rows([['k' => 'a', 'v' => 1]], $schema); })(), )[0]->schema(), ); diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Bucketing/KeyValuesTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Bucketing/KeyValuesTest.php index ac7da009f4..39dbca4b29 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Unit/Bucketing/KeyValuesTest.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Bucketing/KeyValuesTest.php @@ -5,69 +5,74 @@ namespace Flow\ETL\Tests\Unit\Bucketing; use Flow\ETL\Bucketing\KeyValues; -use Flow\ETL\Exception\InvalidArgumentException; +use Flow\ETL\Exception\SchemaDefinitionNotFoundException; use Flow\ETL\Tests\FlowTestCase; +use function Flow\ETL\DSL\array_to_rows; use function Flow\ETL\DSL\int_schema; use function Flow\ETL\DSL\ref; -use function Flow\ETL\DSL\row; use function Flow\ETL\DSL\rows; use function Flow\ETL\DSL\schema; use function Flow\ETL\DSL\str_schema; final class KeyValuesTest extends FlowTestCase { - public function test_of_projects_each_row(): void + public function test_a_missing_ref_throws(): void { - static::assertSame( - [[1], [2]], - (new KeyValues([ref('id')]))->of(rows(schema(int_schema('id')), row(['id' => 1]), row(['id' => 2]))), - ); + $this->expectException(SchemaDefinitionNotFoundException::class); + $this->expectExceptionMessage('Schema definition for entry "id" not found'); + + (new KeyValues([ref('id')]))->of(array_to_rows([['name' => 'flow']], schema(str_schema('name')))); } - public function test_of_row_preserves_null(): void + public function test_duplicated_refs_are_extracted_per_position(): void { static::assertSame( - [null], - (new KeyValues([ref('id')]))->ofRow(row(['id' => null]), schema(int_schema('id', nullable: true))), + [[1, 1]], + (new KeyValues([ref('id'), ref('id')]))->of(array_to_rows([['id' => 1]], schema(int_schema('id')))), ); } - public function test_of_row_projects_refs_in_order(): void + public function test_no_refs_is_an_empty_key_per_row(): void { static::assertSame( - [1, 'PL'], - (new KeyValues([ref('id'), ref('country')]))->ofRow( - row(['id' => 1, 'country' => 'PL']), - schema(int_schema('id'), str_schema('country')), - ), + [[], []], + (new KeyValues([]))->of(array_to_rows([['id' => 1], ['id' => 2]], schema(int_schema('id')))), ); } - public function test_duplicated_refs_are_extracted_per_position(): void + public function test_of_an_empty_batch_is_empty(): void + { + static::assertSame([], (new KeyValues([ref('id')]))->of(rows(schema(str_schema('name'))))); + } + + public function test_of_preserves_null(): void { static::assertSame( - [1, 1], - (new KeyValues([ref('id'), ref('id')]))->ofRow(row(['id' => 1]), schema(int_schema('id'))), + [[null], ['a']], + (new KeyValues([ref('id')]))->of(array_to_rows([ + ['id' => null], + ['id' => 'a'], + ], schema(str_schema('id', nullable: true)))), ); } - public function test_a_missing_not_null_ref_throws(): void + public function test_of_projects_each_row(): void { - $this->expectException(InvalidArgumentException::class); - $this->expectExceptionMessage('Column "id" does not exist'); - - (new KeyValues([ref('id')]))->ofRow(row(['name' => 'flow']), schema(int_schema('id'), str_schema('name'))); + static::assertSame( + [[1], [2]], + (new KeyValues([ref('id')]))->of(array_to_rows([['id' => 1], ['id' => 2]], schema(int_schema('id')))), + ); } - public function test_a_missing_nullable_ref_is_null_when_the_schema_allows_it(): void + public function test_of_projects_refs_in_order(): void { static::assertSame( - [null, 'flow'], - (new KeyValues([ref('id'), ref('name')]))->ofRow( - row(['name' => 'flow']), - schema(int_schema('id', nullable: true), str_schema('name')), - ), + [[1, 'PL'], [2, 'DE']], + (new KeyValues([ref('id'), ref('country')]))->of(array_to_rows( + [['id' => 1, 'country' => 'PL'], ['id' => 2, 'country' => 'DE']], + schema(int_schema('id'), str_schema('country')), + )), ); } } diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Bucketing/NativeHasherTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Bucketing/NativeHasherTest.php index b8e47bbe4f..8831f572c3 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Unit/Bucketing/NativeHasherTest.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Bucketing/NativeHasherTest.php @@ -80,4 +80,11 @@ public function test_normalize_distinguishes_null_from_string(): void { static::assertNotSame(NativeHasher::normalize(null), NativeHasher::normalize('null')); } + + public function test_values_do_not_run_into_the_next_one(): void + { + [$first, $second] = (new NativeHasher())->hash([['ab', 'c'], ['a', 'bc']]); + + static::assertNotSame($first, $second); + } } diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Bucketing/SortedRunBucketingTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Bucketing/SortedRunBucketingTest.php deleted file mode 100644 index c9da5f21ad..0000000000 --- a/src/core/etl/tests/Flow/ETL/Tests/Unit/Bucketing/SortedRunBucketingTest.php +++ /dev/null @@ -1,145 +0,0 @@ - 1]), row(['id' => 2])); - yield rows(schema(int_schema('id')), row(['id' => 3]), row(['id' => 4])); - })(); - - $ids = []; - - foreach ($strategy->bucketize($generator, new MemoryBuckets()) as $bucket) { - static::assertMatchesRegularExpression('/^sort-[0-9a-f]{16}-\d+$/', $bucket->id); - $ids[] = $bucket->id; - } - - static::assertCount(2, $ids); - static::assertSame($ids, array_unique($ids)); - } - - public function test_emits_final_partial_run(): void - { - $strategy = new SortedRunBucketing([ref('id')], 2, new NativePHPRandomValueGenerator()); - - $generator = (static function () { - yield rows(schema(int_schema('id')), row(['id' => 3]), row(['id' => 1]), row(['id' => 2])); - })(); - - $sizes = []; - - foreach ($strategy->bucketize($generator, new MemoryBuckets()) as $bucket) { - $sizes[] = $bucket->totalRows; - } - - static::assertSame([2, 1], $sizes); - } - - public function test_runs_are_spilled_sorted_by_refs(): void - { - $strategy = new SortedRunBucketing([ref('id')], 3, new NativePHPRandomValueGenerator()); - $storage = new MemoryBuckets(); - - $generator = (static function () { - yield rows(schema(int_schema('id')), row(['id' => 3]), row(['id' => 1]), row(['id' => 2])); - })(); - - $buckets = iterator_to_array($strategy->bucketize($generator, $storage)); - - static::assertCount(1, $buckets); - static::assertSame( - [1, 2, 3], - array_map(static fn(Row $r): mixed => $r->get( - 'id', - ), BucketsStorageContext::rows($storage->get($buckets[0]->id))), - ); - } - - public function test_runs_are_yielded_as_soon_as_they_are_complete(): void - { - $strategy = new SortedRunBucketing([ref('id')], 2, new NativePHPRandomValueGenerator()); - $storage = new MemoryBuckets(); - - $generator = (static function () { - yield rows(schema(int_schema('id')), row(['id' => 1]), row(['id' => 2])); - yield rows(schema(int_schema('id')), row(['id' => 3]), row(['id' => 4])); - })(); - - $firstRun = $strategy->bucketize($generator, $storage)->current(); - - static::assertNotNull($firstRun); - static::assertSame(2, $firstRun->totalRows); - static::assertCount(2, BucketsStorageContext::rows($storage->get($firstRun->id))); - } - - public function test_splits_buffer_into_runs_of_run_size(): void - { - $strategy = new SortedRunBucketing([ref('id')], 2, new NativePHPRandomValueGenerator()); - - $generator = (static function () { - yield rows(schema(int_schema('id')), row(['id' => 1]), row(['id' => 2])); - yield rows(schema(int_schema('id')), row(['id' => 3]), row(['id' => 4])); - })(); - - $sizes = []; - - foreach ($strategy->bucketize($generator, new MemoryBuckets()) as $bucket) { - $sizes[] = $bucket->totalRows; - } - - static::assertSame([2, 2], $sizes); - } - - public function test_runs_are_numbered_sequentially(): void - { - $strategy = new SortedRunBucketing([ref('id')], 2, new NativePHPRandomValueGenerator()); - - $generator = (static function () { - yield rows(schema(int_schema('id')), row(['id' => 1]), row(['id' => 2])); - yield rows(schema(int_schema('id')), row(['id' => 3]), row(['id' => 4])); - yield rows(schema(int_schema('id')), row(['id' => 5])); - })(); - - $indexes = []; - - foreach ($strategy->bucketize($generator, new MemoryBuckets()) as $bucket) { - $indexes[] = $bucket->index; - } - - static::assertSame([0, 1, 2], $indexes); - } - - public function test_throws_when_run_size_below_one(): void - { - $this->expectException(InvalidArgumentException::class); - $this->expectExceptionMessage('Run size must be greater than 0, given: 0'); - - // @mago-ignore analysis:invalid-argument - new SortedRunBucketing([ref('id')], 0, new NativePHPRandomValueGenerator()); - } -} diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Bucketing/Storage/MemoryBucketsTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Bucketing/Storage/MemoryBucketsTest.php index a302fc339e..cfa808aca5 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Unit/Bucketing/Storage/MemoryBucketsTest.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Bucketing/Storage/MemoryBucketsTest.php @@ -6,14 +6,11 @@ use Flow\ETL\Bucketing\ResidentBucketsStorage; use Flow\ETL\Bucketing\Storage\MemoryBuckets; -use Flow\ETL\Row; use Flow\ETL\Tests\Context\BucketsStorageContext; use Flow\ETL\Tests\FlowTestCase; -use function array_map; +use function Flow\ETL\DSL\array_to_rows; use function Flow\ETL\DSL\int_schema; -use function Flow\ETL\DSL\row; -use function Flow\ETL\DSL\rows; use function Flow\ETL\DSL\schema; final class MemoryBucketsTest extends FlowTestCase @@ -21,13 +18,10 @@ final class MemoryBucketsTest extends FlowTestCase public function test_append_accumulates_rows_in_order(): void { $storage = new MemoryBuckets(); - $storage->append('bucket', rows(schema(int_schema('id')), row(['id' => 1]), row(['id' => 2]))); - $storage->append('bucket', rows(schema(int_schema('id')), row(['id' => 3]))); + $storage->append('bucket', array_to_rows([['id' => 1], ['id' => 2]], schema(int_schema('id')))); + $storage->append('bucket', array_to_rows([['id' => 3]], schema(int_schema('id')))); - static::assertSame( - [1, 2, 3], - array_map(static fn(Row $r): mixed => $r->get('id'), BucketsStorageContext::rows($storage->get('bucket'))), - ); + static::assertSame([1, 2, 3], array_column(BucketsStorageContext::rows($storage->get('bucket')), 'id')); } public function test_get_unknown_bucket_yields_nothing(): void @@ -43,7 +37,7 @@ public function test_is_resident_storage(): void public function test_remove_drops_the_bucket(): void { $storage = new MemoryBuckets(); - $storage->append('bucket', rows(schema(int_schema('id')), row(['id' => 1]))); + $storage->append('bucket', array_to_rows([['id' => 1]], schema(int_schema('id')))); $storage->remove('bucket'); static::assertSame([], BucketsStorageContext::rows($storage->get('bucket'))); @@ -52,12 +46,9 @@ public function test_remove_drops_the_bucket(): void public function test_set_replaces_previous_rows(): void { $storage = new MemoryBuckets(); - $storage->append('bucket', rows(schema(int_schema('id')), row(['id' => 1]))); - $storage->set('bucket', rows(schema(int_schema('id')), row(['id' => 42]))); + $storage->append('bucket', array_to_rows([['id' => 1]], schema(int_schema('id')))); + $storage->set('bucket', array_to_rows([['id' => 42]], schema(int_schema('id')))); - static::assertSame( - [42], - array_map(static fn(Row $r): mixed => $r->get('id'), BucketsStorageContext::rows($storage->get('bucket'))), - ); + static::assertSame([42], array_column(BucketsStorageContext::rows($storage->get('bucket')), 'id')); } } diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Bucketing/Storage/PSRCacheBucketsTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Bucketing/Storage/PSRCacheBucketsTest.php index 9a02f0086e..a951db54fe 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Unit/Bucketing/Storage/PSRCacheBucketsTest.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Bucketing/Storage/PSRCacheBucketsTest.php @@ -5,40 +5,38 @@ namespace Flow\ETL\Tests\Unit\Bucketing\Storage; use Flow\ETL\Bucketing\Storage\PSRCacheBuckets; -use Flow\ETL\Row; +use Flow\ETL\Column\AdaptiveBackend; use Flow\ETL\Tests\Context\BucketsStorageContext; use Flow\ETL\Tests\Double\ArrayCache; use Flow\ETL\Tests\FlowTestCase; +use Flow\Floe\FloeSerializer; -use function array_map; +use function Flow\ETL\DSL\array_to_rows; use function Flow\ETL\DSL\int_schema; -use function Flow\ETL\DSL\row; -use function Flow\ETL\DSL\rows; use function Flow\ETL\DSL\schema; use function Flow\ETL\DSL\str_schema; -use function iterator_to_array; final class PSRCacheBucketsTest extends FlowTestCase { public function test_append_then_get_returns_rows_in_order_across_chunks(): void { - $storage = new PSRCacheBuckets(new ArrayCache()); - $storage->append('bucket', rows(schema(int_schema('id')), row(['id' => 1]), row(['id' => 2]))); - $storage->append('bucket', rows(schema(int_schema('id')), row(['id' => 3]))); + $storage = new PSRCacheBuckets(new ArrayCache(), new FloeSerializer(new AdaptiveBackend())); + $storage->append('bucket', array_to_rows([['id' => 1], ['id' => 2]], schema(int_schema('id')))); + $storage->append('bucket', array_to_rows([['id' => 3]], schema(int_schema('id')))); - static::assertSame( - [1, 2, 3], - array_map(static fn(Row $r): mixed => $r->get('id'), BucketsStorageContext::rows($storage->get('bucket'))), - ); + static::assertSame([1, 2, 3], array_column(BucketsStorageContext::rows($storage->get('bucket')), 'id')); } public function test_custom_prefix_is_used_in_keys(): void { $cache = new ArrayCache(); - (new PSRCacheBuckets($cache, prefix: 'custom:pfx'))->append('bucket', rows( - schema(int_schema('id')), - row(['id' => 1]), - )); + (new PSRCacheBuckets( + $cache, + new FloeSerializer(new AdaptiveBackend()), + prefix: 'custom:pfx', + ))->append('bucket', array_to_rows([[ + 'id' => 1, + ]], schema(int_schema('id')))); static::assertTrue($cache->has('custom:pfx:bucket:chunks')); static::assertTrue($cache->has('custom:pfx:bucket:chunk:0')); @@ -50,7 +48,12 @@ public function test_get_skips_non_string_chunk_payloads(): void $cache->set('flow:buckets:bucket:chunks', 1); $cache->set('flow:buckets:bucket:chunk:0', 123); - static::assertSame([], BucketsStorageContext::rows((new PSRCacheBuckets($cache))->get('bucket'))); + static::assertSame( + [], + BucketsStorageContext::rows((new PSRCacheBuckets($cache, new FloeSerializer(new AdaptiveBackend())))->get( + 'bucket', + )), + ); } public function test_get_treats_non_integer_chunk_counter_as_empty(): void @@ -58,20 +61,31 @@ public function test_get_treats_non_integer_chunk_counter_as_empty(): void $cache = new ArrayCache(); $cache->set('flow:buckets:bucket:chunks', 'garbage'); - static::assertSame([], BucketsStorageContext::rows((new PSRCacheBuckets($cache))->get('bucket'))); + static::assertSame( + [], + BucketsStorageContext::rows((new PSRCacheBuckets($cache, new FloeSerializer(new AdaptiveBackend())))->get( + 'bucket', + )), + ); } public function test_get_unknown_bucket_yields_nothing(): void { - static::assertSame([], BucketsStorageContext::rows((new PSRCacheBuckets(new ArrayCache()))->get('unknown'))); + static::assertSame( + [], + BucketsStorageContext::rows((new PSRCacheBuckets( + new ArrayCache(), + new FloeSerializer(new AdaptiveBackend()), + ))->get('unknown')), + ); } public function test_remove_deletes_all_chunk_keys_and_the_counter(): void { $cache = new ArrayCache(); - $storage = new PSRCacheBuckets($cache); - $storage->append('bucket', rows(schema(int_schema('id')), row(['id' => 1]))); - $storage->append('bucket', rows(schema(int_schema('id')), row(['id' => 2]))); + $storage = new PSRCacheBuckets($cache, new FloeSerializer(new AdaptiveBackend())); + $storage->append('bucket', array_to_rows([['id' => 1]], schema(int_schema('id')))); + $storage->append('bucket', array_to_rows([['id' => 2]], schema(int_schema('id')))); $storage->remove('bucket'); @@ -83,30 +97,22 @@ public function test_remove_deletes_all_chunk_keys_and_the_counter(): void public function test_round_trips_schema_changing_rows(): void { - $storage = new PSRCacheBuckets(new ArrayCache()); - $input = rows( + $storage = new PSRCacheBuckets(new ArrayCache(), new FloeSerializer(new AdaptiveBackend())); + $input = array_to_rows( + [['id' => 1], ['id' => 2, 'name' => 'John'], ['name' => 'Jane']], schema(int_schema('id', nullable: true), str_schema('name', nullable: true)), - row(['id' => 1]), - row(['id' => 2, 'name' => 'John']), - row(['name' => 'Jane']), ); $storage->append('bucket', $input); - static::assertSame( - array_map(static fn(Row $r): array => $r->toArray(), iterator_to_array($input, false)), - array_map(static fn(Row $r): array => $r->toArray(), BucketsStorageContext::rows($storage->get('bucket'))), - ); + static::assertSame($input->toArray(), BucketsStorageContext::rows($storage->get('bucket'))); } public function test_set_replaces_previous_content(): void { - $storage = new PSRCacheBuckets(new ArrayCache()); - $storage->append('bucket', rows(schema(int_schema('id')), row(['id' => 1]), row(['id' => 2]))); - $storage->set('bucket', rows(schema(int_schema('id')), row(['id' => 42]))); + $storage = new PSRCacheBuckets(new ArrayCache(), new FloeSerializer(new AdaptiveBackend())); + $storage->append('bucket', array_to_rows([['id' => 1], ['id' => 2]], schema(int_schema('id')))); + $storage->set('bucket', array_to_rows([['id' => 42]], schema(int_schema('id')))); - static::assertSame( - [42], - array_map(static fn(Row $r): mixed => $r->get('id'), BucketsStorageContext::rows($storage->get('bucket'))), - ); + static::assertSame([42], array_column(BucketsStorageContext::rows($storage->get('bucket')), 'id')); } } diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Bucketing/Storage/SpillingBucketsTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Bucketing/Storage/SpillingBucketsTest.php new file mode 100644 index 0000000000..e0a1a3a50d --- /dev/null +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Bucketing/Storage/SpillingBucketsTest.php @@ -0,0 +1,111 @@ +append('a', $batch); + } + + static::assertSame([], $disk->appendedRows()); + static::assertSame([1, 2, 3, 4], IdRows::ids($buckets->get('a'))); + } + + public function test_past_the_limit_every_frame_holds_a_full_batch_and_the_rest_is_read_after_them(): void + { + $disk = new SpyBucketsStorage(new MemoryBuckets()); + $buckets = new SpillingBuckets($disk, Unit::fromBytes(1), new PhpBackend(), 2); + + foreach (IdRows::batches([1], [2], [3], [4], [5]) as $batch) { + $buckets->append('a', $batch); + } + + static::assertSame( + ['a' => [2, 2]], + array_map(static fn(array $frames): array => array_map( + static fn(Rows $rows): int => $rows->count(), + $frames, + ), $disk->appendedRows()), + ); + static::assertSame([1, 2, 3, 4, 5], IdRows::ids($buckets->get('a'))); + } + + public function test_crossing_the_limit_moves_every_bucket_that_holds_a_full_batch(): void + { + $disk = new SpyBucketsStorage(new MemoryBuckets()); + $buckets = new SpillingBuckets($disk, Unit::fromBytes(1), new PhpBackend(), 2); + + $buckets->append('a', IdRows::batches([1, 2, 3])->current()); + + static::assertSame([1, 2], IdRows::ids($disk->get('a'))); + static::assertSame([1, 2, 3], IdRows::ids($buckets->get('a'))); + } + + public function test_remove_drops_the_buffer_and_the_frames(): void + { + $disk = new SpyBucketsStorage(new MemoryBuckets()); + $buckets = new SpillingBuckets($disk, Unit::fromBytes(1), new PhpBackend(), 1); + $buckets->append('a', IdRows::batches([1, 2])->current()); + + $buckets->remove('a'); + + static::assertSame([], IdRows::ids($buckets->get('a'))); + static::assertSame([], $disk->liveBucketIds()); + } + + public function test_set_replaces_what_the_bucket_held(): void + { + $buckets = new SpillingBuckets(new MemoryBuckets(), Unit::fromGb(64), new PhpBackend(), 2); + $buckets->append('a', IdRows::batches([1])->current()); + + $buckets->set('a', IdRows::batches([7, 8])->current()); + + static::assertSame([7, 8], IdRows::ids($buckets->get('a'))); + } + + public function test_an_unknown_bucket_reads_nothing(): void + { + static::assertSame( + [], + IdRows::ids((new SpillingBuckets(new MemoryBuckets(), Unit::fromGb(64), new PhpBackend()))->get('missing')), + ); + } + + public function test_around_keeps_a_resident_storage_and_wraps_any_other(): void + { + $resident = new MemoryBuckets(); + $disk = new SpyBucketsStorage(new MemoryBuckets()); + + static::assertSame($resident, SpillingBuckets::around( + new BucketingConfig($resident, 64, 1000), + Unit::fromMb(1), + new PhpBackend(), + )); + + $wrapped = SpillingBuckets::around(new BucketingConfig($disk, 64, 500), Unit::fromMb(3), new PhpBackend()); + + static::assertInstanceOf(SpillingBuckets::class, $wrapped); + static::assertSame($disk, $wrapped->disk); + static::assertSame(Unit::fromMb(3)->inBytes(), $wrapped->memoryLimit->inBytes()); + } +} diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Cache/CacheIndexTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Cache/CacheIndexTest.php index 7f36e3845c..8fe275a0bd 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Unit/Cache/CacheIndexTest.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Cache/CacheIndexTest.php @@ -6,12 +6,12 @@ use Flow\ETL\Cache\CacheIndex; use Flow\ETL\Cardinality; +use Flow\ETL\Column\PhpBackend; use Flow\ETL\Exception\InvalidArgumentException; use Flow\ETL\Tests\FlowTestCase; +use function Flow\ETL\DSL\array_to_rows; use function Flow\ETL\DSL\int_schema; -use function Flow\ETL\DSL\row; -use function Flow\ETL\DSL\rows; use function Flow\ETL\DSL\schema; use function Flow\ETL\DSL\str_schema; @@ -21,7 +21,7 @@ public function test_empty_index_round_trip(): void { $index = new CacheIndex('dataset-id'); - $indexRows = $index->toRows(); + $indexRows = $index->toRows(new PhpBackend()); static::assertCount(0, $indexRows); static::assertEquals($index, CacheIndex::fromRows('dataset-id', $indexRows)); @@ -34,7 +34,7 @@ public function test_an_empty_index_declares_zero_rows_exactly(): void public function test_an_index_without_a_rows_column_declares_unknown_rows(): void { - $index = CacheIndex::fromRows('dataset-id', rows(schema(str_schema('key')), row(['key' => 'chunk-1']))); + $index = CacheIndex::fromRows('dataset-id', array_to_rows([['key' => 'chunk-1']], schema(str_schema('key')))); static::assertSame(['chunk-1'], $index->values()); static::assertEquals(Cardinality::unknown(), $index->rows()); @@ -54,7 +54,7 @@ public function test_from_rows_key_comes_from_the_argument(): void $index = new CacheIndex('original-key'); $index->add('chunk-1'); - $reconstructed = CacheIndex::fromRows('different-key', $index->toRows()); + $reconstructed = CacheIndex::fromRows('different-key', $index->toRows(new PhpBackend())); static::assertSame('different-key', $reconstructed->key); static::assertSame(['chunk-1'], $reconstructed->values()); @@ -64,7 +64,7 @@ public function test_from_rows_with_missing_key_entry_throws(): void { $this->expectException(InvalidArgumentException::class); - CacheIndex::fromRows('dataset-id', rows(schema(int_schema('id')), row(['id' => 1]))); + CacheIndex::fromRows('dataset-id', array_to_rows([['id' => 1]], schema(int_schema('id')))); } public function test_from_rows_with_non_string_key_entry_throws(): void @@ -72,7 +72,7 @@ public function test_from_rows_with_non_string_key_entry_throws(): void $this->expectException(InvalidArgumentException::class); $this->expectExceptionMessage('CacheIndex expects rows with a string "key" entry, got: null'); - CacheIndex::fromRows('dataset-id', rows(schema(str_schema('key', nullable: true)), row(['key' => null]))); + CacheIndex::fromRows('dataset-id', array_to_rows([['key' => null]], schema(str_schema('key', nullable: true)))); } public function test_from_rows_with_non_integer_rows_entry_throws(): void @@ -80,9 +80,9 @@ public function test_from_rows_with_non_integer_rows_entry_throws(): void $this->expectException(InvalidArgumentException::class); $this->expectExceptionMessage('CacheIndex expects rows with an integer "rows" entry, got: string'); - CacheIndex::fromRows('dataset-id', rows( + CacheIndex::fromRows('dataset-id', array_to_rows( + [['key' => 'chunk-1', 'rows' => 'three']], schema(str_schema('key'), str_schema('rows')), - row(['key' => 'chunk-1', 'rows' => 'three']), )); } @@ -93,7 +93,10 @@ public function test_rows_sum_the_chunk_counts(): void $index->add('chunk-2', 0); $index->add('chunk-3', 4); - static::assertEquals(Cardinality::exact(7), CacheIndex::fromRows('dataset-id', $index->toRows())->rows()); + static::assertEquals( + Cardinality::exact(7), + CacheIndex::fromRows('dataset-id', $index->toRows(new PhpBackend()))->rows(), + ); } public function test_to_rows_from_rows_round_trip_preserves_order(): void @@ -103,7 +106,7 @@ public function test_to_rows_from_rows_round_trip_preserves_order(): void $index->add('chunk-a'); $index->add('chunk-c'); - $reconstructed = CacheIndex::fromRows('dataset-id', $index->toRows()); + $reconstructed = CacheIndex::fromRows('dataset-id', $index->toRows(new PhpBackend())); static::assertEquals($index, $reconstructed); static::assertSame(['chunk-b', 'chunk-a', 'chunk-c'], $reconstructed->values()); @@ -116,12 +119,11 @@ public function test_to_rows_stores_one_key_and_rows_entry_per_chunk(): void $index->add('chunk-2'); static::assertEquals( - rows( + array_to_rows( + [['key' => 'chunk-1', 'rows' => 2], ['key' => 'chunk-2', 'rows' => null]], schema(str_schema('key'), int_schema('rows', nullable: true)), - row(['key' => 'chunk-1', 'rows' => 2]), - row(['key' => 'chunk-2', 'rows' => null]), ), - $index->toRows(), + $index->toRows(new PhpBackend()), ); } } diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Cache/Implementation/ApcuCacheTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Cache/Implementation/ApcuCacheTest.php index 27564fd095..c39d2ff38c 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Unit/Cache/Implementation/ApcuCacheTest.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Cache/Implementation/ApcuCacheTest.php @@ -5,18 +5,19 @@ namespace Flow\ETL\Tests\Unit\Cache\Implementation; use Flow\ETL\Cache\Implementation\ApcuCache; +use Flow\ETL\Column\PhpBackend; use Flow\ETL\Exception\KeyNotInCacheException; use Flow\ETL\Exception\RuntimeException; use Flow\ETL\Tests\FlowTestCase; +use Flow\Floe\FloeSerializer; use Override; use PHPUnit\Framework\Attributes\CoversClass; use PHPUnit\Framework\Attributes\RequiresPhpExtension; use function apcu_enabled; use function apcu_store; +use function Flow\ETL\DSL\array_to_rows; use function Flow\ETL\DSL\int_schema; -use function Flow\ETL\DSL\row; -use function Flow\ETL\DSL\rows; use function Flow\ETL\DSL\schema; use function Flow\ETL\DSL\str_schema; @@ -35,8 +36,8 @@ protected function setUp(): void static::markTestSkipped('APCu is not enabled for CLI (apc.enable_cli=0).'); } - $this->cache = new ApcuCache('flow_php_cache_test'); - $this->otherCache = new ApcuCache('flow_php_cache_test_other'); + $this->cache = new ApcuCache(new FloeSerializer(new PhpBackend()), 'flow_php_cache_test'); + $this->otherCache = new ApcuCache(new FloeSerializer(new PhpBackend()), 'flow_php_cache_test_other'); $this->cache->clear(); $this->otherCache->clear(); @@ -59,10 +60,9 @@ public function test_caching_rows(): void $this->cache->set( 'rows', - $rows = rows( + $rows = array_to_rows( + [['id' => 1, 'name' => 'John'], ['id' => 2, 'name' => 'Jane']], schema(int_schema('id'), str_schema('name')), - row(['id' => 1, 'name' => 'John']), - row(['id' => 2, 'name' => 'Jane']), ), ); @@ -74,7 +74,7 @@ public function test_caching_schema(): void { $this->cache->set( 'rows', - $rows = rows(schema(int_schema('id'), str_schema('name')), row(['id' => 1, 'name' => 'John'])), + $rows = array_to_rows([['id' => 1, 'name' => 'John']], schema(int_schema('id'), str_schema('name'))), ); static::assertEquals($rows->schema(), $this->cache->schema('rows')); @@ -87,8 +87,8 @@ public function test_checking_on_non_existing_cache_key(): void public function test_clearing_cache_removes_only_its_own_namespace(): void { - $this->cache->set('rows', rows(schema(int_schema('id')), row(['id' => 1]))); - $this->otherCache->set('rows', rows(schema(int_schema('id')), row(['id' => 2]))); + $this->cache->set('rows', array_to_rows([['id' => 1]], schema(int_schema('id')))); + $this->otherCache->set('rows', array_to_rows([['id' => 2]], schema(int_schema('id')))); $this->cache->clear(); @@ -98,7 +98,7 @@ public function test_clearing_cache_removes_only_its_own_namespace(): void public function test_clearing_cache_removes_schemas(): void { - $this->cache->set('rows', rows(schema(int_schema('id')), row(['id' => 1]))); + $this->cache->set('rows', array_to_rows([['id' => 1]], schema(int_schema('id')))); $this->cache->clear(); @@ -107,6 +107,16 @@ public function test_clearing_cache_removes_schemas(): void $this->cache->schema('rows'); } + public function test_getting_an_entry_another_writer_stored(): void + { + apcu_store('flow_php_cache_test:rows', array_to_rows([['id' => 1]], schema(int_schema('id')))); + + $this->expectException(RuntimeException::class); + $this->expectExceptionMessage('Cache entry for key "rows" is corrupted or was not written by ApcuCache.'); + + $this->cache->get('rows'); + } + public function test_getting_corrupted_schema_entry(): void { apcu_store('flow_php_cache_test:rows:schema', 'not a schema'); @@ -133,8 +143,8 @@ public function test_getting_schema_of_non_existing_cache_key(): void public function test_removing_from_cache(): void { - $this->cache->set('first', rows(schema(int_schema('id')), row(['id' => 1]))); - $this->cache->set('second', rows(schema(int_schema('id')), row(['id' => 2]))); + $this->cache->set('first', array_to_rows([['id' => 1]], schema(int_schema('id')))); + $this->cache->set('second', array_to_rows([['id' => 2]], schema(int_schema('id')))); $this->cache->delete('second'); @@ -144,8 +154,8 @@ public function test_removing_from_cache(): void public function test_removing_from_cache_removes_its_schema(): void { - $this->cache->set('first', $rows = rows(schema(int_schema('id')), row(['id' => 1]))); - $this->cache->set('second', rows(schema(int_schema('id')), row(['id' => 2]))); + $this->cache->set('first', $rows = array_to_rows([['id' => 1]], schema(int_schema('id')))); + $this->cache->set('second', array_to_rows([['id' => 2]], schema(int_schema('id')))); $this->cache->delete('second'); @@ -165,7 +175,7 @@ public function test_removing_non_existing_cache_key(): void public function test_schema_of_an_entry_without_a_stored_schema_is_a_cache_miss(): void { - apcu_store('flow_php_cache_test:orphan', rows(schema(int_schema('id')), row(['id' => 1]))); + apcu_store('flow_php_cache_test:orphan', array_to_rows([['id' => 1]], schema(int_schema('id')))); static::assertTrue($this->cache->has('orphan')); diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Cache/Implementation/TraceableCacheTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Cache/Implementation/TraceableCacheTest.php index f40857fe16..44316996fd 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Unit/Cache/Implementation/TraceableCacheTest.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Cache/Implementation/TraceableCacheTest.php @@ -24,8 +24,8 @@ use PHPUnit\Framework\Attributes\CoversClass; use RuntimeException; +use function Flow\ETL\DSL\array_to_rows; use function Flow\ETL\DSL\int_schema; -use function Flow\ETL\DSL\row; use function Flow\ETL\DSL\rows; use function Flow\ETL\DSL\schema; @@ -101,7 +101,7 @@ public function test_get_increments_hit_counter_on_existing_key(): void $innerCache = new InMemoryCache(); $cache = new TraceableCache($innerCache, $this->telemetry, 'test_dataframe'); - $innerCache->set('existing-key', rows(schema(), row([]))); + $innerCache->set('existing-key', array_to_rows([[]], schema())); $cache->get('existing-key'); $this->telemetry->flush(); @@ -141,7 +141,7 @@ public function test_has_increments_hit_counter_when_key_exists(): void $innerCache = new InMemoryCache(); $cache = new TraceableCache($innerCache, $this->telemetry, 'test_dataframe'); - $innerCache->set('existing-key', rows(schema(), row([]))); + $innerCache->set('existing-key', array_to_rows([[]], schema())); $exists = $cache->has('existing-key'); static::assertTrue($exists); @@ -181,7 +181,7 @@ public function test_schema_increments_hit_counter_on_existing_key(): void $innerCache = new InMemoryCache(); $cache = new TraceableCache($innerCache, $this->telemetry, 'test_dataframe'); - $innerCache->set('existing-key', $rows = rows(schema(int_schema('id')), row(['id' => 1]))); + $innerCache->set('existing-key', $rows = array_to_rows([['id' => 1]], schema(int_schema('id')))); static::assertEquals($rows->schema(), $cache->schema('existing-key')); @@ -290,7 +290,7 @@ public function test_set_records_exception_on_error(): void $this->expectExceptionMessage('Test error'); try { - $cache->set('test-key', rows(schema(), row([]))); + $cache->set('test-key', array_to_rows([[]], schema())); } finally { $this->telemetry->flush(); $spans = $this->spanProcessor->endedSpans(); diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Column/AdaptiveBackendTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Column/AdaptiveBackendTest.php new file mode 100644 index 0000000000..07f406c467 --- /dev/null +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Column/AdaptiveBackendTest.php @@ -0,0 +1,90 @@ +adopt(int_schema('a'), $column)); + static::assertEquals( + (new PhpBackend())->adopt(int_schema('a'), $foreign), + (new AdaptiveBackend())->adopt(int_schema('a'), $foreign), + ); + } + + public function test_adopt_refuses_an_untyped_column(): void + { + $this->expectException(ColumnMismatchException::class); + $this->expectExceptionMessage('column "a": mixed cannot be a batch column'); + + (new AdaptiveBackend())->adopt(int_schema('a'), new ValueColumn([1])); + } + + public function test_allocated_bytes_is_what_the_php_backend_allocates(): void + { + static::assertSame((new PhpBackend())->allocatedBytes(), (new AdaptiveBackend())->allocatedBytes()); + } + + public function test_builder_builds_what_the_php_backend_builds(): void + { + $default = (new AdaptiveBackend())->builder(str_schema('a')); + $default->append('x'); + $php = (new PhpBackend())->builder(str_schema('a')); + $php->append('x'); + + static::assertEquals($php->finish(), $default->finish()); + } + + public function test_constant_is_what_the_php_backend_builds(): void + { + static::assertEquals( + (new PhpBackend())->constant(int_schema('a'), 7, 3), + (new AdaptiveBackend())->constant(int_schema('a'), 7, 3), + ); + } + + public function test_decode_is_what_the_php_backend_decodes(): void + { + $buffers = ColumnMother::of(int_schema('a', nullable: true), [1, null])->encode(); + + static::assertEquals( + (new PhpBackend())->decode(int_schema('a', nullable: true), $buffers, 2, 1), + (new AdaptiveBackend())->decode(int_schema('a', nullable: true), $buffers, 2, 1), + ); + } + + public function test_a_flow_php_of_another_abi_is_refused(): void + { + $this->expectException(RuntimeException::class); + $this->expectExceptionMessage('does not match flow-php/etl'); + + new AdaptiveBackend(new FlowPhpExtension(true, FlowPhpExtension::ABI + 1, '0.46.0')); + } +} diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Column/ColumnConcatTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Column/ColumnConcatTest.php new file mode 100644 index 0000000000..541fa57135 --- /dev/null +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Column/ColumnConcatTest.php @@ -0,0 +1,115 @@ +concat(ColumnMother::of( + int_schema('a', nullable: true), + [null, 4], + )); + + static::assertSame([1, null, null, 4], $column->values()); + static::assertSame(2, $column->nullCount()); + } + + public function test_list_columns_rebase_offsets_and_rekey_nulls(): void + { + $definition = list_schema('a', type_list(type_integer()), nullable: true); + $column = ColumnMother::of($definition, [[1], null])->concat(ColumnMother::of($definition, [null, [2, 3]])); + + static::assertSame([[1], null, null, [2, 3]], $column->values()); + static::assertSame(2, $column->nullCount()); + static::assertTrue($column->isNull(2)); + } + + public function test_map_columns_rebase_offsets_and_rekey_nulls(): void + { + $definition = map_schema('a', type_map(type_string(), type_integer()), nullable: true); + $column = ColumnMother::of($definition, [['x' => 1], null])->concat(ColumnMother::of($definition, [ + null, + ['y' => 2, 'z' => 3], + ])); + + static::assertSame([['x' => 1], null, null, ['y' => 2, 'z' => 3]], $column->values()); + static::assertTrue($column->isNull(2)); + } + + public function test_structure_columns(): void + { + $definition = structure_schema('a', type_structure(['x' => type_integer()]), nullable: true); + $column = ColumnMother::of($definition, [['x' => 1]])->concat(ColumnMother::of($definition, [ + null, + ['x' => 2], + ])); + + static::assertSame([['x' => 1], null, ['x' => 2]], $column->values()); + static::assertTrue($column->isNull(1)); + } + + public function test_constant_columns_with_the_same_value_collapse(): void + { + $backend = new PhpBackend(); + $column = $backend->constant(int_schema('a'), 7, 2)->concat($backend->constant(int_schema('a'), 7, 3)); + + static::assertInstanceOf(ConstantColumn::class, $column); + static::assertSame(5, $column->count()); + } + + public function test_constant_columns_with_different_values_expand(): void + { + $backend = new PhpBackend(); + $column = $backend->constant(int_schema('a'), 7, 2)->concat($backend->constant(int_schema('a'), 8, 1)); + + static::assertInstanceOf(ScalarColumn::class, $column); + static::assertSame([7, 7, 8], $column->values()); + } + + public function test_a_mixed_class_concat_is_refused(): void + { + $this->expectException(InvalidArgumentException::class); + $this->expectExceptionMessage('cannot concat'); + + ColumnMother::of(int_schema('a'), [1])->concat((new PhpBackend())->constant(int_schema('a'), 7, 2)); + } + + public function test_container_columns_refuse_another_column_class(): void + { + $constant = (new PhpBackend())->constant(int_schema('a'), 1, 1); + $containers = [ + ColumnMother::of(list_schema('a', type_list(type_integer())), [[1]]), + ColumnMother::of(map_schema('a', type_map(type_string(), type_integer())), [['x' => 1]]), + ColumnMother::of(structure_schema('a', type_structure(['x' => type_integer()])), [['x' => 1]]), + ]; + + foreach ($containers as $container) { + try { + $container->concat($constant); + static::fail($container::class . ' concatenated another column class'); + } catch (InvalidArgumentException $e) { + static::assertStringContainsString('cannot concat', $e->getMessage()); + } + } + } +} diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Column/ColumnEncodeTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Column/ColumnEncodeTest.php new file mode 100644 index 0000000000..48babf1a09 --- /dev/null +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Column/ColumnEncodeTest.php @@ -0,0 +1,115 @@ +}> + */ + public static function columns(): Generator + { + yield 'integer' => [ + ColumnMother::of(int_schema('id'), [1, 2]), + ['', '01000000000000000200000000000000'], + ]; + yield 'nullable string' => [ + ColumnMother::of(str_schema('name', nullable: true), ['ab', null]), + ['01', '000000000200000002000000', '6162'], + ]; + yield 'float' => [ColumnMother::of(float_schema('a'), [1.5]), ['', '000000000000f83f']]; + yield 'date before the epoch' => [ + ColumnMother::of(date_schema('a'), [new DateTimeImmutable('1969-12-29 00:00:00 UTC')]), + ['', 'fdffffff'], + ]; + yield 'boolean' => [ColumnMother::of(bool_schema('a'), [true, false, true]), ['', '05']]; + yield 'nullable boolean with a null slot at 0' => [ + ColumnMother::of(bool_schema('a', nullable: true), [true, null, true]), + ['05', '05'], + ]; + yield 'uuid' => [ + ColumnMother::of(uuid_schema('a'), ['6c2f1d4e-8b3a-4c5d-9e6f-0a1b2c3d4e5f']), + ['', '6c2f1d4e8b3a4c5d9e6f0a1b2c3d4e5f'], + ]; + yield 'list with a null slot' => [ + ColumnMother::of(list_schema('a', type_list(type_integer()), nullable: true), [[1], null, [2, 3]]), + ['05', '00000000010000000100000003000000', '', '010000000000000002000000000000000300000000000000'], + ]; + yield 'map' => [ + ColumnMother::of(map_schema('a', type_map(type_string(), type_integer())), [['a' => 1]]), + ['', '0000000001000000', '', '', '0000000001000000', '61', '', '0100000000000000'], + ]; + yield 'structure: the child bit is 0 under a null parent' => [ + ColumnMother::of(structure_schema('a', type_structure(['x' => type_integer()]), nullable: true), [ + ['x' => 1], + null, + ]), + ['01', '01', '01000000000000000000000000000000'], + ]; + yield 'a null column has no buffers' => [ColumnMother::of(null_schema('a'), [null, null, null]), []]; + yield 'a null element of a structure has no buffers' => [ + ColumnMother::of(structure_schema('a', type_structure(['x' => type_integer(), 'n' => type_null()])), [ + ['x' => 1, 'n' => null], + ]), + ['', '', '0100000000000000'], + ]; + yield 'an optional null element of a structure has no buffers' => [ + ColumnMother::of( + structure_schema('a', type_structure(['x' => type_integer(), 'n' => type_optional(type_null())])), + [['x' => 1, 'n' => null]], + ), + ['', '', '0100000000000000'], + ]; + yield 'a constant is expanded' => [ + (new PhpBackend())->constant(int_schema('a'), 7, 2), + ['', '07000000000000000700000000000000'], + ]; + yield 'a slice re-bases offsets and re-packs validity from bit 0' => [ + ColumnMother::of(list_schema('a', type_list(type_integer()), nullable: true), [ + [1], + [2, 3], + null, + ])->slice(1, 2), + ['01', '000000000200000002000000', '', '02000000000000000300000000000000'], + ]; + } + + /** + * @param list $hex + */ + #[DataProvider('columns')] + public function test_encodes(Column $column, array $hex): void + { + static::assertSame($hex, array_map(bin2hex(...), $column->encode())); + } +} diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Column/ColumnWithTypeTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Column/ColumnWithTypeTest.php new file mode 100644 index 0000000000..cfb0667ea6 --- /dev/null +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Column/ColumnWithTypeTest.php @@ -0,0 +1,138 @@ +withType(type_integer())->values()); + } + + public function test_a_datetime_zone_change_keeps_the_instants(): void + { + $column = ColumnMother::of(datetime_schema('a'), [new DateTimeImmutable('2024-01-02 03:04:05 UTC')]); + $warsaw = $column->withType(type_datetime('Europe/Warsaw')); + $value = type_instance_of(DateTimeImmutable::class)->assert($warsaw->value(0)); + + static::assertSame($column->physicals(), $warsaw->physicals()); + static::assertInstanceOf(DateTimeImmutable::class, $value); + static::assertSame('Europe/Warsaw', $value->getTimezone()->getName()); + static::assertSame('2024-01-02 04:04:05', $value->format('Y-m-d H:i:s')); + } + + public function test_a_nullable_list_element_narrows_without_nulls(): void + { + $column = ColumnMother::of(list_schema('a', type_list(type_optional(type_integer()))), [[1, 2]]); + + static::assertSame([[1, 2]], $column->withType(type_list(type_integer()))->values()); + } + + public function test_a_nullable_list_element_with_a_null_does_not_narrow(): void + { + $column = ColumnMother::of(list_schema('a', type_list(type_optional(type_integer()))), [[1, null]]); + + $this->expectException(InvalidArgumentException::class); + $this->expectExceptionMessage('1 null values under a required type'); + + $column->withType(type_list(type_integer())); + } + + public function test_a_map_retypes_its_values(): void + { + $column = ColumnMother::of(map_schema('a', type_map(type_string(), type_optional(type_integer()))), [[ + 'x' => 1, + ]]); + + static::assertSame([['x' => 1]], $column->withType(type_map(type_string(), type_integer()))->values()); + } + + public function test_an_optional_structure_element_with_an_absent_value_does_not_become_required(): void + { + $column = ColumnMother::of( + structure_schema('a', type_structure([ + 'x' => type_integer(), + 'y' => structure_element('y', type_integer(), true), + ])), + [['x' => 1]], + ); + + $this->expectException(InvalidArgumentException::class); + $this->expectExceptionMessage('1 absent values under the required element "y"'); + + $column->withType(type_structure(['x' => type_integer(), 'y' => type_integer()])); + } + + public function test_an_optional_structure_element_without_absent_values_becomes_required(): void + { + $column = ColumnMother::of( + structure_schema( + 'a', + type_structure(['x' => type_integer(), 'y' => structure_element('y', type_integer(), true)]), + nullable: true, + ), + [['x' => 1, 'y' => 2], null], + ); + + static::assertSame( + [['x' => 1, 'y' => 2], null], + $column->withType(type_structure(['x' => type_integer(), 'y' => type_integer()]))->values(), + ); + } + + public function test_a_renamed_structure_element_is_another_kind(): void + { + $column = ColumnMother::of(structure_schema('a', type_structure(['x' => type_integer()])), [['x' => 1]]); + + $this->expectException(InvalidArgumentException::class); + $this->expectExceptionMessage('structure{x: integer} and structure{z: integer} are different column kinds'); + + $column->withType(type_structure(['z' => type_integer()])); + } + + public function test_an_integer_does_not_become_a_string(): void + { + $this->expectException(InvalidArgumentException::class); + $this->expectExceptionMessage('integer and string are different column kinds'); + + ColumnMother::of(int_schema('id'), [1])->withType(type_string()); + } + + public function test_an_enum_does_not_change_its_class(): void + { + $this->expectException(InvalidArgumentException::class); + $this->expectExceptionMessage('are different column kinds'); + + ColumnMother::of(enum_schema('a', BackedStringEnum::class), [BackedStringEnum::one])->withType(type_enum( + BackedIntEnum::class, + )); + } +} diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Column/ComparableValuesTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Column/ComparableValuesTest.php new file mode 100644 index 0000000000..8330844dd2 --- /dev/null +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Column/ComparableValuesTest.php @@ -0,0 +1,164 @@ +, list}> + */ + public static function ordered(): Generator + { + yield 'integer' => [int_schema('a', nullable: true), [2, null, 1]]; + yield 'float' => [float_schema('a'), [1.5, 0.5]]; + yield 'boolean' => [bool_schema('a'), [true, false]]; + yield 'string' => [str_schema('a'), ['b', 'a']]; + yield 'datetime' => [ + datetime_schema('a'), + [new DateTimeImmutable('2024-01-02'), new DateTimeImmutable('2024-01-01')], + ]; + yield 'time' => [time_schema('a'), ['PT2H', 'PT1H']]; + } + + /** + * @return Generator, list}> + */ + public static function equal_by_physical(): Generator + { + yield 'uuid' => [uuid_schema('a'), ['00000000-0000-4000-8000-000000000001']]; + yield 'json' => [json_schema('a'), ['{"a":1}']]; + yield 'enum' => [enum_schema('a', BackedStringEnum::class), [BackedStringEnum::one]]; + yield 'xml' => [xml_schema('a'), ['1']]; + } + + /** + * @return Generator, list}> + */ + public static function by_value(): Generator + { + yield 'timezone' => [time_zone_schema('a'), ['UTC']]; + yield 'list' => [list_schema('a', type_list(type_integer())), [[1, 2]]]; + } + + #[RequiresPhp('>= 8.4.0')] + public function test_html_compares_by_physical(): void + { + $document = ColumnMother::of( + html_schema('a'), + ['

a

'], + ); + $element = ColumnMother::of(html_element_schema('a'), [ + type_html_element()->cast('

a

')->firstElementChild, + type_html_element()->cast('

a

')->firstElementChild, + ]); + + static::assertSame($document->physicals(), (new ComparableValues())->equality($document)); + static::assertSame(['

a

', '

a

'], (new ComparableValues())->equality($element)); + } + + public function test_xml_elements_of_different_documents_compare_by_their_markup(): void + { + $first = type_xml_element()->cast('

a

')->firstElementChild; + $second = type_xml_element()->cast('

a

')->firstElementChild; + $elements = ColumnMother::of(xml_element_schema('a', nullable: true), [$first, $second, null]); + $lists = ColumnMother::of(list_schema('a', type_list(type_xml_element())), [[$first], [$second]]); + + static::assertTrue((new ComparableValues())->equalByPhysical($elements->type())); + static::assertEquals($elements->values(), (new ComparableValues())->ordering($elements)); + static::assertSame(['

a

', '

a

', null], (new ComparableValues())->equality($elements)); + static::assertSame( + [['

a

'], ['

a

']], + (new ComparableValues())->equalities($lists->type(), $lists->physicals()), + ); + } + + /** + * @param Definition $definition + * @param list $values + */ + #[DataProvider('by_value')] + public function test_other_types_compare_their_values(Definition $definition, array $values): void + { + $column = ColumnMother::of($definition, $values); + + static::assertFalse((new ComparableValues())->orderedByPhysical($column->type())); + static::assertEquals($column->values(), (new ComparableValues())->ordering($column)); + static::assertEquals($column->values(), (new ComparableValues())->equality($column)); + } + + /** + * @param Definition $definition + * @param list $values + */ + #[DataProvider('equal_by_physical')] + public function test_equality_only_types_order_by_value(Definition $definition, array $values): void + { + $column = ColumnMother::of($definition, $values); + + static::assertFalse((new ComparableValues())->orderedByPhysical($column->type())); + static::assertEquals($column->values(), (new ComparableValues())->ordering($column)); + static::assertSame($column->physicals(), (new ComparableValues())->equality($column)); + } + + /** + * @param Definition $definition + * @param list $values + */ + #[DataProvider('ordered')] + public function test_ordered_types_read_physicals(Definition $definition, array $values): void + { + $column = ColumnMother::of($definition, $values); + + static::assertTrue((new ComparableValues())->orderedByPhysical($column->type())); + static::assertTrue((new ComparableValues())->equalByPhysical($column->type())); + static::assertSame($column->physicals(), (new ComparableValues())->ordering($column)); + static::assertSame($column->physicals(), (new ComparableValues())->equality($column)); + } + + public function test_a_date_compares_on_the_datetime_microsecond_scale(): void + { + $column = ColumnMother::of(date_schema('a', nullable: true), [new DateTimeImmutable('2024-01-02'), null]); + + static::assertSame([1_704_153_600_000_000, null], (new ComparableValues())->ordering($column)); + static::assertSame([1_704_153_600_000_000, null], (new ComparableValues())->equality($column)); + static::assertSame( + (new ComparableValues())->equality(ColumnMother::of(datetime_schema('a'), [new DateTimeImmutable( + '2024-01-02 00:00:00 UTC', + )])), + [(new ComparableValues())->equality($column)[0]], + ); + } +} diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Column/DateFormatPatternTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Column/DateFormatPatternTest.php new file mode 100644 index 0000000000..5b90635573 --- /dev/null +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Column/DateFormatPatternTest.php @@ -0,0 +1,48 @@ + $letters + */ + #[TestWith(['Y-m-d\TH:i:s.uP', ['Y', 'm', 'd', 'H', 'i', 's', 'u', 'P']])] + #[TestWith(['\Y\m', []])] + #[TestWith(['', []])] + #[TestWith(['1 2-3', []])] + #[TestWith(['\\\\T', ['T']])] + #[TestWith(['Y\\', ['Y']])] + public function test_letters(string $format, array $letters): void + { + static::assertSame($letters, (new DateFormatPattern($format))->letters()); + } + + /** + * @param list $segments + */ + #[TestWith([ + 'Y-m-d\TH:i:s.uP', + [ + ['Y-m-d\TH:i:s.', 'u'], + ['P', ''], + ], + ])] + #[TestWith(['Y-m-d', [['Y-m-d', '']]])] + #[TestWith(['', []])] + #[TestWith(['u', [['', 'u']]])] + #[TestWith(['uv', [['', 'u'], ['', 'v']]])] + #[TestWith(['H.v', [['H.', 'v']]])] + #[TestWith(['\u \v u', [['\u \v ', 'u']]])] + #[TestWith(['Y\\', [['Y\\', '']]])] + public function test_segments(string $format, array $segments): void + { + static::assertSame($segments, (new DateFormatPattern($format))->segments()); + } +} diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Column/ElementsTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Column/ElementsTest.php new file mode 100644 index 0000000000..58ef75f5cb --- /dev/null +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Column/ElementsTest.php @@ -0,0 +1,45 @@ +flatten([[1, null], null, [], ['a' => 2, 5 => 3], [[4]]]), + ); + static::assertSame([], (new Elements())->flatten([])); + } + + public function test_shape_cuts_a_flat_list_back_into_the_containers(): void + { + static::assertSame( + [['a', 'b'], null, [], ['a' => 'c', 5 => 'd']], + (new Elements())->shape(['a', 'b', 'c', 'd'], [[1, null], null, [], ['a' => 2, 5 => 3]]), + ); + } + + public function test_field_takes_the_element_of_every_structure_that_has_it(): void + { + static::assertSame( + [1, null], + (new Elements())->field([['a' => 1, 'b' => 2], ['b' => 3], null, ['a' => null]], 'a'), + ); + static::assertSame(['x'], (new Elements())->field([[0 => 'x']], 0)); + } + + public function test_with_field_puts_rendered_values_back_where_the_field_was(): void + { + static::assertSame( + [['a' => 'one', 'b' => 2], ['b' => 3], null, ['a' => 'two']], + (new Elements())->withField([['a' => 1, 'b' => 2], ['b' => 3], null, ['a' => null]], 'a', ['one', 'two']), + ); + } +} diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Column/Layout/BooleanLayoutTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Column/Layout/BooleanLayoutTest.php new file mode 100644 index 0000000000..20ab3e713d --- /dev/null +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Column/Layout/BooleanLayoutTest.php @@ -0,0 +1,50 @@ +encode([true, null, true, false]))); + static::assertSame(['ff01'], array_map(bin2hex(...), (new BooleanLayout())->encode(array_fill(0, 9, true)))); + static::assertSame(['ff'], array_map(bin2hex(...), (new BooleanLayout())->encode(array_fill(0, 8, true)))); + } + + public function test_round_trips(): void + { + $layout = new BooleanLayout(); + $values = [true, null, false, true, true, true, true, true, false]; + + static::assertSame($values, $layout->decode(new Buffers($layout->encode($values)), 9, [ + true, + false, + true, + true, + true, + true, + true, + true, + true, + ])); + } + + public function test_refuses_a_short_buffer(): void + { + $this->expectException(InvalidArgumentException::class); + $this->expectExceptionMessage('Boolean values buffer of 1 bytes, expected 2 for 9 rows'); + + (new BooleanLayout())->decode(new Buffers(["\xFF"]), 9, array_fill(0, 9, true)); + } +} diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Column/Layout/BufferLayoutTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Column/Layout/BufferLayoutTest.php new file mode 100644 index 0000000000..e5c7dec134 --- /dev/null +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Column/Layout/BufferLayoutTest.php @@ -0,0 +1,137 @@ +, int, int}> + */ + public static function counts(): Generator + { + yield 'int' => [type_integer(), 1, 2]; + yield 'float' => [type_float(), 1, 2]; + yield 'date' => [type_date(), 1, 2]; + yield 'bool' => [type_boolean(), 1, 2]; + yield 'string' => [type_string(), 1, 3]; + yield 'uuid' => [type_uuid(), 1, 2]; + yield 'null' => [type_null(), 1, 0]; + yield 'optional' => [type_optional(type_string()), 1, 3]; + yield 'list' => [type_list(type_string()), 2, 5]; + yield 'map' => [type_map(type_string(), type_integer()), 4, 8]; + yield 'structure' => [type_structure(['a' => type_integer(), 'b' => type_list(type_null())]), 4, 5]; + } + + /** + * @param Type $type + */ + #[DataProvider('counts')] + public function test_counts_nodes_and_buffers(Type $type, int $nodes, int $buffers): void + { + static::assertSame($nodes, (new BufferLayout())->nodeCount($type)); + static::assertSame($buffers, (new BufferLayout())->bufferCount($type)); + } + + public function test_the_first_node_is_given(): void + { + static::assertSame([[3, 3]], (new BufferLayout())->nodes(type_null(), 3, 3, [])); + static::assertSame([[2, 1]], (new BufferLayout())->nodes(type_integer(), 2, 1, ['', ''])); + } + + public function test_a_list_child_spans_the_last_offset(): void + { + $type = type_list(type_optional(type_integer())); + $column = ColumnMother::of(list_schema('a', $type, nullable: true), [[1, null], null, [2, 3, null]]); + + static::assertSame([[3, 1], [5, 2]], (new BufferLayout())->nodes($column->type(), 3, 1, $column->encode())); + } + + public function test_a_map_entries_node_holds_no_nulls(): void + { + $type = type_map(type_string(), type_optional(type_integer())); + $column = ColumnMother::of(map_schema('a', $type), [['a' => 1, 'b' => null], ['c' => null]]); + + static::assertSame( + [[2, 0], [3, 0], [3, 0], [3, 2]], + (new BufferLayout())->nodes($type, 2, 0, $column->encode()), + ); + } + + public function test_structure_children_span_their_parent(): void + { + $type = type_structure(['a' => type_integer(), 'n' => type_optional(type_null())]); + $column = ColumnMother::of(structure_schema('s', $type, nullable: true), [['a' => 1, 'n' => null], null]); + + static::assertSame( + [[2, 1], [2, 1], [2, 2]], + (new BufferLayout())->nodes($column->type(), 2, 1, $column->encode()), + ); + } + + public function test_refuses_a_buffer_count_other_than_the_layout(): void + { + $this->expectException(InvalidArgumentException::class); + $this->expectExceptionMessage('Column of type integer holds 1 buffers, its layout needs 2'); + + (new BufferLayout())->nodes(type_integer(), 1, 0, ['']); + } + + public function test_refuses_an_offsets_buffer_of_another_length(): void + { + $this->expectException(InvalidArgumentException::class); + $this->expectExceptionMessage('List offsets buffer of 4 bytes, expected 8 for 1 rows'); + + (new BufferLayout())->nodes(type_list(type_integer()), 1, 0, ['', pack('V', 0), '', '']); + } + + public function test_refuses_a_map_entries_validity(): void + { + $this->expectException(InvalidArgumentException::class); + $this->expectExceptionMessage('Map entries validity must be omitted, got 1 bytes'); + + (new BufferLayout())->nodes(type_map(type_string(), type_integer()), 1, 0, [ + '', + pack('V2', 0, 0), + "\x01", + '', + pack('V', 0), + '', + '', + '', + ]); + } + + public function test_refuses_a_short_child_validity(): void + { + $this->expectException(InvalidArgumentException::class); + $this->expectExceptionMessage('Validity bitmap of 1 bytes is too short for 9 rows'); + + (new BufferLayout())->nodes(type_list(type_optional(type_integer())), 1, 0, ['', pack('V2', 0, 9), "\x01", '']); + } +} diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Column/Layout/BuffersTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Column/Layout/BuffersTest.php new file mode 100644 index 0000000000..e361b91e77 --- /dev/null +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Column/Layout/BuffersTest.php @@ -0,0 +1,34 @@ +remaining()); + static::assertSame('a', $buffers->next()); + static::assertSame(1, $buffers->remaining()); + static::assertSame('b', $buffers->next()); + static::assertSame(0, $buffers->remaining()); + } + + public function test_refuses_when_exhausted(): void + { + $buffers = new Buffers(['a']); + $buffers->next(); + + $this->expectException(InvalidArgumentException::class); + $this->expectExceptionMessage('Column buffers exhausted'); + + $buffers->next(); + } +} diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Column/Layout/FixedBinary16LayoutTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Column/Layout/FixedBinary16LayoutTest.php new file mode 100644 index 0000000000..c762de9b2c --- /dev/null +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Column/Layout/FixedBinary16LayoutTest.php @@ -0,0 +1,40 @@ +encode([str_repeat('a', 16), null]), + ); + } + + public function test_round_trips(): void + { + $layout = new FixedBinary16Layout(); + $values = [str_repeat('a', 16), null, str_repeat('b', 16)]; + + static::assertSame($values, $layout->decode(new Buffers($layout->encode($values)), 3, [true, false, true])); + static::assertSame([], $layout->decode(new Buffers(['']), 0, [])); + } + + public function test_refuses_a_wrong_length(): void + { + $this->expectException(InvalidArgumentException::class); + $this->expectExceptionMessage('FixedBinary16 values buffer of 15 bytes, expected 16 for 1 rows'); + + (new FixedBinary16Layout())->decode(new Buffers([str_repeat('a', 15)]), 1, [true]); + } +} diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Column/Layout/Float64LayoutTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Column/Layout/Float64LayoutTest.php new file mode 100644 index 0000000000..8502ac0a52 --- /dev/null +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Column/Layout/Float64LayoutTest.php @@ -0,0 +1,43 @@ +encode([1.5, null])), + ); + } + + public function test_round_trips(): void + { + $layout = new Float64Layout(); + + static::assertSame( + [-0.25, null], + $layout->decode(new Buffers($layout->encode([-0.25, null])), 2, [true, false]), + ); + static::assertSame([], $layout->decode(new Buffers(['']), 0, [])); + } + + public function test_refuses_a_wrong_length(): void + { + $this->expectException(InvalidArgumentException::class); + $this->expectExceptionMessage('Float64 values buffer of 9 bytes, expected 8 for 1 rows'); + + (new Float64Layout())->decode(new Buffers([str_repeat("\0", 9)]), 1, [true]); + } +} diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Column/Layout/Int32LayoutTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Column/Layout/Int32LayoutTest.php new file mode 100644 index 0000000000..4289ba3400 --- /dev/null +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Column/Layout/Int32LayoutTest.php @@ -0,0 +1,47 @@ +encode([-3, null, 1])), + ); + } + + public function test_round_trips_signed_values(): void + { + $layout = new Int32Layout(); + $values = [-2_147_483_648, null, 2_147_483_647, -3, 0]; + + static::assertSame($values, $layout->decode(new Buffers($layout->encode($values)), 5, [ + true, + false, + true, + true, + true, + ])); + static::assertSame([], $layout->decode(new Buffers(['']), 0, [])); + } + + public function test_refuses_a_wrong_length(): void + { + $this->expectException(InvalidArgumentException::class); + $this->expectExceptionMessage('Int32 values buffer of 3 bytes, expected 4 for 1 rows'); + + (new Int32Layout())->decode(new Buffers(["\0\0\0"]), 1, [true]); + } +} diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Column/Layout/Int64LayoutTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Column/Layout/Int64LayoutTest.php new file mode 100644 index 0000000000..f72b88f103 --- /dev/null +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Column/Layout/Int64LayoutTest.php @@ -0,0 +1,47 @@ +encode([1, null, -1])), + ); + } + + public function test_round_trips_the_extremes(): void + { + $layout = new Int64Layout(); + + static::assertSame( + [PHP_INT_MIN, null, PHP_INT_MAX], + $layout->decode(new Buffers($layout->encode([PHP_INT_MIN, null, PHP_INT_MAX])), 3, [true, false, true]), + ); + } + + public function test_decodes_zero_rows(): void + { + static::assertSame([], (new Int64Layout())->decode(new Buffers(['']), 0, [])); + } + + public function test_refuses_a_wrong_length(): void + { + $this->expectException(InvalidArgumentException::class); + $this->expectExceptionMessage('Int64 values buffer of 7 bytes, expected 8 for 1 rows'); + + (new Int64Layout())->decode(new Buffers(["\0\0\0\0\0\0\0"]), 1, [true]); + } +} diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Column/Layout/LayoutForTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Column/Layout/LayoutForTest.php new file mode 100644 index 0000000000..e98384bf2d --- /dev/null +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Column/Layout/LayoutForTest.php @@ -0,0 +1,89 @@ +, class-string}> + */ + public static function types(): Generator + { + yield 'integer' => [type_integer(), Int64Layout::class]; + yield 'positive integer' => [type_positive_integer(), Int64Layout::class]; + yield 'datetime' => [type_datetime(), Int64Layout::class]; + yield 'time' => [type_time(), Int64Layout::class]; + yield 'optional integer' => [type_optional(type_integer()), Int64Layout::class]; + yield 'float' => [type_float(), Float64Layout::class]; + yield 'date' => [type_date(), Int32Layout::class]; + yield 'boolean' => [type_boolean(), BooleanLayout::class]; + yield 'string' => [type_string(), Utf8Layout::class]; + yield 'non-empty string' => [type_non_empty_string(), Utf8Layout::class]; + yield 'numeric string' => [type_numeric_string(), Utf8Layout::class]; + yield 'class string' => [type_class_string(), Utf8Layout::class]; + yield 'timezone' => [type_time_zone(), Utf8Layout::class]; + yield 'json' => [type_json(), Utf8Layout::class]; + yield 'enum' => [type_enum(BasicEnum::class), Utf8Layout::class]; + yield 'xml' => [type_xml(), Utf8Layout::class]; + yield 'xml element' => [type_xml_element(), Utf8Layout::class]; + yield 'html' => [type_html(), Utf8Layout::class]; + yield 'html element' => [type_html_element(), Utf8Layout::class]; + yield 'uuid' => [type_uuid(), FixedBinary16Layout::class]; + } + + /** + * @param Type $type + * @param class-string $class + */ + #[DataProvider('types')] + public function test_maps_the_type_to_its_layout(Type $type, string $class): void + { + static::assertInstanceOf($class, (new LayoutFor())->type($type)); + } + + public function test_a_container_has_no_scalar_layout(): void + { + $this->expectException(InvalidArgumentException::class); + $this->expectExceptionMessage('type list has no scalar layout'); + + (new LayoutFor())->type(type_list(type_integer())); + } +} diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Column/Layout/NullCountTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Column/Layout/NullCountTest.php new file mode 100644 index 0000000000..bb9551bef6 --- /dev/null +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Column/Layout/NullCountTest.php @@ -0,0 +1,17 @@ +of([null, 0, '', null, false])); + static::assertSame(0, (new NullCount())->of([])); + } +} diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Column/Layout/OffsetsTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Column/Layout/OffsetsTest.php new file mode 100644 index 0000000000..c4a1d6cbcf --- /dev/null +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Column/Layout/OffsetsTest.php @@ -0,0 +1,63 @@ +pack([0, 2, 2]))); + } + + public function test_packs_the_last_i32_offset(): void + { + static::assertSame('00000000ffffff7f', bin2hex((new Offsets())->pack([0, 2_147_483_647]))); + } + + public function test_refuses_an_offset_past_the_i32_range(): void + { + $this->expectException(OffsetOverflow::class); + $this->expectExceptionMessage('Offsets exceed the i32 range of a batch buffer: the last offset is 2147483648'); + + (new Offsets())->pack([0, 2_147_483_648]); + } + + public function test_unpacks(): void + { + static::assertSame([0, 2, 2], (new Offsets())->unpack((string) hex2bin('000000000200000002000000'), 2, 'List')); + } + + public function test_unpack_refuses_a_wrong_length(): void + { + $this->expectException(InvalidArgumentException::class); + $this->expectExceptionMessage('List offsets buffer of 8 bytes, expected 12 for 2 rows'); + + (new Offsets())->unpack((string) hex2bin('0000000002000000'), 2, 'List'); + } + + public function test_unpack_refuses_offsets_that_do_not_start_at_0(): void + { + $this->expectException(InvalidArgumentException::class); + $this->expectExceptionMessage('List offsets start at 1, not 0'); + + (new Offsets())->unpack((string) hex2bin('0100000002000000'), 1, 'List'); + } + + public function test_unpack_refuses_offsets_that_decrease(): void + { + $this->expectException(InvalidArgumentException::class); + $this->expectExceptionMessage('List offsets are not monotonic: offset 1 is 3, offset 2 is 2'); + + (new Offsets())->unpack((string) hex2bin('000000000300000002000000'), 2, 'List'); + } +} diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Column/Layout/Utf8LayoutTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Column/Layout/Utf8LayoutTest.php new file mode 100644 index 0000000000..c0675f16bd --- /dev/null +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Column/Layout/Utf8LayoutTest.php @@ -0,0 +1,42 @@ +encode(['ab', null, 'c'])), + ); + } + + public function test_round_trips(): void + { + $layout = new Utf8Layout(); + + static::assertSame( + ['ab', null, '', 'c'], + $layout->decode(new Buffers($layout->encode(['ab', null, '', 'c'])), 4, [true, false, true, true]), + ); + } + + public function test_refuses_a_last_offset_other_than_the_data_length(): void + { + $this->expectException(InvalidArgumentException::class); + $this->expectExceptionMessage('Utf8 data buffer of 3 bytes, the last offset is 2'); + + (new Utf8Layout())->decode(new Buffers(["\0\0\0\0\x02\0\0\0", 'abc']), 1, [true]); + } +} diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Column/Layout/ValidityTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Column/Layout/ValidityTest.php new file mode 100644 index 0000000000..44d7a04291 --- /dev/null +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Column/Layout/ValidityTest.php @@ -0,0 +1,73 @@ +fromValues([1, 2, 3])); + static::assertSame('', (new Validity())->fromValues([])); + static::assertSame('', (new Validity())->fromNulls([], 3)); + } + + #[TestWith([7, '3f'])] + #[TestWith([8, '7f'])] + #[TestWith([9, 'ff00'])] + public function test_the_last_row_null_packs_lsb_first(int $count, string $hex): void + { + $values = array_fill(0, 9, 1); + $values = array_slice($values, 0, $count); + $values[$count - 1] = null; + + static::assertSame($hex, bin2hex((new Validity())->fromValues($values))); + static::assertSame($hex, bin2hex((new Validity())->fromNulls([$count - 1 => true], $count))); + } + + public function test_the_first_row_null(): void + { + static::assertSame('fe01', bin2hex((new Validity())->fromValues([null, 1, 1, 1, 1, 1, 1, 1, 1]))); + static::assertSame('fe01', bin2hex((new Validity())->fromNulls([0 => true], 9))); + } + + public function test_reads_the_valid_flags(): void + { + static::assertSame( + [false, true, true, true, true, true, true, true, true], + (new Validity())->valid("\xFE\x01", 9), + ); + static::assertSame([true, true], (new Validity())->valid('', 2)); + static::assertSame([], (new Validity())->valid('', 0)); + } + + public function test_counts_nulls_ignoring_padding_bits(): void + { + static::assertSame(1, (new Validity())->nullCount("\xFE\x01", 9)); + static::assertSame(0, (new Validity())->nullCount('', 9)); + static::assertSame(7, (new Validity())->nullCount("\x01", 8)); + } + + public function test_the_nulls_of_the_flags(): void + { + static::assertSame([0 => true, 2 => true], (new Validity())->nulls([false, true, false])); + } + + public function test_refuses_a_short_bitmap(): void + { + $this->expectException(InvalidArgumentException::class); + $this->expectExceptionMessage('Validity bitmap of 1 bytes is too short for 9 rows'); + + (new Validity())->valid("\xFF", 9); + } +} diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Column/Php/CastingColumnBuilderTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Column/Php/CastingColumnBuilderTest.php new file mode 100644 index 0000000000..28f7416796 --- /dev/null +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Column/Php/CastingColumnBuilderTest.php @@ -0,0 +1,169 @@ +builder(int_schema('a', nullable: true)); + $builder->append('1'); + $builder->appendMany(['2', null]); + + static::assertSame(3, $builder->count()); + static::assertSame([1, 2, null], $builder->finish()->physicals()); + } + + public function test_append_many_is_atomic_on_a_refusal(): void + { + $builder = (new PhpBackend())->builder(int_schema('a')); + + try { + $builder->appendMany(['1', 'x']); + } catch (SchemaMismatchException) { + } + + static::assertSame(0, $builder->count()); + } + + public function test_append_many_refuses_null_under_not_null(): void + { + $this->expectException(SchemaMismatchException::class); + $this->expectExceptionMessage('column "a" (row 1): could not convert null to integer, column is not nullable'); + + (new PhpBackend()) + ->builder(int_schema('a')) + ->appendMany([1, null]); + } + + public function test_the_null_definition_accepts_null(): void + { + $builder = (new PhpBackend())->builder(null_schema('a')); + $builder->append(null); + $builder->appendMany([null]); + + static::assertSame(2, $builder->count()); + } + + public function test_append_from_copies_a_physical_cell(): void + { + $source = ColumnMother::of(int_schema('a', nullable: true), [1, null, 3]); + $builder = (new PhpBackend())->builder(int_schema('a', nullable: true)); + $builder->appendFrom($source, 2); + $builder->appendFrom($source, 1); + $builder->appendTake($source, [0, 1, 2]); + + static::assertSame([3, null, 1, null, 3], $builder->finish()->physicals()); + } + + public function test_append_from_refuses_null_under_not_null(): void + { + $this->expectException(ColumnMismatchException::class); + $this->expectExceptionMessage('could not convert null to integer, column is not nullable'); + + (new PhpBackend()) + ->builder(int_schema('a')) + ->appendFrom(ColumnMother::of(int_schema('a', nullable: true), [null]), 0); + } + + public function test_append_take_refuses_null_under_not_null(): void + { + $this->expectException(ColumnMismatchException::class); + $this->expectExceptionMessage('could not convert null to integer, column is not nullable'); + + (new PhpBackend()) + ->builder(int_schema('a')) + ->appendTake(ColumnMother::of(int_schema('a', nullable: true), [1, null]), [0, 1]); + } + + #[TestWith([2])] + #[TestWith([null])] + public function test_append_physicals_takes_physicals_with_nulls(?int $nullCount): void + { + $builder = (new PhpBackend())->builder(int_schema('id', nullable: true)); + $builder->appendPhysicals([1, null, 3, null], $nullCount); + $column = $builder->finish(); + + static::assertSame([1, null, 3, null], $column->values()); + static::assertSame(2, $column->nullCount()); + } + + public function test_append_physicals_refuses_null_under_not_null_at_its_position(): void + { + $builder = (new PhpBackend())->builder(int_schema('id')); + + try { + $builder->appendPhysicals([1, 2, null]); + static::fail('a null under NOT NULL must be refused'); + } catch (SchemaMismatchException $e) { + static::assertSame(2, $e->rowIndex); + } + + static::assertSame(0, $builder->count()); + } + + public function test_append_physicals_of_nothing_appends_nothing(): void + { + $builder = (new PhpBackend())->builder(int_schema('id')); + $builder->appendPhysicals([], 0); + + static::assertSame(0, $builder->finish()->count()); + } + + #[TestWith(['integer', '1'])] + #[TestWith(['float', 1])] + #[TestWith(['datetime', 'not micros'])] + #[TestWith(['uuid', 'too short'])] + #[TestWith(['list', 'not an array'])] + #[TestWith(['list_of_int', ['1']])] + public function test_append_physicals_refuses_a_physical_of_another_kind_and_appends_nothing( + string $definition, + mixed $physical, + ): void { + $builder = (new PhpBackend())->builder(match ($definition) { + 'integer' => int_schema('c'), + 'float' => float_schema('c'), + 'datetime' => datetime_schema('c'), + 'uuid' => uuid_schema('c'), + 'list' => list_schema('c', type_list(type_string())), + default => list_schema('c', type_list(type_integer())), + }); + + try { + $builder->appendPhysicals([$physical]); + static::fail('a physical of another kind must be refused'); + } catch (InvalidArgumentException $e) { + static::assertStringContainsString('physical in a', $e->getMessage()); + } + + static::assertSame(0, $builder->count()); + } + + public function test_append_physicals_stores_any_physical_of_a_null_column_as_null(): void + { + $builder = (new PhpBackend())->builder(null_schema('c')); + $builder->appendPhysicals([null]); + + static::assertSame([null], $builder->finish()->physicals()); + } +} diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Column/Php/ColumnDecoderTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Column/Php/ColumnDecoderTest.php new file mode 100644 index 0000000000..05e62194b9 --- /dev/null +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Column/Php/ColumnDecoderTest.php @@ -0,0 +1,136 @@ +decode($type, new Buffers($column->encode()), 2, 0); + + static::assertSame([[1, null], [null]], $decoded->values()); + } + + public function test_a_null_kind_consumes_no_buffer(): void + { + $type = type_structure(['n' => type_optional(type_null()), 'a' => type_integer()]); + $column = ColumnMother::of(structure_schema('s', $type), [['n' => null, 'a' => 1]]); + $buffers = new Buffers($column->encode()); + $decoded = (new ColumnDecoder())->decode($type, $buffers, 1, 0); + + static::assertSame(0, $buffers->remaining()); + static::assertSame([['n' => null, 'a' => 1]], $decoded->values()); + } + + public function test_a_top_level_null_column(): void + { + static::assertInstanceOf(ConstantColumn::class, (new ColumnDecoder())->decode( + type_null(), + new Buffers([]), + 3, + 3, + )); + } + + public function test_refuses_a_null_count_other_than_the_bitmap(): void + { + $this->expectException(InvalidArgumentException::class); + $this->expectExceptionMessage('Column null count 1 disagrees with its validity bitmap (0 nulls in 1 rows)'); + + (new ColumnDecoder())->decode(type_integer(), new Buffers(['', "\0\0\0\0\0\0\0\0"]), 1, 1); + } + + public function test_refuses_nulls_in_a_non_optional_list_element(): void + { + $this->expectException(InvalidArgumentException::class); + $this->expectExceptionMessage('List child holds 1 nulls in a non-nullable integer'); + + (new ColumnDecoder())->decode( + type_list(type_integer()), + new Buffers(['', pack('V2', 0, 1), "\x00", "\0\0\0\0\0\0\0\0"]), + 1, + 0, + ); + } + + public function test_refuses_nulls_in_a_map_key(): void + { + $this->expectException(InvalidArgumentException::class); + $this->expectExceptionMessage('Map key child holds 1 nulls in a non-nullable string'); + + (new ColumnDecoder())->decode( + type_map(type_string(), type_integer()), + new Buffers(['', pack('V2', 0, 1), '', "\x00", pack('V2', 0, 0), '', '', "\0\0\0\0\0\0\0\0"]), + 1, + 0, + ); + } + + public function test_refuses_nulls_in_a_non_optional_map_value(): void + { + $this->expectException(InvalidArgumentException::class); + $this->expectExceptionMessage('Map value child holds 1 nulls in a non-nullable integer'); + + (new ColumnDecoder())->decode( + type_map(type_string(), type_integer()), + new Buffers(['', pack('V2', 0, 1), '', '', pack('V2', 0, 1), 'k', "\x00", "\0\0\0\0\0\0\0\0"]), + 1, + 0, + ); + } + + public function test_null_kind_children_hold_nulls(): void + { + $list = type_list(type_null()); + $map = type_map(type_string(), type_null()); + + static::assertSame( + [[null, null]], + (new ColumnDecoder()) + ->decode($list, new Buffers(ColumnMother::of(list_schema('a', $list), [[null, null]])->encode()), 1, 0) + ->values(), + ); + static::assertSame( + [['k' => null]], + (new ColumnDecoder()) + ->decode($map, new Buffers(ColumnMother::of(map_schema('a', $map), [['k' => null]])->encode()), 1, 0) + ->values(), + ); + } + + public function test_a_structure_child_under_a_null_parent_holds_nulls(): void + { + $type = type_structure(['a' => type_integer()]); + $column = ColumnMother::of(structure_schema('s', $type, nullable: true), [['a' => 1], null]); + + static::assertSame( + [['a' => 1], null], + (new ColumnDecoder()) + ->decode($column->type(), new Buffers($column->encode()), 2, 1) + ->values(), + ); + } +} diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Column/Php/ConstantColumnTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Column/Php/ConstantColumnTest.php new file mode 100644 index 0000000000..4dc098ded7 --- /dev/null +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Column/Php/ConstantColumnTest.php @@ -0,0 +1,126 @@ +count()); + static::assertSame('UTC', $column->at(2)); + static::assertSame(['UTC', 'UTC', 'UTC'], $column->physicals()); + static::assertEquals(new DateTimeZone('UTC'), $column->value(1)); + static::assertEquals( + [new DateTimeZone('UTC'), new DateTimeZone('UTC'), new DateTimeZone('UTC')], + $column->values(), + ); + static::assertFalse($column->isNull(0)); + static::assertSame(0, $column->nullCount()); + static::assertSame(2, $column->slice(1, 2)->count()); + static::assertSame(4, $column->take([0, 0, 1, 2])->count()); + static::assertEquals(type_time_zone(), $column->type()); + } + + public function test_zero_rows(): void + { + $column = new ConstantColumn(type_integer(), new IdentityPhysical(), 1, 0); + + static::assertSame([], $column->physicals()); + static::assertSame([], $column->values()); + } + + public function test_null(): void + { + $column = new ConstantColumn(type_optional(type_integer()), new NullPhysical(), null, 2); + + static::assertTrue($column->isNull(0)); + static::assertSame(2, $column->nullCount()); + static::assertNull($column->value(0)); + static::assertSame(['00', str_repeat('0', 32)], array_map(bin2hex(...), $column->encode())); + } + + public function test_expands_to_the_column_kind(): void + { + $column = new ConstantColumn(type_list(type_integer()), new IdentityPhysical(), [1, 2], 2); + $expanded = $column->expand(); + + static::assertInstanceOf(ListColumn::class, $expanded); + static::assertSame([[1, 2], [1, 2]], $expanded->physicals()); + } + + public function test_the_null_kind_does_not_expand_and_has_no_buffers(): void + { + foreach ([type_null(), type_optional(type_null())] as $type) { + $column = new ConstantColumn($type, new NullPhysical(), null, 2); + + static::assertTrue($column->isNullKind()); + static::assertSame($column, $column->expand()); + static::assertSame([], $column->encode()); + } + } + + public function test_concat_with_a_scalar_column_expands(): void + { + $constant = new ConstantColumn(type_integer(), new IdentityPhysical(), 7, 1); + + static::assertSame([7, 1, 2], $constant->concat(ColumnMother::of(int_schema('a'), [1, 2]))->physicals()); + static::assertSame( + [7, 8, 8], + $constant->concat(new ConstantColumn(type_integer(), new IdentityPhysical(), 8, 2))->physicals(), + ); + } + + public function test_with_type_restamps_the_value(): void + { + $column = (new PhpBackend())->constant( + datetime_schema('a'), + new DateTimeImmutable('2024-01-02 03:04:05 UTC'), + 2, + ); + $warsaw = type_instance_of(DateTimeImmutable::class)->assert( + $column->withType(type_datetime('Europe/Warsaw'))->value(1), + ); + + static::assertSame('2024-01-02 04:04:05 Europe/Warsaw', $warsaw->format('Y-m-d H:i:s e')); + } + + public function test_with_type_refuses_another_column_kind(): void + { + $this->expectException(InvalidArgumentException::class); + $this->expectExceptionMessage('integer and string are different column kinds'); + + (new PhpBackend()) + ->constant(int_schema('a'), 1, 2) + ->withType(type_string()); + } +} diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Column/Php/ListColumnBuilderTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Column/Php/ListColumnBuilderTest.php new file mode 100644 index 0000000000..edfc094938 --- /dev/null +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Column/Php/ListColumnBuilderTest.php @@ -0,0 +1,58 @@ +appendPhysical([5 => 1, 9 => 2]); + $builder->appendPhysicals([null, [], [3]]); + + static::assertSame(4, $builder->count()); + + $column = $builder->finish(); + + static::assertInstanceOf(ListColumn::class, $column); + static::assertSame([0, 2, 2, 2, 3], $column->offsets); + static::assertSame([1 => true], $column->nulls); + static::assertSame([[1, 2], null, [], [3]], $column->physicals()); + } + + public function test_a_bulk_append_builds_the_column_row_by_row_appends_build(): void + { + $physicals = [[[1, 2], []], null, [], [null, [3]], [5 => [4]]]; + $builder = static fn(): ListColumnBuilder => new ListColumnBuilder( + type_list(type_list(type_integer())), + new ListColumnBuilder( + type_list(type_integer()), + new ScalarColumnBuilder(type_integer(), new IdentityPhysical()), + ), + ); + $bulk = $builder(); + $bulk->appendPhysicals($physicals); + $rowByRow = $builder(); + + foreach ($physicals as $physical) { + $rowByRow->appendPhysical($physical); + } + + static::assertEquals($rowByRow->finish(), $bulk->finish()); + static::assertSame([[[1, 2], []], null, [], [null, [3]], [[4]]], $bulk->finish()->physicals()); + } +} diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Column/Php/ListColumnTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Column/Php/ListColumnTest.php new file mode 100644 index 0000000000..2a80ce5027 --- /dev/null +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Column/Php/ListColumnTest.php @@ -0,0 +1,80 @@ +count()); + static::assertSame(1, $column->nullCount()); + static::assertSame(['UTC', 'Europe/Warsaw'], $column->at(3)); + static::assertNull($column->at(1)); + static::assertSame([], $column->at(2)); + static::assertSame([['UTC'], null, [], ['UTC', 'Europe/Warsaw']], $column->physicals()); + static::assertEquals([new DateTimeZone('UTC'), new DateTimeZone('Europe/Warsaw')], $column->value(3)); + static::assertNull($column->value(1)); + static::assertEquals( + [[new DateTimeZone('UTC')], null, [], [new DateTimeZone('UTC'), new DateTimeZone('Europe/Warsaw')]], + $column->values(), + ); + static::assertTrue($column->isNull(1)); + static::assertFalse($column->isNull(2)); + static::assertEquals(type_list(type_time_zone()), $column->type()); + } + + public function test_reads_cells_over_a_constant_element(): void + { + $column = new ListColumn( + type_list(type_time_zone()), + [0, 2, 2, 3], + (new PhpBackend())->constant(definition_from_type('element', type_time_zone()), 'UTC', 3), + [1 => true], + ); + + static::assertSame(['UTC', 'UTC'], $column->at(0)); + static::assertNull($column->at(1)); + static::assertSame(['UTC'], $column->at(2)); + static::assertEquals([new DateTimeZone('UTC'), new DateTimeZone('UTC')], $column->value(0)); + static::assertNull($column->value(1)); + static::assertEquals([new DateTimeZone('UTC')], $column->value(2)); + } + + public function test_slice_and_take_rebuild_offsets_from_zero(): void + { + $column = ColumnMother::of(list_schema('a', type_list(type_time_zone()), nullable: true), [ + ['UTC'], + null, + ['Europe/Warsaw', 'UTC'], + ]); + $taken = $column->take([2, 1, 0]); + + static::assertInstanceOf(ListColumn::class, $taken); + static::assertSame([0, 2, 2, 3], $taken->offsets); + static::assertSame([1 => true], $taken->nulls); + static::assertSame([['Europe/Warsaw', 'UTC'], null, ['UTC']], $taken->physicals()); + static::assertSame([null, ['Europe/Warsaw', 'UTC']], $column->slice(1, 2)->physicals()); + static::assertSame(0, $column->slice(0, 0)->count()); + } +} diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Column/Php/MapColumnBuilderTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Column/Php/MapColumnBuilderTest.php new file mode 100644 index 0000000000..4735093aca --- /dev/null +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Column/Php/MapColumnBuilderTest.php @@ -0,0 +1,39 @@ +appendPhysical(['a' => 1, 'b' => 2]); + $builder->appendPhysicals([null, ['c' => 3]]); + + static::assertSame(3, $builder->count()); + + $column = $builder->finish(); + + static::assertInstanceOf(MapColumn::class, $column); + static::assertSame([0, 2, 2, 3], $column->offsets); + static::assertSame([1 => true], $column->nulls); + static::assertSame(['a', 'b', 'c'], $column->keys->physicals()); + static::assertSame([1, 2, 3], $column->values->physicals()); + } +} diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Column/Php/MapColumnTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Column/Php/MapColumnTest.php new file mode 100644 index 0000000000..65e0b224f8 --- /dev/null +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Column/Php/MapColumnTest.php @@ -0,0 +1,91 @@ + 'UTC'], + null, + [], + ['y' => 'UTC', 'z' => 'Europe/Warsaw'], + ]); + + static::assertInstanceOf(MapColumn::class, $column); + static::assertSame(4, $column->count()); + static::assertSame(1, $column->nullCount()); + static::assertSame(['y' => 'UTC', 'z' => 'Europe/Warsaw'], $column->at(3)); + static::assertNull($column->at(1)); + static::assertSame([], $column->at(2)); + static::assertSame([['x' => 'UTC'], null, [], ['y' => 'UTC', 'z' => 'Europe/Warsaw']], $column->physicals()); + static::assertEquals( + ['y' => new DateTimeZone('UTC'), 'z' => new DateTimeZone('Europe/Warsaw')], + $column->value(3), + ); + static::assertNull($column->value(1)); + static::assertEquals( + [ + ['x' => new DateTimeZone('UTC')], + null, + [], + ['y' => new DateTimeZone('UTC'), 'z' => new DateTimeZone('Europe/Warsaw')], + ], + $column->values(), + ); + static::assertTrue($column->isNull(1)); + static::assertEquals(type_map(type_string(), type_time_zone()), $column->type()); + } + + public function test_reads_cells_over_constant_keys_and_values(): void + { + $backend = new PhpBackend(); + $keys = $backend->builder(definition_from_type('key', type_string())); + $keys->appendMany(['x', 'y']); + $column = new MapColumn( + type_map(type_string(), type_time_zone()), + [0, 1, 1, 2], + $keys->finish(), + $backend->constant(definition_from_type('value', type_time_zone()), 'UTC', 2), + [1 => true], + ); + + static::assertSame(['x' => 'UTC'], $column->at(0)); + static::assertNull($column->at(1)); + static::assertSame(['y' => 'UTC'], $column->at(2)); + static::assertEquals(['x' => new DateTimeZone('UTC')], $column->value(0)); + static::assertNull($column->value(1)); + static::assertEquals(['y' => new DateTimeZone('UTC')], $column->value(2)); + } + + public function test_slice_and_take_rebuild_offsets_from_zero(): void + { + $column = ColumnMother::of(map_schema('a', type_map(type_string(), type_string()), nullable: true), [ + ['a' => 'x'], + null, + ['b' => 'y', 'c' => 'z'], + ]); + $taken = $column->take([2, 1, 0]); + + static::assertInstanceOf(MapColumn::class, $taken); + static::assertSame([0, 2, 2, 3], $taken->offsets); + static::assertSame([['b' => 'y', 'c' => 'z'], null, ['a' => 'x']], $taken->values()); + static::assertSame([null, ['b' => 'y', 'c' => 'z']], $column->slice(1, 2)->values()); + static::assertSame(0, $column->slice(0, 0)->count()); + } +} diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Column/Php/NullColumnBuilderTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Column/Php/NullColumnBuilderTest.php new file mode 100644 index 0000000000..fb7b43d519 --- /dev/null +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Column/Php/NullColumnBuilderTest.php @@ -0,0 +1,28 @@ +appendPhysical(null); + $builder->appendPhysicals([null, null]); + + static::assertSame(3, $builder->count()); + + $column = $builder->finish(); + + static::assertInstanceOf(ConstantColumn::class, $column); + static::assertSame(3, $column->nullCount()); + } +} diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Column/Php/ScalarColumnBuilderTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Column/Php/ScalarColumnBuilderTest.php new file mode 100644 index 0000000000..44fdcddc95 --- /dev/null +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Column/Php/ScalarColumnBuilderTest.php @@ -0,0 +1,53 @@ +appendPhysical(1); + $builder->appendPhysical(null); + $builder->appendPhysicals([null, 4]); + + static::assertSame(4, $builder->count()); + + $column = $builder->finish(); + + static::assertSame([1, null, null, 4], $column->physicals()); + static::assertSame(2, $column->nullCount()); + } + + public function test_takes_the_null_count_the_caller_already_counted(): void + { + $builder = new ScalarColumnBuilder(type_optional(type_integer()), new IdentityPhysical()); + $builder->appendPhysicals([1, null], 1); + $builder->appendPhysicals([null, null, 5], 2); + + $column = $builder->finish(); + + static::assertSame([1, null, null, null, 5], $column->physicals()); + static::assertSame(3, $column->nullCount()); + } + + public function test_a_bulk_append_counts_only_nulls(): void + { + $builder = new ScalarColumnBuilder(type_optional(type_integer()), new IdentityPhysical()); + $builder->appendPhysicals([0, '', false, null, '0']); + + $column = $builder->finish(); + + static::assertSame([0, '', false, null, '0'], $column->physicals()); + static::assertSame(1, $column->nullCount()); + } +} diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Column/Php/ScalarColumnTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Column/Php/ScalarColumnTest.php new file mode 100644 index 0000000000..3d33417579 --- /dev/null +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Column/Php/ScalarColumnTest.php @@ -0,0 +1,55 @@ +count()); + static::assertSame('UTC', $column->at(0)); + static::assertNull($column->at(1)); + static::assertSame(['UTC', null], $column->physicals()); + static::assertEquals(new DateTimeZone('UTC'), $column->value(0)); + static::assertNull($column->value(1)); + static::assertTrue($column->isNull(1)); + static::assertFalse($column->isNull(0)); + static::assertSame(1, $column->nullCount()); + static::assertEquals(type_time_zone(), $column->type()); + } + + public function test_slice_and_take_recount_nulls(): void + { + $column = ColumnMother::of(int_schema('a', nullable: true), [1, null, 3, null]); + + static::assertSame([null, 3], $column->slice(1, 2)->physicals()); + static::assertSame(1, $column->slice(1, 2)->nullCount()); + static::assertSame(0, $column->slice(2, 1)->nullCount()); + static::assertSame([3, 1, 3], $column->take([2, 0, 2])->physicals()); + static::assertSame(0, $column->take([2, 0, 2])->nullCount()); + static::assertSame(2, $column->take([1, 3, 0])->nullCount()); + } + + public function test_a_column_without_nulls_slices_without_counting(): void + { + $column = new ScalarColumn(type_time_zone(), new TimeZonePhysical(), ['UTC', 'UTC'], 0); + + static::assertSame(0, $column->slice(0, 1)->nullCount()); + static::assertSame(0, $column->take([1])->nullCount()); + } +} diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Column/Php/StructColumnBuilderTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Column/Php/StructColumnBuilderTest.php new file mode 100644 index 0000000000..5bb7d4e1ea --- /dev/null +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Column/Php/StructColumnBuilderTest.php @@ -0,0 +1,72 @@ + type_integer(), 'b' => type_integer()]), [ + 'a' => new ScalarColumnBuilder(type_integer(), new IdentityPhysical()), + 'b' => new ScalarColumnBuilder(type_integer(), new IdentityPhysical()), + ]); + $builder->appendPhysical(['b' => 2, 'a' => 1]); + $builder->appendPhysicals([null, ['a' => 3]]); + + static::assertSame(3, $builder->count()); + + $column = $builder->finish(); + + static::assertInstanceOf(StructColumn::class, $column); + static::assertSame([1 => true], $column->nulls); + static::assertSame([1, null, 3], $column->children['a']->physicals()); + static::assertSame([2, null, null], $column->children['b']->physicals()); + } + + public function test_a_bulk_append_builds_the_column_row_by_row_appends_build(): void + { + $physicals = [['a' => 1, 'b' => [1, 2]], null, ['b' => []], ['a' => null, 'b' => null], ['b' => [3], 'a' => 4]]; + $builder = static fn(): StructColumnBuilder => new StructColumnBuilder( + type_structure(['a' => type_integer(), 'b' => type_list(type_integer())]), + [ + 'a' => new ScalarColumnBuilder(type_integer(), new IdentityPhysical()), + 'b' => new ListColumnBuilder( + type_list(type_integer()), + new ScalarColumnBuilder(type_integer(), new IdentityPhysical()), + ), + ], + ); + $bulk = $builder(); + $bulk->appendPhysicals($physicals); + $rowByRow = $builder(); + + foreach ($physicals as $physical) { + $rowByRow->appendPhysical($physical); + } + + static::assertEquals($rowByRow->finish(), $bulk->finish()); + static::assertSame( + [ + ['a' => 1, 'b' => [1, 2]], + null, + ['a' => null, 'b' => []], + ['a' => null, 'b' => null], + ['a' => 4, 'b' => [3]], + ], + $bulk->finish()->physicals(), + ); + } +} diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Column/Php/StructColumnTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Column/Php/StructColumnTest.php new file mode 100644 index 0000000000..2249845e5d --- /dev/null +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Column/Php/StructColumnTest.php @@ -0,0 +1,71 @@ + type_integer(), + 'zone' => structure_element('zone', type_optional(type_time_zone()), optional: true), + 'tag' => type_optional(type_integer()), + ]); + $column = ColumnMother::of(structure_schema('a', $type, nullable: true), [ + ['id' => 1, 'zone' => 'UTC', 'tag' => 2], + null, + ['id' => 3, 'tag' => null], + ]); + + static::assertInstanceOf(StructColumn::class, $column); + static::assertSame(3, $column->count()); + static::assertSame(1, $column->nullCount()); + static::assertTrue($column->isNull(1)); + static::assertSame(['id' => 1, 'zone' => 'UTC', 'tag' => 2], $column->at(0)); + static::assertNull($column->at(1)); + static::assertSame(['id' => 3, 'tag' => null], $column->at(2)); + static::assertSame( + [['id' => 1, 'zone' => 'UTC', 'tag' => 2], null, ['id' => 3, 'tag' => null]], + $column->physicals(), + ); + static::assertEquals(['id' => 1, 'zone' => new DateTimeZone('UTC'), 'tag' => 2], $column->value(0)); + static::assertNull($column->value(1)); + static::assertSame(['id' => 3, 'tag' => null], $column->value(2)); + static::assertEquals( + [['id' => 1, 'zone' => new DateTimeZone('UTC'), 'tag' => 2], null, ['id' => 3, 'tag' => null]], + $column->values(), + ); + static::assertSame($type, $column->structure()); + static::assertSame($type, $column->type()); + } + + public function test_slice_and_take_rekey_nulls(): void + { + $column = ColumnMother::of(structure_schema('a', type_structure(['id' => type_integer()]), nullable: true), [ + ['id' => 1], + null, + ['id' => 3], + ]); + $taken = $column->take([1, 2]); + + static::assertInstanceOf(StructColumn::class, $taken); + static::assertSame([0 => true], $taken->nulls); + static::assertSame([null, ['id' => 3]], $taken->values()); + static::assertSame([null, ['id' => 3]], $column->slice(1, 2)->values()); + static::assertSame(0, $column->slice(0, 0)->count()); + } +} diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Column/PhpBackendTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Column/PhpBackendTest.php new file mode 100644 index 0000000000..921c386c9f --- /dev/null +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Column/PhpBackendTest.php @@ -0,0 +1,628 @@ +, mixed, mixed}> + */ + public static function definitions(): Generator + { + $structure = type_structure([ + 'id' => type_integer(), + 'note' => structure_element('note', type_optional(type_string()), optional: true), + 'tag' => type_optional(type_string()), + ]); + + yield 'integer' => [int_schema('a'), '5', 5]; + yield 'float' => [float_schema('a'), 1.5, 1.5]; + yield 'boolean' => [bool_schema('a'), true, true]; + yield 'string' => [str_schema('a'), 'text', 'text']; + yield 'datetime' => [ + datetime_schema('a'), + '2024-01-02 03:04:05.123456', + new DateTimeImmutable('2024-01-02 03:04:05.123456 UTC'), + ]; + yield 'datetime in the column zone' => [ + datetime_schema('a', zone: 'Europe/Warsaw'), + new DateTimeImmutable('2024-01-02 03:04:05.5 UTC'), + new DateTimeImmutable('2024-01-02 04:04:05.5', new DateTimeZone('Europe/Warsaw')), + ]; + yield 'datetime one microsecond before the epoch' => [ + datetime_schema('a'), + new DateTimeImmutable('@-0.000001'), + new DateTimeImmutable('1969-12-31 23:59:59.999999 UTC'), + ]; + yield 'datetime one and a half seconds before the epoch' => [ + datetime_schema('a'), + new DateTimeImmutable('@-1.500000'), + new DateTimeImmutable('1969-12-31 23:59:58.5 UTC'), + ]; + yield 'date east of UTC keeps its calendar day' => [ + date_schema('a'), + new DateTimeImmutable('2024-03-05 00:30:00', new DateTimeZone('Asia/Tokyo')), + new DateTimeImmutable('2024-03-05 00:00:00 UTC'), + ]; + yield 'date before the epoch' => [ + date_schema('a'), + new DateTimeImmutable('1969-12-29 00:00:00 UTC'), + new DateTimeImmutable('1969-12-29 00:00:00 UTC'), + ]; + yield 'time folds days into hours' => [ + time_schema('a'), + DateIntervalMother::of('P1DT2H3M4S', 0.25, 1), + DateIntervalMother::of('PT26H3M4S', 0.25, 1), + ]; + yield 'uuid' => [ + uuid_schema('a'), + '6c2f1d4e-8b3a-4c5d-9e6f-0a1b2c3d4e5f', + new Uuid('6c2f1d4e-8b3a-4c5d-9e6f-0a1b2c3d4e5f'), + ]; + yield 'timezone' => [time_zone_schema('a'), 'Europe/Warsaw', new DateTimeZone('Europe/Warsaw')]; + yield 'json object' => [json_schema('a'), '{"a":1}', Json::fromString('{"a":1}')]; + yield 'json list' => [json_schema('a'), [1, 2], Json::fromString('[1,2]')]; + yield 'enum' => [enum_schema('a', BackedStringEnum::class), 'one', BackedStringEnum::one]; + yield 'xml' => [xml_schema('a'), '1', XmlMother::document('1')]; + yield 'xml element' => [ + xml_element_schema('a'), + XmlMother::document('1')->documentElement, + XmlMother::document('1')->documentElement, + ]; + yield 'list with a null element' => [ + list_schema('a', type_list(type_optional(type_integer()))), + [1, null, 3], + [1, null, 3], + ]; + yield 'list of datetimes in the column zone' => [ + list_schema('a', type_list(type_datetime('Europe/Warsaw'))), + [new DateTimeImmutable('2024-01-02 03:04:05 UTC')], + [new DateTimeImmutable('2024-01-02 04:04:05', new DateTimeZone('Europe/Warsaw'))], + ]; + yield 'map with a null value' => [ + map_schema('a', type_map(type_string(), type_optional(type_integer()))), + ['x' => 1, 'y' => null], + ['x' => 1, 'y' => null], + ]; + yield 'structure with every element' => [ + structure_schema('a', $structure), + ['id' => 1, 'note' => 'x', 'tag' => 'y'], + ['id' => 1, 'note' => 'x', 'tag' => 'y'], + ]; + yield 'structure: an optional nullable element given null reads back absent' => [ + structure_schema('a', $structure), + ['id' => 1, 'note' => null, 'tag' => null], + ['id' => 1, 'tag' => null], + ]; + yield 'structure: an absent optional element stays absent' => [ + structure_schema('a', $structure), + ['id' => 1, 'tag' => 'y'], + ['id' => 1, 'tag' => 'y'], + ]; + } + + /** + * @return Generator}> + */ + public static function not_null_definitions(): Generator + { + foreach (self::definitions() as $name => [$definition]) { + yield $name => [$definition]; + } + } + + /** + * @return Generator, string}> + */ + public static function refusals(): Generator + { + yield 'a definition outside the nineteen' => [ + new ForeignTypeDefinition('a', type_integer()), + 'Row does not match its schema: column "a": integer cannot be a batch column, only the 19 Flow definitions have a column kind', + ]; + yield 'a mixed element' => [ + list_schema('a', type_list(type_mixed())), + 'Row does not match its schema: column "a": list cannot be a batch column, a mixed element has no column kind, declare it or use json', + ]; + yield 'a wildcard enum' => [ + enum_schema('a', UnitEnum::class), + 'Row does not match its schema: column "a": enum cannot be a batch column, declare the concrete enum class', + ]; + yield 'a type without a column kind' => [ + list_schema('a', type_list(type_instance_of(stdClass::class))), + 'Row does not match its schema: column "a": list> cannot be a batch column, type object has no column kind', + ]; + } + + /** + * @return Generator, mixed, bool}> + */ + public static function temporal_range(): Generator + { + yield 'datetime at the upper bound' => [ + datetime_schema('a'), + new DateTimeImmutable('@9223372036854.775807'), + true, + ]; + yield 'datetime past the upper bound' => [ + datetime_schema('a'), + new DateTimeImmutable('@9223372036854.775808'), + false, + ]; + yield 'datetime at the lower bound' => [datetime_schema('a'), new DateTimeImmutable('@-9223372036854'), true]; + yield 'datetime past the lower bound' => [ + datetime_schema('a'), + new DateTimeImmutable('@-9223372036855'), + false, + ]; + yield 'date at the upper bound' => [ + date_schema('a'), + new DateTimeImmutable('@' . ((2 ** 31 - 1) * 86_400)), + true, + ]; + yield 'date past the upper bound' => [ + date_schema('a'), + new DateTimeImmutable('@' . ((2 ** 31) * 86_400)), + false, + ]; + yield 'date at the lower bound' => [date_schema('a'), new DateTimeImmutable('@' . (-2 ** 31 * 86_400)), true]; + yield 'date past the lower bound' => [ + date_schema('a'), + new DateTimeImmutable('@' . ((-2 ** 31 - 1) * 86_400)), + false, + ]; + yield 'time at the upper bound' => [ + time_schema('a'), + DateIntervalMother::seconds(9_223_372_036_854, 0.775807, 0), + true, + ]; + yield 'time past the upper bound' => [ + time_schema('a'), + DateIntervalMother::seconds(9_223_372_036_854, 0.775808, 0), + false, + ]; + yield 'negative time at the bound' => [ + time_schema('a'), + DateIntervalMother::seconds(9_223_372_036_854, 0.775807, 1), + true, + ]; + yield 'negative time past the bound' => [ + time_schema('a'), + DateIntervalMother::seconds(9_223_372_036_854, 0.775808, 1), + false, + ]; + yield 'time with a negative component past the bound' => [ + time_schema('a'), + DateInterval::createFromDateString('-9223372036860 seconds'), + false, + ]; + } + + /** + * @param Definition $definition + */ + #[DataProvider('definitions')] + public function test_round_trips(Definition $definition, mixed $value, mixed $expected): void + { + static::assertEquals($expected, ColumnMother::of($definition, [$value])->value(0)); + static::assertEquals([$expected], ColumnMother::of($definition, [$value])->values()); + } + + /** + * @param Definition $definition + */ + #[DataProvider('definitions')] + public function test_round_trips_null_under_a_nullable_definition( + Definition $definition, + mixed $value, + mixed $expected, + ): void { + $column = ColumnMother::of($definition->makeNullable(), [$value, null]); + + static::assertEquals([$expected, null], $column->values()); + static::assertSame(1, $column->nullCount()); + static::assertTrue($column->isNull(1)); + static::assertFalse($column->isNull(0)); + } + + /** + * @param Definition $definition + */ + #[DataProvider('not_null_definitions')] + public function test_refuses_null_under_a_not_null_definition(Definition $definition): void + { + $this->expectException(ColumnMismatchException::class); + $this->expectExceptionMessage('could not convert null to'); + + (new PhpBackend()) + ->builder($definition) + ->append(null); + } + + #[RequiresPhp('>= 8.4.0')] + public function test_html_round_trips(): void + { + $document = type_instance_of(HTMLDocument::class)->assert(ColumnMother::of( + html_schema('a'), + ['

a

'], + )->value(0)); + $element = type_instance_of(HTMLElement::class)->assert(ColumnMother::of(html_element_schema('a'), [ + type_html_element()->cast('

a

'), + ])->value(0)); + + static::assertStringContainsString( + '

a

', + type_string()->assert((new HtmlDocumentPhysical())->toPhysical($document)), + ); + static::assertSame( + '

a

', + (new HtmlElementPhysical())->markup(type_string()->assert((new HtmlElementPhysical())->toPhysical( + $element, + ))), + ); + } + + public function test_the_null_definition_holds_nulls_only(): void + { + $column = ColumnMother::of(null_schema('a'), [null, null, null]); + + static::assertSame(3, $column->count()); + static::assertSame(3, $column->nullCount()); + static::assertSame([null, null, null], $column->values()); + } + + /** + * @param Definition $definition + */ + #[DataProvider('refusals')] + public function test_refuses(Definition $definition, string $message): void + { + $this->expectException(ColumnMismatchException::class); + $this->expectExceptionMessage($message); + + (new PhpBackend())->builder($definition); + } + + public function test_time_refuses_months(): void + { + $this->expectException(CastingException::class); + $this->expectExceptionMessage("Relative DateInterval (with months/years) can't be stored in a time column"); + + (new PhpBackend()) + ->builder(time_schema('a')) + ->append('P1M'); + } + + /** + * @param Definition $definition + */ + #[DataProvider('temporal_range')] + public function test_temporal_range(Definition $definition, mixed $value, bool $accepted): void + { + if (!$accepted) { + $this->expectException(SchemaMismatchException::class); + $this->expectExceptionMessage('outside the'); + } + + static::assertIsInt(ColumnMother::of($definition, [$value])->at(0)); + } + + public function test_date_reads_back_in_utc(): void + { + $date = type_instance_of(DateTimeImmutable::class)->assert(ColumnMother::of(date_schema('a'), [ + new DateTimeImmutable('2024-03-05 00:30:00', new DateTimeZone('Asia/Tokyo')), + ])->value(0)); + + static::assertInstanceOf(DateTimeImmutable::class, $date); + static::assertSame('UTC', $date->getTimezone()->getName()); + static::assertSame('2024-03-05 00:00:00', $date->format('Y-m-d H:i:s')); + } + + public function test_datetime_reads_back_in_the_column_zone(): void + { + $datetime = type_instance_of(DateTimeImmutable::class)->assert(ColumnMother::of( + datetime_schema('a', zone: 'Europe/Warsaw'), + [ + new DateTimeImmutable('2024-01-02 03:04:05 UTC'), + ], + )->value(0)); + + static::assertInstanceOf(DateTimeImmutable::class, $datetime); + static::assertSame('Europe/Warsaw', $datetime->getTimezone()->getName()); + } + + public function test_time_reads_back_with_days_folded_and_no_days_count(): void + { + $time = type_instance_of(DateInterval::class)->assert(ColumnMother::of(time_schema('a'), [new DateInterval( + 'P2DT1H', + )])->value(0)); + + static::assertInstanceOf(DateInterval::class, $time); + static::assertSame(0, $time->d); + static::assertSame(49, $time->h); + static::assertFalse($time->days); + } + + public function test_constant_holds_one_value(): void + { + $column = (new PhpBackend())->constant(int_schema('a'), '7', 3); + + static::assertSame(3, $column->count()); + static::assertSame(0, $column->nullCount()); + static::assertSame([7, 7, 7], $column->values()); + static::assertSame(7, $column->value(2)); + static::assertSame(7, $column->at(1)); + } + + public function test_constant_null(): void + { + $column = (new PhpBackend())->constant(int_schema('a', nullable: true), null, 2); + + static::assertSame(2, $column->nullCount()); + static::assertSame([null, null], $column->values()); + static::assertTrue($column->isNull(0)); + } + + public function test_constant_refuses_null_under_a_not_null_definition(): void + { + $this->expectException(ColumnMismatchException::class); + $this->expectExceptionMessage('column "a": could not convert null to integer, column is not nullable'); + + (new PhpBackend())->constant(int_schema('a'), null, 2); + } + + public function test_constant_null_under_the_null_definition(): void + { + static::assertSame( + 2, + (new PhpBackend()) + ->constant(null_schema('a'), null, 2) + ->nullCount(), + ); + } + + public function test_constant_slice_take_and_concat(): void + { + $backend = new PhpBackend(); + $column = $backend->constant(int_schema('a'), 7, 5); + + static::assertSame([7, 7], $column->slice(1, 2)->values()); + static::assertSame([7, 7, 7], $column->take([0, 4, 2])->values()); + static::assertSame([7, 7, 7, 7, 7, 7], $column->concat($backend->constant(int_schema('a'), 7, 1))->values()); + static::assertSame([7, 7, 7, 7, 7, 8], $column->concat($backend->constant(int_schema('a'), 8, 1))->values()); + } + + /** + * @param Definition $definition + */ + #[DataProvider('definitions')] + public function test_decode_round_trips(Definition $definition, mixed $value, mixed $expected): void + { + $nullable = $definition->makeNullable(); + $column = ColumnMother::of($nullable, [$value, null, $value]); + $backend = new PhpBackend(); + + foreach ([$column, $column->slice(1, 2), $column->take([2, 1])] as $variant) { + static::assertEquals( + $variant->values(), + $backend->decode($nullable, $variant->encode(), $variant->count(), $variant->nullCount())->values(), + ); + } + } + + public function test_decode_round_trips_a_null_column(): void + { + $column = ColumnMother::of(null_schema('a'), [null, null]); + + static::assertSame( + [null, null], + (new PhpBackend()) + ->decode(null_schema('a'), $column->encode(), 2, 2) + ->values(), + ); + } + + /** + * @return Generator, list, int, int, string}> + */ + public static function corrupt_buffers(): Generator + { + yield 'too few buffers' => [int_schema('a'), [''], 1, 0, 'Column buffers exhausted']; + yield 'short fixed buffer' => [ + int_schema('a'), + ['', "\x01\x00"], + 1, + 0, + 'Int64 values buffer of 2 bytes, expected 8 for 1 rows', + ]; + yield 'offsets not from 0' => [ + str_schema('a'), + ['', "\x01\x00\x00\x00\x02\x00\x00\x00", 'ab'], + 1, + 0, + 'Utf8 offsets start at 1, not 0', + ]; + yield 'offsets not monotonic' => [ + str_schema('a'), + ['', "\x00\x00\x00\x00\x02\x00\x00\x00\x01\x00\x00\x00", 'ab'], + 2, + 0, + 'Utf8 offsets are not monotonic', + ]; + yield 'offsets past the data' => [ + str_schema('a'), + ['', "\x00\x00\x00\x00\x03\x00\x00\x00", 'ab'], + 1, + 0, + 'Utf8 data buffer of 2 bytes, the last offset is 3', + ]; + yield 'map entries validity present' => [ + map_schema('a', type_map(type_string(), type_integer())), + ['', "\x00\x00\x00\x00\x00\x00\x00\x00", "\x01", '', "\x00\x00\x00\x00", '', '', ''], + 1, + 0, + 'Map entries validity must be omitted, got 1 bytes', + ]; + yield 'null count disagreeing with the bitmap' => [ + int_schema('a', nullable: true), + ["\x01", "\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00"], + 2, + 0, + 'Column null count 0 disagrees with its validity bitmap (1 nulls in 2 rows)', + ]; + yield 'short validity bitmap' => [ + int_schema('a', nullable: true), + ['', ''], + 9, + 0, + 'Int64 values buffer of 0 bytes, expected 72 for 9 rows', + ]; + yield 'null column with a wrong null count' => [ + null_schema('a'), + [], + 2, + 1, + 'Null column of 2 rows carries a null count of 1', + ]; + yield 'leftover buffers' => [ + int_schema('a'), + ['', "\x01\x00\x00\x00\x00\x00\x00\x00", ''], + 1, + 0, + 'Column "a": 1 buffers left after decoding', + ]; + } + + /** + * @param Definition $definition + * @param list $buffers + */ + #[DataProvider('corrupt_buffers')] + public function test_decode_refuses( + Definition $definition, + array $buffers, + int $count, + int $nullCount, + string $message, + ): void { + $this->expectException(InvalidArgumentException::class); + $this->expectExceptionMessage($message); + + (new PhpBackend())->decode($definition, $buffers, $count, $nullCount); + } + + public function test_adopt_returns_its_own_column(): void + { + $column = ColumnMother::of(int_schema('a'), [1, 2]); + + static::assertSame($column, (new PhpBackend())->adopt(int_schema('a'), $column)); + } + + public function test_adopt_copies_a_foreign_column(): void + { + $foreign = new ForeignColumnStub(ColumnMother::of(str_schema('a', nullable: true), ['x', null, 'z'])); + + $adopted = (new PhpBackend())->adopt(str_schema('a', nullable: true), $foreign); + + static::assertNotSame($foreign, $adopted); + static::assertSame(['x', null, 'z'], $adopted->values()); + static::assertSame(1, $adopted->nullCount()); + } + + public function test_adopt_copies_an_empty_foreign_column(): void + { + $foreign = new ForeignColumnStub(ColumnMother::of(int_schema('a'), [])); + + $adopted = (new PhpBackend())->adopt(int_schema('a'), $foreign); + + static::assertNotSame($foreign, $adopted); + static::assertSame(0, $adopted->count()); + } + + public function test_adopt_refuses_a_null_under_not_null(): void + { + $foreign = new ForeignColumnStub(ColumnMother::of(int_schema('a', nullable: true), [1, null])); + + $this->expectException(ColumnMismatchException::class); + $this->expectExceptionMessage(ColumnMismatchException::valueDoesNotMatch(int_schema('a'), null)->getMessage()); + + (new PhpBackend())->adopt(int_schema('a'), $foreign); + } + + public function test_adopt_refuses_an_untyped_column(): void + { + $this->expectException(ColumnMismatchException::class); + $this->expectExceptionMessage( + 'column "a": mixed cannot be a batch column, an untyped function result exists only inside function evaluation', + ); + + (new PhpBackend())->adopt(int_schema('a'), new ValueColumn([1, 2])); + } + + public function test_allocated_bytes_is_zero(): void + { + static::assertSame(0, (new PhpBackend())->allocatedBytes()); + } +} diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Column/Physical/DatePhysicalTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Column/Physical/DatePhysicalTest.php new file mode 100644 index 0000000000..bd5b7ee498 --- /dev/null +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Column/Physical/DatePhysicalTest.php @@ -0,0 +1,46 @@ +toPhysical( + new DateTimeImmutable('2024-03-05 00:30:00', new DateTimeZone('Asia/Tokyo')), + ), + ); + static::assertSame(-1, (new DatePhysical())->toPhysical(new DateTimeImmutable('1969-12-31 23:59:59 UTC'))); + } + + public function test_reads_back_midnight_utc(): void + { + $physical = new DatePhysical(); + $value = type_instance_of(DateTimeImmutable::class)->assert($physical->fromPhysical(-1)); + $all = type_instance_of(DateTimeImmutable::class)->assert($physical->fromPhysicalAll([-1, null])[0]); + + static::assertSame('1969-12-31 00:00:00 UTC', $value->format('Y-m-d H:i:s e')); + static::assertSame('1969-12-31 00:00:00 UTC', $all->format('Y-m-d H:i:s e')); + static::assertNull($physical->fromPhysicalAll([-1, null])[1]); + } + + public function test_refuses_a_day_outside_the_i32_range(): void + { + $this->expectException(CastingException::class); + $this->expectExceptionMessage('outside the day range of a date column'); + + (new DatePhysical())->toPhysical(new DateTimeImmutable('@' . ((2 ** 31) * 86_400))); + } +} diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Column/Physical/DateTimePhysicalTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Column/Physical/DateTimePhysicalTest.php new file mode 100644 index 0000000000..3012d1fa61 --- /dev/null +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Column/Physical/DateTimePhysicalTest.php @@ -0,0 +1,50 @@ +toPhysical($value)); + static::assertEquals($value, $physical->fromPhysical($microseconds)); + static::assertEquals([$value, null], $physical->fromPhysicalAll([$microseconds, null])); + } + + public function test_reads_back_in_the_zone(): void + { + $physical = new DateTimePhysical(new DateTimeZone('Asia/Tokyo')); + $value = type_instance_of(DateTimeImmutable::class)->assert($physical->fromPhysical(0)); + $all = type_instance_of(DateTimeImmutable::class)->assert($physical->fromPhysicalAll([0])[0]); + + static::assertSame('1970-01-01 09:00:00 Asia/Tokyo', $value->format('Y-m-d H:i:s e')); + static::assertSame('1970-01-01 09:00:00 Asia/Tokyo', $all->format('Y-m-d H:i:s e')); + } + + public function test_refuses_a_value_outside_the_microsecond_range(): void + { + $this->expectException(CastingException::class); + $this->expectExceptionMessage('outside the microsecond range of a datetime column'); + + (new DateTimePhysical(new DateTimeZone('UTC')))->toPhysical(new DateTimeImmutable('@9223372036855')); + } +} diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Column/Physical/DaysFromCivilTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Column/Physical/DaysFromCivilTest.php new file mode 100644 index 0000000000..86123b068a --- /dev/null +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Column/Physical/DaysFromCivilTest.php @@ -0,0 +1,26 @@ +of($year, $month, $day)); + } +} diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Column/Physical/ElementPositionTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Column/Physical/ElementPositionTest.php new file mode 100644 index 0000000000..f26d982423 --- /dev/null +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Column/Physical/ElementPositionTest.php @@ -0,0 +1,67 @@ +loadXML('text'); + $position = new ElementPosition(); + $root = type_instance_of(DOMElement::class)->assert($document->documentElement); + $d = type_instance_of(DOMElement::class)->assert($root->lastElementChild?->lastElementChild); + + static::assertSame('', $position->path($root)); + static::assertSame('1/1', $position->path($d)); + static::assertSame($d, $position->element($root, '1/1')); + static::assertSame($root, $position->element($root, '')); + static::assertNull($position->element($root, '2')); + static::assertNull($position->element($root, '0/0')); + } + + public function test_an_element_outside_the_document_tree_has_no_path(): void + { + $document = new DOMDocument(); + $document->loadXML(''); + + static::assertNull((new ElementPosition())->path(type_instance_of( + DOMElement::class, + )->assert($document->createElement('detached')))); + } + + public function test_the_physical_splits_into_the_path_the_markup_and_the_document(): void + { + $position = new ElementPosition(); + $physical = $position->physical('0/2', "\0", "\0"); + + static::assertSame("\x010/2\x008\x00\0\0", $physical); + static::assertSame('0/2', $position->pathOf($physical)); + static::assertSame("\0", $position->markupOf($physical)); + static::assertSame("\0", $position->documentOf($physical)); + } + + public function test_the_document_without_the_markup_splits_into_the_path_and_the_document(): void + { + $position = new ElementPosition(); + + static::assertSame('0/2', $position->pathOf("\x000/2\x00")); + static::assertNull($position->markupOf("\x000/2\x00")); + static::assertSame('', $position->documentOf("\x000/2\x00")); + } + + public function test_bare_markup_has_no_position(): void + { + static::assertNull((new ElementPosition())->pathOf('')); + static::assertNull((new ElementPosition())->markupOf('')); + } +} diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Column/Physical/EnumPhysicalTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Column/Physical/EnumPhysicalTest.php new file mode 100644 index 0000000000..834e2dc79f --- /dev/null +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Column/Physical/EnumPhysicalTest.php @@ -0,0 +1,21 @@ +toPhysical(BasicEnum::one)); + static::assertSame(BasicEnum::one, $physical->fromPhysical('one')); + static::assertSame([BasicEnum::two, null], $physical->fromPhysicalAll(['two', null])); + } +} diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Column/Physical/HtmlDocumentPhysicalTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Column/Physical/HtmlDocumentPhysicalTest.php new file mode 100644 index 0000000000..8997be5485 --- /dev/null +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Column/Physical/HtmlDocumentPhysicalTest.php @@ -0,0 +1,38 @@ += 8.4.0')] + public function test_the_markup(): void + { + $document = type_html()->cast('

a

'); + $physical = new HtmlDocumentPhysical(); + $markup = type_string()->assert($physical->toPhysical($document)); + + static::assertIsString($markup); + static::assertStringContainsString('

a

', $markup); + static::assertEquals($markup, $physical->toPhysical($physical->fromPhysical($markup))); + static::assertNull($physical->fromPhysicalAll([$markup, null])[1]); + } + + #[RequiresPhp('< 8.4.0')] + public function test_restoring_below_php84_throws(): void + { + $this->expectException(InvalidArgumentException::class); + $this->expectExceptionMessage('requires PHP 8.4+'); + + (new HtmlDocumentPhysical())->fromPhysical('

x

'); + } +} diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Column/Physical/HtmlElementPhysicalTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Column/Physical/HtmlElementPhysicalTest.php new file mode 100644 index 0000000000..f1a2bd6c6e --- /dev/null +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Column/Physical/HtmlElementPhysicalTest.php @@ -0,0 +1,108 @@ += 8.4.0')] + public function test_the_markup(): void + { + $physical = new HtmlElementPhysical(); + $bytes = type_string()->assert($physical->toPhysical(type_html_element()->cast('

a

'))); + + static::assertSame("\x01\x008\x00

a

a

", $bytes); + static::assertSame('

a

', $physical->markup($bytes)); + static::assertSame('

a

', $physical->markup('

a

')); + static::assertNull($physical->fromPhysicalAll([$bytes, null])[1]); + } + + #[RequiresPhp('>= 8.4.0')] + public function test_a_round_trip_keeps_the_parent_and_the_siblings(): void + { + $physical = new HtmlElementPhysical(); + $section = type_html_element() + ->cast('

Name

01
') + ->querySelector('section'); + + static::assertNotNull($section); + + $element = type_html_element()->assert($physical->fromPhysical($physical->toPhysical($section))); + + static::assertSame( + '

Name

', + $physical->markup(type_string()->assert($physical->toPhysical($element))), + ); + static::assertSame('ARTICLE', $element->parentElement?->tagName); + static::assertNull($element->parentElement->parentElement); + static::assertSame('01', $element->nextElementSibling?->textContent); + } + + #[RequiresPhp('>= 8.4.0')] + public function test_bare_element_markup_of_earlier_builds_still_loads(): void + { + $element = type_html_element()->assert((new HtmlElementPhysical())->fromPhysical( + '

Name

', + )); + + static::assertSame('Name', $element->firstElementChild?->textContent); + static::assertSame('BODY', $element->parentElement?->tagName); + } + + #[RequiresPhp('>= 8.4.0')] + public function test_the_document_without_the_element_markup_still_loads(): void + { + $physical = new HtmlElementPhysical(); + $bytes = "\x000\x00

Name

01
"; + $element = type_html_element()->assert($physical->fromPhysical($bytes)); + + static::assertSame('SECTION', $element->tagName); + static::assertSame('ARTICLE', $element->parentElement?->tagName); + static::assertSame('

Name

', $physical->markup($bytes)); + } + + #[RequiresPhp('>= 8.4.0')] + public function test_a_detached_element_is_stored_as_its_markup(): void + { + $element = type_html_element()->cast('

a

'); + $element->remove(); + + static::assertSame('

a

', (new HtmlElementPhysical())->toPhysical($element)); + } + + #[RequiresPhp('>= 8.4.0')] + public function test_a_path_outside_the_document_is_refused(): void + { + $this->expectException(InvalidArgumentException::class); + $this->expectExceptionMessage('Floe failed to restore HTMLElement'); + + (new HtmlElementPhysical())->fromPhysical("\x015\x000\x00"); + } + + #[RequiresPhp('>= 8.4.0')] + public function test_markup_without_an_element_is_refused(): void + { + $this->expectException(InvalidArgumentException::class); + $this->expectExceptionMessage('Floe failed to restore HTMLElement from "text"'); + + (new HtmlElementPhysical())->fromPhysical('text'); + } + + #[RequiresPhp('< 8.4.0')] + public function test_restoring_below_php84_throws(): void + { + $this->expectException(InvalidArgumentException::class); + $this->expectExceptionMessage('requires PHP 8.4+'); + + (new HtmlElementPhysical())->fromPhysical('

x

'); + } +} diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Column/Physical/IdentityPhysicalTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Column/Physical/IdentityPhysicalTest.php new file mode 100644 index 0000000000..12def22f3d --- /dev/null +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Column/Physical/IdentityPhysicalTest.php @@ -0,0 +1,20 @@ +toPhysical('a')); + static::assertSame(1.5, $physical->fromPhysical(1.5)); + static::assertSame([1, null, true], $physical->fromPhysicalAll([1, null, true])); + } +} diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Column/Physical/JsonFactoryTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Column/Physical/JsonFactoryTest.php new file mode 100644 index 0000000000..45f6455f20 --- /dev/null +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Column/Physical/JsonFactoryTest.php @@ -0,0 +1,28 @@ +create('{"a":1}')); + static::assertEquals(Json::fromString('[1,2]'), (new JsonFactory())->create('[1,2]')); + static::assertTrue( + (new JsonFactory()) + ->create('{}') + ->isObject(), + ); + static::assertFalse( + (new JsonFactory()) + ->create('[]') + ->isObject(), + ); + } +} diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Column/Physical/JsonPhysicalTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Column/Physical/JsonPhysicalTest.php new file mode 100644 index 0000000000..5ac45a85ee --- /dev/null +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Column/Physical/JsonPhysicalTest.php @@ -0,0 +1,21 @@ +toPhysical(Json::fromString('{"a":1}'))); + static::assertEquals(Json::fromString('{"a":1}'), $physical->fromPhysical('{"a":1}')); + static::assertEquals([Json::fromString('[1]'), null], $physical->fromPhysicalAll(['[1]', null])); + } +} diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Column/Physical/ListPhysicalTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Column/Physical/ListPhysicalTest.php new file mode 100644 index 0000000000..d49b266a4f --- /dev/null +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Column/Physical/ListPhysicalTest.php @@ -0,0 +1,22 @@ +toPhysical([1 => new DateTimeZone('UTC'), 3 => null])); + static::assertEquals([new DateTimeZone('UTC'), null], $physical->fromPhysical(['UTC', null])); + static::assertEquals([[new DateTimeZone('UTC')], null], $physical->fromPhysicalAll([['UTC'], null])); + } +} diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Column/Physical/MapPhysicalTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Column/Physical/MapPhysicalTest.php new file mode 100644 index 0000000000..d67c2a94eb --- /dev/null +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Column/Physical/MapPhysicalTest.php @@ -0,0 +1,28 @@ + 'UTC', 'b' => null], + $physical->toPhysical(['a' => new DateTimeZone('UTC'), 'b' => null]), + ); + static::assertEquals( + ['a' => new DateTimeZone('UTC'), 'b' => null], + $physical->fromPhysical(['a' => 'UTC', 'b' => null]), + ); + static::assertEquals([[5 => new DateTimeZone('UTC')], null], $physical->fromPhysicalAll([[5 => 'UTC'], null])); + } +} diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Column/Physical/NullPhysicalTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Column/Physical/NullPhysicalTest.php new file mode 100644 index 0000000000..99c256fef5 --- /dev/null +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Column/Physical/NullPhysicalTest.php @@ -0,0 +1,21 @@ +toPhysical('a')); + static::assertNull($physical->fromPhysical('a')); + static::assertSame([null, null], $physical->fromPhysicalAll([1, null])); + static::assertSame([], $physical->fromPhysicalAll([])); + } +} diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Column/Physical/PhysicalBuilderForTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Column/Physical/PhysicalBuilderForTest.php new file mode 100644 index 0000000000..4360db8cd6 --- /dev/null +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Column/Physical/PhysicalBuilderForTest.php @@ -0,0 +1,53 @@ +type(type_integer())); + static::assertInstanceOf(ListColumnBuilder::class, $builders->type(type_optional(type_list(type_integer())))); + static::assertInstanceOf(MapColumnBuilder::class, $builders->type(type_map(type_string(), type_integer()))); + static::assertInstanceOf(StructColumnBuilder::class, $builders->type(type_structure(['a' => type_integer()]))); + static::assertInstanceOf(NullColumnBuilder::class, $builders->type(type_null())); + } + + public function test_an_optional_container_keeps_its_optional_type(): void + { + $builder = (new PhysicalBuilderFor())->type(type_optional(type_list(type_integer()))); + $builder->appendPhysical(null); + + static::assertEquals(type_optional(type_list(type_integer())), $builder->finish()->type()); + } + + public function test_refuses_a_mixed_element(): void + { + $this->expectException(InvalidArgumentException::class); + $this->expectExceptionMessage('a mixed element has no column kind'); + + (new PhysicalBuilderFor())->type(type_list(type_mixed())); + } +} diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Column/Physical/PhysicalForTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Column/Physical/PhysicalForTest.php new file mode 100644 index 0000000000..42ee0b927d --- /dev/null +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Column/Physical/PhysicalForTest.php @@ -0,0 +1,211 @@ +, class-string}> + */ + public static function types(): Generator + { + yield 'integer' => [type_integer(), IdentityPhysical::class]; + yield 'positive integer' => [type_positive_integer(), IdentityPhysical::class]; + yield 'float' => [type_float(), IdentityPhysical::class]; + yield 'boolean' => [type_boolean(), IdentityPhysical::class]; + yield 'string' => [type_string(), IdentityPhysical::class]; + yield 'non-empty string' => [type_non_empty_string(), IdentityPhysical::class]; + yield 'numeric string' => [type_numeric_string(), IdentityPhysical::class]; + yield 'class string' => [type_class_string(), IdentityPhysical::class]; + yield 'optional integer' => [type_optional(type_integer()), IdentityPhysical::class]; + yield 'datetime' => [type_datetime(), DateTimePhysical::class]; + yield 'date' => [type_date(), DatePhysical::class]; + yield 'time' => [type_time(), TimePhysical::class]; + yield 'uuid' => [type_uuid(), UuidPhysical::class]; + yield 'timezone' => [type_time_zone(), TimeZonePhysical::class]; + yield 'json' => [type_json(), JsonPhysical::class]; + yield 'enum' => [type_enum(BasicEnum::class), EnumPhysical::class]; + yield 'xml' => [type_xml(), XmlDocumentPhysical::class]; + yield 'xml element' => [type_xml_element(), XmlElementPhysical::class]; + yield 'html' => [type_html(), HtmlDocumentPhysical::class]; + yield 'html element' => [type_html_element(), HtmlElementPhysical::class]; + yield 'list of identities' => [type_list(type_optional(type_integer())), IdentityPhysical::class]; + yield 'list of lists of identities' => [type_list(type_list(type_string())), IdentityPhysical::class]; + yield 'list' => [type_list(type_date()), ListPhysical::class]; + yield 'map of identities' => [type_map(type_string(), type_integer()), IdentityPhysical::class]; + yield 'map' => [type_map(type_string(), type_uuid()), MapPhysical::class]; + yield 'structure of identities' => [ + type_structure(['a' => type_integer(), 'b' => type_list(type_string())]), + IdentityPhysical::class, + ]; + yield 'structure' => [type_structure(['a' => type_integer(), 'b' => type_date()]), StructPhysical::class]; + yield 'structure of a list' => [type_structure(['a' => type_list(type_date())]), StructPhysical::class]; + yield 'null' => [type_null(), NullPhysical::class]; + } + + /** + * @return Generator, string}> + */ + public static function refusals(): Generator + { + yield 'mixed' => [type_mixed(), 'a mixed element has no column kind, declare it or use json']; + yield 'nested mixed' => [type_map(type_string(), type_mixed()), 'a mixed element has no column kind']; + yield 'unit enum wildcard' => [type_enum(UnitEnum::class), 'declare the concrete enum class']; + yield 'backed enum wildcard' => [type_enum(BackedEnum::class), 'declare the concrete enum class']; + yield 'instance of' => [type_instance_of(stdClass::class), 'type object has no column kind']; + yield 'structure element' => [type_structure(['a' => type_mixed()]), 'a mixed element has no column kind']; + } + + /** + * @return Generator, bool}> + */ + public static function supported(): Generator + { + foreach (self::types() as $name => [$type]) { + yield $name => [ + $type, + !in_array($name, ['positive integer', 'non-empty string', 'numeric string', 'class string'], true), + ]; + } + + yield 'nullable union' => [type_union(type_string(), type_null()), true]; + yield 'array' => [type_array(), true]; + yield 'optional array' => [type_optional(type_array()), true]; + yield 'list of non-empty strings' => [type_list(type_non_empty_string()), true]; + yield 'list of optional integers' => [type_list(type_optional(type_integer())), true]; + yield 'structure of lists' => [ + type_structure(['a' => type_list(type_uuid()), 'b' => type_map(type_string(), type_date())]), + true, + ]; + yield 'mixed' => [type_mixed(), false]; + yield 'union' => [type_union(type_string(), type_integer()), false]; + yield 'object' => [type_object(), false]; + yield 'instance of' => [type_instance_of(ArrayObject::class), false]; + yield 'empty array' => [type_empty_array(), false]; + yield 'callable' => [type_callable(), false]; + yield 'scalar' => [type_scalar(), false]; + yield 'unit enum wildcard' => [type_enum(UnitEnum::class), false]; + yield 'backed enum wildcard' => [type_enum(BackedEnum::class), false]; + yield 'list of mixed' => [type_list(type_mixed()), false]; + yield 'list of arrays' => [type_list(type_array()), true]; + yield 'list of empty arrays' => [type_list(type_empty_array()), true]; + yield 'list of nullable unions' => [type_list(type_union(type_string(), type_null())), true]; + yield 'list of unions' => [type_list(type_union(type_string(), type_integer())), false]; + yield 'map of mixed' => [type_map(type_string(), type_mixed()), false]; + yield 'structure element' => [type_structure(['a' => type_integer(), 'b' => type_mixed()]), false]; + yield 'list of unit enums' => [type_list(type_enum(UnitEnum::class)), false]; + } + + /** + * @param Type $type + */ + #[DataProvider('supported')] + public function test_supports_agrees_with_definition_from_type_and_definition(Type $type, bool $supported): void + { + try { + (new PhysicalFor())->definition(definition_from_type('value', $type)); + $defined = true; + } catch (InvalidArgumentException|RuntimeException) { + $defined = false; + } + + static::assertSame($supported, (new PhysicalFor())->supports($type)); + static::assertSame($supported, $defined); + } + + /** + * @param Type $type + * @param class-string $class + */ + #[DataProvider('types')] + public function test_maps_the_type_to_its_physical(Type $type, string $class): void + { + static::assertInstanceOf($class, (new PhysicalFor())->type($type)); + } + + /** + * @param Type $type + */ + #[DataProvider('refusals')] + public function test_refuses(Type $type, string $message): void + { + $this->expectException(InvalidArgumentException::class); + $this->expectExceptionMessage($message); + + (new PhysicalFor())->type($type); + } + + public function test_structure_elements_are_keyed_by_name(): void + { + static::assertSame( + ['a', 'b'], + array_keys((new PhysicalFor())->elements(type_structure(['a' => type_integer(), 'b' => type_string()]))), + ); + } +} diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Column/Physical/PhysicalKindTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Column/Physical/PhysicalKindTest.php new file mode 100644 index 0000000000..3fc28a58ea --- /dev/null +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Column/Physical/PhysicalKindTest.php @@ -0,0 +1,63 @@ +, mixed, bool}> + */ + public static function physicals(): Generator + { + yield 'null is any type\'s' => [type_integer(), null, true]; + yield 'int in an integer column' => [type_integer(), 1, true]; + yield 'string in an integer column' => [type_integer(), '1', false]; + yield 'int in a float column' => [type_float(), 1, false]; + yield 'float in a float column' => [type_float(), 1.5, true]; + yield 'bool' => [type_boolean(), true, true]; + yield 'datetime micros' => [type_datetime(), 1_700_000_000_000_000, true]; + yield 'datetime text' => [type_datetime(), '2026-01-01', false]; + yield 'uuid 16 bytes' => [type_uuid(), str_repeat("\0", 16), true]; + yield 'uuid text' => [type_uuid(), '00000000-0000-4000-8000-000000000000', false]; + yield 'json text' => [type_json(), '{"a":1}', true]; + yield 'optional unwrapped' => [type_optional(type_integer()), 1, true]; + yield 'anything in a null column' => [type_null(), 'x', true]; + yield 'list of ints' => [type_list(type_integer()), [1, null, 2], true]; + yield 'list with a wrong element' => [type_list(type_integer()), [1, '2'], false]; + yield 'list that is not an array' => [type_list(type_integer()), 'x', false]; + yield 'map values checked' => [type_map(type_string(), type_integer()), ['a' => 'b'], false]; + yield 'structure element checked' => [type_structure(['a' => type_integer()]), ['a' => 'x'], false]; + yield 'structure element absent' => [type_structure(['a' => type_integer()]), [], true]; + } + + /** + * @param Type $type + */ + #[DataProvider('physicals')] + public function test_accepts_the_kind_flow_php_stores(Type $type, mixed $physical, bool $accepts): void + { + static::assertSame($accepts, (new PhysicalKind())->accepts($type, $physical)); + } +} diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Column/Physical/StructPhysicalTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Column/Physical/StructPhysicalTest.php new file mode 100644 index 0000000000..0cf40c2281 --- /dev/null +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Column/Physical/StructPhysicalTest.php @@ -0,0 +1,37 @@ + new IdentityPhysical(), + 'zone' => new TimeZonePhysical(), + 'note' => new IdentityPhysical(), + ]); + + static::assertSame( + ['id' => 1, 'zone' => 'UTC'], + $physical->toPhysical(['zone' => new DateTimeZone('UTC'), 'id' => 1]), + ); + static::assertSame(['id' => 1, 'zone' => null], $physical->toPhysical(['id' => 1, 'zone' => null])); + static::assertEquals( + ['id' => 1, 'zone' => new DateTimeZone('UTC')], + $physical->fromPhysical(['id' => 1, 'zone' => 'UTC']), + ); + static::assertEquals( + [['id' => 1, 'zone' => null], null], + $physical->fromPhysicalAll([['id' => 1, 'zone' => null], null]), + ); + } +} diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Column/Physical/TimePhysicalTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Column/Physical/TimePhysicalTest.php new file mode 100644 index 0000000000..189035b1aa --- /dev/null +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Column/Physical/TimePhysicalTest.php @@ -0,0 +1,72 @@ +f = 0.5; + + static::assertSame(93_784_500_000, (new TimePhysical())->toPhysical($interval)); + + // @mago-ignore analysis:invalid-property-write + $interval->invert = 1; + + static::assertSame(-93_784_500_000, (new TimePhysical())->toPhysical($interval)); + } + + public function test_reads_back_hours_minutes_seconds_and_fraction(): void + { + $physical = new TimePhysical(); + $intervals = type_list(type_instance_of(DateInterval::class))->assert([ + $physical->fromPhysical(-93_784_500_000), + $physical->fromPhysicalAll([-93_784_500_000, null])[0], + ]); + + foreach ($intervals as $interval) { + static::assertSame([0, 26, 3, 4, 0.5, 1], [ + $interval->d, + $interval->h, + $interval->i, + $interval->s, + $interval->f, + $interval->invert, + ]); + } + + static::assertNull($physical->fromPhysicalAll([-93_784_500_000, null])[1]); + } + + public function test_refuses_months(): void + { + $this->expectException(CastingException::class); + $this->expectExceptionMessage("Relative DateInterval (with months/years) can't be stored in a time column"); + + (new TimePhysical())->toPhysical(new DateInterval('P1Y')); + } + + public function test_refuses_a_duration_outside_the_microsecond_range(): void + { + $this->expectException(CastingException::class); + $this->expectExceptionMessage('outside the microsecond range of a time column'); + + $interval = new DateInterval('PT0S'); + // @mago-ignore analysis:invalid-property-write + $interval->s = 9_223_372_036_855; + + (new TimePhysical())->toPhysical($interval); + } +} diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Column/Physical/TimeZonePhysicalTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Column/Physical/TimeZonePhysicalTest.php new file mode 100644 index 0000000000..b83c72038d --- /dev/null +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Column/Physical/TimeZonePhysicalTest.php @@ -0,0 +1,21 @@ +toPhysical(new DateTimeZone('Europe/Warsaw'))); + static::assertEquals(new DateTimeZone('Europe/Warsaw'), $physical->fromPhysical('Europe/Warsaw')); + static::assertEquals([new DateTimeZone('UTC'), null], $physical->fromPhysicalAll(['UTC', null])); + } +} diff --git a/src/core/etl/tests/Flow/Floe/Tests/Unit/Decoding/TimeZonesTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Column/Physical/TimeZonesTest.php similarity index 81% rename from src/core/etl/tests/Flow/Floe/Tests/Unit/Decoding/TimeZonesTest.php rename to src/core/etl/tests/Flow/ETL/Tests/Unit/Column/Physical/TimeZonesTest.php index 090a154f9a..0f370b00f1 100644 --- a/src/core/etl/tests/Flow/Floe/Tests/Unit/Decoding/TimeZonesTest.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Column/Physical/TimeZonesTest.php @@ -2,9 +2,9 @@ declare(strict_types=1); -namespace Flow\Floe\Tests\Unit\Decoding; +namespace Flow\ETL\Tests\Unit\Column\Physical; -use Flow\Floe\Decoding\TimeZones; +use Flow\ETL\Column\Physical\TimeZones; use PHPUnit\Framework\TestCase; final class TimeZonesTest extends TestCase diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Column/Physical/UuidPhysicalTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Column/Physical/UuidPhysicalTest.php new file mode 100644 index 0000000000..bb438f05de --- /dev/null +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Column/Physical/UuidPhysicalTest.php @@ -0,0 +1,32 @@ +assert($physical->toPhysical(new Uuid('6c2f1d4e-8b3a-4c5d-9e6f-0a1b2c3d4e5f'))); + + static::assertIsString($bytes); + static::assertSame('6c2f1d4e8b3a4c5d9e6f0a1b2c3d4e5f', bin2hex($bytes)); + static::assertEquals(new Uuid('6c2f1d4e-8b3a-4c5d-9e6f-0a1b2c3d4e5f'), $physical->fromPhysical($bytes)); + static::assertEquals( + [new Uuid('6c2f1d4e-8b3a-4c5d-9e6f-0a1b2c3d4e5f'), null], + $physical->fromPhysicalAll([ + $bytes, + null, + ]), + ); + } +} diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Column/Physical/XmlCharacterReferencesTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Column/Physical/XmlCharacterReferencesTest.php new file mode 100644 index 0000000000..43acc0d390 --- /dev/null +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Column/Physical/XmlCharacterReferencesTest.php @@ -0,0 +1,24 @@ +', ''])] + #[TestWith(['', ''])] + #[TestWith(['', ''])] + #[TestWith(['', ''])] + #[TestWith(['', ''])] + #[TestWith(['', ''])] + #[TestWith(['&', '&'])] + public function test_decodes_non_ascii_references_outside_cdata_and_comments(string $xml, string $decoded): void + { + static::assertSame($decoded, (new XmlCharacterReferences())->decoded($xml)); + } +} diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Column/Physical/XmlDocumentPhysicalTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Column/Physical/XmlDocumentPhysicalTest.php new file mode 100644 index 0000000000..6d791269d1 --- /dev/null +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Column/Physical/XmlDocumentPhysicalTest.php @@ -0,0 +1,85 @@ +loadXML('
1'); + $physical = new XmlDocumentPhysical(); + $markup = type_string()->assert($physical->toPhysical($document)); + + static::assertIsString($markup); + static::assertStringContainsString('1', $markup); + static::assertEquals($document, $physical->fromPhysical($markup)); + static::assertEquals([$document, null], $physical->fromPhysicalAll([$markup, null])); + } + + public function test_malformed_markup_is_refused(): void + { + $this->expectException(InvalidArgumentException::class); + $this->expectExceptionMessage('Floe failed to restore DOMDocument from ""'); + + (new XmlDocumentPhysical())->fromPhysical(''); + } + + public static function nodes(): Generator + { + foreach (XmlNodeMother::edges() as $label => ['node' => $node]) { + yield $label => [$node]; + } + } + + #[DataProvider('nodes')] + public function test_text_is_the_inverse_of_physical(string $node): void + { + $physical = new XmlDocumentPhysical(); + + static::assertSame($node, $physical->text($physical->physical($node))); + } + + #[TestWith(["\n\n"])] + #[TestWith(["\n\n\n"])] + #[TestWith(["\n\n\n"])] + #[TestWith(["\n\n\n"])] + #[TestWith(["\n\n\n"])] + #[TestWith(["\n\n\n"])] + #[TestWith(["\n\n"])] + #[TestWith(["\n"])] + #[TestWith(["\n\n"])] + #[TestWith([''])] + public function test_bytes_physical_did_not_produce_have_no_text(string $physical): void + { + static::assertNull((new XmlDocumentPhysical())->text($physical)); + } + + public function test_the_physical_of_a_node_is_the_markup_of_its_utf8_document(): void + { + $document = new DOMDocument('1.0', 'UTF-8'); + $row = new DOMElement('row'); + $document->appendChild($row); + $row->setAttribute('a', 'żółć'); + $row->appendChild(new DOMElement('b', 'ść')); + + static::assertSame( + (new XmlDocumentPhysical())->toPhysical($document), + (new XmlDocumentPhysical())->physical('ść'), + ); + } +} diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Column/Physical/XmlElementPhysicalTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Column/Physical/XmlElementPhysicalTest.php new file mode 100644 index 0000000000..40a7dc7892 --- /dev/null +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Column/Physical/XmlElementPhysicalTest.php @@ -0,0 +1,122 @@ +loadXML('1'); + $element = $document->documentElement; + $physical = new XmlElementPhysical(); + + static::assertNotNull($element); + + $bytes = type_string()->assert($physical->toPhysical($element)); + + static::assertSame( + "\x01\x0021\x001\n1\n", + $bytes, + ); + static::assertSame('1', $physical->markup($bytes)); + static::assertEquals($element, $physical->fromPhysical($bytes)); + static::assertEquals([$element, null], $physical->fromPhysicalAll([$bytes, null])); + } + + public function test_a_round_trip_keeps_the_parent_and_the_siblings(): void + { + $document = new DOMDocument(); + $document->loadXML('Name01'); + $physical = new XmlElementPhysical(); + $user = $document->documentElement?->firstElementChild; + + static::assertNotNull($user); + + $element = type_instance_of(DOMElement::class)->assert($physical->fromPhysical($physical->toPhysical($user))); + + static::assertSame('user', $element->tagName); + static::assertSame('Name', $element->textContent); + static::assertSame('root', $element->parentElement?->tagName); + static::assertSame('01', $element->nextElementSibling?->textContent); + } + + public function test_bare_element_markup_of_earlier_builds_still_loads(): void + { + static::assertSame( + 'Name', + type_instance_of(DOMElement::class)->assert((new XmlElementPhysical())->fromPhysical( + 'Name', + ))->textContent, + ); + } + + public function test_the_document_without_the_element_markup_still_loads(): void + { + $physical = new XmlElementPhysical(); + $bytes = "\x000\x00Name01"; + $element = type_instance_of(DOMElement::class)->assert($physical->fromPhysical($bytes)); + + static::assertSame('user', $element->tagName); + static::assertSame('root', $element->parentElement?->tagName); + static::assertSame('Name', $physical->markup($bytes)); + static::assertSame('value', $physical->markup('value')); + } + + public function test_a_path_outside_the_document_is_refused(): void + { + $this->expectException(InvalidArgumentException::class); + $this->expectExceptionMessage('Floe failed to restore DOMElement'); + + (new XmlElementPhysical())->fromPhysical("\x015\x000\x00"); + } + + #[RequiresPhp('>= 8.4.0')] + public function test_a_dom_element_of_the_new_api_reads_back_as_a_dom_element(): void + { + // @mago-ignore analysis:unavailable-method + $element = XMLDocument::createFromString('1')->documentElement?->firstElementChild; + $physical = new XmlElementPhysical(); + $bytes = type_string()->assert($physical->toPhysical($element)); + + static::assertSame( + "\x010\x008\x001\n1", + $bytes, + ); + static::assertSame( + 'root', + type_instance_of(DOMElement::class)->assert($physical->fromPhysical($bytes))->parentElement?->tagName, + ); + } + + public function test_restoring_xml_element_from_string(): void + { + static::assertSame( + 'item', + type_instance_of(DOMElement::class)->assert((new XmlElementPhysical())->fromPhysical( + 'value', + ))->tagName, + ); + } + + public function test_a_detached_dom_element_is_refused(): void + { + $this->expectException(InvalidArgumentException::class); + $this->expectExceptionMessage('failed to convert DOMElement to XML string'); + + (new XmlElementPhysical())->toPhysical(new DOMElement('detached')); + } +} diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Column/RetypeTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Column/RetypeTest.php new file mode 100644 index 0000000000..8fdc837744 --- /dev/null +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Column/RetypeTest.php @@ -0,0 +1,156 @@ +, Type, int}> + */ + public static function accepted(): Generator + { + yield 'same type' => [type_integer(), type_integer(), 0]; + yield 'optional to required without nulls' => [type_optional(type_integer()), type_integer(), 0]; + yield 'required to optional with nulls' => [type_integer(), type_optional(type_integer()), 3]; + yield 'datetime zone change' => [type_datetime('UTC'), type_datetime('Europe/Warsaw'), 0]; + yield 'same enum class' => [type_enum(BackedIntEnum::class), type_enum(BackedIntEnum::class), 0]; + yield 'same structure element names' => [ + type_structure(['a' => type_integer()]), + type_structure(['a' => type_optional(type_integer())]), + 0, + ]; + } + + /** + * @return Generator, Type, int, string}> + */ + public static function refused(): Generator + { + yield 'optional to required with nulls' => [ + type_optional(type_integer()), + type_integer(), + 2, + '2 null values under a required type', + ]; + yield 'integer to string' => [ + type_integer(), + type_string(), + 0, + 'integer and string are different column kinds', + ]; + yield 'enum class change' => [ + type_enum(BackedIntEnum::class), + type_enum(BackedStringEnum::class), + 0, + 'are different column kinds', + ]; + yield 'structure element rename' => [ + type_structure(['a' => type_integer()]), + type_structure(['b' => type_integer()]), + 0, + 'are different column kinds', + ]; + yield 'structure element reorder' => [ + type_structure(['a' => type_integer(), 'b' => type_integer()]), + type_structure(['b' => type_integer(), 'a' => type_integer()]), + 0, + 'are different column kinds', + ]; + } + + /** + * @param Type $from + * @param Type $to + */ + #[DataProvider('accepted')] + public function test_accepts(Type $from, Type $to, int $nullCount): void + { + (new Retype())->assert($from, $to, $nullCount); + + $this->addToAssertionCount(1); + } + + /** + * @param Type $from + * @param Type $to + */ + #[DataProvider('refused')] + public function test_refuses(Type $from, Type $to, int $nullCount, string $message): void + { + $this->expectException(InvalidArgumentException::class); + $this->expectExceptionMessage($message); + + (new Retype())->assert($from, $to, $nullCount); + } + + /** + * @return Generator, Type, bool}> + */ + public static function proofs(): Generator + { + yield 'same scalar kind' => [type_integer(), type_optional(type_integer()), true]; + yield 'integer to string' => [type_integer(), type_string(), false]; + yield 'enum of another class' => [type_enum(BackedIntEnum::class), type_enum(BackedStringEnum::class), false]; + yield 'structure with a renamed element' => [ + type_structure(['a' => type_integer()]), + type_structure(['b' => type_integer()]), + false, + ]; + yield 'optional list of optional ints to list of ints' => [ + type_optional(type_list(type_optional(type_integer()))), + type_list(type_integer()), + false, + ]; + yield 'list of ints to optional list of optional ints' => [ + type_list(type_integer()), + type_optional(type_list(type_optional(type_integer()))), + true, + ]; + yield 'map value optional to required' => [ + type_map(type_string(), type_optional(type_integer())), + type_map(type_string(), type_integer()), + false, + ]; + yield 'structure element optional to required' => [ + type_structure(['a' => structure_element('a', type_integer(), optional: true)]), + type_structure(['a' => structure_element('a', type_integer())]), + false, + ]; + yield 'structure element type optional to required' => [ + type_structure(['a' => type_optional(type_integer())]), + type_structure(['a' => type_integer()]), + false, + ]; + } + + /** + * @param Type $from + * @param Type $to + */ + #[DataProvider('proofs')] + public function test_proves(Type $from, Type $to, bool $proven): void + { + static::assertSame($proven, (new Retype())->proves($from, $to)); + } +} diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Column/RoundTripPrecisionTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Column/RoundTripPrecisionTest.php new file mode 100644 index 0000000000..c071cd1f28 --- /dev/null +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Column/RoundTripPrecisionTest.php @@ -0,0 +1,71 @@ +force(); + + static::assertSame('-1', ini_get('serialize_precision')); + static::assertSame('0.1', json_encode(0.1)); + + $precision->restore(); + + static::assertSame('17', ini_get('serialize_precision')); + static::assertSame('0.10000000000000001', json_encode(0.1)); + } finally { + ini_set('serialize_precision', $previous); + } + } + + public function test_force_changes_nothing_when_the_ini_is_already_round_trip(): void + { + $previous = (string) ini_get('serialize_precision'); + ini_set('serialize_precision', '-1'); + + try { + $precision = new RoundTripPrecision(); + $precision->force(); + ini_set('serialize_precision', '5'); + $precision->restore(); + + static::assertSame('5', ini_get('serialize_precision')); + } finally { + ini_set('serialize_precision', $previous); + } + } + + public function test_restore_is_applied_once(): void + { + $previous = (string) ini_get('serialize_precision'); + ini_set('serialize_precision', '17'); + + try { + $precision = new RoundTripPrecision(); + $precision->force(); + $precision->restore(); + ini_set('serialize_precision', '9'); + $precision->restore(); + + static::assertSame('9', ini_get('serialize_precision')); + } finally { + ini_set('serialize_precision', $previous); + } + } +} diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Column/TextValuesTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Column/TextValuesTest.php new file mode 100644 index 0000000000..dcf10a83d7 --- /dev/null +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Column/TextValuesTest.php @@ -0,0 +1,592 @@ +, list, list}> + */ + public static function plain_values(): Generator + { + yield 'integer' => [type_integer(), [1, -2], [1, -2]]; + yield 'float' => [type_float(), [1.5, -0.0], [1.5, -0.0]]; + yield 'boolean' => [type_boolean(), [true, false], [true, false]]; + yield 'string' => [type_string(), ['a', ''], ['a', '']]; + yield 'optional string' => [type_optional(type_string()), ['a', null], ['a', null]]; + yield 'datetime' => [type_datetime(), [1_767_323_045_123_456, null], ['2026-01-02T03:04:05+00:00', null]]; + yield 'optional datetime' => [type_optional(type_datetime()), [0, null], ['1970-01-01T00:00:00+00:00', null]]; + yield 'zoned datetime' => [ + type_datetime('Europe/Warsaw'), + [1_767_323_045_000_000], + ['2026-01-02T04:04:05+01:00'], + ]; + yield 'date' => [type_date(), [20_455, null], ['2026-01-02', null]]; + yield 'time' => [type_time(), [3_600_000_000, null], [3_600_000_000, null]]; + yield 'uuid' => [type_uuid(), [self::UUID_BYTES, null], [self::UUID_TEXT, null]]; + yield 'enum' => [type_enum(BasicEnum::class), ['one', null], ['one', null]]; + yield 'time_zone' => [type_time_zone(), ['Europe/Warsaw', null], ['Europe/Warsaw', null]]; + yield 'json' => [ + type_json(), + ['{"a":1}', '[1,2]', '5', null], + [(object) ['a' => 1], [1, 2], 5, null], + ]; + yield 'xml' => [ + type_xml(), + ['' . "\n" . '' . "\n", null], + ['', null], + ]; + yield 'xml node documents' => [ + type_xml(), + [ + XmlDocumentPhysical::DECLARATION . '' . "\n", + XmlDocumentPhysical::DECLARATION . "\n\n", + null, + ], + ['', '', null], + ]; + yield 'xml_element' => [type_xml_element(), ['', null], ['', null]]; + yield 'html' => [type_html(), ['x', null], ['x', null]]; + yield 'html_element' => [type_html_element(), ['

x

', null], ['

x

', null]]; + yield 'null' => [type_null(), [null, null], [null, null]]; + yield 'list' => [ + type_list(type_datetime()), + [[1_767_323_045_000_000, null], null, []], + [['2026-01-02T03:04:05+00:00', null], null, []], + ]; + yield 'list>' => [ + type_list(type_list(type_date())), + [[[20_455], [], null], [[0, 1]]], + [[['2026-01-02'], [], null], [['1970-01-01', '1970-01-02']]], + ]; + yield 'map' => [ + type_map(type_string(), type_uuid()), + [['a' => self::UUID_BYTES, 'b' => null], null], + [(object) ['a' => self::UUID_TEXT, 'b' => null], null], + ]; + yield 'structure with an absent optional element' => [ + type_structure(['day' => type_date(), 'id' => structure_element('id', type_optional(type_uuid()), true)]), + [['day' => 0, 'id' => self::UUID_BYTES], ['day' => 1], ['day' => 2, 'id' => null], null], + [ + (object) ['day' => '1970-01-01', 'id' => self::UUID_TEXT], + (object) ['day' => '1970-01-02'], + (object) ['day' => '1970-01-03', 'id' => null], + null, + ], + ]; + yield 'structure of a list and a map' => [ + type_structure(['l' => type_list(type_date()), 'm' => type_map(type_integer(), type_json())]), + [['l' => [0], 'm' => [5 => '{}']], ['l' => [], 'm' => []]], + [ + (object) ['l' => ['1970-01-01'], 'm' => (object) [5 => (object) []]], + (object) ['l' => [], 'm' => (object) []], + ], + ]; + } + + /** + * @return Generator, list, string}> + */ + public static function shapes(): Generator + { + yield 'a map keyed 0, 1' => [ + type_map(type_integer(), type_string()), + [[0 => 'a', 1 => 'b']], + '[{"0":"a","1":"b"}]', + ]; + yield 'a map keyed 5, 7' => [ + type_map(type_integer(), type_string()), + [[5 => 'a', 7 => 'b']], + '[{"5":"a","7":"b"}]', + ]; + yield 'a map keyed by numeric strings' => [ + type_map(type_string(), type_string()), + [['0' => 'a', '1' => 'b']], + '[{"0":"a","1":"b"}]', + ]; + yield 'an empty map' => [type_map(type_string(), type_integer()), [[]], '[{}]']; + yield 'a structure' => [type_structure(['a' => type_integer()]), [['a' => 1]], '[{"a":1}]']; + yield 'a structure keyed 0, 1' => [ + type_structure([type_integer(), type_integer()]), + [[0 => 1, 1 => 2]], + '[{"0":1,"1":2}]', + ]; + yield 'a list' => [type_list(type_integer()), [[1, 2], []], '[[1,2],[]]']; + yield 'json objects and lists' => [type_json(), ['{}', '[]', '{"a":{}}'], '[{},[],{"a":{}}]']; + } + + /** + * @return Generator, list, list}> + */ + public static function scalar_texts(): Generator + { + yield 'integer' => [type_optional(type_integer()), [1, -2, null], ['1', '-2', null]]; + yield 'float' => [ + type_optional(type_float()), + [ + 1.0, + -0.0, + 1.0e25, + 1.0e-7, + PHP_FLOAT_MAX, + PHP_FLOAT_MIN, + 5.0e-324, + 0.1 + 0.2, + 1 / 3, + 123_456_789_012_345.67, + null, + ], + [ + '1.0', + '-0.0', + '1.0e+25', + '1.0e-7', + '1.7976931348623157e+308', + '2.2250738585072014e-308', + '5.0e-324', + '0.30000000000000004', + '0.3333333333333333', + '123456789012345.67', + null, + ], + ]; + yield 'a float that is not finite' => [type_float(), [NAN, INF, -INF], ['NAN', 'INF', '-INF']]; + yield 'boolean' => [type_optional(type_boolean()), [true, false, null], ['true', 'false', null]]; + yield 'string' => [type_optional(type_string()), ['Norbert', '', null], ['Norbert', '', null]]; + yield 'datetime' => [ + type_optional(type_datetime()), + [1_767_323_045_123_456, null], + ['2026-01-02T03:04:05+00:00', null], + ]; + yield 'zoned datetime' => [ + type_datetime('Europe/Warsaw'), + [1_767_323_045_000_000], + ['2026-01-02T04:04:05+01:00'], + ]; + yield 'date' => [type_optional(type_date()), [20_455, null], ['2026-01-02', null]]; + yield 'time' => [type_optional(type_time()), [3_600_000_000, null], ['3600000000', null]]; + yield 'uuid' => [type_optional(type_uuid()), [self::UUID_BYTES, null], [self::UUID_TEXT, null]]; + yield 'enum' => [type_optional(type_enum(BasicEnum::class)), ['two', null], ['two', null]]; + yield 'time_zone' => [type_optional(type_time_zone()), ['Europe/Warsaw', null], ['Europe/Warsaw', null]]; + yield 'json keeps its stored text' => [ + type_optional(type_json()), + ['{"a": 1}', '[]', '{}', null], + ['{"a": 1}', '[]', '{}', null], + ]; + yield 'xml' => [ + type_optional(type_xml()), + ['' . "\n" . '1' . "\n", null], + ['1', null], + ]; + yield 'xml_element' => [type_xml_element(), [''], ['']]; + yield 'html' => [ + type_html(), + ['x'], + ['x'], + ]; + yield 'html_element' => [type_html_element(), ['

x

'], ['

x

']]; + yield 'null' => [type_null(), [null, null], [null, null]]; + yield 'list' => [ + type_optional(type_list(type_integer())), + [[1, 2, 3], [], null], + ['[1,2,3]', '[]', null], + ]; + yield 'list' => [ + type_list(type_datetime()), + [[1_767_323_045_000_000]], + ['["2026-01-02T03:04:05+00:00"]'], + ]; + yield 'list' => [type_list(type_date()), [[20_455]], ['["2026-01-02"]']]; + yield 'list' => [type_list(type_uuid()), [[self::UUID_BYTES]], ['["' . self::UUID_TEXT . '"]']]; + yield 'list
'], (new TextValues())->bindable($column->type(), $column)); + } + + public function test_bindable_hands_over_every_other_type_as_logical_values(): void + { + $column = ColumnMother::of(datetime_schema('at'), [new DateTimeImmutable('2026-01-02 03:04:05 UTC')]); + + static::assertEquals( + [new DateTimeImmutable('2026-01-02 03:04:05 UTC')], + (new TextValues())->bindable($column->type(), $column), + ); + } +} diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Column/ValueColumnTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Column/ValueColumnTest.php new file mode 100644 index 0000000000..c00ccee30f --- /dev/null +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Column/ValueColumnTest.php @@ -0,0 +1,72 @@ +expectException(InvalidArgumentException::class); + $this->expectExceptionMessage( + 'An untyped function result (mixed) cannot be concatenated; it exists only inside function evaluation', + ); + + (new ValueColumn([1]))->concat(new ValueColumn([2])); + } + + public function test_encode_is_refused(): void + { + $this->expectException(RuntimeException::class); + $this->expectExceptionMessage( + 'An untyped function result (mixed) cannot be encoded; it exists only inside function evaluation', + ); + + (new ValueColumn([1]))->encode(); + } + + public function test_reads_values_as_given(): void + { + $object = new ArrayObject([1]); + $column = new ValueColumn([['a' => 1], null, $object, 'x']); + + static::assertEquals(type_mixed(), $column->type()); + static::assertSame(4, $column->count()); + static::assertSame(1, $column->nullCount()); + static::assertTrue($column->isNull(1)); + static::assertFalse($column->isNull(0)); + static::assertSame(['a' => 1], $column->at(0)); + static::assertSame($object, $column->value(2)); + static::assertSame([['a' => 1], null, $object, 'x'], $column->physicals()); + static::assertSame([['a' => 1], null, $object, 'x'], $column->values()); + } + + public function test_slice_and_take(): void + { + $column = new ValueColumn(['a', null, 'c', 'd']); + + static::assertSame([null, 'c'], $column->slice(1, 2)->values()); + static::assertSame(['d', 'a', 'd'], $column->take([3, 0, 3])->values()); + static::assertSame(0, $column->take([])->count()); + } + + public function test_with_type_is_refused(): void + { + $this->expectException(InvalidArgumentException::class); + $this->expectExceptionMessage( + 'An untyped function result (mixed) cannot be retyped to integer; it exists only inside function evaluation', + ); + + (new ValueColumn([1]))->withType(type_integer()); + } +} diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Comparator/ColumnComparatorTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Comparator/ColumnComparatorTest.php new file mode 100644 index 0000000000..6844146eb4 --- /dev/null +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Comparator/ColumnComparatorTest.php @@ -0,0 +1,65 @@ +accepts($column, $column)); + static::assertFalse((new ColumnComparator())->accepts($column, [1])); + static::assertFalse((new ColumnComparator())->accepts([1], $column)); + } + + public function test_columns_of_both_backends_with_equal_values_pass(): void + { + $comparator = new ColumnComparator(); + $comparator->setFactory(Factory::getInstance()); + $default = (new AdaptiveBackend())->builder(int_schema('a', nullable: true)); + $default->appendMany([1, null]); + + $comparator->assertEquals(ColumnMother::of(int_schema('a', nullable: true), [1, null]), $default->finish()); + + $this->addToAssertionCount(1); + } + + public function test_different_values_fail(): void + { + $comparator = new ColumnComparator(); + $comparator->setFactory(Factory::getInstance()); + + $this->expectException(ComparisonFailure::class); + $this->expectExceptionMessage('Failed asserting that two columns are equal.'); + + $comparator->assertEquals( + (new PhpBackend())->constant(int_schema('a'), 1, 2), + ColumnMother::of(int_schema('a'), [1, 3]), + ); + } + + public function test_different_types_fail(): void + { + $comparator = new ColumnComparator(); + $comparator->setFactory(Factory::getInstance()); + + $this->expectException(ComparisonFailure::class); + $this->expectExceptionMessage('Failed asserting that two columns have the same type.'); + + $comparator->assertEquals(ColumnMother::of(int_schema('a'), [1]), ColumnMother::of(str_schema('a'), ['1'])); + } +} diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Comparator/ComparatorExtensionTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Comparator/ComparatorExtensionTest.php new file mode 100644 index 0000000000..facb943252 --- /dev/null +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Comparator/ComparatorExtensionTest.php @@ -0,0 +1,28 @@ + 1]], schema(int_schema('id'))); + + static::assertInstanceOf(RowsComparator::class, Factory::getInstance()->getComparatorFor($rows, $rows)); + static::assertInstanceOf(ColumnComparator::class, Factory::getInstance()->getComparatorFor( + $rows->column('id'), + $rows->column('id'), + )); + } +} diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Comparator/RowsComparatorTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Comparator/RowsComparatorTest.php new file mode 100644 index 0000000000..e09e47b58d --- /dev/null +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Comparator/RowsComparatorTest.php @@ -0,0 +1,107 @@ + 1]], schema(int_schema('id'))); + + static::assertTrue($comparator->accepts($rows, $rows)); + static::assertFalse($comparator->accepts($rows, ['id' => 1])); + static::assertFalse($comparator->accepts([], $rows)); + } + + public function test_equal_rows_of_two_batches_pass(): void + { + $comparator = new RowsComparator(); + $comparator->setFactory(Factory::getInstance()); + + $comparator->assertEquals( + array_to_rows([['id' => 1], ['id' => 2]], schema(int_schema('id'))), + array_to_rows([['id' => 1], ['id' => 2]], schema(int_schema('id'))), + ); + + $this->addToAssertionCount(1); + } + + public function test_equal_across_column_classes(): void + { + $comparator = new RowsComparator(); + $comparator->setFactory(Factory::getInstance()); + + $comparator->assertEquals( + Rows::fromColumns( + schema(int_schema('id')), + ['id' => (new PhpBackend())->constant(int_schema('id'), 7, 2)], + 2, + ), + array_to_rows([['id' => 7], ['id' => 7]], schema(int_schema('id'))), + ); + + $this->addToAssertionCount(1); + } + + public function test_a_schema_difference_fails(): void + { + $comparator = new RowsComparator(); + $comparator->setFactory(Factory::getInstance()); + + $this->expectException(ComparisonFailure::class); + $this->expectExceptionMessage('Failed asserting that two Rows have the same schema.'); + + $comparator->assertEquals( + array_to_rows([['id' => 1]], schema(int_schema('id'))), + array_to_rows([['id' => 1]], schema(int_schema('id', nullable: true))), + ); + } + + public function test_a_value_difference_fails(): void + { + $comparator = new RowsComparator(); + $comparator->setFactory(Factory::getInstance()); + + try { + $comparator->assertEquals( + array_to_rows([['id' => 1, 'name' => 'a']], schema(int_schema('id'), str_schema('name'))), + array_to_rows([['id' => 1, 'name' => 'b']], schema(int_schema('id'), str_schema('name'))), + ); + } catch (ComparisonFailure $failure) { + static::assertSame('Failed asserting that two Rows are equal.', $failure->getMessage()); + static::assertStringContainsString("'a'", $failure->getExpectedAsString()); + static::assertStringContainsString("'b'", $failure->getActualAsString()); + + return; + } + + static::fail('two Rows with different values compared equal'); + } + + public function test_assert_equals_uses_the_registered_comparator(): void + { + static::assertEquals( + array_to_rows([['id' => 1]], schema(int_schema('id'))), + array_to_rows([['id' => 1]], schema(int_schema('id'))), + ); + static::assertNotEquals( + array_to_rows([['id' => 1]], schema(int_schema('id'))), + array_to_rows([['id' => 2]], schema(int_schema('id'))), + ); + } +} diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Config/Bucketing/BucketingConfigBuilderTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Config/Bucketing/BucketingConfigBuilderTest.php index 0159f8501f..d1a6f6c7f9 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Unit/Config/Bucketing/BucketingConfigBuilderTest.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Config/Bucketing/BucketingConfigBuilderTest.php @@ -6,6 +6,7 @@ use Flow\ETL\Bucketing\Storage\FilesystemBuckets; use Flow\ETL\Bucketing\Storage\MemoryBuckets; +use Flow\ETL\Column\PhpBackend; use Flow\ETL\Config\Bucketing\BucketingConfigBuilder; use Flow\ETL\Exception\InvalidArgumentException; use Flow\ETL\Tests\FlowTestCase; @@ -27,7 +28,7 @@ public function test_buckets_count_and_batch_size_are_set(): void $config = (new BucketingConfigBuilder('/flow-php-join/', 64)) ->bucketsCount(4) ->batchSize(250) - ->build(Path::realpath(__DIR__)); + ->build(Path::realpath(__DIR__), new PhpBackend()); static::assertSame(4, $config->bucketsCount); static::assertSame(250, $config->batchSize); @@ -44,7 +45,7 @@ public function test_buckets_count_below_one_is_rejected(): void public function test_default_builds_a_filesystem_buckets_storage(): void { - $config = (new BucketingConfigBuilder('/flow-php-join/', 64))->build(Path::realpath(__DIR__)); + $config = (new BucketingConfigBuilder('/flow-php-join/', 64))->build(Path::realpath(__DIR__), new PhpBackend()); static::assertInstanceOf(FilesystemBuckets::class, $config->storage); static::assertSame(64, $config->bucketsCount); @@ -53,7 +54,10 @@ public function test_default_builds_a_filesystem_buckets_storage(): void public function test_default_buckets_count_comes_from_the_constructor(): void { - $config = (new BucketingConfigBuilder('/flow-php-sort/', 100))->build(Path::realpath(__DIR__)); + $config = (new BucketingConfigBuilder('/flow-php-sort/', 100))->build( + Path::realpath(__DIR__), + new PhpBackend(), + ); static::assertSame(100, $config->bucketsCount); } @@ -64,7 +68,7 @@ public function test_injected_storage_wins_over_the_default(): void $config = (new BucketingConfigBuilder('/flow-php-join/', 64)) ->storage($storage) - ->build(Path::realpath(__DIR__)); + ->build(Path::realpath(__DIR__), new PhpBackend()); static::assertSame($storage, $config->storage); } diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Config/Grouping/HashGroupByBuilderTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Config/Grouping/HashGroupByBuilderTest.php index 134370e877..d5c0650659 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Unit/Config/Grouping/HashGroupByBuilderTest.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Config/Grouping/HashGroupByBuilderTest.php @@ -6,6 +6,7 @@ use Flow\ETL\Bucketing\Storage\FilesystemBuckets; use Flow\ETL\Bucketing\Storage\MemoryBuckets; +use Flow\ETL\Column\PhpBackend; use Flow\ETL\Tests\FlowTestCase; use Flow\Filesystem\Path; @@ -15,7 +16,7 @@ final class HashGroupByBuilderTest extends FlowTestCase { public function test_bucketing_options_are_set(): void { - $config = hash_group_by()->bucketsCount(4)->batchSize(250)->build(Path::realpath(__DIR__)); + $config = hash_group_by()->bucketsCount(4)->batchSize(250)->build(Path::realpath(__DIR__), new PhpBackend()); static::assertSame(4, $config->bucketing->bucketsCount); static::assertSame(250, $config->bucketing->batchSize); @@ -23,7 +24,7 @@ public function test_bucketing_options_are_set(): void public function test_default_builds_a_filesystem_buckets_storage(): void { - $config = hash_group_by()->build(Path::realpath(__DIR__)); + $config = hash_group_by()->build(Path::realpath(__DIR__), new PhpBackend()); static::assertInstanceOf(FilesystemBuckets::class, $config->bucketing->storage); static::assertSame(64, $config->bucketing->bucketsCount); @@ -36,7 +37,7 @@ public function test_injected_storage_wins_over_the_default(): void static::assertSame( $storage, - hash_group_by()->storage($storage)->build(Path::realpath(__DIR__))->bucketing->storage, + hash_group_by()->storage($storage)->build(Path::realpath(__DIR__), new PhpBackend())->bucketing->storage, ); } } diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Config/Join/HashJoinBuilderTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Config/Join/HashJoinBuilderTest.php index 311f034df7..c18935818e 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Unit/Config/Join/HashJoinBuilderTest.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Config/Join/HashJoinBuilderTest.php @@ -6,6 +6,10 @@ use Flow\ETL\Bucketing\Storage\FilesystemBuckets; use Flow\ETL\Bucketing\Storage\MemoryBuckets; +use Flow\ETL\Column\PhpBackend; +use Flow\ETL\Config\MemoryLimit; +use Flow\ETL\Dataset\Memory\Unit; +use Flow\ETL\Exception\InvalidArgumentException; use Flow\ETL\Tests\FlowTestCase; use Flow\Filesystem\Path; @@ -15,7 +19,7 @@ final class HashJoinBuilderTest extends FlowTestCase { public function test_bucketing_options_are_set(): void { - $config = hash_join()->bucketsCount(4)->batchSize(250)->build(Path::realpath(__DIR__)); + $config = hash_join()->bucketsCount(4)->batchSize(250)->build(Path::realpath(__DIR__), new PhpBackend()); static::assertSame(4, $config->bucketing->bucketsCount); static::assertSame(250, $config->bucketing->batchSize); @@ -23,11 +27,31 @@ public function test_bucketing_options_are_set(): void public function test_default_builds_a_filesystem_buckets_storage(): void { - $config = hash_join()->build(Path::realpath(__DIR__)); + $config = hash_join()->build(Path::realpath(__DIR__), new PhpBackend()); static::assertInstanceOf(FilesystemBuckets::class, $config->bucketing->storage); static::assertSame(64, $config->bucketing->bucketsCount); static::assertSame(1000, $config->bucketing->batchSize); + static::assertSame(MemoryLimit::default()->inBytes(), $config->memoryLimit->inBytes()); + } + + public function test_memory_limit_below_one_byte_is_rejected(): void + { + $this->expectException(InvalidArgumentException::class); + $this->expectExceptionMessage('Join memory limit must be greater than 0 bytes'); + + hash_join()->memoryLimit(Unit::fromBytes(0)); + } + + public function test_memory_limit_is_set(): void + { + static::assertSame( + Unit::fromMb(128)->inBytes(), + hash_join() + ->memoryLimit(Unit::fromMb(128)) + ->build(Path::realpath(__DIR__), new PhpBackend()) + ->memoryLimit->inBytes(), + ); } public function test_injected_storage_wins_over_the_default(): void @@ -36,7 +60,7 @@ public function test_injected_storage_wins_over_the_default(): void static::assertSame( $storage, - hash_join()->storage($storage)->build(Path::realpath(__DIR__))->bucketing->storage, + hash_join()->storage($storage)->build(Path::realpath(__DIR__), new PhpBackend())->bucketing->storage, ); } } diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Config/Sort/ExternalSortBuilderTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Config/Sort/ExternalSortBuilderTest.php index 6af61444be..c550309da9 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Unit/Config/Sort/ExternalSortBuilderTest.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Config/Sort/ExternalSortBuilderTest.php @@ -6,6 +6,9 @@ use Flow\ETL\Bucketing\Storage\FilesystemBuckets; use Flow\ETL\Bucketing\Storage\MemoryBuckets; +use Flow\ETL\Column\PhpBackend; +use Flow\ETL\Config\MemoryLimit; +use Flow\ETL\Dataset\Memory\Unit; use Flow\ETL\Exception\InvalidArgumentException; use Flow\ETL\Tests\FlowTestCase; use Flow\Filesystem\Path; @@ -16,7 +19,7 @@ final class ExternalSortBuilderTest extends FlowTestCase { public function test_bucketing_options_are_set(): void { - $config = external_sort()->bucketsCount(10)->batchSize(250)->build(Path::realpath(__DIR__)); + $config = external_sort()->bucketsCount(10)->batchSize(250)->build(Path::realpath(__DIR__), new PhpBackend()); static::assertSame(10, $config->bucketing->bucketsCount); static::assertSame(250, $config->bucketing->batchSize); @@ -24,24 +27,29 @@ public function test_bucketing_options_are_set(): void public function test_default_builds_a_filesystem_buckets_storage(): void { - $config = external_sort()->build(Path::realpath(__DIR__)); + $config = external_sort()->build(Path::realpath(__DIR__), new PhpBackend()); static::assertInstanceOf(FilesystemBuckets::class, $config->bucketing->storage); static::assertSame(100, $config->bucketing->bucketsCount); static::assertSame(1000, $config->bucketing->batchSize); - static::assertSame(10_000, $config->runSize); + static::assertSame(MemoryLimit::default()->inBytes(), $config->memoryLimit->inBytes()); } public function test_merge_storage_defaults_to_null_so_the_spill_storage_is_reused(): void { - static::assertNull(external_sort()->storage(new MemoryBuckets())->build(Path::realpath(__DIR__))->merge); + static::assertNull( + external_sort()->storage(new MemoryBuckets())->build(Path::realpath(__DIR__), new PhpBackend())->merge, + ); } public function test_merge_storage_is_carried_into_the_config(): void { $merge = new MemoryBuckets(); - static::assertSame($merge, external_sort()->mergeStorage($merge)->build(Path::realpath(__DIR__))->merge); + static::assertSame( + $merge, + external_sort()->mergeStorage($merge)->build(Path::realpath(__DIR__), new PhpBackend())->merge, + ); } public function test_injected_storage_wins_over_the_default(): void @@ -50,21 +58,26 @@ public function test_injected_storage_wins_over_the_default(): void static::assertSame( $storage, - external_sort()->storage($storage)->build(Path::realpath(__DIR__))->bucketing->storage, + external_sort()->storage($storage)->build(Path::realpath(__DIR__), new PhpBackend())->bucketing->storage, ); } - public function test_run_size_below_one_is_rejected(): void + public function test_memory_limit_below_one_byte_is_rejected(): void { $this->expectException(InvalidArgumentException::class); - $this->expectExceptionMessage('Run size must be greater than 0'); + $this->expectExceptionMessage('Sort memory limit must be greater than 0 bytes'); - // @mago-ignore analysis:invalid-argument - external_sort()->runSize(0); + external_sort()->memoryLimit(Unit::fromBytes(0)); } - public function test_run_size_is_set(): void + public function test_memory_limit_is_set(): void { - static::assertSame(500, external_sort()->runSize(500)->build(Path::realpath(__DIR__))->runSize); + static::assertSame( + Unit::fromMb(500)->inBytes(), + external_sort() + ->memoryLimit(Unit::fromMb(500)) + ->build(Path::realpath(__DIR__), new PhpBackend()) + ->memoryLimit->inBytes(), + ); } } diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Config/Sort/MemorySortBuilderTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Config/Sort/MemorySortBuilderTest.php index 751df697b4..1343b647d9 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Unit/Config/Sort/MemorySortBuilderTest.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Config/Sort/MemorySortBuilderTest.php @@ -4,6 +4,7 @@ namespace Flow\ETL\Tests\Unit\Config\Sort; +use Flow\ETL\Column\PhpBackend; use Flow\ETL\Config\Sort\MemorySortConfig; use Flow\ETL\Tests\FlowTestCase; use Flow\Filesystem\Path; @@ -14,6 +15,9 @@ final class MemorySortBuilderTest extends FlowTestCase { public function test_builds_a_memory_sort_config(): void { - static::assertInstanceOf(MemorySortConfig::class, memory_sort()->build(Path::realpath(__DIR__))); + static::assertInstanceOf(MemorySortConfig::class, memory_sort()->build( + Path::realpath(__DIR__), + new PhpBackend(), + )); } } diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Config/Telemetry/TelemetryContextTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Config/Telemetry/TelemetryContextTest.php index 64da37b0d7..1ce41a6ec2 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Unit/Config/Telemetry/TelemetryContextTest.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Config/Telemetry/TelemetryContextTest.php @@ -5,6 +5,7 @@ namespace Flow\ETL\Tests\Unit\Config\Telemetry; use DateTimeImmutable; +use Flow\ETL\Column\PhpBackend; use Flow\ETL\Config\Telemetry\TelemetryContext; use Flow\ETL\Config\Telemetry\TelemetryOptions; use Flow\ETL\Loader\StreamLoader; @@ -34,13 +35,12 @@ use function array_map; use function count; +use function Flow\ETL\DSL\array_to_rows; use function Flow\ETL\DSL\config_builder; use function Flow\ETL\DSL\flow_context; use function Flow\ETL\DSL\int_schema; use function Flow\ETL\DSL\lit; use function Flow\ETL\DSL\ref; -use function Flow\ETL\DSL\row; -use function Flow\ETL\DSL\rows; use function Flow\ETL\DSL\schema; use function Flow\ETL\DSL\telemetry_options; use function Flow\ETL\DSL\to_array; @@ -61,10 +61,9 @@ public function test_dataframe_batch_processed_drains_abandoned_spans(): void $context->telemetryContext->transformationStarted($inner); $context->telemetryContext->transformationCompleted($inner); - $context->telemetryContext->dataFrameBatchProcessed( - rows(schema(int_schema('id')), row(['id' => 1])), - $context->flowContext, - ); + $context->telemetryContext->dataFrameBatchProcessed(array_to_rows([[ + 'id' => 1, + ]], schema(int_schema('id'))), $context->flowContext); $endedSpans = $context->spans->endedSpans(); @@ -96,6 +95,7 @@ public function test_dataframe_batch_processed_tracks_rows_and_memory(): void ); $telemetryContext = new TelemetryContext( + new PhpBackend(), $telemetry->logger('flow-php'), $telemetry->tracer('flow-php'), $telemetry->meter('flow-php'), @@ -108,7 +108,7 @@ public function test_dataframe_batch_processed_tracks_rows_and_memory(): void $telemetryContext->dataFrameStarted($context); - $rows = rows(schema(int_schema('id')), row(['id' => 1]), row(['id' => 2]), row(['id' => 3])); + $rows = array_to_rows([['id' => 1], ['id' => 2], ['id' => 3]], schema(int_schema('id'))); $telemetryContext->dataFrameBatchProcessed($rows, $context); @@ -168,6 +168,7 @@ public function test_dataframe_completed_finalizes_span_with_statistics(): void ); $telemetryContext = new TelemetryContext( + new PhpBackend(), $telemetry->logger('flow-php'), $telemetry->tracer('flow-php'), $telemetry->meter('flow-php'), @@ -180,7 +181,7 @@ public function test_dataframe_completed_finalizes_span_with_statistics(): void $telemetryContext->dataFrameStarted($context); - $rows = rows(schema(int_schema('id')), row(['id' => 1]), row(['id' => 2])); + $rows = array_to_rows([['id' => 1], ['id' => 2]], schema(int_schema('id'))); $telemetryContext->dataFrameBatchProcessed($rows, $context); $telemetryContext->dataFrameCompleted($context); @@ -244,6 +245,7 @@ public function test_dataframe_failed_logs_error_and_sets_span_status(): void ); $telemetryContext = new TelemetryContext( + new PhpBackend(), $telemetry->logger('flow-php'), $telemetry->tracer('flow-php'), $telemetry->meter('flow-php'), @@ -256,7 +258,7 @@ public function test_dataframe_failed_logs_error_and_sets_span_status(): void $telemetryContext->dataFrameStarted($context); - $rows = rows(schema(int_schema('id')), row(['id' => 1])); + $rows = array_to_rows([['id' => 1]], schema(int_schema('id'))); $telemetryContext->dataFrameBatchProcessed($rows, $context); $exception = new RuntimeException('Processing failed due to invalid data'); @@ -295,6 +297,7 @@ public function test_dataframe_started_creates_span_and_logs_debug_message(): vo ); $telemetryContext = new TelemetryContext( + new PhpBackend(), $telemetry->logger('flow-php'), $telemetry->tracer('flow-php'), $telemetry->meter('flow-php'), @@ -354,6 +357,7 @@ public function test_loading_completed_finalizes_span_with_ok_status(): void ); $telemetryContext = new TelemetryContext( + new PhpBackend(), $telemetry->logger('flow-php'), $telemetry->tracer('flow-php'), $telemetry->meter('flow-php'), @@ -420,6 +424,7 @@ public function test_loading_failed_logs_error_and_sets_span_status(): void ); $telemetryContext = new TelemetryContext( + new PhpBackend(), $telemetry->logger('flow-php'), $telemetry->tracer('flow-php'), $telemetry->meter('flow-php'), @@ -492,6 +497,7 @@ public function test_loading_started_creates_span_when_trace_loading_enabled(): ); $telemetryContext = new TelemetryContext( + new PhpBackend(), $telemetry->logger('flow-php'), $telemetry->tracer('flow-php'), $telemetry->meter('flow-php'), @@ -529,6 +535,7 @@ public function test_loading_started_does_not_create_span_when_trace_loading_dis ); $telemetryContext = new TelemetryContext( + new PhpBackend(), $telemetry->logger('flow-php'), $telemetry->tracer('flow-php'), $telemetry->meter('flow-php'), @@ -565,6 +572,7 @@ public function test_metrics_collected_when_collect_metrics_enabled(): void ); $telemetryContext = new TelemetryContext( + new PhpBackend(), $telemetry->logger('flow-php'), $telemetry->tracer('flow-php'), $telemetry->meter('flow-php'), @@ -577,7 +585,7 @@ public function test_metrics_collected_when_collect_metrics_enabled(): void $telemetryContext->dataFrameStarted($context); - $rows = rows(schema(int_schema('id')), row(['id' => 1]), row(['id' => 2])); + $rows = array_to_rows([['id' => 1], ['id' => 2]], schema(int_schema('id'))); $telemetryContext->dataFrameBatchProcessed($rows, $context); $telemetryContext->dataFrameCompleted($context); $telemetry->flush(); @@ -605,6 +613,7 @@ public function test_metrics_include_dataframe_name_attribute(): void ); $telemetryContext = new TelemetryContext( + new PhpBackend(), $telemetry->logger('flow-php'), $telemetry->tracer('flow-php'), $telemetry->meter('flow-php'), @@ -620,7 +629,7 @@ public function test_metrics_include_dataframe_name_attribute(): void $telemetryContext->dataFrameStarted($context); - $rows = rows(schema(int_schema('id')), row(['id' => 1]), row(['id' => 2]), row(['id' => 3])); + $rows = array_to_rows([['id' => 1], ['id' => 2], ['id' => 3]], schema(int_schema('id'))); $telemetryContext->dataFrameBatchProcessed($rows, $context); $telemetryContext->dataFrameCompleted($context); $telemetry->flush(); @@ -654,6 +663,7 @@ public function test_metrics_not_collected_when_collect_metrics_disabled(): void ); $telemetryContext = new TelemetryContext( + new PhpBackend(), $telemetry->logger('flow-php'), $telemetry->tracer('flow-php'), $telemetry->meter('flow-php'), @@ -666,7 +676,7 @@ public function test_metrics_not_collected_when_collect_metrics_disabled(): void $telemetryContext->dataFrameStarted($context); - $rows = rows(schema(int_schema('id')), row(['id' => 1])); + $rows = array_to_rows([['id' => 1]], schema(int_schema('id'))); $telemetryContext->dataFrameBatchProcessed($rows, $context); $telemetryContext->dataFrameCompleted($context); $telemetry->flush(); @@ -737,6 +747,7 @@ public function test_transformation_completed_finalizes_span(): void ); $telemetryContext = new TelemetryContext( + new PhpBackend(), $telemetry->logger('flow-php'), $telemetry->tracer('flow-php'), $telemetry->meter('flow-php'), @@ -803,6 +814,7 @@ public function test_transformation_failed_logs_error_and_sets_span_status(): vo ); $telemetryContext = new TelemetryContext( + new PhpBackend(), $telemetry->logger('flow-php'), $telemetry->tracer('flow-php'), $telemetry->meter('flow-php'), @@ -874,6 +886,7 @@ public function test_transformation_started_creates_span_when_trace_transformati ); $telemetryContext = new TelemetryContext( + new PhpBackend(), $telemetry->logger('flow-php'), $telemetry->tracer('flow-php'), $telemetry->meter('flow-php'), @@ -911,6 +924,7 @@ public function test_transformation_started_does_not_create_span_when_trace_tran ); $telemetryContext = new TelemetryContext( + new PhpBackend(), $telemetry->logger('flow-php'), $telemetry->tracer('flow-php'), $telemetry->meter('flow-php'), @@ -959,6 +973,7 @@ public function test_optimizer_rules_are_logged(): void ); $telemetryContext = new TelemetryContext( + new PhpBackend(), $telemetry->logger('flow-php'), $telemetry->tracer('flow-php'), $telemetry->meter('flow-php'), diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Constraint/SortedConstraintTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Constraint/SortedConstraintTest.php index 7548605d0c..78a1677b9f 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Unit/Constraint/SortedConstraintTest.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Constraint/SortedConstraintTest.php @@ -6,16 +6,19 @@ use DateTimeImmutable; use Flow\ETL\Constraint\SortedByConstraint; +use Flow\ETL\Row\NullsOrder; use Flow\ETL\Tests\FlowTestCase; -use function Flow\ETL\DSL\date_schema; +use function Flow\ETL\DSL\array_to_rows; +use function Flow\ETL\DSL\datetime_schema; use function Flow\ETL\DSL\float_schema; use function Flow\ETL\DSL\int_schema; use function Flow\ETL\DSL\ref; -use function Flow\ETL\DSL\row; use function Flow\ETL\DSL\schema; use function Flow\ETL\DSL\str_schema; +use const NAN; + final class SortedConstraintTest extends FlowTestCase { public function test_sorted_constraint_as_a_string(): void @@ -29,78 +32,140 @@ public function test_sorted_constraint_ascending_integers(): void { $constraint = new SortedByConstraint(ref('id')->asc()); - static::assertTrue($constraint->isSatisfiedBy(row(['id' => 1]), schema(int_schema('id')))); - static::assertTrue($constraint->isSatisfiedBy(row(['id' => 2]), schema(int_schema('id')))); - static::assertTrue($constraint->isSatisfiedBy(row(['id' => 3]), schema(int_schema('id')))); - static::assertTrue($constraint->isSatisfiedBy(row(['id' => 3]), schema(int_schema('id')))); - static::assertFalse($constraint->isSatisfiedBy(row(['id' => 2]), schema(int_schema('id')))); + static::assertNull($constraint->firstViolation(array_to_rows([[ + 'id' => 1, + ]], schema(int_schema('id'))))); + static::assertNull($constraint->firstViolation(array_to_rows([[ + 'id' => 2, + ]], schema(int_schema('id'))))); + static::assertNull($constraint->firstViolation(array_to_rows([[ + 'id' => 3, + ]], schema(int_schema('id'))))); + static::assertNull($constraint->firstViolation(array_to_rows([[ + 'id' => 3, + ]], schema(int_schema('id'))))); + static::assertSame( + 0, + $constraint->firstViolation(array_to_rows([[ + 'id' => 2, + ]], schema(int_schema('id')))), + ); } public function test_sorted_constraint_ascending_strings(): void { $constraint = new SortedByConstraint(ref('name')->asc()); - static::assertTrue($constraint->isSatisfiedBy(row(['name' => 'Alice']), schema(str_schema('name')))); - static::assertTrue($constraint->isSatisfiedBy(row(['name' => 'Bob']), schema(str_schema('name')))); - static::assertTrue($constraint->isSatisfiedBy(row(['name' => 'Charlie']), schema(str_schema('name')))); - static::assertFalse($constraint->isSatisfiedBy(row(['name' => 'Alice']), schema(str_schema('name')))); + static::assertNull($constraint->firstViolation(array_to_rows([[ + 'name' => 'Alice', + ]], schema(str_schema('name'))))); + static::assertNull($constraint->firstViolation(array_to_rows([[ + 'name' => 'Bob', + ]], schema(str_schema('name'))))); + static::assertNull($constraint->firstViolation(array_to_rows([[ + 'name' => 'Charlie', + ]], schema(str_schema('name'))))); + static::assertSame( + 0, + $constraint->firstViolation(array_to_rows([[ + 'name' => 'Alice', + ]], schema(str_schema('name')))), + ); } public function test_sorted_constraint_default_order_is_ascending(): void { $constraint = new SortedByConstraint(ref('id')); - static::assertTrue($constraint->isSatisfiedBy(row(['id' => 1]), schema(int_schema('id')))); - static::assertTrue($constraint->isSatisfiedBy(row(['id' => 2]), schema(int_schema('id')))); - static::assertTrue($constraint->isSatisfiedBy(row(['id' => 3]), schema(int_schema('id')))); - static::assertFalse($constraint->isSatisfiedBy(row(['id' => 1]), schema(int_schema('id')))); + static::assertNull($constraint->firstViolation(array_to_rows([[ + 'id' => 1, + ]], schema(int_schema('id'))))); + static::assertNull($constraint->firstViolation(array_to_rows([[ + 'id' => 2, + ]], schema(int_schema('id'))))); + static::assertNull($constraint->firstViolation(array_to_rows([[ + 'id' => 3, + ]], schema(int_schema('id'))))); + static::assertSame( + 0, + $constraint->firstViolation(array_to_rows([[ + 'id' => 1, + ]], schema(int_schema('id')))), + ); } public function test_sorted_constraint_descending_floats(): void { $constraint = new SortedByConstraint(ref('price')->desc()); - static::assertTrue($constraint->isSatisfiedBy(row(['price' => 99.99]), schema(float_schema('price')))); - static::assertTrue($constraint->isSatisfiedBy(row(['price' => 49.99]), schema(float_schema('price')))); - static::assertTrue($constraint->isSatisfiedBy(row(['price' => 19.99]), schema(float_schema('price')))); - static::assertTrue($constraint->isSatisfiedBy(row(['price' => 19.99]), schema(float_schema('price')))); - static::assertFalse($constraint->isSatisfiedBy(row(['price' => 29.99]), schema(float_schema('price')))); + static::assertNull($constraint->firstViolation(array_to_rows([[ + 'price' => 99.99, + ]], schema(float_schema('price'))))); + static::assertNull($constraint->firstViolation(array_to_rows([[ + 'price' => 49.99, + ]], schema(float_schema('price'))))); + static::assertNull($constraint->firstViolation(array_to_rows([[ + 'price' => 19.99, + ]], schema(float_schema('price'))))); + static::assertNull($constraint->firstViolation(array_to_rows([[ + 'price' => 19.99, + ]], schema(float_schema('price'))))); + static::assertSame( + 0, + $constraint->firstViolation(array_to_rows([[ + 'price' => 29.99, + ]], schema(float_schema('price')))), + ); } public function test_sorted_constraint_descending_integers(): void { $constraint = new SortedByConstraint(ref('id')->desc()); - static::assertTrue($constraint->isSatisfiedBy(row(['id' => 10]), schema(int_schema('id')))); - static::assertTrue($constraint->isSatisfiedBy(row(['id' => 5]), schema(int_schema('id')))); - static::assertTrue($constraint->isSatisfiedBy(row(['id' => 1]), schema(int_schema('id')))); - static::assertFalse($constraint->isSatisfiedBy(row(['id' => 3]), schema(int_schema('id')))); + static::assertNull($constraint->firstViolation(array_to_rows([[ + 'id' => 10, + ]], schema(int_schema('id'))))); + static::assertNull($constraint->firstViolation(array_to_rows([[ + 'id' => 5, + ]], schema(int_schema('id'))))); + static::assertNull($constraint->firstViolation(array_to_rows([[ + 'id' => 1, + ]], schema(int_schema('id'))))); + static::assertSame( + 0, + $constraint->firstViolation(array_to_rows([[ + 'id' => 3, + ]], schema(int_schema('id')))), + ); } public function test_sorted_constraint_multiple_columns(): void { $constraint = new SortedByConstraint(ref('category')->asc(), ref('price')->desc()); - static::assertTrue($constraint->isSatisfiedBy( - row(['category' => 'A', 'price' => 100.0]), - schema(str_schema('category'), float_schema('price')), - )); - static::assertTrue($constraint->isSatisfiedBy( - row(['category' => 'A', 'price' => 50.0]), + static::assertNull($constraint->firstViolation(array_to_rows( + [['category' => 'A', 'price' => 100.0]], schema(str_schema('category'), float_schema('price')), - )); - static::assertTrue($constraint->isSatisfiedBy( - row(['category' => 'B', 'price' => 200.0]), + ))); + static::assertNull($constraint->firstViolation(array_to_rows( + [['category' => 'A', 'price' => 50.0]], schema(str_schema('category'), float_schema('price')), - )); - static::assertTrue($constraint->isSatisfiedBy( - row(['category' => 'B', 'price' => 150.0]), + ))); + static::assertNull($constraint->firstViolation(array_to_rows( + [['category' => 'B', 'price' => 200.0]], schema(str_schema('category'), float_schema('price')), - )); - static::assertFalse($constraint->isSatisfiedBy( - row(['category' => 'B', 'price' => 250.0]), + ))); + static::assertNull($constraint->firstViolation(array_to_rows( + [['category' => 'B', 'price' => 150.0]], schema(str_schema('category'), float_schema('price')), - )); + ))); + static::assertSame( + 0, + $constraint->firstViolation(array_to_rows( + [['category' => 'B', 'price' => 250.0]], + schema(str_schema('category'), float_schema('price')), + )), + ); } public function test_sorted_constraint_multiple_columns_as_a_string(): void @@ -113,10 +178,12 @@ public function test_sorted_constraint_multiple_columns_as_a_string(): void public function test_sorted_constraint_violation_ascending(): void { $constraint = new SortedByConstraint(ref('id')->asc()); - $constraint->isSatisfiedBy(row(['id' => 5]), schema(int_schema('id'))); - $constraint->isSatisfiedBy(row(['id' => 10]), schema(int_schema('id'))); + $constraint->firstViolation(array_to_rows([['id' => 5]], schema(int_schema('id')))); + $constraint->firstViolation(array_to_rows([['id' => 10]], schema(int_schema('id')))); - $violation = $constraint->violation(row(['id' => 3]), schema(int_schema('id'))); + $violation = $constraint->violation(array_to_rows([[ + 'id' => 3, + ]], schema(int_schema('id'))), 0); static::assertStringContainsString('expected ASC order', $violation); static::assertStringContainsString('id', $violation); @@ -127,10 +194,12 @@ public function test_sorted_constraint_violation_ascending(): void public function test_sorted_constraint_violation_descending(): void { $constraint = new SortedByConstraint(ref('id')->desc()); - $constraint->isSatisfiedBy(row(['id' => 10]), schema(int_schema('id'))); - $constraint->isSatisfiedBy(row(['id' => 5]), schema(int_schema('id'))); + $constraint->firstViolation(array_to_rows([['id' => 10]], schema(int_schema('id')))); + $constraint->firstViolation(array_to_rows([['id' => 5]], schema(int_schema('id')))); - $violation = $constraint->violation(row(['id' => 8]), schema(int_schema('id'))); + $violation = $constraint->violation(array_to_rows([[ + 'id' => 8, + ]], schema(int_schema('id'))), 0); static::assertStringContainsString('expected DESC order', $violation); static::assertStringContainsString('id', $violation); @@ -142,24 +211,127 @@ public function test_sorted_constraint_with_dates(): void { $constraint = new SortedByConstraint(ref('date')->asc()); - static::assertTrue($constraint->isSatisfiedBy(row([ + static::assertNull($constraint->firstViolation(array_to_rows([[ 'date' => new DateTimeImmutable('2025-01-01'), - ]), schema(date_schema('date')))); - static::assertTrue($constraint->isSatisfiedBy(row([ + ]], schema(datetime_schema('date'))))); + static::assertNull($constraint->firstViolation(array_to_rows([[ 'date' => new DateTimeImmutable('2025-01-02'), - ]), schema(date_schema('date')))); - static::assertTrue($constraint->isSatisfiedBy(row([ + ]], schema(datetime_schema('date'))))); + static::assertNull($constraint->firstViolation(array_to_rows([[ 'date' => new DateTimeImmutable('2025-01-03'), - ]), schema(date_schema('date')))); - static::assertFalse($constraint->isSatisfiedBy(row([ - 'date' => new DateTimeImmutable('2025-01-01'), - ]), schema(date_schema('date')))); + ]], schema(datetime_schema('date'))))); + static::assertSame( + 0, + $constraint->firstViolation(array_to_rows([[ + 'date' => new DateTimeImmutable('2025-01-01'), + ]], schema(datetime_schema('date')))), + ); } public function test_sorted_constraint_with_single_row(): void { $constraint = new SortedByConstraint(ref('id')->asc()); - static::assertTrue($constraint->isSatisfiedBy(row(['id' => 42]), schema(int_schema('id')))); + static::assertNull($constraint->firstViolation(array_to_rows([[ + 'id' => 42, + ]], schema(int_schema('id'))))); + } + + public function test_first_violation_carries_the_previous_batch_last_row(): void + { + $constraint = new SortedByConstraint(ref('id')->asc()); + + static::assertNull($constraint->firstViolation(array_to_rows([ + ['id' => 1], + ['id' => 5], + ], schema(int_schema('id'))))); + + $batch = array_to_rows([['id' => 4], ['id' => 6]], schema(int_schema('id'))); + + static::assertSame(0, $constraint->firstViolation($batch)); + static::assertStringContainsString('current: 4, previous: 5', $constraint->violation($batch, 0)); + } + + public function test_first_violation_stops_advancing_at_the_violation(): void + { + $constraint = new SortedByConstraint(ref('id')->asc()); + $batch = array_to_rows([['id' => 1], ['id' => 3], ['id' => 2], ['id' => 9]], schema(int_schema('id'))); + + static::assertSame(2, $constraint->firstViolation($batch)); + static::assertStringContainsString('current: 2, previous: 3', $constraint->violation($batch, 2)); + } + + public function test_an_empty_batch_has_no_violation(): void + { + static::assertNull((new SortedByConstraint(ref('id')))->firstViolation(array_to_rows( + [], + schema(int_schema('id')), + ))); + } + + public function test_datetimes_are_compared_by_instant(): void + { + $constraint = new SortedByConstraint(ref('at')->asc()); + + static::assertSame( + 2, + $constraint->firstViolation(array_to_rows([ + ['at' => new DateTimeImmutable('2024-01-01 00:00:00.000001')], + ['at' => new DateTimeImmutable('2024-01-01 00:00:00.000002')], + ['at' => new DateTimeImmutable('2024-01-01 00:00:00.000001')], + ], schema(datetime_schema('at')))), + ); + } + + public function test_strings_are_in_byte_order_never_numeric(): void + { + static::assertSame( + 1, + (new SortedByConstraint(ref('code')->asc()))->firstViolation(array_to_rows([ + ['code' => '9'], + ['code' => '10'], + ], schema(str_schema('code')))), + ); + static::assertNull((new SortedByConstraint(ref('code')->asc()))->firstViolation(array_to_rows([ + ['code' => '10'], + ['code' => '9'], + ], schema(str_schema('code'))))); + } + + public function test_nulls_come_first_ascending_and_nan_last(): void + { + static::assertNull((new SortedByConstraint(ref('v')->asc()))->firstViolation(array_to_rows([ + ['v' => null], + ['v' => -1.5], + ['v' => 2.0], + ['v' => NAN], + ['v' => NAN], + ], schema(float_schema('v', true))))); + static::assertSame( + 1, + (new SortedByConstraint(ref('v')->desc()))->firstViolation(array_to_rows([ + ['v' => null], + ['v' => NAN], + ], schema(float_schema('v', true)))), + ); + } + + public function test_a_moved_null_placement_is_part_of_the_order(): void + { + $constraint = new SortedByConstraint(ref('v')->asc(NullsOrder::LAST)); + + static::assertSame('Sorted constraint on [v ASC NULLS LAST]', $constraint->toString()); + static::assertNull($constraint->firstViolation(array_to_rows([ + ['v' => 1], + ['v' => 2], + ['v' => null], + ], schema(int_schema('v', true))))); + static::assertSame( + 1, + (new SortedByConstraint(ref('v')->asc(NullsOrder::LAST)))->firstViolation(array_to_rows([ + ['v' => null], + ['v' => 1], + ], schema(int_schema('v', true)))), + ); } } diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Constraint/UniqueConstraintTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Constraint/UniqueConstraintTest.php index b9fef7efe1..9b72c33f81 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Unit/Constraint/UniqueConstraintTest.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Constraint/UniqueConstraintTest.php @@ -8,9 +8,9 @@ use Flow\ETL\Constraint\UniqueConstraint; use Flow\ETL\Tests\FlowTestCase; +use function Flow\ETL\DSL\array_to_rows; use function Flow\ETL\DSL\date_schema; use function Flow\ETL\DSL\int_schema; -use function Flow\ETL\DSL\row; use function Flow\ETL\DSL\schema; final class UniqueConstraintTest extends FlowTestCase @@ -26,31 +26,43 @@ public function test_unique_constraint_on_multiple_columns(): void { $constraint = new UniqueConstraint('id', 'sub_id'); - static::assertTrue($constraint->isSatisfiedBy( - row(['id' => 1, 'sub_id' => 1]), + static::assertNull($constraint->firstViolation(array_to_rows( + [['id' => 1, 'sub_id' => 1]], schema(int_schema('id'), int_schema('sub_id')), - )); - static::assertTrue($constraint->isSatisfiedBy( - row(['id' => 1, 'sub_id' => 2]), + ))); + static::assertNull($constraint->firstViolation(array_to_rows( + [['id' => 1, 'sub_id' => 2]], schema(int_schema('id'), int_schema('sub_id')), - )); - static::assertTrue($constraint->isSatisfiedBy( - row(['id' => 2, 'sub_id' => 1]), + ))); + static::assertNull($constraint->firstViolation(array_to_rows( + [['id' => 2, 'sub_id' => 1]], schema(int_schema('id'), int_schema('sub_id')), - )); - static::assertFalse($constraint->isSatisfiedBy( - row(['id' => 1, 'sub_id' => 1]), - schema(int_schema('id'), int_schema('sub_id')), - )); + ))); + static::assertSame( + 0, + $constraint->firstViolation(array_to_rows( + [['id' => 1, 'sub_id' => 1]], + schema(int_schema('id'), int_schema('sub_id')), + )), + ); } public function test_unique_constraint_on_single_column(): void { $constraint = new UniqueConstraint('id'); - static::assertTrue($constraint->isSatisfiedBy(row(['id' => 1]), schema(int_schema('id')))); - static::assertTrue($constraint->isSatisfiedBy(row(['id' => 2]), schema(int_schema('id')))); - static::assertFalse($constraint->isSatisfiedBy(row(['id' => 1]), schema(int_schema('id')))); + static::assertNull($constraint->firstViolation(array_to_rows([[ + 'id' => 1, + ]], schema(int_schema('id'))))); + static::assertNull($constraint->firstViolation(array_to_rows([[ + 'id' => 2, + ]], schema(int_schema('id'))))); + static::assertSame( + 0, + $constraint->firstViolation(array_to_rows([[ + 'id' => 1, + ]], schema(int_schema('id')))), + ); } public function test_unique_constraint_violation(): void @@ -58,8 +70,8 @@ public function test_unique_constraint_violation(): void $constraint = new UniqueConstraint('id', 'sub_id'); static::assertSame('Values: [id = 1, sub_id = 1]', $constraint->violation( - row(['id' => 1, 'sub_id' => 1]), - schema(int_schema('id'), int_schema('sub_id')), + array_to_rows([['id' => 1, 'sub_id' => 1]], schema(int_schema('id'), int_schema('sub_id'))), + 0, )); } @@ -67,8 +79,43 @@ public function test_unique_constraint_violation_on_dates(): void { $constraint = new UniqueConstraint('date'); - static::assertSame('Values: [date = 2025-01-01]', $constraint->violation(row([ + static::assertSame('Values: [date = 2025-01-01]', $constraint->violation(array_to_rows([[ 'date' => new DateTimeImmutable('2025-01-01'), - ]), schema(date_schema('date')))); + ]], schema(date_schema('date'))), 0)); + } + + public function test_first_violation_is_the_first_repeated_key_across_batches(): void + { + $constraint = new UniqueConstraint('id'); + + static::assertNull($constraint->firstViolation(array_to_rows([ + ['id' => 1], + ['id' => 2], + ], schema(int_schema('id'))))); + static::assertSame( + 1, + $constraint->firstViolation(array_to_rows([ + ['id' => 3], + ['id' => 2], + ['id' => 3], + ], schema(int_schema('id')))), + ); + } + + public function test_first_violation_within_one_batch(): void + { + static::assertSame( + 2, + (new UniqueConstraint('id'))->firstViolation(array_to_rows([ + ['id' => 1], + ['id' => 2], + ['id' => 1], + ], schema(int_schema('id')))), + ); + } + + public function test_an_empty_batch_has_no_violation(): void + { + static::assertNull((new UniqueConstraint('id'))->firstViolation(array_to_rows([], schema(int_schema('id'))))); } } diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/DataFrameTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/DataFrameTest.php index dc403af5a5..e9e50824f2 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Unit/DataFrameTest.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/DataFrameTest.php @@ -23,7 +23,6 @@ use Flow\ETL\Plan\Node\Read; use Flow\ETL\Plan\Stage; use Flow\ETL\Plan\Trigger; -use Flow\ETL\Row\RowRenaming; use Flow\ETL\Rows; use Flow\ETL\Schema; use Flow\ETL\Schema\Validator\SelectiveValidator; @@ -56,6 +55,7 @@ use function array_column; use function array_merge; use function Flow\ETL\DSL\add_row_index; +use function Flow\ETL\DSL\array_to_rows; use function Flow\ETL\DSL\average; use function Flow\ETL\DSL\bool_schema; use function Flow\ETL\DSL\config_builder; @@ -76,8 +76,6 @@ use function Flow\ETL\DSL\lit; use function Flow\ETL\DSL\ref; use function Flow\ETL\DSL\refs; -use function Flow\ETL\DSL\row; -use function Flow\ETL\DSL\rows; use function Flow\ETL\DSL\schema; use function Flow\ETL\DSL\select; use function Flow\ETL\DSL\str_schema; @@ -186,21 +184,25 @@ public function test_count_still_writes_the_frames_sinks(): void public function test_drop(): void { $rows = df() - ->process(rows( + ->process(array_to_rows( + [ + ['id' => 1, 'name' => 'foo', 'active' => true], + ['id' => 2, 'name' => null, 'active' => false], + ['id' => 2, 'name' => 'bar', 'active' => false], + ], schema(int_schema('id'), str_schema('name', nullable: true), bool_schema('active')), - row(['id' => 1, 'name' => 'foo', 'active' => true]), - row(['id' => 2, 'name' => null, 'active' => false]), - row(['id' => 2, 'name' => 'bar', 'active' => false]), )) ->drop('id') ->fetch(); static::assertEquals( - rows( + array_to_rows( + [ + ['name' => 'foo', 'active' => true], + ['name' => null, 'active' => false], + ['name' => 'bar', 'active' => false], + ], schema(str_schema('name', nullable: true), bool_schema('active')), - row(['name' => 'foo', 'active' => true]), - row(['name' => null, 'active' => false]), - row(['name' => 'bar', 'active' => false]), ), $rows, ); @@ -209,20 +211,21 @@ public function test_drop(): void public function test_drop_duplicates(): void { $rows = df() - ->process(rows( + ->process(array_to_rows( + [ + ['id' => 1, 'name' => 'foo', 'active' => true], + ['id' => 2, 'name' => 'bar', 'active' => false], + ['id' => 2, 'name' => 'bar', 'active' => false], + ], schema(int_schema('id'), str_schema('name'), bool_schema('active')), - row(['id' => 1, 'name' => 'foo', 'active' => true]), - row(['id' => 2, 'name' => 'bar', 'active' => false]), - row(['id' => 2, 'name' => 'bar', 'active' => false]), )) ->dropDuplicates(ref('id')) ->fetch(); static::assertEquals( - rows( + array_to_rows( + [['id' => 1, 'name' => 'foo', 'active' => true], ['id' => 2, 'name' => 'bar', 'active' => false]], schema(int_schema('id'), str_schema('name'), bool_schema('active')), - row(['id' => 1, 'name' => 'foo', 'active' => true]), - row(['id' => 2, 'name' => 'bar', 'active' => false]), ), $rows, ); @@ -231,16 +234,18 @@ public function test_drop_duplicates(): void public function test_encapsulate_transformations(): void { $rows = df() - ->process(rows( + ->process(array_to_rows( + [ + ['id' => 1, 'country' => 'PL', 'age' => 20, 'gender' => 'male'], + ['id' => 2, 'country' => 'PL', 'age' => 20, 'gender' => 'male'], + ['id' => 3, 'country' => 'PL', 'age' => 25, 'gender' => 'male'], + ['id' => 4, 'country' => 'PL', 'age' => 30, 'gender' => 'female'], + ['id' => 5, 'country' => 'US', 'age' => 40, 'gender' => 'female'], + ['id' => 6, 'country' => 'US', 'age' => 40, 'gender' => 'male'], + ['id' => 7, 'country' => 'US', 'age' => 45, 'gender' => 'female'], + ['id' => 9, 'country' => 'US', 'age' => 50, 'gender' => 'male'], + ], schema(int_schema('id'), str_schema('country'), int_schema('age'), str_schema('gender')), - row(['id' => 1, 'country' => 'PL', 'age' => 20, 'gender' => 'male']), - row(['id' => 2, 'country' => 'PL', 'age' => 20, 'gender' => 'male']), - row(['id' => 3, 'country' => 'PL', 'age' => 25, 'gender' => 'male']), - row(['id' => 4, 'country' => 'PL', 'age' => 30, 'gender' => 'female']), - row(['id' => 5, 'country' => 'US', 'age' => 40, 'gender' => 'female']), - row(['id' => 6, 'country' => 'US', 'age' => 40, 'gender' => 'male']), - row(['id' => 7, 'country' => 'US', 'age' => 45, 'gender' => 'female']), - row(['id' => 9, 'country' => 'US', 'age' => 50, 'gender' => 'male']), )) ->rows(new class implements Transformation { public function transform(DataFrame $dataFrame): DataFrame @@ -259,16 +264,18 @@ public function transform(DataFrame $dataFrame): DataFrame ->fetch(); static::assertEquals( - rows( + array_to_rows( + [ + ['country' => 'pl', 'age' => 2.0], + ['country' => 'pl', 'age' => 2.0], + ['country' => 'pl', 'age' => 2.5], + ['country' => 'pl', 'age' => 3.0], + ['country' => 'us', 'age' => 4.0], + ['country' => 'us', 'age' => 4.0], + ['country' => 'us', 'age' => 4.5], + ['country' => 'us', 'age' => 5.0], + ], schema(str_schema('country'), float_schema('age')), - row(['country' => 'pl', 'age' => 2.0]), - row(['country' => 'pl', 'age' => 2.0]), - row(['country' => 'pl', 'age' => 2.5]), - row(['country' => 'pl', 'age' => 3.0]), - row(['country' => 'us', 'age' => 4.0]), - row(['country' => 'us', 'age' => 4.0]), - row(['country' => 'us', 'age' => 4.5]), - row(['country' => 'us', 'age' => 5.0]), ), $rows, ); @@ -296,7 +303,7 @@ public function schema(): Schema public function extract(FlowContext $context, ?int $limit = null): Generator { for ($i = 1; $i <= 10; $i++) { - yield rows(schema(integer_schema('id')), row(['id' => $i])); + yield array_to_rows([['id' => $i]], schema(integer_schema('id'))); } } @@ -315,19 +322,23 @@ public function statistics(): Statistics public function test_foreach(): void { df() - ->process(rows( + ->process(array_to_rows( + [ + ['id' => 1, 'name' => 'foo', 'active' => true], + ['id' => 2, 'name' => null, 'active' => false], + ['id' => 2, 'name' => 'bar', 'active' => false], + ], schema(int_schema('id'), str_schema('name', nullable: true), bool_schema('active')), - row(['id' => 1, 'name' => 'foo', 'active' => true]), - row(['id' => 2, 'name' => null, 'active' => false]), - row(['id' => 2, 'name' => 'bar', 'active' => false]), )) ->foreach(function (Rows $rows): void { $this->assertEquals( - rows( + array_to_rows( + [ + ['id' => 1, 'name' => 'foo', 'active' => true], + ['id' => 2, 'name' => null, 'active' => false], + ['id' => 2, 'name' => 'bar', 'active' => false], + ], schema(int_schema('id'), str_schema('name', nullable: true), bool_schema('active')), - row(['id' => 1, 'name' => 'foo', 'active' => true]), - row(['id' => 2, 'name' => null, 'active' => false]), - row(['id' => 2, 'name' => 'bar', 'active' => false]), ), $rows, ); @@ -337,14 +348,16 @@ public function test_foreach(): void public function test_get(): void { $rows = df()->read(from_rows( - $extractedRows = rows( + $extractedRows = array_to_rows( + [ + ['id' => 1, 'name' => 'foo'], + ['id' => 2, 'name' => 'bar'], + ['id' => 3, 'name' => 'baz'], + ['id' => 4, 'name' => 'foo'], + ['id' => 5, 'name' => 'bar'], + ['id' => 6, 'name' => 'baz'], + ], schema(int_schema('id'), str_schema('name')), - row(['id' => 1, 'name' => 'foo']), - row(['id' => 2, 'name' => 'bar']), - row(['id' => 3, 'name' => 'baz']), - row(['id' => 4, 'name' => 'foo']), - row(['id' => 5, 'name' => 'bar']), - row(['id' => 6, 'name' => 'baz']), ), ))->get(); @@ -355,14 +368,16 @@ public function test_get_as_array(): void { $rows = df() ->read(from_rows( - $extractedRows = rows( + $extractedRows = array_to_rows( + [ + ['id' => 1, 'name' => 'foo'], + ['id' => 2, 'name' => 'bar'], + ['id' => 3, 'name' => 'baz'], + ['id' => 4, 'name' => 'foo'], + ['id' => 5, 'name' => 'bar'], + ['id' => 6, 'name' => 'baz'], + ], schema(int_schema('id'), str_schema('name')), - row(['id' => 1, 'name' => 'foo']), - row(['id' => 2, 'name' => 'bar']), - row(['id' => 3, 'name' => 'baz']), - row(['id' => 4, 'name' => 'foo']), - row(['id' => 5, 'name' => 'bar']), - row(['id' => 6, 'name' => 'baz']), ), )) ->getAsArray(); @@ -375,44 +390,19 @@ public function test_get_as_array(): void ); } - public function test_get_each(): void - { - $rows = df() - ->read(from_rows(rows( - schema(int_schema('id'), str_schema('name')), - row(['id' => 1, 'name' => 'foo']), - row(['id' => 2, 'name' => 'bar']), - row(['id' => 3, 'name' => 'baz']), - row(['id' => 4, 'name' => 'foo']), - row(['id' => 5, 'name' => 'bar']), - row(['id' => 6, 'name' => 'baz']), - ))) - ->getEach(); - - static::assertEquals( - [ - row(['id' => 1, 'name' => 'foo']), - row(['id' => 2, 'name' => 'bar']), - row(['id' => 3, 'name' => 'baz']), - row(['id' => 4, 'name' => 'foo']), - row(['id' => 5, 'name' => 'bar']), - row(['id' => 6, 'name' => 'baz']), - ], - iterator_to_array($rows), - ); - } - public function test_get_each_as_array(): void { $rows = df() - ->read(from_rows(rows( + ->read(from_rows(array_to_rows( + [ + ['id' => 1, 'name' => 'foo'], + ['id' => 2, 'name' => 'bar'], + ['id' => 3, 'name' => 'baz'], + ['id' => 4, 'name' => 'foo'], + ['id' => 5, 'name' => 'bar'], + ['id' => 6, 'name' => 'baz'], + ], schema(int_schema('id'), str_schema('name')), - row(['id' => 1, 'name' => 'foo']), - row(['id' => 2, 'name' => 'bar']), - row(['id' => 3, 'name' => 'baz']), - row(['id' => 4, 'name' => 'foo']), - row(['id' => 5, 'name' => 'bar']), - row(['id' => 6, 'name' => 'baz']), ))) ->getEachAsArray(); @@ -451,7 +441,7 @@ public function schema(): Schema public function extract(FlowContext $context, ?int $limit = null): Generator { for ($i = 1; $i <= 10; $i++) { - yield rows(schema(integer_schema('id')), row(['id' => $i])); + yield array_to_rows([['id' => $i]], schema(integer_schema('id'))); } } @@ -506,34 +496,34 @@ public function schema(): Schema */ public function extract(FlowContext $context, ?int $limit = null): Generator { - yield rows( - schema( - int_schema('id'), - bool_schema('deleted'), - datetime_schema('expiration-date'), - str_schema('phase', nullable: true), - ), - row([ + yield array_to_rows( + [[ 'id' => 101, 'deleted' => false, 'expiration-date' => new DateTimeImmutable('2020-08-24'), 'phase' => null, - ]), - ); - - yield rows( + ]], schema( int_schema('id'), bool_schema('deleted'), datetime_schema('expiration-date'), str_schema('phase', nullable: true), ), - row([ + ); + + yield array_to_rows( + [[ 'id' => 102, 'deleted' => true, 'expiration-date' => new DateTimeImmutable('2020-08-25'), 'phase' => null, - ]), + ]], + schema( + int_schema('id'), + bool_schema('deleted'), + datetime_schema('expiration-date'), + str_schema('phase', nullable: true), + ), ); } @@ -553,11 +543,11 @@ public function transform(Rows $rows, FlowContext $context): Rows { $stamped = []; - foreach ($rows->all() as $row) { - $stamped[] = row([...$row->values(), 'stamp' => 'zero']); + for ($i = 0; $i < $rows->count(); $i++) { + $stamped[] = [...$rows->values($i), 'stamp' => 'zero']; } - return new Rows($rows->schema()->add(str_schema('stamp')), ...$stamped); + return array_to_rows($stamped, $rows->schema()->add(str_schema('stamp'))); } }; @@ -600,19 +590,19 @@ public function test_process_constructor(): void { $collectedRows = data_frame() ->process( - $rows = rows( + $rows = array_to_rows( + [[ + 'id' => 101, + 'deleted' => false, + 'expiration-date' => new DateTimeImmutable('2020-08-24'), + 'phase' => null, + ]], schema( int_schema('id'), bool_schema('deleted'), datetime_schema('expiration-date'), str_schema('phase', nullable: true), ), - row([ - 'id' => 101, - 'deleted' => false, - 'expiration-date' => new DateTimeImmutable('2020-08-24'), - 'phase' => null, - ]), ), ) ->fetch(); @@ -623,21 +613,21 @@ public function test_process_constructor(): void public function test_select(): void { $rows = data_frame() - ->process(rows( + ->process(array_to_rows( + [ + ['id' => 1, 'name' => 'foo', 'active' => true], + ['id' => 2, 'name' => null, 'active' => false], + ['id' => 2, 'name' => 'bar', 'active' => false], + ], schema(int_schema('id'), str_schema('name', nullable: true), bool_schema('active')), - row(['id' => 1, 'name' => 'foo', 'active' => true]), - row(['id' => 2, 'name' => null, 'active' => false]), - row(['id' => 2, 'name' => 'bar', 'active' => false]), )) ->select('name', 'id') ->fetch(); static::assertEquals( - rows( + array_to_rows( + [['name' => 'foo', 'id' => 1], ['name' => null, 'id' => 2], ['name' => 'bar', 'id' => 2]], schema(str_schema('name', nullable: true), int_schema('id')), - row(['name' => 'foo', 'id' => 1]), - row(['name' => null, 'id' => 2]), - row(['name' => 'bar', 'id' => 2]), ), $rows, ); @@ -646,31 +636,35 @@ public function test_select(): void public function test_selective_validation_against_schema(): void { $rows = data_frame() - ->process(rows( + ->process(array_to_rows( + [ + ['id' => 1, 'name' => 'foo', 'active' => true], + ['id' => 2, 'name' => null, 'tags' => type_json()->cast(['foo', 'bar'])], + ['id' => 2, 'name' => 'bar', 'active' => false], + ], schema( int_schema('id'), str_schema('name', nullable: true), bool_schema('active', nullable: true), json_schema('tags', nullable: true), ), - row(['id' => 1, 'name' => 'foo', 'active' => true]), - row(['id' => 2, 'name' => null, 'tags' => type_json()->cast(['foo', 'bar'])]), - row(['id' => 2, 'name' => 'bar', 'active' => false]), )) ->match(schema(integer_schema('id', false)), new SelectiveValidator()) ->fetch(); static::assertSame( - rows( + array_to_rows( + [ + ['id' => 1, 'name' => 'foo', 'active' => true], + ['id' => 2, 'name' => null, 'tags' => type_json()->cast(['foo', 'bar'])], + ['id' => 2, 'name' => 'bar', 'active' => false], + ], schema( int_schema('id'), str_schema('name', nullable: true), bool_schema('active', nullable: true), json_schema('tags', nullable: true), ), - row(['id' => 1, 'name' => 'foo', 'active' => true]), - row(['id' => 2, 'name' => null, 'tags' => type_json()->cast(['foo', 'bar'])]), - row(['id' => 2, 'name' => 'bar', 'active' => false]), )->toArray(), $rows->toArray(), ); @@ -707,21 +701,25 @@ public function test_sort_by_accepts_column_name_strings_inside_the_array_form() public function test_strict_validation_against_schema(): void { $rows = data_frame() - ->process(rows( + ->process(array_to_rows( + [ + ['id' => 1, 'name' => 'foo', 'active' => true], + ['id' => 2, 'name' => null, 'active' => false], + ['id' => 2, 'name' => 'bar', 'active' => false], + ], schema(int_schema('id'), str_schema('name', nullable: true), bool_schema('active')), - row(['id' => 1, 'name' => 'foo', 'active' => true]), - row(['id' => 2, 'name' => null, 'active' => false]), - row(['id' => 2, 'name' => 'bar', 'active' => false]), )) ->match(schema(integer_schema('id', false), string_schema('name', true), bool_schema('active', false))) ->fetch(); static::assertSame( - rows( + array_to_rows( + [ + ['id' => 1, 'name' => 'foo', 'active' => true], + ['id' => 2, 'name' => null, 'active' => false], + ['id' => 2, 'name' => 'bar', 'active' => false], + ], schema(int_schema('id'), str_schema('name', nullable: true), bool_schema('active')), - row(['id' => 1, 'name' => 'foo', 'active' => true]), - row(['id' => 2, 'name' => null, 'active' => false]), - row(['id' => 2, 'name' => 'bar', 'active' => false]), )->toArray(), $rows->toArray(), ); @@ -763,16 +761,18 @@ public function test_until(): void public function test_void(): void { $rows = data_frame() - ->process(rows( + ->process(array_to_rows( + [ + ['id' => 1, 'country' => 'PL', 'age' => 20], + ['id' => 2, 'country' => 'PL', 'age' => 20], + ['id' => 3, 'country' => 'PL', 'age' => 25], + ['id' => 4, 'country' => 'PL', 'age' => 30], + ['id' => 5, 'country' => 'US', 'age' => 40], + ['id' => 6, 'country' => 'US', 'age' => 40], + ['id' => 7, 'country' => 'US', 'age' => 45], + ['id' => 9, 'country' => 'US', 'age' => 50], + ], schema(int_schema('id'), str_schema('country'), int_schema('age')), - row(['id' => 1, 'country' => 'PL', 'age' => 20]), - row(['id' => 2, 'country' => 'PL', 'age' => 20]), - row(['id' => 3, 'country' => 'PL', 'age' => 25]), - row(['id' => 4, 'country' => 'PL', 'age' => 30]), - row(['id' => 5, 'country' => 'US', 'age' => 40]), - row(['id' => 6, 'country' => 'US', 'age' => 40]), - row(['id' => 7, 'country' => 'US', 'age' => 45]), - row(['id' => 9, 'country' => 'US', 'age' => 50]), )) ->rename('country', 'country_code') ->void() @@ -783,7 +783,7 @@ public function test_void(): void // void() drops rows, not columns - the batch it yields carries the plan's declared schema, and // the global aggregate that follows it reads zero rows, so it emits its initial accumulators static::assertEquals( - rows(schema(float_schema('average_age', nullable: true)), row(['average_age' => null])), + array_to_rows([['average_age' => null]], schema(float_schema('average_age', nullable: true))), $rows, ); } @@ -809,19 +809,18 @@ public function schema(): Schema */ public function extract(FlowContext $context, ?int $limit = null): Generator { - yield rows( - schema(integer_schema('id')), - row(['id' => 1]), - row(['id' => 2]), - row(['id' => 3]), - row(['id' => 4]), - row(['id' => 5]), - row(['id' => 6]), - row(['id' => 7]), - row(['id' => 8]), - row(['id' => 9]), - row(['id' => 10]), - ); + yield array_to_rows([ + ['id' => 1], + ['id' => 2], + ['id' => 3], + ['id' => 4], + ['id' => 5], + ['id' => 6], + ['id' => 7], + ['id' => 8], + ['id' => 9], + ['id' => 10], + ], schema(integer_schema('id'))); } public function statistics(): Statistics @@ -837,13 +836,7 @@ public function bind(Schema $input): BoundStep public function transform(Rows $rows, FlowContext $context): Rows { - $renamed = []; - - foreach ($rows->all() as $row) { - $renamed[] = RowRenaming::of(['id' => 'new_id'])->apply($row); - } - - return new Rows($rows->schema()->rename('id', 'new_id'), ...$renamed); + return $rows->withSchema($rows->schema()->rename('id', 'new_id')); } }) ->batchSize(2) @@ -877,9 +870,9 @@ public function schema(): Schema */ public function extract(FlowContext $context, ?int $limit = null): Generator { - yield rows(schema(integer_schema('id')), row(['id' => 1])); - yield rows(schema(integer_schema('id')), row(['id' => 2])); - yield rows(schema(integer_schema('id')), row(['id' => 3])); + yield array_to_rows([['id' => 1]], schema(integer_schema('id'))); + yield array_to_rows([['id' => 2]], schema(integer_schema('id'))); + yield array_to_rows([['id' => 3]], schema(integer_schema('id'))); } public function statistics(): Statistics @@ -895,13 +888,7 @@ public function bind(Schema $input): BoundStep public function transform(Rows $rows, FlowContext $context): Rows { - $renamed = []; - - foreach ($rows->all() as $row) { - $renamed[] = RowRenaming::of(['id' => 'new_id'])->apply($row); - } - - return new Rows($rows->schema()->rename('id', 'new_id'), ...$renamed); + return $rows->withSchema($rows->schema()->rename('id', 'new_id')); } }) ->collect() @@ -960,7 +947,7 @@ public function test_a_pushed_limit_reaches_extract_and_leaves_the_users_extract { $extractor = new RecordingFileExtractor( schema(int_schema('id')), - rows(schema(int_schema('id')), row(['id' => 1]), row(['id' => 2])), + array_to_rows([['id' => 1], ['id' => 2]], schema(int_schema('id'))), ); $frame = df() ->read($extractor) @@ -975,11 +962,11 @@ public function test_a_pushed_limit_reaches_extract_and_leaves_the_users_extract static::assertSame(1, $extractor->limits[0]); static::assertSame(1, $extractor->limits[1]); static::assertStringEndsWith( - "#1 Read\n Extractor: RecordingFileExtractor\n Source: file://dev/null", + "#1 Read\n Extractor: RecordingFileExtractor\n Source: file:///dev/null", $frame->explain()->toString(Stage::unoptimized), ); static::assertStringEndsWith( - "#1 Read\n Extractor: RecordingFileExtractor\n Source: file://dev/null" + "#1 Read\n Extractor: RecordingFileExtractor\n Source: file:///dev/null" . "\n Limit: 1", $frame->explain()->toString(), ); @@ -989,7 +976,7 @@ public function test_a_limit_inside_a_joins_right_side_is_pushed_into_that_sourc { $extractor = new RecordingFileExtractor( schema(int_schema('id')), - rows(schema(int_schema('id')), row(['id' => 1]), row(['id' => 2])), + array_to_rows([['id' => 1], ['id' => 2]], schema(int_schema('id'))), ); $right = df()->read($extractor)->limit(1); @@ -1015,17 +1002,17 @@ public function test_an_empty_fetch_plans_once(): void static::assertCount(1, $log); } - public function test_an_abandoned_get_each_leaves_no_consumed_step_for_the_next_run(): void + public function test_an_abandoned_get_each_as_array_leaves_no_consumed_step_for_the_next_run(): void { $dataFrame = df() ->read(from_rows( - rows(schema(int_schema('id')), row(['id' => 1]), row(['id' => 2])), - rows(schema(int_schema('id')), row(['id' => 3]), row(['id' => 4])), + array_to_rows([['id' => 1], ['id' => 2]], schema(int_schema('id'))), + array_to_rows([['id' => 3], ['id' => 4]], schema(int_schema('id'))), )) ->limit(3); // the reference keeps the generator parked, so its steps stay consumed - $parked = $dataFrame->getEach(); + $parked = $dataFrame->getEachAsArray(); $parked->current(); static::assertCount(3, $dataFrame->fetch()); @@ -1120,7 +1107,7 @@ public function test_a_limit_inside_a_read_frame_is_pushed_into_its_source(): vo { $extractor = new RecordingFileExtractor( schema(int_schema('id')), - rows(schema(int_schema('id')), row(['id' => 1]), row(['id' => 2])), + array_to_rows([['id' => 1], ['id' => 2]], schema(int_schema('id'))), ); $inner = df()->read($extractor)->limit(1); @@ -1222,7 +1209,7 @@ public function test_explain_prints_the_logical_plan_without_reading_a_row(): vo │ Condition: IsNotNull └─ #1 Read source · transparent · streaming Extractor: RecordingFileExtractor - Source: file://dev/null + Source: file:///dev/null PLAN, $dataFrame->explain()->toString(format: Format::declarations)); static::assertSame([], $extractor->limits); } diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Dataset/Memory/BoundedReadTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Dataset/Memory/BoundedReadTest.php new file mode 100644 index 0000000000..1b5f5a336b --- /dev/null +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Dataset/Memory/BoundedReadTest.php @@ -0,0 +1,44 @@ +read(IdRows::batches([1, 2], [3])); + + static::assertTrue($fits); + static::assertSame([[1, 2], [3]], array_map(static fn($rows): array => $rows->reduceToArray('id'), $read)); + } + + public function test_past_the_limit_reading_stops_and_the_rest_continues_where_it_stopped(): void + { + $rows = IdRows::batches([1, 2], [3], [4]); + $bounded = new BoundedRead(Unit::fromBytes(1), new PhpBackend()); + + [$read, $fits] = $bounded->read($rows); + + static::assertFalse($fits); + static::assertCount(1, $read); + static::assertSame([1, 2, 3, 4], IdRows::ids($bounded->followedBy($read, $rows))); + } + + public function test_followed_by_a_finished_stream_yields_only_what_was_read(): void + { + $rows = IdRows::batches([1]); + $bounded = new BoundedRead(Unit::fromGb(64), new PhpBackend()); + + [$read] = $bounded->read($rows); + + static::assertSame([1], IdRows::ids($bounded->followedBy($read, $rows))); + } +} diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Dataset/Memory/ConsumptionTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Dataset/Memory/ConsumptionTest.php index 66a0b872fd..9ddb9be2ad 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Unit/Dataset/Memory/ConsumptionTest.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Dataset/Memory/ConsumptionTest.php @@ -4,18 +4,29 @@ namespace Flow\ETL\Tests\Unit\Dataset\Memory; +use Flow\ETL\Column\PhpBackend; use Flow\ETL\Dataset\Memory\Consumption; +use Flow\ETL\Tests\Double\SpyBackend; use Flow\ETL\Tests\FlowTestCase; final class ConsumptionTest extends FlowTestCase { public function test_capturing_memory_consumption(): void { - $consumption = new Consumption(); + $consumption = new Consumption(new PhpBackend()); $consumption->capture(); static::assertGreaterThan(0, $consumption->max()->inBytes()); static::assertGreaterThan(0, $consumption->min()->inBytes()); } + + public function test_consumption_adds_the_backend_allocations(): void + { + $php = new Consumption(new PhpBackend()); + $native = new Consumption(new SpyBackend(1_048_576)); + + static::assertSame(1_048_576, $native->initial()->inBytes() - $php->initial()->inBytes()); + static::assertSame(1_048_576, $native->capture()->inBytes() - $php->capture()->inBytes()); + } } diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Dataset/Memory/MemoryBudgetTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Dataset/Memory/MemoryBudgetTest.php new file mode 100644 index 0000000000..819365eea6 --- /dev/null +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Dataset/Memory/MemoryBudgetTest.php @@ -0,0 +1,48 @@ +exceeded()); + } + + public function test_native_memory_counts_towards_the_limit(): void + { + $budget = new MemoryBudget( + new ReportedMemoryBackend(2_000_000_000), + Unit::fromBytes(memory_get_usage(false) + 1_000_000_000), + ); + + static::assertTrue($budget->exceeded()); + } + + public function test_the_limit_follows_the_memory_the_process_holds_now(): void + { + $backend = new ReportedMemoryBackend(2_000_000_000); + $budget = new MemoryBudget($backend, Unit::fromBytes(memory_get_usage(false) + 1_000_000_000)); + + static::assertTrue($budget->exceeded()); + + $backend->reported = 0; + $budget->released(); + + static::assertFalse($budget->exceeded()); + } +} diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Dataset/Statistics/ColumnTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Dataset/Statistics/ColumnTest.php index 0dde29d1c5..c17931a24d 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Unit/Dataset/Statistics/ColumnTest.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Dataset/Statistics/ColumnTest.php @@ -7,13 +7,27 @@ use DateTimeImmutable; use Flow\ETL\Dataset\Statistics\Column; use Flow\ETL\Tests\FlowTestCase; +use Flow\ETL\Tests\Mother\ColumnMother; +use function Flow\ETL\DSL\bool_schema; use function Flow\ETL\DSL\date_schema; use function Flow\ETL\DSL\datetime_schema; +use function Flow\ETL\DSL\float_schema; use function Flow\ETL\DSL\int_schema; +use function Flow\ETL\DSL\json_schema; +use function Flow\ETL\DSL\list_schema; +use function Flow\ETL\DSL\map_schema; +use function Flow\ETL\DSL\str_schema; +use function Flow\ETL\DSL\structure_schema; use function Flow\ETL\DSL\time_zone_schema; +use function Flow\ETL\DSL\uuid_schema; use function Flow\Types\DSL\type_date; use function Flow\Types\DSL\type_datetime; +use function Flow\Types\DSL\type_integer; +use function Flow\Types\DSL\type_list; +use function Flow\Types\DSL\type_map; +use function Flow\Types\DSL\type_string; +use function Flow\Types\DSL\type_structure; use function Flow\Types\DSL\type_time_zone; final class ColumnTest extends FlowTestCase @@ -22,20 +36,20 @@ public function test_collecting_column_statistics(): void { $definition = int_schema('a', true); - $statistics = new Column($definition, 1); + $statistics = new Column($definition, ColumnMother::of($definition, [1])); static::assertEquals(1, $statistics->distinctCount()); static::assertEquals(0, $statistics->nullCount()); static::assertEquals(1, $statistics->max()); static::assertEquals(1, $statistics->min()); - $statistics->add($definition, 2); + $statistics->add($definition, ColumnMother::of($definition, [2])); static::assertEquals(2, $statistics->distinctCount()); static::assertEquals(0, $statistics->nullCount()); static::assertEquals(2, $statistics->max()); static::assertEquals(1, $statistics->min()); - $statistics->add($definition, null); + $statistics->add($definition, ColumnMother::of($definition, [null])); static::assertEquals(2, $statistics->distinctCount()); static::assertEquals(1, $statistics->nullCount()); @@ -47,16 +61,16 @@ public function test_collecting_column_statistics_for_time_zone_entries(): void { $definition = time_zone_schema('tz', true); - $statistics = new Column($definition, type_time_zone()->cast('UTC')); + $statistics = new Column($definition, ColumnMother::of($definition, [type_time_zone()->cast('UTC')])); static::assertEquals(1, $statistics->distinctCount()); - $statistics->add($definition, type_time_zone()->cast('Europe/Warsaw')); + $statistics->add($definition, ColumnMother::of($definition, [type_time_zone()->cast('Europe/Warsaw')])); static::assertEquals(2, $statistics->distinctCount()); - $statistics->add($definition, type_time_zone()->cast('UTC')); + $statistics->add($definition, ColumnMother::of($definition, [type_time_zone()->cast('UTC')])); static::assertEquals(2, $statistics->distinctCount()); - $statistics->add($definition, null); + $statistics->add($definition, ColumnMother::of($definition, [null])); static::assertEquals(2, $statistics->distinctCount()); static::assertEquals(1, $statistics->nullCount()); } @@ -65,20 +79,20 @@ public function test_collecting_column_statistics_for_date_entries(): void { $definition = date_schema('a', true); - $statistics = new Column($definition, type_date()->cast('2024-01-01')); + $statistics = new Column($definition, ColumnMother::of($definition, [type_date()->cast('2024-01-01')])); static::assertEquals(1, $statistics->distinctCount()); static::assertEquals(0, $statistics->nullCount()); static::assertEquals(new DateTimeImmutable('2024-01-01'), $statistics->max()); static::assertEquals(new DateTimeImmutable('2024-01-01'), $statistics->min()); - $statistics->add($definition, type_date()->cast('2024-01-05')); + $statistics->add($definition, ColumnMother::of($definition, [type_date()->cast('2024-01-05')])); static::assertEquals(2, $statistics->distinctCount()); static::assertEquals(0, $statistics->nullCount()); static::assertEquals(new DateTimeImmutable('2024-01-05'), $statistics->max()); static::assertEquals(new DateTimeImmutable('2024-01-01'), $statistics->min()); - $statistics->add($definition, null); + $statistics->add($definition, ColumnMother::of($definition, [null])); static::assertEquals(2, $statistics->distinctCount()); static::assertEquals(1, $statistics->nullCount()); @@ -90,24 +104,125 @@ public function test_collecting_column_statistics_for_datetime_entries(): void { $definition = datetime_schema('a', true); - $statistics = new Column($definition, type_datetime()->cast('2024-01-01 00:00:01')); + $statistics = new Column($definition, ColumnMother::of($definition, [type_datetime()->cast( + '2024-01-01 00:00:01', + )])); static::assertEquals(1, $statistics->distinctCount()); static::assertEquals(0, $statistics->nullCount()); static::assertEquals(new DateTimeImmutable('2024-01-01 00:00:01'), $statistics->max()); static::assertEquals(new DateTimeImmutable('2024-01-01 00:00:01'), $statistics->min()); - $statistics->add($definition, type_datetime()->cast('2024-01-01 01:00:00')); + $statistics->add($definition, ColumnMother::of($definition, [type_datetime()->cast('2024-01-01 01:00:00')])); static::assertEquals(2, $statistics->distinctCount()); static::assertEquals(0, $statistics->nullCount()); static::assertEquals(new DateTimeImmutable('2024-01-01 01:00:00'), $statistics->max()); static::assertEquals(new DateTimeImmutable('2024-01-01 00:00:01'), $statistics->min()); - $statistics->add($definition, null); + $statistics->add($definition, ColumnMother::of($definition, [null])); static::assertEquals(2, $statistics->distinctCount()); static::assertEquals(1, $statistics->nullCount()); static::assertEquals(new DateTimeImmutable('2024-01-01 01:00:00'), $statistics->max()); static::assertEquals(new DateTimeImmutable('2024-01-01 00:00:01'), $statistics->min()); } + + public function test_a_column_of_many_values_with_nulls(): void + { + $definition = str_schema('s', true); + + $statistics = new Column($definition, ColumnMother::of($definition, ['abc', null, 'a', 'abc', null])); + + static::assertSame(2, $statistics->nullCount()); + static::assertSame(2, $statistics->distinctCount()); + static::assertSame(1, $statistics->minLength()); + static::assertSame(3, $statistics->maxLength()); + } + + public function test_a_column_of_only_nulls_counts_nulls(): void + { + $definition = int_schema('a', true); + + $statistics = new Column($definition, ColumnMother::of($definition, [null, null])); + + static::assertSame(2, $statistics->nullCount()); + static::assertSame(0, $statistics->distinctCount()); + static::assertNull($statistics->min()); + } + + public function test_another_column_is_ignored(): void + { + $statistics = new Column(int_schema('a'), ColumnMother::of(int_schema('a'), [1])); + + $statistics->add(int_schema('b'), ColumnMother::of(int_schema('b'), [5])); + + static::assertSame(1, $statistics->max()); + } + + public function test_booleans_and_floats(): void + { + $flags = new Column(bool_schema('b'), ColumnMother::of(bool_schema('b'), [true, false, true])); + $amounts = new Column(float_schema('f'), ColumnMother::of(float_schema('f'), [1.5, -0.5])); + + static::assertSame(2, $flags->distinctCount()); + static::assertFalse($flags->min()); + static::assertTrue($flags->max()); + static::assertSame(-0.5, $amounts->min()); + static::assertSame(1.5, $amounts->max()); + } + + public function test_datetimes_before_the_epoch_count_distinct_seconds(): void + { + $definition = datetime_schema('a'); + + $statistics = new Column($definition, ColumnMother::of($definition, [ + new DateTimeImmutable('1969-12-31 23:59:59.5'), + new DateTimeImmutable('1969-12-31 23:59:59.25'), + new DateTimeImmutable('1970-01-01 00:00:00'), + ])); + + static::assertSame(2, $statistics->distinctCount()); + static::assertEquals(new DateTimeImmutable('1969-12-31 23:59:59.25'), $statistics->min()); + static::assertEquals(new DateTimeImmutable('1970-01-01 00:00:00'), $statistics->max()); + } + + public function test_lists_and_maps_count_elements(): void + { + $lists = new Column( + list_schema('l', type_list(type_integer())), + ColumnMother::of(list_schema('l', type_list(type_integer())), [[1, 2, 3], [], [1, 2, 3]]), + ); + $maps = new Column( + map_schema('m', type_map(type_string(), type_integer())), + ColumnMother::of(map_schema('m', type_map(type_string(), type_integer())), [['a' => 1]]), + ); + + static::assertSame(2, $lists->distinctCount()); + static::assertSame(0, $lists->minElementsCount()); + static::assertSame(3, $lists->maxElementsCount()); + static::assertSame(1, $maps->maxElementsCount()); + } + + public function test_uuids_structures_and_json_count_by_value(): void + { + $uuids = new Column( + uuid_schema('u'), + ColumnMother::of( + uuid_schema('u'), + [ + '00000000-0000-4000-8000-000000000001', + '00000000-0000-4000-8000-000000000001', + ], + ), + ); + $structures = new Column( + structure_schema('s', type_structure(['a' => type_integer()])), + ColumnMother::of(structure_schema('s', type_structure(['a' => type_integer()])), [['a' => 1], ['a' => 2]]), + ); + $json = new Column(json_schema('j'), ColumnMother::of(json_schema('j'), ['{"a":1}'])); + + static::assertSame(1, $uuids->distinctCount()); + static::assertSame(2, $structures->distinctCount()); + static::assertSame(0, $json->distinctCount()); + } } diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Dataset/Statistics/ColumnsTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Dataset/Statistics/ColumnsTest.php index 129dafc920..17d5b5a9b8 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Unit/Dataset/Statistics/ColumnsTest.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Dataset/Statistics/ColumnsTest.php @@ -6,6 +6,7 @@ use Flow\ETL\Dataset\Statistics\Columns; use Flow\ETL\Tests\FlowTestCase; +use Flow\ETL\Tests\Mother\ColumnMother; use InvalidArgumentException; use function Flow\ETL\DSL\integer_schema; @@ -17,12 +18,12 @@ public function test_columns_statistics(): void { $columns = new Columns(); - $columns->add(integer_schema('a'), 1); - $columns->add(integer_schema('a'), 100); - $columns->add(integer_schema('a'), -5); - $columns->add(string_schema('b', true), 'a'); - $columns->add(string_schema('b', true), 'some text'); - $columns->add(string_schema('b', true), null); + $columns->add(integer_schema('a'), ColumnMother::of(integer_schema('a'), [1])); + $columns->add(integer_schema('a'), ColumnMother::of(integer_schema('a'), [100])); + $columns->add(integer_schema('a'), ColumnMother::of(integer_schema('a'), [-5])); + $columns->add(string_schema('b', true), ColumnMother::of(string_schema('b', true), ['a'])); + $columns->add(string_schema('b', true), ColumnMother::of(string_schema('b', true), ['some text'])); + $columns->add(string_schema('b', true), ColumnMother::of(string_schema('b', true), [null])); static::assertCount(2, $columns->all()); static::assertSame(3, $columns->get('a')->distinctCount()); diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/ETLErrorHandlingTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/ETLErrorHandlingTest.php index f5ae242a1b..b336040b35 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Unit/ETLErrorHandlingTest.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/ETLErrorHandlingTest.php @@ -20,6 +20,7 @@ use RuntimeException; use function array_merge; +use function Flow\ETL\DSL\array_to_rows; use function Flow\ETL\DSL\bool_schema; use function Flow\ETL\DSL\data_frame; use function Flow\ETL\DSL\datetime_schema; @@ -29,8 +30,6 @@ use function Flow\ETL\DSL\int_schema; use function Flow\ETL\DSL\lit; use function Flow\ETL\DSL\ref; -use function Flow\ETL\DSL\row; -use function Flow\ETL\DSL\rows; use function Flow\ETL\DSL\schema; use function Flow\ETL\DSL\skip_rows_handler; use function Flow\ETL\DSL\str_schema; @@ -75,19 +74,19 @@ public function extract(FlowContext $context, ?int $limit = null): Generator str_schema('phase', true), ); - yield rows($schema, row([ + yield array_to_rows([[ 'id' => 101, 'deleted' => false, 'expiration-date' => new DateTimeImmutable('2020-08-24'), 'phase' => null, - ])); + ]], $schema); - yield rows($schema, row([ + yield array_to_rows([[ 'id' => 102, 'deleted' => true, 'expiration-date' => new DateTimeImmutable('2020-08-25'), 'phase' => null, - ])); + ]], $schema); } public function statistics(): Statistics @@ -156,19 +155,19 @@ public function extract(FlowContext $context, ?int $limit = null): Generator str_schema('phase', true), ); - yield rows($schema, row([ + yield array_to_rows([[ 'id' => 101, 'deleted' => false, 'expiration-date' => new DateTimeImmutable('2020-08-24'), 'phase' => null, - ])); + ]], $schema); - yield rows($schema, row([ + yield array_to_rows([[ 'id' => 102, 'deleted' => true, 'expiration-date' => new DateTimeImmutable('2020-08-25'), 'phase' => null, - ])); + ]], $schema); } public function statistics(): Statistics @@ -259,19 +258,19 @@ public function extract(FlowContext $context, ?int $limit = null): Generator str_schema('phase', true), ); - yield rows($schema, row([ + yield array_to_rows([[ 'id' => 101, 'deleted' => false, 'expiration-date' => new DateTimeImmutable('2020-08-24'), 'phase' => null, - ])); + ]], $schema); - yield rows($schema, row([ + yield array_to_rows([[ 'id' => 102, 'deleted' => true, 'expiration-date' => new DateTimeImmutable('2020-08-25'), 'phase' => null, - ])); + ]], $schema); } public function statistics(): Statistics @@ -288,7 +287,7 @@ public function bind(Schema $input): BoundStep public function transform(Rows $rows, FlowContext $context): Rows { - if ($rows->first()->get('id') === 101) { + if ($rows->column('id')->value(0) === 101) { throw new RuntimeException('Transformer Exception'); } diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Exception/ColumnMismatchExceptionTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Exception/ColumnMismatchExceptionTest.php index b23fb9d150..a837c861c9 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Unit/Exception/ColumnMismatchExceptionTest.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Exception/ColumnMismatchExceptionTest.php @@ -4,6 +4,8 @@ namespace Flow\ETL\Tests\Unit\Exception; +use DateTimeImmutable; +use DateTimeZone; use Flow\ETL\Exception\ColumnMismatchException; use Flow\ETL\Tests\FlowTestCase; use Generator; @@ -29,6 +31,10 @@ public static function provide_rendered_values(): Generator yield 'true' => [true, 'could not convert true (boolean) to integer']; yield 'false' => [false, 'could not convert false (boolean) to integer']; yield 'array' => [[1, 2], 'could not convert array (list) to integer']; + yield 'datetime with its offset' => [ + new DateTimeImmutable('2026-01-01 00:00:00.5', new DateTimeZone('Europe/Warsaw')), + 'could not convert 2026-01-01T00:00:00.500000+01:00 (datetime) to integer', + ]; } public function test_a_message_carries_no_row_coordinate(): void diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Exception/EvaluationExceptionTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Exception/EvaluationExceptionTest.php new file mode 100644 index 0000000000..df088f4a7f --- /dev/null +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Exception/EvaluationExceptionTest.php @@ -0,0 +1,44 @@ +rowIndex); + static::assertSame($cause, $exception->getPrevious()); + static::assertSame('division by zero (row 5)', $exception->getMessage()); + } + + public function test_at_wraps_any_other_exception(): void + { + $cause = new RuntimeException('division by zero'); + + $exception = EvaluationException::at(2, $cause); + + static::assertSame(2, $exception->rowIndex); + static::assertSame($cause, $exception->getPrevious()); + } + + public function test_carries_the_row_and_the_cause(): void + { + $cause = new RuntimeException('division by zero'); + + $exception = new EvaluationException(3, $cause); + + static::assertSame(3, $exception->rowIndex); + static::assertSame($cause, $exception->getPrevious()); + static::assertSame('division by zero (row 3)', $exception->getMessage()); + } +} diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Exception/InferredSchemaExceptionTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Exception/InferredSchemaExceptionTest.php index 240db3cc00..6b024e7166 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Unit/Exception/InferredSchemaExceptionTest.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Exception/InferredSchemaExceptionTest.php @@ -34,13 +34,13 @@ public function test_columns_diverge_names_the_source_both_diffs_the_knobs_and_b static::assertStringContainsString('infer_schema()->unionByName()', $exception->getMessage()); } - public function test_files_diverge_names_both_files_the_difference_and_both_remedies(): void + public function test_files_diverge_names_both_files_the_difference_and_the_remedy(): void { static::assertSame( "Columns of file:///orders/part-02.parquet do not match the schema read from file:///orders/part-01.parquet:\n" . " Unexpected Definitions: \n" . " |-- extra\n" - . 'Read the files as one wider schema with ->unionByName(), or declare the schema with ->withSchema(...).', + . 'Read the files as one wider schema with ->unionByName().', InferredSchemaException::filesDiverge( 'file:///orders/part-02.parquet', 'file:///orders/part-01.parquet', diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Executor/AdoptedBatchesTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Executor/AdoptedBatchesTest.php new file mode 100644 index 0000000000..7f54bfa8e6 --- /dev/null +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Executor/AdoptedBatchesTest.php @@ -0,0 +1,69 @@ + $source */ + $source = (static function () use ($schema) { + yield array_to_rows([['id' => 1, 'name' => 'a']], $schema, new PhpBackend()); + yield array_to_rows([['id' => 2, 'name' => 'b']], $schema, new PhpBackend()); + })(); + + $batches = iterator_to_array((new AdoptedBatches($backend))->of($source), false); + + static::assertCount(2, $batches); + static::assertSame(4, $backend->adopts()); + } + + public function test_a_batch_the_backend_already_owns_passes_as_the_same_instance(): void + { + $batch = array_to_rows([['id' => 1]], schema(int_schema('id')), new PhpBackend()); + + /** @var Generator $source */ + $source = (static function () use ($batch) { + yield $batch; + })(); + $adopted = (new AdoptedBatches(new PhpBackend()))->of($source); + + static::assertSame($batch, $adopted->current()); + } + + public function test_a_stop_reaches_the_source(): void + { + $extractor = new CountingExtractor( + schema(int_schema('id')), + array_to_rows([['id' => 1], ['id' => 2], ['id' => 3]], schema(int_schema('id'))), + ); + $extractor->withBatchSize(1); + $adopted = (new AdoptedBatches(new PhpBackend()))->of($extractor->extract(flow_context())); + + static::assertCount(1, $adopted->current()); + $adopted->send(Signal::STOP); + + static::assertFalse($adopted->valid()); + static::assertSame(1, $extractor->batchesYielded); + } +} diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Executor/OffsetPipelineTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Executor/OffsetPipelineTest.php index 6e3e8cafac..27cce26e98 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Unit/Executor/OffsetPipelineTest.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Executor/OffsetPipelineTest.php @@ -19,6 +19,7 @@ use function array_map; use function array_sum; +use function Flow\ETL\DSL\array_to_rows; use function Flow\ETL\DSL\bool_schema; use function Flow\ETL\DSL\config; use function Flow\ETL\DSL\flow_context; @@ -26,7 +27,6 @@ use function Flow\ETL\DSL\int_schema; use function Flow\ETL\DSL\lit; use function Flow\ETL\DSL\ref; -use function Flow\ETL\DSL\row; use function Flow\ETL\DSL\rows; use function Flow\ETL\DSL\schema; use function iterator_to_array; @@ -52,23 +52,23 @@ public function test_constructor_with_negative_offset_throws_exception(): void public function test_process_maintains_row_structure_with_mixed_entry_types(): void { - $segments = new Segments(from_rows(rows( + $segments = new Segments(from_rows(array_to_rows( + [ + ['id' => 1, 'active' => true], + ['id' => 2, 'active' => false], + ['id' => 3, 'active' => true], + ['id' => 4, 'active' => false], + ], schema(int_schema('id'), bool_schema('active')), - row(['id' => 1, 'active' => true]), - row(['id' => 2, 'active' => false]), - row(['id' => 3, 'active' => true]), - row(['id' => 4, 'active' => false]), ))); $segments->add(new OffsetProcessor(1)); $result = iterator_to_array(ExecutedSegments::of($segments, flow_context(config()))); static::assertCount(1, $result); static::assertCount(3, $result[0]); static::assertEquals( - rows( + array_to_rows( + [['id' => 2, 'active' => false], ['id' => 3, 'active' => true], ['id' => 4, 'active' => false]], schema(int_schema('id'), bool_schema('active')), - row(['id' => 2, 'active' => false]), - row(['id' => 3, 'active' => true]), - row(['id' => 4, 'active' => false]), ), $result[0], ); @@ -97,9 +97,9 @@ public function schema(): Schema public function extract(FlowContext $context, ?int $limit = null): Generator { - yield rows(schema(int_schema('id')), row(['id' => 1]), row(['id' => 2])); - yield rows(schema(int_schema('id')), row(['id' => 3]), row(['id' => 4])); - yield rows(schema(int_schema('id')), row(['id' => 5]), row(['id' => 6])); + yield array_to_rows([['id' => 1], ['id' => 2]], schema(int_schema('id'))); + yield array_to_rows([['id' => 3], ['id' => 4]], schema(int_schema('id'))); + yield array_to_rows([['id' => 5], ['id' => 6]], schema(int_schema('id'))); } public function statistics(): Statistics @@ -110,7 +110,7 @@ public function statistics(): Statistics $segments->add(new OffsetProcessor(4)); $result = iterator_to_array(ExecutedSegments::of($segments, flow_context(config()))); static::assertCount(1, $result); - static::assertEquals(rows(schema(int_schema('id')), row(['id' => 5]), row(['id' => 6])), $result[0]); + static::assertEquals(array_to_rows([['id' => 5], ['id' => 6]], schema(int_schema('id'))), $result[0]); } public function test_process_with_multiple_batches_offset_spanning_batches(): void @@ -128,9 +128,9 @@ public function schema(): Schema public function extract(FlowContext $context, ?int $limit = null): Generator { - yield rows(schema(int_schema('id')), row(['id' => 1]), row(['id' => 2])); - yield rows(schema(int_schema('id')), row(['id' => 3]), row(['id' => 4]), row(['id' => 5])); - yield rows(schema(int_schema('id')), row(['id' => 6])); + yield array_to_rows([['id' => 1], ['id' => 2]], schema(int_schema('id'))); + yield array_to_rows([['id' => 3], ['id' => 4], ['id' => 5]], schema(int_schema('id'))); + yield array_to_rows([['id' => 6]], schema(int_schema('id'))); } public function statistics(): Statistics @@ -141,8 +141,8 @@ public function statistics(): Statistics $segments->add(new OffsetProcessor(3)); $result = iterator_to_array(ExecutedSegments::of($segments, flow_context(config()))); static::assertCount(2, $result); - static::assertEquals(rows(schema(int_schema('id')), row(['id' => 4]), row(['id' => 5])), $result[0]); - static::assertEquals(rows(schema(int_schema('id')), row(['id' => 6])), $result[1]); + static::assertEquals(array_to_rows([['id' => 4], ['id' => 5]], schema(int_schema('id'))), $result[0]); + static::assertEquals(array_to_rows([['id' => 6]], schema(int_schema('id'))), $result[1]); } public function test_process_with_multiple_batches_offset_within_first_batch(): void @@ -160,8 +160,8 @@ public function schema(): Schema public function extract(FlowContext $context, ?int $limit = null): Generator { - yield rows(schema(int_schema('id')), row(['id' => 1]), row(['id' => 2]), row(['id' => 3])); - yield rows(schema(int_schema('id')), row(['id' => 4]), row(['id' => 5])); + yield array_to_rows([['id' => 1], ['id' => 2], ['id' => 3]], schema(int_schema('id'))); + yield array_to_rows([['id' => 4], ['id' => 5]], schema(int_schema('id'))); } public function statistics(): Statistics @@ -172,18 +172,17 @@ public function statistics(): Statistics $segments->add(new OffsetProcessor(1)); $result = iterator_to_array(ExecutedSegments::of($segments, flow_context(config()))); static::assertCount(2, $result); - static::assertEquals(rows(schema(int_schema('id')), row(['id' => 2]), row(['id' => 3])), $result[0]); - static::assertEquals(rows(schema(int_schema('id')), row(['id' => 4]), row(['id' => 5])), $result[1]); + static::assertEquals(array_to_rows([['id' => 2], ['id' => 3]], schema(int_schema('id'))), $result[0]); + static::assertEquals(array_to_rows([['id' => 4], ['id' => 5]], schema(int_schema('id'))), $result[1]); } public function test_process_with_offset_equal_to_batch_size(): void { - $segments = new Segments(from_rows(rows( - schema(int_schema('id')), - row(['id' => 1]), - row(['id' => 2]), - row(['id' => 3]), - ))); + $segments = new Segments(from_rows(array_to_rows([ + ['id' => 1], + ['id' => 2], + ['id' => 3], + ], schema(int_schema('id'))))); $segments->add(new OffsetProcessor(3)); $result = iterator_to_array(ExecutedSegments::of($segments, flow_context(config()))); static::assertCount(0, $result); @@ -191,7 +190,7 @@ public function test_process_with_offset_equal_to_batch_size(): void public function test_process_with_offset_larger_than_batch_size(): void { - $segments = new Segments(from_rows(rows(schema(int_schema('id')), row(['id' => 1]), row(['id' => 2])))); + $segments = new Segments(from_rows(array_to_rows([['id' => 1], ['id' => 2]], schema(int_schema('id'))))); $segments->add(new OffsetProcessor(5)); $result = iterator_to_array(ExecutedSegments::of($segments, flow_context(config()))); static::assertCount(0, $result); @@ -212,9 +211,9 @@ public function schema(): Schema public function extract(FlowContext $context, ?int $limit = null): Generator { - yield rows(schema(int_schema('id')), row(['id' => 1]), row(['id' => 2])); + yield array_to_rows([['id' => 1], ['id' => 2]], schema(int_schema('id'))); yield rows(schema()); - yield rows(schema(int_schema('id')), row(['id' => 3])); + yield array_to_rows([['id' => 3]], schema(int_schema('id'))); } public function statistics(): Statistics @@ -225,48 +224,46 @@ public function statistics(): Statistics $segments->add(new OffsetProcessor(2)); $result = iterator_to_array(ExecutedSegments::of($segments, flow_context(config()))); static::assertCount(1, $result); - static::assertEquals(rows(schema(int_schema('id')), row(['id' => 3])), $result[0]); + static::assertEquals(array_to_rows([['id' => 3]], schema(int_schema('id'))), $result[0]); } public function test_process_with_offset_smaller_than_batch_size(): void { - $segments = new Segments(from_rows(rows( - schema(int_schema('id')), - row(['id' => 1]), - row(['id' => 2]), - row(['id' => 3]), - row(['id' => 4]), - row(['id' => 5]), - ))); + $segments = new Segments(from_rows(array_to_rows([ + ['id' => 1], + ['id' => 2], + ['id' => 3], + ['id' => 4], + ['id' => 5], + ], schema(int_schema('id'))))); $segments->add(new OffsetProcessor(2)); $result = iterator_to_array(ExecutedSegments::of($segments, flow_context(config()))); static::assertCount(1, $result); static::assertCount(3, $result[0]); static::assertEquals( - rows(schema(int_schema('id')), row(['id' => 3]), row(['id' => 4]), row(['id' => 5])), + array_to_rows([['id' => 3], ['id' => 4], ['id' => 5]], schema(int_schema('id'))), $result[0], ); } public function test_process_with_transformer_before_offset(): void { - $segments = new Segments(from_rows(rows( - schema(int_schema('id')), - row(['id' => 1]), - row(['id' => 2]), - row(['id' => 3]), - row(['id' => 4]), - ))); + $segments = new Segments(from_rows(array_to_rows([ + ['id' => 1], + ['id' => 2], + ['id' => 3], + ['id' => 4], + ], schema(int_schema('id'))))); $segments->add(new ScalarFunctionTransformer('doubled', ref('id')->multiply(lit(2)))); $segments->add(new OffsetProcessor(1)); $result = iterator_to_array(ExecutedSegments::of($segments, flow_context(config()))); static::assertCount(1, $result); static::assertCount(3, $result[0]); $rows = $result[0]; - static::assertEquals(2, $rows->all()[0]->get('id')); - static::assertEquals(4, $rows->all()[0]->get('doubled')); - static::assertEquals(3, $rows->all()[1]->get('id')); - static::assertEquals(6, $rows->all()[1]->get('doubled')); + static::assertEquals(2, $rows->column('id')->value(0)); + static::assertEquals(4, $rows->column('doubled')->value(0)); + static::assertEquals(3, $rows->column('id')->value(1)); + static::assertEquals(6, $rows->column('doubled')->value(1)); } #[DataProvider('offset_values_data_provider')] @@ -274,34 +271,34 @@ public function test_process_with_various_offset_values(int $offset): void { $rowsData = []; for ($i = 1; $i <= 20; $i++) { - $rowsData[] = row(['id' => $i]); + $rowsData[] = ['id' => $i]; } - $segments = new Segments(from_rows(rows(schema(int_schema('id')), ...$rowsData))); + $segments = new Segments(from_rows(array_to_rows($rowsData, schema(int_schema('id'))))); $segments->add(new OffsetProcessor($offset >= 0 ? $offset : 0)); $result = iterator_to_array(ExecutedSegments::of($segments, flow_context(config()))); $expectedCount = max(0, 20 - $offset); $totalRows = array_sum(array_map(static fn($batch) => $batch->count(), $result)); static::assertEquals($expectedCount, $totalRows); if ($expectedCount > 0) { - $firstRowId = $result[0]->first()->get('id'); + // @mago-ignore analysis:mixed-assignment + $firstRowId = $result[0]->column('id')->value(0); static::assertEquals($offset + 1, $firstRowId); } } public function test_process_with_zero_offset_returns_all_data(): void { - $segments = new Segments(from_rows(rows( - schema(int_schema('id')), - row(['id' => 1]), - row(['id' => 2]), - row(['id' => 3]), - ))); + $segments = new Segments(from_rows(array_to_rows([ + ['id' => 1], + ['id' => 2], + ['id' => 3], + ], schema(int_schema('id'))))); $segments->add(new OffsetProcessor(0)); $result = iterator_to_array(ExecutedSegments::of($segments, flow_context(config()))); static::assertCount(1, $result); static::assertCount(3, $result[0]); static::assertEquals( - rows(schema(int_schema('id')), row(['id' => 1]), row(['id' => 2]), row(['id' => 3])), + array_to_rows([['id' => 1], ['id' => 2], ['id' => 3]], schema(int_schema('id'))), $result[0], ); } diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Executor/SourceRowsTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Executor/SourceRowsTest.php index 59bc66e0d8..1fbcd00b14 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Unit/Executor/SourceRowsTest.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Executor/SourceRowsTest.php @@ -12,11 +12,10 @@ use Flow\ETL\Tests\Double\CountingExtractor; use Flow\ETL\Tests\FlowTestCase; +use function Flow\ETL\DSL\array_to_rows; use function Flow\ETL\DSL\flow_context; use function Flow\ETL\DSL\from_array; use function Flow\ETL\DSL\int_schema; -use function Flow\ETL\DSL\row; -use function Flow\ETL\DSL\rows; use function Flow\ETL\DSL\schema; use function iterator_to_array; @@ -40,7 +39,7 @@ public function test_a_stop_reaches_the_source_and_ends_the_count(): void { $extractor = new CountingExtractor( schema(int_schema('id')), - rows(schema(int_schema('id')), row(['id' => 1]), row(['id' => 2]), row(['id' => 3])), + array_to_rows([['id' => 1], ['id' => 2], ['id' => 3]], schema(int_schema('id'))), ); $extractor->withBatchSize(1); $sources = new SourceRows(); diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Executor/StatisticsCollectorTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Executor/StatisticsCollectorTest.php index 9c466dac8a..5beb8b28cd 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Unit/Executor/StatisticsCollectorTest.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Executor/StatisticsCollectorTest.php @@ -17,11 +17,10 @@ use Flow\ETL\Tests\FlowTestCase; use function Flow\ETL\DSL\analyze; +use function Flow\ETL\DSL\array_to_rows; use function Flow\ETL\DSL\flow_context; use function Flow\ETL\DSL\from_array; use function Flow\ETL\DSL\int_schema; -use function Flow\ETL\DSL\row; -use function Flow\ETL\DSL\rows; use function Flow\ETL\DSL\schema; use function Flow\ETL\DSL\str_schema; use function iterator_to_array; @@ -32,10 +31,9 @@ public function test_capture_collects_column_statistics_when_enabled(): void { $collector = new StatisticsCollector(analyze()->withColumnStatistics(), flow_context()); - $collector->capture(rows( + $collector->capture(array_to_rows( + [['id' => 1, 'name' => 'Alice'], ['id' => 2, 'name' => 'Bob']], schema(int_schema('id'), str_schema('name')), - row(['id' => 1, 'name' => 'Alice']), - row(['id' => 2, 'name' => 'Bob']), )); $report = $collector->report(); @@ -51,7 +49,10 @@ public function test_capture_collects_schema_when_enabled(): void { $collector = new StatisticsCollector(analyze()->withSchema(), flow_context()); - $collector->capture(rows(schema(int_schema('id'), str_schema('name')), row(['id' => 1, 'name' => 'Alice']))); + $collector->capture(array_to_rows( + [['id' => 1, 'name' => 'Alice']], + schema(int_schema('id'), str_schema('name')), + )); $report = $collector->report(); @@ -67,8 +68,8 @@ public function test_capture_increments_row_count_correctly(): void { $collector = new StatisticsCollector(true, flow_context()); - $collector->capture(rows(schema(int_schema('id')), row(['id' => 1]), row(['id' => 2]))); - $collector->capture(rows(schema(int_schema('id')), row(['id' => 3]))); + $collector->capture(array_to_rows([['id' => 1], ['id' => 2]], schema(int_schema('id')))); + $collector->capture(array_to_rows([['id' => 3]], schema(int_schema('id')))); $report = $collector->report(); @@ -80,7 +81,7 @@ public function test_capture_is_noop_when_analyze_is_false(): void { $collector = new StatisticsCollector(false, flow_context()); - $collector->capture(rows(schema(int_schema('id')), row(['id' => 1]))); + $collector->capture(array_to_rows([['id' => 1]], schema(int_schema('id')))); static::assertNull($collector->report()); } @@ -89,7 +90,7 @@ public function test_column_statistics_is_null_when_not_enabled(): void { $collector = new StatisticsCollector(analyze(), flow_context()); - $collector->capture(rows(schema(int_schema('id')), row(['id' => 1]))); + $collector->capture(array_to_rows([['id' => 1]], schema(int_schema('id')))); $report = $collector->report(); @@ -135,7 +136,7 @@ public function test_report_returns_report_with_correct_execution_time(): void $collector = new StatisticsCollector(true, $context); $clock->modify('+5 minutes'); - $collector->capture(rows(schema(int_schema('id')), row(['id' => 1]))); + $collector->capture(array_to_rows([['id' => 1]], schema(int_schema('id')))); $clock->modify('+5 minutes'); $report = $collector->report(); @@ -156,7 +157,7 @@ public function test_report_returns_report_with_high_resolution_time(): void { $collector = new StatisticsCollector(true, flow_context()); - $collector->capture(rows(schema(int_schema('id')), row(['id' => 1]))); + $collector->capture(array_to_rows([['id' => 1]], schema(int_schema('id')))); $report = $collector->report(); @@ -168,7 +169,7 @@ public function test_report_returns_report_with_memory_consumption(): void { $collector = new StatisticsCollector(true, flow_context()); - $collector->capture(rows(schema(int_schema('id')), row(['id' => 1]))); + $collector->capture(array_to_rows([['id' => 1]], schema(int_schema('id')))); $report = $collector->report(); @@ -208,7 +209,7 @@ public function test_schema_is_null_when_not_enabled(): void { $collector = new StatisticsCollector(analyze(), flow_context()); - $collector->capture(rows(schema(int_schema('id')), row(['id' => 1]))); + $collector->capture(array_to_rows([['id' => 1]], schema(int_schema('id')))); $report = $collector->report(); diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Executor/WindowFunctionPipelineTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Executor/WindowFunctionPipelineTest.php index eef87678f6..a2622b4c6a 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Unit/Executor/WindowFunctionPipelineTest.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Executor/WindowFunctionPipelineTest.php @@ -9,12 +9,12 @@ use Flow\ETL\Tests\Context\ExecutedSegments; use PHPUnit\Framework\TestCase; +use function Flow\ETL\DSL\array_to_rows; use function Flow\ETL\DSL\config; use function Flow\ETL\DSL\flow_context; use function Flow\ETL\DSL\from_rows; use function Flow\ETL\DSL\int_schema; use function Flow\ETL\DSL\ref; -use function Flow\ETL\DSL\row; use function Flow\ETL\DSL\row_number; use function Flow\ETL\DSL\rows; use function Flow\ETL\DSL\schema; @@ -39,10 +39,9 @@ public function test_handles_empty_input(): void public function test_handles_no_order_by(): void { - $segments = new Segments(from_rows(rows( + $segments = new Segments(from_rows(array_to_rows( + [['dept' => 'IT', 'value' => 100], ['dept' => 'IT', 'value' => 150]], schema(str_schema('dept'), int_schema('value')), - row(['dept' => 'IT', 'value' => 100]), - row(['dept' => 'IT', 'value' => 150]), ))); $window = window()->partitionBy(ref('dept')); @@ -55,8 +54,8 @@ public function test_handles_no_order_by(): void static::assertCount(1, $result); static::assertCount(2, $result[0]); - static::assertEquals(250, $result[0][0]->get('total')); - static::assertEquals(250, $result[0][1]->get('total')); + static::assertEquals(250, $result[0]->column('total')->value(0)); + static::assertEquals(250, $result[0]->column('total')->value(1)); } public function test_handles_single_row_partition(): void @@ -65,8 +64,8 @@ public function test_handles_single_row_partition(): void // one Rows per partition key $schema = schema(str_schema('dept'), int_schema('salary')); $segments = new Segments(from_rows( - rows($schema, row(['dept' => 'IT', 'salary' => 5000])), - rows($schema, row(['dept' => 'HR', 'salary' => 4000])), + array_to_rows([['dept' => 'IT', 'salary' => 5000]], $schema), + array_to_rows([['dept' => 'HR', 'salary' => 4000]], $schema), )); $window = window()->partitionBy(ref('dept'))->orderBy(ref('salary')); @@ -85,8 +84,8 @@ public function test_processes_multiple_partitions_separately(): void { $schema = schema(str_schema('dept'), int_schema('salary')); $segments = new Segments(from_rows( - rows($schema, row(['dept' => 'IT', 'salary' => 5000]), row(['dept' => 'IT', 'salary' => 6000])), - rows($schema, row(['dept' => 'HR', 'salary' => 4000]), row(['dept' => 'HR', 'salary' => 4500])), + array_to_rows([['dept' => 'IT', 'salary' => 5000], ['dept' => 'IT', 'salary' => 6000]], $schema), + array_to_rows([['dept' => 'HR', 'salary' => 4000], ['dept' => 'HR', 'salary' => 4500]], $schema), )); $window = window()->partitionBy(ref('dept'))->orderBy(ref('salary')); @@ -99,21 +98,19 @@ public function test_processes_multiple_partitions_separately(): void static::assertCount(2, $result); static::assertCount(2, $result[0]); - static::assertEquals(1, $result[0][0]->get('row_num')); - static::assertEquals(2, $result[0][1]->get('row_num')); + static::assertEquals(1, $result[0]->column('row_num')->value(0)); + static::assertEquals(2, $result[0]->column('row_num')->value(1)); static::assertCount(2, $result[1]); - static::assertEquals(1, $result[1][0]->get('row_num')); - static::assertEquals(2, $result[1][1]->get('row_num')); + static::assertEquals(1, $result[1]->column('row_num')->value(0)); + static::assertEquals(2, $result[1]->column('row_num')->value(1)); } public function test_processes_single_partition_without_partition_by(): void { - $segments = new Segments(from_rows(rows( + $segments = new Segments(from_rows(array_to_rows( + [['id' => 1, 'value' => 100], ['id' => 2, 'value' => 150], ['id' => 3, 'value' => 200]], schema(int_schema('id'), int_schema('value')), - row(['id' => 1, 'value' => 100]), - row(['id' => 2, 'value' => 150]), - row(['id' => 3, 'value' => 200]), ))); $window = window()->orderBy(ref('id')); @@ -125,18 +122,20 @@ public function test_processes_single_partition_without_partition_by(): void static::assertCount(1, $result); static::assertCount(3, $result[0]); - static::assertEquals(1, $result[0][0]->get('row_num')); - static::assertEquals(2, $result[0][1]->get('row_num')); - static::assertEquals(3, $result[0][2]->get('row_num')); + static::assertEquals(1, $result[0]->column('row_num')->value(0)); + static::assertEquals(2, $result[0]->column('row_num')->value(1)); + static::assertEquals(3, $result[0]->column('row_num')->value(2)); } public function test_sorts_partition_by_order_by(): void { - $segments = new Segments(from_rows(rows( + $segments = new Segments(from_rows(array_to_rows( + [ + ['dept' => 'IT', 'salary' => 6000], + ['dept' => 'IT', 'salary' => 5000], + ['dept' => 'IT', 'salary' => 7000], + ], schema(str_schema('dept'), int_schema('salary')), - row(['dept' => 'IT', 'salary' => 6000]), - row(['dept' => 'IT', 'salary' => 5000]), - row(['dept' => 'IT', 'salary' => 7000]), ))); $window = window()->partitionBy(ref('dept'))->orderBy(ref('salary')); @@ -146,12 +145,12 @@ public function test_sorts_partition_by_order_by(): void $context = flow_context(config()); $result = iterator_to_array(ExecutedSegments::of($segments, $context)); - static::assertEquals(5000, $result[0][0]->get('salary')); - static::assertEquals(6000, $result[0][1]->get('salary')); - static::assertEquals(7000, $result[0][2]->get('salary')); + static::assertEquals(5000, $result[0]->column('salary')->value(0)); + static::assertEquals(6000, $result[0]->column('salary')->value(1)); + static::assertEquals(7000, $result[0]->column('salary')->value(2)); - static::assertEquals(1, $result[0][0]->get('row_num')); - static::assertEquals(2, $result[0][1]->get('row_num')); - static::assertEquals(3, $result[0][2]->get('row_num')); + static::assertEquals(1, $result[0]->column('row_num')->value(0)); + static::assertEquals(2, $result[0]->column('row_num')->value(1)); + static::assertEquals(3, $result[0]->column('row_num')->value(2)); } } diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/ExecutorTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/ExecutorTest.php index ccdab6b92e..efeb84abb4 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Unit/ExecutorTest.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/ExecutorTest.php @@ -23,30 +23,74 @@ use Flow\ETL\Tests\Double\RecordingExtractor; use Flow\ETL\Tests\Double\RecordingFileExtractor; use Flow\ETL\Tests\Double\RecordingRule; +use Flow\ETL\Tests\Double\SpyColumn; use Flow\ETL\Tests\Double\SpyLoader; use Flow\ETL\Tests\Double\ThrowingTransformer; use Flow\ETL\Tests\FlowTestCase; use Flow\ETL\Tests\Mother\NodeMother; +use Flow\ETL\Tests\Mother\PhysicalPlanMother; use Flow\ETL\Tests\Mother\RowsMother; use Flow\ETL\Transformer\LimitTransformer; use Flow\ETL\Transformer\RenameEntryTransformer; use Flow\Filesystem\Tests\Double\RejectingFilter; use RuntimeException; +use function array_filter; +use function array_keys; use function array_map; use function array_sum; +use function array_values; +use function Flow\ETL\DSL\array_to_rows; use function Flow\ETL\DSL\config; use function Flow\ETL\DSL\config_builder; use function Flow\ETL\DSL\from_array; use function Flow\ETL\DSL\from_rows; use function Flow\ETL\DSL\int_schema; -use function Flow\ETL\DSL\row; use function Flow\ETL\DSL\rows; use function Flow\ETL\DSL\schema; use function iterator_to_array; final class ExecutorTest extends FlowTestCase { + public function test_merge_concatenates_the_batches_once(): void + { + $calls = new ArrayObject(); + $schema = schema(int_schema('id')); + $batches = (static function () use ($schema, $calls) { + foreach ([[1], [2], [3], [4], [5]] as $ids) { + $column = array_to_rows([['id' => $ids[0]]], $schema)->column('id'); + + yield Rows::fromColumns($schema, ['id' => new SpyColumn($column, $calls)], 1); + } + })(); + + $merged = (new Executor())->merge($batches, PhysicalPlanMother::reading(from_array([]))); + + static::assertSame([1, 2, 3, 4, 5], $merged->reduceToArray('id')); + static::assertSame( + ['concat'], + array_values(array_filter($calls->getArrayCopy(), static fn(string $call): bool => $call === 'concat')), + ); + } + + public function test_merge_of_empty_batches_keeps_the_last_batch(): void + { + $batches = (static function () { + yield array_to_rows([], schema(int_schema('id'))); + yield array_to_rows([], schema(int_schema('other'))); + })(); + + static::assertSame( + ['other'], + array_keys( + (new Executor()) + ->merge($batches, PhysicalPlanMother::reading(from_array([]))) + ->schema() + ->definitions(), + ), + ); + } + public function test_a_single_pipeline_runs_its_segments_in_order(): void { $segments = new Segments(from_rows(RowsMother::sequentialIds(3))); @@ -241,7 +285,7 @@ public function test_merge_merges_every_batch_into_one_rows(): void ); $batches = (static function () { yield RowsMother::sequentialIds(1); - yield rows(schema(int_schema('id')), row(['id' => 2])); + yield array_to_rows([['id' => 2]], schema(int_schema('id'))); })(); static::assertSame( @@ -351,8 +395,8 @@ public function test_an_abandoned_run_leaves_no_consumed_step_for_the_next_run() { $plan = NodeMother::plan(NodeMother::limit( NodeMother::read(from_rows( - rows(schema(int_schema('id')), row(['id' => 1]), row(['id' => 2])), - rows(schema(int_schema('id')), row(['id' => 3]), row(['id' => 4])), + array_to_rows([['id' => 1], ['id' => 2]], schema(int_schema('id'))), + array_to_rows([['id' => 3], ['id' => 4]], schema(int_schema('id'))), )), 3, )); @@ -449,8 +493,8 @@ public function test_an_abandoned_run_closes_its_span(): void $telemetry = new MemoryTelemetryContext(); $generator = ExecutedPlan::of( NodeMother::plan(NodeMother::read(from_rows( - rows(schema(int_schema('id')), row(['id' => 1])), - rows(schema(int_schema('id')), row(['id' => 2])), + array_to_rows([['id' => 1]], schema(int_schema('id'))), + array_to_rows([['id' => 2]], schema(int_schema('id'))), ))), $telemetry->flowContext, ); diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Extractor/ArrayExtractorTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Extractor/ArrayExtractorTest.php index 0fb182c67f..6537e21283 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Unit/Extractor/ArrayExtractorTest.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Extractor/ArrayExtractorTest.php @@ -7,11 +7,11 @@ use ArrayIterator; use ArrayObject; use Flow\ETL\Cardinality; +use Flow\ETL\Column\PhpBackend; use Flow\ETL\Exception\InferredSchemaException; use Flow\ETL\Exception\InvalidLogicException; use Flow\ETL\Extractor\ArrayExtractor; use Flow\ETL\Extractor\Statistics; -use Flow\ETL\Row\PhpRowHydrator; use Flow\ETL\Schema\Definition; use Flow\ETL\Tests\Double\FixedTmpDirFilesystem; use Flow\ETL\Tests\Double\FreshRowsAggregate; @@ -280,10 +280,7 @@ public function test_a_generator_is_described_exactly_and_extracted_once(): void public function test_positional_rows_are_named_like_the_hydrator_names_them(array $dataset, array $expected): void { static::assertSame($expected, array_keys(from_array($dataset)->schema()->definitions())); - static::assertSame( - $expected, - array_keys(data_frame()->read(from_array($dataset))->fetch()->first()->toArray()), - ); + static::assertSame($expected, array_keys(data_frame()->read(from_array($dataset))->fetch()->toArray()[0])); } public function test_a_bounded_sample_types_only_its_prefix(): void @@ -310,7 +307,7 @@ public function test_a_bounded_sample_types_only_its_prefix(): void iterator_to_array( from_array($dataset) ->inferSchema(infer_schema()->sampleSize(3)) - ->extract(execution_context(config_builder()->hydrator(new PhpRowHydrator())->build())), + ->extract(execution_context(config_builder()->backend(new PhpBackend())->build())), ); } diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Extractor/BatchByExtractorTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Extractor/BatchByExtractorTest.php index e927239705..096a841b5c 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Unit/Extractor/BatchByExtractorTest.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Extractor/BatchByExtractorTest.php @@ -4,26 +4,32 @@ namespace Flow\ETL\Tests\Unit\Extractor; +use DateTimeImmutable; use Flow\ETL\Cardinality; use Flow\ETL\Exception\InvalidArgumentException; use Flow\ETL\Exception\SchemaMismatchException; use Flow\ETL\Extractor\Statistics; use Flow\ETL\Memory\ArrayMemory; +use Flow\ETL\Rows; use Flow\ETL\Tests\Double\DeclaringExtractor; use Flow\ETL\Tests\Double\VaryingBatchesExtractor; +use Flow\Types\Value\Uuid; use PHPUnit\Framework\TestCase; +use function array_map; +use function Flow\ETL\DSL\array_to_rows; use function Flow\ETL\DSL\batched_by; use function Flow\ETL\DSL\config; +use function Flow\ETL\DSL\datetime_schema; use function Flow\ETL\DSL\flow_context; use function Flow\ETL\DSL\from_memory; use function Flow\ETL\DSL\from_rows; use function Flow\ETL\DSL\int_schema; use function Flow\ETL\DSL\ref; -use function Flow\ETL\DSL\row; use function Flow\ETL\DSL\rows; use function Flow\ETL\DSL\schema; use function Flow\ETL\DSL\str_schema; +use function Flow\ETL\DSL\uuid_schema; use function iterator_to_array; final class BatchByExtractorTest extends TestCase @@ -31,8 +37,8 @@ final class BatchByExtractorTest extends TestCase public function test_a_group_spanning_child_batches_answers_to_the_first_batch_schema(): void { $child = new VaryingBatchesExtractor( - rows(schema(int_schema('g'), str_schema('name', nullable: true)), row(['g' => 1, 'name' => 'a'])), - rows(schema(int_schema('g')), row(['g' => 1])), + array_to_rows([['g' => 1, 'name' => 'a']], schema(int_schema('g'), str_schema('name', nullable: true))), + array_to_rows([['g' => 1]], schema(int_schema('g'))), ); $batches = iterator_to_array(batched_by($child, ref('g'))->extract(flow_context(config())), false); @@ -41,11 +47,106 @@ public function test_a_group_spanning_child_batches_answers_to_the_first_batch_s static::assertSame([['g' => 1, 'name' => 'a'], ['g' => 1, 'name' => null]], $batches[0]->toArray()); } + public function test_equal_uuids_stay_in_one_batch(): void + { + $batches = iterator_to_array( + batched_by( + from_rows(array_to_rows( + [ + ['id' => 1, 'u' => new Uuid('00000000-0000-4000-8000-000000000001')], + ['id' => 2, 'u' => new Uuid('00000000-0000-4000-8000-000000000001')], + ['id' => 3, 'u' => new Uuid('00000000-0000-4000-8000-000000000002')], + ], + schema(int_schema('id'), uuid_schema('u')), + )), + ref('u'), + )->extract(flow_context(config())), + false, + ); + + static::assertSame( + [[1, 2], [3]], + array_map(static fn(Rows $rows): array => $rows->reduceToArray('id'), $batches), + ); + } + + public function test_equal_datetimes_stay_in_one_batch(): void + { + $batches = iterator_to_array( + batched_by( + from_rows(array_to_rows( + [ + ['id' => 1, 'at' => new DateTimeImmutable('2026-01-01 00:00:00')], + ['id' => 2, 'at' => new DateTimeImmutable('2026-01-01 00:00:00')], + ['id' => 3, 'at' => new DateTimeImmutable('2026-01-02 00:00:00')], + ], + schema(int_schema('id'), datetime_schema('at')), + )), + ref('at'), + )->extract(flow_context(config())), + false, + ); + + static::assertSame( + [[1, 2], [3]], + array_map(static fn(Rows $rows): array => $rows->reduceToArray('id'), $batches), + ); + } + + public function test_nulls_are_a_group_of_their_own(): void + { + $batches = iterator_to_array( + batched_by( + from_rows(array_to_rows( + [['id' => 1, 'g' => null], ['id' => 2, 'g' => null], ['id' => 3, 'g' => 1]], + schema(int_schema('id'), int_schema('g', nullable: true)), + )), + ref('g'), + )->extract(flow_context(config())), + false, + ); + + static::assertSame( + [[1, 2], [3]], + array_map(static fn(Rows $rows): array => $rows->reduceToArray('id'), $batches), + ); + } + + public function test_a_group_spanning_two_child_batches_is_one_batch(): void + { + $child = new VaryingBatchesExtractor( + array_to_rows([['g' => 1], ['g' => 1]], schema(int_schema('g'))), + array_to_rows([['g' => 1], ['g' => 2]], schema(int_schema('g'))), + ); + + $batches = iterator_to_array(batched_by($child, ref('g'))->extract(flow_context(config())), false); + + static::assertSame( + [[1, 1, 1], [2]], + array_map(static fn(Rows $rows): array => $rows->reduceToArray('g'), $batches), + ); + } + + public function test_a_group_spanning_two_child_batches_counts_toward_the_min_size(): void + { + $child = new VaryingBatchesExtractor( + array_to_rows([['g' => 1], ['g' => 1]], schema(int_schema('g'))), + array_to_rows([['g' => 1], ['g' => 2]], schema(int_schema('g'))), + ); + + $batches = iterator_to_array(batched_by($child, ref('g'), 3)->extract(flow_context(config())), false); + + static::assertSame( + [[1, 1, 1], [2]], + array_map(static fn(Rows $rows): array => $rows->reduceToArray('g'), $batches), + ); + } + public function test_a_later_child_batch_that_widens_the_shape_is_refused(): void { $child = new VaryingBatchesExtractor( - rows(schema(int_schema('g')), row(['g' => 1])), - rows(schema(int_schema('g'), str_schema('extra')), row(['g' => 1, 'extra' => 'x'])), + array_to_rows([['g' => 1]], schema(int_schema('g'))), + array_to_rows([['g' => 1, 'extra' => 'x']], schema(int_schema('g'), str_schema('extra'))), ); $this->expectException(SchemaMismatchException::class); @@ -57,13 +158,15 @@ public function test_a_later_child_batch_that_widens_the_shape_is_refused(): voi public function test_grouping_by_column_with_min_size(): void { $extractor = batched_by( - from_rows(rows( + from_rows(array_to_rows( + [ + ['order_id' => 1, 'item' => 1], + ['order_id' => 2, 'item' => 2], + ['order_id' => 3, 'item' => 3], + ['order_id' => 4, 'item' => 4], + ['order_id' => 5, 'item' => 5], + ], schema(int_schema('order_id'), int_schema('item')), - row(['order_id' => 1, 'item' => 1]), - row(['order_id' => 2, 'item' => 2]), - row(['order_id' => 3, 'item' => 3]), - row(['order_id' => 4, 'item' => 4]), - row(['order_id' => 5, 'item' => 5]), )), ref('order_id'), 3, @@ -77,13 +180,15 @@ public function test_grouping_by_column_with_min_size(): void public function test_grouping_by_column_without_min_size(): void { $extractor = batched_by( - from_rows(rows( + from_rows(array_to_rows( + [ + ['order_id' => 1, 'item' => 1], + ['order_id' => 1, 'item' => 2], + ['order_id' => 2, 'item' => 3], + ['order_id' => 2, 'item' => 4], + ['order_id' => 3, 'item' => 5], + ], schema(int_schema('order_id'), int_schema('item')), - row(['order_id' => 1, 'item' => 1]), - row(['order_id' => 1, 'item' => 2]), - row(['order_id' => 2, 'item' => 3]), - row(['order_id' => 2, 'item' => 4]), - row(['order_id' => 3, 'item' => 5]), )), ref('order_id'), null, @@ -98,11 +203,9 @@ public function test_grouping_by_column_without_min_size(): void public function test_grouping_with_all_unique_values(): void { $extractor = batched_by( - from_rows(rows( + from_rows(array_to_rows( + [['order_id' => 1, 'item' => 1], ['order_id' => 2, 'item' => 2], ['order_id' => 3, 'item' => 3]], schema(int_schema('order_id'), int_schema('item')), - row(['order_id' => 1, 'item' => 1]), - row(['order_id' => 2, 'item' => 2]), - row(['order_id' => 3, 'item' => 3]), )), ref('order_id'), null, @@ -124,14 +227,16 @@ public function test_grouping_with_empty_data(): void public function test_grouping_with_large_group_exceeding_min_size(): void { $extractor = batched_by( - from_rows(rows( + from_rows(array_to_rows( + [ + ['order_id' => 1, 'item' => 1], + ['order_id' => 1, 'item' => 2], + ['order_id' => 1, 'item' => 3], + ['order_id' => 1, 'item' => 4], + ['order_id' => 1, 'item' => 5], + ['order_id' => 2, 'item' => 6], + ], schema(int_schema('order_id'), int_schema('item')), - row(['order_id' => 1, 'item' => 1]), - row(['order_id' => 1, 'item' => 2]), - row(['order_id' => 1, 'item' => 3]), - row(['order_id' => 1, 'item' => 4]), - row(['order_id' => 1, 'item' => 5]), - row(['order_id' => 2, 'item' => 6]), )), ref('order_id'), 2, @@ -145,11 +250,9 @@ public function test_grouping_with_large_group_exceeding_min_size(): void public function test_grouping_with_single_group(): void { $extractor = batched_by( - from_rows(rows( + from_rows(array_to_rows( + [['order_id' => 1, 'item' => 1], ['order_id' => 1, 'item' => 2], ['order_id' => 1, 'item' => 3]], schema(int_schema('order_id'), int_schema('item')), - row(['order_id' => 1, 'item' => 1]), - row(['order_id' => 1, 'item' => 2]), - row(['order_id' => 1, 'item' => 3]), )), ref('order_id'), null, @@ -169,7 +272,7 @@ public function test_min_size_validation(): void public function test_with_schema_does_not_leak_into_a_second_pipeline(): void { - $child = from_rows(rows(schema(int_schema('id')), row(['id' => 1]))); + $child = from_rows(array_to_rows([['id' => 1]], schema(int_schema('id')))); iterator_to_array( batched_by($child, ref('id')) @@ -188,7 +291,7 @@ public function test_with_schema_does_not_leak_into_a_second_pipeline(): void public function test_is_repeatable(): void { static::assertTrue( - batched_by(from_rows(rows(schema(int_schema('id')), row(['id' => 1]))), ref('id'))->isRepeatable(), + batched_by(from_rows(array_to_rows([['id' => 1]], schema(int_schema('id')))), ref('id'))->isRepeatable(), ); } diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Extractor/BatchExtractorTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Extractor/BatchExtractorTest.php index 5b34d76fd4..2086810667 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Unit/Extractor/BatchExtractorTest.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Extractor/BatchExtractorTest.php @@ -10,32 +10,58 @@ use Flow\ETL\Extractor\BatchExtractor; use Flow\ETL\Extractor\Statistics; use Flow\ETL\Memory\ArrayMemory; +use Flow\ETL\Rows; use Flow\ETL\Tests\Double\DeclaringExtractor; use Flow\ETL\Tests\Double\FakeExtractor; use Flow\ETL\Tests\Double\VaryingBatchesExtractor; use Flow\ETL\Tests\FlowTestCase; use Flow\ETL\Tests\Mother\RowsMother; +use function array_map; +use function array_merge; +use function Flow\ETL\DSL\array_to_rows; use function Flow\ETL\DSL\batches; use function Flow\ETL\DSL\flow_context; use function Flow\ETL\DSL\from_memory; use function Flow\ETL\DSL\from_rows; use function Flow\ETL\DSL\int_schema; -use function Flow\ETL\DSL\row; -use function Flow\ETL\DSL\rows; use function Flow\ETL\DSL\schema; use function Flow\ETL\DSL\str_schema; use function iterator_to_array; +use function range; final class BatchExtractorTest extends FlowTestCase { + public function test_windows_built_from_many_child_batches_keep_counts_and_order(): void + { + $children = []; + + for ($batch = 0; $batch < 25; $batch++) { + $children[] = array_to_rows( + array_map(static fn(int $id): array => ['id' => $id], range(($batch * 100) + 1, ($batch * 100) + 100)), + schema(int_schema('id')), + ); + } + + $windows = iterator_to_array( + batches(new VaryingBatchesExtractor(...$children), 1_000)->extract(flow_context()), + false, + ); + + static::assertSame([1000, 1000, 500], array_map(static fn(Rows $rows): int => $rows->count(), $windows)); + static::assertSame( + range(1, 2_500), + array_merge(...array_map(static fn(Rows $rows): array => $rows->reduceToArray('id'), $windows)), + ); + } + public function test_a_buffer_spanning_child_batches_answers_to_the_first_batch_schema(): void { // the second child batch omits a column the first declares nullable - the buffer that spans // both must carry one schema, and the later rows are matched to it rather than mislabelled $child = new VaryingBatchesExtractor( - rows(schema(int_schema('id'), str_schema('name', nullable: true)), row(['id' => 1, 'name' => 'a'])), - rows(schema(int_schema('id')), row(['id' => 2])), + array_to_rows([['id' => 1, 'name' => 'a']], schema(int_schema('id'), str_schema('name', nullable: true))), + array_to_rows([['id' => 2]], schema(int_schema('id'))), ); $batches = iterator_to_array(batches($child, 2)->extract(flow_context()), false); @@ -48,8 +74,8 @@ public function test_a_buffer_spanning_child_batches_answers_to_the_first_batch_ public function test_a_later_child_batch_that_widens_the_shape_is_refused(): void { $child = new VaryingBatchesExtractor( - rows(schema(int_schema('id')), row(['id' => 1])), - rows(schema(int_schema('id'), str_schema('extra')), row(['id' => 2, 'extra' => 'x'])), + array_to_rows([['id' => 1]], schema(int_schema('id'))), + array_to_rows([['id' => 2, 'extra' => 'x']], schema(int_schema('id'), str_schema('extra'))), ); $this->expectException(SchemaMismatchException::class); @@ -98,7 +124,7 @@ public function test_throws_when_constructed_with_zero_batch_size(): void public function test_with_schema_does_not_leak_into_a_second_pipeline(): void { - $child = from_rows(rows(schema(int_schema('id')), row(['id' => 1]))); + $child = from_rows(array_to_rows([['id' => 1]], schema(int_schema('id')))); iterator_to_array( batches($child, 1) diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Extractor/ChainExtractorTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Extractor/ChainExtractorTest.php index 92768cfdb4..80658384f0 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Unit/Extractor/ChainExtractorTest.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Extractor/ChainExtractorTest.php @@ -15,6 +15,7 @@ use Flow\ETL\Tests\FlowTestCase; use Generator; +use function Flow\ETL\DSL\array_to_rows; use function Flow\ETL\DSL\df; use function Flow\ETL\DSL\flow_context; use function Flow\ETL\DSL\from_all; @@ -23,7 +24,6 @@ use function Flow\ETL\DSL\from_memory; use function Flow\ETL\DSL\from_rows; use function Flow\ETL\DSL\int_schema; -use function Flow\ETL\DSL\row; use function Flow\ETL\DSL\rows; use function Flow\ETL\DSL\schema; use function Flow\ETL\DSL\str_schema; @@ -36,8 +36,8 @@ public function test_a_chain_containing_a_data_frame_source_folds_its_schema(): static::assertEquals( schema(int_schema('id', true), str_schema('name', true)), from_all( - from_rows(rows(schema(int_schema('id')), row(['id' => 1]))), - from_data_frame(df()->read(from_rows(rows(schema(str_schema('name')), row(['name' => 'a']))))), + from_rows(array_to_rows([['id' => 1]], schema(int_schema('id')))), + from_data_frame(df()->read(from_rows(array_to_rows([['name' => 'a']], schema(str_schema('name')))))), )->schema(), ); } @@ -63,8 +63,8 @@ public function schema(): Schema public function extract(FlowContext $context, ?int $limit = null): Generator { - yield rows(schema(int_schema('id')), row(['id' => 1])); - yield rows(schema(int_schema('id')), row(['id' => 2])); + yield array_to_rows([['id' => 1]], schema(int_schema('id'))); + yield array_to_rows([['id' => 2]], schema(int_schema('id'))); } public function statistics(): Statistics @@ -84,8 +84,8 @@ public function schema(): Schema public function extract(FlowContext $context, ?int $limit = null): Generator { - yield rows(schema(int_schema('id')), row(['id' => 3])); - yield rows(schema(int_schema('id')), row(['id' => 4])); + yield array_to_rows([['id' => 3]], schema(int_schema('id'))); + yield array_to_rows([['id' => 4]], schema(int_schema('id'))); } public function statistics(): Statistics @@ -95,14 +95,14 @@ public function statistics(): Statistics }); self::assertExtractedRowsEquals( - rows(schema(int_schema('id')), row(['id' => 1]), row(['id' => 2]), row(['id' => 3]), row(['id' => 4])), + array_to_rows([['id' => 1], ['id' => 2], ['id' => 3], ['id' => 4]], schema(int_schema('id'))), $extractor, ); } public function test_with_schema_does_not_leak_into_a_second_pipeline(): void { - $child = from_rows(rows(schema(int_schema('id')), row(['id' => 1]))); + $child = from_rows(array_to_rows([['id' => 1]], schema(int_schema('id')))); iterator_to_array( from_all($child) @@ -120,7 +120,7 @@ public function test_with_schema_does_not_leak_into_a_second_pipeline(): void public function test_is_repeatable(): void { - static::assertTrue(from_all(from_rows(rows(schema(int_schema('id')), row(['id' => 1]))))->isRepeatable()); + static::assertTrue(from_all(from_rows(array_to_rows([['id' => 1]], schema(int_schema('id')))))->isRepeatable()); } public function test_it_sums_the_statistics_of_its_children(): void diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Extractor/CollectingExtractorTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Extractor/CollectingExtractorTest.php index 8c4877f1fb..69f3b49bf5 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Unit/Extractor/CollectingExtractorTest.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Extractor/CollectingExtractorTest.php @@ -11,12 +11,11 @@ use Flow\ETL\Tests\Double\DeclaringExtractor; use Flow\ETL\Tests\FlowTestCase; +use function Flow\ETL\DSL\array_to_rows; use function Flow\ETL\DSL\flow_context; use function Flow\ETL\DSL\from_memory; use function Flow\ETL\DSL\from_rows; use function Flow\ETL\DSL\int_schema; -use function Flow\ETL\DSL\row; -use function Flow\ETL\DSL\rows; use function Flow\ETL\DSL\schema; use function Flow\ETL\DSL\str_schema; use function iterator_to_array; @@ -27,8 +26,8 @@ public function test_collecting_every_batch_into_one(): void { $collected = iterator_to_array( (new CollectingExtractor(from_rows( - rows(schema(int_schema('id')), row(['id' => 1])), - rows(schema(int_schema('id')), row(['id' => 2])), + array_to_rows([['id' => 1]], schema(int_schema('id'))), + array_to_rows([['id' => 2]], schema(int_schema('id'))), )))->extract(flow_context()), false, ); @@ -39,7 +38,7 @@ public function test_collecting_every_batch_into_one(): void public function test_with_schema_does_not_leak_into_a_second_pipeline(): void { - $child = from_rows(rows(schema(int_schema('id')), row(['id' => 1]))); + $child = from_rows(array_to_rows([['id' => 1]], schema(int_schema('id')))); iterator_to_array( (new CollectingExtractor($child)) @@ -57,7 +56,9 @@ public function test_with_schema_does_not_leak_into_a_second_pipeline(): void public function test_is_repeatable(): void { static::assertTrue( - (new CollectingExtractor(from_rows(rows(schema(int_schema('id')), row(['id' => 1])))))->isRepeatable(), + (new CollectingExtractor(from_rows(array_to_rows([[ + 'id' => 1, + ]], schema(int_schema('id'))))))->isRepeatable(), ); } diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Extractor/DataFrameExtractorTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Extractor/DataFrameExtractorTest.php index dcb1388227..dc607111f5 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Unit/Extractor/DataFrameExtractorTest.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Extractor/DataFrameExtractorTest.php @@ -25,6 +25,7 @@ use function array_map; use function array_merge; +use function Flow\ETL\DSL\array_to_rows; use function Flow\ETL\DSL\config; use function Flow\ETL\DSL\df; use function Flow\ETL\DSL\flow_context; @@ -35,8 +36,6 @@ use function Flow\ETL\DSL\int_schema; use function Flow\ETL\DSL\lit; use function Flow\ETL\DSL\ref; -use function Flow\ETL\DSL\row; -use function Flow\ETL\DSL\rows; use function Flow\ETL\DSL\schema; use function Flow\ETL\DSL\str_schema; use function iterator_to_array; @@ -46,7 +45,7 @@ final class DataFrameExtractorTest extends FlowTestCase public function test_a_build_error_in_the_wrapped_frame_is_not_a_schema_refusal(): void { $extractor = from_data_frame( - df()->read(from_rows(rows(schema(int_schema('id')), row(['id' => 1]))))->select('nope'), + df()->read(from_rows(array_to_rows([['id' => 1]], schema(int_schema('id')))))->select('nope'), ); try { @@ -61,11 +60,10 @@ public function test_a_build_error_in_the_wrapped_frame_is_not_a_schema_refusal( public function test_a_declared_schema_wins_over_the_wrapped_frame(): void { - $extractor = from_data_frame(df()->read(from_rows(rows( - schema(int_schema('id')), - row(['id' => 1]), - row(['id' => 2]), - )))) + $extractor = from_data_frame(df()->read(from_rows(array_to_rows([ + ['id' => 1], + ['id' => 2], + ], schema(int_schema('id')))))) ->withSchema(schema(str_schema('id'))); $batches = iterator_to_array($extractor->extract(flow_context(config()))); @@ -75,6 +73,19 @@ public function test_a_declared_schema_wins_over_the_wrapped_frame(): void static::assertSame([['id' => '1'], ['id' => '2']], $batches[0]->toArray()); } + public function test_a_declared_schema_narrower_than_the_frame_projects_its_columns(): void + { + $extractor = from_data_frame(df()->read(from_rows(array_to_rows( + [['id' => 1, 'name' => 'one'], ['id' => 2, 'name' => 'two']], + schema(int_schema('id'), str_schema('name')), + )))) + ->withSchema(schema(int_schema('id'))); + + $batches = iterator_to_array($extractor->extract(flow_context(config()))); + + static::assertSame([['id' => 1], ['id' => 2]], $batches[0]->toArray()); + } + public function test_a_declared_schema_describes_a_frame_that_cannot_describe_itself(): void { $declared = schema(int_schema('id')); @@ -146,7 +157,7 @@ public function test_a_limit_given_to_extract_reaches_the_frames_source(): void { $source = new RecordingFileExtractor( schema(int_schema('id')), - rows(schema(int_schema('id')), row(['id' => 1]), row(['id' => 2]), row(['id' => 3])), + array_to_rows([['id' => 1], ['id' => 2], ['id' => 3]], schema(int_schema('id'))), ); $batches = iterator_to_array(from_data_frame(df()->read($source))->extract(flow_context(config()), 2), false); @@ -174,7 +185,7 @@ public function test_a_refusal_from_the_wrapped_frame_is_re_raised_unchanged(): public function test_the_frame_is_frozen_at_construction(): void { - $inner = df()->read(from_rows(rows(schema(int_schema('id')), row(['id' => 1])))); + $inner = df()->read(from_rows(array_to_rows([['id' => 1]], schema(int_schema('id'))))); $extractor = from_data_frame($inner); $inner->withEntry('doubled', ref('id')->multiply(lit(2))); @@ -187,7 +198,9 @@ public function test_it_is_constructed_from_a_frame(): void static::assertSame( [['id' => 1]], df() - ->read(new DataFrameExtractor(df()->read(from_rows(rows(schema(int_schema('id')), row(['id' => 1])))))) + ->read(new DataFrameExtractor(df()->read(from_rows(array_to_rows([[ + 'id' => 1, + ]], schema(int_schema('id'))))))) ->fetch() ->toArray(), ); @@ -209,7 +222,7 @@ public function test_an_error_handler_set_after_embedding_does_not_reach_the_fra public function test_extract_runs_the_snapshot_plan_when_reached_through_a_chain_wrapper(): void { - $inner = df()->read(from_rows(rows(schema(int_schema('id')), row(['id' => 1]), row(['id' => 2]))))->limit(1); + $inner = df()->read(from_rows(array_to_rows([['id' => 1], ['id' => 2]], schema(int_schema('id')))))->limit(1); static::assertSame( [['id' => 1], ['id' => 3]], @@ -224,8 +237,8 @@ public function test_extract_stops_on_signal_stop(): void { $counting = new CountingExtractor( schema(int_schema('id')), - rows(schema(int_schema('id')), row(['id' => 1])), - rows(schema(int_schema('id')), row(['id' => 2])), + array_to_rows([['id' => 1]], schema(int_schema('id'))), + array_to_rows([['id' => 2]], schema(int_schema('id'))), ); $counting->withBatchSize(1); $generator = from_data_frame(df()->read($counting))->extract(flow_context(config())); @@ -239,7 +252,10 @@ public function test_extract_stops_on_signal_stop(): void public function test_the_snapshot_source_is_read_once_across_schema_and_extract(): void { - $counting = new CountingExtractor(schema(int_schema('id')), rows(schema(int_schema('id')), row(['id' => 1]))); + $counting = new CountingExtractor( + schema(int_schema('id')), + array_to_rows([['id' => 1]], schema(int_schema('id'))), + ); $extractor = from_data_frame(df()->read($counting)->limit(1)); $extractor->schema(); @@ -254,7 +270,7 @@ public function test_extract_opens_and_closes_one_dataframe_span_for_the_wrapped { $telemetry = new MemoryTelemetryContext(); $extractor = from_data_frame( - df($telemetry->config)->read(from_rows(rows(schema(int_schema('id')), row(['id' => 1])))), + df($telemetry->config)->read(from_rows(array_to_rows([['id' => 1]], schema(int_schema('id'))))), ) ->withSchema(schema(int_schema('id'))); @@ -267,18 +283,17 @@ public function test_extract_opens_and_closes_one_dataframe_span_for_the_wrapped public function test_extracting_from_another_data_frame(): void { $extractor = from_data_frame(df()->read(from_rows( - rows(schema(str_schema('value')), row(['value' => 'test']), row(['value' => 'test'])), - rows(schema(str_schema('value')), row(['value' => 'test']), row(['value' => 'test'])), + array_to_rows([['value' => 'test'], ['value' => 'test']], schema(str_schema('value'))), + array_to_rows([['value' => 'test'], ['value' => 'test']], schema(str_schema('value'))), ))); self::assertExtractedRowsEquals( - rows( - schema(str_schema('value')), - row(['value' => 'test']), - row(['value' => 'test']), - row(['value' => 'test']), - row(['value' => 'test']), - ), + array_to_rows([ + ['value' => 'test'], + ['value' => 'test'], + ['value' => 'test'], + ['value' => 'test'], + ], schema(str_schema('value'))), $extractor, ); } diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Extractor/FeedExtractorTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Extractor/FeedExtractorTest.php index 9e09913886..0aeb21e637 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Unit/Extractor/FeedExtractorTest.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Extractor/FeedExtractorTest.php @@ -12,11 +12,10 @@ use Flow\ETL\Tests\FlowTestCase; use function array_map; +use function Flow\ETL\DSL\array_to_rows; use function Flow\ETL\DSL\config; use function Flow\ETL\DSL\flow_context; use function Flow\ETL\DSL\int_schema; -use function Flow\ETL\DSL\row; -use function Flow\ETL\DSL\rows; use function Flow\ETL\DSL\schema; use function Flow\ETL\DSL\str_schema; @@ -38,7 +37,7 @@ public function test_finish_ends_the_stream(): void $collected = []; $completed = false; - $extractor->feed(rows(schema(int_schema('id')), row(['id' => 1]))); + $extractor->feed(array_to_rows([['id' => 1]], schema(int_schema('id')))); $fiber = new Fiber(static function () use ($extractor, $context, &$collected, &$completed): void { foreach ($extractor->extract($context) as $rows) { @@ -71,7 +70,7 @@ public function test_resumes_with_the_next_fed_batch(): void $context = flow_context(config()); $collected = []; - $extractor->feed(rows(schema(int_schema('id')), row(['id' => 1]))); + $extractor->feed(array_to_rows([['id' => 1]], schema(int_schema('id')))); $fiber = new Fiber(static function () use ($extractor, $context, &$collected): void { foreach ($extractor->extract($context) as $rows) { @@ -80,7 +79,7 @@ public function test_resumes_with_the_next_fed_batch(): void }); $fiber->start(); - $extractor->feed(rows(schema(int_schema('id')), row(['id' => 2]))); + $extractor->feed(array_to_rows([['id' => 2]], schema(int_schema('id')))); $fiber->resume(); static::assertTrue($fiber->isSuspended()); @@ -97,7 +96,7 @@ public function test_stop_signal_at_a_batch_yield_ends_the_stream(): void $batch = null; $valid = null; - $extractor->feed(rows(schema(int_schema('id')), row(['id' => 1]))); + $extractor->feed(array_to_rows([['id' => 1]], schema(int_schema('id')))); $fiber = new Fiber(static function () use ($extractor, $context, &$batch, &$valid): void { $generator = $extractor->extract($context); @@ -120,7 +119,7 @@ public function test_stop_signal_at_the_trailing_yield_ends_the_stream(): void $trailing = null; $valid = null; - $extractor->feed(rows(schema(int_schema('id')), row(['id' => 1]))); + $extractor->feed(array_to_rows([['id' => 1]], schema(int_schema('id')))); $fiber = new Fiber(static function () use ($extractor, $context, &$trailing, &$valid): void { $generator = $extractor->extract($context); @@ -153,7 +152,7 @@ public function test_yields_the_fed_batch_and_a_trailing_empty_rows_then_suspend $context = flow_context(config()); $collected = []; - $extractor->feed(rows(schema(int_schema('id')), row(['id' => 1]), row(['id' => 2]))); + $extractor->feed(array_to_rows([['id' => 1], ['id' => 2]], schema(int_schema('id')))); $fiber = new Fiber(static function () use ($extractor, $context, &$collected): void { foreach ($extractor->extract($context) as $rows) { diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Extractor/File/DerivedSchemaTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Extractor/File/DerivedSchemaTest.php new file mode 100644 index 0000000000..eda8360df9 --- /dev/null +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Extractor/File/DerivedSchemaTest.php @@ -0,0 +1,37 @@ +expectNotToPerformAssertions(); + + (new DerivedSchema(schema(int_schema('id')), 'memory://a.floe'))->refuseDivergence( + 'memory://b.floe', + schema(int_schema('id')), + ); + } + + public function test_a_file_of_another_schema_is_refused_naming_both_files(): void + { + $this->expectException(InferredSchemaException::class); + $this->expectExceptionMessage('memory://b.floe'); + + (new DerivedSchema(schema(int_schema('id')), 'memory://a.floe'))->refuseDivergence( + 'memory://b.floe', + schema(str_schema('name')), + ); + } +} diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Extractor/FileColumnsTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Extractor/File/FileColumnsTest.php similarity index 92% rename from src/core/etl/tests/Flow/ETL/Tests/Unit/Extractor/FileColumnsTest.php rename to src/core/etl/tests/Flow/ETL/Tests/Unit/Extractor/File/FileColumnsTest.php index 88366141bc..4a59e11ad2 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Unit/Extractor/FileColumnsTest.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Extractor/File/FileColumnsTest.php @@ -2,10 +2,10 @@ declare(strict_types=1); -namespace Flow\ETL\Tests\Unit\Extractor; +namespace Flow\ETL\Tests\Unit\Extractor\File; use Flow\ETL\Exception\InvalidArgumentException; -use Flow\ETL\Extractor\SourceFile; +use Flow\ETL\Extractor\File\SourceFile; use Flow\ETL\Tests\Context\FileColumnsContext; use Flow\ETL\Tests\FlowTestCase; @@ -24,13 +24,13 @@ public function test_a_hive_null_partition_value_stays_null(): void $fileColumns = FileColumnsContext::discovering(names: ['year' => true]); static::assertSame( - ['name' => 'Norbert', 'year' => null], + ['year' => null], $fileColumns ->forFile( new SourceFile(path('memory://orders/year=__HIVE_DEFAULT_PARTITION__/data.csv')), $fileColumns->declare(schema()), ) - ->fill(['name' => 'Norbert']), + ->values(), ); } @@ -39,10 +39,10 @@ public function test_a_partition_missing_from_the_path_stays_null(): void $fileColumns = FileColumnsContext::discovering(names: ['year' => true]); static::assertSame( - ['name' => 'Norbert', 'year' => null], + ['year' => null], $fileColumns ->forFile(new SourceFile(path('memory://orders/data.csv')), $fileColumns->declare(schema())) - ->fill(['name' => 'Norbert']), + ->values(), ); } @@ -94,10 +94,10 @@ public function test_forFile_types_the_value_with_the_declared_partition_type(): $fileColumns = FileColumnsContext::discovering(types: partition_types(year: type_integer())); static::assertSame( - ['name' => 'Norbert', 'year' => 2024], + ['year' => 2024], $fileColumns ->forFile(new SourceFile(path('memory://orders/year=2024/data.csv')), $fileColumns->declare(schema())) - ->fill(['name' => 'Norbert']), + ->values(), ); } @@ -106,10 +106,10 @@ public function test_forFile_without_a_declared_type_leaves_the_path_value_a_str $fileColumns = FileColumnsContext::discovering(); static::assertSame( - ['name' => 'Norbert', 'year' => '2024'], + ['year' => '2024'], $fileColumns ->forFile(new SourceFile(path('memory://orders/year=2024/data.csv')), $fileColumns->declare(schema())) - ->fill(['name' => 'Norbert']), + ->values(), ); } @@ -129,13 +129,13 @@ public function test_the_declared_schema_types_the_value_forFile_produces(): voi $fileColumns = FileColumnsContext::discovering(); static::assertSame( - ['name' => 'Norbert', 'year' => 2024], + ['year' => 2024], $fileColumns ->forFile( new SourceFile(path('memory://orders/year=2024/data.csv')), $fileColumns->declare(schema(str_schema('name'), int_schema('year'))), ) - ->fill(['name' => 'Norbert']), + ->values(), ); } diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Extractor/File/FileConstantsTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Extractor/File/FileConstantsTest.php new file mode 100644 index 0000000000..0e901b20c3 --- /dev/null +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Extractor/File/FileConstantsTest.php @@ -0,0 +1,73 @@ +values()); + } + + public function test_values_carry_the_uri_when_metadata_columns_are_on(): void + { + static::assertSame( + ['_input_file_uri' => 'memory://orders/data.csv'], + (new FileConstants('memory://orders/data.csv', [], []))->values(), + ); + } + + public function test_values_carry_every_partition_and_null_for_one_the_path_lacks(): void + { + static::assertSame( + ['_input_file_uri' => 'memory://orders/year=2024/data.csv', 'year' => 2024, 'month' => null], + (new FileConstants( + 'memory://orders/year=2024/data.csv', + ['year' => false, 'month' => true], + ['year' => 2024], + ))->values(), + ); + } + + public function test_fill_rows_returns_a_batch_with_nothing_to_add_as_it_is(): void + { + $rows = array_to_rows([['name' => 'Norbert']], schema(str_schema('name'))); + + static::assertSame($rows, (new FileConstants(null, [], []))->fillRows( + $rows, + $rows->schema(), + new PhpBackend(), + )); + } + + public function test_fill_rows_adds_the_constants_to_every_row_under_the_declared_schema(): void + { + $declared = schema(str_schema('name'), str_schema('_input_file_uri'), int_schema('year')); + + static::assertEquals( + array_to_rows([ + ['name' => 'Norbert', '_input_file_uri' => 'memory://orders/data.csv', 'year' => 2024], + ['name' => 'Flow', '_input_file_uri' => 'memory://orders/data.csv', 'year' => 2024], + ], $declared), + (new FileConstants('memory://orders/data.csv', ['year' => false], ['year' => 2024]))->fillRows( + array_to_rows([ + ['name' => 'Norbert'], + ['name' => 'Flow'], + ], schema(str_schema('name'))), + $declared, + new PhpBackend(), + ), + ); + } +} diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Extractor/File/FileReadLoopTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Extractor/File/FileReadLoopTest.php new file mode 100644 index 0000000000..7357642e5f --- /dev/null +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Extractor/File/FileReadLoopTest.php @@ -0,0 +1,217 @@ + [1, 2], + 'memory://orders/year=2024/b.csv' => [3], + ]); + + $rows = iterator_to_array($loop->read( + [ + new SourceFile(path('memory://orders/year=2023/a.csv')), + new SourceFile(path('memory://orders/year=2024/b.csv')), + ], + $batches, + 10, + new PhpBackend(), + new ReadWindow(), + )); + + static::assertSame( + [[['id' => 1, 'year' => '2023'], ['id' => 2, 'year' => '2023']], [['id' => 3, 'year' => '2024']]], + array_map(static fn(Rows $batch): array => $batch->toArray(), $rows), + ); + } + + public function test_stop_after_the_first_batch_ends_the_read(): void + { + $read = (new FileReadLoop(FileColumnsContext::discovering(names: []), schema(int_schema('id'))))->read( + [new SourceFile(path('memory://a.csv')), new SourceFile(path('memory://b.csv'))], + new InMemoryFileBatches(['memory://a.csv' => [1, 2, 3], 'memory://b.csv' => [4]]), + 1, + new PhpBackend(), + new ReadWindow(), + ); + + static::assertSame([['id' => 1]], $read->current()->toArray()); + + $read->send(Signal::STOP); + + static::assertFalse($read->valid()); + } + + public function test_a_limit_inside_a_file_ends_the_read_and_is_pushed_down(): void + { + $batches = new InMemoryFileBatches(['memory://a.csv' => [1, 2, 3], 'memory://b.csv' => [4]]); + + $rows = iterator_to_array((new FileReadLoop( + FileColumnsContext::discovering(names: []), + schema(int_schema('id')), + ))->read( + [new SourceFile(path('memory://a.csv')), new SourceFile(path('memory://b.csv'))], + $batches, + 10, + new PhpBackend(), + new ReadWindow(limit: 2), + )); + + static::assertSame( + [[['id' => 1], ['id' => 2]]], + array_map(static fn(Rows $batch): array => $batch->toArray(), $rows), + ); + static::assertEquals(['memory://a.csv' => [new ReadWindow(0, 2)]], $batches->windows); + } + + public function test_a_limit_at_a_file_boundary_opens_no_further_file(): void + { + $batches = new InMemoryFileBatches(['memory://a.csv' => [1, 2], 'memory://b.csv' => [3]]); + + $rows = iterator_to_array((new FileReadLoop( + FileColumnsContext::discovering(names: []), + schema(int_schema('id')), + ))->read( + [new SourceFile(path('memory://a.csv')), new SourceFile(path('memory://b.csv'))], + $batches, + 1, + new PhpBackend(), + new ReadWindow(limit: 2), + )); + + static::assertCount(2, $rows); + static::assertSame(['memory://a.csv'], array_keys($batches->windows)); + } + + public function test_the_offset_a_file_leaves_and_the_rows_still_wanted_reach_the_next_file(): void + { + $batches = new InMemoryFileBatches([ + 'memory://a.csv' => [1, 2], + 'memory://b.csv' => [3, 4], + 'memory://c.csv' => [5, 6, 7], + ]); + + $rows = iterator_to_array((new FileReadLoop( + FileColumnsContext::discovering(names: []), + schema(int_schema('id')), + ))->read( + [ + new SourceFile(path('memory://a.csv')), + new SourceFile(path('memory://b.csv')), + new SourceFile(path('memory://c.csv')), + ], + $batches, + 10, + new PhpBackend(), + new ReadWindow(offset: 3, limit: 2), + )); + + static::assertSame( + [[['id' => 4]], [['id' => 5]]], + array_map(static fn(Rows $batch): array => $batch->toArray(), $rows), + ); + static::assertEquals( + [ + 'memory://a.csv' => [new ReadWindow(3, 2)], + 'memory://b.csv' => [new ReadWindow(1, 2)], + 'memory://c.csv' => [new ReadWindow(0, 1)], + ], + $batches->windows, + ); + } + + public function test_no_sources_read_nothing(): void + { + static::assertSame( + [], + iterator_to_array((new FileReadLoop( + FileColumnsContext::discovering(names: []), + schema(int_schema('id')), + ))->read([], new InMemoryFileBatches([]), 10, new PhpBackend(), new ReadWindow())), + ); + } + + public function test_the_loop_slices_the_offset_a_format_cannot_skip_across_files(): void + { + $batches = new InMemoryUnskippableFileBatches([ + 'memory://a.csv' => [1, 2], + 'memory://b.csv' => [3, 4], + 'memory://c.csv' => [5, 6, 7], + ]); + + $rows = iterator_to_array( + (new FileReadLoop(FileColumnsContext::discovering(names: []), schema(int_schema('id'))))->read( + [ + new SourceFile(path('memory://a.csv')), + new SourceFile(path('memory://b.csv')), + new SourceFile(path('memory://c.csv')), + ], + $batches, + 10, + new PhpBackend(), + new ReadWindow(offset: 3, limit: 2), + ), + false, + ); + + static::assertSame( + [[['id' => 4]], [['id' => 5]]], + array_map(static fn(Rows $batch): array => $batch->toArray(), $rows), + ); + static::assertEquals( + [ + 'memory://a.csv' => [new ReadWindow(0, 5)], + 'memory://b.csv' => [new ReadWindow(0, 3)], + 'memory://c.csv' => [new ReadWindow(0, 1)], + ], + $batches->windows, + ); + } + + public function test_the_loop_skips_the_offset_and_trims_to_the_limit_for_a_format_that_cannot_skip(): void + { + $batches = new InMemoryUnskippableFileBatches(['memory://a.csv' => [1, 2, 3, 4, 5, 6, 7, 8, 9, 10]]); + + $rows = iterator_to_array( + (new FileReadLoop(FileColumnsContext::discovering(names: []), schema(int_schema('id'))))->read( + [new SourceFile(path('memory://a.csv'))], + $batches, + 10, + new PhpBackend(), + new ReadWindow(offset: 3, limit: 2), + ), + false, + ); + + static::assertSame( + [[['id' => 4], ['id' => 5]]], + array_map(static fn(Rows $batch): array => $batch->toArray(), $rows), + ); + static::assertEquals(['memory://a.csv' => [new ReadWindow(0, 5)]], $batches->windows); + } +} diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Extractor/FileReadingTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Extractor/File/FileReadingTest.php similarity index 97% rename from src/core/etl/tests/Flow/ETL/Tests/Unit/Extractor/FileReadingTest.php rename to src/core/etl/tests/Flow/ETL/Tests/Unit/Extractor/File/FileReadingTest.php index ee21c126c1..f822dd8cb0 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Unit/Extractor/FileReadingTest.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Extractor/File/FileReadingTest.php @@ -2,10 +2,10 @@ declare(strict_types=1); -namespace Flow\ETL\Tests\Unit\Extractor; +namespace Flow\ETL\Tests\Unit\Extractor\File; use Flow\ETL\Exception\RuntimeException; -use Flow\ETL\Extractor\SourceFile; +use Flow\ETL\Extractor\File\SourceFile; use Flow\ETL\Tests\Context\SelfDescribingFilesContext; use Flow\ETL\Tests\Double\FileReadingExtractor; use Flow\ETL\Tests\FlowTestCase; diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Extractor/FooterStatisticsTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Extractor/File/FooterStatisticsTest.php similarity index 95% rename from src/core/etl/tests/Flow/ETL/Tests/Unit/Extractor/FooterStatisticsTest.php rename to src/core/etl/tests/Flow/ETL/Tests/Unit/Extractor/File/FooterStatisticsTest.php index c8bbc86615..3f577f7a86 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Unit/Extractor/FooterStatisticsTest.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Extractor/File/FooterStatisticsTest.php @@ -2,11 +2,11 @@ declare(strict_types=1); -namespace Flow\ETL\Tests\Unit\Extractor; +namespace Flow\ETL\Tests\Unit\Extractor\File; use Flow\ETL\Cardinality; use Flow\ETL\Exception\InvalidArgumentException; -use Flow\ETL\Extractor\FooterStatistics; +use Flow\ETL\Extractor\File\FooterStatistics; use Flow\ETL\Extractor\Statistics; use Flow\ETL\Tests\FlowTestCase; diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Extractor/File/InferredColumnsTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Extractor/File/InferredColumnsTest.php new file mode 100644 index 0000000000..c4f068b361 --- /dev/null +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Extractor/File/InferredColumnsTest.php @@ -0,0 +1,56 @@ +expectNotToPerformAssertions(); + + (new InferredColumns( + schema(int_schema('id'), str_schema('name')), + ['_input_file_uri'], + new SchemaInference(), + 'memory://a.csv', + ))->refuseDivergence('memory://b.csv', ['name', 'id', 'id', '_input_file_uri']); + } + + public function test_a_file_without_columns_passes(): void + { + $this->expectNotToPerformAssertions(); + + (new InferredColumns( + schema(int_schema('id'), str_schema('name')), + ['_input_file_uri'], + new SchemaInference(), + 'memory://a.csv', + ))->refuseDivergence('memory://empty.csv', []); + } + + public function test_a_file_whose_columns_diverge_is_refused_naming_both_sources(): void + { + $this->expectException(InferredSchemaException::class); + $this->expectExceptionMessage( + 'Columns of memory://b.csv do not match the schema inferred from memory://a.csv: unexpected [age], missing [name]', + ); + + (new InferredColumns( + schema(int_schema('id'), str_schema('name')), + ['_input_file_uri'], + new SchemaInference(), + 'memory://a.csv', + ))->refuseDivergence('memory://b.csv', ['id', 'age']); + } +} diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Extractor/ListedFilesTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Extractor/File/ListedFilesTest.php similarity index 94% rename from src/core/etl/tests/Flow/ETL/Tests/Unit/Extractor/ListedFilesTest.php rename to src/core/etl/tests/Flow/ETL/Tests/Unit/Extractor/File/ListedFilesTest.php index 74e434f185..184d1081fd 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Unit/Extractor/ListedFilesTest.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Extractor/File/ListedFilesTest.php @@ -2,12 +2,12 @@ declare(strict_types=1); -namespace Flow\ETL\Tests\Unit\Extractor; +namespace Flow\ETL\Tests\Unit\Extractor\File; use Flow\ETL\Cardinality; use Flow\ETL\Exception\InvalidArgumentException; -use Flow\ETL\Extractor\ListedFiles; -use Flow\ETL\Extractor\SourceFile; +use Flow\ETL\Extractor\File\ListedFiles; +use Flow\ETL\Extractor\File\SourceFile; use Flow\ETL\Tests\FlowTestCase; use Flow\Filesystem\FileStatus; @@ -18,7 +18,7 @@ final class ListedFilesTest extends FlowTestCase { public function test_a_zero_byte_member_reported_as_null_makes_the_total_unknown(): void { - $empty = native_local_filesystem()->status(path(__DIR__ . '/../../Fixtures/empty.txt')); + $empty = native_local_filesystem()->status(path(__DIR__ . '/../../../Fixtures/empty.txt')); static::assertNotNull($empty); static::assertEquals( diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Extractor/MetadataColumnsExtractorTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Extractor/File/MetadataColumnsExtractorTest.php similarity index 90% rename from src/core/etl/tests/Flow/ETL/Tests/Unit/Extractor/MetadataColumnsExtractorTest.php rename to src/core/etl/tests/Flow/ETL/Tests/Unit/Extractor/File/MetadataColumnsExtractorTest.php index 9adeff0f4b..d40f37b3a9 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Unit/Extractor/MetadataColumnsExtractorTest.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Extractor/File/MetadataColumnsExtractorTest.php @@ -2,12 +2,12 @@ declare(strict_types=1); -namespace Flow\ETL\Tests\Unit\Extractor; +namespace Flow\ETL\Tests\Unit\Extractor\File; use Flow\ETL\Config; use Flow\ETL\Config\ConfigBuilder; use Flow\ETL\Extractor; -use Flow\ETL\Extractor\MetadataColumnsExtractor; +use Flow\ETL\Extractor\File\MetadataColumnsExtractor; use Flow\ETL\Tests\Context\ExtractorClasses; use Flow\ETL\Tests\FlowTestCase; use Generator; @@ -66,11 +66,9 @@ public function test_the_capability_is_declared_only_by_sources_that_have_metada 'Flow\ETL\Adapter\Excel\ExcelExtractor', 'Flow\ETL\Adapter\GoogleSheet\GoogleSheetExtractor', 'Flow\ETL\Adapter\JSON\JSONMachine\JsonExtractor', - 'Flow\ETL\Adapter\JSON\JSONMachine\JsonLinesExtractor', 'Flow\ETL\Adapter\Parquet\ParquetExtractor', 'Flow\ETL\Adapter\Text\TextExtractor', 'Flow\ETL\Adapter\XML\XMLParserExtractor', - 'Flow\ETL\Adapter\XML\XMLReaderExtractor', 'Flow\Floe\FloeExtractor', ], self::classesDeclaringMetadataColumns(), @@ -79,7 +77,7 @@ public function test_the_capability_is_declared_only_by_sources_that_have_metada public function test_the_census_covers_every_extractor_in_src(): void { - static::assertCount(33, ExtractorClasses::all()); + static::assertCount(31, ExtractorClasses::all()); } public function test_the_config_delegator_is_gone(): void diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Extractor/File/PartitionColumnsTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Extractor/File/PartitionColumnsTest.php new file mode 100644 index 0000000000..e4d35a6b2c --- /dev/null +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Extractor/File/PartitionColumnsTest.php @@ -0,0 +1,105 @@ +declare(schema(int_schema('date')), ['date' => true]), + ); + } + + public function test_declare_adds_each_name_with_the_nullability_it_was_given(): void + { + static::assertEquals( + schema(int_schema('id'), str_schema('date'), str_schema('region', nullable: true)), + (new PartitionColumns(new MemoryFilesystem()))->declare(schema(int_schema('id')), [ + 'date' => false, + 'region' => true, + ]), + ); + } + + public function test_declare_moves_a_partition_column_out_of_its_body_position_and_keeps_its_type(): void + { + // the file's own header puts `group` first; the partition block is appended, so the declared + // int type survives but the position does not + static::assertEquals( + schema(int_schema('id'), str_schema('value'), int_schema('group')), + (new PartitionColumns(new MemoryFilesystem()))->declare( + schema(int_schema('group'), int_schema('id'), str_schema('value')), + ['group' => false], + ), + ); + } + + public function test_declare_leaves_a_schema_alone_when_nothing_was_discovered(): void + { + static::assertEquals( + schema(int_schema('id')), + (new PartitionColumns(new MemoryFilesystem()))->declare(schema(int_schema('id')), []), + ); + } + + public function test_names_of_a_listing_without_partitions_is_empty(): void + { + $filesystem = new MemoryFilesystem(); + $filesystem->appendTo(path('memory://plain/data.csv'))->append('id')->close(); + + static::assertSame( + [], + (new PartitionColumns($filesystem))->names(path('memory://plain/*.csv'), new OnlyFiles()), + ); + } + + public function test_names_reports_a_partition_every_path_carries_as_non_nullable(): void + { + $filesystem = new MemoryFilesystem(); + $filesystem->appendTo(path('memory://all/date=2026-01-01/data.csv'))->append('id')->close(); + $filesystem->appendTo(path('memory://all/date=2026-01-02/data.csv'))->append('id')->close(); + + static::assertSame( + ['date' => false], + (new PartitionColumns($filesystem))->names(path('memory://all/*/*.csv'), new OnlyFiles()), + ); + } + + public function test_names_are_sorted_by_name_not_by_path_order(): void + { + $filesystem = new MemoryFilesystem(); + $filesystem->appendTo(path('memory://sorted/year=2026/month=01/region=eu/data.csv'))->append('id')->close(); + + static::assertSame( + ['month' => false, 'region' => false, 'year' => false], + (new PartitionColumns($filesystem))->names(path('memory://sorted/*/*/*/*.csv'), new OnlyFiles()), + ); + } + + public function test_names_reports_a_partition_only_some_paths_carry_as_nullable(): void + { + $filesystem = new MemoryFilesystem(); + $filesystem->appendTo(path('memory://mixed/date=2026-01-01/data.csv'))->append('id')->close(); + $filesystem->appendTo(path('memory://mixed/plain/data.csv'))->append('id')->close(); + + static::assertSame( + ['date' => true], + (new PartitionColumns($filesystem))->names(path('memory://mixed/*/*.csv'), new OnlyFiles()), + ); + } +} diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Extractor/PartitionTypesTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Extractor/File/PartitionTypesTest.php similarity index 95% rename from src/core/etl/tests/Flow/ETL/Tests/Unit/Extractor/PartitionTypesTest.php rename to src/core/etl/tests/Flow/ETL/Tests/Unit/Extractor/File/PartitionTypesTest.php index ae3d5883e9..e8d2ad1843 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Unit/Extractor/PartitionTypesTest.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Extractor/File/PartitionTypesTest.php @@ -2,11 +2,11 @@ declare(strict_types=1); -namespace Flow\ETL\Tests\Unit\Extractor; +namespace Flow\ETL\Tests\Unit\Extractor\File; use Flow\ETL\Exception\InvalidArgumentException; -use Flow\ETL\Extractor\PartitionColumns; -use Flow\ETL\Extractor\PartitionTypes; +use Flow\ETL\Extractor\File\PartitionColumns; +use Flow\ETL\Extractor\File\PartitionTypes; use Flow\ETL\Tests\FlowTestCase; use Flow\Filesystem\Local\MemoryFilesystem; diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Extractor/File/ReadWindowTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Extractor/File/ReadWindowTest.php new file mode 100644 index 0000000000..ab0333e226 --- /dev/null +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Extractor/File/ReadWindowTest.php @@ -0,0 +1,36 @@ +expectException(InvalidArgumentException::class); + $this->expectExceptionMessage('ReadWindow offset must be 0 or more, got -1'); + + new ReadWindow(offset: -1); + } + + public function test_a_negative_limit_is_refused(): void + { + $this->expectException(InvalidArgumentException::class); + $this->expectExceptionMessage('ReadWindow limit must be 0 or more, got -1'); + + new ReadWindow(limit: -1); + } + + public function test_the_default_window_skips_nothing_and_wants_everything(): void + { + $window = new ReadWindow(); + + static::assertSame(0, $window->offset); + static::assertNull($window->limit); + } +} diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Extractor/SourceFileTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Extractor/File/SourceFileTest.php similarity index 93% rename from src/core/etl/tests/Flow/ETL/Tests/Unit/Extractor/SourceFileTest.php rename to src/core/etl/tests/Flow/ETL/Tests/Unit/Extractor/File/SourceFileTest.php index a87045a380..a3b6035d28 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Unit/Extractor/SourceFileTest.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Extractor/File/SourceFileTest.php @@ -2,9 +2,9 @@ declare(strict_types=1); -namespace Flow\ETL\Tests\Unit\Extractor; +namespace Flow\ETL\Tests\Unit\Extractor\File; -use Flow\ETL\Extractor\SourceFile; +use Flow\ETL\Extractor\File\SourceFile; use Flow\ETL\Tests\FlowTestCase; use function Flow\Filesystem\DSL\path; diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Extractor/FileConstantsTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Extractor/FileConstantsTest.php deleted file mode 100644 index 4b204a152d..0000000000 --- a/src/core/etl/tests/Flow/ETL/Tests/Unit/Extractor/FileConstantsTest.php +++ /dev/null @@ -1,95 +0,0 @@ - 'memory://orders/data.csv', 'name' => 'Norbert'], - (new FileConstants(new PartitionColumns(memory_filesystem()), 'memory://orders/data.csv', [], []))->fill([ - '_input_file_uri' => 'hijacked', - 'name' => 'Norbert', - ]), - ); - } - - public function test_partition_columns_leave_their_body_position_and_are_re_appended(): void - { - static::assertSame( - ['name' => 'Norbert', 'year' => 2024], - (new FileConstants( - new PartitionColumns(memory_filesystem()), - null, - ['year' => false], - ['year' => 2024], - ))->fill(['year' => 'from the body', 'name' => 'Norbert']), - ); - } - - public function test_the_uri_is_stamped_only_when_metadata_columns_are_on(): void - { - static::assertSame( - ['name' => 'Norbert'], - (new FileConstants(new PartitionColumns(memory_filesystem()), null, [], []))->fill(['name' => 'Norbert']), - ); - } - - public function test_the_uri_joins_the_body_when_metadata_columns_are_on(): void - { - static::assertSame( - ['name' => 'Norbert', '_input_file_uri' => 'memory://orders/data.csv'], - (new FileConstants(new PartitionColumns(memory_filesystem()), 'memory://orders/data.csv', [], []))->fill([ - 'name' => 'Norbert', - ]), - ); - } - - public function test_fill_rows_returns_a_batch_with_nothing_to_add_as_it_is(): void - { - $rows = rows(schema(str_schema('name')), row(['name' => 'Norbert'])); - - static::assertSame($rows, (new FileConstants( - new PartitionColumns(memory_filesystem()), - null, - [], - [], - ))->fillRows($rows, $rows->schema())); - } - - public function test_fill_rows_adds_the_constants_to_every_row_under_the_declared_schema(): void - { - $declared = schema(str_schema('name'), str_schema('_input_file_uri'), int_schema('year')); - - static::assertEquals( - rows( - $declared, - row(['name' => 'Norbert', '_input_file_uri' => 'memory://orders/data.csv', 'year' => 2024]), - row(['name' => 'Flow', '_input_file_uri' => 'memory://orders/data.csv', 'year' => 2024]), - ), - (new FileConstants( - new PartitionColumns(memory_filesystem()), - 'memory://orders/data.csv', - ['year' => false], - ['year' => 2024], - ))->fillRows( - rows(schema(str_schema('name')), row(['name' => 'Norbert']), row(['name' => 'Flow'])), - $declared, - ), - ); - } -} diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Extractor/FilesExtractorTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Extractor/FilesExtractorTest.php index 85c6c045b6..62e717d70f 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Unit/Extractor/FilesExtractorTest.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Extractor/FilesExtractorTest.php @@ -68,7 +68,7 @@ public function test_a_zero_string_extension_is_not_null(): void { $batches = iterator_to_array(files(__DIR__ . '/Fixtures/ZeroExtension/*')->extract(flow_context())); - static::assertSame('0', $batches[0]->first()->get('extension')); + static::assertSame('0', $batches[0]->column('extension')->value(0)); } public function test_partition_directories_become_string_columns(): void @@ -77,14 +77,14 @@ public function test_partition_directories_become_string_columns(): void . '/../../Integration/DataFrame/Fixtures/Partitioning/multi_partition_pruning_test/**/*.txt'); $partitions = schema(str_schema('day'), str_schema('month'), str_schema('year')); - $first = iterator_to_array($extractor->extract(flow_context()), false)[0]->first(); + $first = iterator_to_array($extractor->extract(flow_context()), false)[0]->toArray()[0]; static::assertEquals($partitions, $extractor->partitionSchema()); static::assertEquals($partitions, $extractor->schema()->keep('day', 'month', 'year')); static::assertSame(['day' => '30', 'month' => '12', 'year' => '2022'], [ - 'day' => $first->get('day'), - 'month' => $first->get('month'), - 'year' => $first->get('year'), + 'day' => $first['day'], + 'month' => $first['month'], + 'year' => $first['year'], ]); } @@ -160,6 +160,41 @@ public function test_batches_at_the_configured_batch_size(): void static::assertSame([2, 2, 2], $sizes); } + public function test_every_row_of_a_batch_carries_its_own_files_partition_value(): void + { + $batches = iterator_to_array( + files('memory://dir/**/*.txt', MemoryFiles::with([ + 'memory://dir/year=2023/a.txt' => 'a', + 'memory://dir/year=2024/b.txt' => 'b', + 'memory://dir/year=2025/c.txt' => 'c', + ])) + ->withBatchSize(3) + ->extract(flow_context()), + false, + ); + + static::assertCount(1, $batches); + static::assertSame(['2023', '2024', '2025'], $batches[0]->column('year')->values()); + static::assertSame(['a.txt', 'b.txt', 'c.txt'], $batches[0]->column('base_name')->values()); + } + + public function test_a_limit_at_the_batch_boundary_keeps_each_rows_partition_value(): void + { + $batches = iterator_to_array( + files('memory://dir/**/*.txt', MemoryFiles::with([ + 'memory://dir/year=2023/a.txt' => 'a', + 'memory://dir/year=2024/b.txt' => 'b', + 'memory://dir/year=2025/c.txt' => 'c', + ])) + ->withBatchSize(2) + ->extract(flow_context(), limit: 2), + false, + ); + + static::assertCount(1, $batches); + static::assertSame(['2023', '2024'], $batches[0]->column('year')->values()); + } + public function test_files_extractor_honours_the_batch_contract(): void { self::assertExtractorHonoursBatchContract( diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Extractor/Grid/HeaderNamesTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Extractor/Grid/HeaderNamesTest.php new file mode 100644 index 0000000000..b211919b9b --- /dev/null +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Extractor/Grid/HeaderNamesTest.php @@ -0,0 +1,25 @@ +of([' id ', 1, 2.5, true])); + } + + public function test_a_blank_or_non_scalar_cell_is_named_by_its_position(): void + { + static::assertSame( + ['e00', 'e01', 'e02', 'name', 'e04'], + (new HeaderNames())->of(['', null, ' ', 'name', new DateTimeImmutable('2026-01-01')]), + ); + } +} diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Extractor/Grid/RecordDecoderTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Extractor/Grid/RecordDecoderTest.php new file mode 100644 index 0000000000..e2892b218d --- /dev/null +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Extractor/Grid/RecordDecoderTest.php @@ -0,0 +1,382 @@ +decode([[' id ', '', null, 'name ']])); + static::assertSame(['id', 'e01', 'e02', 'name'], $records->headers()); + } + + public function test_a_numeric_header_cell_is_its_string(): void + { + $records = new RecordDecoder(withHeader: true, emptyToNull: true); + $records->decode([[1, 2.5, true]]); + + static::assertSame(['1', '2.5', '1'], $records->headers()); + } + + public function test_a_non_scalar_header_cell_is_named_by_its_position(): void + { + $records = new RecordDecoder(withHeader: true, emptyToNull: true); + $records->decode([[new DateTimeImmutable('2026-01-01'), 'name']]); + + static::assertSame(['e00', 'name'], $records->headers()); + } + + public function test_without_a_header_the_first_record_sizes_the_generated_names(): void + { + $records = new RecordDecoder(withHeader: false, emptyToNull: true); + + static::assertSame([['e00' => 'a', 'e01' => 'b']], $records->decode([['a', 'b']])); + static::assertSame(['e00', 'e01'], $records->headers()); + } + + #[TestWith([true])] + #[TestWith([false])] + public function test_a_cell_a_short_record_lacks_is_null(bool $emptyToNull): void + { + static::assertSame( + [['id' => '1', 'name' => 'a', 'age' => null]], + (new RecordDecoder(withHeader: true, emptyToNull: $emptyToNull))->decode([ + ['id', 'name', 'age'], + ['1', 'a'], + ]), + ); + } + + #[TestWith([true, null])] + #[TestWith([false, ''])] + public function test_a_present_empty_cell_is_null_only_under_empty_to_null( + bool $emptyToNull, + ?string $expected, + ): void { + static::assertSame( + [['id' => '1', 'name' => $expected]], + (new RecordDecoder(withHeader: true, emptyToNull: $emptyToNull))->decode([['id', 'name'], ['1', '']]), + ); + } + + public function test_a_non_string_cell_is_kept_as_read(): void + { + $at = new DateTimeImmutable('2026-01-01'); + + static::assertSame( + [['at' => $at, 'n' => 0]], + (new RecordDecoder(withHeader: true, emptyToNull: true))->decode([['at', 'n'], [$at, 0]]), + ); + } + + public function test_extra_cells_are_dropped_by_default(): void + { + static::assertSame( + [['id' => '1']], + (new RecordDecoder(withHeader: true, emptyToNull: true))->decode([['id'], ['1', 'extra']]), + ); + } + + public function test_extra_cells_are_refused_when_they_are_not_dropped(): void + { + $this->expectException(InvalidArgumentException::class); + $this->expectExceptionMessage('Row has more columns (2) than headers (1)'); + + (new RecordDecoder(withHeader: true, emptyToNull: true, dropExtraColumns: false))->decode([ + ['id'], + ['1', 'extra'], + ]); + } + + public function test_leading_empty_records_are_skipped_before_the_header(): void + { + $records = new RecordDecoder(withHeader: true, emptyToNull: true); + + static::assertSame([['id' => '1']], $records->decode([[], [], ['id'], ['1']])); + static::assertSame(['id'], $records->headers()); + } + + public function test_an_empty_batch_decodes_to_nothing(): void + { + static::assertSame([], (new RecordDecoder(withHeader: true, emptyToNull: true))->decode([])); + } + + public function test_headers_are_null_before_the_first_decode(): void + { + static::assertNull((new RecordDecoder(withHeader: true, emptyToNull: true))->headers()); + } + + public function test_the_header_is_kept_between_batches(): void + { + $records = new RecordDecoder(withHeader: true, emptyToNull: true); + $records->decode([['id']]); + + static::assertSame([['id' => '2']], $records->decode([['2']])); + } + + public function test_decode_captures_headers_from_the_first_row_and_emits_no_row_for_it(): void + { + $decoder = new RecordDecoder(withHeader: true, emptyToNull: true); + + static::assertSame([], $decoder->decode([['id', 'name']])); + static::assertSame(['id' => 1, 'name' => 'Norbert'], $decoder->decode([[1, 'Norbert']])[0]); + } + + public function test_decode_generates_auto_headers_when_header_is_disabled(): void + { + static::assertSame( + ['e00' => 1, 'e01' => 'Norbert'], + (new RecordDecoder(withHeader: false, emptyToNull: true))->decode([[1, 'Norbert']])[0], + ); + } + + public function test_headers_are_generated_when_header_is_disabled(): void + { + $decoder = new RecordDecoder(withHeader: false, emptyToNull: true); + + static::assertCount(1, $decoder->decode([[1, 2, 3]])); + static::assertSame(['e00', 'e01', 'e02'], $decoder->headers()); + } + + public function test_excel_headers_are_null_before_the_first_decode(): void + { + static::assertNull((new RecordDecoder(withHeader: true, emptyToNull: true))->headers()); + } + + public function test_headers_are_stringified_and_named_by_position_for_blank_cells(): void + { + $decoder = new RecordDecoder(withHeader: true, emptyToNull: true); + $decoder->decode([[1, null, 'x']]); + + static::assertSame(['1', 'e01', 'x'], $decoder->headers()); + } + + public function test_headers_are_the_header_row_after_decode(): void + { + $decoder = new RecordDecoder(withHeader: true, emptyToNull: true); + + static::assertSame([], $decoder->decode([['id', 'name']])); + static::assertSame(['id', 'name'], $decoder->headers()); + } + + public function test_decode_keeps_empty_cells_when_convert_empty_to_null_is_disabled(): void + { + static::assertSame( + ['id' => 1, 'name' => ''], + (new RecordDecoder(withHeader: true, emptyToNull: false))->decode([['id', 'name'], [1, '']])[0], + ); + } + + public function test_decode_maps_cells_to_headers(): void + { + static::assertSame( + ['id' => 1, 'name' => 'Norbert'], + (new RecordDecoder(withHeader: true, emptyToNull: true))->decode([['id', 'name'], [1, 'Norbert']])[0], + ); + } + + public function test_decode_turns_empty_cells_into_null_by_default(): void + { + static::assertSame( + ['id' => 1, 'name' => null], + (new RecordDecoder(withHeader: true, emptyToNull: true))->decode([['id', 'name'], [1, '']])[0], + ); + } + + public function test_header_names_follow_the_csv_rule(): void + { + $decoder = new RecordDecoder(withHeader: true, emptyToNull: true); + $decoder->decode([[' id ', '', 'name']]); + + static::assertSame(['id', 'e01', 'name'], $decoder->headers()); + } + + public function test_decodes_rows_with_first_row_as_headers(): void + { + $decoder = new RecordDecoder(withHeader: true, emptyToNull: true); + + $decoded = $decoder->decode([ + ['id', 'name'], + ['1', 'Norbert'], + ['2', 'Tomek'], + ]); + + static::assertSame( + [ + ['id' => '1', 'name' => 'Norbert'], + ['id' => '2', 'name' => 'Tomek'], + ], + $decoded, + ); + } + + public function test_skips_empty_rows_before_headers(): void + { + $decoder = new RecordDecoder(withHeader: true, emptyToNull: true); + + $decoded = $decoder->decode([ + [], + ['id', 'name'], + ['1', 'Norbert'], + ]); + + static::assertSame([['id' => '1', 'name' => 'Norbert']], $decoded); + } + + public function test_keeps_headers_between_decode_calls(): void + { + $decoder = new RecordDecoder(withHeader: true, emptyToNull: true); + + $decoder->decode([['id', 'name'], ['1', 'Norbert']]); + + static::assertSame([['id' => '2', 'name' => 'Tomek']], $decoder->decode([['2', 'Tomek']])); + } + + public function test_pads_shorter_rows_with_nulls(): void + { + $decoder = new RecordDecoder(withHeader: true, emptyToNull: true); + + $decoded = $decoder->decode([ + ['id', 'name'], + ['1'], + ]); + + static::assertSame([['id' => '1', 'name' => null]], $decoded); + } + + public function test_drops_extra_columns(): void + { + $decoder = new RecordDecoder(withHeader: true, emptyToNull: true); + + $decoded = $decoder->decode([ + ['id', 'name'], + ['1', 'Norbert', 'extra'], + ]); + + static::assertSame([['id' => '1', 'name' => 'Norbert']], $decoded); + } + + public function test_throws_on_extra_columns_when_drop_is_disabled(): void + { + $decoder = new RecordDecoder(withHeader: true, emptyToNull: true, dropExtraColumns: false); + + $this->expectException(InvalidArgumentException::class); + $this->expectExceptionMessage('Row has more columns (3) than headers (2)'); + + $decoder->decode([ + ['id', 'name'], + ['1', 'Norbert', 'extra'], + ]); + } + + public function test_generates_auto_headers_without_header_row(): void + { + $decoder = new RecordDecoder(withHeader: false, emptyToNull: true); + + $decoded = $decoder->decode([ + ['1', 'Norbert'], + ['2', 'Tomek'], + ]); + + static::assertSame( + [ + ['e00' => '1', 'e01' => 'Norbert'], + ['e00' => '2', 'e01' => 'Tomek'], + ], + $decoded, + ); + } + + public function test_empty_cells_become_null_by_default(): void + { + $decoder = new RecordDecoder(withHeader: true, emptyToNull: true); + + static::assertSame([['a' => null, 'b' => 'x']], $decoder->decode([['a', 'b'], ['', 'x']])); + } + + public function test_empty_cells_stay_strings_when_asked(): void + { + $decoder = new RecordDecoder(withHeader: true, emptyToNull: false); + + static::assertSame([['a' => '', 'b' => 'x']], $decoder->decode([['a', 'b'], ['', 'x']])); + } + + public function test_omitted_trailing_cells_are_null_regardless_of_empty_to_null(): void + { + static::assertSame( + [['a' => 'x', 'b' => null]], + (new RecordDecoder(withHeader: true, emptyToNull: true))->decode([ + ['a', 'b'], + ['x'], + ]), + ); + static::assertSame( + [['a' => 'x', 'b' => null]], + (new RecordDecoder(withHeader: true, emptyToNull: false))->decode([['a', 'b'], ['x']]), + ); + } + + public function test_headers_are_the_consumed_header_row(): void + { + $decoder = new RecordDecoder(withHeader: true, emptyToNull: true); + $decoder->decode([['a', 'b'], ['1', '2']]); + + static::assertSame(['a', 'b'], $decoder->headers()); + } + + public function test_headers_skip_a_leading_empty_row(): void + { + $decoder = new RecordDecoder(withHeader: true, emptyToNull: true); + $decoder->decode([[], ['a', 'b']]); + + static::assertSame(['a', 'b'], $decoder->headers()); + } + + public function test_headers_are_generated_without_a_header_row(): void + { + $decoder = new RecordDecoder(withHeader: false, emptyToNull: true); + $decoder->decode([['1', '2', '3']]); + + static::assertSame(['e00', 'e01', 'e02'], $decoder->headers()); + } + + public function test_headers_are_strings_even_when_the_api_typed_them(): void + { + $decoder = new RecordDecoder(withHeader: true, emptyToNull: true); + $decoder->decode([[1, '2'], ['x', 'y']]); + + static::assertSame(['1', '2'], $decoder->headers()); + } + + /** + * b88, pinned as it behaves rather than as it should: `1` and `true` both stringify to `'1'`, and + * array_combine() keeps the last value under a repeated key, so the row loses a column. Not specific to a + * typed header - `a,a` does the same in CSV - and not introduced by inference. + */ + public function test_two_header_cells_with_the_same_name_collapse_the_row_to_one_column(): void + { + $decoder = new RecordDecoder(withHeader: true, emptyToNull: true); + $decoded = $decoder->decode([[1, true], ['x', 'y']]); + + static::assertSame(['1', '1'], $decoder->headers()); + static::assertSame(['1' => 'y'], $decoded[0]); + } + + public function test_google_sheet_header_names_follow_the_csv_rule(): void + { + $decoder = new RecordDecoder(withHeader: true, emptyToNull: true); + $decoder->decode([[' id ', '', 'name']]); + + static::assertSame(['id', 'e01', 'name'], $decoder->headers()); + } +} diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Extractor/InMemoryRowsTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Extractor/InMemoryRowsTest.php deleted file mode 100644 index 4aafa9a547..0000000000 --- a/src/core/etl/tests/Flow/ETL/Tests/Unit/Extractor/InMemoryRowsTest.php +++ /dev/null @@ -1,49 +0,0 @@ - 1], ['id' => 2]]); - - static::assertEquals( - [new RawRowValues(['id' => 1]), new RawRowValues(['id' => 2])], - iterator_to_array($rows->values(), false), - ); - static::assertEquals( - [new RawRowValues(['id' => 1]), new RawRowValues(['id' => 2])], - iterator_to_array($rows->values(), false), - ); - } - - public function test_positional_keys_are_named_like_the_hydrator_names_them(): void - { - static::assertSame( - ['e00', 'e01'], - array_keys(iterator_to_array((new InMemoryRows([[1, 2]]))->values(), false)[0]->values), - ); - } - - public function test_samples_yields_exactly_one_inner_iterable(): void - { - static::assertCount(1, iterator_to_array((new InMemoryRows([['id' => 1]]))->samples(-1), false)); - } - - public function test_the_row_budget_is_never_rationed(): void - { - $units = iterator_to_array((new InMemoryRows([['id' => 1], ['id' => 2], ['id' => 3]]))->samples(1), false); - - static::assertCount(3, iterator_to_array($units[0], false)); - } -} diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Extractor/Memory/InMemoryRowsTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Extractor/Memory/InMemoryRowsTest.php new file mode 100644 index 0000000000..dc0aad7362 --- /dev/null +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Extractor/Memory/InMemoryRowsTest.php @@ -0,0 +1,42 @@ + 1], ['id' => 2]]); + + static::assertEquals([['id' => 1], ['id' => 2]], iterator_to_array($rows->values(), false)); + static::assertEquals([['id' => 1], ['id' => 2]], iterator_to_array($rows->values(), false)); + } + + public function test_positional_keys_are_named_like_the_hydrator_names_them(): void + { + static::assertSame( + ['e00', 'e01'], + array_keys(iterator_to_array((new InMemoryRows([[1, 2]]))->values(), false)[0]), + ); + } + + public function test_samples_yields_exactly_one_inner_iterable(): void + { + static::assertCount(1, iterator_to_array((new InMemoryRows([['id' => 1]]))->samples(-1), false)); + } + + public function test_the_row_budget_is_never_rationed(): void + { + $units = iterator_to_array((new InMemoryRows([['id' => 1], ['id' => 2], ['id' => 3]]))->samples(1), false); + + static::assertCount(3, iterator_to_array($units[0], false)); + } +} diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Extractor/InferredRowsTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Extractor/Memory/InferredRowsTest.php similarity index 76% rename from src/core/etl/tests/Flow/ETL/Tests/Unit/Extractor/InferredRowsTest.php rename to src/core/etl/tests/Flow/ETL/Tests/Unit/Extractor/Memory/InferredRowsTest.php index c72c9ddb93..a3609e2042 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Unit/Extractor/InferredRowsTest.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Extractor/Memory/InferredRowsTest.php @@ -2,12 +2,12 @@ declare(strict_types=1); -namespace Flow\ETL\Tests\Unit\Extractor; +namespace Flow\ETL\Tests\Unit\Extractor\Memory; +use Flow\ETL\Column\PhpBackend; use Flow\ETL\Exception\InferredSchemaException; use Flow\ETL\Exception\SchemaMismatchException; -use Flow\ETL\Extractor\InferredRows; -use Flow\ETL\Row\PhpRowHydrator; +use Flow\ETL\Extractor\Memory\InferredRows; use Flow\ETL\Schema\Inference\SchemaInference; use Flow\ETL\Tests\FlowTestCase; use PHPUnit\Framework\Attributes\TestWith; @@ -20,7 +20,7 @@ final class InferredRowsTest extends FlowTestCase public function test_a_row_past_the_sample_is_reported_at_its_position_in_the_whole_source(): void { $rows = new InferredRows('from_array()', new SchemaInference(sampleSize: 2)); - $rows->of([['code' => 1], ['code' => 2]], schema(int_schema('code', true)), new PhpRowHydrator()); + $rows->of([['code' => 1], ['code' => 2]], schema(int_schema('code', true)), new PhpBackend()); $this->expectException(InferredSchemaException::class); $this->expectExceptionMessage( @@ -29,7 +29,7 @@ public function test_a_row_past_the_sample_is_reported_at_its_position_in_the_wh . 'or declare the schema with ->withSchema(...).', ); - $rows->of([['code' => 3], ['code' => 'x']], schema(int_schema('code', true)), new PhpRowHydrator()); + $rows->of([['code' => 3], ['code' => 'x']], schema(int_schema('code', true)), new PhpBackend()); } public function test_a_row_past_the_sample_keeps_the_refusal_it_was_raised_from(): void @@ -38,7 +38,7 @@ public function test_a_row_past_the_sample_keeps_the_refusal_it_was_raised_from( (new InferredRows('from_array()', new SchemaInference(sampleSize: 1)))->of( [['code' => 1], ['code' => 'x']], schema(int_schema('code', true)), - new PhpRowHydrator(), + new PhpBackend(), ); static::fail('The row past the sample was not refused.'); @@ -61,6 +61,18 @@ public function test_a_refusal_the_sample_does_not_explain_stays_the_gate_s_own( (new InferredRows( 'from_array()', $sampleSize === null ? null : new SchemaInference(sampleSize: $sampleSize), - ))->of([['code' => 1], ['code' => 'x']], schema(int_schema('code', true)), new PhpRowHydrator()); + ))->of([['code' => 1], ['code' => 'x']], schema(int_schema('code', true)), new PhpBackend()); + } + + public function test_a_declared_schema_narrower_than_the_rows_projects_its_columns(): void + { + static::assertSame( + [['code' => 1], ['code' => 2]], + (new InferredRows('from_array()', null))->of( + [['code' => 1, 'name' => 'one'], ['code' => 2, 'name' => 'two']], + schema(int_schema('code')), + new PhpBackend(), + )->toArray(), + ); } } diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Extractor/SpilledRowsTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Extractor/Memory/SpilledRowsTest.php similarity index 98% rename from src/core/etl/tests/Flow/ETL/Tests/Unit/Extractor/SpilledRowsTest.php rename to src/core/etl/tests/Flow/ETL/Tests/Unit/Extractor/Memory/SpilledRowsTest.php index 5dbfdd9786..0651990aa5 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Unit/Extractor/SpilledRowsTest.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Extractor/Memory/SpilledRowsTest.php @@ -2,12 +2,12 @@ declare(strict_types=1); -namespace Flow\ETL\Tests\Unit\Extractor; +namespace Flow\ETL\Tests\Unit\Extractor\Memory; use DateTimeImmutable; use Flow\ETL\Exception\InvalidLogicException; -use Flow\ETL\Extractor\SpilledRows; -use Flow\ETL\Extractor\SpillState; +use Flow\ETL\Extractor\Memory\SpilledRows; +use Flow\ETL\Extractor\Memory\SpillState; use Flow\ETL\Tests\Double\RecordingFilesystem; use Flow\ETL\Tests\Double\ThrowingRmFilesystem; use Flow\ETL\Tests\Fixtures\Enum\BackedStringEnum; diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Extractor/MemoryExtractorTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Extractor/MemoryExtractorTest.php index c8300a73a2..c79d2a680f 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Unit/Extractor/MemoryExtractorTest.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Extractor/MemoryExtractorTest.php @@ -13,13 +13,12 @@ use Flow\ETL\Tests\Mother\RowsMother; use function array_map; +use function Flow\ETL\DSL\array_to_rows; use function Flow\ETL\DSL\config; use function Flow\ETL\DSL\flow_context; use function Flow\ETL\DSL\from_memory; use function Flow\ETL\DSL\infer_schema; use function Flow\ETL\DSL\int_schema; -use function Flow\ETL\DSL\row; -use function Flow\ETL\DSL\rows; use function Flow\ETL\DSL\schema; use function Flow\ETL\DSL\str_schema; use function Flow\ETL\DSL\to_memory; @@ -33,25 +32,27 @@ public function test_declared_schema_is_used_to_hydrate_extracted_rows(): void $memory = new ArrayMemory(); to_memory($memory)->load( - rows(schema(int_schema('number'), str_schema('name')), row(['number' => 1, 'name' => 'one'])), + array_to_rows([['number' => 1, 'name' => 'one']], schema(int_schema('number'), str_schema('name'))), flow_context(config()), ); self::assertExtractedRowsEquals( - rows(schema(str_schema('number'), str_schema('name')), row(['number' => '1', 'name' => 'one'])), + array_to_rows([['number' => '1', 'name' => 'one']], schema(str_schema('number'), str_schema('name'))), from_memory($memory)->withSchema(schema(str_schema('number'), str_schema('name'))), ); } public function test_memory_extractor(): void { - $rows = rows( + $rows = array_to_rows( + [ + ['number' => 1, 'name' => 'one'], + ['number' => 2, 'name' => 'two'], + ['number' => 3, 'name' => 'tree'], + ['number' => 4, 'name' => 'four'], + ['number' => 5, 'name' => 'five'], + ], schema(int_schema('number'), str_schema('name')), - row(['number' => 1, 'name' => 'one']), - row(['number' => 2, 'name' => 'two']), - row(['number' => 3, 'name' => 'tree']), - row(['number' => 4, 'name' => 'four']), - row(['number' => 5, 'name' => 'five']), ); $memory = new ArrayMemory(); diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Extractor/PartitionColumnsTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Extractor/PartitionColumnsTest.php deleted file mode 100644 index 932d726021..0000000000 --- a/src/core/etl/tests/Flow/ETL/Tests/Unit/Extractor/PartitionColumnsTest.php +++ /dev/null @@ -1,209 +0,0 @@ -declare(schema(int_schema('date')), ['date' => true]), - ); - } - - public function test_apply_adds_a_column_the_inferred_schema_is_missing(): void - { - static::assertEquals( - schema(int_schema('id'), str_schema('date', nullable: true)), - (new PartitionColumns(new MemoryFilesystem())) - ->apply(rows(schema(int_schema('id')), row(['id' => 1])), ['date' => true]) - ->schema(), - ); - } - - public function test_apply_keeps_the_rows_it_was_given(): void - { - static::assertEquals( - [row(['id' => 1, 'date' => null])], - (new PartitionColumns(new MemoryFilesystem()))->apply( - rows(schema(int_schema('id'), str_schema('date', nullable: true)), row(['id' => 1, 'date' => null])), - ['date' => true], - )->all(), - ); - } - - public function test_apply_replaces_the_definition_inference_guessed(): void - { - static::assertEquals( - schema(int_schema('id'), str_schema('date', nullable: true)), - (new PartitionColumns(new MemoryFilesystem())) - ->apply(rows(schema(int_schema('id'), str_schema('date')), row(['id' => 1, 'date' => '2026-01-01'])), [ - 'date' => true, - ]) - ->schema(), - ); - } - - public function test_apply_takes_nullability_from_the_names_map(): void - { - static::assertEquals( - schema(str_schema('date'), str_schema('region', nullable: true)), - (new PartitionColumns(new MemoryFilesystem())) - ->apply(rows(schema(str_schema('date', nullable: true)), row(['date' => '2026-01-01'])), [ - 'date' => false, - 'region' => true, - ]) - ->schema(), - ); - } - - public function test_declare_adds_each_name_with_the_nullability_it_was_given(): void - { - static::assertEquals( - schema(int_schema('id'), str_schema('date'), str_schema('region', nullable: true)), - (new PartitionColumns(new MemoryFilesystem()))->declare(schema(int_schema('id')), [ - 'date' => false, - 'region' => true, - ]), - ); - } - - public function test_declare_moves_a_partition_column_out_of_its_body_position_and_keeps_its_type(): void - { - // the file's own header puts `group` first; the partition block is appended, so the declared - // int type survives but the position does not - static::assertEquals( - schema(int_schema('id'), str_schema('value'), int_schema('group')), - (new PartitionColumns(new MemoryFilesystem()))->declare( - schema(int_schema('group'), int_schema('id'), str_schema('value')), - ['group' => false], - ), - ); - } - - public function test_apply_moves_a_partition_column_out_of_its_body_position(): void - { - static::assertEquals( - schema(int_schema('id'), str_schema('value'), str_schema('group')), - (new PartitionColumns(new MemoryFilesystem())) - ->apply( - rows( - schema(str_schema('group'), int_schema('id'), str_schema('value')), - row(['group' => '1', 'id' => 1, 'value' => 'a']), - ), - ['group' => false], - ) - ->schema(), - ); - } - - public function test_declare_leaves_a_schema_alone_when_nothing_was_discovered(): void - { - static::assertEquals( - schema(int_schema('id')), - (new PartitionColumns(new MemoryFilesystem()))->declare(schema(int_schema('id')), []), - ); - } - - public function test_apply_leaves_rows_alone_when_nothing_was_discovered(): void - { - static::assertEquals( - rows(schema(int_schema('id')), row(['id' => 1])), - (new PartitionColumns(new MemoryFilesystem()))->apply(rows(schema(int_schema('id')), row(['id' => 1])), []), - ); - } - - public function test_fill_leaves_a_row_untouched_when_there_are_no_partition_columns(): void - { - static::assertSame(['id' => 1], (new PartitionColumns(new MemoryFilesystem()))->fill(['id' => 1], [], [])); - } - - public function test_fill_nulls_a_column_the_path_does_not_carry(): void - { - static::assertSame( - ['id' => 1, 'date' => '2026-01-01', 'region' => null], - (new PartitionColumns(new MemoryFilesystem()))->fill( - ['id' => 1], - ['date' => false, 'region' => true], - ['date' => '2026-01-01'], - ), - ); - } - - public function test_names_of_a_listing_without_partitions_is_empty(): void - { - $filesystem = new MemoryFilesystem(); - $filesystem->appendTo(path('memory://plain/data.csv'))->append('id')->close(); - - static::assertSame( - [], - (new PartitionColumns($filesystem))->names(path('memory://plain/*.csv'), new OnlyFiles()), - ); - } - - public function test_names_reports_a_partition_every_path_carries_as_non_nullable(): void - { - $filesystem = new MemoryFilesystem(); - $filesystem->appendTo(path('memory://all/date=2026-01-01/data.csv'))->append('id')->close(); - $filesystem->appendTo(path('memory://all/date=2026-01-02/data.csv'))->append('id')->close(); - - static::assertSame( - ['date' => false], - (new PartitionColumns($filesystem))->names(path('memory://all/*/*.csv'), new OnlyFiles()), - ); - } - - public function test_names_are_sorted_by_name_not_by_path_order(): void - { - $filesystem = new MemoryFilesystem(); - $filesystem->appendTo(path('memory://sorted/year=2026/month=01/region=eu/data.csv'))->append('id')->close(); - - static::assertSame( - ['month' => false, 'region' => false, 'year' => false], - (new PartitionColumns($filesystem))->names(path('memory://sorted/*/*/*/*.csv'), new OnlyFiles()), - ); - } - - public function test_names_reports_a_partition_only_some_paths_carry_as_nullable(): void - { - $filesystem = new MemoryFilesystem(); - $filesystem->appendTo(path('memory://mixed/date=2026-01-01/data.csv'))->append('id')->close(); - $filesystem->appendTo(path('memory://mixed/plain/data.csv'))->append('id')->close(); - - static::assertSame( - ['date' => true], - (new PartitionColumns($filesystem))->names(path('memory://mixed/*/*.csv'), new OnlyFiles()), - ); - } - - public function test_fill_moves_a_colliding_body_column_into_the_partition_block(): void - { - // declare() re-appends every partition column at the tail, so fill() has to do the same or - // schema() and extract() disagree for a file that also carries a column of that name - static::assertSame( - ['id', 'date'], - array_keys((new PartitionColumns(new MemoryFilesystem()))->fill( - ['date' => '1999-01-01', 'id' => 1], - ['date' => false], - ['date' => '2024-01-01'], - )), - ); - } -} diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Extractor/RowsExtractorTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Extractor/RowsExtractorTest.php index dad9e64077..64e3a5fef4 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Unit/Extractor/RowsExtractorTest.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Extractor/RowsExtractorTest.php @@ -8,11 +8,10 @@ use Flow\ETL\Extractor\Statistics; use Flow\ETL\Tests\FlowTestCase; +use function Flow\ETL\DSL\array_to_rows; use function Flow\ETL\DSL\flow_context; use function Flow\ETL\DSL\from_rows; use function Flow\ETL\DSL\int_schema; -use function Flow\ETL\DSL\row; -use function Flow\ETL\DSL\rows; use function Flow\ETL\DSL\schema; use function Flow\ETL\DSL\str_schema; @@ -20,13 +19,15 @@ final class RowsExtractorTest extends FlowTestCase { public function test_process_extractor(): void { - $rows = rows( + $rows = array_to_rows( + [ + ['number' => 1, 'name' => 'one'], + ['number' => 2, 'name' => 'two'], + ['number' => 3, 'name' => 'tree'], + ['number' => 4, 'name' => 'four'], + ['number' => 5, 'name' => 'five'], + ], schema(int_schema('number'), str_schema('name')), - row(['number' => 1, 'name' => 'one']), - row(['number' => 2, 'name' => 'two']), - row(['number' => 3, 'name' => 'tree']), - row(['number' => 4, 'name' => 'four']), - row(['number' => 5, 'name' => 'five']), ); $extractor = from_rows($rows); @@ -46,8 +47,8 @@ public function test_process_extractor(): void public function test_extract_yields_batches_matching_schema(): void { $extractor = from_rows( - rows(schema(int_schema('number')), row(['number' => 1])), - rows(schema(int_schema('number'), str_schema('name')), row(['number' => 2, 'name' => 'two'])), + array_to_rows([['number' => 1]], schema(int_schema('number'))), + array_to_rows([['number' => 2, 'name' => 'two']], schema(int_schema('number'), str_schema('name'))), ); foreach ($extractor->extract(flow_context()) as $batch) { @@ -57,14 +58,14 @@ public function test_extract_yields_batches_matching_schema(): void public function test_is_repeatable(): void { - static::assertTrue(from_rows(rows(schema(int_schema('number')), row(['number' => 1])))->isRepeatable()); + static::assertTrue(from_rows(array_to_rows([['number' => 1]], schema(int_schema('number'))))->isRepeatable()); } public function test_it_declares_an_exact_row_count(): void { $extractor = from_rows( - rows(schema(int_schema('number')), row(['number' => 1]), row(['number' => 2])), - rows(schema(int_schema('number')), row(['number' => 3])), + array_to_rows([['number' => 1], ['number' => 2]], schema(int_schema('number'))), + array_to_rows([['number' => 3]], schema(int_schema('number'))), ); static::assertEquals(new Statistics(rows: Cardinality::exact(3)), $extractor->statistics()); diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Extractor/SequenceExtractorTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Extractor/SequenceExtractorTest.php index 9ae9fb4bc6..a487cb4a25 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Unit/Extractor/SequenceExtractorTest.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Extractor/SequenceExtractorTest.php @@ -15,6 +15,7 @@ use Flow\ETL\Tests\FlowTestCase; use Flow\ETL\Tests\Mother\RowsMother; +use function Flow\ETL\DSL\array_to_rows; use function Flow\ETL\DSL\config; use function Flow\ETL\DSL\date_schema; use function Flow\ETL\DSL\float_schema; @@ -23,8 +24,6 @@ use function Flow\ETL\DSL\from_sequence_date_period_recurrences; use function Flow\ETL\DSL\from_sequence_number; use function Flow\ETL\DSL\infer_schema; -use function Flow\ETL\DSL\row; -use function Flow\ETL\DSL\rows; use function Flow\ETL\DSL\schema; use function Flow\ETL\DSL\str_schema; use function iterator_to_array; @@ -34,7 +33,7 @@ final class SequenceExtractorTest extends FlowTestCase public function test_declared_schema_is_used_to_hydrate_extracted_rows(): void { self::assertExtractedRowsEquals( - rows(schema(str_schema('num')), row(['num' => '1']), row(['num' => '2']), row(['num' => '3'])), + array_to_rows([['num' => '1'], ['num' => '2'], ['num' => '3']], schema(str_schema('num'))), from_sequence_number('num', 1, 3)->withSchema(schema(str_schema('num'))), ); } @@ -50,18 +49,17 @@ public function test_extracting_from_date_period(): void ); self::assertExtractedRowsEquals( - rows( - schema(date_schema('day', true)), - row(['day' => new DateTimeImmutable('2023-01-02')]), - row(['day' => new DateTimeImmutable('2023-01-03')]), - row(['day' => new DateTimeImmutable('2023-01-04')]), - row(['day' => new DateTimeImmutable('2023-01-05')]), - row(['day' => new DateTimeImmutable('2023-01-06')]), - row(['day' => new DateTimeImmutable('2023-01-07')]), - row(['day' => new DateTimeImmutable('2023-01-08')]), - row(['day' => new DateTimeImmutable('2023-01-09')]), - row(['day' => new DateTimeImmutable('2023-01-10')]), - ), + array_to_rows([ + ['day' => new DateTimeImmutable('2023-01-02')], + ['day' => new DateTimeImmutable('2023-01-03')], + ['day' => new DateTimeImmutable('2023-01-04')], + ['day' => new DateTimeImmutable('2023-01-05')], + ['day' => new DateTimeImmutable('2023-01-06')], + ['day' => new DateTimeImmutable('2023-01-07')], + ['day' => new DateTimeImmutable('2023-01-08')], + ['day' => new DateTimeImmutable('2023-01-09')], + ['day' => new DateTimeImmutable('2023-01-10')], + ], schema(date_schema('day', true))), $extractor, ); } @@ -77,19 +75,18 @@ public function test_extracting_from_date_period_recurrences(): void ); self::assertExtractedRowsEquals( - rows( - schema(date_schema('day', true)), - row(['day' => new DateTimeImmutable('2023-01-02')]), - row(['day' => new DateTimeImmutable('2023-01-03')]), - row(['day' => new DateTimeImmutable('2023-01-04')]), - row(['day' => new DateTimeImmutable('2023-01-05')]), - row(['day' => new DateTimeImmutable('2023-01-06')]), - row(['day' => new DateTimeImmutable('2023-01-07')]), - row(['day' => new DateTimeImmutable('2023-01-08')]), - row(['day' => new DateTimeImmutable('2023-01-09')]), - row(['day' => new DateTimeImmutable('2023-01-10')]), - row(['day' => new DateTimeImmutable('2023-01-11')]), - ), + array_to_rows([ + ['day' => new DateTimeImmutable('2023-01-02')], + ['day' => new DateTimeImmutable('2023-01-03')], + ['day' => new DateTimeImmutable('2023-01-04')], + ['day' => new DateTimeImmutable('2023-01-05')], + ['day' => new DateTimeImmutable('2023-01-06')], + ['day' => new DateTimeImmutable('2023-01-07')], + ['day' => new DateTimeImmutable('2023-01-08')], + ['day' => new DateTimeImmutable('2023-01-09')], + ['day' => new DateTimeImmutable('2023-01-10')], + ['day' => new DateTimeImmutable('2023-01-11')], + ], schema(date_schema('day', true))), $extractor, ); } @@ -99,16 +96,15 @@ public function test_extracting_from_numbers_range(): void $extractor = from_sequence_number('num', 0, 10, 1.5); self::assertExtractedRowsEquals( - rows( - schema(float_schema('num', true)), - row(['num' => 0.0]), - row(['num' => 1.5]), - row(['num' => 3.0]), - row(['num' => 4.5]), - row(['num' => 6.0]), - row(['num' => 7.5]), - row(['num' => 9.0]), - ), + array_to_rows([ + ['num' => 0.0], + ['num' => 1.5], + ['num' => 3.0], + ['num' => 4.5], + ['num' => 6.0], + ['num' => 7.5], + ['num' => 9.0], + ], schema(float_schema('num', true))), $extractor, ); } diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Filesystem/FilesSinkTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Filesystem/FilesSinkTest.php deleted file mode 100644 index 4bde83dddd..0000000000 --- a/src/core/etl/tests/Flow/ETL/Tests/Unit/Filesystem/FilesSinkTest.php +++ /dev/null @@ -1,79 +0,0 @@ -writeTo()->append('half written'); - - $files->abandon(); - - // a format writer flushes its footer on close, so the file must not be removed before that happens - static::assertLessThan( - array_search('rm', $filesystem->calls, true), - array_search('close', $filesystem->calls, true), - ); - } - - public function test_a_destination_without_an_extension_is_refused(): void - { - $this->expectException(RuntimeException::class); - $this->expectExceptionMessage('Stream path must have an extension'); - - (new FilesSink(memory_filesystem(), path('memory://out'), exception_if_exists()))->writeTo(); - } - - public function test_a_pattern_destination_is_refused(): void - { - $this->expectException(RuntimeException::class); - $this->expectExceptionMessage("Destination path can't be pattern"); - - (new FilesSink(memory_filesystem(), path('memory://out/*.csv'), exception_if_exists()))->writeTo(); - } - - public function test_a_placeholder_destination_without_partitions_is_refused(): void - { - $this->expectException(RuntimeException::class); - $this->expectExceptionMessage('contains partition placeholders but rows are not partitioned'); - - (new FilesSink(memory_filesystem(), path('memory://out/{name}.csv'), exception_if_exists()))->writeTo(); - } - - public function test_the_same_destination_returns_the_same_stream(): void - { - $files = new FilesSink(memory_filesystem(), path('memory://out.csv'), exception_if_exists()); - - static::assertSame($files->writeTo(), $files->writeTo()); - } - - public function test_touched_is_answered_per_partition(): void - { - $files = new FilesSink(memory_filesystem(), path('memory://out.csv'), exception_if_exists()); - $partition = [new Partition('day', '2024-01-01')]; - - static::assertFalse($files->touched($partition)); - - $files->writeTo($partition); - - static::assertTrue($files->touched($partition)); - static::assertFalse($files->touched([new Partition('day', '2024-01-02')])); - } -} diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Filesystem/ScalarFunctionFilterTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Filesystem/ScalarFunctionFilterTest.php index 6200d30f80..d4b6a6d869 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Unit/Filesystem/ScalarFunctionFilterTest.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Filesystem/ScalarFunctionFilterTest.php @@ -107,4 +107,14 @@ public function test_partition_filter_with_cast_rejects_non_matching_files(): vo static::assertTrue($filter->accept(new FileStatus(path('flow-file://data/year=2024/file.csv'), true))); static::assertFalse($filter->accept(new FileStatus(path('flow-file://data/year=2025/file.csv'), true))); } + + public function test_an_undeclared_null_partition_value_reads_as_null(): void + { + $filter = new ScalarFunctionFilter(ref('year')->isNull(), schema(), flow_context(config())); + + static::assertTrue($filter->accept( + new FileStatus(path('flow-file://data/year=__HIVE_DEFAULT_PARTITION__/f.csv'), true), + )); + static::assertFalse($filter->accept(new FileStatus(path('flow-file://data/year=2024/f.csv'), true))); + } } diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/FlowContextTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/FlowContextTest.php index 880deb6336..95cf322b0e 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Unit/FlowContextTest.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/FlowContextTest.php @@ -77,7 +77,7 @@ public function test_flow_context_exposes_no_filesystem_and_no_streams(): void 'cache', 'calculator', 'errorHandler', - 'hydrator', + 'backend', 'setErrorHandler', 'withErrorHandler', 'telemetry', diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/FlowPhpExtensionTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/FlowPhpExtensionTest.php new file mode 100644 index 0000000000..1ecd4d604a --- /dev/null +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/FlowPhpExtensionTest.php @@ -0,0 +1,51 @@ +available()); + } + + public function test_a_loaded_extension_of_the_expected_abi_is_available(): void + { + static::assertTrue((new FlowPhpExtension(true, FlowPhpExtension::ABI))->available()); + } + + public function test_a_loaded_extension_of_another_abi_is_refused(): void + { + $this->expectException(RuntimeException::class); + $this->expectExceptionMessage( + 'The loaded flow_php extension (0.46.0, ABI 2) does not match flow-php/etl (ABI 1). Rebuild or upgrade ' + . 'flow_php to the version of flow-php/etl, or stop loading it to use the PHP implementation.', + ); + + (new FlowPhpExtension(true, FlowPhpExtension::ABI + 1, '0.46.0'))->available(); + } + + public function test_a_loaded_extension_without_an_abi_is_refused(): void + { + $this->expectException(RuntimeException::class); + $this->expectExceptionMessage('The loaded flow_php extension (0.44.1, ABI none) does not match flow-php/etl'); + + (new FlowPhpExtension(true, null, '0.44.1'))->available(); + } + + public function test_detect_reports_whether_flow_php_is_loaded(): void + { + $extension = FlowPhpExtension::detect(); + + static::assertSame(extension_loaded('flow_php'), $extension->loaded); + static::assertSame(extension_loaded('flow_php') ? FlowPhpExtension::ABI : null, $extension->abi); + } +} diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/FlowTestCaseBatchContractTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/FlowTestCaseBatchContractTest.php index e7bb74dcc7..ce0f14defd 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Unit/FlowTestCaseBatchContractTest.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/FlowTestCaseBatchContractTest.php @@ -4,6 +4,7 @@ namespace Flow\ETL\Tests\Unit; +use Flow\ETL\Column\PhpBackend; use Flow\ETL\Tests\Double\CountingExtractor; use Flow\ETL\Tests\Double\OneRowBatchesExtractor; use Flow\ETL\Tests\Double\StopIgnoringExtractor; @@ -49,7 +50,7 @@ public function test_contract_holds_for_a_varying_batch_source(): void $expected = RowsMother::sequentialIds(0); foreach ($batches as $batch) { - $expected = $expected->merge($batch); + $expected = $expected->concat(new PhpBackend(), $batch); } self::assertExtractorHonoursBatchContract( diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Formatter/ASCII/ASCIIBodyTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Formatter/ASCII/ASCIIBodyTest.php index 089c6382d1..89ccbea5d3 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Unit/Formatter/ASCII/ASCIIBodyTest.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Formatter/ASCII/ASCIIBodyTest.php @@ -10,10 +10,9 @@ use Flow\ETL\Tests\CommandOutputNormalizer; use Flow\ETL\Tests\FlowTestCase; +use function Flow\ETL\DSL\array_to_rows; use function Flow\ETL\DSL\float_schema; use function Flow\ETL\DSL\int_schema; -use function Flow\ETL\DSL\row; -use function Flow\ETL\DSL\rows; use function Flow\ETL\DSL\schema; final class ASCIIBodyTest extends FlowTestCase @@ -22,10 +21,9 @@ final class ASCIIBodyTest extends FlowTestCase public function test_printing_ascii_body(): void { - $rows = rows( + $rows = array_to_rows( + [['id' => 1, 'value' => 1.4], ['id' => 2, 'value' => 3.4]], schema(int_schema('id'), float_schema('value')), - row(['id' => 1, 'value' => 1.4]), - row(['id' => 2, 'value' => 3.4]), ); $headers = new ASCIIBody(new Headers($rows), new Body($rows)); diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Formatter/ASCII/ASCIIHeadersTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Formatter/ASCII/ASCIIHeadersTest.php index 6177a6e6ae..0f7355de51 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Unit/Formatter/ASCII/ASCIIHeadersTest.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Formatter/ASCII/ASCIIHeadersTest.php @@ -10,10 +10,9 @@ use Flow\ETL\Tests\CommandOutputNormalizer; use Flow\ETL\Tests\FlowTestCase; +use function Flow\ETL\DSL\array_to_rows; use function Flow\ETL\DSL\float_schema; use function Flow\ETL\DSL\int_schema; -use function Flow\ETL\DSL\row; -use function Flow\ETL\DSL\rows; use function Flow\ETL\DSL\schema; final class ASCIIHeadersTest extends FlowTestCase @@ -22,10 +21,9 @@ final class ASCIIHeadersTest extends FlowTestCase public function test_printing_ascii_headers(): void { - $rows = rows( + $rows = array_to_rows( + [['id' => 1, 'value' => 1.4], ['id' => 2, 'value' => 3.4]], schema(int_schema('id'), float_schema('value')), - row(['id' => 1, 'value' => 1.4]), - row(['id' => 2, 'value' => 3.4]), ); $headers = new ASCIIHeaders(new Headers($rows), new Body($rows)); diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Formatter/ASCIITableTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Formatter/ASCIITableTest.php index 76ecf94220..82a83b4598 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Unit/Formatter/ASCIITableTest.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Formatter/ASCIITableTest.php @@ -8,9 +8,8 @@ use Flow\ETL\Tests\CommandOutputNormalizer; use Flow\ETL\Tests\FlowTestCase; +use function Flow\ETL\DSL\array_to_rows; use function Flow\ETL\DSL\int_schema; -use function Flow\ETL\DSL\row; -use function Flow\ETL\DSL\rows; use function Flow\ETL\DSL\schema; use function Flow\ETL\DSL\str_schema; @@ -20,16 +19,15 @@ final class ASCIITableTest extends FlowTestCase public function test_ascii_table_with_mb_strings(): void { - $rows = rows( - schema(str_schema('row')), - row(['row' => '[498][534]/Wiele z tego,|/co niegdyś było, przepadło.']), - row(['row' => '[540][572]/A nie żyje już nikt z tych,|/którzy by o tym pamiętali.']), - row(['row' => '[572][647]WŁADCA PIERŚCIENI']), - row(['row' => '[701][741]/Wszystko zaczęło się|/od wykucia Pierścieni Władzy.']), - row(['row' => '[742][762]/Trzy zostały dane elfom...']), - row(['row' => '[763][805]/nieśmiertelnym, najmędrszym|/i najbliższym magii spośród wszystkich ras.']), - row(['row' => '[816][853]/Siedem - władcom krasnoludów,|/wspaniałym górnikom']), - ); + $rows = array_to_rows([ + ['row' => '[498][534]/Wiele z tego,|/co niegdyś było, przepadło.'], + ['row' => '[540][572]/A nie żyje już nikt z tych,|/którzy by o tym pamiętali.'], + ['row' => '[572][647]WŁADCA PIERŚCIENI'], + ['row' => '[701][741]/Wszystko zaczęło się|/od wykucia Pierścieni Władzy.'], + ['row' => '[742][762]/Trzy zostały dane elfom...'], + ['row' => '[763][805]/nieśmiertelnym, najmędrszym|/i najbliższym magii spośród wszystkich ras.'], + ['row' => '[816][853]/Siedem - władcom krasnoludów,|/wspaniałym górnikom'], + ], schema(str_schema('row'))); self::assertCommandOutputContains(<<<'TABLE' +-------------------------------------------------------------------------------------+ @@ -48,16 +46,15 @@ public function test_ascii_table_with_mb_strings(): void public function test_ascii_table_with_mb_strings_truncate(): void { - $rows = rows( - schema(str_schema('row')), - row(['row' => '[498][534]/Wiele z tego,|/co niegdyś było, przepadło.']), - row(['row' => '[540][572]/A nie żyje już nikt z tych,|/którzy by o tym pamiętali.']), - row(['row' => '[572][647]WŁADCA PIERŚCIENI']), - row(['row' => '[701][741]/Wszystko zaczęło się|/od wykucia Pierścieni Władzy.']), - row(['row' => '[742][762]/Trzy zostały dane elfom...']), - row(['row' => '[763][805]/nieśmiertelnym, najmędrszym|/i najbliższym magii spośród wszystkich ras.']), - row(['row' => '[816][853]/Siedem - władcom krasnoludów,|/wspaniałym górnikom']), - ); + $rows = array_to_rows([ + ['row' => '[498][534]/Wiele z tego,|/co niegdyś było, przepadło.'], + ['row' => '[540][572]/A nie żyje już nikt z tych,|/którzy by o tym pamiętali.'], + ['row' => '[572][647]WŁADCA PIERŚCIENI'], + ['row' => '[701][741]/Wszystko zaczęło się|/od wykucia Pierścieni Władzy.'], + ['row' => '[742][762]/Trzy zostały dane elfom...'], + ['row' => '[763][805]/nieśmiertelnym, najmędrszym|/i najbliższym magii spośród wszystkich ras.'], + ['row' => '[816][853]/Siedem - władcom krasnoludów,|/wspaniałym górnikom'], + ], schema(str_schema('row'))); self::assertCommandOutputContains(<<<'TABLE' +----------------------+ @@ -76,15 +73,17 @@ public function test_ascii_table_with_mb_strings_truncate(): void public function test_ascii_table_with_non_symmetric_entries(): void { - $rows = rows( + $rows = array_to_rows( + [ + ['row' => '[498][534]/Wiele z tego,|/co niegdyś było, przepadło.'], + ['row' => '[540][572]/A nie żyje już nikt z tych,|/którzy by o tym pamiętali.'], + ['row' => '[572][647]WŁADCA PIERŚCIENI'], + ['row' => '[701][741]/Wszystko zaczęło się|/od wykucia Pierścieni Władzy.'], + ['row' => '[742][762]/Trzy zostały dane elfom...'], + ['row' => '[763][805]/nieśmiertelnym, najmędrszym|/i najbliższym magii spośród wszystkich ras.'], + ['test' => '[816][853]/Siedem - władcom krasnoludów,|/wspaniałym górnikom'], + ], schema(str_schema('row', nullable: true), str_schema('test', nullable: true)), - row(['row' => '[498][534]/Wiele z tego,|/co niegdyś było, przepadło.']), - row(['row' => '[540][572]/A nie żyje już nikt z tych,|/którzy by o tym pamiętali.']), - row(['row' => '[572][647]WŁADCA PIERŚCIENI']), - row(['row' => '[701][741]/Wszystko zaczęło się|/od wykucia Pierścieni Władzy.']), - row(['row' => '[742][762]/Trzy zostały dane elfom...']), - row(['row' => '[763][805]/nieśmiertelnym, najmędrszym|/i najbliższym magii spośród wszystkich ras.']), - row(['test' => '[816][853]/Siedem - władcom krasnoludów,|/wspaniałym górnikom']), ); self::assertCommandOutputContains(<<<'TABLE' @@ -111,10 +110,9 @@ public function test_ascii_table_with_single_row(): void | 1 | EN | | 2 | PL | +----+------+ - TABLE, (new ASCIITable(rows( + TABLE, (new ASCIITable(array_to_rows( + [['id' => 1, 'name' => 'EN'], ['id' => 2, 'name' => 'PL']], schema(int_schema('id'), str_schema('name')), - row(['id' => 1, 'name' => 'EN']), - row(['id' => 2, 'name' => 'PL']), )))->print(false)); } } diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/AllFunctionsEvaluateOnTheConfiguredBackendTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/AllFunctionsEvaluateOnTheConfiguredBackendTest.php new file mode 100644 index 0000000000..fd1b941863 --- /dev/null +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/AllFunctionsEvaluateOnTheConfiguredBackendTest.php @@ -0,0 +1,113 @@ +}> + */ + public static function functions(): Generator + { + foreach (array_keys(ScalarFunctionFixtures::all()) as $class) { + yield $class => [$class]; + } + } + + /** + * @param class-string $class + */ + #[DataProvider('functions')] + public function test_a_function_gives_the_same_values_and_a_column_of_the_configured_backend(string $class): void + { + try { + // a fresh function per run: a function may keep what it evaluated (Literal keeps its column) + $resolve = static fn(): ScalarFunction => (new ReferenceResolver())->resolve( + ScalarFunctionFixtures::instance($class), + ScalarFunctionFixtures::schema(), + ); + $resolve(); + } catch (RequiredPHPVersionException $e) { + static::markTestSkipped($e->getMessage()); + } + + try { + $php = $resolve()->eval( + ScalarFunctionFixtures::rows(new PhpBackend()), + flow_context(config_builder()->backend(new PhpBackend())->build()), + ); + } catch (Throwable $phpFailure) { + $this->expectException($phpFailure::class); + $this->expectExceptionMessage($phpFailure->getMessage()); + + $resolve()->eval( + ScalarFunctionFixtures::rows(new AdaptiveBackend()), + flow_context(config_builder()->backend(new AdaptiveBackend())->build()), + ); + + return; + } + + $function = $resolve(); + $configured = $function->eval( + ScalarFunctionFixtures::rows(new AdaptiveBackend()), + flow_context(config_builder()->backend(new AdaptiveBackend())->build()), + ); + // the batch's columns come from another backend than the configured one + $adaptiveUnderPhp = $resolve()->eval( + ScalarFunctionFixtures::rows(new AdaptiveBackend()), + flow_context(config_builder()->backend(new PhpBackend())->build()), + ); + + if ($function->deterministic()) { + static::assertEquals($php->values(), $configured->values()); + static::assertEquals($php->values(), $adaptiveUnderPhp->values()); + } else { + static::assertSame($php->count(), $configured->count()); + static::assertSame($php->count(), $adaptiveUnderPhp->count()); + } + + static::assertStringStartsWith('Flow\ETL\Column\Php\\', $php::class, $class . ' under PhpBackend'); + + // a reference hands back the batch's own column, whatever backend built it + if (!$function instanceof Reference) { + static::assertStringStartsWith( + 'Flow\ETL\Column\Php\\', + $adaptiveUnderPhp::class, + $class . ' over Adaptive columns under PhpBackend', + ); + } + + if ($php instanceof ValueColumn) { + return; + } + + static::assertTrue( + extension_loaded('flow_php') + ? $configured::class === 'Flow\ETL\Column\RustColumn' + : str_starts_with($configured::class, 'Flow\ETL\Column\Php\\'), + $class . ' evaluated into a ' . $configured::class, + ); + } +} diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/AllTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/AllTest.php index 6b166b23d8..684eb7e14c 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/AllTest.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/AllTest.php @@ -5,15 +5,22 @@ namespace Flow\ETL\Tests\Unit\Function; use Flow\ETL\Function\All; +use Flow\ETL\Function\ReferenceResolver; +use Flow\ETL\Tests\Context\FunctionContext; +use Flow\ETL\Tests\Double\FailingOnValuesFunction; use Flow\ETL\Tests\FlowTestCase; use Generator; use PHPUnit\Framework\Attributes\DataProvider; use function Flow\ETL\DSL\all; +use function Flow\ETL\DSL\array_to_rows; +use function Flow\ETL\DSL\bool_schema; use function Flow\ETL\DSL\flow_context; +use function Flow\ETL\DSL\int_schema; use function Flow\ETL\DSL\lit; use function Flow\ETL\DSL\ref; -use function Flow\ETL\DSL\row; +use function Flow\ETL\DSL\schema; +use function Flow\ETL\DSL\str_schema; final class AllTest extends FlowTestCase { @@ -41,31 +48,63 @@ public static function three_valued_and(): Generator #[DataProvider('three_valued_and')] public function test_three_valued_truth_table(?bool $left, ?bool $right, ?bool $expected): void { - static::assertSame($expected, (new All(lit($left), lit($right)))->eval(row([]), flow_context())); + static::assertSame($expected, (new FunctionContext(flow_context()))->eval( + new All(lit($left), lit($right)), + [], + schema(), + )); } public function test_all_expression_on_is_null_expression(): void { - static::assertTrue(all(ref('value')->isNull())->eval(row(['value' => null]), flow_context())); + static::assertTrue((new FunctionContext(flow_context()))->eval( + all(ref('value')->isNull()), + ['value' => null], + schema(str_schema('value', nullable: true)), + )); } public function test_all_expression_on_multiple_boolean_values(): void { - static::assertTrue(all(lit(true), lit(true), lit(true))->eval(row([]), flow_context())); + static::assertTrue((new FunctionContext(flow_context()))->eval( + all(lit(true), lit(true), lit(true)), + [], + schema(), + )); } public function test_all_expression_on_multiple_random_boolean_values(): void { - static::assertFalse(all(lit(true), lit(false), lit(true))->eval(row([]), flow_context())); + static::assertFalse((new FunctionContext(flow_context()))->eval( + all(lit(true), lit(false), lit(true)), + [], + schema(), + )); } public function test_all_function_on_boolean_false_value(): void { - static::assertFalse(all(lit(false))->eval(row([]), flow_context())); + static::assertFalse((new FunctionContext(flow_context()))->eval(all(lit(false)), [], schema())); } public function test_all_function_on_boolean_true_value(): void { - static::assertTrue(all(lit(true))->eval(row([]), flow_context())); + static::assertTrue((new FunctionContext(flow_context()))->eval(all(lit(true)), [], schema())); + } + + public function test_an_undecided_row_only_reaches_the_next_argument(): void + { + $rows = array_to_rows( + [['c' => false, 'v' => 99], ['c' => true, 'v' => 1]], + schema(bool_schema('c'), int_schema('v')), + ); + + static::assertSame( + [false, true], + (new ReferenceResolver()) + ->resolve(all(ref('c'), (new FailingOnValuesFunction(ref('v'), [99]))->isNotNull()), $rows->schema()) + ->eval($rows, flow_context()) + ->values(), + ); } } diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/AnyTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/AnyTest.php index 9075da0381..a4ed0a497f 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/AnyTest.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/AnyTest.php @@ -5,15 +5,22 @@ namespace Flow\ETL\Tests\Unit\Function; use Flow\ETL\Function\Any; +use Flow\ETL\Function\ReferenceResolver; +use Flow\ETL\Tests\Context\FunctionContext; +use Flow\ETL\Tests\Double\FailingOnValuesFunction; use Flow\ETL\Tests\FlowTestCase; use Generator; use PHPUnit\Framework\Attributes\DataProvider; use function Flow\ETL\DSL\any; +use function Flow\ETL\DSL\array_to_rows; +use function Flow\ETL\DSL\bool_schema; use function Flow\ETL\DSL\flow_context; +use function Flow\ETL\DSL\int_schema; use function Flow\ETL\DSL\lit; use function Flow\ETL\DSL\ref; -use function Flow\ETL\DSL\row; +use function Flow\ETL\DSL\schema; +use function Flow\ETL\DSL\str_schema; final class AnyTest extends FlowTestCase { @@ -41,26 +48,54 @@ public static function three_valued_or(): Generator #[DataProvider('three_valued_or')] public function test_three_valued_truth_table(?bool $left, ?bool $right, ?bool $expected): void { - static::assertSame($expected, (new Any(lit($left), lit($right)))->eval(row([]), flow_context())); + static::assertSame($expected, (new FunctionContext(flow_context()))->eval( + new Any(lit($left), lit($right)), + [], + schema(), + )); } public function test_any_expression_on_boolean_false_value(): void { - static::assertFalse(any(lit(false))->eval(row([]), flow_context())); + static::assertFalse((new FunctionContext(flow_context()))->eval(any(lit(false)), [], schema())); } public function test_any_expression_on_boolean_true_value(): void { - static::assertTrue(any(lit(true))->eval(row([]), flow_context())); + static::assertTrue((new FunctionContext(flow_context()))->eval(any(lit(true)), [], schema())); } public function test_any_expression_on_is_null_expression(): void { - static::assertTrue(any(ref('value')->isNull())->eval(row(['value' => null]), flow_context())); + static::assertTrue((new FunctionContext(flow_context()))->eval( + any(ref('value')->isNull()), + ['value' => null], + schema(str_schema('value', nullable: true)), + )); } public function test_any_expression_on_multiple_boolean_values(): void { - static::assertTrue(any(lit(false), lit(true), lit(false))->eval(row([]), flow_context())); + static::assertTrue((new FunctionContext(flow_context()))->eval( + any(lit(false), lit(true), lit(false)), + [], + schema(), + )); + } + + public function test_an_undecided_row_only_reaches_the_next_argument(): void + { + $rows = array_to_rows( + [['c' => true, 'v' => 99], ['c' => false, 'v' => 1]], + schema(bool_schema('c'), int_schema('v')), + ); + + static::assertSame( + [true, true], + (new ReferenceResolver()) + ->resolve(any(ref('c'), (new FailingOnValuesFunction(ref('v'), [99]))->isNotNull()), $rows->schema()) + ->eval($rows, flow_context()) + ->values(), + ); } } diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/AppendTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/AppendTest.php index bb48210e92..a57b43d41b 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/AppendTest.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/AppendTest.php @@ -5,41 +5,51 @@ namespace Flow\ETL\Tests\Unit\Function; use Flow\ETL\Exception\InvalidArgumentException; +use Flow\ETL\Tests\Context\FunctionContext; use Flow\ETL\Tests\FlowTestCase; use function Flow\ETL\DSL\config; use function Flow\ETL\DSL\flow_context; use function Flow\ETL\DSL\ref; -use function Flow\ETL\DSL\row; +use function Flow\ETL\DSL\schema; +use function Flow\ETL\DSL\str_schema; final class AppendTest extends FlowTestCase { public function test_append_empty_string_to_content(): void { - $result = ref('str')->append('')->eval(row(['str' => 'hello']), flow_context()); - - static::assertEquals('hello', $result); + static::assertEquals('hello', (new FunctionContext(flow_context()))->eval( + ref('str')->append(''), + ['str' => 'hello'], + schema(str_schema('str')), + )); } public function test_append_to_empty_string(): void { - $result = ref('str')->append('hello')->eval(row(['str' => '']), flow_context()); - - static::assertEquals('hello', $result); + static::assertEquals('hello', (new FunctionContext(flow_context()))->eval( + ref('str')->append('hello'), + ['str' => ''], + schema(str_schema('str')), + )); } public function test_append_to_non_empty_string(): void { - $result = ref('str')->append(' world')->eval(row(['str' => 'hello']), flow_context()); - - static::assertEquals('hello world', $result); + static::assertEquals('hello world', (new FunctionContext(flow_context()))->eval( + ref('str')->append(' world'), + ['str' => 'hello'], + schema(str_schema('str')), + )); } public function test_append_with_null_suffix(): void { - $result = ref('str')->append(ref('suffix'))->eval(row(['str' => 'hello', 'suffix' => null]), flow_context()); - - static::assertEquals('hello', $result); + static::assertEquals('hello', (new FunctionContext(flow_context()))->eval( + ref('str')->append(ref('suffix')), + ['str' => 'hello', 'suffix' => null], + schema(str_schema('str'), str_schema('suffix', nullable: true)), + )); } public function test_append_with_null_value(): void @@ -47,9 +57,11 @@ public function test_append_with_null_value(): void $this->expectException(InvalidArgumentException::class); $this->expectExceptionMessage('Append function requires non-null value'); - $result = ref('str')->append(' world')->eval(row(['str' => null]), flow_context()); - - static::assertNull($result); + static::assertNull((new FunctionContext(flow_context()))->eval( + ref('str')->append(' world'), + ['str' => null], + schema(str_schema('str', nullable: true)), + )); } public function test_append_with_null_value_in_strict_mode(): void @@ -58,15 +70,19 @@ public function test_append_with_null_value_in_strict_mode(): void $this->expectExceptionMessage('Append function requires non-null value'); $context = flow_context(config()); - ref('str')->append(' world')->eval(row(['str' => null]), $context); + (new FunctionContext($context))->eval( + ref('str')->append(' world'), + ['str' => null], + schema(str_schema('str', nullable: true)), + ); } public function test_append_with_scalar_function_parameter(): void { - $result = ref('str') - ->append(ref('suffix')) - ->eval(row(['str' => 'hello', 'suffix' => ' world']), flow_context()); - - static::assertEquals('hello world', $result); + static::assertEquals('hello world', (new FunctionContext(flow_context()))->eval( + ref('str')->append(ref('suffix')), + ['str' => 'hello', 'suffix' => ' world'], + schema(str_schema('str'), str_schema('suffix')), + )); } } diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/ArrayExpandTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/ArrayExpandTest.php index d0856661d1..aef0d82eb6 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/ArrayExpandTest.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/ArrayExpandTest.php @@ -8,17 +8,19 @@ use Flow\ETL\Exception\SchemaNotDerivableException; use Flow\ETL\Function\ArrayExpand\ArrayExpand; use Flow\ETL\Function\ReferenceResolver; +use Flow\ETL\Tests\Context\FunctionContext; use Flow\ETL\Tests\FlowTestCase; use PHPUnit\Framework\Attributes\TestWith; use function Flow\ETL\DSL\array_expand; use function Flow\ETL\DSL\config; use function Flow\ETL\DSL\flow_context; +use function Flow\ETL\DSL\int_schema; use function Flow\ETL\DSL\list_schema; use function Flow\ETL\DSL\map_schema; use function Flow\ETL\DSL\ref; -use function Flow\ETL\DSL\row; use function Flow\ETL\DSL\schema; +use function Flow\ETL\DSL\str_schema; use function Flow\ETL\DSL\structure_schema; use function Flow\Types\DSL\type_boolean; use function Flow\Types\DSL\type_integer; @@ -31,30 +33,55 @@ final class ArrayExpandTest extends FlowTestCase { public function test_expand_both(): void { - $row = row(['array' => ['a' => 1, 'b' => 2, 'c' => 3]]); - static::assertSame( [ ['a' => 1], ['b' => 2], ['c' => 3], ], - array_expand(ref('array'), ArrayExpand::BOTH)->eval($row, flow_context()), + (new FunctionContext(flow_context()))->eval( + array_expand(ref('array'), ArrayExpand::BOTH), + ['array' => [ + 'a' => 1, + 'b' => 2, + 'c' => 3, + ]], + schema(map_schema('array', type_map(type_string(), type_integer()))), + ), ); } public function test_expand_keys(): void { - $row = row(['array' => ['a' => 1, 'b' => 2, 'c' => 3]]); - - static::assertSame(['a', 'b', 'c'], array_expand(ref('array'), ArrayExpand::KEYS)->eval($row, flow_context())); + static::assertSame( + ['a', 'b', 'c'], + (new FunctionContext(flow_context()))->eval( + array_expand(ref('array'), ArrayExpand::KEYS), + ['array' => [ + 'a' => 1, + 'b' => 2, + 'c' => 3, + ]], + schema(map_schema('array', type_map(type_string(), type_integer()))), + ), + ); } public function test_expand_values(): void { - $row = row(['array' => ['a' => 1, 'b' => 2, 'c' => 3]]); - - static::assertSame(['a' => 1, 'b' => 2, 'c' => 3], array_expand(ref('array'))->eval($row, flow_context())); + // an expand hands the step a list: the keys never reached the exploded column + static::assertSame( + [1, 2, 3], + (new FunctionContext(flow_context()))->eval( + array_expand(ref('array')), + ['array' => [ + 'a' => 1, + 'b' => 2, + 'c' => 3, + ]], + schema(map_schema('array', type_map(type_string(), type_integer()))), + ), + ); } public function test_for_not_array_entry(): void @@ -62,7 +89,11 @@ public function test_for_not_array_entry(): void $this->expectException(InvalidArgumentException::class); $this->expectExceptionMessage('Expected type "array", got "integer".'); - array_expand(ref('integer_entry'))->eval(row(['integer_entry' => 1]), flow_context()); + (new FunctionContext(flow_context()))->eval( + array_expand(ref('integer_entry')), + ['integer_entry' => 1], + schema(int_schema('integer_entry')), + ); } public function test_for_not_array_entry_in_strict_mode(): void @@ -71,7 +102,11 @@ public function test_for_not_array_entry_in_strict_mode(): void $this->expectExceptionMessage('Expected type "array", got "integer".'); $context = flow_context(config()); - array_expand(ref('integer_entry'))->eval(row(['integer_entry' => 1]), $context); + (new FunctionContext($context))->eval( + array_expand(ref('integer_entry')), + ['integer_entry' => 1], + schema(int_schema('integer_entry')), + ); } public function test_values_over_a_structure_declare_the_unified_field_type(): void @@ -177,6 +212,15 @@ public function test_values_over_a_structure_with_non_unifying_fields_refuse_to_ #[TestWith([ArrayExpand::BOTH])] public function test_a_null_array_expands_to_nothing(ArrayExpand $expand): void { - static::assertSame([], array_expand(ref('list'), $expand)->eval(row(['list' => null]), flow_context())); + static::assertSame( + [], + (new FunctionContext(flow_context()))->eval( + array_expand(ref('list'), $expand), + [ + 'list' => null, + ], + schema(str_schema('list', nullable: true)), + ), + ); } } diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/ArrayFilterTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/ArrayFilterTest.php index 1a9afe3324..b8c943b13c 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/ArrayFilterTest.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/ArrayFilterTest.php @@ -5,26 +5,46 @@ namespace Flow\ETL\Tests\Unit\Function; use Flow\ETL\Exception\InvalidArgumentException; +use Flow\ETL\Tests\Context\FunctionContext; use Flow\ETL\Tests\FlowTestCase; use function Flow\ETL\DSL\config; use function Flow\ETL\DSL\flow_context; +use function Flow\ETL\DSL\int_schema; +use function Flow\ETL\DSL\list_schema; use function Flow\ETL\DSL\lit; use function Flow\ETL\DSL\ref; -use function Flow\ETL\DSL\row; +use function Flow\ETL\DSL\schema; +use function Flow\ETL\DSL\str_schema; +use function Flow\Types\DSL\type_integer; +use function Flow\Types\DSL\type_list; final class ArrayFilterTest extends FlowTestCase { public function test_array_filter(): void { - static::assertSame([1], ref('list')->arrayFilter(lit(2))->eval(row(['list' => [1, 2]]), flow_context())); + static::assertSame( + [1], + (new FunctionContext(flow_context()))->eval( + ref('list')->arrayFilter(lit(2)), + ['list' => [ + 1, + 2, + ]], + schema(list_schema('list', type_list(type_integer()))), + ), + ); } public function test_array_filter_by_entry_reference(): void { static::assertSame( [1], - ref('list')->arrayFilter(ref('int'))->eval(row(['list' => [1, 2], 'int' => 2]), flow_context()), + (new FunctionContext(flow_context()))->eval( + ref('list')->arrayFilter(ref('int')), + ['list' => [1, 2], 'int' => 2], + schema(list_schema('list', type_list(type_integer())), int_schema('int')), + ), ); } @@ -34,12 +54,26 @@ public function test_array_filter_in_strict_mode(): void $this->expectExceptionMessage('Expected type "array", got "string".'); $context = flow_context(config()); - ref('map')->arrayFilter(lit(1))->eval(row(['map' => 'test']), $context); + (new FunctionContext($context))->eval( + ref('map')->arrayFilter(lit(1)), + ['map' => 'test'], + schema(str_schema('map')), + ); } public function test_array_filter_not_existing_value(): void { - static::assertSame([1, 2], ref('list')->arrayFilter(lit(5))->eval(row(['list' => [1, 2]]), flow_context())); + static::assertSame( + [1, 2], + (new FunctionContext(flow_context()))->eval( + ref('list')->arrayFilter(lit(5)), + ['list' => [ + 1, + 2, + ]], + schema(list_schema('list', type_list(type_integer()))), + ), + ); } public function test_array_filter_on_non_array(): void @@ -47,6 +81,10 @@ public function test_array_filter_on_non_array(): void $this->expectException(InvalidArgumentException::class); $this->expectExceptionMessage('Expected type "array", got "string".'); - ref('map')->arrayFilter(lit(1))->eval(row(['map' => 'test']), flow_context()); + (new FunctionContext(flow_context()))->eval( + ref('map')->arrayFilter(lit(1)), + ['map' => 'test'], + schema(str_schema('map')), + ); } } diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/ArrayGetCollectionTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/ArrayGetCollectionTest.php index 42aa8ddd64..1fba097f60 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/ArrayGetCollectionTest.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/ArrayGetCollectionTest.php @@ -5,17 +5,29 @@ namespace Flow\ETL\Tests\Unit\Function; use Flow\ETL\Exception\InvalidArgumentException; +use Flow\ETL\Tests\Context\FunctionContext; use Flow\ETL\Tests\FlowTestCase; +use Flow\ETL\Tests\Mother\RowsMother; use PHPUnit\Framework\Attributes\TestWith; use function Flow\ETL\DSL\array_get_collection; use function Flow\ETL\DSL\array_get_collection_first; use function Flow\ETL\DSL\config; use function Flow\ETL\DSL\flow_context; +use function Flow\ETL\DSL\int_schema; +use function Flow\ETL\DSL\json_schema; +use function Flow\ETL\DSL\list_schema; +use function Flow\ETL\DSL\map_schema; use function Flow\ETL\DSL\ref; -use function Flow\ETL\DSL\row; +use function Flow\ETL\DSL\schema; use function Flow\ETL\DSL\structure_get_collection; use function Flow\ETL\DSL\structure_get_collection_first; +use function Flow\Types\DSL\type_boolean; +use function Flow\Types\DSL\type_integer; +use function Flow\Types\DSL\type_list; +use function Flow\Types\DSL\type_map; +use function Flow\Types\DSL\type_string; +use function Flow\Types\DSL\type_structure; final class ArrayGetCollectionTest extends FlowTestCase { @@ -25,9 +37,12 @@ public function test_array_get_collection_in_strict_mode(): void $this->expectExceptionMessage('ArrayGetCollection function failed to evaluate parameters'); $context = flow_context(config()); - $row = row(['invalid_entry' => 1]); - array_get_collection(ref('invalid_entry'), ['id'])->eval($row, $context); + (new FunctionContext($context))->eval( + array_get_collection(ref('invalid_entry'), ['id']), + ['invalid_entry' => 1], + schema(int_schema('invalid_entry')), + ); } public function test_for_not_array_entry(): void @@ -35,9 +50,11 @@ public function test_for_not_array_entry(): void $this->expectException(InvalidArgumentException::class); $this->expectExceptionMessage('ArrayGetCollection function failed to evaluate parameters.'); - $row = row(['invalid_entry' => 1]); - - array_get_collection(ref('invalid_entry'), ['id'])->eval($row, flow_context()); + (new FunctionContext(flow_context()))->eval( + array_get_collection(ref('invalid_entry'), ['id']), + ['invalid_entry' => 1], + schema(int_schema('invalid_entry')), + ); } public function test_getting_keys_from_simple_array(): void @@ -45,101 +62,127 @@ public function test_getting_keys_from_simple_array(): void $this->expectException(InvalidArgumentException::class); $this->expectExceptionMessage('ArrayGetCollection function failed to evaluate parameters.'); - $row = row([ - 'array_entry' => [ - 'id' => 1, - 'status' => 'PENDING', - 'enabled' => true, - 'array' => ['foo' => 'bar'], - ], - ]); + $rows = RowsMother::arrayEntry(); - array_get_collection(ref('array_entry'), ['id'])->eval($row, flow_context()); + (new FunctionContext(flow_context()))->eval( + array_get_collection(ref('array_entry'), ['id']), + $rows->values(0), + $rows->schema(), + ); } public function test_getting_specific_keys_from_collection_of_array(): void { - $row = row([ - 'array_entry' => [ - [ - 'id' => 1, - 'status' => 'PENDING', - 'enabled' => true, - 'array' => ['foo' => 'bar'], - ], - [ - 'id' => 2, - 'status' => 'NEW', - 'enabled' => true, - 'array' => ['foo' => 'bar'], - ], - ], - ]); - static::assertEquals( [ ['id' => 1, 'status' => 'PENDING'], ['id' => 2, 'status' => 'NEW'], ], - array_get_collection(ref('array_entry'), ['id', 'status'])->eval($row, flow_context()), + (new FunctionContext(flow_context()))->eval( + array_get_collection(ref('array_entry'), ['id', 'status']), + [ + 'array_entry' => [ + [ + 'id' => 1, + 'status' => 'PENDING', + 'enabled' => true, + 'array' => ['foo' => 'bar'], + ], + [ + 'id' => 2, + 'status' => 'NEW', + 'enabled' => true, + 'array' => ['foo' => 'bar'], + ], + ], + ], + schema(list_schema( + 'array_entry', + type_list(type_structure([ + 'id' => type_integer(), + 'status' => type_string(), + 'enabled' => type_boolean(), + 'array' => type_structure(['foo' => type_string()]), + ])), + )), + ), ); } public function test_getting_specific_keys_from_first_element_in_collection_of_array(): void { - $row = row([ - 'array_entry' => [ - [ - 'parent_id' => 1, - 'id' => 1, - 'status' => 'PENDING', - 'enabled' => true, - 'array' => ['foo' => 'bar'], - ], - [ - 'parent_id' => 1, - 'id' => 2, - 'status' => 'NEW', - 'enabled' => true, - 'array' => ['foo' => 'bar'], - ], - ], - ]); - static::assertEquals( [ 'parent_id' => 1, ], - ref('array_entry')->arrayGetCollectionFirst('parent_id')->eval($row, flow_context()), + (new FunctionContext(flow_context()))->eval( + ref('array_entry')->arrayGetCollectionFirst('parent_id'), + [ + 'array_entry' => [ + [ + 'parent_id' => 1, + 'id' => 1, + 'status' => 'PENDING', + 'enabled' => true, + 'array' => ['foo' => 'bar'], + ], + [ + 'parent_id' => 1, + 'id' => 2, + 'status' => 'NEW', + 'enabled' => true, + 'array' => ['foo' => 'bar'], + ], + ], + ], + schema(list_schema( + 'array_entry', + type_list(type_structure([ + 'parent_id' => type_integer(), + 'id' => type_integer(), + 'status' => type_string(), + 'enabled' => type_boolean(), + 'array' => type_structure(['foo' => type_string()]), + ])), + )), + ), ); } public function test_getting_specific_keys_from_first_element_in_collection_of_array_when_first_index_does_not_exists(): void { - $row = row([ - 'array_entry' => [ - 2 => [ - 'parent_id' => 1, - 'id' => 1, - 'status' => 'PENDING', - 'enabled' => true, - 'array' => ['foo' => 'bar'], - ], - 3 => [ - 'parent_id' => 1, - 'id' => 2, - 'status' => 'NEW', - 'enabled' => true, - 'array' => ['foo' => 'bar'], - ], - ], - ]); - static::assertEquals( [ 'parent_id' => 1, ], - array_get_collection_first(ref('array_entry'), 'parent_id')->eval($row, flow_context()), + (new FunctionContext(flow_context()))->eval( + array_get_collection_first(ref('array_entry'), 'parent_id'), + [ + 'array_entry' => [ + 2 => [ + 'parent_id' => 1, + 'id' => 1, + 'status' => 'PENDING', + 'enabled' => true, + 'array' => ['foo' => 'bar'], + ], + 3 => [ + 'parent_id' => 1, + 'id' => 2, + 'status' => 'NEW', + 'enabled' => true, + 'array' => ['foo' => 'bar'], + ], + ], + ], + schema(map_schema('array_entry', type_map(type_integer(), type_structure([ + 'parent_id' => type_integer(), + 'id' => type_integer(), + 'status' => type_string(), + 'enabled' => type_boolean(), + 'array' => type_structure(['foo' => type_string()]), + ])))), + ), ); } @@ -153,9 +196,13 @@ public function test_getting_specific_keys_from_first_element_in_collection_of_a #[TestWith([['a.b'], [['a' => ['b' => 1]]], [['a.b' => null]]])] public function test_keys_are_literal_keys(array $keys, array $collection, array $expected): void { - static::assertSame($expected, array_get_collection(ref('array_entry'), $keys)->eval(row([ - 'array_entry' => $collection, - ]), flow_context())); + static::assertSame($expected, (new FunctionContext(flow_context()))->eval( + array_get_collection(ref('array_entry'), $keys), + [ + 'array_entry' => $collection, + ], + schema(json_schema('array_entry')), + )); } public function test_a_non_scalar_key_fails(): void @@ -163,16 +210,22 @@ public function test_a_non_scalar_key_fails(): void $this->expectException(InvalidArgumentException::class); $this->expectExceptionMessage('ArrayGetCollection function failed to evaluate parameters.'); - array_get_collection(ref('array_entry'), [['x']])->eval(row(['array_entry' => [['x' => 1]]]), flow_context()); + (new FunctionContext(flow_context()))->eval(array_get_collection(ref('array_entry'), [[ + 'x', + ]]), ['array_entry' => [['x' => 1]]], schema(list_schema('array_entry', type_list(type_structure(['x' => type_integer()]))))); } public function test_an_empty_element_reads_null_keys(): void { static::assertSame( [['id' => null], ['id' => null]], - array_get_collection(ref('array_entry'), ['id'])->eval(row([ - 'array_entry' => [['name' => 'a'], []], - ]), flow_context()), + (new FunctionContext(flow_context()))->eval( + array_get_collection(ref('array_entry'), ['id']), + [ + 'array_entry' => [['name' => 'a'], []], + ], + schema(json_schema('array_entry')), + ), ); } diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/ArrayGetTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/ArrayGetTest.php index 7435e91ca3..99882b85f1 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/ArrayGetTest.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/ArrayGetTest.php @@ -6,25 +6,30 @@ use DateTimeImmutable; use Flow\ArrayDot\Exception\InvalidPathException; +use Flow\ETL\Exception\EvaluationException; use Flow\ETL\Exception\InvalidArgumentException; use Flow\ETL\Exception\SchemaNotDerivableException; use Flow\ETL\Function\ArrayGet; use Flow\ETL\Function\ReferenceResolver; use Flow\ETL\Function\ScalarFunction; +use Flow\ETL\Tests\Context\FunctionContext; use Flow\ETL\Tests\FlowTestCase; +use Flow\ETL\Tests\Mother\RowsMother; use Flow\Types\Type\Logical\StructureType; use PHPUnit\Framework\Attributes\DataProvider; use function Flow\ETL\DSL\array_exists; use function Flow\ETL\DSL\array_get; use function Flow\ETL\DSL\flow_context; +use function Flow\ETL\DSL\int_schema; use function Flow\ETL\DSL\list_schema; use function Flow\ETL\DSL\ref; -use function Flow\ETL\DSL\row; use function Flow\ETL\DSL\schema; use function Flow\ETL\DSL\structure_get; use function Flow\ETL\DSL\structure_schema; use function Flow\Types\DSL\structure_element; +use function Flow\Types\DSL\type_boolean; +use function Flow\Types\DSL\type_datetime; use function Flow\Types\DSL\type_integer; use function Flow\Types\DSL\type_list; use function Flow\Types\DSL\type_mixed; @@ -108,8 +113,20 @@ public function test_array_access_for_not_array_entry(): void $this->expectException(InvalidArgumentException::class); $this->expectExceptionMessage('ArrayGet function failed to get value from array.'); - array_get(ref('integer_entry'), 'invalid_path')->eval(row(['integer_entry' => 1]), flow_context()); - array_exists(ref('integer_entry'), 'invalid_path')->eval(row(['integer_entry' => 1]), flow_context()); + (new FunctionContext(flow_context()))->eval( + array_get(ref('integer_entry'), 'invalid_path'), + [ + 'integer_entry' => 1, + ], + schema(int_schema('integer_entry')), + ); + (new FunctionContext(flow_context()))->eval( + array_exists(ref('integer_entry'), 'invalid_path'), + [ + 'integer_entry' => 1, + ], + schema(int_schema('integer_entry')), + ); } public function test_array_access_for_not_array_entry_strict_mode(): void @@ -118,53 +135,118 @@ public function test_array_access_for_not_array_entry_strict_mode(): void $this->expectException(InvalidArgumentException::class); $this->expectExceptionMessage('ArrayGet function failed to get value from array'); - array_get(ref('integer_entry'), 'invalid_path')->eval(row(['integer_entry' => 1]), $context); + (new FunctionContext($context))->eval( + array_get(ref('integer_entry'), 'invalid_path'), + [ + 'integer_entry' => 1, + ], + schema(int_schema('integer_entry')), + ); } public function test_array_accessor_transformer(): void { - $row = row([ - 'array_entry' => [ - 'id' => 1, - 'status' => 'PENDING', - 'enabled' => true, - 'array' => ['foo' => 'bar'], - ], - ]); - static::assertEquals('bar', array_get(ref('array_entry'), 'array.foo')->eval($row, flow_context())); - static::assertTrue(array_exists(ref('array_entry'), 'array.foo')->eval($row, flow_context())); + $rows = RowsMother::arrayEntry(); + static::assertEquals('bar', (new FunctionContext(flow_context()))->eval( + array_get(ref('array_entry'), 'array.foo'), + $rows->values(0), + $rows->schema(), + )); + static::assertTrue((new FunctionContext(flow_context()))->eval( + array_exists(ref('array_entry'), 'array.foo'), + $rows->values(0), + $rows->schema(), + )); } public function test_array_accessor_transformer_with_invalid_and_without_strict_path(): void { - $row = row([ - 'array_entry' => [ - 'id' => 1, - 'status' => 'PENDING', - 'enabled' => true, - 'datetime' => new DateTimeImmutable('2020-01-01 00:00:00 UTC'), - 'array' => ['foo' => 'bar'], + static::assertNull((new FunctionContext(flow_context()))->eval( + array_get(ref('array_entry'), '?invalid_path'), + [ + 'array_entry' => [ + 'id' => 1, + 'status' => 'PENDING', + 'enabled' => true, + 'datetime' => new DateTimeImmutable('2020-01-01 00:00:00 UTC'), + 'array' => ['foo' => 'bar'], + ], + ], + schema(structure_schema('array_entry', type_structure([ + 'id' => type_integer(), + 'status' => type_string(), + 'enabled' => type_boolean(), + 'datetime' => type_datetime(), + 'array' => type_structure(['foo' => type_string()]), + ]))), + )); + static::assertTrue((new FunctionContext(flow_context()))->eval( + array_exists(ref('array_entry'), '?invalid_path'), + [ + 'array_entry' => [ + 'id' => 1, + 'status' => 'PENDING', + 'enabled' => true, + 'datetime' => new DateTimeImmutable('2020-01-01 00:00:00 UTC'), + 'array' => ['foo' => 'bar'], + ], ], - ]); - static::assertNull(array_get(ref('array_entry'), '?invalid_path')->eval($row, flow_context())); - static::assertTrue(array_exists(ref('array_entry'), '?invalid_path')->eval($row, flow_context())); - static::assertFalse(array_exists(ref('array_entry'), 'invalid_path')->eval($row, flow_context())); + schema(structure_schema('array_entry', type_structure([ + 'id' => type_integer(), + 'status' => type_string(), + 'enabled' => type_boolean(), + 'datetime' => type_datetime(), + 'array' => type_structure(['foo' => type_string()]), + ]))), + )); + static::assertFalse((new FunctionContext(flow_context()))->eval( + array_exists(ref('array_entry'), 'invalid_path'), + [ + 'array_entry' => [ + 'id' => 1, + 'status' => 'PENDING', + 'enabled' => true, + 'datetime' => new DateTimeImmutable('2020-01-01 00:00:00 UTC'), + 'array' => ['foo' => 'bar'], + ], + ], + schema(structure_schema('array_entry', type_structure([ + 'id' => type_integer(), + 'status' => type_string(), + 'enabled' => type_boolean(), + 'datetime' => type_datetime(), + 'array' => type_structure(['foo' => type_string()]), + ]))), + )); } public function test_array_accessor_transformer_with_invalid_but_strict_path(): void { - $this->expectException(InvalidPathException::class); - $this->expectExceptionMessage('Path "invalid_path" does not exists in array '); - - array_get(ref('array_entry'), 'invalid_path')->eval(row([ - 'array_entry' => [ - 'id' => 1, - 'status' => 'PENDING', - 'enabled' => true, - 'datetime' => new DateTimeImmutable('2020-01-01 00:00:00 UTC'), - 'array' => ['foo' => 'bar'], - ], - ]), flow_context()); + try { + (new FunctionContext(flow_context()))->eval( + array_get(ref('array_entry'), 'invalid_path'), + [ + 'array_entry' => [ + 'id' => 1, + 'status' => 'PENDING', + 'enabled' => true, + 'datetime' => new DateTimeImmutable('2020-01-01 00:00:00 UTC'), + 'array' => ['foo' => 'bar'], + ], + ], + schema(structure_schema('array_entry', type_structure([ + 'id' => type_integer(), + 'status' => type_string(), + 'enabled' => type_boolean(), + 'datetime' => type_datetime(), + 'array' => type_structure(['foo' => type_string()]), + ]))), + ); + static::fail('expected an EvaluationException'); + } catch (EvaluationException $e) { + static::assertInstanceOf(InvalidPathException::class, $e->getPrevious()); + static::assertStringContainsString('Path "invalid_path" does not exists in array ', $e->getMessage()); + } } public function test_returns_rejects_a_non_structure_input(): void diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/ArrayKeepTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/ArrayKeepTest.php index 50dce04302..7ce25035d1 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/ArrayKeepTest.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/ArrayKeepTest.php @@ -5,26 +5,46 @@ namespace Flow\ETL\Tests\Unit\Function; use Flow\ETL\Exception\InvalidArgumentException; +use Flow\ETL\Tests\Context\FunctionContext; use Flow\ETL\Tests\FlowTestCase; use function Flow\ETL\DSL\config; use function Flow\ETL\DSL\flow_context; +use function Flow\ETL\DSL\int_schema; +use function Flow\ETL\DSL\list_schema; use function Flow\ETL\DSL\lit; use function Flow\ETL\DSL\ref; -use function Flow\ETL\DSL\row; +use function Flow\ETL\DSL\schema; +use function Flow\ETL\DSL\str_schema; +use function Flow\Types\DSL\type_integer; +use function Flow\Types\DSL\type_list; final class ArrayKeepTest extends FlowTestCase { public function test_array_keep(): void { - static::assertSame([1 => 2], ref('list')->arrayKeep(lit(2))->eval(row(['list' => [1, 2]]), flow_context())); + static::assertSame( + [1 => 2], + (new FunctionContext(flow_context()))->eval( + ref('list')->arrayKeep(lit(2)), + ['list' => [ + 1, + 2, + ]], + schema(list_schema('list', type_list(type_integer()))), + ), + ); } public function test_array_keep_by_entry_reference(): void { static::assertSame( [1 => 2], - ref('list')->arrayKeep(ref('int'))->eval(row(['list' => [1, 2], 'int' => 2]), flow_context()), + (new FunctionContext(flow_context()))->eval( + ref('list')->arrayKeep(ref('int')), + ['list' => [1, 2], 'int' => 2], + schema(list_schema('list', type_list(type_integer())), int_schema('int')), + ), ); } @@ -34,12 +54,26 @@ public function test_array_keep_in_strict_mode(): void $this->expectExceptionMessage('Expected type "array", got "string".'); $context = flow_context(config()); - ref('map')->arrayKeep(lit(1))->eval(row(['map' => 'test']), $context); + (new FunctionContext($context))->eval( + ref('map')->arrayKeep(lit(1)), + ['map' => 'test'], + schema(str_schema('map')), + ); } public function test_array_keep_not_existing_value(): void { - static::assertSame([], ref('list')->arrayKeep(lit(5))->eval(row(['list' => [1, 2]]), flow_context())); + static::assertSame( + [], + (new FunctionContext(flow_context()))->eval( + ref('list')->arrayKeep(lit(5)), + ['list' => [ + 1, + 2, + ]], + schema(list_schema('list', type_list(type_integer()))), + ), + ); } public function test_array_keep_on_non_array(): void @@ -47,6 +81,10 @@ public function test_array_keep_on_non_array(): void $this->expectException(InvalidArgumentException::class); $this->expectExceptionMessage('Expected type "array", got "string".'); - ref('map')->arrayKeep(lit(1))->eval(row(['map' => 'test']), flow_context()); + (new FunctionContext(flow_context()))->eval( + ref('map')->arrayKeep(lit(1)), + ['map' => 'test'], + schema(str_schema('map')), + ); } } diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/ArrayKeyRenameTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/ArrayKeyRenameTest.php index b746fba6e3..2f18854641 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/ArrayKeyRenameTest.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/ArrayKeyRenameTest.php @@ -5,14 +5,21 @@ namespace Flow\ETL\Tests\Unit\Function; use Flow\ArrayDot\Exception\InvalidPathException; +use Flow\ETL\Exception\EvaluationException; use Flow\ETL\Exception\InvalidArgumentException; +use Flow\ETL\Tests\Context\FunctionContext; use Flow\ETL\Tests\FlowTestCase; +use Flow\ETL\Tests\Mother\RowsMother; use function Flow\ETL\DSL\array_key_rename; use function Flow\ETL\DSL\config; use function Flow\ETL\DSL\flow_context; +use function Flow\ETL\DSL\int_schema; use function Flow\ETL\DSL\ref; -use function Flow\ETL\DSL\row; +use function Flow\ETL\DSL\schema; +use function Flow\ETL\DSL\structure_schema; +use function Flow\Types\DSL\type_string; +use function Flow\Types\DSL\type_structure; final class ArrayKeyRenameTest extends FlowTestCase { @@ -22,9 +29,12 @@ public function test_array_key_rename_in_strict_mode(): void $this->expectExceptionMessage('Expected type "array", got "integer".'); $context = flow_context(config()); - $row = row(['integer_entry' => 1]); - array_key_rename(ref('integer_entry'), 'invalid_path', 'new_name')->eval($row, $context); + (new FunctionContext($context))->eval( + array_key_rename(ref('integer_entry'), 'invalid_path', 'new_name'), + ['integer_entry' => 1], + schema(int_schema('integer_entry')), + ); } public function test_for_not_array_entry(): void @@ -32,33 +42,43 @@ public function test_for_not_array_entry(): void $this->expectException(InvalidArgumentException::class); $this->expectExceptionMessage('Expected type "array", got "integer".'); - $row = row(['integer_entry' => 1]); - - array_key_rename(ref('integer_entry'), 'invalid_path', 'new_name')->eval($row, flow_context()); + (new FunctionContext(flow_context()))->eval( + array_key_rename(ref('integer_entry'), 'invalid_path', 'new_name'), + ['integer_entry' => 1], + schema(int_schema('integer_entry')), + ); } public function test_renames_array_entry_keys_in_multiple_array_entry(): void { - $row = row([ - 'customer' => [ - 'first' => 'John', - 'last' => 'Snow', - ], - 'shipping' => [ - 'address' => [ - 'line' => '3644 Clement Street', - 'city' => 'Atalanta', - ], - 'estimated_delivery_date' => '2023-04-01T10:00:00+00:00', - ], - ]); - static::assertEquals( [ 'first_name' => 'John', 'last' => 'Snow', ], - array_key_rename(ref('customer'), 'first', 'first_name')->eval($row, flow_context()), + (new FunctionContext(flow_context()))->eval( + array_key_rename(ref('customer'), 'first', 'first_name'), + [ + 'customer' => [ + 'first' => 'John', + 'last' => 'Snow', + ], + 'shipping' => [ + 'address' => [ + 'line' => '3644 Clement Street', + 'city' => 'Atalanta', + ], + 'estimated_delivery_date' => '2023-04-01T10:00:00+00:00', + ], + ], + schema( + structure_schema('customer', type_structure(['first' => type_string(), 'last' => type_string()])), + structure_schema('shipping', type_structure([ + 'address' => type_structure(['line' => type_string(), 'city' => type_string()]), + 'estimated_delivery_date' => type_string(), + ])), + ), + ), ); static::assertEquals( @@ -69,20 +89,35 @@ public function test_renames_array_entry_keys_in_multiple_array_entry(): void ], 'estimated_delivery_date' => '2023-04-01T10:00:00+00:00', ], - array_key_rename(ref('shipping'), 'address.line', 'street')->eval($row, flow_context()), + (new FunctionContext(flow_context()))->eval( + array_key_rename(ref('shipping'), 'address.line', 'street'), + [ + 'customer' => [ + 'first' => 'John', + 'last' => 'Snow', + ], + 'shipping' => [ + 'address' => [ + 'line' => '3644 Clement Street', + 'city' => 'Atalanta', + ], + 'estimated_delivery_date' => '2023-04-01T10:00:00+00:00', + ], + ], + schema( + structure_schema('customer', type_structure(['first' => type_string(), 'last' => type_string()])), + structure_schema('shipping', type_structure([ + 'address' => type_structure(['line' => type_string(), 'city' => type_string()]), + 'estimated_delivery_date' => type_string(), + ])), + ), + ), ); } public function test_renames_array_entry_keys_in_single_array_entry(): void { - $row = row([ - 'array_entry' => [ - 'id' => 1, - 'status' => 'PENDING', - 'enabled' => true, - 'array' => ['foo' => 'bar'], - ], - ]); + $rows = RowsMother::arrayEntry(); static::assertEquals( [ @@ -91,24 +126,28 @@ public function test_renames_array_entry_keys_in_single_array_entry(): void 'enabled' => true, 'array' => ['new_name' => 'bar'], ], - array_key_rename(ref('array_entry'), 'array.foo', 'new_name')->eval($row, flow_context()), + (new FunctionContext(flow_context()))->eval( + array_key_rename(ref('array_entry'), 'array.foo', 'new_name'), + $rows->values(0), + $rows->schema(), + ), ); } public function test_throws_exception_for_invalid_path(): void { - $row = row([ - 'array_entry' => [ - 'id' => 1, - 'status' => 'PENDING', - 'enabled' => true, - 'array' => ['foo' => 'bar'], - ], - ]); - - $this->expectException(InvalidPathException::class); - $this->expectExceptionMessage('Path "invalid_path" does not exists in array '); - - array_key_rename(ref('array_entry'), 'invalid_path', 'new_name')->eval($row, flow_context()); + $rows = RowsMother::arrayEntry(); + + try { + (new FunctionContext(flow_context()))->eval( + array_key_rename(ref('array_entry'), 'invalid_path', 'new_name'), + $rows->values(0), + $rows->schema(), + ); + static::fail('expected an EvaluationException'); + } catch (EvaluationException $e) { + static::assertInstanceOf(InvalidPathException::class, $e->getPrevious()); + static::assertStringContainsString('Path "invalid_path" does not exists in array ', $e->getMessage()); + } } } diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/ArrayKeysStyleConverterTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/ArrayKeysStyleConverterTest.php index 90061098df..07e34a2c7e 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/ArrayKeysStyleConverterTest.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/ArrayKeysStyleConverterTest.php @@ -8,15 +8,19 @@ use Flow\ETL\Exception\SchemaNotDerivableException; use Flow\ETL\Function\ReferenceResolver; use Flow\ETL\String\StringStyles; +use Flow\ETL\Tests\Context\FunctionContext; use Flow\ETL\Tests\FlowTestCase; use function Flow\ETL\DSL\array_keys_style_convert; use function Flow\ETL\DSL\flow_context; +use function Flow\ETL\DSL\int_schema; +use function Flow\ETL\DSL\json_schema; use function Flow\ETL\DSL\ref; -use function Flow\ETL\DSL\row; use function Flow\ETL\DSL\schema; use function Flow\ETL\DSL\structure_schema; +use function Flow\Types\DSL\type_boolean; use function Flow\Types\DSL\type_integer; +use function Flow\Types\DSL\type_list; use function Flow\Types\DSL\type_string; use function Flow\Types\DSL\type_structure; @@ -27,9 +31,11 @@ public function test_for_invalid_style(): void $this->expectException(InvalidArgumentException::class); $this->expectExceptionMessage('Unrecognized style invalid, please use one of following:'); - $row = row(['invalid_entry' => []]); - - array_keys_style_convert(ref('invalid_entry'), 'invalid')->eval($row, flow_context()); + (new FunctionContext(flow_context()))->eval( + array_keys_style_convert(ref('invalid_entry'), 'invalid'), + ['invalid_entry' => []], + schema(json_schema('invalid_entry')), + ); } public function test_for_not_array_entry(): void @@ -37,34 +43,15 @@ public function test_for_not_array_entry(): void $this->expectException(InvalidArgumentException::class); $this->expectExceptionMessage('Expected type "array", got "integer".'); - $row = row(['invalid_entry' => 1]); - - array_keys_style_convert(ref('invalid_entry'), 'snake')->eval($row, flow_context()); + (new FunctionContext(flow_context()))->eval( + array_keys_style_convert(ref('invalid_entry'), 'snake'), + ['invalid_entry' => 1], + schema(int_schema('invalid_entry')), + ); } public function test_transforms_case_style_for_all_keys_in_array_entry(): void { - $row = row([ - 'arrayEntry' => [ - 'itemId' => 1, - 'itemStatus' => 'PENDING', - 'itemEnabled' => true, - 'itemVariants' => [ - 'variantStatuses' => [ - [ - 'statusId' => 1000, - 'statusName' => 'NEW', - ], - [ - 'statusId' => 2000, - 'statusName' => 'ACTIVE', - ], - ], - 'variantName' => 'Variant Name', - ], - ], - ]); - static::assertEquals( [ 'item_id' => 1, @@ -84,7 +71,41 @@ public function test_transforms_case_style_for_all_keys_in_array_entry(): void 'variant_name' => 'Variant Name', ], ], - array_keys_style_convert(ref('arrayEntry'), 'snake')->eval($row, flow_context()), + (new FunctionContext(flow_context()))->eval( + array_keys_style_convert(ref('arrayEntry'), 'snake'), + [ + 'arrayEntry' => [ + 'itemId' => 1, + 'itemStatus' => 'PENDING', + 'itemEnabled' => true, + 'itemVariants' => [ + 'variantStatuses' => [ + [ + 'statusId' => 1000, + 'statusName' => 'NEW', + ], + [ + 'statusId' => 2000, + 'statusName' => 'ACTIVE', + ], + ], + 'variantName' => 'Variant Name', + ], + ], + ], + schema(structure_schema('arrayEntry', type_structure([ + 'itemId' => type_integer(), + 'itemStatus' => type_string(), + 'itemEnabled' => type_boolean(), + 'itemVariants' => type_structure([ + 'variantStatuses' => type_list(type_structure([ + 'statusId' => type_integer(), + 'statusName' => type_string(), + ])), + 'variantName' => type_string(), + ]), + ]))), + ), ); } diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/ArrayKeysTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/ArrayKeysTest.php index 34732fc2ad..a2e54ebcc2 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/ArrayKeysTest.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/ArrayKeysTest.php @@ -8,18 +8,21 @@ use Flow\ETL\Exception\SchemaNotDerivableException; use Flow\ETL\Function\ReferenceResolver; use Flow\ETL\Function\ScalarFunction; +use Flow\ETL\Tests\Context\FunctionContext; use Flow\ETL\Tests\FlowTestCase; use function Flow\ETL\DSL\config; use function Flow\ETL\DSL\flow_context; use function Flow\ETL\DSL\json_schema; use function Flow\ETL\DSL\list_schema; +use function Flow\ETL\DSL\map_schema; use function Flow\ETL\DSL\ref; -use function Flow\ETL\DSL\row; use function Flow\ETL\DSL\schema; +use function Flow\ETL\DSL\str_schema; use function Flow\ETL\DSL\structure_schema; use function Flow\Types\DSL\type_integer; use function Flow\Types\DSL\type_list; +use function Flow\Types\DSL\type_map; use function Flow\Types\DSL\type_string; use function Flow\Types\DSL\type_structure; @@ -62,7 +65,14 @@ public function test_array_keys(): void { static::assertSame( ['a', 'b'], - ref('map')->arrayKeys()->eval(row(['map' => ['a' => 1, 'b' => 2]]), flow_context()), + (new FunctionContext(flow_context()))->eval( + ref('map')->arrayKeys(), + ['map' => [ + 'a' => 1, + 'b' => 2, + ]], + schema(map_schema('map', type_map(type_string(), type_integer()))), + ), ); } @@ -72,7 +82,7 @@ public function test_array_keys_in_strict_mode(): void $this->expectExceptionMessage('Expected type "array", got "string".'); $context = flow_context(config()); - ref('map')->arrayKeys()->eval(row(['map' => 'test']), $context); + (new FunctionContext($context))->eval(ref('map')->arrayKeys(), ['map' => 'test'], schema(str_schema('map'))); } public function test_array_keys_on_non_array(): void @@ -80,6 +90,10 @@ public function test_array_keys_on_non_array(): void $this->expectException(InvalidArgumentException::class); $this->expectExceptionMessage('Expected type "array", got "string".'); - ref('map')->arrayKeys()->eval(row(['map' => 'test']), flow_context()); + (new FunctionContext(flow_context()))->eval( + ref('map')->arrayKeys(), + ['map' => 'test'], + schema(str_schema('map')), + ); } } diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/ArrayMergeCollectionTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/ArrayMergeCollectionTest.php index 446937d0cc..cb7154f010 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/ArrayMergeCollectionTest.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/ArrayMergeCollectionTest.php @@ -6,14 +6,16 @@ use Flow\ETL\Exception\InvalidArgumentException; use Flow\ETL\Function\ReferenceResolver; +use Flow\ETL\Tests\Context\FunctionContext; use Flow\ETL\Tests\FlowTestCase; use function Flow\ETL\DSL\array_merge_collection; use function Flow\ETL\DSL\config; use function Flow\ETL\DSL\flow_context; +use function Flow\ETL\DSL\int_schema; +use function Flow\ETL\DSL\json_schema; use function Flow\ETL\DSL\list_schema; use function Flow\ETL\DSL\ref; -use function Flow\ETL\DSL\row; use function Flow\ETL\DSL\schema; use function Flow\Types\DSL\structure_element; use function Flow\Types\DSL\type_integer; @@ -29,9 +31,12 @@ public function test_array_merge_collection_in_strict_mode_with_non_array(): voi $this->expectExceptionMessage('Expected type "array", got "integer".'); $context = flow_context(config()); - $row = row(['invalid_entry' => 1]); - array_merge_collection(ref('invalid_entry'))->eval($row, $context); + (new FunctionContext($context))->eval( + array_merge_collection(ref('invalid_entry')), + ['invalid_entry' => 1], + schema(int_schema('invalid_entry')), + ); } public function test_array_merge_collection_in_strict_mode_with_non_array_elements(): void @@ -40,14 +45,17 @@ public function test_array_merge_collection_in_strict_mode_with_non_array_elemen $this->expectExceptionMessage('ArrayMergeCollection function requires array elements to be arrays'); $context = flow_context(config()); - $row = row([ - 'array_entry' => [ - ['foo' => 'bar'], - 1, - ], - ]); - array_merge_collection(ref('array_entry'))->eval($row, $context); + (new FunctionContext($context))->eval( + array_merge_collection(ref('array_entry')), + [ + 'array_entry' => [ + ['foo' => 'bar'], + 1, + ], + ], + schema(json_schema('array_entry')), + ); } public function test_attempt_of_merging_collection_where_not_every_element_is_array(): void @@ -55,14 +63,16 @@ public function test_attempt_of_merging_collection_where_not_every_element_is_ar $this->expectException(InvalidArgumentException::class); $this->expectExceptionMessage('ArrayMergeCollection function requires array elements to be arrays'); - $row = row([ - 'array_entry' => [ - ['foo' => 'bar'], - 1, + (new FunctionContext(flow_context()))->eval( + array_merge_collection(ref('array_entry')), + [ + 'array_entry' => [ + ['foo' => 'bar'], + 1, + ], ], - ]); - - array_merge_collection(ref('array_entry'))->eval($row, flow_context()); + schema(json_schema('array_entry')), + ); } public function test_for_not_array_entry(): void @@ -70,26 +80,33 @@ public function test_for_not_array_entry(): void $this->expectException(InvalidArgumentException::class); $this->expectExceptionMessage('Expected type "array", got "integer".'); - $row = row(['invalid_entry' => 1]); - - array_merge_collection(ref('invalid_entry'))->eval($row, flow_context()); + (new FunctionContext(flow_context()))->eval( + array_merge_collection(ref('invalid_entry')), + ['invalid_entry' => 1], + schema(int_schema('invalid_entry')), + ); } public function test_merging_collection_of_arrays(): void { - $row = row([ - 'array_entry' => [ + static::assertEquals( + [1, 2], + (new FunctionContext(flow_context()))->eval( + array_merge_collection(ref('array_entry')), [ - 1, + 'array_entry' => [ + [ + 1, + ], + [ + 2, + ], + [], + ], ], - [ - 2, - ], - [], - ], - ]); - - static::assertEquals([1, 2], array_merge_collection(ref('array_entry'))->eval($row, flow_context())); + schema(list_schema('array_entry', type_list(type_list(type_integer())))), + ), + ); } public function test_a_list_of_structures_declares_the_structure(): void diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/ArrayMergeTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/ArrayMergeTest.php index ea8c656b17..ef762a0f60 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/ArrayMergeTest.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/ArrayMergeTest.php @@ -6,13 +6,19 @@ use Flow\ETL\Exception\InvalidArgumentException; use Flow\ETL\Function\ArrayMerge; +use Flow\ETL\Tests\Context\FunctionContext; use Flow\ETL\Tests\FlowTestCase; use function Flow\ETL\DSL\config; use function Flow\ETL\DSL\flow_context; +use function Flow\ETL\DSL\int_schema; use function Flow\ETL\DSL\lit; +use function Flow\ETL\DSL\map_schema; use function Flow\ETL\DSL\ref; -use function Flow\ETL\DSL\row; +use function Flow\ETL\DSL\schema; +use function Flow\Types\DSL\type_integer; +use function Flow\Types\DSL\type_map; +use function Flow\Types\DSL\type_string; final class ArrayMergeTest extends FlowTestCase { @@ -22,14 +28,25 @@ public function test_array_merge_in_strict_mode(): void $this->expectExceptionMessage('Expected type "array", got "integer".'); $context = flow_context(config()); - ref('a')->arrayMerge(ref('b'))->eval(row(['a' => 1, 'b' => ['b' => 2]]), $context); + (new FunctionContext($context))->eval( + ref('a')->arrayMerge(ref('b')), + ['a' => 1, 'b' => ['b' => 2]], + schema(int_schema('a'), map_schema('b', type_map(type_string(), type_integer()))), + ); } public function test_array_merge_two_array_row_entries(): void { static::assertSame( ['a' => 1, 'b' => 2], - ref('a')->arrayMerge(ref('b'))->eval(row(['a' => ['a' => 1], 'b' => ['b' => 2]]), flow_context()), + (new FunctionContext(flow_context()))->eval( + ref('a')->arrayMerge(ref('b')), + ['a' => ['a' => 1], 'b' => ['b' => 2]], + schema( + map_schema('a', type_map(type_string(), type_integer())), + map_schema('b', type_map(type_string(), type_integer())), + ), + ), ); } @@ -37,7 +54,7 @@ public function test_array_merge_two_lit_functions(): void { $function = new ArrayMerge(lit(['a' => 1]), lit(['b' => 2])); - static::assertSame(['a' => 1, 'b' => 2], $function->eval(row([]), flow_context())); + static::assertSame(['a' => 1, 'b' => 2], (new FunctionContext(flow_context()))->eval($function, [], schema())); } public function test_array_merge_when_left_side_is_not_an_array(): void @@ -45,7 +62,11 @@ public function test_array_merge_when_left_side_is_not_an_array(): void $this->expectException(InvalidArgumentException::class); $this->expectExceptionMessage('Expected type "array", got "integer".'); - ref('a')->arrayMerge(ref('b'))->eval(row(['a' => 1, 'b' => ['b' => 2]]), flow_context()); + (new FunctionContext(flow_context()))->eval( + ref('a')->arrayMerge(ref('b')), + ['a' => 1, 'b' => ['b' => 2]], + schema(int_schema('a'), map_schema('b', type_map(type_string(), type_integer()))), + ); } public function test_array_merge_when_right_side_is_not_an_array(): void @@ -53,6 +74,10 @@ public function test_array_merge_when_right_side_is_not_an_array(): void $this->expectException(InvalidArgumentException::class); $this->expectExceptionMessage('Expected type "array", got "integer".'); - ref('a')->arrayMerge(ref('b'))->eval(row(['a' => ['a' => 1], 'b' => 2]), flow_context()); + (new FunctionContext(flow_context()))->eval( + ref('a')->arrayMerge(ref('b')), + ['a' => ['a' => 1], 'b' => 2], + schema(map_schema('a', type_map(type_string(), type_integer())), int_schema('b')), + ); } } diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/ArrayReverseTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/ArrayReverseTest.php index 58bb9cd3a4..303e1ba232 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/ArrayReverseTest.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/ArrayReverseTest.php @@ -6,16 +6,19 @@ use Flow\ETL\Exception\InvalidArgumentException; use Flow\ETL\Function\ReferenceResolver; +use Flow\ETL\Tests\Context\FunctionContext; use Flow\ETL\Tests\FlowTestCase; use function Flow\ETL\DSL\config; use function Flow\ETL\DSL\flow_context; +use function Flow\ETL\DSL\int_schema; +use function Flow\ETL\DSL\list_schema; use function Flow\ETL\DSL\ref; -use function Flow\ETL\DSL\row; use function Flow\ETL\DSL\schema; use function Flow\ETL\DSL\structure_schema; use function Flow\Types\DSL\structure_element; use function Flow\Types\DSL\type_integer; +use function Flow\Types\DSL\type_list; use function Flow\Types\DSL\type_string; use function Flow\Types\DSL\type_structure; @@ -23,7 +26,19 @@ final class ArrayReverseTest extends FlowTestCase { public function test_array_reverse_array_entry(): void { - static::assertSame([5, 3, 10, 4], ref('a')->arrayReverse()->eval(row(['a' => [4, 10, 3, 5]]), flow_context())); + static::assertSame( + [5, 3, 10, 4], + (new FunctionContext(flow_context()))->eval( + ref('a')->arrayReverse(), + ['a' => [ + 4, + 10, + 3, + 5, + ]], + schema(list_schema('a', type_list(type_integer()))), + ), + ); } public function test_array_reverse_in_strict_mode(): void @@ -32,7 +47,7 @@ public function test_array_reverse_in_strict_mode(): void $this->expectExceptionMessage('Expected type "array", got "integer".'); $context = flow_context(config()); - ref('a')->arrayReverse()->eval(row(['a' => 123]), $context); + (new FunctionContext($context))->eval(ref('a')->arrayReverse(), ['a' => 123], schema(int_schema('a'))); } public function test_array_reverse_non_array_entry(): void @@ -40,7 +55,7 @@ public function test_array_reverse_non_array_entry(): void $this->expectException(InvalidArgumentException::class); $this->expectExceptionMessage('Expected type "array", got "integer".'); - ref('a')->arrayReverse()->eval(row(['a' => 123]), flow_context()); + (new FunctionContext(flow_context()))->eval(ref('a')->arrayReverse(), ['a' => 123], schema(int_schema('a'))); } public function test_a_structure_operand_declares_reversed_fields(): void diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/ArraySortTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/ArraySortTest.php index 624c2c2d47..94fbea451f 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/ArraySortTest.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/ArraySortTest.php @@ -7,16 +7,22 @@ use Flow\ETL\Exception\InvalidArgumentException; use Flow\ETL\Function\ArraySort\Sort; use Flow\ETL\Function\ReferenceResolver; +use Flow\ETL\Tests\Context\FunctionContext; use Flow\ETL\Tests\FlowTestCase; +use Flow\Types\Value\Json; +use Generator; +use PHPUnit\Framework\Attributes\DataProvider; use function Flow\ETL\DSL\config; use function Flow\ETL\DSL\flow_context; +use function Flow\ETL\DSL\json_schema; use function Flow\ETL\DSL\ref; -use function Flow\ETL\DSL\row; use function Flow\ETL\DSL\schema; +use function Flow\ETL\DSL\str_schema; use function Flow\ETL\DSL\structure_schema; use function Flow\Types\DSL\structure_element; use function Flow\Types\DSL\type_array; +use function Flow\Types\DSL\type_instance_of; use function Flow\Types\DSL\type_integer; use function Flow\Types\DSL\type_structure; use function json_decode; @@ -29,33 +35,138 @@ public function test_array_sort_in_strict_mode(): void $this->expectExceptionMessage('Expected type "array", got "string".'); $context = flow_context(config()); - ref('array')->arraySort()->eval(row(['array' => 'string']), $context); + (new FunctionContext($context))->eval( + ref('array')->arraySort(), + ['array' => 'string'], + schema(str_schema('array')), + ); } - public function test_sorting_big_arrays(): void + public static function one_document_in_two_orders(): Generator + { + yield 'as stored' => [<<<'JSON' + [ + { + "asin": "B00PHQB8EE", + "images": [ + { + "images": [ + { + "link": "https://m.media-amazon.com/images/I/419NipwmTaL.jpg", + "width": 190, + "height": 500, + "variant": "MAIN" + }, + { + "link": "https://m.media-amazon.com/images/I/419NipwmTaL._SL75_.jpg", + "width": 29, + "height": 75, + "variant": "MAIN" + }, + { + "link": "https://m.media-amazon.com/images/I/61AQupUe5pL.jpg", + "width": 418, + "height": 1100, + "variant": "MAIN" + }, + { + "link": "https://m.media-amazon.com/images/I/51XvsWDOBuS.jpg", + "width": 500, + "height": 279, + "variant": "PT01" + }, + { + "link": "https://m.media-amazon.com/images/I/51XvsWDOBuS._SL75_.jpg", + "width": 75, + "height": 42, + "variant": "PT01" + } + ], + "marketplaceId": "ATVPDKIKX0DER" + } + ], + "synchronized_at": "2023-04-21 11:33:25" + } + ] + JSON]; + yield 'reordered' => [<<<'JSON' + [ + { + "asin": "B00PHQB8EE", + "images": [ + { + "images": [ + { + "link": "https://m.media-amazon.com/images/I/51XvsWDOBuS._SL75_.jpg", + "width": 75, + "height": 42, + "variant": "PT01" + }, + { + "link": "https://m.media-amazon.com/images/I/419NipwmTaL.jpg", + "width": 190, + "height": 500, + "variant": "MAIN" + }, + { + "link": "https://m.media-amazon.com/images/I/61AQupUe5pL.jpg", + "width": 418, + "height": 1100, + "variant": "MAIN" + }, + { + "link": "https://m.media-amazon.com/images/I/419NipwmTaL._SL75_.jpg", + "width": 29, + "height": 75, + "variant": "MAIN" + }, + { + "link": "https://m.media-amazon.com/images/I/51XvsWDOBuS.jpg", + "width": 500, + "height": 279, + "variant": "PT01" + } + ], + "marketplaceId": "ATVPDKIKX0DER" + } + ], + "synchronized_at": "2023-04-21 11:33:25" + } + ] + JSON]; + } + + #[DataProvider('one_document_in_two_orders')] + public function test_sorting_big_arrays(string $json): void { static::assertSame( - ref('array') - ->arraySort() - ->eval(row(['array' => type_array()->assert(json_decode( - $this->jsonDifferentOrder(), - true, - 512, - JSON_THROW_ON_ERROR, - ))]), flow_context()), - ref('array') - ->arraySort() - ->eval(row(['array' => type_array()->assert(json_decode( - $this->json(), - true, - 512, - JSON_THROW_ON_ERROR, - ))]), flow_context()), + [[ + '2023-04-21 11:33:25', + 'B00PHQB8EE', + [[ + 'ATVPDKIKX0DER', + [ + [29, 75, 'MAIN', 'https://m.media-amazon.com/images/I/419NipwmTaL._SL75_.jpg'], + [42, 75, 'PT01', 'https://m.media-amazon.com/images/I/51XvsWDOBuS._SL75_.jpg'], + [190, 500, 'MAIN', 'https://m.media-amazon.com/images/I/419NipwmTaL.jpg'], + [279, 500, 'PT01', 'https://m.media-amazon.com/images/I/51XvsWDOBuS.jpg'], + [418, 1100, 'MAIN', 'https://m.media-amazon.com/images/I/61AQupUe5pL.jpg'], + ], + ]], + ]], + type_instance_of(Json::class) + ->assert((new FunctionContext(flow_context()))->eval( + ref('array')->arraySort(), + ['array' => type_array()->assert(json_decode($json, true, 512, JSON_THROW_ON_ERROR))], + schema(json_schema('array')), + )) + ->toArray(), ); } public function test_sorting_nested_array_using_asort_algo(): void { + // a json operand gives a json column: the sorted array is read back from it static::assertSame( [ 'a' => [ @@ -66,24 +177,29 @@ public function test_sorting_nested_array_using_asort_algo(): void ], ], ], - ref('array') - ->arraySort(Sort::asort) - ->eval(row([ - 'array' => [ - 'a' => [ - 'b' => [ - 'e' => 'f', - 'c' => 'd', + type_instance_of(Json::class) + ->assert((new FunctionContext(flow_context()))->eval( + ref('array')->arraySort(Sort::asort), + [ + 'array' => [ + 'a' => [ + 'b' => [ + 'e' => 'f', + 'c' => 'd', + ], + 'g' => 'h', ], - 'g' => 'h', ], ], - ]), flow_context()), + schema(json_schema('array')), + )) + ->toArray(), ); } public function test_sorting_nested_associative_array(): void { + // a json operand gives a json column: the sorted array is read back from it static::assertSame( [ 'a' => [ @@ -94,19 +210,23 @@ public function test_sorting_nested_associative_array(): void 'g' => 'h', ], ], - ref('array') - ->arraySort(Sort::ksort) - ->eval(row([ - 'array' => [ - 'a' => [ - 'g' => 'h', - 'b' => [ - 'e' => 'f', - 'c' => 'd', + type_instance_of(Json::class) + ->assert((new FunctionContext(flow_context()))->eval( + ref('array')->arraySort(Sort::ksort), + [ + 'array' => [ + 'a' => [ + 'g' => 'h', + 'b' => [ + 'e' => 'f', + 'c' => 'd', + ], ], ], ], - ]), flow_context()), + schema(json_schema('array')), + )) + ->toArray(), ); } @@ -115,105 +235,11 @@ public function test_sorting_non_array_value(): void $this->expectException(InvalidArgumentException::class); $this->expectExceptionMessage('Expected type "array", got "string".'); - ref('array')->arraySort()->eval(row(['array' => 'string']), flow_context()); - } - - private function json(): string - { - return <<<'JSON' - [ - { - "asin": "B00PHQB8EE", - "images": [ - { - "images": [ - { - "link": "https://m.media-amazon.com/images/I/419NipwmTaL.jpg", - "width": 190, - "height": 500, - "variant": "MAIN" - }, - { - "link": "https://m.media-amazon.com/images/I/419NipwmTaL._SL75_.jpg", - "width": 29, - "height": 75, - "variant": "MAIN" - }, - { - "link": "https://m.media-amazon.com/images/I/61AQupUe5pL.jpg", - "width": 418, - "height": 1100, - "variant": "MAIN" - }, - { - "link": "https://m.media-amazon.com/images/I/51XvsWDOBuS.jpg", - "width": 500, - "height": 279, - "variant": "PT01" - }, - { - "link": "https://m.media-amazon.com/images/I/51XvsWDOBuS._SL75_.jpg", - "width": 75, - "height": 42, - "variant": "PT01" - } - ], - "marketplaceId": "ATVPDKIKX0DER" - } - ], - "synchronized_at": "2023-04-21 11:33:25" - } - ] - JSON; - } - - private function jsonDifferentOrder(): string - { - return <<<'JSON' - [ - { - "asin": "B00PHQB8EE", - "images": [ - { - "images": [ - { - "link": "https://m.media-amazon.com/images/I/51XvsWDOBuS._SL75_.jpg", - "width": 75, - "height": 42, - "variant": "PT01" - }, - { - "link": "https://m.media-amazon.com/images/I/419NipwmTaL.jpg", - "width": 190, - "height": 500, - "variant": "MAIN" - }, - { - "link": "https://m.media-amazon.com/images/I/61AQupUe5pL.jpg", - "width": 418, - "height": 1100, - "variant": "MAIN" - }, - { - "link": "https://m.media-amazon.com/images/I/419NipwmTaL._SL75_.jpg", - "width": 29, - "height": 75, - "variant": "MAIN" - }, - { - "link": "https://m.media-amazon.com/images/I/51XvsWDOBuS.jpg", - "width": 500, - "height": 279, - "variant": "PT01" - } - ], - "marketplaceId": "ATVPDKIKX0DER" - } - ], - "synchronized_at": "2023-04-21 11:33:25" - } - ] - JSON; + (new FunctionContext(flow_context()))->eval( + ref('array')->arraySort(), + ['array' => 'string'], + schema(str_schema('array')), + ); } public function test_ksort_declares_key_sorted_structure_fields(): void diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/ArrayUnpackTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/ArrayUnpackTest.php index 5d45c9d16b..12fa101270 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/ArrayUnpackTest.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/ArrayUnpackTest.php @@ -6,16 +6,18 @@ use Flow\ETL\Exception\InvalidArgumentException; use Flow\ETL\Function\ArrayUnpack; +use Flow\ETL\Tests\Context\FunctionContext; use Flow\ETL\Tests\FlowTestCase; use function Flow\ETL\DSL\bool_schema; use function Flow\ETL\DSL\config; use function Flow\ETL\DSL\flow_context; +use function Flow\ETL\DSL\int_schema; use function Flow\ETL\DSL\json_schema; use function Flow\ETL\DSL\ref; -use function Flow\ETL\DSL\row; use function Flow\ETL\DSL\schema; use function Flow\ETL\DSL\str_schema; +use function Flow\ETL\DSL\structure_schema; use function Flow\Types\DSL\structure_element; use function Flow\Types\DSL\type_boolean; use function Flow\Types\DSL\type_json; @@ -27,20 +29,31 @@ final class ArrayUnpackTest extends FlowTestCase { public function test_array_unpack(): void { + // the StructureType column returns() declares: the undeclared "array" field is not unpacked static::assertSame( [ 'status' => 'PENDING', 'enabled' => true, - 'array' => ['foo' => 'bar'], ], - (new ArrayUnpack(ref('array_entry'), schema(str_schema('status'), bool_schema('enabled'))))->eval(row([ - 'id' => 1, - 'array_entry' => [ - 'status' => 'PENDING', - 'enabled' => true, - 'array' => ['foo' => 'bar'], + (new FunctionContext(flow_context()))->eval( + new ArrayUnpack(ref('array_entry'), schema(str_schema('status'), bool_schema('enabled'))), + [ + 'id' => 1, + 'array_entry' => [ + 'status' => 'PENDING', + 'enabled' => true, + 'array' => ['foo' => 'bar'], + ], ], - ]), flow_context()), + schema( + int_schema('id'), + structure_schema('array_entry', type_structure([ + 'status' => type_string(), + 'enabled' => type_boolean(), + 'array' => type_structure(['foo' => type_string()]), + ])), + ), + ), ); } @@ -49,10 +62,14 @@ public function test_array_unpack_with_null_in_strict_mode(): void $this->expectException(InvalidArgumentException::class); $this->expectExceptionMessage('array_unpack() requires a non-null array'); - (new ArrayUnpack(ref('array_entry'), schema(str_schema('status'))))->eval(row([ - 'id' => 1, - 'array_entry' => null, - ]), flow_context(config())); + (new FunctionContext(flow_context(config())))->eval( + new ArrayUnpack(ref('array_entry'), schema(str_schema('status'))), + [ + 'id' => 1, + 'array_entry' => null, + ], + schema(int_schema('id'), str_schema('array_entry', nullable: true)), + ); } public function test_it_refuses_an_empty_schema(): void diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/ArrayValuesTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/ArrayValuesTest.php index 14fc364ea1..3ef7bbb8cb 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/ArrayValuesTest.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/ArrayValuesTest.php @@ -7,17 +7,21 @@ use Flow\ETL\Exception\InvalidArgumentException; use Flow\ETL\Exception\SchemaNotDerivableException; use Flow\ETL\Function\ReferenceResolver; +use Flow\ETL\Tests\Context\FunctionContext; use Flow\ETL\Tests\FlowTestCase; use function Flow\ETL\DSL\config; use function Flow\ETL\DSL\flow_context; +use function Flow\ETL\DSL\map_schema; use function Flow\ETL\DSL\ref; -use function Flow\ETL\DSL\row; use function Flow\ETL\DSL\schema; +use function Flow\ETL\DSL\str_schema; use function Flow\ETL\DSL\structure_schema; use function Flow\Types\DSL\type_boolean; use function Flow\Types\DSL\type_integer; use function Flow\Types\DSL\type_list; +use function Flow\Types\DSL\type_map; +use function Flow\Types\DSL\type_string; use function Flow\Types\DSL\type_structure; final class ArrayValuesTest extends FlowTestCase @@ -26,7 +30,14 @@ public function test_array_values(): void { static::assertSame( [1, 2], - ref('map')->arrayValues()->eval(row(['map' => ['a' => 1, 'b' => 2]]), flow_context()), + (new FunctionContext(flow_context()))->eval( + ref('map')->arrayValues(), + ['map' => [ + 'a' => 1, + 'b' => 2, + ]], + schema(map_schema('map', type_map(type_string(), type_integer()))), + ), ); } @@ -36,7 +47,7 @@ public function test_array_values_in_strict_mode(): void $this->expectExceptionMessage('Expected type "array", got "string".'); $context = flow_context(config()); - ref('map')->arrayValues()->eval(row(['map' => 'test']), $context); + (new FunctionContext($context))->eval(ref('map')->arrayValues(), ['map' => 'test'], schema(str_schema('map'))); } public function test_array_values_on_non_array(): void @@ -44,7 +55,11 @@ public function test_array_values_on_non_array(): void $this->expectException(InvalidArgumentException::class); $this->expectExceptionMessage('Expected type "array", got "string".'); - ref('map')->arrayValues()->eval(row(['map' => 'test']), flow_context()); + (new FunctionContext(flow_context()))->eval( + ref('map')->arrayValues(), + ['map' => 'test'], + schema(str_schema('map')), + ); } public function test_a_structure_operand_declares_the_unified_field_type(): void diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/AsciiTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/AsciiTest.php index 47dc3345e7..0278246f14 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/AsciiTest.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/AsciiTest.php @@ -5,17 +5,23 @@ namespace Flow\ETL\Tests\Unit\Function; use Flow\ETL\Exception\InvalidArgumentException; +use Flow\ETL\Tests\Context\FunctionContext; use Flow\ETL\Tests\FlowTestCase; use function Flow\ETL\DSL\flow_context; use function Flow\ETL\DSL\ref; -use function Flow\ETL\DSL\row; +use function Flow\ETL\DSL\schema; +use function Flow\ETL\DSL\str_schema; final class AsciiTest extends FlowTestCase { public function test_ascii(): void { - static::assertSame('azcz', ref('str')->ascii()->eval(row(['str' => 'ąźćż']), flow_context())); + static::assertSame('azcz', (new FunctionContext(flow_context()))->eval( + ref('str')->ascii(), + ['str' => 'ąźćż'], + schema(str_schema('str')), + )); } public function test_ascii_on_null(): void @@ -23,6 +29,10 @@ public function test_ascii_on_null(): void $this->expectException(InvalidArgumentException::class); $this->expectExceptionMessage('Ascii function requires non-null value'); - ref('str')->ascii()->eval(row(['str' => null]), flow_context()); + (new FunctionContext(flow_context()))->eval( + ref('str')->ascii(), + ['str' => null], + schema(str_schema('str', nullable: true)), + ); } } diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/AverageTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/AverageTest.php index 31f1d300c4..ca5f2f02ae 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/AverageTest.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/AverageTest.php @@ -7,16 +7,22 @@ use Flow\ETL\Exception\InvalidArgumentException; use Flow\ETL\Tests\FlowTestCase; use Flow\ETL\Tests\Mother\WindowContextMother; +use PHPUnit\Framework\Attributes\TestWith; +use function array_fill; +use function array_map; +use function Flow\ETL\DSL\array_to_rows; use function Flow\ETL\DSL\average; use function Flow\ETL\DSL\config; +use function Flow\ETL\DSL\float_schema; use function Flow\ETL\DSL\flow_context; use function Flow\ETL\DSL\int_schema; use function Flow\ETL\DSL\ref; -use function Flow\ETL\DSL\row; -use function Flow\ETL\DSL\rows; use function Flow\ETL\DSL\schema; +use function Flow\ETL\DSL\str_schema; +use function Flow\ETL\DSL\sum; use function Flow\ETL\DSL\window; +use function range; final class AverageTest extends FlowTestCase { @@ -29,11 +35,17 @@ public function test_aggregation_average_from_numeric_values(): void { $aggregator = average(ref('int')); - $aggregator->aggregate(row(['int' => '10']), flow_context()); - $aggregator->aggregate(row(['int' => '20']), flow_context()); - $aggregator->aggregate(row(['int' => '30']), flow_context()); - $aggregator->aggregate(row(['int' => '25']), flow_context()); - $aggregator->aggregate(row(['not_int' => null]), flow_context()); + $aggregator->aggregate(array_to_rows([['int' => '10']], schema(str_schema('int'))), [0], flow_context()); + $aggregator->aggregate(array_to_rows([['int' => '20']], schema(str_schema('int'))), [0], flow_context()); + $aggregator->aggregate(array_to_rows([['int' => '30']], schema(str_schema('int'))), [0], flow_context()); + $aggregator->aggregate(array_to_rows([['int' => '25']], schema(str_schema('int'))), [0], flow_context()); + $aggregator->aggregate( + array_to_rows([[ + 'not_int' => null, + ]], schema(str_schema('not_int', nullable: true))), + [0], + flow_context(), + ); static::assertSame(21.25, $aggregator->value()); } @@ -42,10 +54,16 @@ public function test_aggregation_average_including_null_value(): void { $aggregator = average(ref('int')); - $aggregator->aggregate(row(['int' => 10]), flow_context()); - $aggregator->aggregate(row(['int' => 20]), flow_context()); - $aggregator->aggregate(row(['int' => 30]), flow_context()); - $aggregator->aggregate(row(['int' => null]), flow_context()); + $aggregator->aggregate(array_to_rows([['int' => 10]], schema(int_schema('int'))), [0], flow_context()); + $aggregator->aggregate(array_to_rows([['int' => 20]], schema(int_schema('int'))), [0], flow_context()); + $aggregator->aggregate(array_to_rows([['int' => 30]], schema(int_schema('int'))), [0], flow_context()); + $aggregator->aggregate( + array_to_rows([[ + 'int' => null, + ]], schema(int_schema('int', nullable: true))), + [0], + flow_context(), + ); static::assertSame(20.0, $aggregator->value()); } @@ -54,10 +72,10 @@ public function test_aggregation_average_with_float_result(): void { $aggregator = average(ref('int')); - $aggregator->aggregate(row(['int' => 10]), flow_context()); - $aggregator->aggregate(row(['int' => 20]), flow_context()); - $aggregator->aggregate(row(['int' => 30]), flow_context()); - $aggregator->aggregate(row(['int' => 25]), flow_context()); + $aggregator->aggregate(array_to_rows([['int' => 10]], schema(int_schema('int'))), [0], flow_context()); + $aggregator->aggregate(array_to_rows([['int' => 20]], schema(int_schema('int'))), [0], flow_context()); + $aggregator->aggregate(array_to_rows([['int' => 30]], schema(int_schema('int'))), [0], flow_context()); + $aggregator->aggregate(array_to_rows([['int' => 25]], schema(int_schema('int'))), [0], flow_context()); static::assertSame(21.25, $aggregator->value()); } @@ -66,10 +84,10 @@ public function test_aggregation_average_with_integer_result(): void { $aggregator = average(ref('int')); - $aggregator->aggregate(row(['int' => 10]), flow_context()); - $aggregator->aggregate(row(['int' => 20]), flow_context()); - $aggregator->aggregate(row(['int' => 30]), flow_context()); - $aggregator->aggregate(row(['int' => 40]), flow_context()); + $aggregator->aggregate(array_to_rows([['int' => 10]], schema(int_schema('int'))), [0], flow_context()); + $aggregator->aggregate(array_to_rows([['int' => 20]], schema(int_schema('int'))), [0], flow_context()); + $aggregator->aggregate(array_to_rows([['int' => 30]], schema(int_schema('int'))), [0], flow_context()); + $aggregator->aggregate(array_to_rows([['int' => 40]], schema(int_schema('int'))), [0], flow_context()); static::assertSame(25.0, $aggregator->value()); } @@ -83,18 +101,20 @@ public function test_aggregation_average_of_nothing_is_null(): void public function test_window_function_average_on_partitioned_rows(): void { - $rows = rows( + $rows = array_to_rows( + [ + ['id' => 1, 'value' => 1], + ['id' => 2, 'value' => 100], + ['id' => 3, 'value' => 25], + ['id' => 4, 'value' => 64], + ['id' => 5, 'value' => 23], + ], schema(int_schema('id'), int_schema('value')), - $row1 = row(['id' => 1, 'value' => 1]), - row(['id' => 2, 'value' => 100]), - row(['id' => 3, 'value' => 25]), - row(['id' => 4, 'value' => 64]), - row(['id' => 5, 'value' => 23]), ); $avg = average(ref('value'))->over(window()->orderBy(ref('value'))); - static::assertSame(42.6, $avg->apply(WindowContextMother::forRow($row1, $rows))); + static::assertSame(42.6, $avg->apply(WindowContextMother::atIndex($rows, 0))); } public function test_window_function_average_with_missing_reference_in_strict_mode(): void @@ -102,12 +122,12 @@ public function test_window_function_average_with_missing_reference_in_strict_mo $this->expectException(InvalidArgumentException::class); $this->expectExceptionMessage('Average window function error:'); - $rows = rows(schema(int_schema('id')), $row1 = row(['id' => 1]), row(['id' => 2])); + $rows = array_to_rows([['id' => 1], ['id' => 2]], schema(int_schema('id'))); $avg = average(ref('missing_column'))->over(window()->orderBy(ref('id'))); $context = flow_context(config()); - $avg->apply(WindowContextMother::forRow($row1, $rows, context: $context)); + $avg->apply(WindowContextMother::atIndex($rows, 0, context: $context)); } public function test_with_children_rebuilds_the_aggregate_with_the_given_reference(): void @@ -121,4 +141,74 @@ public function test_with_children_rebuilds_the_aggregate_with_the_given_referen static::assertNotSame($aggregate, $rebuilt); static::assertEquals([ref('b')], $rebuilt->references()); } + + public function test_aggregate_reads_only_given_indices(): void + { + $aggregator = average(ref('v')); + $aggregator->aggregate( + array_to_rows(array_map(static fn(mixed $v): array => ['v' => $v], [ + 0.5, + 1.25, + 2.0, + 3.0, + ]), schema(float_schema('v'))), + [0, 2], + flow_context(), + ); + + static::assertSame(1.25, $aggregator->value()); + } + + public function test_merge_of_two_halves_equals_one_pass(): void + { + $rows = array_to_rows(array_map(static fn(mixed $v): array => ['v' => $v], [ + 0.5, + 1.25, + 2.0, + 3.0, + ]), schema(float_schema('v'))); + $onePass = average(ref('v')); + $onePass->aggregate($rows, [0, 1, 2, 3], flow_context()); + $left = average(ref('v')); + $left->aggregate($rows, [0, 1], flow_context()); + $right = average(ref('v')); + $right->aggregate($rows, [2, 3], flow_context()); + + $left->merge($right, flow_context()); + + static::assertSame($onePass->value(), $left->value()); + } + + public function test_merge_refuses_another_class(): void + { + $this->expectException(InvalidArgumentException::class); + + average(ref('v'))->merge(sum(ref('v')), flow_context()); + } + + #[TestWith([false])] + #[TestWith([true])] + public function test_ten_tenths_average_to_a_tenth(bool $exact): void + { + $aggregator = average(ref('v'), 17, exact: $exact); + $aggregator->aggregate(array_to_rows(array_fill(0, 10, [ + 'v' => 0.1, + ]), schema(float_schema('v'))), range(0, 9), flow_context()); + + // naive IEEE summation would give 0.09999999999999999 + static::assertSame(0.1, $aggregator->value()); + } + + public function test_merged_partial_averages_keep_their_compensation(): void + { + $left = average(ref('v'), 17); + $right = average(ref('v'), 17); + $rows = array_to_rows(array_fill(0, 5, ['v' => 0.1]), schema(float_schema('v'))); + $left->aggregate($rows, range(0, 4), flow_context()); + $right->aggregate($rows, range(0, 4), flow_context()); + + $left->merge($right, flow_context()); + + static::assertSame(0.1, $left->value()); + } } diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/BetweenTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/BetweenTest.php index 6fc53681a4..3bbcb9b6de 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/BetweenTest.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/BetweenTest.php @@ -6,72 +6,134 @@ use Flow\ETL\Exception\InvalidArgumentException; use Flow\ETL\Function\Between\Boundary; +use Flow\ETL\Tests\Context\FunctionContext; use Flow\ETL\Tests\FlowTestCase; use function Flow\ETL\DSL\between; use function Flow\ETL\DSL\flow_context; +use function Flow\ETL\DSL\int_schema; use function Flow\ETL\DSL\lit; use function Flow\ETL\DSL\ref; -use function Flow\ETL\DSL\row; +use function Flow\ETL\DSL\schema; final class BetweenTest extends FlowTestCase { public function test_a_definite_false_short_circuits_past_a_null(): void { - static::assertFalse(ref('value')->between(lit(10), lit(null))->eval(row(['value' => 5]), flow_context())); + static::assertFalse((new FunctionContext(flow_context()))->eval( + ref('value')->between(lit(10), lit(null)), + ['value' => 5], + schema(int_schema('value')), + )); } public function test_a_null_bound_that_could_change_the_answer_is_null(): void { - static::assertNull(ref('value')->between(lit(10), lit(null))->eval(row(['value' => 50]), flow_context())); + static::assertNull((new FunctionContext(flow_context()))->eval( + ref('value')->between(lit(10), lit(null)), + ['value' => 50], + schema(int_schema('value')), + )); } public function test_between_exclusive(): void { - static::assertTrue(between(ref('value'), lit(10), lit(50), Boundary::EXCLUSIVE)->eval(row([ - 'value' => 11, - ]), flow_context())); - static::assertTrue(between(ref('value'), lit(10), lit(50), Boundary::EXCLUSIVE)->eval(row([ - 'value' => 49, - ]), flow_context())); - static::assertFalse(between(ref('value'), lit(10), lit(50), Boundary::EXCLUSIVE)->eval(row([ - 'value' => 10, - ]), flow_context())); - static::assertFalse(between(ref('value'), lit(10), lit(50), Boundary::EXCLUSIVE)->eval(row([ - 'value' => 50, - ]), flow_context())); + static::assertTrue((new FunctionContext(flow_context()))->eval( + between(ref('value'), lit(10), lit(50), Boundary::EXCLUSIVE), + [ + 'value' => 11, + ], + schema(int_schema('value')), + )); + static::assertTrue((new FunctionContext(flow_context()))->eval( + between(ref('value'), lit(10), lit(50), Boundary::EXCLUSIVE), + [ + 'value' => 49, + ], + schema(int_schema('value')), + )); + static::assertFalse((new FunctionContext(flow_context()))->eval( + between(ref('value'), lit(10), lit(50), Boundary::EXCLUSIVE), + [ + 'value' => 10, + ], + schema(int_schema('value')), + )); + static::assertFalse((new FunctionContext(flow_context()))->eval( + between(ref('value'), lit(10), lit(50), Boundary::EXCLUSIVE), + [ + 'value' => 50, + ], + schema(int_schema('value')), + )); } public function test_between_inclusive(): void { - static::assertTrue(between(ref('value'), lit(10), lit(50), Boundary::INCLUSIVE)->eval(row([ - 'value' => 10, - ]), flow_context())); - static::assertTrue(between(ref('value'), lit(10), lit(50), Boundary::INCLUSIVE)->eval(row([ - 'value' => 50, - ]), flow_context())); - static::assertFalse(between(ref('value'), lit(10), lit(50), Boundary::INCLUSIVE)->eval(row([ - 'value' => 9, - ]), flow_context())); - static::assertFalse(between(ref('value'), lit(10), lit(50), Boundary::INCLUSIVE)->eval(row([ - 'value' => 51, - ]), flow_context())); + static::assertTrue((new FunctionContext(flow_context()))->eval( + between(ref('value'), lit(10), lit(50), Boundary::INCLUSIVE), + [ + 'value' => 10, + ], + schema(int_schema('value')), + )); + static::assertTrue((new FunctionContext(flow_context()))->eval( + between(ref('value'), lit(10), lit(50), Boundary::INCLUSIVE), + [ + 'value' => 50, + ], + schema(int_schema('value')), + )); + static::assertFalse((new FunctionContext(flow_context()))->eval( + between(ref('value'), lit(10), lit(50), Boundary::INCLUSIVE), + [ + 'value' => 9, + ], + schema(int_schema('value')), + )); + static::assertFalse((new FunctionContext(flow_context()))->eval( + between(ref('value'), lit(10), lit(50), Boundary::INCLUSIVE), + [ + 'value' => 51, + ], + schema(int_schema('value')), + )); } public function test_between_left_inclusive(): void { - static::assertTrue(between(ref('value'), lit(10), lit(50))->eval(row(['value' => 10]), flow_context())); - static::assertFalse(between(ref('value'), lit(10), lit(50))->eval(row(['value' => 9]), flow_context())); + static::assertTrue((new FunctionContext(flow_context()))->eval( + between(ref('value'), lit(10), lit(50)), + [ + 'value' => 10, + ], + schema(int_schema('value')), + )); + static::assertFalse((new FunctionContext(flow_context()))->eval( + between(ref('value'), lit(10), lit(50)), + [ + 'value' => 9, + ], + schema(int_schema('value')), + )); } public function test_between_right_inclusive(): void { - static::assertTrue(between(ref('value'), lit(10), lit(50), Boundary::RIGHT_INCLUSIVE)->eval(row([ - 'value' => 50, - ]), flow_context())); - static::assertFalse(between(ref('value'), lit(10), lit(50), Boundary::RIGHT_INCLUSIVE)->eval(row([ - 'value' => 51, - ]), flow_context())); + static::assertTrue((new FunctionContext(flow_context()))->eval( + between(ref('value'), lit(10), lit(50), Boundary::RIGHT_INCLUSIVE), + [ + 'value' => 50, + ], + schema(int_schema('value')), + )); + static::assertFalse((new FunctionContext(flow_context()))->eval( + between(ref('value'), lit(10), lit(50), Boundary::RIGHT_INCLUSIVE), + [ + 'value' => 51, + ], + schema(int_schema('value')), + )); } public function test_between_with_invalid_boundary_in_strict_mode(): void @@ -80,6 +142,12 @@ public function test_between_with_invalid_boundary_in_strict_mode(): void $this->expectExceptionMessage('Expected type "object", got "string".'); $context = flow_context(); - between(ref('value'), lit(10), lit(50), lit('invalid'))->eval(row(['value' => 20]), $context); + (new FunctionContext($context))->eval( + between(ref('value'), lit(10), lit(50), lit('invalid')), + [ + 'value' => 20, + ], + schema(int_schema('value')), + ); } } diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/BinaryComparisonsTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/BinaryComparisonsTest.php index d0eaa9db2d..0552afe5d8 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/BinaryComparisonsTest.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/BinaryComparisonsTest.php @@ -22,216 +22,708 @@ use Flow\ETL\Function\NotSame; use Flow\ETL\Function\Same; use Flow\ETL\Function\StartsWith; +use Flow\ETL\Tests\Context\FunctionContext; use Flow\ETL\Tests\FlowTestCase; +use function Flow\ETL\DSL\datetime_schema; use function Flow\ETL\DSL\flow_context; +use function Flow\ETL\DSL\int_schema; +use function Flow\ETL\DSL\list_schema; use function Flow\ETL\DSL\lit; use function Flow\ETL\DSL\ref; -use function Flow\ETL\DSL\row; +use function Flow\ETL\DSL\schema; +use function Flow\ETL\DSL\str_schema; use function Flow\Types\DSL\type_datetime; +use function Flow\Types\DSL\type_integer; +use function Flow\Types\DSL\type_list; use function Flow\Types\DSL\type_string; final class BinaryComparisonsTest extends FlowTestCase { public function test_equals(): void { - $row = row([ - 'a' => 100, - 'b' => 100, - 'c' => 10, - 'd' => type_datetime()->cast('2023-01-01 00:00:00 UTC'), - 'e' => type_datetime()->cast('2023-01-01 00:00:00 UTC'), - ]); - - static::assertTrue((new Equals(ref('a'), ref('b')))->eval($row, flow_context())); - static::assertTrue((new Equals(ref('d'), ref('e')))->eval($row, flow_context())); - static::assertFalse((new Equals(ref('a'), ref('c')))->eval($row, flow_context())); + static::assertTrue((new FunctionContext(flow_context()))->eval( + new Equals(ref('a'), ref('b')), + [ + 'a' => 100, + 'b' => 100, + 'c' => 10, + 'd' => type_datetime()->cast('2023-01-01 00:00:00 UTC'), + 'e' => type_datetime()->cast('2023-01-01 00:00:00 UTC'), + ], + schema(int_schema('a'), int_schema('b'), int_schema('c'), datetime_schema('d'), datetime_schema('e')), + )); + static::assertTrue((new FunctionContext(flow_context()))->eval( + new Equals(ref('d'), ref('e')), + [ + 'a' => 100, + 'b' => 100, + 'c' => 10, + 'd' => type_datetime()->cast('2023-01-01 00:00:00 UTC'), + 'e' => type_datetime()->cast('2023-01-01 00:00:00 UTC'), + ], + schema(int_schema('a'), int_schema('b'), int_schema('c'), datetime_schema('d'), datetime_schema('e')), + )); + static::assertFalse((new FunctionContext(flow_context()))->eval( + new Equals(ref('a'), ref('c')), + [ + 'a' => 100, + 'b' => 100, + 'c' => 10, + 'd' => type_datetime()->cast('2023-01-01 00:00:00 UTC'), + 'e' => type_datetime()->cast('2023-01-01 00:00:00 UTC'), + ], + schema(int_schema('a'), int_schema('b'), int_schema('c'), datetime_schema('d'), datetime_schema('e')), + )); } public function test_greater_than(): void { - $row = row([ - 'a' => 100, - 'b' => 100, - 'c' => 10, - 'd' => type_datetime()->cast('2023-01-01 00:00:00 UTC'), - 'e' => type_datetime()->cast('2023-01-02 00:00:00 UTC'), - 'f' => null, - ]); - - static::assertTrue((new GreaterThan(ref('a'), ref('c')))->eval($row, flow_context())); - static::assertNull((new GreaterThan(ref('a'), ref('f')))->eval($row, flow_context())); - static::assertNull((new GreaterThan(ref('f'), ref('c')))->eval($row, flow_context())); - static::assertNull((new GreaterThan(ref('f'), ref('f')))->eval($row, flow_context())); - static::assertFalse((new GreaterThan(ref('a'), ref('b')))->eval($row, flow_context())); - static::assertTrue((new GreaterThanEqual(ref('a'), ref('c')))->eval($row, flow_context())); - static::assertTrue((new GreaterThanEqual(ref('a'), ref('b')))->eval($row, flow_context())); - static::assertTrue((new GreaterThanEqual(ref('e'), ref('d')))->eval($row, flow_context())); - static::assertTrue((new GreaterThanEqual( - ref('e'), - lit(new DateTimeImmutable('2022-01-01 00:00:00 UTC')), - ))->eval($row, flow_context())); - static::assertFalse((new GreaterThanEqual( - ref('e'), - lit(new DateTimeImmutable('2024-01-01 00:00:00 UTC')), - ))->eval($row, flow_context())); - static::assertNull((new GreaterThanEqual(ref('a'), ref('f')))->eval($row, flow_context())); - static::assertNull((new GreaterThanEqual(ref('f'), ref('c')))->eval($row, flow_context())); - static::assertNull((new GreaterThanEqual(ref('f'), ref('f')))->eval($row, flow_context())); + static::assertTrue((new FunctionContext(flow_context()))->eval( + new GreaterThan(ref('a'), ref('c')), + [ + 'a' => 100, + 'b' => 100, + 'c' => 10, + 'd' => type_datetime()->cast('2023-01-01 00:00:00 UTC'), + 'e' => type_datetime()->cast('2023-01-02 00:00:00 UTC'), + 'f' => null, + ], + schema( + int_schema('a'), + int_schema('b'), + int_schema('c'), + datetime_schema('d'), + datetime_schema('e'), + str_schema('f', nullable: true), + ), + )); + static::assertNull((new FunctionContext(flow_context()))->eval( + new GreaterThan(ref('a'), ref('f')), + [ + 'a' => 100, + 'b' => 100, + 'c' => 10, + 'd' => type_datetime()->cast('2023-01-01 00:00:00 UTC'), + 'e' => type_datetime()->cast('2023-01-02 00:00:00 UTC'), + 'f' => null, + ], + schema( + int_schema('a'), + int_schema('b'), + int_schema('c'), + datetime_schema('d'), + datetime_schema('e'), + str_schema('f', nullable: true), + ), + )); + static::assertNull((new FunctionContext(flow_context()))->eval( + new GreaterThan(ref('f'), ref('c')), + [ + 'a' => 100, + 'b' => 100, + 'c' => 10, + 'd' => type_datetime()->cast('2023-01-01 00:00:00 UTC'), + 'e' => type_datetime()->cast('2023-01-02 00:00:00 UTC'), + 'f' => null, + ], + schema( + int_schema('a'), + int_schema('b'), + int_schema('c'), + datetime_schema('d'), + datetime_schema('e'), + str_schema('f', nullable: true), + ), + )); + static::assertNull((new FunctionContext(flow_context()))->eval( + new GreaterThan(ref('f'), ref('f')), + [ + 'a' => 100, + 'b' => 100, + 'c' => 10, + 'd' => type_datetime()->cast('2023-01-01 00:00:00 UTC'), + 'e' => type_datetime()->cast('2023-01-02 00:00:00 UTC'), + 'f' => null, + ], + schema( + int_schema('a'), + int_schema('b'), + int_schema('c'), + datetime_schema('d'), + datetime_schema('e'), + str_schema('f', nullable: true), + ), + )); + static::assertFalse((new FunctionContext(flow_context()))->eval( + new GreaterThan(ref('a'), ref('b')), + [ + 'a' => 100, + 'b' => 100, + 'c' => 10, + 'd' => type_datetime()->cast('2023-01-01 00:00:00 UTC'), + 'e' => type_datetime()->cast('2023-01-02 00:00:00 UTC'), + 'f' => null, + ], + schema( + int_schema('a'), + int_schema('b'), + int_schema('c'), + datetime_schema('d'), + datetime_schema('e'), + str_schema('f', nullable: true), + ), + )); + static::assertTrue((new FunctionContext(flow_context()))->eval( + new GreaterThanEqual(ref('a'), ref('c')), + [ + 'a' => 100, + 'b' => 100, + 'c' => 10, + 'd' => type_datetime()->cast('2023-01-01 00:00:00 UTC'), + 'e' => type_datetime()->cast('2023-01-02 00:00:00 UTC'), + 'f' => null, + ], + schema( + int_schema('a'), + int_schema('b'), + int_schema('c'), + datetime_schema('d'), + datetime_schema('e'), + str_schema('f', nullable: true), + ), + )); + static::assertTrue((new FunctionContext(flow_context()))->eval( + new GreaterThanEqual(ref('a'), ref('b')), + [ + 'a' => 100, + 'b' => 100, + 'c' => 10, + 'd' => type_datetime()->cast('2023-01-01 00:00:00 UTC'), + 'e' => type_datetime()->cast('2023-01-02 00:00:00 UTC'), + 'f' => null, + ], + schema( + int_schema('a'), + int_schema('b'), + int_schema('c'), + datetime_schema('d'), + datetime_schema('e'), + str_schema('f', nullable: true), + ), + )); + static::assertTrue((new FunctionContext(flow_context()))->eval( + new GreaterThanEqual(ref('e'), ref('d')), + [ + 'a' => 100, + 'b' => 100, + 'c' => 10, + 'd' => type_datetime()->cast('2023-01-01 00:00:00 UTC'), + 'e' => type_datetime()->cast('2023-01-02 00:00:00 UTC'), + 'f' => null, + ], + schema( + int_schema('a'), + int_schema('b'), + int_schema('c'), + datetime_schema('d'), + datetime_schema('e'), + str_schema('f', nullable: true), + ), + )); + static::assertTrue((new FunctionContext(flow_context()))->eval( + new GreaterThanEqual(ref('e'), lit(new DateTimeImmutable('2022-01-01 00:00:00 UTC'))), + [ + 'a' => 100, + 'b' => 100, + 'c' => 10, + 'd' => type_datetime()->cast('2023-01-01 00:00:00 UTC'), + 'e' => type_datetime()->cast('2023-01-02 00:00:00 UTC'), + 'f' => null, + ], + schema( + int_schema('a'), + int_schema('b'), + int_schema('c'), + datetime_schema('d'), + datetime_schema('e'), + str_schema('f', nullable: true), + ), + )); + static::assertFalse((new FunctionContext(flow_context()))->eval( + new GreaterThanEqual(ref('e'), lit(new DateTimeImmutable('2024-01-01 00:00:00 UTC'))), + [ + 'a' => 100, + 'b' => 100, + 'c' => 10, + 'd' => type_datetime()->cast('2023-01-01 00:00:00 UTC'), + 'e' => type_datetime()->cast('2023-01-02 00:00:00 UTC'), + 'f' => null, + ], + schema( + int_schema('a'), + int_schema('b'), + int_schema('c'), + datetime_schema('d'), + datetime_schema('e'), + str_schema('f', nullable: true), + ), + )); + static::assertNull((new FunctionContext(flow_context()))->eval( + new GreaterThanEqual(ref('a'), ref('f')), + [ + 'a' => 100, + 'b' => 100, + 'c' => 10, + 'd' => type_datetime()->cast('2023-01-01 00:00:00 UTC'), + 'e' => type_datetime()->cast('2023-01-02 00:00:00 UTC'), + 'f' => null, + ], + schema( + int_schema('a'), + int_schema('b'), + int_schema('c'), + datetime_schema('d'), + datetime_schema('e'), + str_schema('f', nullable: true), + ), + )); + static::assertNull((new FunctionContext(flow_context()))->eval( + new GreaterThanEqual(ref('f'), ref('c')), + [ + 'a' => 100, + 'b' => 100, + 'c' => 10, + 'd' => type_datetime()->cast('2023-01-01 00:00:00 UTC'), + 'e' => type_datetime()->cast('2023-01-02 00:00:00 UTC'), + 'f' => null, + ], + schema( + int_schema('a'), + int_schema('b'), + int_schema('c'), + datetime_schema('d'), + datetime_schema('e'), + str_schema('f', nullable: true), + ), + )); + static::assertNull((new FunctionContext(flow_context()))->eval( + new GreaterThanEqual(ref('f'), ref('f')), + [ + 'a' => 100, + 'b' => 100, + 'c' => 10, + 'd' => type_datetime()->cast('2023-01-01 00:00:00 UTC'), + 'e' => type_datetime()->cast('2023-01-02 00:00:00 UTC'), + 'f' => null, + ], + schema( + int_schema('a'), + int_schema('b'), + int_schema('c'), + datetime_schema('d'), + datetime_schema('e'), + str_schema('f', nullable: true), + ), + )); } public function test_greater_than_equal_with_null_in_strict_mode(): void { $context = flow_context(); - $row = row(['a' => 100, 'f' => null]); - static::assertNull((new GreaterThanEqual(ref('a'), ref('f')))->eval($row, $context)); + static::assertNull((new FunctionContext($context))->eval( + new GreaterThanEqual(ref('a'), ref('f')), + ['a' => 100, 'f' => null], + schema(int_schema('a'), str_schema('f', nullable: true)), + )); } public function test_greater_than_with_null_in_strict_mode(): void { $context = flow_context(); - $row = row(['a' => 100, 'f' => null]); - static::assertNull((new GreaterThan(ref('a'), ref('f')))->eval($row, $context)); + static::assertNull((new FunctionContext($context))->eval( + new GreaterThan(ref('a'), ref('f')), + ['a' => 100, 'f' => null], + schema(int_schema('a'), str_schema('f', nullable: true)), + )); } public function test_is_in(): void { - $row = row([ - 'a' => [1, 2, 3, 4, 5], - 'b' => ['a', 'b', 'c'], - 'c' => 'another', - 'd' => 4, - 'e' => 'b', - ]); - - static::assertTrue((new IsIn(ref('a'), lit(1)))->eval($row, flow_context())); - static::assertFalse((new IsIn(ref('a'), lit(10)))->eval($row, flow_context())); - static::assertTrue((new IsIn(ref('a'), ref('d')))->eval($row, flow_context())); - static::assertTrue((new IsIn(ref('b'), ref('e')))->eval($row, flow_context())); + static::assertTrue((new FunctionContext(flow_context()))->eval( + new IsIn(ref('a'), lit(1)), + [ + 'a' => [1, 2, 3, 4, 5], + 'b' => ['a', 'b', 'c'], + 'c' => 'another', + 'd' => 4, + 'e' => 'b', + ], + schema( + list_schema('a', type_list(type_integer())), + list_schema('b', type_list(type_string())), + str_schema('c'), + int_schema('d'), + str_schema('e'), + ), + )); + static::assertFalse((new FunctionContext(flow_context()))->eval( + new IsIn(ref('a'), lit(10)), + [ + 'a' => [1, 2, 3, 4, 5], + 'b' => ['a', 'b', 'c'], + 'c' => 'another', + 'd' => 4, + 'e' => 'b', + ], + schema( + list_schema('a', type_list(type_integer())), + list_schema('b', type_list(type_string())), + str_schema('c'), + int_schema('d'), + str_schema('e'), + ), + )); + static::assertTrue((new FunctionContext(flow_context()))->eval( + new IsIn(ref('a'), ref('d')), + [ + 'a' => [1, 2, 3, 4, 5], + 'b' => ['a', 'b', 'c'], + 'c' => 'another', + 'd' => 4, + 'e' => 'b', + ], + schema( + list_schema('a', type_list(type_integer())), + list_schema('b', type_list(type_string())), + str_schema('c'), + int_schema('d'), + str_schema('e'), + ), + )); + static::assertTrue((new FunctionContext(flow_context()))->eval( + new IsIn(ref('b'), ref('e')), + [ + 'a' => [1, 2, 3, 4, 5], + 'b' => ['a', 'b', 'c'], + 'c' => 'another', + 'd' => 4, + 'e' => 'b', + ], + schema( + list_schema('a', type_list(type_integer())), + list_schema('b', type_list(type_string())), + str_schema('c'), + int_schema('d'), + str_schema('e'), + ), + )); } public function test_is_in_with_null_array_in_strict_mode(): void { $context = flow_context(); - $row = row(['a' => null, 'd' => 1]); - static::assertNull((new IsIn(ref('a'), ref('d')))->eval($row, $context)); + static::assertNull((new FunctionContext($context))->eval( + new IsIn(ref('a'), ref('d')), + ['a' => null, 'd' => 1], + schema(str_schema('a', nullable: true), int_schema('d')), + )); } public function test_is_numeric(): void { - $row = row(['a' => 100, 'b' => null]); - static::assertTrue((new IsNumeric(ref('a')))->eval($row, flow_context())); - static::assertNull((new IsNumeric(ref('b')))->eval($row, flow_context())); - static::assertFalse((new IsNotNumeric(ref('a')))->eval($row, flow_context())); - static::assertNull((new IsNotNumeric(ref('b')))->eval($row, flow_context())); - static::assertNull((new IsNotNumeric(lit(null)))->eval($row, flow_context())); - static::assertTrue((new IsNumeric(lit(1000)))->eval($row, flow_context())); + static::assertTrue((new FunctionContext(flow_context()))->eval( + new IsNumeric(ref('a')), + ['a' => 100, 'b' => null], + schema(int_schema('a'), str_schema('b', nullable: true)), + )); + static::assertNull((new FunctionContext(flow_context()))->eval( + new IsNumeric(ref('b')), + ['a' => 100, 'b' => null], + schema(int_schema('a'), str_schema('b', nullable: true)), + )); + static::assertFalse((new FunctionContext(flow_context()))->eval( + new IsNotNumeric(ref('a')), + ['a' => 100, 'b' => null], + schema(int_schema('a'), str_schema('b', nullable: true)), + )); + static::assertNull((new FunctionContext(flow_context()))->eval( + new IsNotNumeric(ref('b')), + ['a' => 100, 'b' => null], + schema(int_schema('a'), str_schema('b', nullable: true)), + )); + static::assertNull((new FunctionContext(flow_context()))->eval( + new IsNotNumeric(lit(null)), + ['a' => 100, 'b' => null], + schema(int_schema('a'), str_schema('b', nullable: true)), + )); + static::assertTrue((new FunctionContext(flow_context()))->eval( + new IsNumeric(lit(1000)), + ['a' => 100, 'b' => null], + schema(int_schema('a'), str_schema('b', nullable: true)), + )); } public function test_is_type(): void { - $row = row(['a' => 100, 'b' => null]); - - static::assertTrue((new IsType(ref('a'), 'integer', 'string'))->eval($row, flow_context())); - static::assertFalse((new IsType(ref('a'), type_string()))->eval($row, flow_context())); + static::assertTrue((new FunctionContext(flow_context()))->eval( + new IsType(ref('a'), 'integer', 'string'), + ['a' => 100, 'b' => null], + schema(int_schema('a'), str_schema('b', nullable: true)), + )); + static::assertFalse((new FunctionContext(flow_context()))->eval( + new IsType(ref('a'), type_string()), + ['a' => 100, 'b' => null], + schema(int_schema('a'), str_schema('b', nullable: true)), + )); } public function test_is_type_with_non_existing_type_class(): void { $this->expectExceptionMessage('Unknown type \'aaa\''); - $row = row(['a' => 100, 'b' => null]); - - static::assertFalse((new IsType(ref('a'), 'aaa'))->eval($row, flow_context())); + static::assertFalse((new FunctionContext(flow_context()))->eval( + new IsType(ref('a'), 'aaa'), + ['a' => 100, 'b' => null], + schema(int_schema('a'), str_schema('b', nullable: true)), + )); } public function test_less_than(): void { - $row = row(['a' => 100, 'b' => 100, 'c' => 10, 'd' => null]); - - static::assertFalse((new LessThan(ref('a'), ref('c')))->eval($row, flow_context())); - static::assertNull((new LessThan(ref('a'), ref('d')))->eval($row, flow_context())); - static::assertNull((new LessThan(ref('d'), ref('d')))->eval($row, flow_context())); - static::assertNull((new LessThan(ref('d'), ref('c')))->eval($row, flow_context())); - static::assertFalse((new LessThan(ref('a'), ref('b')))->eval($row, flow_context())); - static::assertTrue((new LessThanEqual(ref('c'), ref('a')))->eval($row, flow_context())); - static::assertTrue((new LessThanEqual(ref('a'), ref('b')))->eval($row, flow_context())); - static::assertNull((new LessThanEqual(ref('a'), ref('d')))->eval($row, flow_context())); - static::assertNull((new LessThanEqual(ref('d'), ref('c')))->eval($row, flow_context())); - static::assertNull((new LessThanEqual(ref('d'), ref('d')))->eval($row, flow_context())); + static::assertFalse((new FunctionContext(flow_context()))->eval( + new LessThan(ref('a'), ref('c')), + ['a' => 100, 'b' => 100, 'c' => 10, 'd' => null], + schema(int_schema('a'), int_schema('b'), int_schema('c'), str_schema('d', nullable: true)), + )); + static::assertNull((new FunctionContext(flow_context()))->eval( + new LessThan(ref('a'), ref('d')), + ['a' => 100, 'b' => 100, 'c' => 10, 'd' => null], + schema(int_schema('a'), int_schema('b'), int_schema('c'), str_schema('d', nullable: true)), + )); + static::assertNull((new FunctionContext(flow_context()))->eval( + new LessThan(ref('d'), ref('d')), + ['a' => 100, 'b' => 100, 'c' => 10, 'd' => null], + schema(int_schema('a'), int_schema('b'), int_schema('c'), str_schema('d', nullable: true)), + )); + static::assertNull((new FunctionContext(flow_context()))->eval( + new LessThan(ref('d'), ref('c')), + ['a' => 100, 'b' => 100, 'c' => 10, 'd' => null], + schema(int_schema('a'), int_schema('b'), int_schema('c'), str_schema('d', nullable: true)), + )); + static::assertFalse((new FunctionContext(flow_context()))->eval( + new LessThan(ref('a'), ref('b')), + ['a' => 100, 'b' => 100, 'c' => 10, 'd' => null], + schema(int_schema('a'), int_schema('b'), int_schema('c'), str_schema('d', nullable: true)), + )); + static::assertTrue((new FunctionContext(flow_context()))->eval( + new LessThanEqual(ref('c'), ref('a')), + ['a' => 100, 'b' => 100, 'c' => 10, 'd' => null], + schema(int_schema('a'), int_schema('b'), int_schema('c'), str_schema('d', nullable: true)), + )); + static::assertTrue((new FunctionContext(flow_context()))->eval( + new LessThanEqual(ref('a'), ref('b')), + ['a' => 100, 'b' => 100, 'c' => 10, 'd' => null], + schema(int_schema('a'), int_schema('b'), int_schema('c'), str_schema('d', nullable: true)), + )); + static::assertNull((new FunctionContext(flow_context()))->eval( + new LessThanEqual(ref('a'), ref('d')), + ['a' => 100, 'b' => 100, 'c' => 10, 'd' => null], + schema(int_schema('a'), int_schema('b'), int_schema('c'), str_schema('d', nullable: true)), + )); + static::assertNull((new FunctionContext(flow_context()))->eval( + new LessThanEqual(ref('d'), ref('c')), + ['a' => 100, 'b' => 100, 'c' => 10, 'd' => null], + schema(int_schema('a'), int_schema('b'), int_schema('c'), str_schema('d', nullable: true)), + )); + static::assertNull((new FunctionContext(flow_context()))->eval( + new LessThanEqual(ref('d'), ref('d')), + ['a' => 100, 'b' => 100, 'c' => 10, 'd' => null], + schema(int_schema('a'), int_schema('b'), int_schema('c'), str_schema('d', nullable: true)), + )); } public function test_less_than_equal_with_null_in_strict_mode(): void { $context = flow_context(); - $row = row(['a' => 100, 'd' => null]); - static::assertNull((new LessThanEqual(ref('a'), ref('d')))->eval($row, $context)); + static::assertNull((new FunctionContext($context))->eval( + new LessThanEqual(ref('a'), ref('d')), + ['a' => 100, 'd' => null], + schema(int_schema('a'), str_schema('d', nullable: true)), + )); } public function test_less_than_with_null_in_strict_mode(): void { $context = flow_context(); - $row = row(['a' => 100, 'd' => null]); - static::assertNull((new LessThan(ref('a'), ref('d')))->eval($row, $context)); + static::assertNull((new FunctionContext($context))->eval( + new LessThan(ref('a'), ref('d')), + ['a' => 100, 'd' => null], + schema(int_schema('a'), str_schema('d', nullable: true)), + )); } public function test_not_equals(): void { - $row = row(['a' => 100, 'b' => 100, 'c' => 10]); - - static::assertFalse((new NotEquals(ref('a'), ref('b')))->eval($row, flow_context())); - static::assertTrue((new NotEquals(ref('a'), ref('c')))->eval($row, flow_context())); + static::assertFalse((new FunctionContext(flow_context()))->eval( + new NotEquals(ref('a'), ref('b')), + ['a' => 100, 'b' => 100, 'c' => 10], + schema(int_schema('a'), int_schema('b'), int_schema('c')), + )); + static::assertTrue((new FunctionContext(flow_context()))->eval( + new NotEquals(ref('a'), ref('c')), + ['a' => 100, 'b' => 100, 'c' => 10], + schema(int_schema('a'), int_schema('b'), int_schema('c')), + )); } public function test_not_same(): void { - $row = row(['a' => 100, 'b' => 100, 'c' => 10]); - - static::assertTrue((new NotSame(ref('a'), ref('c')))->eval($row, flow_context())); - static::assertFalse((new NotSame(ref('a'), ref('b')))->eval($row, flow_context())); + static::assertTrue((new FunctionContext(flow_context()))->eval( + new NotSame(ref('a'), ref('c')), + ['a' => 100, 'b' => 100, 'c' => 10], + schema(int_schema('a'), int_schema('b'), int_schema('c')), + )); + static::assertFalse((new FunctionContext(flow_context()))->eval( + new NotSame(ref('a'), ref('b')), + ['a' => 100, 'b' => 100, 'c' => 10], + schema(int_schema('a'), int_schema('b'), int_schema('c')), + )); } public function test_null(): void { - $row = row(['a' => 100, 'b' => null]); - - static::assertFalse((new IsNull(ref('a')))->eval($row, flow_context())); - static::assertTrue((new IsNull(ref('b')))->eval($row, flow_context())); - static::assertTrue((new IsNotNull(ref('a')))->eval($row, flow_context())); - static::assertFalse((new IsNotNull(ref('b')))->eval($row, flow_context())); - static::assertTrue((new IsNull(lit(null)))->eval($row, flow_context())); - static::assertTrue((new IsNotNull(lit(1000)))->eval($row, flow_context())); + static::assertFalse((new FunctionContext(flow_context()))->eval( + new IsNull(ref('a')), + ['a' => 100, 'b' => null], + schema(int_schema('a'), str_schema('b', nullable: true)), + )); + static::assertTrue((new FunctionContext(flow_context()))->eval( + new IsNull(ref('b')), + ['a' => 100, 'b' => null], + schema(int_schema('a'), str_schema('b', nullable: true)), + )); + static::assertTrue((new FunctionContext(flow_context()))->eval( + new IsNotNull(ref('a')), + ['a' => 100, 'b' => null], + schema(int_schema('a'), str_schema('b', nullable: true)), + )); + static::assertFalse((new FunctionContext(flow_context()))->eval( + new IsNotNull(ref('b')), + ['a' => 100, 'b' => null], + schema(int_schema('a'), str_schema('b', nullable: true)), + )); + static::assertTrue((new FunctionContext(flow_context()))->eval( + new IsNull(lit(null)), + ['a' => 100, 'b' => null], + schema(int_schema('a'), str_schema('b', nullable: true)), + )); + static::assertTrue((new FunctionContext(flow_context()))->eval( + new IsNotNull(lit(1000)), + ['a' => 100, 'b' => null], + schema(int_schema('a'), str_schema('b', nullable: true)), + )); } public function test_same(): void { - $row = row([ - 'a' => 100, - 'b' => 100, - 'c' => 10, - 'd' => type_datetime()->cast('2023-01-01 00:00:00 UTC'), - 'e' => type_datetime()->cast('2023-01-01 00:00:00 UTC'), - ]); - - static::assertTrue((new Same(ref('a'), ref('b')))->eval($row, flow_context())); - static::assertFalse((new Same(ref('d'), ref('e')))->eval($row, flow_context())); - static::assertFalse((new Same(ref('a'), ref('c')))->eval($row, flow_context())); + static::assertTrue((new FunctionContext(flow_context()))->eval( + new Same(ref('a'), ref('b')), + [ + 'a' => 100, + 'b' => 100, + 'c' => 10, + 'd' => type_datetime()->cast('2023-01-01 00:00:00 UTC'), + 'e' => type_datetime()->cast('2023-01-01 00:00:00 UTC'), + ], + schema(int_schema('a'), int_schema('b'), int_schema('c'), datetime_schema('d'), datetime_schema('e')), + )); + // two datetimes of the same instant are the same value + static::assertTrue((new FunctionContext(flow_context()))->eval( + new Same(ref('d'), ref('e')), + [ + 'a' => 100, + 'b' => 100, + 'c' => 10, + 'd' => type_datetime()->cast('2023-01-01 00:00:00 UTC'), + 'e' => type_datetime()->cast('2023-01-01 00:00:00 UTC'), + ], + schema(int_schema('a'), int_schema('b'), int_schema('c'), datetime_schema('d'), datetime_schema('e')), + )); + static::assertFalse((new FunctionContext(flow_context()))->eval( + new Same(ref('a'), ref('c')), + [ + 'a' => 100, + 'b' => 100, + 'c' => 10, + 'd' => type_datetime()->cast('2023-01-01 00:00:00 UTC'), + 'e' => type_datetime()->cast('2023-01-01 00:00:00 UTC'), + ], + schema(int_schema('a'), int_schema('b'), int_schema('c'), datetime_schema('d'), datetime_schema('e')), + )); } public function test_starts_ends_with(): void { - $row = row([ - 'a' => 'some not too long string', - 'b' => 'another not too long text', - 'c' => 'another', - 'd' => 'text', - ]); - - static::assertTrue((new StartsWith(ref('a'), lit('some not')))->eval($row, flow_context())); - static::assertTrue((new EndsWith(ref('a'), lit('long string')))->eval($row, flow_context())); - static::assertTrue((new StartsWith(ref('b'), ref('c')))->eval($row, flow_context())); - static::assertTrue((new EndsWith(ref('b'), ref('d')))->eval($row, flow_context())); - static::assertTrue((new Contains(ref('a'), lit('too long')))->eval($row, flow_context())); - static::assertFalse((new Contains(ref('a'), lit('blablabla')))->eval($row, flow_context())); + static::assertTrue((new FunctionContext(flow_context()))->eval( + new StartsWith(ref('a'), lit('some not')), + [ + 'a' => 'some not too long string', + 'b' => 'another not too long text', + 'c' => 'another', + 'd' => 'text', + ], + schema(str_schema('a'), str_schema('b'), str_schema('c'), str_schema('d')), + )); + static::assertTrue((new FunctionContext(flow_context()))->eval( + new EndsWith(ref('a'), lit('long string')), + [ + 'a' => 'some not too long string', + 'b' => 'another not too long text', + 'c' => 'another', + 'd' => 'text', + ], + schema(str_schema('a'), str_schema('b'), str_schema('c'), str_schema('d')), + )); + static::assertTrue((new FunctionContext(flow_context()))->eval( + new StartsWith(ref('b'), ref('c')), + [ + 'a' => 'some not too long string', + 'b' => 'another not too long text', + 'c' => 'another', + 'd' => 'text', + ], + schema(str_schema('a'), str_schema('b'), str_schema('c'), str_schema('d')), + )); + static::assertTrue((new FunctionContext(flow_context()))->eval( + new EndsWith(ref('b'), ref('d')), + [ + 'a' => 'some not too long string', + 'b' => 'another not too long text', + 'c' => 'another', + 'd' => 'text', + ], + schema(str_schema('a'), str_schema('b'), str_schema('c'), str_schema('d')), + )); + static::assertTrue((new FunctionContext(flow_context()))->eval( + new Contains(ref('a'), lit('too long')), + [ + 'a' => 'some not too long string', + 'b' => 'another not too long text', + 'c' => 'another', + 'd' => 'text', + ], + schema(str_schema('a'), str_schema('b'), str_schema('c'), str_schema('d')), + )); + static::assertFalse((new FunctionContext(flow_context()))->eval( + new Contains(ref('a'), lit('blablabla')), + [ + 'a' => 'some not too long string', + 'b' => 'another not too long text', + 'c' => 'another', + 'd' => 'text', + ], + schema(str_schema('a'), str_schema('b'), str_schema('c'), str_schema('d')), + )); } } diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/BinaryLengthTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/BinaryLengthTest.php index 4ee9b58dd8..266b05164d 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/BinaryLengthTest.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/BinaryLengthTest.php @@ -5,29 +5,43 @@ namespace Flow\ETL\Tests\Unit\Function; use Flow\ETL\Exception\InvalidArgumentException; +use Flow\ETL\Tests\Context\FunctionContext; use Flow\ETL\Tests\FlowTestCase; use function Flow\ETL\DSL\flow_context; use function Flow\ETL\DSL\ref; -use function Flow\ETL\DSL\row; +use function Flow\ETL\DSL\schema; +use function Flow\ETL\DSL\str_schema; final class BinaryLengthTest extends FlowTestCase { public function test_binary_length_ascii_string(): void { - static::assertSame(5, ref('str')->binaryLength()->eval(row(['str' => 'hello']), flow_context())); + static::assertSame(5, (new FunctionContext(flow_context()))->eval( + ref('str')->binaryLength(), + ['str' => 'hello'], + schema(str_schema('str')), + )); } public function test_binary_length_binary_data(): void { $binaryData = "\x00\x01\x02\x03\xFF"; - static::assertSame(5, ref('str')->binaryLength()->eval(row(['str' => $binaryData]), flow_context())); + static::assertSame(5, (new FunctionContext(flow_context()))->eval( + ref('str')->binaryLength(), + ['str' => $binaryData], + schema(str_schema('str')), + )); } public function test_binary_length_empty_string(): void { - static::assertSame(0, ref('str')->binaryLength()->eval(row(['str' => '']), flow_context())); + static::assertSame(0, (new FunctionContext(flow_context()))->eval( + ref('str')->binaryLength(), + ['str' => ''], + schema(str_schema('str')), + )); } public function test_binary_length_throws_on_null_input(): void @@ -35,11 +49,19 @@ public function test_binary_length_throws_on_null_input(): void $this->expectException(InvalidArgumentException::class); $this->expectExceptionMessage('BinaryLength function requires non-null value'); - ref('str')->binaryLength()->eval(row(['str' => null]), flow_context()); + (new FunctionContext(flow_context()))->eval( + ref('str')->binaryLength(), + ['str' => null], + schema(str_schema('str', nullable: true)), + ); } public function test_binary_length_string_with_newlines_and_tabs(): void { - static::assertSame(12, ref('str')->binaryLength()->eval(row(['str' => "hello\nworld\t"]), flow_context())); + static::assertSame(12, (new FunctionContext(flow_context()))->eval( + ref('str')->binaryLength(), + ['str' => "hello\nworld\t"], + schema(str_schema('str')), + )); } } diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/CallUserFuncTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/CallUserFuncTest.php index 16f5711bc1..aa7b3e94e9 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/CallUserFuncTest.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/CallUserFuncTest.php @@ -6,17 +6,18 @@ use Flow\ETL\Exception\InvalidArgumentException; use Flow\ETL\Function\CallUserFunc; +use Flow\ETL\Tests\Context\FunctionContext; use Flow\ETL\Tests\FlowTestCase; use Flow\ETL\Tests\Unit\Function\Fixtures\CallUserFunc\StaticCalculator; use Flow\ETL\Transformer\ScalarFunctionTransformer; +use function Flow\ETL\DSL\array_to_rows; use function Flow\ETL\DSL\call; use function Flow\ETL\DSL\flow_context; +use function Flow\ETL\DSL\list_schema; use function Flow\ETL\DSL\lit; use function Flow\ETL\DSL\ref; -use function Flow\ETL\DSL\row; use function Flow\ETL\DSL\row_number; -use function Flow\ETL\DSL\rows; use function Flow\ETL\DSL\schema; use function Flow\ETL\DSL\str_schema; use function Flow\Types\DSL\type_integer; @@ -38,13 +39,20 @@ public function test_named_parameters_survive_a_rebuild(): void ]); $rebuilt = $function->withChildren($function->children()); - static::assertEquals($function->eval(row([]), flow_context()), $rebuilt->eval(row([]), flow_context())); - static::assertEquals(['1', '2', '3'], $rebuilt->eval(row([]), flow_context())); + static::assertEquals( + (new FunctionContext(flow_context()))->eval($function, [], schema()), + (new FunctionContext(flow_context()))->eval($rebuilt, [], schema()), + ); + static::assertEquals(['1', '2', '3'], (new FunctionContext(flow_context()))->eval($rebuilt, [], schema())); } public function test_call_user_func_as_dsl(): void { - static::assertIsInt(call(lit('time'), type_integer())->eval(row([]), flow_context())); + static::assertIsInt((new FunctionContext(flow_context()))->eval( + call(lit('time'), type_integer()), + [], + schema(), + )); } public function test_a_non_scalar_function_cannot_replace_the_callable_child(): void @@ -59,49 +67,67 @@ public function test_a_non_scalar_function_cannot_replace_the_callable_child(): public function test_call_user_func_with_array_callable(): void { - static::assertSame(3, ref('list') - ->call(lit([new StaticCalculator(), 'count']), type_integer()) - ->eval(row(['list' => [1, 2, 3]]), flow_context())); + static::assertSame(3, (new FunctionContext(flow_context()))->eval( + ref('list')->call(lit([new StaticCalculator(), 'count']), type_integer()), + ['list' => [ + 1, + 2, + 3, + ]], + schema(list_schema('list', type_list(type_integer()))), + )); } public function test_call_user_func_with_native_function(): void { - $row = row(['list' => [1, 2, 3]]); - - static::assertSame(3, ref('list')->call(lit('count'), type_integer())->eval($row, flow_context())); + static::assertSame(3, (new FunctionContext(flow_context()))->eval( + ref('list')->call(lit('count'), type_integer()), + ['list' => [1, 2, 3]], + schema(list_schema('list', type_list(type_integer()))), + )); } public function test_call_user_func_with_ref_alias_and_optional_arguments(): void { - $row = row(['item_ids' => '1,2,3']); - static::assertEquals( ['1', '2', '3'], - ref('item_ids') - ->call(lit('explode'), type_list(type_string()), ['separator' => ','], refAlias: 'string') - ->eval($row, flow_context()), + (new FunctionContext(flow_context()))->eval( + ref('item_ids')->call( + lit('explode'), + type_list(type_string()), + ['separator' => ','], + refAlias: 'string', + ), + ['item_ids' => '1,2,3'], + schema(str_schema('item_ids')), + ), ); } public function test_call_user_func_with_ref_alias_and_optional_arguments_and_return_type(): void { - $row = row(['item_ids' => '1,2,3']); - static::assertEquals( [1, 2, 3], - ref('item_ids') - ->call(lit('explode'), type_list(type_integer()), ['separator' => ','], refAlias: 'string') - ->eval($row, flow_context()), + (new FunctionContext(flow_context()))->eval( + ref('item_ids')->call( + lit('explode'), + type_list(type_integer()), + ['separator' => ','], + refAlias: 'string', + ), + ['item_ids' => '1,2,3'], + schema(str_schema('item_ids')), + ), ); } public function test_call_user_func_with_static_method(): void { - $row = row(['list' => [1, 2, 3]]); - - static::assertSame(3, ref('list') - ->call(lit(StaticCalculator::class . '::count'), type_integer()) - ->eval($row, flow_context())); + static::assertSame(3, (new FunctionContext(flow_context()))->eval( + ref('list')->call(lit(StaticCalculator::class . '::count'), type_integer()), + ['list' => [1, 2, 3]], + schema(list_schema('list', type_list(type_integer()))), + )); } public function test_a_callable_that_can_return_null_declares_a_nullable_column(): void @@ -116,9 +142,9 @@ public function test_a_null_return_lands_in_a_nullable_column(): void $result = (new ScalarFunctionTransformer( 'out', new CallUserFunc(lit([StaticCalculator::class, 'alwaysNull']), type_string(), []), - ))->transform(rows(schema(str_schema('name')), row(['name' => 'a'])), flow_context()); + ))->transform(array_to_rows([['name' => 'a']], schema(str_schema('name'))), flow_context()); static::assertTrue($result->schema()->get('out')->isNullable()); - static::assertNull($result->first()->get('out')); + static::assertNull($result->column('out')->value(0)); } } diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/CapitalizeTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/CapitalizeTest.php index 7acc24a0c1..72e040e5cb 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/CapitalizeTest.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/CapitalizeTest.php @@ -4,18 +4,22 @@ namespace Flow\ETL\Tests\Unit\Function; +use Flow\ETL\Tests\Context\FunctionContext; use Flow\ETL\Tests\FlowTestCase; use function Flow\ETL\DSL\flow_context; use function Flow\ETL\DSL\ref; -use function Flow\ETL\DSL\row; +use function Flow\ETL\DSL\schema; +use function Flow\ETL\DSL\str_schema; final class CapitalizeTest extends FlowTestCase { public function test_capitalize_valid_string(): void { - static::assertSame('This Is A Value', ref('string') - ->capitalize() - ->eval(row(['string' => 'this is a value']), flow_context())); + static::assertSame('This Is A Value', (new FunctionContext(flow_context()))->eval( + ref('string')->capitalize(), + ['string' => 'this is a value'], + schema(str_schema('string')), + )); } } diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/CastTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/CastTest.php index ccc8c83a88..960137fb5f 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/CastTest.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/CastTest.php @@ -7,18 +7,31 @@ use DateTimeImmutable; use DateTimeZone; use DOMDocument; +use DOMElement; +use Flow\ETL\Exception\EvaluationException; use Flow\ETL\Exception\InvalidArgumentException; +use Flow\ETL\Schema\Definition; +use Flow\ETL\Tests\Context\FunctionContext; use Flow\ETL\Tests\FlowTestCase; +use Flow\Types\Exception\CastingException; use Flow\Types\Type; use Flow\Types\Value\Json; use Flow\Types\Value\Uuid; use Generator; use PHPUnit\Framework\Attributes\DataProvider; +use function Flow\ETL\DSL\bool_schema; use function Flow\ETL\DSL\cast; +use function Flow\ETL\DSL\date_schema; +use function Flow\ETL\DSL\datetime_schema; use function Flow\ETL\DSL\flow_context; +use function Flow\ETL\DSL\int_schema; +use function Flow\ETL\DSL\list_schema; use function Flow\ETL\DSL\ref; -use function Flow\ETL\DSL\row; +use function Flow\ETL\DSL\schema; +use function Flow\ETL\DSL\str_schema; +use function Flow\ETL\DSL\uuid_schema; +use function Flow\ETL\DSL\xml_schema; use function Flow\Types\DSL\type_datetime; use function Flow\Types\DSL\type_integer; use function Flow\Types\DSL\type_list; @@ -34,14 +47,88 @@ public function test_cast_of_a_null_value_throws(): void $this->expectException(InvalidArgumentException::class); $this->expectExceptionMessage('Cast function requires non-null value'); - cast(ref('value'), 'int')->eval(row(['value' => null]), flow_context()); + (new FunctionContext(flow_context()))->eval( + cast(ref('value'), 'int'), + [ + 'value' => null, + ], + schema(str_schema('value', nullable: true)), + ); + } + + public function test_casting_a_datetime_column_to_string_equals_casting_each_value(): void + { + $rows = [ + ['at' => new DateTimeImmutable('2026-03-29T00:59:59.123456Z')], + ['at' => new DateTimeImmutable('2026-03-29T01:00:00Z')], + ]; + + foreach (['UTC', 'Europe/Warsaw'] as $zone) { + static::assertSame( + [ + type_string()->cast($rows[0]['at']->setTimezone(new DateTimeZone($zone))), + type_string()->cast($rows[1]['at']->setTimezone(new DateTimeZone($zone))), + ], + (new FunctionContext(flow_context())) + ->column(cast(ref('at'), 'string'), $rows, schema(datetime_schema('at', zone: $zone))) + ->values(), + ); + } + } + + public function test_casting_a_date_column_to_string(): void + { + static::assertSame( + ['2026-01-02T00:00:00+00:00', null], + (new FunctionContext(flow_context())) + ->column( + cast(ref('on'), type_optional(type_string())), + [['on' => new DateTimeImmutable('2026-01-02')], ['on' => null]], + schema(date_schema('on', nullable: true)), + ) + ->values(), + ); + } + + public function test_casting_a_null_datetime_to_string_is_refused_with_its_row_index(): void + { + try { + (new FunctionContext(flow_context()))->column( + cast(ref('at'), 'string'), + [['at' => new DateTimeImmutable('2026-01-02')], ['at' => null]], + schema(datetime_schema('at', nullable: true)), + ); + static::fail('a null datetime was cast to a non-nullable string'); + } catch (EvaluationException $e) { + static::assertSame(1, $e->rowIndex); + static::assertInstanceOf(InvalidArgumentException::class, $e->getPrevious()); + static::assertSame('Cast function requires non-null value', $e->getPrevious()->getMessage()); + } } public function test_cast_of_a_null_value_to_an_optional_target_is_null(): void { - static::assertNull(cast(ref('value'), type_optional(type_integer()))->eval(row([ - 'value' => null, - ]), flow_context())); + static::assertNull((new FunctionContext(flow_context()))->eval( + cast(ref('value'), type_optional(type_integer())), + [ + 'value' => null, + ], + schema(str_schema('value', nullable: true)), + )); + } + + public function test_a_refused_cast_keeps_the_casting_error_as_the_cause(): void + { + try { + (new FunctionContext(flow_context()))->eval( + cast(ref('value'), type_optional(type_datetime())), + ['value' => 'abc'], + schema(str_schema('value')), + ); + static::fail('An unconvertible value must be refused.'); + } catch (EvaluationException $e) { + static::assertInstanceOf(CastingException::class, $e->getPrevious()?->getPrevious()); + } } public function test_casting_an_unconvertible_value_to_an_optional_target_throws(): void @@ -49,7 +136,13 @@ public function test_casting_an_unconvertible_value_to_an_optional_target_throws $this->expectException(InvalidArgumentException::class); $this->expectExceptionMessage('Cast function failed: Can\'t cast "string" into "datetime" type'); - cast(ref('value'), type_optional(type_datetime()))->eval(row(['value' => 'abc']), flow_context()); + (new FunctionContext(flow_context()))->eval( + cast(ref('value'), type_optional(type_datetime())), + [ + 'value' => 'abc', + ], + schema(str_schema('value')), + ); } public function test_constructor_rejects_a_non_representable_target(): void @@ -62,8 +155,20 @@ public function test_constructor_rejects_a_non_representable_target(): void public function test_double_and_real_resolve_to_float(): void { - static::assertSame(1.0, cast(ref('value'), 'double')->eval(row(['value' => '1']), flow_context())); - static::assertSame(1.0, cast(ref('value'), 'real')->eval(row(['value' => '1']), flow_context())); + static::assertSame(1.0, (new FunctionContext(flow_context()))->eval( + cast(ref('value'), 'double'), + [ + 'value' => '1', + ], + schema(str_schema('value')), + )); + static::assertSame(1.0, (new FunctionContext(flow_context()))->eval( + cast(ref('value'), 'real'), + [ + 'value' => '1', + ], + schema(str_schema('value')), + )); } public function test_constructor_rejects_json_pretty(): void @@ -106,6 +211,9 @@ public static function cast_provider(): array $xml = new DOMDocument(); $xml->loadXML($xmlString = 'bar'); + $node = new DOMDocument('1.0', 'UTF-8'); + $node->appendChild(new DOMElement('row', 'zażółć')); + $fullXMLString = <<<'XML' bar @@ -113,52 +221,79 @@ public static function cast_provider(): array XML; return [ - 'int' => ['1', 'int', 1], - 'integer' => ['1', 'integer', 1], - 'float' => ['1', 'float', 1.0], - 'double' => ['1', 'double', 1.0], - 'real' => ['1', 'real', 1.0], - 'string' => [1, 'string', '1'], - 'bool' => ['1', 'bool', true], - 'boolean' => ['1', 'boolean', true], - 'json' => [[1], 'json', new Json('[1]')], + 'int' => ['1', str_schema('value'), 'int', 1], + 'integer' => ['1', str_schema('value'), 'integer', 1], + 'float' => ['1', str_schema('value'), 'float', 1.0], + 'double' => ['1', str_schema('value'), 'double', 1.0], + 'real' => ['1', str_schema('value'), 'real', 1.0], + 'string' => [1, int_schema('value'), 'string', '1'], + 'bool' => ['1', str_schema('value'), 'bool', true], + 'boolean' => ['1', str_schema('value'), 'boolean', true], + 'json' => [[1], list_schema('value', type_list(type_integer())), 'json', new Json('[1]')], 'xml_to_array' => [ $xml, + xml_schema('value'), 'array', ['root' => ['foo' => ['@attributes' => ['baz' => 'buz'], '@value' => 'bar']]], ], - 'string_to_xml' => [$xmlString, 'xml', $xml], - 'xml_to_string' => [$xml, 'string', 'bar'], - 'full_xml_to_string' => [$fullXMLString, 'string', $fullXMLString], - 'datetime' => [new DateTimeImmutable('2023-01-01 00:00:00 UTC'), 'string', '2023-01-01T00:00:00+00:00'], + 'string_to_xml' => [$xmlString, str_schema('value'), 'xml', $xml], + 'xml_to_string' => [$xml, xml_schema('value'), 'string', 'bar'], + 'xml_node_document_to_string' => [$node, xml_schema('value'), 'string', 'zażółć'], + 'full_xml_to_string' => [$fullXMLString, str_schema('value'), 'string', $fullXMLString], + 'datetime' => [ + new DateTimeImmutable('2023-01-01 00:00:00 UTC'), + datetime_schema('value'), + 'string', + '2023-01-01T00:00:00+00:00', + ], 'datetime_to_date' => [ new DateTimeImmutable('2023-01-01 00:01:00 UTC'), + datetime_schema('value'), 'date', new DateTimeImmutable('2023-01-01T00:00:00+00:00'), ], - 'string_to_timezone' => ['UTC', 'timezone', new DateTimeZone('UTC')], - 'string_to_timezone_america' => ['America/New_York', 'timezone', new DateTimeZone('America/New_York')], + 'string_to_timezone' => ['UTC', str_schema('value'), 'timezone', new DateTimeZone('UTC')], + 'string_to_timezone_america' => [ + 'America/New_York', + str_schema('value'), + 'timezone', + new DateTimeZone('America/New_York'), + ], + // a UTC datetime column, so the value reads back under the column zone 'datetime_to_timezone' => [ new DateTimeImmutable('2023-01-01 00:00:00', new DateTimeZone('Europe/London')), + datetime_schema('value'), 'timezone', - new DateTimeZone('Europe/London'), + new DateTimeZone('UTC'), ], 'uuid' => [ Uuid::fromString('a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a11'), + uuid_schema('value'), 'string', 'a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a11', ], - 'bool_to_string' => [true, 'string', 'true'], + 'bool_to_string' => [true, bool_schema('value'), 'string', 'true'], ]; } + /** + * @param Definition $definition + */ #[DataProvider('cast_provider')] - public function test_cast(mixed $from, string $to, mixed $expected): void + public function test_cast(mixed $from, Definition $definition, string $to, mixed $expected): void { // @mago-ignore analysis:mixed-assignment - $resultRefCast = ref('value')->cast($to)->eval(row(['value' => $from]), flow_context()); + $resultRefCast = (new FunctionContext(flow_context()))->eval( + ref('value')->cast($to), + ['value' => $from], + schema($definition), + ); // @mago-ignore analysis:mixed-assignment - $resultCastRef = cast(ref('value'), $to)->eval(row(['value' => $from]), flow_context()); + $resultCastRef = (new FunctionContext(flow_context()))->eval( + cast(ref('value'), $to), + ['value' => $from], + schema($definition), + ); if (is_object($expected) || is_object($from)) { static::assertEquals($expected, $resultRefCast); @@ -176,7 +311,11 @@ public function test_casting_a_scalar_to_array_names_the_remedy(): void 'Cast function failed: Can\'t cast "string" into "array" type: wrap the value first, e.g. type_list(type_string())', ); - ref('value')->cast('array')->eval(row(['value' => '1']), flow_context()); + (new FunctionContext(flow_context()))->eval( + ref('value')->cast('array'), + ['value' => '1'], + schema(str_schema('value')), + ); } public function test_casting_integer_to_timezone(): void @@ -184,7 +323,11 @@ public function test_casting_integer_to_timezone(): void $this->expectException(InvalidArgumentException::class); $this->expectExceptionMessage('Cast function failed: Can\'t cast "int" into "timezone" type'); - ref('value')->cast('timezone')->eval(row(['value' => 123]), flow_context()); + (new FunctionContext(flow_context()))->eval( + ref('value')->cast('timezone'), + ['value' => 123], + schema(int_schema('value')), + ); } public function test_casting_integer_to_xml(): void @@ -192,7 +335,11 @@ public function test_casting_integer_to_xml(): void $this->expectException(InvalidArgumentException::class); $this->expectExceptionMessage('Cast function failed: Can\'t cast "int" into "xml" type'); - ref('value')->cast('xml')->eval(row(['value' => 1]), flow_context()); + (new FunctionContext(flow_context()))->eval( + ref('value')->cast('xml'), + ['value' => 1], + schema(int_schema('value')), + ); } public function test_casting_invalid_string_to_timezone(): void @@ -200,7 +347,11 @@ public function test_casting_invalid_string_to_timezone(): void $this->expectException(InvalidArgumentException::class); $this->expectExceptionMessage('Cast function failed: Can\'t cast "string" into "timezone" type'); - ref('value')->cast('timezone')->eval(row(['value' => 'invalid-timezone']), flow_context()); + (new FunctionContext(flow_context()))->eval( + ref('value')->cast('timezone'), + ['value' => 'invalid-timezone'], + schema(str_schema('value')), + ); } public function test_casting_non_xml_string_to_xml(): void @@ -208,6 +359,10 @@ public function test_casting_non_xml_string_to_xml(): void $this->expectException(InvalidArgumentException::class); $this->expectExceptionMessage('Cast function failed: Can\'t cast "string" into "xml" type'); - ref('value')->cast('xml')->eval(row(['value' => 'foo']), flow_context()); + (new FunctionContext(flow_context()))->eval( + ref('value')->cast('xml'), + ['value' => 'foo'], + schema(str_schema('value')), + ); } } diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/ChunkTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/ChunkTest.php index ab3bb5790f..3e0c5ce6b1 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/ChunkTest.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/ChunkTest.php @@ -5,27 +5,47 @@ namespace Flow\ETL\Tests\Unit\Function; use Flow\ETL\Exception\InvalidArgumentException; +use Flow\ETL\Tests\Context\FunctionContext; use Flow\ETL\Tests\FlowTestCase; use function Flow\ETL\DSL\flow_context; +use function Flow\ETL\DSL\int_schema; use function Flow\ETL\DSL\ref; -use function Flow\ETL\DSL\row; +use function Flow\ETL\DSL\schema; +use function Flow\ETL\DSL\str_schema; final class ChunkTest extends FlowTestCase { public function test_chunk_empty_string(): void { - static::assertSame([], ref('str')->chunk(3)->eval(row(['str' => '']), flow_context())); + static::assertSame( + [], + (new FunctionContext(flow_context()))->eval(ref('str')->chunk(3), ['str' => ''], schema(str_schema('str'))), + ); } public function test_chunk_equal_to_string_length(): void { - static::assertSame(['hello'], ref('str')->chunk(5)->eval(row(['str' => 'hello']), flow_context())); + static::assertSame( + ['hello'], + (new FunctionContext(flow_context()))->eval( + ref('str')->chunk(5), + ['str' => 'hello'], + schema(str_schema('str')), + ), + ); } public function test_chunk_larger_than_string_length(): void { - static::assertSame(['hello'], ref('str')->chunk(10)->eval(row(['str' => 'hello']), flow_context())); + static::assertSame( + ['hello'], + (new FunctionContext(flow_context()))->eval( + ref('str')->chunk(10), + ['str' => 'hello'], + schema(str_schema('str')), + ), + ); } public function test_chunk_negative_size(): void @@ -33,7 +53,11 @@ public function test_chunk_negative_size(): void $this->expectException(InvalidArgumentException::class); $this->expectExceptionMessage('Chunk function requires non-null, positive size'); - ref('str')->chunk(-1)->eval(row(['str' => 'hello']), flow_context()); + (new FunctionContext(flow_context()))->eval( + ref('str')->chunk(-1), + ['str' => 'hello'], + schema(str_schema('str')), + ); } public function test_chunk_null_input(): void @@ -41,12 +65,23 @@ public function test_chunk_null_input(): void $this->expectException(InvalidArgumentException::class); $this->expectExceptionMessage('Chunk function requires non-null value'); - ref('str')->chunk(3)->eval(row(['str' => null]), flow_context()); + (new FunctionContext(flow_context()))->eval( + ref('str')->chunk(3), + ['str' => null], + schema(str_schema('str', nullable: true)), + ); } public function test_chunk_string_basic_functionality(): void { - static::assertSame(['hel', 'lo '], ref('str')->chunk(3)->eval(row(['str' => 'hello ']), flow_context())); + static::assertSame( + ['hel', 'lo '], + (new FunctionContext(flow_context()))->eval( + ref('str')->chunk(3), + ['str' => 'hello '], + schema(str_schema('str')), + ), + ); } public function test_chunk_with_null_size(): void @@ -54,14 +89,22 @@ public function test_chunk_with_null_size(): void $this->expectException(InvalidArgumentException::class); $this->expectExceptionMessage('Chunk function requires non-null, positive size'); - ref('str')->chunk(ref('size'))->eval(row(['str' => 'hello', 'size' => null]), flow_context()); + (new FunctionContext(flow_context()))->eval( + ref('str')->chunk(ref('size')), + ['str' => 'hello', 'size' => null], + schema(str_schema('str'), str_schema('size', nullable: true)), + ); } public function test_chunk_with_scalar_function_size(): void { static::assertSame( ['he', 'll', 'o'], - ref('str')->chunk(ref('size'))->eval(row(['str' => 'hello', 'size' => 2]), flow_context()), + (new FunctionContext(flow_context()))->eval( + ref('str')->chunk(ref('size')), + ['str' => 'hello', 'size' => 2], + schema(str_schema('str'), int_schema('size')), + ), ); } @@ -70,6 +113,10 @@ public function test_chunk_zero_size(): void $this->expectException(InvalidArgumentException::class); $this->expectExceptionMessage('Chunk function requires non-null, positive size'); - ref('str')->chunk(0)->eval(row(['str' => 'hello']), flow_context()); + (new FunctionContext(flow_context()))->eval( + ref('str')->chunk(0), + ['str' => 'hello'], + schema(str_schema('str')), + ); } } diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/CoalesceTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/CoalesceTest.php index 9f6198fe5b..3c0e133457 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/CoalesceTest.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/CoalesceTest.php @@ -7,13 +7,16 @@ use Flow\ETL\Exception\InvalidArgumentException; use Flow\ETL\Function\ReferenceResolver; use Flow\ETL\Function\ScalarFunction; +use Flow\ETL\Tests\Context\FunctionContext; +use Flow\ETL\Tests\Double\FailingOnValuesFunction; use Flow\ETL\Tests\FlowTestCase; +use function Flow\ETL\DSL\array_to_rows; use function Flow\ETL\DSL\coalesce; use function Flow\ETL\DSL\flow_context; +use function Flow\ETL\DSL\int_schema; use function Flow\ETL\DSL\lit; use function Flow\ETL\DSL\ref; -use function Flow\ETL\DSL\row; use function Flow\ETL\DSL\schema; use function Flow\ETL\DSL\str_schema; @@ -32,31 +35,63 @@ public function test_a_non_nullable_branch_makes_the_result_not_nullable(): void public function test_coalesce_entries(): void { - static::assertSame(1, coalesce(ref('name'), ref('id'), lit('N/A'))->eval(row([ - 'name' => null, - 'id' => 1, - ]), flow_context())); + // coalesce() of string and integer declares string: the value is cast to it + static::assertSame('1', (new FunctionContext(flow_context()))->eval( + coalesce(ref('name'), ref('id'), lit('N/A')), + [ + 'name' => null, + 'id' => 1, + ], + schema(str_schema('name', nullable: true), int_schema('id')), + )); } public function test_coalesce_on_null_entries_falls_through_to_lit(): void { - static::assertSame('N/A', coalesce(ref('name'), ref('string'), lit('N/A'))->eval(row([ - 'name' => null, - 'string' => null, - ]), flow_context())); + static::assertSame('N/A', (new FunctionContext(flow_context()))->eval( + coalesce(ref('name'), ref('string'), lit('N/A')), + [ + 'name' => null, + 'string' => null, + ], + schema(str_schema('name', nullable: true), str_schema('string', nullable: true)), + )); } public function test_coalesce_on_ref(): void { - static::assertSame(1, ref('name') - ->coalesce(ref('id'), lit('N/A')) - ->eval(row(['name' => null, 'id' => 1]), flow_context())); + // coalesce() of string and integer declares string: the value is cast to it + static::assertSame('1', (new FunctionContext(flow_context()))->eval( + ref('name')->coalesce(ref('id'), lit('N/A')), + ['name' => null, 'id' => 1], + schema(str_schema('name', nullable: true), int_schema('id')), + )); } public function test_a_throwing_branch_is_not_swallowed(): void { $this->expectException(InvalidArgumentException::class); - coalesce(ref('a')->upper(), ref('b')->upper())->eval(row(['a' => 1, 'b' => 2]), flow_context()); + (new FunctionContext(flow_context()))->eval( + coalesce(ref('a')->upper(), ref('b')->upper()), + ['a' => 1, 'b' => 2], + schema(int_schema('a'), int_schema('b')), + ); + } + + public function test_an_unreached_argument_that_would_throw_is_not_evaluated(): void + { + $rows = array_to_rows( + [['a' => 1, 'v' => 99], ['a' => null, 'v' => 2]], + schema(int_schema('a', nullable: true), int_schema('v')), + ); + + static::assertSame( + [1, 2], + (new ReferenceResolver()) + ->resolve(coalesce(ref('a'), new FailingOnValuesFunction(ref('v'), [99])), $rows->schema()) + ->eval($rows, flow_context()) + ->values(), + ); } } diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/CodePointLengthTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/CodePointLengthTest.php index 290321d443..d7e7d8ea7a 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/CodePointLengthTest.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/CodePointLengthTest.php @@ -5,22 +5,32 @@ namespace Flow\ETL\Tests\Unit\Function; use Flow\ETL\Exception\InvalidArgumentException; +use Flow\ETL\Tests\Context\FunctionContext; use Flow\ETL\Tests\FlowTestCase; use function Flow\ETL\DSL\flow_context; use function Flow\ETL\DSL\ref; -use function Flow\ETL\DSL\row; +use function Flow\ETL\DSL\schema; +use function Flow\ETL\DSL\str_schema; final class CodePointLengthTest extends FlowTestCase { public function test_code_point_length_ascii_string(): void { - static::assertSame(5, ref('str')->codePointLength()->eval(row(['str' => 'hello']), flow_context())); + static::assertSame(5, (new FunctionContext(flow_context()))->eval( + ref('str')->codePointLength(), + ['str' => 'hello'], + schema(str_schema('str')), + )); } public function test_code_point_length_empty_string(): void { - static::assertSame(0, ref('str')->codePointLength()->eval(row(['str' => '']), flow_context())); + static::assertSame(0, (new FunctionContext(flow_context()))->eval( + ref('str')->codePointLength(), + ['str' => ''], + schema(str_schema('str')), + )); } public function test_code_point_length_throws_on_null_input(): void @@ -28,6 +38,10 @@ public function test_code_point_length_throws_on_null_input(): void $this->expectException(InvalidArgumentException::class); $this->expectExceptionMessage('CodePointLength function requires non-null value'); - ref('str')->codePointLength()->eval(row(['str' => null]), flow_context()); + (new FunctionContext(flow_context()))->eval( + ref('str')->codePointLength(), + ['str' => null], + schema(str_schema('str', nullable: true)), + ); } } diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/CollapseWhitespaceTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/CollapseWhitespaceTest.php index 6a46f796c3..5b4ff74f47 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/CollapseWhitespaceTest.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/CollapseWhitespaceTest.php @@ -5,47 +5,59 @@ namespace Flow\ETL\Tests\Unit\Function; use Flow\ETL\Exception\InvalidArgumentException; +use Flow\ETL\Tests\Context\FunctionContext; use Flow\ETL\Tests\FlowTestCase; use function Flow\ETL\DSL\flow_context; use function Flow\ETL\DSL\ref; -use function Flow\ETL\DSL\row; +use function Flow\ETL\DSL\schema; +use function Flow\ETL\DSL\str_schema; final class CollapseWhitespaceTest extends FlowTestCase { public function test_empty_string(): void { - $result = ref('str')->collapseWhitespace()->eval(row(['str' => '']), flow_context()); - - static::assertEquals('', $result); + static::assertEquals('', (new FunctionContext(flow_context()))->eval( + ref('str')->collapseWhitespace(), + ['str' => ''], + schema(str_schema('str')), + )); } public function test_leading_and_trailing_whitespace(): void { - $result = ref('str')->collapseWhitespace()->eval(row(['str' => ' Hello world ']), flow_context()); - - static::assertEquals('Hello world', $result); + static::assertEquals('Hello world', (new FunctionContext(flow_context()))->eval( + ref('str')->collapseWhitespace(), + ['str' => ' Hello world '], + schema(str_schema('str')), + )); } public function test_leading_whitespace(): void { - $result = ref('str')->collapseWhitespace()->eval(row(['str' => ' Hello world']), flow_context()); - - static::assertEquals('Hello world', $result); + static::assertEquals('Hello world', (new FunctionContext(flow_context()))->eval( + ref('str')->collapseWhitespace(), + ['str' => ' Hello world'], + schema(str_schema('str')), + )); } public function test_mixed_whitespace_types(): void { - $result = ref('str')->collapseWhitespace()->eval(row(['str' => "Hello\t\tworld\n\ntest"]), flow_context()); - - static::assertEquals('Hello world test', $result); + static::assertEquals('Hello world test', (new FunctionContext(flow_context()))->eval( + ref('str')->collapseWhitespace(), + ['str' => "Hello\t\tworld\n\ntest"], + schema(str_schema('str')), + )); } public function test_multiple_spaces_between_words(): void { - $result = ref('str')->collapseWhitespace()->eval(row(['str' => 'Hello world test']), flow_context()); - - static::assertEquals('Hello world test', $result); + static::assertEquals('Hello world test', (new FunctionContext(flow_context()))->eval( + ref('str')->collapseWhitespace(), + ['str' => 'Hello world test'], + schema(str_schema('str')), + )); } public function test_null_value(): void @@ -53,29 +65,37 @@ public function test_null_value(): void $this->expectException(InvalidArgumentException::class); $this->expectExceptionMessage('CollapseWhitespace function requires non-null value'); - $result = ref('str')->collapseWhitespace()->eval(row(['str' => null]), flow_context()); - - static::assertNull($result); + static::assertNull((new FunctionContext(flow_context()))->eval( + ref('str')->collapseWhitespace(), + ['str' => null], + schema(str_schema('str', nullable: true)), + )); } public function test_single_spaces(): void { - $result = ref('str')->collapseWhitespace()->eval(row(['str' => 'Hello world test']), flow_context()); - - static::assertEquals('Hello world test', $result); + static::assertEquals('Hello world test', (new FunctionContext(flow_context()))->eval( + ref('str')->collapseWhitespace(), + ['str' => 'Hello world test'], + schema(str_schema('str')), + )); } public function test_single_word(): void { - $result = ref('str')->collapseWhitespace()->eval(row(['str' => 'Hello']), flow_context()); - - static::assertEquals('Hello', $result); + static::assertEquals('Hello', (new FunctionContext(flow_context()))->eval( + ref('str')->collapseWhitespace(), + ['str' => 'Hello'], + schema(str_schema('str')), + )); } public function test_trailing_whitespace(): void { - $result = ref('str')->collapseWhitespace()->eval(row(['str' => 'Hello world ']), flow_context()); - - static::assertEquals('Hello world', $result); + static::assertEquals('Hello world', (new FunctionContext(flow_context()))->eval( + ref('str')->collapseWhitespace(), + ['str' => 'Hello world '], + schema(str_schema('str')), + )); } } diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/CollectTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/CollectTest.php index 77516fe010..dcf73348cb 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/CollectTest.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/CollectTest.php @@ -4,15 +4,19 @@ namespace Flow\ETL\Tests\Unit\Function; +use Flow\ETL\Exception\InvalidArgumentException; use Flow\ETL\Function\ReferenceResolver; use Flow\ETL\Tests\FlowTestCase; +use function array_map; +use function Flow\ETL\DSL\array_to_rows; use function Flow\ETL\DSL\collect; +use function Flow\ETL\DSL\count; use function Flow\ETL\DSL\flow_context; use function Flow\ETL\DSL\int_schema; use function Flow\ETL\DSL\ref; -use function Flow\ETL\DSL\row; use function Flow\ETL\DSL\schema; +use function Flow\ETL\DSL\str_schema; final class CollectTest extends FlowTestCase { @@ -25,10 +29,10 @@ public function test_aggregation_collect_entry_values(): void { $aggregator = collect(ref('data')); - $aggregator->aggregate(row(['data' => 'a']), flow_context()); - $aggregator->aggregate(row(['data' => 'b']), flow_context()); - $aggregator->aggregate(row(['data' => 'b']), flow_context()); - $aggregator->aggregate(row(['data' => 'c']), flow_context()); + $aggregator->aggregate(array_to_rows([['data' => 'a']], schema(str_schema('data'))), [0], flow_context()); + $aggregator->aggregate(array_to_rows([['data' => 'b']], schema(str_schema('data'))), [0], flow_context()); + $aggregator->aggregate(array_to_rows([['data' => 'b']], schema(str_schema('data'))), [0], flow_context()); + $aggregator->aggregate(array_to_rows([['data' => 'c']], schema(str_schema('data'))), [0], flow_context()); static::assertSame( [ @@ -64,4 +68,48 @@ public function test_a_nullable_source_column_yields_a_nullable_element_type(): static::assertSame('?list', $resolved->returns()->toString()); } + + public function test_aggregate_reads_only_given_indices(): void + { + $aggregator = collect(ref('v')); + $aggregator->aggregate( + array_to_rows(array_map(static fn(mixed $v): array => ['v' => $v], [ + 'a', + 'b', + 'c', + 'd', + ]), schema(str_schema('v'))), + [1, 3], + flow_context(), + ); + + static::assertSame(['b', 'd'], $aggregator->value()); + } + + public function test_merge_of_two_halves_equals_one_pass(): void + { + $rows = array_to_rows(array_map(static fn(mixed $v): array => ['v' => $v], [ + 'a', + 'b', + 'c', + 'd', + ]), schema(str_schema('v'))); + $onePass = collect(ref('v')); + $onePass->aggregate($rows, [0, 1, 2, 3], flow_context()); + $left = collect(ref('v')); + $left->aggregate($rows, [0, 1], flow_context()); + $right = collect(ref('v')); + $right->aggregate($rows, [2, 3], flow_context()); + + $left->merge($right, flow_context()); + + static::assertSame($onePass->value(), $left->value()); + } + + public function test_merge_refuses_another_class(): void + { + $this->expectException(InvalidArgumentException::class); + + collect(ref('v'))->merge(count(ref('v')), flow_context()); + } } diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/CollectUniqueTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/CollectUniqueTest.php index 9132bdd1f9..0e3d41e024 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/CollectUniqueTest.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/CollectUniqueTest.php @@ -4,12 +4,17 @@ namespace Flow\ETL\Tests\Unit\Function; +use Flow\ETL\Exception\InvalidArgumentException; use Flow\ETL\Tests\FlowTestCase; +use function array_map; +use function Flow\ETL\DSL\array_to_rows; +use function Flow\ETL\DSL\collect; use function Flow\ETL\DSL\collect_unique; use function Flow\ETL\DSL\flow_context; use function Flow\ETL\DSL\ref; -use function Flow\ETL\DSL\row; +use function Flow\ETL\DSL\schema; +use function Flow\ETL\DSL\str_schema; final class CollectUniqueTest extends FlowTestCase { @@ -22,10 +27,10 @@ public function test_aggregation_collect_unique_values(): void { $aggregator = collect_unique(ref('data')); - $aggregator->aggregate(row(['data' => 'a']), flow_context()); - $aggregator->aggregate(row(['data' => 'b']), flow_context()); - $aggregator->aggregate(row(['data' => 'b']), flow_context()); - $aggregator->aggregate(row(['data' => 'c']), flow_context()); + $aggregator->aggregate(array_to_rows([['data' => 'a']], schema(str_schema('data'))), [0], flow_context()); + $aggregator->aggregate(array_to_rows([['data' => 'b']], schema(str_schema('data'))), [0], flow_context()); + $aggregator->aggregate(array_to_rows([['data' => 'b']], schema(str_schema('data'))), [0], flow_context()); + $aggregator->aggregate(array_to_rows([['data' => 'c']], schema(str_schema('data'))), [0], flow_context()); static::assertSame( [ @@ -53,4 +58,48 @@ public function test_collecting_nothing_yields_an_empty_list(): void { static::assertSame([], collect_unique(ref('data'))->value()); } + + public function test_aggregate_reads_only_given_indices(): void + { + $aggregator = collect_unique(ref('v')); + $aggregator->aggregate( + array_to_rows(array_map(static fn(mixed $v): array => ['v' => $v], [ + 'a', + 'b', + 'a', + 'c', + ]), schema(str_schema('v'))), + [0, 2], + flow_context(), + ); + + static::assertSame(['a'], $aggregator->value()); + } + + public function test_merge_of_two_halves_equals_one_pass(): void + { + $rows = array_to_rows(array_map(static fn(mixed $v): array => ['v' => $v], [ + 'a', + 'b', + 'a', + 'c', + ]), schema(str_schema('v'))); + $onePass = collect_unique(ref('v')); + $onePass->aggregate($rows, [0, 1, 2, 3], flow_context()); + $left = collect_unique(ref('v')); + $left->aggregate($rows, [0, 1], flow_context()); + $right = collect_unique(ref('v')); + $right->aggregate($rows, [2, 3], flow_context()); + + $left->merge($right, flow_context()); + + static::assertSame($onePass->value(), $left->value()); + } + + public function test_merge_refuses_another_class(): void + { + $this->expectException(InvalidArgumentException::class); + + collect_unique(ref('v'))->merge(collect(ref('v')), flow_context()); + } } diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/CombineTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/CombineTest.php index 5362ead779..4b95bf8139 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/CombineTest.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/CombineTest.php @@ -8,6 +8,7 @@ use Flow\ETL\Exception\SchemaNotDerivableException; use Flow\ETL\Function\ReferenceResolver; use Flow\ETL\Function\ScalarFunction; +use Flow\ETL\Tests\Context\FunctionContext; use Flow\ETL\Tests\FlowTestCase; use function Flow\ETL\DSL\combine; @@ -15,7 +16,6 @@ use function Flow\ETL\DSL\list_schema; use function Flow\ETL\DSL\lit; use function Flow\ETL\DSL\ref; -use function Flow\ETL\DSL\row; use function Flow\ETL\DSL\schema; use function Flow\ETL\DSL\str_schema; use function Flow\Types\DSL\type_list; @@ -41,13 +41,13 @@ public function test_array_combine(): void { static::assertSame( ['a' => 1, 'b' => 2, 'c' => 3], - combine(lit(['a', 'b', 'c']), lit([1, 2, 3]))->eval(row([]), flow_context()), + (new FunctionContext(flow_context()))->eval(combine(lit(['a', 'b', 'c']), lit([1, 2, 3])), [], schema()), ); } public function test_array_combine_when_arrays_are_empty(): void { - static::assertSame([], combine(lit([]), lit([]))->eval(row([]), flow_context())); + static::assertSame([], (new FunctionContext(flow_context()))->eval(combine(lit([]), lit([])), [], schema())); } public function test_array_combine_when_keys_are_not_array(): void @@ -55,14 +55,18 @@ public function test_array_combine_when_keys_are_not_array(): void $this->expectException(InvalidArgumentException::class); $this->expectExceptionMessage('Expected type "array", got "string".'); - combine(lit('a'), lit([1, 2, 3]))->eval(row([]), flow_context()); + (new FunctionContext(flow_context()))->eval(combine(lit('a'), lit([1, 2, 3])), [], schema()); } public function test_array_combine_when_keys_are_not_unique(): void { static::assertSame( ['a' => 4, 'b' => 2, 'c' => 3], - combine(lit(['a', 'b', 'c', 'a']), lit([1, 2, 3, 4]))->eval(row([]), flow_context()), + (new FunctionContext(flow_context()))->eval( + combine(lit(['a', 'b', 'c', 'a']), lit([1, 2, 3, 4])), + [], + schema(), + ), ); } @@ -71,6 +75,6 @@ public function test_array_combine_when_one_of_arrays_is_empty(): void $this->expectException(InvalidArgumentException::class); $this->expectExceptionMessage('Combine function requires keys and values arrays to have the same length'); - combine(lit(['a', 'b', 'c']), lit([]))->eval(row([]), flow_context()); + (new FunctionContext(flow_context()))->eval(combine(lit(['a', 'b', 'c']), lit([])), [], schema()); } } diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/ConcatTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/ConcatTest.php index c7c3bf40bb..33f944c3e3 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/ConcatTest.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/ConcatTest.php @@ -4,34 +4,47 @@ namespace Flow\ETL\Tests\Unit\Function; +use Flow\ETL\Tests\Context\FunctionContext; use Flow\ETL\Tests\FlowTestCase; use function Flow\ETL\DSL\concat; use function Flow\ETL\DSL\flow_context; +use function Flow\ETL\DSL\list_schema; use function Flow\ETL\DSL\lit; use function Flow\ETL\DSL\ref; -use function Flow\ETL\DSL\row; +use function Flow\ETL\DSL\schema; +use function Flow\Types\DSL\type_list; +use function Flow\Types\DSL\type_string; final class ConcatTest extends FlowTestCase { public function test_concat_arrays(): void { - static::assertSame('["a"]["b","c"]', concat(ref('array_1'), ref('array_2'))->eval(row([ - 'array_1' => ['a'], - 'array_2' => ['b', 'c'], - ]), flow_context())); + static::assertSame('["a"]["b","c"]', (new FunctionContext(flow_context()))->eval( + concat(ref('array_1'), ref('array_2')), + [ + 'array_1' => ['a'], + 'array_2' => ['b', 'c'], + ], + schema(list_schema('array_1', type_list(type_string())), list_schema('array_2', type_list(type_string()))), + )); } public function test_concat_different_types_of_values(): void { - static::assertSame('1abc["a","b"]', concat(lit(1), lit('a'), lit('b'), lit('c'), lit(['a', 'b']))->eval( - row([]), - flow_context(), + static::assertSame('1abc["a","b"]', (new FunctionContext(flow_context()))->eval( + concat(lit(1), lit('a'), lit('b'), lit('c'), lit(['a', 'b'])), + [], + schema(), )); } public function test_concat_string_values(): void { - static::assertSame('abc', concat(lit('a'), lit('b'), lit('c'))->eval(row([]), flow_context())); + static::assertSame('abc', (new FunctionContext(flow_context()))->eval( + concat(lit('a'), lit('b'), lit('c')), + [], + schema(), + )); } } diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/ConcatWithSeparatorTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/ConcatWithSeparatorTest.php index 8a536b2036..ad6e12fcf9 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/ConcatWithSeparatorTest.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/ConcatWithSeparatorTest.php @@ -4,29 +4,40 @@ namespace Flow\ETL\Tests\Unit\Function; +use Flow\ETL\Tests\Context\FunctionContext; use Flow\ETL\Tests\FlowTestCase; use function Flow\ETL\DSL\concat_ws; use function Flow\ETL\DSL\flow_context; +use function Flow\ETL\DSL\int_schema; use function Flow\ETL\DSL\lit; use function Flow\ETL\DSL\ref; -use function Flow\ETL\DSL\row; +use function Flow\ETL\DSL\schema; +use function Flow\ETL\DSL\str_schema; final class ConcatWithSeparatorTest extends FlowTestCase { public function test_concat_with_separator(): void { - static::assertSame('a,1', concat_ws(lit(','), ref('string'), ref('integer'))->eval(row([ - 'string' => 'a', - 'integer' => 1, - ]), flow_context())); + static::assertSame('a,1', (new FunctionContext(flow_context()))->eval( + concat_ws(lit(','), ref('string'), ref('integer')), + [ + 'string' => 'a', + 'integer' => 1, + ], + schema(str_schema('string'), int_schema('integer')), + )); } public function test_a_null_value_is_skipped(): void { - static::assertSame('a', concat_ws(lit(','), ref('string'), ref('missing_value'))->eval(row([ - 'string' => 'a', - 'missing_value' => null, - ]), flow_context())); + static::assertSame('a', (new FunctionContext(flow_context()))->eval( + concat_ws(lit(','), ref('string'), ref('missing_value')), + [ + 'string' => 'a', + 'missing_value' => null, + ], + schema(str_schema('string'), str_schema('missing_value', nullable: true)), + )); } } diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/CountTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/CountTest.php index a17be7e289..efbdfb3a88 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/CountTest.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/CountTest.php @@ -8,14 +8,17 @@ use Flow\ETL\Tests\FlowTestCase; use Flow\ETL\Tests\Mother\WindowContextMother; +use function array_map; +use function Flow\ETL\DSL\array_to_rows; use function Flow\ETL\DSL\config; use function Flow\ETL\DSL\count; +use function Flow\ETL\DSL\float_schema; use function Flow\ETL\DSL\flow_context; use function Flow\ETL\DSL\int_schema; use function Flow\ETL\DSL\ref; -use function Flow\ETL\DSL\row; -use function Flow\ETL\DSL\rows; use function Flow\ETL\DSL\schema; +use function Flow\ETL\DSL\str_schema; +use function Flow\ETL\DSL\sum; use function Flow\ETL\DSL\window; final class CountTest extends FlowTestCase @@ -34,11 +37,17 @@ public function test_aggregation_count_from_numeric_values(): void { $aggregator = count(ref('int')); - $aggregator->aggregate(row(['int' => '10']), flow_context()); - $aggregator->aggregate(row(['int' => '20']), flow_context()); - $aggregator->aggregate(row(['int' => '55']), flow_context()); - $aggregator->aggregate(row(['int' => '25']), flow_context()); - $aggregator->aggregate(row(['not_int' => null]), flow_context()); + $aggregator->aggregate(array_to_rows([['int' => '10']], schema(str_schema('int'))), [0], flow_context()); + $aggregator->aggregate(array_to_rows([['int' => '20']], schema(str_schema('int'))), [0], flow_context()); + $aggregator->aggregate(array_to_rows([['int' => '55']], schema(str_schema('int'))), [0], flow_context()); + $aggregator->aggregate(array_to_rows([['int' => '25']], schema(str_schema('int'))), [0], flow_context()); + $aggregator->aggregate( + array_to_rows([[ + 'not_int' => null, + ]], schema(str_schema('not_int', nullable: true))), + [0], + flow_context(), + ); static::assertSame(4, $aggregator->value()); } @@ -47,10 +56,10 @@ public function test_aggregation_count_with_float_result(): void { $aggregator = count(ref('int')); - $aggregator->aggregate(row(['int' => 10.25]), flow_context()); - $aggregator->aggregate(row(['int' => 20]), flow_context()); - $aggregator->aggregate(row(['int' => 305]), flow_context()); - $aggregator->aggregate(row(['int' => 25]), flow_context()); + $aggregator->aggregate(array_to_rows([['int' => 10.25]], schema(float_schema('int'))), [0], flow_context()); + $aggregator->aggregate(array_to_rows([['int' => 20]], schema(int_schema('int'))), [0], flow_context()); + $aggregator->aggregate(array_to_rows([['int' => 305]], schema(int_schema('int'))), [0], flow_context()); + $aggregator->aggregate(array_to_rows([['int' => 25]], schema(int_schema('int'))), [0], flow_context()); static::assertSame(4, $aggregator->value()); } @@ -59,11 +68,17 @@ public function test_aggregation_count_without_reference(): void { $aggregator = count(); - $aggregator->aggregate(row(['int' => '10']), flow_context()); - $aggregator->aggregate(row(['int' => '20']), flow_context()); - $aggregator->aggregate(row(['int' => '55']), flow_context()); - $aggregator->aggregate(row(['int' => '25']), flow_context()); - $aggregator->aggregate(row(['not_int' => null]), flow_context()); + $aggregator->aggregate(array_to_rows([['int' => '10']], schema(str_schema('int'))), [0], flow_context()); + $aggregator->aggregate(array_to_rows([['int' => '20']], schema(str_schema('int'))), [0], flow_context()); + $aggregator->aggregate(array_to_rows([['int' => '55']], schema(str_schema('int'))), [0], flow_context()); + $aggregator->aggregate(array_to_rows([['int' => '25']], schema(str_schema('int'))), [0], flow_context()); + $aggregator->aggregate( + array_to_rows([[ + 'not_int' => null, + ]], schema(str_schema('not_int', nullable: true))), + [0], + flow_context(), + ); static::assertSame(5, $aggregator->value()); } @@ -72,41 +87,49 @@ public function test_aggregation_when_row_does_not_have_entry(): void { $aggregator = count(ref('int')); - $aggregator->aggregate(row(['int' => 10]), flow_context()); - $aggregator->aggregate(row(['int' => 20]), flow_context()); - $aggregator->aggregate(row(['int' => 30]), flow_context()); - $aggregator->aggregate(row(['int' => null]), flow_context()); - $aggregator->aggregate(row(['test' => null]), flow_context()); + $aggregator->aggregate(array_to_rows([['int' => 10]], schema(int_schema('int'))), [0], flow_context()); + $aggregator->aggregate(array_to_rows([['int' => 20]], schema(int_schema('int'))), [0], flow_context()); + $aggregator->aggregate(array_to_rows([['int' => 30]], schema(int_schema('int'))), [0], flow_context()); + $aggregator->aggregate( + array_to_rows([[ + 'int' => null, + ]], schema(int_schema('int', nullable: true))), + [0], + flow_context(), + ); + $aggregator->aggregate( + array_to_rows([[ + 'test' => null, + ]], schema(str_schema('test', nullable: true))), + [0], + flow_context(), + ); static::assertSame(4, $aggregator->value()); } public function test_window_function_count_of_a_reference_skips_nulls(): void { - $rows = rows( + $rows = array_to_rows( + [['id' => 1, 'value' => 10], ['id' => 2, 'value' => null], ['id' => 3, 'value' => 30]], schema(int_schema('id'), int_schema('value', nullable: true)), - $row1 = row(['id' => 1, 'value' => 10]), - row(['id' => 2, 'value' => null]), - row(['id' => 3, 'value' => 30]), ); $count = count(ref('value'))->over(window()->orderBy(ref('id'))); - static::assertSame(2, $count->apply(WindowContextMother::forRow($row1, $rows))); + static::assertSame(2, $count->apply(WindowContextMother::atIndex($rows, 0))); } public function test_window_function_count_without_a_reference_counts_every_row_in_the_frame(): void { - $rows = rows( + $rows = array_to_rows( + [['id' => 1, 'value' => 10], ['id' => 2, 'value' => null], ['id' => 3, 'value' => 30]], schema(int_schema('id'), int_schema('value', nullable: true)), - $row1 = row(['id' => 1, 'value' => 10]), - row(['id' => 2, 'value' => null]), - row(['id' => 3, 'value' => 30]), ); $count = count()->over(window()->orderBy(ref('id'))); - static::assertSame(3, $count->apply(WindowContextMother::forRow($row1, $rows))); + static::assertSame(3, $count->apply(WindowContextMother::atIndex($rows, 0))); } public function test_window_function_count_with_missing_reference_in_strict_mode(): void @@ -114,12 +137,12 @@ public function test_window_function_count_with_missing_reference_in_strict_mode $this->expectException(InvalidArgumentException::class); $this->expectExceptionMessage('Count window function error:'); - $rows = rows(schema(int_schema('id')), $row1 = row(['id' => 1]), row(['id' => 2])); + $rows = array_to_rows([['id' => 1], ['id' => 2]], schema(int_schema('id'))); $count = count(ref('missing_column'))->over(window()->orderBy(ref('id'))); $context = flow_context(config()); - $count->apply(WindowContextMother::forRow($row1, $rows, context: $context)); + $count->apply(WindowContextMother::atIndex($rows, 0, context: $context)); } public function test_with_children_rebuilds_the_aggregate_with_the_given_reference(): void @@ -146,4 +169,43 @@ public function test_counting_nothing_is_zero(): void { static::assertSame(0, count(ref('int'))->value()); } + + public function test_aggregate_reads_only_given_indices(): void + { + $aggregator = count(ref('v')); + $aggregator->aggregate( + array_to_rows(array_map(static fn(mixed $v): array => ['v' => $v], [1, 2, 3, 4]), schema(int_schema('v'))), + [1, 3], + flow_context(), + ); + + static::assertSame(2, $aggregator->value()); + } + + public function test_merge_of_two_halves_equals_one_pass(): void + { + $rows = array_to_rows(array_map(static fn(mixed $v): array => ['v' => $v], [ + 1, + 2, + 3, + 4, + ]), schema(int_schema('v'))); + $onePass = count(ref('v')); + $onePass->aggregate($rows, [0, 1, 2, 3], flow_context()); + $left = count(ref('v')); + $left->aggregate($rows, [0, 1], flow_context()); + $right = count(ref('v')); + $right->aggregate($rows, [2, 3], flow_context()); + + $left->merge($right, flow_context()); + + static::assertSame($onePass->value(), $left->value()); + } + + public function test_merge_refuses_another_class(): void + { + $this->expectException(InvalidArgumentException::class); + + count(ref('v'))->merge(sum(ref('v')), flow_context()); + } } diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/DOMElementAttributeValueTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/DOMElementAttributeValueTest.php index 312063f7c6..ca38fdac04 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/DOMElementAttributeValueTest.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/DOMElementAttributeValueTest.php @@ -8,12 +8,15 @@ use Dom\HTMLElement; use DOMDocument; use DOMElement; +use Flow\ETL\Tests\Context\FunctionContext; use PHPUnit\Framework\Attributes\RequiresPhp; use PHPUnit\Framework\TestCase; use function Flow\ETL\DSL\flow_context; +use function Flow\ETL\DSL\html_element_schema; use function Flow\ETL\DSL\ref; -use function Flow\ETL\DSL\row; +use function Flow\ETL\DSL\schema; +use function Flow\ETL\DSL\xml_element_schema; use const LIBXML_HTML_NOIMPLIED; use const LIBXML_NOERROR; @@ -30,9 +33,13 @@ public function test_html_extracting_attribute_from_dom_element_entry(): void ); static::assertInstanceOf(HTMLElement::class, $element->documentElement); - static::assertEquals('foobar', ref('value') - ->domElementAttributeValue('id') - ->eval(row(['value' => $element->documentElement]), flow_context())); + static::assertEquals('foobar', (new FunctionContext(flow_context()))->eval( + ref('value')->domElementAttributeValue('id'), + [ + 'value' => $element->documentElement, + ], + schema(html_element_schema('value')), + )); } #[RequiresPhp('>= 8.4.0')] @@ -42,11 +49,13 @@ public function test_html_extracting_non_existing_attribute_from_dom_element_ent $element = HTMLDocument::createFromString('foobar', LIBXML_HTML_NOIMPLIED | LIBXML_NOERROR); static::assertInstanceOf(HTMLElement::class, $element->documentElement); - static::assertNull( - ref('value') - ->domElementAttributeValue('id') - ->eval(row(['value' => $element->documentElement]), flow_context()), - ); + static::assertNull((new FunctionContext(flow_context()))->eval( + ref('value')->domElementAttributeValue('id'), + [ + 'value' => $element->documentElement, + ], + schema(html_element_schema('value')), + )); } public function test_xml_extracting_attribute_from_dom_element_entry(): void @@ -55,9 +64,13 @@ public function test_xml_extracting_attribute_from_dom_element_entry(): void $xml->loadXML('bar'); static::assertInstanceOf(DOMElement::class, $xml->documentElement); - static::assertEquals('buz', ref('value') - ->domElementAttributeValue('baz') - ->eval(row(['value' => $xml->documentElement->firstChild]), flow_context())); + static::assertEquals('buz', (new FunctionContext(flow_context()))->eval( + ref('value')->domElementAttributeValue('baz'), + [ + 'value' => $xml->documentElement->firstChild, + ], + schema(xml_element_schema('value')), + )); } public function test_xml_extracting_non_existing_attribute_from_dom_element_entry(): void @@ -66,10 +79,12 @@ public function test_xml_extracting_non_existing_attribute_from_dom_element_entr $xml->loadXML('bar'); static::assertInstanceOf(DOMElement::class, $xml->documentElement); - static::assertNull( - ref('value') - ->domElementAttributeValue('bar') - ->eval(row(['value' => $xml->documentElement->firstChild]), flow_context()), - ); + static::assertNull((new FunctionContext(flow_context()))->eval( + ref('value')->domElementAttributeValue('bar'), + [ + 'value' => $xml->documentElement->firstChild, + ], + schema(xml_element_schema('value')), + )); } } diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/DOMElementAttributesCountTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/DOMElementAttributesCountTest.php index e43c5b6960..2cc62b2ff1 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/DOMElementAttributesCountTest.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/DOMElementAttributesCountTest.php @@ -8,12 +8,15 @@ use Dom\HTMLElement; use DOMDocument; use DOMElement; +use Flow\ETL\Tests\Context\FunctionContext; use PHPUnit\Framework\Attributes\RequiresPhp; use PHPUnit\Framework\TestCase; use function Flow\ETL\DSL\flow_context; +use function Flow\ETL\DSL\html_element_schema; use function Flow\ETL\DSL\ref; -use function Flow\ETL\DSL\row; +use function Flow\ETL\DSL\schema; +use function Flow\ETL\DSL\xml_element_schema; use const LIBXML_HTML_NOIMPLIED; use const LIBXML_NOERROR; @@ -30,9 +33,13 @@ public function test_html_attributes_count_on_element_with_multiple_attributes() ); static::assertInstanceOf(HTMLElement::class, $element->documentElement); - static::assertSame(3, ref('value') - ->domElementAttributesCount() - ->eval(row(['value' => $element->documentElement]), flow_context())); + static::assertSame(3, (new FunctionContext(flow_context()))->eval( + ref('value')->domElementAttributesCount(), + [ + 'value' => $element->documentElement, + ], + schema(html_element_schema('value')), + )); } #[RequiresPhp('>= 8.4.0')] @@ -45,9 +52,13 @@ public function test_html_attributes_count_on_element_with_one_attribute(): void ); static::assertInstanceOf(HTMLElement::class, $element->documentElement); - static::assertSame(1, ref('value') - ->domElementAttributesCount() - ->eval(row(['value' => $element->documentElement]), flow_context())); + static::assertSame(1, (new FunctionContext(flow_context()))->eval( + ref('value')->domElementAttributesCount(), + [ + 'value' => $element->documentElement, + ], + schema(html_element_schema('value')), + )); } #[RequiresPhp('>= 8.4.0')] @@ -57,9 +68,13 @@ public function test_html_attributes_count_on_element_with_zero_attributes(): vo $element = HTMLDocument::createFromString('foobar', LIBXML_HTML_NOIMPLIED | LIBXML_NOERROR); static::assertInstanceOf(HTMLElement::class, $element->documentElement); - static::assertSame(0, ref('value') - ->domElementAttributesCount() - ->eval(row(['value' => $element->documentElement]), flow_context())); + static::assertSame(0, (new FunctionContext(flow_context()))->eval( + ref('value')->domElementAttributesCount(), + [ + 'value' => $element->documentElement, + ], + schema(html_element_schema('value')), + )); } public function test_xml_attributes_count_on_element_with_multiple_attributes(): void @@ -68,9 +83,13 @@ public function test_xml_attributes_count_on_element_with_multiple_attributes(): $xml->loadXML('bar'); static::assertInstanceOf(DOMElement::class, $xml->documentElement); - static::assertSame(3, ref('value') - ->domElementAttributesCount() - ->eval(row(['value' => $xml->documentElement->firstChild]), flow_context())); + static::assertSame(3, (new FunctionContext(flow_context()))->eval( + ref('value')->domElementAttributesCount(), + [ + 'value' => $xml->documentElement->firstChild, + ], + schema(xml_element_schema('value')), + )); } public function test_xml_attributes_count_on_element_with_one_attribute(): void @@ -79,9 +98,13 @@ public function test_xml_attributes_count_on_element_with_one_attribute(): void $xml->loadXML('bar'); static::assertInstanceOf(DOMElement::class, $xml->documentElement); - static::assertSame(1, ref('value') - ->domElementAttributesCount() - ->eval(row(['value' => $xml->documentElement->firstChild]), flow_context())); + static::assertSame(1, (new FunctionContext(flow_context()))->eval( + ref('value')->domElementAttributesCount(), + [ + 'value' => $xml->documentElement->firstChild, + ], + schema(xml_element_schema('value')), + )); } public function test_xml_attributes_count_on_element_with_zero_attributes(): void @@ -90,8 +113,12 @@ public function test_xml_attributes_count_on_element_with_zero_attributes(): voi $xml->loadXML('bar'); static::assertInstanceOf(DOMElement::class, $xml->documentElement); - static::assertSame(0, ref('value') - ->domElementAttributesCount() - ->eval(row(['value' => $xml->documentElement->firstChild]), flow_context())); + static::assertSame(0, (new FunctionContext(flow_context()))->eval( + ref('value')->domElementAttributesCount(), + [ + 'value' => $xml->documentElement->firstChild, + ], + schema(xml_element_schema('value')), + )); } } diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/DOMElementNamespaceTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/DOMElementNamespaceTest.php index 51c2f23568..193f6cbc02 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/DOMElementNamespaceTest.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/DOMElementNamespaceTest.php @@ -8,12 +8,14 @@ use Dom\XMLDocument; use DOMDocument; use DOMElement; +use Flow\ETL\Tests\Context\FunctionContext; use PHPUnit\Framework\Attributes\RequiresPhp; use PHPUnit\Framework\TestCase; use function Flow\ETL\DSL\flow_context; use function Flow\ETL\DSL\ref; -use function Flow\ETL\DSL\row; +use function Flow\ETL\DSL\schema; +use function Flow\ETL\DSL\xml_element_schema; use const LIBXML_HTML_NOIMPLIED; use const LIBXML_NOERROR; @@ -37,9 +39,15 @@ public function test_html_getting_element_namespace(): void $element = XMLDocument::createFromString(self::XML, LIBXML_HTML_NOIMPLIED | LIBXML_NOERROR); static::assertInstanceOf(Element::class, $element->documentElement); - static::assertSame('urn:oasis:names:specification:ubl:schema:xsd:Invoice-2', ref('node') - ->domElementNamespace() - ->eval(row(['node' => $element->documentElement]), flow_context())); + static::assertSame('urn:oasis:names:specification:ubl:schema:xsd:Invoice-2', (new FunctionContext( + flow_context(), + ))->eval( + ref('node')->domElementNamespace(), + [ + 'node' => $element->documentElement, + ], + schema(xml_element_schema('node')), + )); } public function test_xml_getting_element_namespace(): void @@ -48,9 +56,15 @@ public function test_xml_getting_element_namespace(): void $xml->loadXML(self::XML); static::assertInstanceOf(DOMElement::class, $xml->documentElement); - static::assertSame('urn:oasis:names:specification:ubl:schema:xsd:Invoice-2', ref('node') - ->domElementNamespace() - ->eval(row(['node' => $xml->documentElement]), flow_context())); + static::assertSame('urn:oasis:names:specification:ubl:schema:xsd:Invoice-2', (new FunctionContext( + flow_context(), + ))->eval( + ref('node')->domElementNamespace(), + [ + 'node' => $xml->documentElement, + ], + schema(xml_element_schema('node')), + )); } public function test_xml_getting_element_non_default_namespace(): void @@ -59,8 +73,14 @@ public function test_xml_getting_element_non_default_namespace(): void $xml->loadXML(self::XML); static::assertInstanceOf(DOMElement::class, $xml->documentElement); - static::assertSame('http://www.unece.org/cefact/nodes/StandardBusinessDocumentHeader', ref('node') - ->domElementNamespace('xmlns:sh') - ->eval(row(['node' => $xml->documentElement]), flow_context())); + static::assertSame('http://www.unece.org/cefact/nodes/StandardBusinessDocumentHeader', (new FunctionContext( + flow_context(), + ))->eval( + ref('node')->domElementNamespace('xmlns:sh'), + [ + 'node' => $xml->documentElement, + ], + schema(xml_element_schema('node')), + )); } } diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/DOMElementParentTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/DOMElementParentTest.php index 7e6ca3acc6..0f11e17a7b 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/DOMElementParentTest.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/DOMElementParentTest.php @@ -7,13 +7,18 @@ use Dom\HTMLDocument; use DOMDocument; use Flow\ETL\Exception\InvalidArgumentException; +use Flow\ETL\Tests\Context\FunctionContext; use PHPUnit\Framework\Attributes\RequiresPhp; use PHPUnit\Framework\TestCase; use function Flow\ETL\DSL\config; use function Flow\ETL\DSL\flow_context; +use function Flow\ETL\DSL\html_element_schema; +use function Flow\ETL\DSL\lit; use function Flow\ETL\DSL\ref; -use function Flow\ETL\DSL\row; +use function Flow\ETL\DSL\schema; +use function Flow\ETL\DSL\xml_element_schema; +use function Flow\ETL\DSL\xml_schema; use const LIBXML_HTML_NOIMPLIED; use const LIBXML_NOERROR; @@ -28,10 +33,14 @@ public function test_html_fails_when_parent_not_available_in_strict_mode(): void $context = flow_context(config()); $this->expectException(InvalidArgumentException::class); $this->expectExceptionMessage('DOMElementParent requires non-null DOMNode or HTMLElement.'); - ref('value') - ->domElementParent() - // @mago-ignore analysis:possibly-null-property-access - ->eval(row(['value' => $element->documentElement->parentElement]), $context); + (new FunctionContext($context))->eval( + ref('value')->domElementParent(), + [ + // @mago-ignore analysis:possibly-null-property-access + 'value' => $element->documentElement->parentElement, + ], + schema(html_element_schema('value', nullable: true)), + ); } #[RequiresPhp('>= 8.4.0')] @@ -44,7 +53,11 @@ public function test_html_getting_parent_element(): void ); static::assertEquals( $element->documentElement, - ref('value')->domElementParent()->eval(row(['value' => $element->querySelector('span')]), flow_context()), + (new FunctionContext(flow_context()))->eval( + lit($element->querySelector('span'))->domElementParent(), + [], + schema(), + ), ); } @@ -53,9 +66,11 @@ public function test_html_getting_parent_element_when_not_available(): void { // @mago-ignore analysis:unavailable-method $element = HTMLDocument::createFromString('bar', LIBXML_HTML_NOIMPLIED | LIBXML_NOERROR); - static::assertNull( - ref('value')->domElementParent()->eval(row(['value' => $element->documentElement]), flow_context()), - ); + static::assertNull((new FunctionContext(flow_context()))->eval( + lit($element->documentElement)->domElementParent(), + [], + schema(), + )); } public function test_xml_fails_when_parent_not_available_in_strict_mode(): void @@ -65,9 +80,13 @@ public function test_xml_fails_when_parent_not_available_in_strict_mode(): void $context = flow_context(config()); $this->expectException(InvalidArgumentException::class); $this->expectExceptionMessage('DOMElementParent requires non-null DOMNode or HTMLElement.'); - static::assertEquals($xml, ref('value') - ->domElementParent() - ->eval(row(['value' => $xml->parentNode]), $context)); + static::assertEquals($xml, (new FunctionContext($context))->eval( + ref('value')->domElementParent(), + [ + 'value' => $xml->parentNode, + ], + schema(xml_element_schema('value', nullable: true)), + )); } public function test_xml_getting_parent_element(): void @@ -76,13 +95,13 @@ public function test_xml_getting_parent_element(): void $xml->loadXML('foobar'); static::assertEquals( $xml->documentElement, - ref('value') - ->domElementParent() - ->eval( - // @mago-ignore analysis:possibly-null-property-access - row(['value' => $xml->documentElement->firstChild]), - flow_context(), - ), + // a batch stores the element's markup, so the parent is only reachable on the live node + // @mago-ignore analysis:possibly-null-property-access + (new FunctionContext(flow_context()))->eval( + lit($xml->documentElement->firstChild)->domElementParent(), + [], + schema(), + ), ); } @@ -90,15 +109,23 @@ public function test_xml_getting_parent_element_when_not_available(): void { $xml = new DOMDocument(); $xml->loadXML('foobar'); - static::assertNull( - ref('value')->domElementParent()->eval(row(['value' => $xml->documentElement]), flow_context()), - ); + static::assertNull((new FunctionContext(flow_context()))->eval( + ref('value')->domElementParent(), + [ + 'value' => $xml->documentElement, + ], + schema(xml_element_schema('value')), + )); } public function test_xml_getting_parent_element_when_passing_document(): void { $xml = new DOMDocument(); $xml->loadXML('foobar'); - static::assertNull(ref('value')->domElementParent()->eval(row(['value' => $xml]), flow_context())); + static::assertNull((new FunctionContext(flow_context()))->eval( + ref('value')->domElementParent(), + ['value' => $xml], + schema(xml_schema('value')), + )); } } diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/DOMElementValueTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/DOMElementValueTest.php index 04367e4640..a780580879 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/DOMElementValueTest.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/DOMElementValueTest.php @@ -8,12 +8,15 @@ use Dom\HTMLElement; use DOMDocument; use DOMElement; +use Flow\ETL\Tests\Context\FunctionContext; use Flow\ETL\Tests\FlowTestCase; use PHPUnit\Framework\Attributes\RequiresPhp; use function Flow\ETL\DSL\flow_context; +use function Flow\ETL\DSL\html_element_schema; use function Flow\ETL\DSL\ref; -use function Flow\ETL\DSL\row; +use function Flow\ETL\DSL\schema; +use function Flow\ETL\DSL\xml_element_schema; use const LIBXML_HTML_NOIMPLIED; use const LIBXML_NOERROR; @@ -27,9 +30,13 @@ public function test_html_getting_element_value_with_children(): void $element = HTMLDocument::createFromString('

foobar

', LIBXML_HTML_NOIMPLIED | LIBXML_NOERROR); static::assertInstanceOf(HTMLElement::class, $element->documentElement); - static::assertEquals('foobar', ref('value') - ->domElementValue() - ->eval(row(['value' => $element->documentElement]), flow_context())); + static::assertEquals('foobar', (new FunctionContext(flow_context()))->eval( + ref('value')->domElementValue(), + [ + 'value' => $element->documentElement, + ], + schema(html_element_schema('value')), + )); } #[RequiresPhp('>= 8.4.0')] @@ -39,9 +46,13 @@ public function test_html_getting_simple_element_value(): void $element = HTMLDocument::createFromString('bar', LIBXML_HTML_NOIMPLIED | LIBXML_NOERROR); static::assertInstanceOf(HTMLElement::class, $element->documentElement); - static::assertEquals('bar', ref('value') - ->domElementValue() - ->eval(row(['value' => $element->documentElement]), flow_context())); + static::assertEquals('bar', (new FunctionContext(flow_context()))->eval( + ref('value')->domElementValue(), + [ + 'value' => $element->documentElement, + ], + schema(html_element_schema('value')), + )); } public function test_xml_getting_element_value_with_children(): void @@ -50,9 +61,13 @@ public function test_xml_getting_element_value_with_children(): void $xml->loadXML('baz'); static::assertInstanceOf(DOMElement::class, $xml->documentElement); - static::assertEquals('baz', ref('value') - ->domElementValue() - ->eval(row(['value' => $xml->documentElement->firstChild]), flow_context())); + static::assertEquals('baz', (new FunctionContext(flow_context()))->eval( + ref('value')->domElementValue(), + [ + 'value' => $xml->documentElement->firstChild, + ], + schema(xml_element_schema('value')), + )); } public function test_xml_getting_simple_element_value(): void @@ -61,8 +76,12 @@ public function test_xml_getting_simple_element_value(): void $xml->loadXML('bar'); static::assertInstanceOf(DOMElement::class, $xml->documentElement); - static::assertEquals('bar', ref('value') - ->domElementValue() - ->eval(row(['value' => $xml->documentElement->firstChild]), flow_context())); + static::assertEquals('bar', (new FunctionContext(flow_context()))->eval( + ref('value')->domElementValue(), + [ + 'value' => $xml->documentElement->firstChild, + ], + schema(xml_element_schema('value')), + )); } } diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/DateTimeFormatTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/DateTimeFormatTest.php index 11db787d5d..ed81a721cc 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/DateTimeFormatTest.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/DateTimeFormatTest.php @@ -6,31 +6,92 @@ use DateTimeImmutable; use DateTimeZone; +use Flow\ETL\Exception\EvaluationException; use Flow\ETL\Exception\InvalidArgumentException; +use Flow\ETL\Tests\Context\FunctionContext; use Flow\ETL\Tests\FlowTestCase; +use function Flow\ETL\DSL\date_schema; use function Flow\ETL\DSL\date_time_format; +use function Flow\ETL\DSL\datetime_schema; use function Flow\ETL\DSL\flow_context; use function Flow\ETL\DSL\now; use function Flow\ETL\DSL\ref; -use function Flow\ETL\DSL\row; +use function Flow\ETL\DSL\schema; +use function Flow\ETL\DSL\str_schema; final class DateTimeFormatTest extends FlowTestCase { public function test_date_time_format(): void { - static::assertEquals('2020-01-01 00:00:00', date_time_format(ref('date_time'), 'Y-m-d H:i:s')->eval( - row(['date_time' => new DateTimeImmutable('2020-01-01 00:00:00', new DateTimeZone('UTC'))]), - flow_context(), + static::assertEquals('2020-01-01 00:00:00', (new FunctionContext(flow_context()))->eval( + date_time_format(ref('date_time'), 'Y-m-d H:i:s'), + ['date_time' => new DateTimeImmutable('2020-01-01 00:00:00', new DateTimeZone('UTC'))], + schema(datetime_schema('date_time')), )); } + public function test_a_literal_format_over_a_zoned_column_equals_formatting_each_value(): void + { + $rows = [ + ['at' => new DateTimeImmutable('2026-03-29T00:59:59.123456Z')], + ['at' => new DateTimeImmutable('2026-03-29T01:00:00Z')], + ]; + + foreach (['UTC', 'Europe/Warsaw'] as $zone) { + static::assertSame( + [ + $rows[0]['at']->setTimezone(new DateTimeZone($zone))->format('Y-m-d\TH:i:s.uP T e'), + $rows[1]['at']->setTimezone(new DateTimeZone($zone))->format('Y-m-d\TH:i:s.uP T e'), + ], + (new FunctionContext(flow_context())) + ->column( + date_time_format(ref('at'), 'Y-m-d\TH:i:s.uP T e'), + $rows, + schema(datetime_schema('at', zone: $zone)), + ) + ->values(), + ); + } + } + + public function test_a_literal_format_over_a_date_column(): void + { + static::assertSame( + ['02/01/2026 00:00', '01/01/1970 00:00'], + (new FunctionContext(flow_context())) + ->column( + date_time_format(ref('on'), 'd/m/Y H:i'), + [['on' => new DateTimeImmutable('2026-01-02')], ['on' => new DateTimeImmutable('1970-01-01')]], + schema(date_schema('on')), + ) + ->values(), + ); + } + + public function test_a_null_in_a_datetime_column_is_refused_with_its_row_index(): void + { + try { + (new FunctionContext(flow_context()))->column( + date_time_format(ref('at'), 'Y-m-d'), + [['at' => new DateTimeImmutable('2026-01-02')], ['at' => null], ['at' => null]], + schema(datetime_schema('at', nullable: true)), + ); + static::fail('a null datetime was formatted'); + } catch (EvaluationException $e) { + static::assertSame(1, $e->rowIndex); + static::assertInstanceOf(InvalidArgumentException::class, $e->getPrevious()); + static::assertSame('DateTimeFormat function requires non-null values', $e->getPrevious()->getMessage()); + } + } + public function test_formatting_now(): void { - static::assertInstanceOf(DateTimeImmutable::class, now()->eval(row(['date_time' => new DateTimeImmutable( - '2020-01-01 00:00:00', - new DateTimeZone('UTC'), - )]), flow_context())); + static::assertInstanceOf(DateTimeImmutable::class, (new FunctionContext(flow_context()))->eval( + now(), + ['date_time' => new DateTimeImmutable('2020-01-01 00:00:00', new DateTimeZone('UTC'))], + schema(datetime_schema('date_time')), + )); } public function test_invalid_date_time_format(): void @@ -38,8 +99,12 @@ public function test_invalid_date_time_format(): void $this->expectException(InvalidArgumentException::class); $this->expectExceptionMessage('Expected type "object", got "string".'); - date_time_format(ref('date_time'), 'Y-m-d H:i:s')->eval(row([ - 'date_time' => '2020-01-01 00:00:00', - ]), flow_context()); + (new FunctionContext(flow_context()))->eval( + date_time_format(ref('date_time'), 'Y-m-d H:i:s'), + [ + 'date_time' => '2020-01-01 00:00:00', + ], + schema(str_schema('date_time')), + ); } } diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/DenseRankTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/DenseRankTest.php index 052b8afa6f..97288077e8 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/DenseRankTest.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/DenseRankTest.php @@ -8,11 +8,10 @@ use Flow\ETL\Tests\FlowTestCase; use Flow\ETL\Tests\Mother\WindowContextMother; +use function Flow\ETL\DSL\array_to_rows; use function Flow\ETL\DSL\dense_rank; use function Flow\ETL\DSL\int_schema; use function Flow\ETL\DSL\ref; -use function Flow\ETL\DSL\row; -use function Flow\ETL\DSL\rows; use function Flow\ETL\DSL\schema; use function Flow\ETL\DSL\window; @@ -71,7 +70,7 @@ public function test_dense_rank_requires_an_order_by(): void { $this->expectExceptionMessage('Dens Rank window function requires to be ordered by one column'); - dense_rank()->over(window())->rankPartition(rows(schema(int_schema('salary')), row(['salary' => 6000]))); + dense_rank()->over(window())->rankPartition(array_to_rows([['salary' => 6000]], schema(int_schema('salary')))); } public function test_dense_rank_without_over_clause(): void diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/EnsureEndTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/EnsureEndTest.php index 25cc5732a4..92c440cab7 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/EnsureEndTest.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/EnsureEndTest.php @@ -5,26 +5,32 @@ namespace Flow\ETL\Tests\Unit\Function; use Flow\ETL\Exception\InvalidArgumentException; +use Flow\ETL\Tests\Context\FunctionContext; use Flow\ETL\Tests\FlowTestCase; use function Flow\ETL\DSL\flow_context; use function Flow\ETL\DSL\ref; -use function Flow\ETL\DSL\row; +use function Flow\ETL\DSL\schema; +use function Flow\ETL\DSL\str_schema; final class EnsureEndTest extends FlowTestCase { public function test_empty_string_with_suffix(): void { - $result = ref('str')->ensureEnd('_suffix')->eval(row(['str' => '']), flow_context()); - - static::assertEquals('_suffix', $result); + static::assertEquals('_suffix', (new FunctionContext(flow_context()))->eval( + ref('str')->ensureEnd('_suffix'), + ['str' => ''], + schema(str_schema('str')), + )); } public function test_null_suffix(): void { - $result = ref('str')->ensureEnd(ref('suffix'))->eval(row(['str' => 'hello', 'suffix' => null]), flow_context()); - - static::assertEquals('hello', $result); + static::assertEquals('hello', (new FunctionContext(flow_context()))->eval( + ref('str')->ensureEnd(ref('suffix')), + ['str' => 'hello', 'suffix' => null], + schema(str_schema('str'), str_schema('suffix', nullable: true)), + )); } public function test_null_value(): void @@ -32,38 +38,46 @@ public function test_null_value(): void $this->expectException(InvalidArgumentException::class); $this->expectExceptionMessage('EnsureEnd function requires non-null value'); - $result = ref('str')->ensureEnd('_suffix')->eval(row(['str' => null]), flow_context()); - - static::assertNull($result); + static::assertNull((new FunctionContext(flow_context()))->eval( + ref('str')->ensureEnd('_suffix'), + ['str' => null], + schema(str_schema('str', nullable: true)), + )); } public function test_string_already_ends_with_suffix(): void { - $result = ref('str')->ensureEnd('.txt')->eval(row(['str' => 'document.txt']), flow_context()); - - static::assertEquals('document.txt', $result); + static::assertEquals('document.txt', (new FunctionContext(flow_context()))->eval( + ref('str')->ensureEnd('.txt'), + ['str' => 'document.txt'], + schema(str_schema('str')), + )); } public function test_string_doesnt_end_with_suffix(): void { - $result = ref('str')->ensureEnd('.txt')->eval(row(['str' => 'document']), flow_context()); - - static::assertEquals('document.txt', $result); + static::assertEquals('document.txt', (new FunctionContext(flow_context()))->eval( + ref('str')->ensureEnd('.txt'), + ['str' => 'document'], + schema(str_schema('str')), + )); } public function test_string_with_empty_suffix(): void { - $result = ref('str')->ensureEnd('')->eval(row(['str' => 'hello']), flow_context()); - - static::assertEquals('hello', $result); + static::assertEquals('hello', (new FunctionContext(flow_context()))->eval( + ref('str')->ensureEnd(''), + ['str' => 'hello'], + schema(str_schema('str')), + )); } public function test_with_scalar_function_parameter(): void { - $result = ref('str') - ->ensureEnd(ref('suffix')) - ->eval(row(['str' => 'document', 'suffix' => '.pdf']), flow_context()); - - static::assertEquals('document.pdf', $result); + static::assertEquals('document.pdf', (new FunctionContext(flow_context()))->eval( + ref('str')->ensureEnd(ref('suffix')), + ['str' => 'document', 'suffix' => '.pdf'], + schema(str_schema('str'), str_schema('suffix')), + )); } } diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/EnsureStartTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/EnsureStartTest.php index 441a442018..edb27d5484 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/EnsureStartTest.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/EnsureStartTest.php @@ -5,28 +5,32 @@ namespace Flow\ETL\Tests\Unit\Function; use Flow\ETL\Exception\InvalidArgumentException; +use Flow\ETL\Tests\Context\FunctionContext; use Flow\ETL\Tests\FlowTestCase; use function Flow\ETL\DSL\flow_context; use function Flow\ETL\DSL\ref; -use function Flow\ETL\DSL\row; +use function Flow\ETL\DSL\schema; +use function Flow\ETL\DSL\str_schema; final class EnsureStartTest extends FlowTestCase { public function test_empty_string_with_prefix(): void { - $result = ref('str')->ensureStart('prefix_')->eval(row(['str' => '']), flow_context()); - - static::assertEquals('prefix_', $result); + static::assertEquals('prefix_', (new FunctionContext(flow_context()))->eval( + ref('str')->ensureStart('prefix_'), + ['str' => ''], + schema(str_schema('str')), + )); } public function test_null_prefix(): void { - $result = ref('str') - ->ensureStart(ref('prefix')) - ->eval(row(['str' => 'hello', 'prefix' => null]), flow_context()); - - static::assertEquals('hello', $result); + static::assertEquals('hello', (new FunctionContext(flow_context()))->eval( + ref('str')->ensureStart(ref('prefix')), + ['str' => 'hello', 'prefix' => null], + schema(str_schema('str'), str_schema('prefix', nullable: true)), + )); } public function test_null_value(): void @@ -34,38 +38,46 @@ public function test_null_value(): void $this->expectException(InvalidArgumentException::class); $this->expectExceptionMessage('EnsureStart function requires non-null value'); - $result = ref('str')->ensureStart('prefix_')->eval(row(['str' => null]), flow_context()); - - static::assertNull($result); + static::assertNull((new FunctionContext(flow_context()))->eval( + ref('str')->ensureStart('prefix_'), + ['str' => null], + schema(str_schema('str', nullable: true)), + )); } public function test_string_already_starts_with_prefix(): void { - $result = ref('str')->ensureStart('https://')->eval(row(['str' => 'https://example.com']), flow_context()); - - static::assertEquals('https://example.com', $result); + static::assertEquals('https://example.com', (new FunctionContext(flow_context()))->eval( + ref('str')->ensureStart('https://'), + ['str' => 'https://example.com'], + schema(str_schema('str')), + )); } public function test_string_doesnt_start_with_prefix(): void { - $result = ref('str')->ensureStart('https://')->eval(row(['str' => 'example.com']), flow_context()); - - static::assertEquals('https://example.com', $result); + static::assertEquals('https://example.com', (new FunctionContext(flow_context()))->eval( + ref('str')->ensureStart('https://'), + ['str' => 'example.com'], + schema(str_schema('str')), + )); } public function test_string_with_empty_prefix(): void { - $result = ref('str')->ensureStart('')->eval(row(['str' => 'hello']), flow_context()); - - static::assertEquals('hello', $result); + static::assertEquals('hello', (new FunctionContext(flow_context()))->eval( + ref('str')->ensureStart(''), + ['str' => 'hello'], + schema(str_schema('str')), + )); } public function test_with_scalar_function_parameter(): void { - $result = ref('str') - ->ensureStart(ref('prefix')) - ->eval(row(['str' => 'example.com', 'prefix' => 'https://']), flow_context()); - - static::assertEquals('https://example.com', $result); + static::assertEquals('https://example.com', (new FunctionContext(flow_context()))->eval( + ref('str')->ensureStart(ref('prefix')), + ['str' => 'example.com', 'prefix' => 'https://'], + schema(str_schema('str'), str_schema('prefix')), + )); } } diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/EnumNameTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/EnumNameTest.php index dd23cceaea..613114484b 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/EnumNameTest.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/EnumNameTest.php @@ -5,6 +5,7 @@ namespace Flow\ETL\Tests\Unit\Function; use Flow\ETL\Exception\InvalidArgumentException; +use Flow\ETL\Tests\Context\FunctionContext; use Flow\ETL\Tests\Fixtures\Enum\BackedIntEnum; use Flow\ETL\Tests\Fixtures\Enum\BackedStringEnum; use Flow\ETL\Tests\Fixtures\Enum\BasicEnum; @@ -14,9 +15,10 @@ use function Flow\ETL\DSL\config; use function Flow\ETL\DSL\enum_name; +use function Flow\ETL\DSL\enum_schema; use function Flow\ETL\DSL\flow_context; use function Flow\ETL\DSL\ref; -use function Flow\ETL\DSL\row; +use function Flow\ETL\DSL\schema; use function Flow\Types\DSL\type_equals; use function Flow\Types\DSL\type_string; @@ -24,7 +26,11 @@ final class EnumNameTest extends FlowTestCase { public function test_enum_name_accepts_literal_enum(): void { - static::assertSame('one', enum_name(BackedStringEnum::one)->eval(row([]), flow_context())); + static::assertSame('one', (new FunctionContext(flow_context()))->eval( + enum_name(BackedStringEnum::one), + [], + schema(), + )); } public function test_enum_name_carries_string_type(): void @@ -34,7 +40,13 @@ public function test_enum_name_carries_string_type(): void public function test_enum_name_from_scalar_function_chain(): void { - static::assertSame('one', ref('e')->enumName()->eval(row(['e' => BackedIntEnum::one]), flow_context())); + static::assertSame('one', (new FunctionContext(flow_context()))->eval( + ref('e')->enumName(), + [ + 'e' => BackedIntEnum::one, + ], + schema(enum_schema('e', BackedIntEnum::class)), + )); } #[TestWith([BackedStringEnum::one])] @@ -42,7 +54,11 @@ public function test_enum_name_from_scalar_function_chain(): void #[TestWith([BasicEnum::one])] public function test_enum_name_returns_case_name(UnitEnum $enum): void { - static::assertSame('one', enum_name(ref('e'))->eval(row(['e' => $enum]), flow_context())); + static::assertSame('one', (new FunctionContext(flow_context()))->eval( + enum_name(ref('e')), + ['e' => $enum], + schema(enum_schema('e', $enum::class)), + )); } #[TestWith([null])] @@ -54,6 +70,6 @@ public function test_enum_name_throws_in_strict_mode(mixed $input): void $this->expectExceptionMessage('EnumName function requires a UnitEnum value'); $context = flow_context(config()); - enum_name($input)->eval(row([]), $context); + (new FunctionContext($context))->eval(enum_name($input), [], schema()); } } diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/EnumValueTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/EnumValueTest.php index e87467bc9f..ab6789320e 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/EnumValueTest.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/EnumValueTest.php @@ -8,6 +8,7 @@ use Flow\ETL\Exception\SchemaNotDerivableException; use Flow\ETL\Function\ReferenceResolver; use Flow\ETL\Function\ScalarFunction; +use Flow\ETL\Tests\Context\FunctionContext; use Flow\ETL\Tests\Fixtures\Enum\BackedIntEnum; use Flow\ETL\Tests\Fixtures\Enum\BackedStringEnum; use Flow\ETL\Tests\Fixtures\Enum\BasicEnum; @@ -20,7 +21,6 @@ use function Flow\ETL\DSL\enum_value; use function Flow\ETL\DSL\flow_context; use function Flow\ETL\DSL\ref; -use function Flow\ETL\DSL\row; use function Flow\ETL\DSL\schema; use function Flow\ETL\DSL\str_schema; use function Flow\Types\DSL\type_equals; @@ -31,19 +31,33 @@ final class EnumValueTest extends FlowTestCase { public function test_enum_value_accepts_literal_enum(): void { - static::assertSame(1, enum_value(BackedIntEnum::one)->eval(row([]), flow_context())); + static::assertSame(1, (new FunctionContext(flow_context()))->eval( + enum_value(BackedIntEnum::one), + [], + schema(), + )); } public function test_enum_value_from_scalar_function_chain(): void { - static::assertSame(1, ref('e')->enumValue()->eval(row(['e' => BackedIntEnum::one]), flow_context())); + static::assertSame(1, (new FunctionContext(flow_context()))->eval( + ref('e')->enumValue(), + [ + 'e' => BackedIntEnum::one, + ], + schema(enum_schema('e', BackedIntEnum::class)), + )); } #[TestWith([BackedStringEnum::one, 'one'])] #[TestWith([BackedIntEnum::one, 1])] public function test_enum_value_returns_backing_value(UnitEnum $enum, int|string $expected): void { - static::assertSame($expected, enum_value(ref('e'))->eval(row(['e' => $enum]), flow_context())); + static::assertSame($expected, (new FunctionContext(flow_context()))->eval( + enum_value(ref('e')), + ['e' => $enum], + schema(enum_schema('e', $enum::class)), + )); } #[TestWith([BasicEnum::one])] @@ -56,7 +70,7 @@ public function test_enum_value_throws_in_strict_mode(mixed $input): void $this->expectExceptionMessage('EnumValue function requires a BackedEnum value'); $context = flow_context(config()); - enum_value($input)->eval(row([]), $context); + (new FunctionContext($context))->eval(enum_value($input), [], schema()); } public function test_a_non_enum_operand_is_refused_at_bind(): void diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/EqualsTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/EqualsTest.php index 58070c9e70..f74056ea56 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/EqualsTest.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/EqualsTest.php @@ -4,25 +4,142 @@ namespace Flow\ETL\Tests\Unit\Function; +use DateInterval; +use DateTimeImmutable; +use DateTimeZone; +use Flow\ETL\Function\ReferenceResolver; +use Flow\ETL\Tests\Context\FunctionContext; +use Flow\ETL\Tests\Fixtures\Enum\BackedStringEnum; use Flow\ETL\Tests\FlowTestCase; +use Flow\Types\Exception\InvalidArgumentException as TypesInvalidArgumentException; +use Flow\Types\Type; +use Generator; +use PHPUnit\Framework\Attributes\DataProvider; +use function Flow\ETL\DSL\definition_from_type; use function Flow\ETL\DSL\flow_context; +use function Flow\ETL\DSL\int_schema; use function Flow\ETL\DSL\lit; use function Flow\ETL\DSL\ref; -use function Flow\ETL\DSL\row; +use function Flow\ETL\DSL\schema; +use function Flow\ETL\DSL\str_schema; +use function Flow\Types\DSL\type_date; +use function Flow\Types\DSL\type_datetime; +use function Flow\Types\DSL\type_enum; +use function Flow\Types\DSL\type_json; +use function Flow\Types\DSL\type_time; +use function Flow\Types\DSL\type_uuid; +use function Flow\Types\DSL\type_xml_element; final class EqualsTest extends FlowTestCase { public function test_null_operand_yields_null(): void { - static::assertNull(ref('a')->equals(lit(1))->eval(row(['a' => null]), flow_context())); - static::assertNull(ref('a')->equals(ref('b'))->eval(row(['a' => null, 'b' => null]), flow_context())); + static::assertNull((new FunctionContext(flow_context()))->eval( + ref('a')->equals(lit(1)), + ['a' => null], + schema(str_schema('a', nullable: true)), + )); + static::assertNull((new FunctionContext(flow_context()))->eval( + ref('a')->equals(ref('b')), + ['a' => null, 'b' => null], + schema(str_schema('a', nullable: true), str_schema('b', nullable: true)), + )); } public function test_equal_values(): void { - static::assertTrue(ref('a')->equals(lit(1))->eval(row(['a' => 1]), flow_context())); - static::assertTrue(ref('a')->equals(lit('x'))->eval(row(['a' => 'x']), flow_context())); - static::assertFalse(ref('a')->equals(lit(2))->eval(row(['a' => 1]), flow_context())); + static::assertTrue((new FunctionContext(flow_context()))->eval( + ref('a')->equals(lit(1)), + ['a' => 1], + schema(int_schema('a')), + )); + static::assertTrue((new FunctionContext(flow_context()))->eval( + ref('a')->equals(lit('x')), + ['a' => 'x'], + schema(str_schema('a')), + )); + static::assertFalse((new FunctionContext(flow_context()))->eval( + ref('a')->equals(lit(2)), + ['a' => 1], + schema(int_schema('a')), + )); + } + + /** + * @return Generator, mixed, mixed, bool}> + */ + public static function values_of_one_type(): Generator + { + yield 'uuid' => [ + type_uuid(), + '00000000-0000-4000-8000-000000000001', + '00000000-0000-4000-8000-000000000001', + true, + ]; + yield 'other uuid' => [ + type_uuid(), + '00000000-0000-4000-8000-000000000001', + '00000000-0000-4000-8000-000000000002', + false, + ]; + yield 'json' => [type_json(), '{"a":1,"b":2}', '{"a":1,"b":2}', true]; + yield 'json compares its stored text' => [type_json(), '{"a":1,"b":2}', '{"b":2,"a":1}', false]; + yield 'enum' => [type_enum(BackedStringEnum::class), BackedStringEnum::one, BackedStringEnum::one, true]; + yield 'datetime of the same instant in another zone' => [ + type_datetime(), + new DateTimeImmutable('2024-01-01 12:00:00', new DateTimeZone('UTC')), + new DateTimeImmutable('2024-01-01 13:00:00', new DateTimeZone('Europe/Warsaw')), + true, + ]; + yield 'date' => [type_date(), new DateTimeImmutable('2024-01-01'), new DateTimeImmutable('2024-01-01'), true]; + yield 'time' => [type_time(), new DateInterval('PT60M'), new DateInterval('PT1H'), true]; + yield 'xml elements of different documents' => [ + type_xml_element(), + type_xml_element()->cast('

a

')->firstElementChild, + type_xml_element()->cast('

a

')->firstElementChild, + true, + ]; + yield 'other xml elements' => [ + type_xml_element(), + type_xml_element()->cast('

a

')->firstElementChild, + type_xml_element()->cast('

b

')->firstElementChild, + false, + ]; + } + + /** + * @param Type $type + */ + #[DataProvider('values_of_one_type')] + public function test_values_of_one_type_compare_by_value(Type $type, mixed $left, mixed $right, bool $equal): void + { + static::assertSame($equal, (new FunctionContext(flow_context()))->eval( + ref('a')->equals(ref('b')), + ['a' => $left, 'b' => $right], + schema(definition_from_type('a', $type), definition_from_type('b', $type)), + )); + } + + public function test_a_datetime_equals_a_literal_of_the_same_instant_in_another_zone(): void + { + static::assertTrue((new FunctionContext(flow_context()))->eval( + ref('a')->equals(lit(new DateTimeImmutable('2024-01-01 13:00:00', new DateTimeZone('Europe/Warsaw')))), + ['a' => new DateTimeImmutable('2024-01-01 12:00:00', new DateTimeZone('UTC'))], + schema(definition_from_type('a', type_datetime())), + )); + } + + public function test_a_uuid_is_not_comparable_with_a_string(): void + { + $this->expectException(TypesInvalidArgumentException::class); + $this->expectExceptionMessage("Can't compare '(uuid == string)' due to data type mismatch"); + + (new ReferenceResolver()) + ->resolve( + ref('u')->equals(lit('00000000-0000-4000-8000-000000000001')), + schema(definition_from_type('u', type_uuid())), + ) + ->returns(); } } diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/Evaluation/ResultColumnTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/Evaluation/ResultColumnTest.php new file mode 100644 index 0000000000..a09ace0f18 --- /dev/null +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/Evaluation/ResultColumnTest.php @@ -0,0 +1,157 @@ +of(lit(5), [5, null]); + + static::assertSame([5, null], $column->values()); + static::assertEquals(type_integer(), $column->type()); + } + + public function test_an_unresolved_tree_throws(): void + { + $this->expectException(InvalidLogicException::class); + + (new ResultColumn(new PhpBackend()))->of(ref('a'), [1]); + } + + public function test_casts_each_value_into_the_returned_type(): void + { + static::assertSame([5, 6], (new ResultColumn(new PhpBackend()))->of(lit(1), ['5', 6])->values()); + } + + public function test_builds_through_the_backend_it_was_given(): void + { + $backend = new SpyBackend(); + + (new ResultColumn($backend))->of(lit(1), [1, 2]); + (new ResultColumn($backend))->of(lit(1), ['1', 2]); + + static::assertSame(2, $backend->builders()); + } + + public function test_null_type_builds_a_null_column(): void + { + $column = (new ResultColumn(new PhpBackend()))->of(lit(null), [null, null]); + + static::assertEquals(type_null(), $column->type()); + static::assertSame(2, $column->nullCount()); + } + + public function test_refusal_names_the_row_of_the_value(): void + { + try { + (new ResultColumn(new PhpBackend()))->of(lit(1), [1, 2, 'x']); + static::fail('expected a SchemaMismatchException'); + } catch (SchemaMismatchException $e) { + static::assertSame(2, $e->rowIndex); + } + } + + public function test_type_without_a_column_kind_is_an_untyped_column(): void + { + $column = (new ResultColumn(new PhpBackend()))->of(call(lit('strval'), type_instance_of(ArrayObject::class)), [ + new ArrayObject(), + null, + ]); + + static::assertInstanceOf(ValueColumn::class, $column); + static::assertSame(2, $column->count()); + } + + public function test_underivable_return_is_an_untyped_column(): void + { + $function = (new ReferenceResolver())->resolve( + ref('j')->jsonDecode()->arrayGet('name'), + schema(json_schema('j')), + ); + + $column = (new ResultColumn(new PhpBackend()))->of($function, ['a', ['b' => 1]]); + + static::assertInstanceOf(ValueColumn::class, $column); + static::assertSame(['a', ['b' => 1]], $column->values()); + } + + /** + * @return Generator, bool}> + */ + public static function intermediates(): Generator + { + yield 'integer' => [type_integer(), true]; + yield 'list of structures' => [type_list(type_structure(['a' => type_string()])), true]; + yield 'untyped array' => [type_array(), false]; + yield 'list of untyped arrays' => [type_list(type_array()), false]; + yield 'xml' => [type_xml(), false]; + yield 'xml element in a map' => [type_map(type_string(), type_xml_element()), false]; + yield 'html element in a structure' => [type_structure(['a' => type_optional(type_html_element())]), false]; + yield 'html in a list' => [type_list(type_html()), false]; + yield 'mixed' => [type_mixed(), false]; + } + + /** + * @param Type $type + */ + #[DataProvider('intermediates')] + public function test_a_typed_column_holds_an_intermediate_only_when_it_keeps_the_values( + Type $type, + bool $holds, + ): void { + static::assertSame($holds, (new ResultColumn(new PhpBackend()))->holdsLosslessly($type)); + } + + public function test_an_untyped_array_keeps_its_objects(): void + { + $object = new ArrayObject(); + $column = (new ResultColumn(new PhpBackend()))->typed(type_optional(type_array()), [[$object, 'count']]); + + static::assertInstanceOf(ValueColumn::class, $column); + static::assertSame([[$object, 'count']], $column->values()); + } + + public function test_lists_of_an_expand_are_a_list_column(): void + { + $column = (new ResultColumn(new PhpBackend()))->lists(lit(1), [[1, 2], []]); + + static::assertEquals(type_list(type_integer()), $column->type()); + static::assertSame([[1, 2], []], $column->values()); + } +} diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/Evaluation/ReturnTypesTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/Evaluation/ReturnTypesTest.php new file mode 100644 index 0000000000..add7c90c87 --- /dev/null +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/Evaluation/ReturnTypesTest.php @@ -0,0 +1,60 @@ + 1]], [['id' => 2], ['id' => 3]], [['id' => 4]]] as $batch) { + static::assertSame( + array_fill(0, count($batch), 1), + $function->eval(array_to_rows($batch, schema(int_schema('id'))), flow_context())->values(), + ); + } + + static::assertSame(1, $function->derivations); + } + + public function test_a_type_that_cannot_be_derived_is_tried_once(): void + { + $function = new ReturnsSpyFunction(derivable: false); + + foreach ([[['id' => 1]], [['id' => 2]]] as $batch) { + static::assertInstanceOf(ValueColumn::class, $function->eval( + array_to_rows($batch, schema(int_schema('id'))), + flow_context(), + )); + } + + static::assertNull(ReturnTypes::of($function)); + static::assertSame(1, $function->derivations); + } + + public function test_every_node_derives_its_own_type(): void + { + $first = new ReturnsSpyFunction(); + $second = new ReturnsSpyFunction(); + + static::assertSame('integer', ReturnTypes::of($first)?->toString()); + static::assertSame('integer', ReturnTypes::of($second)?->toString()); + static::assertSame(1, $first->derivations); + static::assertSame(1, $second->derivations); + } +} diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/Evaluation/SelectionTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/Evaluation/SelectionTest.php new file mode 100644 index 0000000000..5070196b0c --- /dev/null +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/Evaluation/SelectionTest.php @@ -0,0 +1,76 @@ + 1, 'name' => 'a'], ['v' => 2, 'name' => 'b']], + schema(int_schema('v'), str_schema('name')), + ); + $spy = new FailingOnValuesFunction(ref('v')); + + $column = (new Selection([0, 1]))->evaluate( + (new ReferenceResolver())->resolve($spy, $rows->schema()), + $rows, + flow_context(), + ); + + static::assertSame([1, 2], $column->values()); + static::assertSame($rows, $spy->seen[0]); + } + + public function test_an_error_names_the_row_of_the_batch(): void + { + $rows = array_to_rows([['v' => 1], ['v' => 2], ['v' => 3], ['v' => 4]], schema(int_schema('v'))); + + try { + (new Selection([1, 3]))->evaluate( + (new ReferenceResolver())->resolve(new FailingOnValuesFunction(ref('v'), [4]), $rows->schema()), + $rows, + flow_context(), + ); + static::fail('expected an EvaluationException'); + } catch (EvaluationException $e) { + static::assertSame(3, $e->rowIndex); + static::assertSame('fails on 4 (row 3)', $e->getMessage()); + } + } + + public function test_a_subset_holds_only_the_selected_rows_of_the_referenced_columns(): void + { + $rows = array_to_rows( + [['v' => 1, 'name' => 'a'], ['v' => 2, 'name' => 'b'], ['v' => 3, 'name' => 'c']], + schema(int_schema('v'), str_schema('name')), + ); + $spy = new FailingOnValuesFunction(ref('v')); + + $column = (new Selection([0, 2]))->evaluate( + (new ReferenceResolver())->resolve($spy, $rows->schema()), + $rows, + flow_context(), + ); + + static::assertSame([1, 3], $column->values()); + static::assertSame(['v'], array_keys($spy->seen[0]->columns())); + static::assertSame(2, $spy->seen[0]->count()); + } +} diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/Evaluation/TolerantEvaluationTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/Evaluation/TolerantEvaluationTest.php new file mode 100644 index 0000000000..265a9330e6 --- /dev/null +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/Evaluation/TolerantEvaluationTest.php @@ -0,0 +1,101 @@ + [], 'values' => []], + (new TolerantEvaluation())->evaluate( + (new ReferenceResolver())->resolve(new FailingOnValuesFunction(ref('id')), $rows->schema()), + $rows, + flow_context(), + ), + ); + } + + public function test_every_row_evaluates_in_one_batch_when_none_fails(): void + { + $spy = new FailingOnValuesFunction(ref('id')); + $rows = IdRows::batches(range(1, 3))->current(); + + static::assertSame( + ['failed' => [], 'values' => [0 => 1, 1 => 2, 2 => 3]], + (new TolerantEvaluation())->evaluate( + (new ReferenceResolver())->resolve($spy, $rows->schema()), + $rows, + flow_context(), + ), + ); + static::assertSame(1, $spy->evals); + } + + public function test_failing_rows_are_reported_and_left_out_of_the_values(): void + { + $spy = new FailingOnValuesFunction(ref('id'), [2, 4]); + $rows = IdRows::batches(range(1, 5))->current(); + + static::assertEquals( + ['failed' => [1 => true, 3 => true], 'values' => [0 => 1, 2 => 3, 4 => 5]], + (new TolerantEvaluation())->evaluate( + (new ReferenceResolver())->resolve($spy, $rows->schema()), + $rows, + flow_context(), + ), + ); + static::assertSame(3, $spy->evals); + } + + public function test_rows_past_the_retry_limit_go_one_by_one(): void + { + $spy = new FailingOnValuesFunction(ref('id'), range(1, 9)); + $rows = IdRows::batches(range(1, 10))->current(); + + $evaluated = (new TolerantEvaluation())->evaluate( + (new ReferenceResolver())->resolve($spy, $rows->schema()), + $rows, + flow_context(), + ); + + static::assertSame(range(0, 8), array_keys($evaluated['failed'])); + static::assertSame([9 => 10], $evaluated['values']); + static::assertSame(TolerantEvaluation::RETRIES + 2, $spy->evals); + } + + public function test_row_by_row_evaluates_each_given_row_alone(): void + { + $spy = new FailingOnValuesFunction(ref('id'), [2]); + $rows = IdRows::batches(range(1, 4))->current(); + + static::assertSame( + ['failed' => [1 => true], 'values' => [0 => 1, 3 => 4]], + (new TolerantEvaluation())->rowByRow( + (new ReferenceResolver())->resolve($spy, $rows->schema()), + $rows, + [0, 1, 3], + flow_context(), + ), + ); + static::assertSame(3, $spy->evals); + } +} diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/ExistsTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/ExistsTest.php index cc38baec29..a80b3c4296 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/ExistsTest.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/ExistsTest.php @@ -5,37 +5,103 @@ namespace Flow\ETL\Tests\Unit\Function; use Flow\ETL\Function\Exists; +use Flow\ETL\Tests\Context\FunctionContext; +use Flow\ETL\Tests\Double\FailingOnValuesFunction; use Flow\ETL\Tests\FlowTestCase; +use Generator; +use PHPUnit\Framework\Attributes\DataProvider; +use function array_map; +use function Flow\ETL\DSL\array_to_rows; +use function Flow\ETL\DSL\exists; use function Flow\ETL\DSL\flow_context; +use function Flow\ETL\DSL\int_schema; use function Flow\ETL\DSL\lit; use function Flow\ETL\DSL\ref; -use function Flow\ETL\DSL\row; +use function Flow\ETL\DSL\schema; +use function Flow\ETL\DSL\str_schema; +use function in_array; +use function range; final class ExistsTest extends FlowTestCase { public function test_a_throwing_operand_means_the_reference_does_not_exist(): void { - static::assertFalse((new Exists(ref('value')->upper()))->eval(row(['value' => 1]), flow_context())); + static::assertFalse((new FunctionContext(flow_context()))->eval( + new Exists(ref('value')->upper()), + [ + 'value' => 1, + ], + schema(int_schema('value')), + )); } public function test_if_reference_exists(): void { - static::assertTrue(ref('value')->exists()->eval(row(['value' => 'test']), flow_context())); + static::assertTrue((new FunctionContext(flow_context()))->eval( + ref('value')->exists(), + ['value' => 'test'], + schema(str_schema('value')), + )); } public function test_that_lit_function_exists(): void { - static::assertTrue((new Exists(lit('val')))->eval(row([]), flow_context())); + static::assertTrue((new FunctionContext(flow_context()))->eval(new Exists(lit('val')), [], schema())); } public function test_that_null_reference_to_null_entry_exists(): void { - static::assertTrue(ref('value')->exists()->eval(row(['value' => null]), flow_context())); + static::assertTrue((new FunctionContext(flow_context()))->eval( + ref('value')->exists(), + ['value' => null], + schema(str_schema('value', nullable: true)), + )); } public function test_that_reference_does_not_exists(): void { - static::assertFalse(ref('value')->exists()->eval(row([]), flow_context())); + static::assertFalse((new FunctionContext(flow_context()))->eval(ref('value')->exists(), [], schema())); + } + + /** + * @return Generator, string, int}> + */ + public static function failing_rows(): Generator + { + yield 'one failing row: one retry' => [[3], FailingOnValuesFunction::WITH_ROW, 2]; + yield 'three failing rows: three retries' => [[2, 4, 6], FailingOnValuesFunction::WITH_ROW, 4]; + yield 'twelve failing rows: eight batch evals, then each remaining row alone' => [ + [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12], + FailingOnValuesFunction::WITH_ROW, + 8 + 6, + ]; + yield 'a failure without a row: each row alone' => [[3], FailingOnValuesFunction::WITHOUT_ROW, 1 + 14]; + } + + /** + * @param list $failing + */ + #[DataProvider('failing_rows')] + public function test_only_the_failing_rows_are_false(array $failing, string $failure, int $evals): void + { + $spy = new FailingOnValuesFunction(ref('v'), $failing, $failure); + $rows = array_to_rows( + array_map(static fn(int $v): array => ['v' => $v], range(1, 14)), + schema(int_schema('v')), + ); + + $values = exists($spy)->eval($rows, flow_context())->values(); + + static::assertSame(array_map(static fn(int $v): bool => !in_array($v, $failing, true), range(1, 14)), $values); + static::assertSame($evals, $spy->evals); + } + + public function test_a_reference_is_a_schema_fact(): void + { + $rows = array_to_rows([['v' => 1], ['v' => 2]], schema(int_schema('v'))); + + static::assertSame([true, true], exists(ref('v'))->eval($rows, flow_context())->values()); + static::assertSame([false, false], exists(ref('missing'))->eval($rows, flow_context())->values()); } } diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/FirstTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/FirstTest.php index 3c10bb7fd7..a1bb30abf7 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/FirstTest.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/FirstTest.php @@ -4,15 +4,19 @@ namespace Flow\ETL\Tests\Unit\Function; +use Flow\ETL\Exception\InvalidArgumentException; use Flow\ETL\Function\ReferenceResolver; use Flow\ETL\Tests\FlowTestCase; +use function array_map; +use function Flow\ETL\DSL\array_to_rows; use function Flow\ETL\DSL\first; use function Flow\ETL\DSL\flow_context; use function Flow\ETL\DSL\int_schema; +use function Flow\ETL\DSL\last; use function Flow\ETL\DSL\ref; -use function Flow\ETL\DSL\row; use function Flow\ETL\DSL\schema; +use function Flow\ETL\DSL\str_schema; final class FirstTest extends FlowTestCase { @@ -25,11 +29,17 @@ public function test_aggregation_firs_value(): void { $aggregator = first(ref('int')); - $aggregator->aggregate(row(['not_int' => null]), flow_context()); - $aggregator->aggregate(row(['int' => '10']), flow_context()); - $aggregator->aggregate(row(['int' => '20']), flow_context()); - $aggregator->aggregate(row(['int' => '55']), flow_context()); - $aggregator->aggregate(row(['int' => '25']), flow_context()); + $aggregator->aggregate( + array_to_rows([[ + 'not_int' => null, + ]], schema(str_schema('not_int', nullable: true))), + [0], + flow_context(), + ); + $aggregator->aggregate(array_to_rows([['int' => '10']], schema(str_schema('int'))), [0], flow_context()); + $aggregator->aggregate(array_to_rows([['int' => '20']], schema(str_schema('int'))), [0], flow_context()); + $aggregator->aggregate(array_to_rows([['int' => '55']], schema(str_schema('int'))), [0], flow_context()); + $aggregator->aggregate(array_to_rows([['int' => '25']], schema(str_schema('int'))), [0], flow_context()); static::assertSame('10', $aggregator->value()); } @@ -64,4 +74,55 @@ public function test_first_over_an_int_column_declares_optional_integer(): void ->toString(), ); } + + public function test_aggregate_reads_only_given_indices(): void + { + $aggregator = first(ref('v')); + $aggregator->aggregate( + array_to_rows(array_map(static fn(mixed $v): array => ['v' => $v], [1, 2, 3, 4]), schema(int_schema('v'))), + [1, 3], + flow_context(), + ); + + static::assertSame(2, $aggregator->value()); + } + + public function test_merge_of_two_halves_equals_one_pass(): void + { + $rows = array_to_rows(array_map(static fn(mixed $v): array => ['v' => $v], [ + 1, + 2, + 3, + 4, + ]), schema(int_schema('v'))); + $onePass = first(ref('v')); + $onePass->aggregate($rows, [0, 1, 2, 3], flow_context()); + $left = first(ref('v')); + $left->aggregate($rows, [0, 1], flow_context()); + $right = first(ref('v')); + $right->aggregate($rows, [2, 3], flow_context()); + + $left->merge($right, flow_context()); + + static::assertSame($onePass->value(), $left->value()); + } + + public function test_merge_refuses_another_class(): void + { + $this->expectException(InvalidArgumentException::class); + + first(ref('v'))->merge(last(ref('v')), flow_context()); + } + + public function test_merge_into_an_empty_half_takes_the_other_value(): void + { + $rows = array_to_rows([['v' => 7]], schema(int_schema('v'))); + $right = first(ref('v')); + $right->aggregate($rows, [0], flow_context()); + $left = first(ref('v')); + + $left->merge($right, flow_context()); + + static::assertSame(7, $left->value()); + } } diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/GreaterThanEqualTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/GreaterThanEqualTest.php index d956455de8..80daa88ae0 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/GreaterThanEqualTest.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/GreaterThanEqualTest.php @@ -6,13 +6,22 @@ use DateInterval; use DateTimeImmutable; +use Flow\ETL\Tests\Context\FunctionContext; use Flow\ETL\Tests\FlowTestCase; use Flow\Types\Exception\InvalidArgumentException; +use function Flow\ETL\DSL\datetime_schema; +use function Flow\ETL\DSL\float_schema; use function Flow\ETL\DSL\flow_context; +use function Flow\ETL\DSL\int_schema; +use function Flow\ETL\DSL\list_schema; use function Flow\ETL\DSL\lit; use function Flow\ETL\DSL\ref; -use function Flow\ETL\DSL\row; +use function Flow\ETL\DSL\schema; +use function Flow\ETL\DSL\str_schema; +use function Flow\ETL\DSL\time_schema; +use function Flow\Types\DSL\type_integer; +use function Flow\Types\DSL\type_list; final class GreaterThanEqualTest extends FlowTestCase { @@ -20,95 +29,165 @@ public function test_greater_than_equal_arrays(): void { $context = flow_context(); - static::assertTrue(ref('v')->greaterThanEqual(lit([1, 0]))->eval(row(['v' => [1, 9]]), $context)); - static::assertTrue(ref('v')->greaterThanEqual(lit([1, 0]))->eval(row(['v' => [1, 0]]), $context)); - static::assertFalse(ref('v')->greaterThanEqual(lit([1, 9]))->eval(row(['v' => [1, 0]]), $context)); + static::assertTrue((new FunctionContext($context))->eval( + ref('v')->greaterThanEqual(lit([1, 0])), + ['v' => [1, 9]], + schema(list_schema('v', type_list(type_integer()))), + )); + static::assertTrue((new FunctionContext($context))->eval( + ref('v')->greaterThanEqual(lit([1, 0])), + ['v' => [1, 0]], + schema(list_schema('v', type_list(type_integer()))), + )); + static::assertFalse((new FunctionContext($context))->eval( + ref('v')->greaterThanEqual(lit([1, 9])), + ['v' => [1, 0]], + schema(list_schema('v', type_list(type_integer()))), + )); } public function test_greater_than_equal_datetimes(): void { $context = flow_context(); - static::assertTrue( - ref('v') - ->greaterThanEqual(lit(new DateTimeImmutable('2024-01-01'))) - ->eval(row(['v' => new DateTimeImmutable('2024-06-01')]), $context), - ); - static::assertTrue( - ref('v') - ->greaterThanEqual(lit(new DateTimeImmutable('2024-01-01'))) - ->eval(row(['v' => new DateTimeImmutable('2024-01-01')]), $context), - ); - static::assertFalse( - ref('v') - ->greaterThanEqual(lit(new DateTimeImmutable('2024-06-01'))) - ->eval(row(['v' => new DateTimeImmutable('2024-01-01')]), $context), - ); + static::assertTrue((new FunctionContext($context))->eval( + ref('v')->greaterThanEqual(lit(new DateTimeImmutable('2024-01-01'))), + [ + 'v' => new DateTimeImmutable('2024-06-01'), + ], + schema(datetime_schema('v')), + )); + static::assertTrue((new FunctionContext($context))->eval( + ref('v')->greaterThanEqual(lit(new DateTimeImmutable('2024-01-01'))), + [ + 'v' => new DateTimeImmutable('2024-01-01'), + ], + schema(datetime_schema('v')), + )); + static::assertFalse((new FunctionContext($context))->eval( + ref('v')->greaterThanEqual(lit(new DateTimeImmutable('2024-06-01'))), + [ + 'v' => new DateTimeImmutable('2024-01-01'), + ], + schema(datetime_schema('v')), + )); } public function test_greater_than_equal_floats(): void { $context = flow_context(); - static::assertTrue(ref('v')->greaterThanEqual(lit(1.5))->eval(row(['v' => 2.5]), $context)); - static::assertTrue(ref('v')->greaterThanEqual(lit(1.5))->eval(row(['v' => 1.5]), $context)); - static::assertFalse(ref('v')->greaterThanEqual(lit(2.5))->eval(row(['v' => 1.5]), $context)); + static::assertTrue((new FunctionContext($context))->eval( + ref('v')->greaterThanEqual(lit(1.5)), + ['v' => 2.5], + schema(float_schema('v')), + )); + static::assertTrue((new FunctionContext($context))->eval( + ref('v')->greaterThanEqual(lit(1.5)), + ['v' => 1.5], + schema(float_schema('v')), + )); + static::assertFalse((new FunctionContext($context))->eval( + ref('v')->greaterThanEqual(lit(2.5)), + ['v' => 1.5], + schema(float_schema('v')), + )); } public function test_greater_than_equal_integers(): void { $context = flow_context(); - static::assertTrue(ref('v')->greaterThanEqual(lit(10))->eval(row(['v' => 20]), $context)); - static::assertTrue(ref('v')->greaterThanEqual(lit(10))->eval(row(['v' => 10]), $context)); - static::assertFalse(ref('v')->greaterThanEqual(lit(20))->eval(row(['v' => 10]), $context)); + static::assertTrue((new FunctionContext($context))->eval( + ref('v')->greaterThanEqual(lit(10)), + ['v' => 20], + schema(int_schema('v')), + )); + static::assertTrue((new FunctionContext($context))->eval( + ref('v')->greaterThanEqual(lit(10)), + ['v' => 10], + schema(int_schema('v')), + )); + static::assertFalse((new FunctionContext($context))->eval( + ref('v')->greaterThanEqual(lit(20)), + ['v' => 10], + schema(int_schema('v')), + )); } public function test_greater_than_equal_returns_null_for_null_array(): void { - static::assertNull(ref('v')->greaterThanEqual(lit([1, 0]))->eval(row(['v' => null]), flow_context())); + static::assertNull((new FunctionContext(flow_context()))->eval( + ref('v')->greaterThanEqual(lit([1, 0])), + ['v' => null], + schema(list_schema('v', type_list(type_integer()), nullable: true)), + )); } public function test_greater_than_equal_returns_null_for_null_datetime(): void { - static::assertNull( - ref('v') - ->greaterThanEqual(lit(new DateTimeImmutable('2024-01-01'))) - ->eval(row(['v' => null]), flow_context()), - ); + static::assertNull((new FunctionContext(flow_context()))->eval( + ref('v')->greaterThanEqual(lit(new DateTimeImmutable('2024-01-01'))), + ['v' => null], + schema(datetime_schema('v', nullable: true)), + )); } public function test_greater_than_equal_returns_null_for_null_left(): void { - static::assertNull(ref('v')->greaterThanEqual(lit(10))->eval(row(['v' => null]), flow_context())); + static::assertNull((new FunctionContext(flow_context()))->eval( + ref('v')->greaterThanEqual(lit(10)), + ['v' => null], + schema(str_schema('v', nullable: true)), + )); } public function test_greater_than_equal_returns_null_for_null_right(): void { - static::assertNull( - ref('v')->greaterThanEqual(ref('other'))->eval(row(['v' => 10, 'other' => null]), flow_context()), - ); + static::assertNull((new FunctionContext(flow_context()))->eval( + ref('v')->greaterThanEqual(ref('other')), + ['v' => 10, 'other' => null], + schema(int_schema('v'), str_schema('other', nullable: true)), + )); } public function test_greater_than_equal_returns_null_for_null_string(): void { - static::assertNull(ref('v')->greaterThanEqual(lit('a'))->eval(row(['v' => null]), flow_context())); + static::assertNull((new FunctionContext(flow_context()))->eval( + ref('v')->greaterThanEqual(lit('a')), + ['v' => null], + schema(str_schema('v', nullable: true)), + )); } public function test_greater_than_equal_returns_null_for_null_time_interval(): void { - static::assertNull( - ref('v')->greaterThanEqual(lit(new DateInterval('PT1H')))->eval(row(['v' => null]), flow_context()), - ); + static::assertNull((new FunctionContext(flow_context()))->eval( + ref('v')->greaterThanEqual(lit(new DateInterval('PT1H'))), + ['v' => null], + schema(time_schema('v', nullable: true)), + )); } public function test_greater_than_equal_strings(): void { $context = flow_context(); - static::assertTrue(ref('v')->greaterThanEqual(lit('apple'))->eval(row(['v' => 'banana']), $context)); - static::assertTrue(ref('v')->greaterThanEqual(lit('apple'))->eval(row(['v' => 'apple']), $context)); - static::assertFalse(ref('v')->greaterThanEqual(lit('banana'))->eval(row(['v' => 'apple']), $context)); + static::assertTrue((new FunctionContext($context))->eval( + ref('v')->greaterThanEqual(lit('apple')), + ['v' => 'banana'], + schema(str_schema('v')), + )); + static::assertTrue((new FunctionContext($context))->eval( + ref('v')->greaterThanEqual(lit('apple')), + ['v' => 'apple'], + schema(str_schema('v')), + )); + static::assertFalse((new FunctionContext($context))->eval( + ref('v')->greaterThanEqual(lit('banana')), + ['v' => 'apple'], + schema(str_schema('v')), + )); } public function test_greater_than_equal_throws_on_incompatible_types(): void @@ -122,20 +201,20 @@ public function test_greater_than_equal_time_intervals(): void { $context = flow_context(); - static::assertTrue( - ref('v') - ->greaterThanEqual(lit(new DateInterval('PT1H'))) - ->eval(row(['v' => new DateInterval('PT5H')]), $context), - ); - static::assertTrue( - ref('v') - ->greaterThanEqual(lit(new DateInterval('PT1H'))) - ->eval(row(['v' => new DateInterval('PT1H')]), $context), - ); - static::assertFalse( - ref('v') - ->greaterThanEqual(lit(new DateInterval('PT5H'))) - ->eval(row(['v' => new DateInterval('PT1H')]), $context), - ); + static::assertTrue((new FunctionContext($context))->eval( + ref('v')->greaterThanEqual(lit(new DateInterval('PT1H'))), + ['v' => new DateInterval('PT5H')], + schema(time_schema('v')), + )); + static::assertTrue((new FunctionContext($context))->eval( + ref('v')->greaterThanEqual(lit(new DateInterval('PT1H'))), + ['v' => new DateInterval('PT1H')], + schema(time_schema('v')), + )); + static::assertFalse((new FunctionContext($context))->eval( + ref('v')->greaterThanEqual(lit(new DateInterval('PT5H'))), + ['v' => new DateInterval('PT1H')], + schema(time_schema('v')), + )); } } diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/GreaterThanTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/GreaterThanTest.php index f5578aa376..a5d92c5ab2 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/GreaterThanTest.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/GreaterThanTest.php @@ -6,113 +6,198 @@ use DateInterval; use DateTimeImmutable; +use Flow\ETL\Tests\Context\FunctionContext; use Flow\ETL\Tests\FlowTestCase; use Flow\Types\Exception\InvalidArgumentException; +use function Flow\ETL\DSL\datetime_schema; +use function Flow\ETL\DSL\float_schema; use function Flow\ETL\DSL\flow_context; +use function Flow\ETL\DSL\int_schema; +use function Flow\ETL\DSL\list_schema; use function Flow\ETL\DSL\lit; use function Flow\ETL\DSL\ref; -use function Flow\ETL\DSL\row; +use function Flow\ETL\DSL\schema; +use function Flow\ETL\DSL\str_schema; +use function Flow\ETL\DSL\time_schema; +use function Flow\Types\DSL\type_integer; +use function Flow\Types\DSL\type_list; final class GreaterThanTest extends FlowTestCase { public function test_an_integer_column_compared_to_a_string_literal_still_works(): void { // Comparator::comparable(integer, string) stays deliberately loose - 5 > '3' keeps working. - static::assertTrue(ref('a')->greaterThan(lit('3'))->eval(row(['a' => 5]), flow_context())); + static::assertTrue((new FunctionContext(flow_context()))->eval( + ref('a')->greaterThan(lit('3')), + ['a' => 5], + schema(int_schema('a')), + )); } public function test_greater_than_arrays(): void { $context = flow_context(); - static::assertTrue(ref('v')->greaterThan(lit([1, 0]))->eval(row(['v' => [1, 9]]), $context)); - static::assertFalse(ref('v')->greaterThan(lit([1, 9]))->eval(row(['v' => [1, 0]]), $context)); - static::assertFalse(ref('v')->greaterThan(lit([1, 0]))->eval(row(['v' => [1, 0]]), $context)); + static::assertTrue((new FunctionContext($context))->eval( + ref('v')->greaterThan(lit([1, 0])), + ['v' => [1, 9]], + schema(list_schema('v', type_list(type_integer()))), + )); + static::assertFalse((new FunctionContext($context))->eval( + ref('v')->greaterThan(lit([1, 9])), + ['v' => [1, 0]], + schema(list_schema('v', type_list(type_integer()))), + )); + static::assertFalse((new FunctionContext($context))->eval( + ref('v')->greaterThan(lit([1, 0])), + ['v' => [1, 0]], + schema(list_schema('v', type_list(type_integer()))), + )); } public function test_greater_than_datetimes(): void { $context = flow_context(); - static::assertTrue( - ref('v') - ->greaterThan(lit(new DateTimeImmutable('2024-01-01'))) - ->eval(row(['v' => new DateTimeImmutable('2024-06-01')]), $context), - ); - static::assertFalse( - ref('v') - ->greaterThan(lit(new DateTimeImmutable('2024-01-01'))) - ->eval(row(['v' => new DateTimeImmutable('2023-06-01')]), $context), - ); - static::assertFalse( - ref('v') - ->greaterThan(lit(new DateTimeImmutable('2024-01-01'))) - ->eval(row(['v' => new DateTimeImmutable('2024-01-01')]), $context), - ); + static::assertTrue((new FunctionContext($context))->eval( + ref('v')->greaterThan(lit(new DateTimeImmutable('2024-01-01'))), + [ + 'v' => new DateTimeImmutable('2024-06-01'), + ], + schema(datetime_schema('v')), + )); + static::assertFalse((new FunctionContext($context))->eval( + ref('v')->greaterThan(lit(new DateTimeImmutable('2024-01-01'))), + [ + 'v' => new DateTimeImmutable('2023-06-01'), + ], + schema(datetime_schema('v')), + )); + static::assertFalse((new FunctionContext($context))->eval( + ref('v')->greaterThan(lit(new DateTimeImmutable('2024-01-01'))), + [ + 'v' => new DateTimeImmutable('2024-01-01'), + ], + schema(datetime_schema('v')), + )); } public function test_greater_than_floats(): void { $context = flow_context(); - static::assertTrue(ref('v')->greaterThan(lit(1.5))->eval(row(['v' => 2.5]), $context)); - static::assertFalse(ref('v')->greaterThan(lit(2.5))->eval(row(['v' => 1.5]), $context)); - static::assertFalse(ref('v')->greaterThan(lit(1.5))->eval(row(['v' => 1.5]), $context)); + static::assertTrue((new FunctionContext($context))->eval( + ref('v')->greaterThan(lit(1.5)), + ['v' => 2.5], + schema(float_schema('v')), + )); + static::assertFalse((new FunctionContext($context))->eval( + ref('v')->greaterThan(lit(2.5)), + ['v' => 1.5], + schema(float_schema('v')), + )); + static::assertFalse((new FunctionContext($context))->eval( + ref('v')->greaterThan(lit(1.5)), + ['v' => 1.5], + schema(float_schema('v')), + )); } public function test_greater_than_integers(): void { $context = flow_context(); - static::assertTrue(ref('v')->greaterThan(lit(10))->eval(row(['v' => 20]), $context)); - static::assertFalse(ref('v')->greaterThan(lit(20))->eval(row(['v' => 10]), $context)); - static::assertFalse(ref('v')->greaterThan(lit(10))->eval(row(['v' => 10]), $context)); + static::assertTrue((new FunctionContext($context))->eval( + ref('v')->greaterThan(lit(10)), + ['v' => 20], + schema(int_schema('v')), + )); + static::assertFalse((new FunctionContext($context))->eval( + ref('v')->greaterThan(lit(20)), + ['v' => 10], + schema(int_schema('v')), + )); + static::assertFalse((new FunctionContext($context))->eval( + ref('v')->greaterThan(lit(10)), + ['v' => 10], + schema(int_schema('v')), + )); } public function test_greater_than_returns_null_for_null_left(): void { - static::assertNull(ref('v')->greaterThan(lit(10))->eval(row(['v' => null]), flow_context())); + static::assertNull((new FunctionContext(flow_context()))->eval( + ref('v')->greaterThan(lit(10)), + ['v' => null], + schema(str_schema('v', nullable: true)), + )); } public function test_greater_than_returns_null_for_null_right(): void { - static::assertNull( - ref('v')->greaterThan(ref('other'))->eval(row(['v' => 10, 'other' => null]), flow_context()), - ); + static::assertNull((new FunctionContext(flow_context()))->eval( + ref('v')->greaterThan(ref('other')), + ['v' => 10, 'other' => null], + schema(int_schema('v'), str_schema('other', nullable: true)), + )); } public function test_greater_than_returns_null_for_null_string(): void { - static::assertNull(ref('v')->greaterThan(lit('a'))->eval(row(['v' => null]), flow_context())); + static::assertNull((new FunctionContext(flow_context()))->eval( + ref('v')->greaterThan(lit('a')), + ['v' => null], + schema(str_schema('v', nullable: true)), + )); } public function test_greater_than_returns_null_for_null_datetime(): void { - static::assertNull( - ref('v')->greaterThan(lit(new DateTimeImmutable('2024-01-01')))->eval(row(['v' => null]), flow_context()), - ); + static::assertNull((new FunctionContext(flow_context()))->eval( + ref('v')->greaterThan(lit(new DateTimeImmutable('2024-01-01'))), + ['v' => null], + schema(datetime_schema('v', nullable: true)), + )); } public function test_greater_than_returns_null_for_null_time_interval(): void { - static::assertNull( - ref('v')->greaterThan(lit(new DateInterval('PT1H')))->eval(row(['v' => null]), flow_context()), - ); + static::assertNull((new FunctionContext(flow_context()))->eval( + ref('v')->greaterThan(lit(new DateInterval('PT1H'))), + ['v' => null], + schema(time_schema('v', nullable: true)), + )); } public function test_greater_than_returns_null_for_null_array(): void { - static::assertNull(ref('v')->greaterThan(lit([1, 0]))->eval(row(['v' => null]), flow_context())); + static::assertNull((new FunctionContext(flow_context()))->eval( + ref('v')->greaterThan(lit([1, 0])), + ['v' => null], + schema(list_schema('v', type_list(type_integer()), nullable: true)), + )); } public function test_greater_than_strings(): void { $context = flow_context(); - static::assertTrue(ref('v')->greaterThan(lit('apple'))->eval(row(['v' => 'banana']), $context)); - static::assertFalse(ref('v')->greaterThan(lit('banana'))->eval(row(['v' => 'apple']), $context)); - static::assertFalse(ref('v')->greaterThan(lit('apple'))->eval(row(['v' => 'apple']), $context)); + static::assertTrue((new FunctionContext($context))->eval( + ref('v')->greaterThan(lit('apple')), + ['v' => 'banana'], + schema(str_schema('v')), + )); + static::assertFalse((new FunctionContext($context))->eval( + ref('v')->greaterThan(lit('banana')), + ['v' => 'apple'], + schema(str_schema('v')), + )); + static::assertFalse((new FunctionContext($context))->eval( + ref('v')->greaterThan(lit('apple')), + ['v' => 'apple'], + schema(str_schema('v')), + )); } public function test_greater_than_throws_on_incompatible_types(): void @@ -126,20 +211,20 @@ public function test_greater_than_time_intervals(): void { $context = flow_context(); - static::assertTrue( - ref('v') - ->greaterThan(lit(new DateInterval('PT1H'))) - ->eval(row(['v' => new DateInterval('PT5H')]), $context), - ); - static::assertFalse( - ref('v') - ->greaterThan(lit(new DateInterval('PT5H'))) - ->eval(row(['v' => new DateInterval('PT1H')]), $context), - ); - static::assertFalse( - ref('v') - ->greaterThan(lit(new DateInterval('PT1H'))) - ->eval(row(['v' => new DateInterval('PT1H')]), $context), - ); + static::assertTrue((new FunctionContext($context))->eval( + ref('v')->greaterThan(lit(new DateInterval('PT1H'))), + ['v' => new DateInterval('PT5H')], + schema(time_schema('v')), + )); + static::assertFalse((new FunctionContext($context))->eval( + ref('v')->greaterThan(lit(new DateInterval('PT5H'))), + ['v' => new DateInterval('PT1H')], + schema(time_schema('v')), + )); + static::assertFalse((new FunctionContext($context))->eval( + ref('v')->greaterThan(lit(new DateInterval('PT1H'))), + ['v' => new DateInterval('PT1H')], + schema(time_schema('v')), + )); } } diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/GreatestTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/GreatestTest.php index 0caf622fa9..8bf15b7382 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/GreatestTest.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/GreatestTest.php @@ -7,6 +7,7 @@ use DateTimeImmutable; use Flow\ETL\Function\ReferenceResolver; use Flow\ETL\Function\ScalarFunction; +use Flow\ETL\Tests\Context\FunctionContext; use Flow\ETL\Tests\FlowTestCase; use function Flow\ETL\DSL\flow_context; @@ -14,7 +15,6 @@ use function Flow\ETL\DSL\int_schema; use function Flow\ETL\DSL\lit; use function Flow\ETL\DSL\ref; -use function Flow\ETL\DSL\row; use function Flow\ETL\DSL\schema; final class GreatestTest extends FlowTestCase @@ -34,7 +34,11 @@ public function test_greatest_value(): void { $greatest = greatest(10, 20, ref('int'), 40); - static::assertSame(55, $greatest->eval(row(['int' => 55]), flow_context())); + static::assertSame(55, (new FunctionContext(flow_context()))->eval( + $greatest, + ['int' => 55], + schema(int_schema('int')), + )); } public function test_greatest_with_non_comparable_values(): void @@ -53,18 +57,36 @@ public function test_an_integer_and_a_string_column_have_no_common_type(): void public function test_nulls_are_skipped(): void { - static::assertSame(20, greatest(null, 20, ref('int'))->eval(row(['int' => 4]), flow_context())); + static::assertSame(20, (new FunctionContext(flow_context()))->eval( + greatest(null, 20, ref('int')), + [ + 'int' => 4, + ], + schema(int_schema('int')), + )); } public function test_null_only_when_every_argument_is_null(): void { - static::assertNull(greatest(null, null)->eval(row(['int' => 4]), flow_context())); + static::assertNull((new FunctionContext(flow_context()))->eval( + greatest(null, null), + [ + 'int' => 4, + ], + schema(int_schema('int')), + )); } public function test_greatest_with_null(): void { $greatest = greatest(null, 20, ref('int'), 1257); - static::assertSame(1257, $greatest->eval(row(['int' => 55]), flow_context())); + static::assertSame(1257, (new FunctionContext(flow_context()))->eval( + $greatest, + [ + 'int' => 55, + ], + schema(int_schema('int')), + )); } } diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/HTMLQuerySelectorAllTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/HTMLQuerySelectorAllTest.php index 2bd4bd6988..791284d339 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/HTMLQuerySelectorAllTest.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/HTMLQuerySelectorAllTest.php @@ -8,12 +8,15 @@ use Dom\HTMLDocument; use Flow\ETL\Exception\InvalidArgumentException; use Flow\ETL\Exception\RequiredPHPVersionException; +use Flow\ETL\Tests\Context\FunctionContext; use PHPUnit\Framework\Attributes\RequiresPhp; use PHPUnit\Framework\TestCase; use function Flow\ETL\DSL\flow_context; +use function Flow\ETL\DSL\html_schema; use function Flow\ETL\DSL\ref; -use function Flow\ETL\DSL\row; +use function Flow\ETL\DSL\schema; +use function Flow\ETL\DSL\str_schema; final class HTMLQuerySelectorAllTest extends TestCase { @@ -21,7 +24,11 @@ final class HTMLQuerySelectorAllTest extends TestCase public function test_getting_element_for_older_versions(): void { $this->expectException(RequiredPHPVersionException::class); - ref('value')->htmlQuerySelectorAll('body div p')->eval(row(['value' => '']), flow_context()); + (new FunctionContext(flow_context()))->eval( + ref('value')->htmlQuerySelectorAll('body div p'), + ['value' => ''], + schema(str_schema('value')), + ); } #[RequiresPhp('>= 8.4.0')] @@ -32,7 +39,11 @@ public function test_getting_elements_for_given_path(): void '
foobar
', ); /** @var array $result */ - $result = ref('value')->htmlQuerySelectorAll('body div span')->eval(row(['value' => $html]), flow_context()); + $result = (new FunctionContext(flow_context()))->eval( + ref('value')->htmlQuerySelectorAll('body div span'), + ['value' => $html], + schema(html_schema('value')), + ); static::assertCount(1, $result); static::assertInstanceOf(Element::class, $result[0]); } @@ -44,8 +55,11 @@ public function test_getting_null_when_nothing_found(): void $html = HTMLDocument::createFromString( '
foobar
', ); - $result = ref('value')->htmlQuerySelectorAll('body div p')->eval(row(['value' => $html]), flow_context()); - static::assertNull($result); + static::assertNull((new FunctionContext(flow_context()))->eval( + ref('value')->htmlQuerySelectorAll('body div p'), + ['value' => $html], + schema(html_schema('value')), + )); } #[RequiresPhp('>= 8.4.0')] @@ -54,6 +68,10 @@ public function test_invalid_value(): void $this->expectException(InvalidArgumentException::class); $this->expectExceptionMessage('Expected one of'); - ref('value')->htmlQuerySelectorAll('body div span')->eval(row(['value' => '']), flow_context()); + (new FunctionContext(flow_context()))->eval( + ref('value')->htmlQuerySelectorAll('body div span'), + ['value' => ''], + schema(str_schema('value')), + ); } } diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/HTMLQuerySelectorTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/HTMLQuerySelectorTest.php index fba9e741fe..51b6c88711 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/HTMLQuerySelectorTest.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/HTMLQuerySelectorTest.php @@ -8,12 +8,15 @@ use Dom\HTMLDocument; use Flow\ETL\Exception\InvalidArgumentException; use Flow\ETL\Exception\RequiredPHPVersionException; +use Flow\ETL\Tests\Context\FunctionContext; use PHPUnit\Framework\Attributes\RequiresPhp; use PHPUnit\Framework\TestCase; use function Flow\ETL\DSL\flow_context; +use function Flow\ETL\DSL\html_schema; use function Flow\ETL\DSL\ref; -use function Flow\ETL\DSL\row; +use function Flow\ETL\DSL\schema; +use function Flow\ETL\DSL\str_schema; final class HTMLQuerySelectorTest extends TestCase { @@ -21,7 +24,11 @@ final class HTMLQuerySelectorTest extends TestCase public function test_getting_element_for_older_versions(): void { $this->expectException(RequiredPHPVersionException::class); - ref('value')->htmlQuerySelector('body div p')->eval(row(['value' => '']), flow_context()); + (new FunctionContext(flow_context()))->eval( + ref('value')->htmlQuerySelector('body div p'), + ['value' => ''], + schema(str_schema('value')), + ); } #[RequiresPhp('>= 8.4.0')] @@ -31,8 +38,11 @@ public function test_getting_elements_for_given_path(): void $html = HTMLDocument::createFromString( '
foobar
', ); - $result = ref('value')->htmlQuerySelector('body div span')->eval(row(['value' => $html]), flow_context()); - static::assertInstanceOf(Element::class, $result); + static::assertInstanceOf(Element::class, (new FunctionContext(flow_context()))->eval( + ref('value')->htmlQuerySelector('body div span'), + ['value' => $html], + schema(html_schema('value')), + )); } #[RequiresPhp('>= 8.4.0')] @@ -42,8 +52,11 @@ public function test_getting_null_when_nothing_found(): void $html = HTMLDocument::createFromString( '
foobar
', ); - $result = ref('value')->htmlQuerySelector('body div p')->eval(row(['value' => $html]), flow_context()); - static::assertNull($result); + static::assertNull((new FunctionContext(flow_context()))->eval( + ref('value')->htmlQuerySelector('body div p'), + ['value' => $html], + schema(html_schema('value')), + )); } #[RequiresPhp('>= 8.4.0')] @@ -52,6 +65,10 @@ public function test_invalid_value(): void $this->expectException(InvalidArgumentException::class); $this->expectExceptionMessage('Expected one of'); - ref('value')->htmlQuerySelector('body div span')->eval(row(['value' => '']), flow_context()); + (new FunctionContext(flow_context()))->eval( + ref('value')->htmlQuerySelector('body div span'), + ['value' => ''], + schema(str_schema('value')), + ); } } diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/HashTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/HashTest.php index 12dade91f1..fabf3824b3 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/HashTest.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/HashTest.php @@ -6,50 +6,74 @@ use DateTimeImmutable; use Flow\ETL\Hash\NativePHPHash; +use Flow\ETL\Tests\Context\FunctionContext; use Flow\ETL\Tests\FlowTestCase; use function Flow\ETL\DSL\concat; +use function Flow\ETL\DSL\datetime_schema; use function Flow\ETL\DSL\flow_context; use function Flow\ETL\DSL\hash; +use function Flow\ETL\DSL\list_schema; use function Flow\ETL\DSL\lit; use function Flow\ETL\DSL\ref; -use function Flow\ETL\DSL\row; +use function Flow\ETL\DSL\schema; +use function Flow\ETL\DSL\str_schema; +use function Flow\Types\DSL\type_list; +use function Flow\Types\DSL\type_string; final class HashTest extends FlowTestCase { public function test_hashing_array_value(): void { - static::assertSame('4450cf82dc53848e2bbe9798b70b0a6a', ref('value') - ->hash() - ->eval(row(['value' => ['test']]), flow_context())); + static::assertSame('4450cf82dc53848e2bbe9798b70b0a6a', (new FunctionContext(flow_context()))->eval( + ref('value')->hash(), + ['value' => [ + 'test', + ]], + schema(list_schema('value', type_list(type_string()))), + )); } public function test_hashing_concat(): void { static::assertSame( NativePHPHash::xxh128('test_test'), - hash(concat(ref('value'), lit('_'), ref('value')), new NativePHPHash('xxh128'))->eval(row([ - 'value' => 'test', - ]), flow_context()), + (new FunctionContext(flow_context()))->eval( + hash(concat(ref('value'), lit('_'), ref('value')), new NativePHPHash('xxh128')), + [ + 'value' => 'test', + ], + schema(str_schema('value')), + ), ); } public function test_hashing_datetime(): void { - static::assertSame('5347d10de38eb5570c044eb710a5120a', ref('value') - ->hash() - ->eval(row(['value' => new DateTimeImmutable('2021-01-01')]), flow_context())); + static::assertSame('5347d10de38eb5570c044eb710a5120a', (new FunctionContext(flow_context()))->eval( + ref('value')->hash(), + [ + 'value' => new DateTimeImmutable('2021-01-01'), + ], + schema(datetime_schema('value')), + )); } public function test_hashing_null_value(): void { - static::assertNull(ref('value')->hash()->eval(row(['value' => null]), flow_context())); + static::assertNull((new FunctionContext(flow_context()))->eval( + ref('value')->hash(), + ['value' => null], + schema(str_schema('value', nullable: true)), + )); } public function test_hashing_string_value(): void { - static::assertSame('6c78e0e3bd51d358d01e758642b85fb8', ref('value') - ->hash() - ->eval(row(['value' => 'test']), flow_context())); + static::assertSame('6c78e0e3bd51d358d01e758642b85fb8', (new FunctionContext(flow_context()))->eval( + ref('value')->hash(), + ['value' => 'test'], + schema(str_schema('value')), + )); } } diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/IndexOfLastTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/IndexOfLastTest.php index 78ce5d3b5a..45eb15ea0f 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/IndexOfLastTest.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/IndexOfLastTest.php @@ -5,22 +5,33 @@ namespace Flow\ETL\Tests\Unit\Function; use Flow\ETL\Exception\InvalidArgumentException; +use Flow\ETL\Tests\Context\FunctionContext; use Flow\ETL\Tests\FlowTestCase; +use function Flow\ETL\DSL\bool_schema; use function Flow\ETL\DSL\flow_context; use function Flow\ETL\DSL\ref; -use function Flow\ETL\DSL\row; +use function Flow\ETL\DSL\schema; +use function Flow\ETL\DSL\str_schema; final class IndexOfLastTest extends FlowTestCase { public function test_index_of_last_basic(): void { - static::assertSame(9, ref('str')->indexOfLast('l')->eval(row(['str' => 'hello world']), flow_context())); + static::assertSame(9, (new FunctionContext(flow_context()))->eval( + ref('str')->indexOfLast('l'), + ['str' => 'hello world'], + schema(str_schema('str')), + )); } public function test_index_of_last_not_found(): void { - static::assertNull(ref('str')->indexOfLast('x')->eval(row(['str' => 'hello world']), flow_context())); + static::assertNull((new FunctionContext(flow_context()))->eval( + ref('str')->indexOfLast('x'), + ['str' => 'hello world'], + schema(str_schema('str')), + )); } public function test_index_of_last_throws_on_null_needle(): void @@ -28,7 +39,11 @@ public function test_index_of_last_throws_on_null_needle(): void $this->expectException(InvalidArgumentException::class); $this->expectExceptionMessage('IndexOfLast function requires non-null string and needle'); - ref('str')->indexOfLast(ref('needle'))->eval(row(['str' => 'hello', 'needle' => null]), flow_context()); + (new FunctionContext(flow_context()))->eval( + ref('str')->indexOfLast(ref('needle')), + ['str' => 'hello', 'needle' => null], + schema(str_schema('str'), str_schema('needle', nullable: true)), + ); } public function test_index_of_last_throws_on_null_string(): void @@ -36,13 +51,19 @@ public function test_index_of_last_throws_on_null_string(): void $this->expectException(InvalidArgumentException::class); $this->expectExceptionMessage('IndexOfLast function requires non-null string and needle'); - ref('str')->indexOfLast('l')->eval(row(['str' => null]), flow_context()); + (new FunctionContext(flow_context()))->eval( + ref('str')->indexOfLast('l'), + ['str' => null], + schema(str_schema('str', nullable: true)), + ); } public function test_index_of_last_with_scalar_function_parameters(): void { - static::assertSame(9, ref('str') - ->indexOfLast(ref('needle'), ref('ignore_case')) - ->eval(row(['str' => 'hello world', 'needle' => 'L', 'ignore_case' => true]), flow_context())); + static::assertSame(9, (new FunctionContext(flow_context()))->eval( + ref('str')->indexOfLast(ref('needle'), ref('ignore_case')), + ['str' => 'hello world', 'needle' => 'L', 'ignore_case' => true], + schema(str_schema('str'), str_schema('needle'), bool_schema('ignore_case')), + )); } } diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/IndexOfTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/IndexOfTest.php index ac82553fde..df96877f7b 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/IndexOfTest.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/IndexOfTest.php @@ -5,25 +5,41 @@ namespace Flow\ETL\Tests\Unit\Function; use Flow\ETL\Exception\InvalidArgumentException; +use Flow\ETL\Tests\Context\FunctionContext; use Flow\ETL\Tests\FlowTestCase; use function Flow\ETL\DSL\flow_context; use function Flow\ETL\DSL\ref; -use function Flow\ETL\DSL\row; +use function Flow\ETL\DSL\schema; +use function Flow\ETL\DSL\str_schema; final class IndexOfTest extends FlowTestCase { public function test_index_of(): void { - static::assertSame(5, ref('str')->indexOf('x', offset: 5)->eval(row(['str' => 'AbBAsxa']), flow_context())); + static::assertSame(5, (new FunctionContext(flow_context()))->eval( + ref('str')->indexOf('x', offset: 5), + ['str' => 'AbBAsxa'], + schema(str_schema('str')), + )); - static::assertSame(0, ref('str') - ->indexOf('A', ignoreCase: true) - ->eval(row(['str' => 'abbbbb']), flow_context())); + static::assertSame(0, (new FunctionContext(flow_context()))->eval( + ref('str')->indexOf('A', ignoreCase: true), + ['str' => 'abbbbb'], + schema(str_schema('str')), + )); - static::assertSame(5, ref('str')->indexOf('x', offset: 5)->eval(row(['str' => 'AbBAsxa']), flow_context())); + static::assertSame(5, (new FunctionContext(flow_context()))->eval( + ref('str')->indexOf('x', offset: 5), + ['str' => 'AbBAsxa'], + schema(str_schema('str')), + )); - static::assertNull(ref('str')->indexOf('x', offset: 2)->eval(row(['str' => 'Abba']), flow_context())); + static::assertNull((new FunctionContext(flow_context()))->eval( + ref('str')->indexOf('x', offset: 2), + ['str' => 'Abba'], + schema(str_schema('str')), + )); } public function test_index_of_throws_on_null_needle(): void @@ -31,7 +47,11 @@ public function test_index_of_throws_on_null_needle(): void $this->expectException(InvalidArgumentException::class); $this->expectExceptionMessage('IndexOf function requires non-null string and needle'); - ref('str')->indexOf(ref('needle'))->eval(row(['str' => 'x', 'needle' => null]), flow_context()); + (new FunctionContext(flow_context()))->eval( + ref('str')->indexOf(ref('needle')), + ['str' => 'x', 'needle' => null], + schema(str_schema('str'), str_schema('needle', nullable: true)), + ); } public function test_index_of_throws_on_null_string(): void @@ -39,6 +59,10 @@ public function test_index_of_throws_on_null_string(): void $this->expectException(InvalidArgumentException::class); $this->expectExceptionMessage('IndexOf function requires non-null string and needle'); - ref('str')->indexOf('x')->eval(row(['str' => null]), flow_context()); + (new FunctionContext(flow_context()))->eval( + ref('str')->indexOf('x'), + ['str' => null], + schema(str_schema('str', nullable: true)), + ); } } diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/IsEmptyTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/IsEmptyTest.php index a5f2299c82..8e96244a90 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/IsEmptyTest.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/IsEmptyTest.php @@ -4,31 +4,49 @@ namespace Flow\ETL\Tests\Unit\Function; +use Flow\ETL\Tests\Context\FunctionContext; use Flow\ETL\Tests\FlowTestCase; use function Flow\ETL\DSL\flow_context; use function Flow\ETL\DSL\ref; -use function Flow\ETL\DSL\row; +use function Flow\ETL\DSL\schema; +use function Flow\ETL\DSL\str_schema; final class IsEmptyTest extends FlowTestCase { public function test_is_empty_empty_string(): void { - static::assertTrue(ref('str')->isEmpty()->eval(row(['str' => '']), flow_context())); + static::assertTrue((new FunctionContext(flow_context()))->eval( + ref('str')->isEmpty(), + ['str' => ''], + schema(str_schema('str')), + )); } public function test_is_empty_non_empty_string(): void { - static::assertFalse(ref('str')->isEmpty()->eval(row(['str' => 'hello']), flow_context())); + static::assertFalse((new FunctionContext(flow_context()))->eval( + ref('str')->isEmpty(), + ['str' => 'hello'], + schema(str_schema('str')), + )); } public function test_is_empty_returns_null_for_null_input(): void { - static::assertNull(ref('str')->isEmpty()->eval(row(['str' => null]), flow_context())); + static::assertNull((new FunctionContext(flow_context()))->eval( + ref('str')->isEmpty(), + ['str' => null], + schema(str_schema('str', nullable: true)), + )); } public function test_is_empty_single_character_string(): void { - static::assertFalse(ref('str')->isEmpty()->eval(row(['str' => 'a']), flow_context())); + static::assertFalse((new FunctionContext(flow_context()))->eval( + ref('str')->isEmpty(), + ['str' => 'a'], + schema(str_schema('str')), + )); } } diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/IsInTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/IsInTest.php index 18de9f64d1..de03f69d32 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/IsInTest.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/IsInTest.php @@ -6,13 +6,31 @@ use DateInterval; use DateTimeImmutable; +use DateTimeZone; +use Flow\ETL\Tests\Context\FunctionContext; +use Flow\ETL\Tests\Fixtures\Enum\BackedStringEnum; use Flow\ETL\Tests\FlowTestCase; use Flow\Types\Exception\InvalidArgumentException; +use Flow\Types\Type; +use Generator; +use PHPUnit\Framework\Attributes\DataProvider; +use function Flow\ETL\DSL\definition_from_type; use function Flow\ETL\DSL\flow_context; use function Flow\ETL\DSL\lit; use function Flow\ETL\DSL\ref; -use function Flow\ETL\DSL\row; +use function Flow\ETL\DSL\schema; +use function Flow\ETL\DSL\str_schema; +use function Flow\Types\DSL\type_date; +use function Flow\Types\DSL\type_datetime; +use function Flow\Types\DSL\type_enum; +use function Flow\Types\DSL\type_json; +use function Flow\Types\DSL\type_list; +use function Flow\Types\DSL\type_map; +use function Flow\Types\DSL\type_string; +use function Flow\Types\DSL\type_time; +use function Flow\Types\DSL\type_uuid; +use function Flow\Types\DSL\type_xml_element; final class IsInTest extends FlowTestCase { @@ -27,42 +45,53 @@ public function test_is_in_accepts_a_container_haystack_without_a_single_element ->returns() ->toString(), ); - static::assertTrue(lit('a')->isIn(lit([1, 'a']))->eval(row([]), flow_context())); + static::assertTrue((new FunctionContext(flow_context()))->eval(lit('a')->isIn(lit([1, 'a'])), [], schema())); } public function test_is_in_matches_two_equal_datetimes(): void { // in_array(strict: true) compared these by identity and answered false - static::assertTrue( + static::assertTrue((new FunctionContext(flow_context()))->eval( lit(new DateTimeImmutable('2024-01-01 10:00:00')) - ->isIn(lit([new DateTimeImmutable('2024-01-01 10:00:00')])) - ->eval(row([]), flow_context()), - ); + ->isIn(lit([new DateTimeImmutable('2024-01-01 10:00:00')])), + [], + schema(), + )); } - public function test_is_in_does_not_match_two_equal_date_intervals(): void + public function test_is_in_matches_two_equal_date_intervals_by_value(): void { - // Equals::eval() has a DateInterval arm and IsIn deliberately does not - static::assertTrue( - lit(new DateInterval('PT1H'))->equals(lit(new DateInterval('PT60M')))->eval(row([]), flow_context()), - ); - static::assertFalse( - lit(new DateInterval('PT1H'))->isIn(lit([new DateInterval('PT60M')]))->eval(row([]), flow_context()), - ); + // a time list of the needle's type compares by the physical form, as equals() does + static::assertTrue((new FunctionContext(flow_context()))->eval( + lit(new DateInterval('PT1H'))->equals(lit(new DateInterval('PT60M'))), + [], + schema(), + )); + static::assertTrue((new FunctionContext(flow_context()))->eval( + lit(new DateInterval('PT1H'))->isIn(lit([new DateInterval('PT60M')])), + [], + schema(), + )); } public function test_is_in_over_an_empty_haystack(): void { static::assertSame('boolean', lit('a')->isIn(lit([]))->returns()->toString()); - static::assertFalse(lit('a')->isIn(lit([]))->eval(row([]), flow_context())); + static::assertFalse((new FunctionContext(flow_context()))->eval(lit('a')->isIn(lit([])), [], schema())); } public function test_is_in_agrees_with_equals_on_numeric_operands(): void { - $row = row(['a' => '1']); - - static::assertTrue(ref('a')->equals(lit(1))->eval($row, flow_context())); - static::assertTrue(ref('a')->isIn(lit([1]))->eval($row, flow_context())); + static::assertTrue((new FunctionContext(flow_context()))->eval( + ref('a')->equals(lit(1)), + ['a' => '1'], + schema(str_schema('a')), + )); + static::assertTrue((new FunctionContext(flow_context()))->eval( + ref('a')->isIn(lit([1])), + ['a' => '1'], + schema(str_schema('a')), + )); } public function test_is_in_refuses_an_incomparable_element_type(): void @@ -77,26 +106,102 @@ public function test_is_in_refuses_an_incomparable_element_type(): void public function test_a_match_beats_a_null_element(): void { - static::assertTrue(lit(5)->isIn(lit([1, null, 5]))->eval(row([]), flow_context())); + static::assertTrue((new FunctionContext(flow_context()))->eval(lit(5)->isIn(lit([1, null, 5])), [], schema())); } public function test_no_match_with_a_null_element_is_null(): void { - static::assertNull(lit(7)->isIn(lit([1, null, 5]))->eval(row([]), flow_context())); + static::assertNull((new FunctionContext(flow_context()))->eval(lit(7)->isIn(lit([1, null, 5])), [], schema())); } public function test_no_match_without_a_null_element_is_false(): void { - static::assertFalse(lit(7)->isIn(lit([1, 3, 5]))->eval(row([]), flow_context())); + static::assertFalse((new FunctionContext(flow_context()))->eval(lit(7)->isIn(lit([1, 3, 5])), [], schema())); } public function test_a_null_needle_is_null(): void { - static::assertNull(lit(null)->isIn(lit([1, 3, 5]))->eval(row([]), flow_context())); + static::assertNull((new FunctionContext(flow_context()))->eval(lit(null)->isIn(lit([1, 3, 5])), [], schema())); } public function test_a_null_haystack_is_null(): void { - static::assertNull(lit(5)->isIn(lit(null))->eval(row(['x' => []]), flow_context())); + static::assertNull((new FunctionContext(flow_context()))->eval(lit(5)->isIn(lit(null)), [], schema())); + } + + /** + * @return Generator, mixed, mixed, bool}> + */ + public static function values_of_one_type(): Generator + { + yield 'uuid' => [ + type_uuid(), + '00000000-0000-4000-8000-000000000001', + '00000000-0000-4000-8000-000000000001', + true, + ]; + yield 'other uuid' => [ + type_uuid(), + '00000000-0000-4000-8000-000000000001', + '00000000-0000-4000-8000-000000000002', + false, + ]; + yield 'json' => [type_json(), '{"a":1,"b":2}', '{"a":1,"b":2}', true]; + yield 'json compares its stored text' => [type_json(), '{"a":1,"b":2}', '{"b":2,"a":1}', false]; + yield 'enum' => [type_enum(BackedStringEnum::class), BackedStringEnum::one, BackedStringEnum::one, true]; + yield 'datetime of the same instant in another zone' => [ + type_datetime(), + new DateTimeImmutable('2024-01-01 12:00:00', new DateTimeZone('UTC')), + new DateTimeImmutable('2024-01-01 13:00:00', new DateTimeZone('Europe/Warsaw')), + true, + ]; + yield 'date' => [type_date(), new DateTimeImmutable('2024-01-01'), new DateTimeImmutable('2024-01-01'), true]; + yield 'time' => [type_time(), new DateInterval('PT60M'), new DateInterval('PT1H'), true]; + yield 'xml elements of different documents' => [ + type_xml_element(), + type_xml_element()->cast('

a

')->firstElementChild, + type_xml_element()->cast('

a

')->firstElementChild, + true, + ]; + yield 'other xml elements' => [ + type_xml_element(), + type_xml_element()->cast('

a

')->firstElementChild, + type_xml_element()->cast('

b

')->firstElementChild, + false, + ]; + } + + /** + * @param Type $type + */ + #[DataProvider('values_of_one_type')] + public function test_a_list_of_the_needle_type_matches_by_value( + Type $type, + mixed $needle, + mixed $element, + bool $in, + ): void { + static::assertSame($in, (new FunctionContext(flow_context()))->eval( + ref('a')->isIn(ref('h')), + ['a' => $needle, 'h' => [$element]], + schema(definition_from_type('a', $type), definition_from_type('h', type_list($type))), + )); + } + + /** + * @param Type $type + */ + #[DataProvider('values_of_one_type')] + public function test_a_map_of_the_needle_type_matches_by_value( + Type $type, + mixed $needle, + mixed $element, + bool $in, + ): void { + static::assertSame($in, (new FunctionContext(flow_context()))->eval( + ref('a')->isIn(ref('h')), + ['a' => $needle, 'h' => ['k' => $element]], + schema(definition_from_type('a', $type), definition_from_type('h', type_map(type_string(), $type))), + )); } } diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/IsUtf8Test.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/IsUtf8Test.php index 85052e04ae..190565014c 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/IsUtf8Test.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/IsUtf8Test.php @@ -4,23 +4,37 @@ namespace Flow\ETL\Tests\Unit\Function; +use Flow\ETL\Tests\Context\FunctionContext; use Flow\ETL\Tests\FlowTestCase; use function Flow\ETL\DSL\flow_context; use function Flow\ETL\DSL\ref; -use function Flow\ETL\DSL\row; +use function Flow\ETL\DSL\schema; +use function Flow\ETL\DSL\str_schema; final class IsUtf8Test extends FlowTestCase { public function test_is_utf8_returns_null(): void { - static::assertNull(ref('str')->isUtf8()->eval(row(['str' => null]), flow_context())); + static::assertNull((new FunctionContext(flow_context()))->eval( + ref('str')->isUtf8(), + ['str' => null], + schema(str_schema('str', nullable: true)), + )); } public function test_is_utf_8(): void { - static::assertTrue(ref('str')->isUtf8()->eval(row(['str' => 'Lorem Ipsum']), flow_context())); + static::assertTrue((new FunctionContext(flow_context()))->eval( + ref('str')->isUtf8(), + ['str' => 'Lorem Ipsum'], + schema(str_schema('str')), + )); - static::assertFalse(ref('str')->isUtf8()->eval(row(['str' => "\xc3\x28"]), flow_context())); + static::assertFalse((new FunctionContext(flow_context()))->eval( + ref('str')->isUtf8(), + ['str' => "\xc3\x28"], + schema(str_schema('str')), + )); } } diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/JsonDecodeTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/JsonDecodeTest.php index eaa65948a9..53b51b6181 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/JsonDecodeTest.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/JsonDecodeTest.php @@ -4,12 +4,17 @@ namespace Flow\ETL\Tests\Unit\Function; +use Flow\ETL\Exception\EvaluationException; use Flow\ETL\Exception\InvalidArgumentException; +use Flow\ETL\Tests\Context\FunctionContext; use Flow\ETL\Tests\FlowTestCase; +use JsonException; use function Flow\ETL\DSL\flow_context; +use function Flow\ETL\DSL\int_schema; use function Flow\ETL\DSL\ref; -use function Flow\ETL\DSL\row; +use function Flow\ETL\DSL\schema; +use function Flow\ETL\DSL\str_schema; final class JsonDecodeTest extends FlowTestCase { @@ -17,7 +22,11 @@ public function test_json_decode_expression(): void { static::assertSame( ['value' => 1], - ref('value')->jsonDecode()->eval(row(['value' => '{"value": 1}']), flow_context()), + (new FunctionContext(flow_context()))->eval( + ref('value')->jsonDecode(), + ['value' => '{"value": 1}'], + schema(str_schema('value')), + ), ); } @@ -26,7 +35,25 @@ public function test_json_decode_expression_with_invalid_json(): void $this->expectException(InvalidArgumentException::class); $this->expectExceptionMessage('JsonDecode error: Syntax error'); - ref('value')->jsonDecode()->eval(row(['value' => '{"value": 1']), flow_context()); + (new FunctionContext(flow_context()))->eval( + ref('value')->jsonDecode(), + ['value' => '{"value": 1'], + schema(str_schema('value')), + ); + } + + public function test_invalid_json_keeps_the_decoder_error_as_the_cause(): void + { + try { + (new FunctionContext(flow_context()))->eval( + ref('value')->jsonDecode(), + ['value' => '{"value": 1'], + schema(str_schema('value')), + ); + static::fail('Invalid JSON must be refused.'); + } catch (EvaluationException $e) { + static::assertInstanceOf(JsonException::class, $e->getPrevious()?->getPrevious()); + } } public function test_json_decode_on_non_json_value(): void @@ -34,7 +61,11 @@ public function test_json_decode_on_non_json_value(): void $this->expectException(InvalidArgumentException::class); $this->expectExceptionMessage('JsonDecode function requires string, array, or Json value'); - ref('value')->jsonDecode()->eval(row(['value' => 125]), flow_context()); + (new FunctionContext(flow_context()))->eval( + ref('value')->jsonDecode(), + ['value' => 125], + schema(int_schema('value')), + ); } public function test_json_decode_on_scalar_json_throws(): void @@ -44,6 +75,10 @@ public function test_json_decode_on_scalar_json_throws(): void 'JsonDecode function requires JSON that decodes to an array, cast scalar JSON instead', ); - ref('value')->jsonDecode()->eval(row(['value' => '5']), flow_context()); + (new FunctionContext(flow_context()))->eval( + ref('value')->jsonDecode(), + ['value' => '5'], + schema(str_schema('value')), + ); } } diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/JsonEncodeTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/JsonEncodeTest.php index 83460a5dd2..18acd19570 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/JsonEncodeTest.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/JsonEncodeTest.php @@ -5,41 +5,65 @@ namespace Flow\ETL\Tests\Unit\Function; use DateTimeImmutable; +use Flow\ETL\Tests\Context\FunctionContext; use Flow\ETL\Tests\FlowTestCase; use Flow\Types\Value\Json; +use function Flow\ETL\DSL\datetime_schema; use function Flow\ETL\DSL\flow_context; +use function Flow\ETL\DSL\int_schema; +use function Flow\ETL\DSL\map_schema; use function Flow\ETL\DSL\ref; -use function Flow\ETL\DSL\row; +use function Flow\ETL\DSL\schema; +use function Flow\ETL\DSL\str_schema; +use function Flow\Types\DSL\type_integer; +use function Flow\Types\DSL\type_map; +use function Flow\Types\DSL\type_string; final class JsonEncodeTest extends FlowTestCase { public function test_json_encode_on_datetime(): void { - $result = ref('value') - ->jsonEncode() - ->eval(row(['value' => new DateTimeImmutable('2021-01-01')]), flow_context()); - - static::assertInstanceOf(Json::class, $result); - static::assertSame( - '{"date":"2021-01-01 00:00:00.000000","timezone_type":3,"timezone":"UTC"}', - $result->toString(), - ); + // json_encode() of a non-container operand declares string, so the encoded object is a string + static::assertSame('{"date":"2021-01-01 00:00:00.000000","timezone_type":3,"timezone":"UTC"}', (new FunctionContext( + flow_context(), + ))->eval( + ref('value')->jsonEncode(), + [ + 'value' => new DateTimeImmutable('2021-01-01'), + ], + schema(datetime_schema('value')), + )); } public function test_json_encode_on_integer(): void { - static::assertSame('125', ref('value')->jsonEncode()->eval(row(['value' => 125]), flow_context())); + static::assertSame('125', (new FunctionContext(flow_context()))->eval( + ref('value')->jsonEncode(), + ['value' => 125], + schema(int_schema('value')), + )); } public function test_json_encode_on_string(): void { - static::assertSame('"test"', ref('value')->jsonEncode()->eval(row(['value' => 'test']), flow_context())); + static::assertSame('"test"', (new FunctionContext(flow_context()))->eval( + ref('value')->jsonEncode(), + ['value' => 'test'], + schema(str_schema('value')), + )); } public function test_json_encode_on_valid_associative_array(): void { - $result = ref('value')->jsonEncode()->eval(row(['value' => ['value' => 1]]), flow_context()); + // @mago-ignore analysis:mixed-assignment + $result = (new FunctionContext(flow_context()))->eval( + ref('value')->jsonEncode(), + ['value' => [ + 'value' => 1, + ]], + schema(map_schema('value', type_map(type_string(), type_integer()))), + ); static::assertInstanceOf(Json::class, $result); static::assertSame('{"value":1}', $result->toString()); diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/LastTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/LastTest.php index bc3496b9a5..eab2e44197 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/LastTest.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/LastTest.php @@ -4,15 +4,19 @@ namespace Flow\ETL\Tests\Unit\Function; +use Flow\ETL\Exception\InvalidArgumentException; use Flow\ETL\Function\ReferenceResolver; use Flow\ETL\Tests\FlowTestCase; +use function array_map; +use function Flow\ETL\DSL\array_to_rows; +use function Flow\ETL\DSL\first; use function Flow\ETL\DSL\flow_context; use function Flow\ETL\DSL\int_schema; use function Flow\ETL\DSL\last; use function Flow\ETL\DSL\ref; -use function Flow\ETL\DSL\row; use function Flow\ETL\DSL\schema; +use function Flow\ETL\DSL\str_schema; final class LastTest extends FlowTestCase { @@ -25,11 +29,17 @@ public function test_aggregation_last_value(): void { $aggregator = last(ref('int')); - $aggregator->aggregate(row(['int' => '10']), flow_context()); - $aggregator->aggregate(row(['int' => '20']), flow_context()); - $aggregator->aggregate(row(['int' => '55']), flow_context()); - $aggregator->aggregate(row(['int' => '25']), flow_context()); - $aggregator->aggregate(row(['not_int' => null]), flow_context()); + $aggregator->aggregate(array_to_rows([['int' => '10']], schema(str_schema('int'))), [0], flow_context()); + $aggregator->aggregate(array_to_rows([['int' => '20']], schema(str_schema('int'))), [0], flow_context()); + $aggregator->aggregate(array_to_rows([['int' => '55']], schema(str_schema('int'))), [0], flow_context()); + $aggregator->aggregate(array_to_rows([['int' => '25']], schema(str_schema('int'))), [0], flow_context()); + $aggregator->aggregate( + array_to_rows([[ + 'not_int' => null, + ]], schema(str_schema('not_int', nullable: true))), + [0], + flow_context(), + ); static::assertSame('25', $aggregator->value()); } @@ -64,4 +74,72 @@ public function test_last_over_an_int_column_declares_optional_integer(): void ->toString(), ); } + + public function test_aggregate_reads_only_given_indices(): void + { + $aggregator = last(ref('v')); + $aggregator->aggregate( + array_to_rows(array_map(static fn(mixed $v): array => ['v' => $v], [1, 2, 3, 4]), schema(int_schema('v'))), + [1, 3], + flow_context(), + ); + + static::assertSame(4, $aggregator->value()); + } + + public function test_merge_of_two_halves_equals_one_pass(): void + { + $rows = array_to_rows(array_map(static fn(mixed $v): array => ['v' => $v], [ + 1, + 2, + 3, + 4, + ]), schema(int_schema('v'))); + $onePass = last(ref('v')); + $onePass->aggregate($rows, [0, 1, 2, 3], flow_context()); + $left = last(ref('v')); + $left->aggregate($rows, [0, 1], flow_context()); + $right = last(ref('v')); + $right->aggregate($rows, [2, 3], flow_context()); + + $left->merge($right, flow_context()); + + static::assertSame($onePass->value(), $left->value()); + } + + public function test_merge_refuses_another_class(): void + { + $this->expectException(InvalidArgumentException::class); + + last(ref('v'))->merge(first(ref('v')), flow_context()); + } + + public function test_merge_after_a_null_only_half_takes_the_null(): void + { + $rows = array_to_rows([ + ['v' => 1], + ['v' => 2], + ['v' => null], + ['v' => null], + ], schema(int_schema('v', nullable: true))); + $left = last(ref('v')); + $left->aggregate($rows, [0, 1], flow_context()); + $right = last(ref('v')); + $right->aggregate($rows, [2, 3], flow_context()); + + $left->merge($right, flow_context()); + + static::assertNull($left->value()); + } + + public function test_merge_with_an_empty_half_keeps_the_value(): void + { + $rows = array_to_rows([['v' => 1]], schema(int_schema('v'))); + $left = last(ref('v')); + $left->aggregate($rows, [0], flow_context()); + + $left->merge(last(ref('v')), flow_context()); + + static::assertSame(1, $left->value()); + } } diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/LeastTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/LeastTest.php index e05ff3df7c..dab159f076 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/LeastTest.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/LeastTest.php @@ -7,6 +7,7 @@ use DateTimeImmutable; use Flow\ETL\Function\ReferenceResolver; use Flow\ETL\Function\ScalarFunction; +use Flow\ETL\Tests\Context\FunctionContext; use Flow\ETL\Tests\FlowTestCase; use function Flow\ETL\DSL\flow_context; @@ -14,7 +15,6 @@ use function Flow\ETL\DSL\least; use function Flow\ETL\DSL\lit; use function Flow\ETL\DSL\ref; -use function Flow\ETL\DSL\row; use function Flow\ETL\DSL\schema; final class LeastTest extends FlowTestCase @@ -46,18 +46,34 @@ public function test_an_integer_and_a_string_column_have_no_common_type(): void public function test_null_only_when_every_argument_is_null(): void { - static::assertNull(least(null, null)->eval(row(['int' => 4]), flow_context())); + static::assertNull((new FunctionContext(flow_context()))->eval( + least(null, null), + [ + 'int' => 4, + ], + schema(int_schema('int')), + )); } public function test_least_value(): void { $lest = least(10, 20, ref('int'), 40); - static::assertSame(10, $lest->eval(row(['int' => 55]), flow_context())); + static::assertSame(10, (new FunctionContext(flow_context()))->eval( + $lest, + ['int' => 55], + schema(int_schema('int')), + )); } public function test_least_with_null(): void { - static::assertSame(4, least(null, 20, ref('int'), 1257)->eval(row(['int' => 4]), flow_context())); + static::assertSame(4, (new FunctionContext(flow_context()))->eval( + least(null, 20, ref('int'), 1257), + [ + 'int' => 4, + ], + schema(int_schema('int')), + )); } } diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/LessThanEqualTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/LessThanEqualTest.php index 7cdd3e2696..7f3b0d1620 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/LessThanEqualTest.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/LessThanEqualTest.php @@ -6,13 +6,22 @@ use DateInterval; use DateTimeImmutable; +use Flow\ETL\Tests\Context\FunctionContext; use Flow\ETL\Tests\FlowTestCase; use Flow\Types\Exception\InvalidArgumentException; +use function Flow\ETL\DSL\datetime_schema; +use function Flow\ETL\DSL\float_schema; use function Flow\ETL\DSL\flow_context; +use function Flow\ETL\DSL\int_schema; +use function Flow\ETL\DSL\list_schema; use function Flow\ETL\DSL\lit; use function Flow\ETL\DSL\ref; -use function Flow\ETL\DSL\row; +use function Flow\ETL\DSL\schema; +use function Flow\ETL\DSL\str_schema; +use function Flow\ETL\DSL\time_schema; +use function Flow\Types\DSL\type_integer; +use function Flow\Types\DSL\type_list; final class LessThanEqualTest extends FlowTestCase { @@ -20,93 +29,165 @@ public function test_less_than_equal_arrays(): void { $context = flow_context(); - static::assertTrue(ref('v')->lessThanEqual(lit([1, 9]))->eval(row(['v' => [1, 0]]), $context)); - static::assertTrue(ref('v')->lessThanEqual(lit([1, 0]))->eval(row(['v' => [1, 0]]), $context)); - static::assertFalse(ref('v')->lessThanEqual(lit([1, 0]))->eval(row(['v' => [1, 9]]), $context)); + static::assertTrue((new FunctionContext($context))->eval( + ref('v')->lessThanEqual(lit([1, 9])), + ['v' => [1, 0]], + schema(list_schema('v', type_list(type_integer()))), + )); + static::assertTrue((new FunctionContext($context))->eval( + ref('v')->lessThanEqual(lit([1, 0])), + ['v' => [1, 0]], + schema(list_schema('v', type_list(type_integer()))), + )); + static::assertFalse((new FunctionContext($context))->eval( + ref('v')->lessThanEqual(lit([1, 0])), + ['v' => [1, 9]], + schema(list_schema('v', type_list(type_integer()))), + )); } public function test_less_than_equal_datetimes(): void { $context = flow_context(); - static::assertTrue( - ref('v') - ->lessThanEqual(lit(new DateTimeImmutable('2024-06-01'))) - ->eval(row(['v' => new DateTimeImmutable('2024-01-01')]), $context), - ); - static::assertTrue( - ref('v') - ->lessThanEqual(lit(new DateTimeImmutable('2024-01-01'))) - ->eval(row(['v' => new DateTimeImmutable('2024-01-01')]), $context), - ); - static::assertFalse( - ref('v') - ->lessThanEqual(lit(new DateTimeImmutable('2024-01-01'))) - ->eval(row(['v' => new DateTimeImmutable('2024-06-01')]), $context), - ); + static::assertTrue((new FunctionContext($context))->eval( + ref('v')->lessThanEqual(lit(new DateTimeImmutable('2024-06-01'))), + [ + 'v' => new DateTimeImmutable('2024-01-01'), + ], + schema(datetime_schema('v')), + )); + static::assertTrue((new FunctionContext($context))->eval( + ref('v')->lessThanEqual(lit(new DateTimeImmutable('2024-01-01'))), + [ + 'v' => new DateTimeImmutable('2024-01-01'), + ], + schema(datetime_schema('v')), + )); + static::assertFalse((new FunctionContext($context))->eval( + ref('v')->lessThanEqual(lit(new DateTimeImmutable('2024-01-01'))), + [ + 'v' => new DateTimeImmutable('2024-06-01'), + ], + schema(datetime_schema('v')), + )); } public function test_less_than_equal_floats(): void { $context = flow_context(); - static::assertTrue(ref('v')->lessThanEqual(lit(2.5))->eval(row(['v' => 1.5]), $context)); - static::assertTrue(ref('v')->lessThanEqual(lit(1.5))->eval(row(['v' => 1.5]), $context)); - static::assertFalse(ref('v')->lessThanEqual(lit(1.5))->eval(row(['v' => 2.5]), $context)); + static::assertTrue((new FunctionContext($context))->eval( + ref('v')->lessThanEqual(lit(2.5)), + ['v' => 1.5], + schema(float_schema('v')), + )); + static::assertTrue((new FunctionContext($context))->eval( + ref('v')->lessThanEqual(lit(1.5)), + ['v' => 1.5], + schema(float_schema('v')), + )); + static::assertFalse((new FunctionContext($context))->eval( + ref('v')->lessThanEqual(lit(1.5)), + ['v' => 2.5], + schema(float_schema('v')), + )); } public function test_less_than_equal_integers(): void { $context = flow_context(); - static::assertTrue(ref('v')->lessThanEqual(lit(20))->eval(row(['v' => 10]), $context)); - static::assertTrue(ref('v')->lessThanEqual(lit(10))->eval(row(['v' => 10]), $context)); - static::assertFalse(ref('v')->lessThanEqual(lit(10))->eval(row(['v' => 20]), $context)); + static::assertTrue((new FunctionContext($context))->eval( + ref('v')->lessThanEqual(lit(20)), + ['v' => 10], + schema(int_schema('v')), + )); + static::assertTrue((new FunctionContext($context))->eval( + ref('v')->lessThanEqual(lit(10)), + ['v' => 10], + schema(int_schema('v')), + )); + static::assertFalse((new FunctionContext($context))->eval( + ref('v')->lessThanEqual(lit(10)), + ['v' => 20], + schema(int_schema('v')), + )); } public function test_less_than_equal_returns_null_for_null_array(): void { - static::assertNull(ref('v')->lessThanEqual(lit([1, 0]))->eval(row(['v' => null]), flow_context())); + static::assertNull((new FunctionContext(flow_context()))->eval( + ref('v')->lessThanEqual(lit([1, 0])), + ['v' => null], + schema(list_schema('v', type_list(type_integer()), nullable: true)), + )); } public function test_less_than_equal_returns_null_for_null_datetime(): void { - static::assertNull( - ref('v')->lessThanEqual(lit(new DateTimeImmutable('2024-01-01')))->eval(row(['v' => null]), flow_context()), - ); + static::assertNull((new FunctionContext(flow_context()))->eval( + ref('v')->lessThanEqual(lit(new DateTimeImmutable('2024-01-01'))), + ['v' => null], + schema(datetime_schema('v', nullable: true)), + )); } public function test_less_than_equal_returns_null_for_null_left(): void { - static::assertNull(ref('v')->lessThanEqual(lit(10))->eval(row(['v' => null]), flow_context())); + static::assertNull((new FunctionContext(flow_context()))->eval( + ref('v')->lessThanEqual(lit(10)), + ['v' => null], + schema(str_schema('v', nullable: true)), + )); } public function test_less_than_equal_returns_null_for_null_right(): void { - static::assertNull( - ref('v')->lessThanEqual(ref('other'))->eval(row(['v' => 10, 'other' => null]), flow_context()), - ); + static::assertNull((new FunctionContext(flow_context()))->eval( + ref('v')->lessThanEqual(ref('other')), + ['v' => 10, 'other' => null], + schema(int_schema('v'), str_schema('other', nullable: true)), + )); } public function test_less_than_equal_returns_null_for_null_string(): void { - static::assertNull(ref('v')->lessThanEqual(lit('a'))->eval(row(['v' => null]), flow_context())); + static::assertNull((new FunctionContext(flow_context()))->eval( + ref('v')->lessThanEqual(lit('a')), + ['v' => null], + schema(str_schema('v', nullable: true)), + )); } public function test_less_than_equal_returns_null_for_null_time_interval(): void { - static::assertNull( - ref('v')->lessThanEqual(lit(new DateInterval('PT1H')))->eval(row(['v' => null]), flow_context()), - ); + static::assertNull((new FunctionContext(flow_context()))->eval( + ref('v')->lessThanEqual(lit(new DateInterval('PT1H'))), + ['v' => null], + schema(time_schema('v', nullable: true)), + )); } public function test_less_than_equal_strings(): void { $context = flow_context(); - static::assertTrue(ref('v')->lessThanEqual(lit('banana'))->eval(row(['v' => 'apple']), $context)); - static::assertTrue(ref('v')->lessThanEqual(lit('apple'))->eval(row(['v' => 'apple']), $context)); - static::assertFalse(ref('v')->lessThanEqual(lit('apple'))->eval(row(['v' => 'banana']), $context)); + static::assertTrue((new FunctionContext($context))->eval( + ref('v')->lessThanEqual(lit('banana')), + ['v' => 'apple'], + schema(str_schema('v')), + )); + static::assertTrue((new FunctionContext($context))->eval( + ref('v')->lessThanEqual(lit('apple')), + ['v' => 'apple'], + schema(str_schema('v')), + )); + static::assertFalse((new FunctionContext($context))->eval( + ref('v')->lessThanEqual(lit('apple')), + ['v' => 'banana'], + schema(str_schema('v')), + )); } public function test_less_than_equal_throws_on_incompatible_types(): void @@ -120,20 +201,20 @@ public function test_less_than_equal_time_intervals(): void { $context = flow_context(); - static::assertTrue( - ref('v') - ->lessThanEqual(lit(new DateInterval('PT5H'))) - ->eval(row(['v' => new DateInterval('PT1H')]), $context), - ); - static::assertTrue( - ref('v') - ->lessThanEqual(lit(new DateInterval('PT1H'))) - ->eval(row(['v' => new DateInterval('PT1H')]), $context), - ); - static::assertFalse( - ref('v') - ->lessThanEqual(lit(new DateInterval('PT1H'))) - ->eval(row(['v' => new DateInterval('PT5H')]), $context), - ); + static::assertTrue((new FunctionContext($context))->eval( + ref('v')->lessThanEqual(lit(new DateInterval('PT5H'))), + ['v' => new DateInterval('PT1H')], + schema(time_schema('v')), + )); + static::assertTrue((new FunctionContext($context))->eval( + ref('v')->lessThanEqual(lit(new DateInterval('PT1H'))), + ['v' => new DateInterval('PT1H')], + schema(time_schema('v')), + )); + static::assertFalse((new FunctionContext($context))->eval( + ref('v')->lessThanEqual(lit(new DateInterval('PT1H'))), + ['v' => new DateInterval('PT5H')], + schema(time_schema('v')), + )); } } diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/LessThanTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/LessThanTest.php index ea78658511..e24c88e4c0 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/LessThanTest.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/LessThanTest.php @@ -6,13 +6,22 @@ use DateInterval; use DateTimeImmutable; +use Flow\ETL\Tests\Context\FunctionContext; use Flow\ETL\Tests\FlowTestCase; use Flow\Types\Exception\InvalidArgumentException; +use function Flow\ETL\DSL\datetime_schema; +use function Flow\ETL\DSL\float_schema; use function Flow\ETL\DSL\flow_context; +use function Flow\ETL\DSL\int_schema; +use function Flow\ETL\DSL\list_schema; use function Flow\ETL\DSL\lit; use function Flow\ETL\DSL\ref; -use function Flow\ETL\DSL\row; +use function Flow\ETL\DSL\schema; +use function Flow\ETL\DSL\str_schema; +use function Flow\ETL\DSL\time_schema; +use function Flow\Types\DSL\type_integer; +use function Flow\Types\DSL\type_list; final class LessThanTest extends FlowTestCase { @@ -20,89 +29,165 @@ public function test_less_than_arrays(): void { $context = flow_context(); - static::assertTrue(ref('v')->lessThan(lit([1, 9]))->eval(row(['v' => [1, 0]]), $context)); - static::assertFalse(ref('v')->lessThan(lit([1, 0]))->eval(row(['v' => [1, 9]]), $context)); - static::assertFalse(ref('v')->lessThan(lit([1, 0]))->eval(row(['v' => [1, 0]]), $context)); + static::assertTrue((new FunctionContext($context))->eval( + ref('v')->lessThan(lit([1, 9])), + ['v' => [1, 0]], + schema(list_schema('v', type_list(type_integer()))), + )); + static::assertFalse((new FunctionContext($context))->eval( + ref('v')->lessThan(lit([1, 0])), + ['v' => [1, 9]], + schema(list_schema('v', type_list(type_integer()))), + )); + static::assertFalse((new FunctionContext($context))->eval( + ref('v')->lessThan(lit([1, 0])), + ['v' => [1, 0]], + schema(list_schema('v', type_list(type_integer()))), + )); } public function test_less_than_datetimes(): void { $context = flow_context(); - static::assertTrue( - ref('v') - ->lessThan(lit(new DateTimeImmutable('2024-06-01'))) - ->eval(row(['v' => new DateTimeImmutable('2024-01-01')]), $context), - ); - static::assertFalse( - ref('v') - ->lessThan(lit(new DateTimeImmutable('2024-01-01'))) - ->eval(row(['v' => new DateTimeImmutable('2024-06-01')]), $context), - ); - static::assertFalse( - ref('v') - ->lessThan(lit(new DateTimeImmutable('2024-01-01'))) - ->eval(row(['v' => new DateTimeImmutable('2024-01-01')]), $context), - ); + static::assertTrue((new FunctionContext($context))->eval( + ref('v')->lessThan(lit(new DateTimeImmutable('2024-06-01'))), + [ + 'v' => new DateTimeImmutable('2024-01-01'), + ], + schema(datetime_schema('v')), + )); + static::assertFalse((new FunctionContext($context))->eval( + ref('v')->lessThan(lit(new DateTimeImmutable('2024-01-01'))), + [ + 'v' => new DateTimeImmutable('2024-06-01'), + ], + schema(datetime_schema('v')), + )); + static::assertFalse((new FunctionContext($context))->eval( + ref('v')->lessThan(lit(new DateTimeImmutable('2024-01-01'))), + [ + 'v' => new DateTimeImmutable('2024-01-01'), + ], + schema(datetime_schema('v')), + )); } public function test_less_than_floats(): void { $context = flow_context(); - static::assertTrue(ref('v')->lessThan(lit(2.5))->eval(row(['v' => 1.5]), $context)); - static::assertFalse(ref('v')->lessThan(lit(1.5))->eval(row(['v' => 2.5]), $context)); - static::assertFalse(ref('v')->lessThan(lit(1.5))->eval(row(['v' => 1.5]), $context)); + static::assertTrue((new FunctionContext($context))->eval( + ref('v')->lessThan(lit(2.5)), + ['v' => 1.5], + schema(float_schema('v')), + )); + static::assertFalse((new FunctionContext($context))->eval( + ref('v')->lessThan(lit(1.5)), + ['v' => 2.5], + schema(float_schema('v')), + )); + static::assertFalse((new FunctionContext($context))->eval( + ref('v')->lessThan(lit(1.5)), + ['v' => 1.5], + schema(float_schema('v')), + )); } public function test_less_than_integers(): void { $context = flow_context(); - static::assertTrue(ref('v')->lessThan(lit(20))->eval(row(['v' => 10]), $context)); - static::assertFalse(ref('v')->lessThan(lit(10))->eval(row(['v' => 20]), $context)); - static::assertFalse(ref('v')->lessThan(lit(10))->eval(row(['v' => 10]), $context)); + static::assertTrue((new FunctionContext($context))->eval( + ref('v')->lessThan(lit(20)), + ['v' => 10], + schema(int_schema('v')), + )); + static::assertFalse((new FunctionContext($context))->eval( + ref('v')->lessThan(lit(10)), + ['v' => 20], + schema(int_schema('v')), + )); + static::assertFalse((new FunctionContext($context))->eval( + ref('v')->lessThan(lit(10)), + ['v' => 10], + schema(int_schema('v')), + )); } public function test_less_than_returns_null_for_null_array(): void { - static::assertNull(ref('v')->lessThan(lit([1, 0]))->eval(row(['v' => null]), flow_context())); + static::assertNull((new FunctionContext(flow_context()))->eval( + ref('v')->lessThan(lit([1, 0])), + ['v' => null], + schema(list_schema('v', type_list(type_integer()), nullable: true)), + )); } public function test_less_than_returns_null_for_null_datetime(): void { - static::assertNull( - ref('v')->lessThan(lit(new DateTimeImmutable('2024-01-01')))->eval(row(['v' => null]), flow_context()), - ); + static::assertNull((new FunctionContext(flow_context()))->eval( + ref('v')->lessThan(lit(new DateTimeImmutable('2024-01-01'))), + ['v' => null], + schema(datetime_schema('v', nullable: true)), + )); } public function test_less_than_returns_null_for_null_left(): void { - static::assertNull(ref('v')->lessThan(lit(10))->eval(row(['v' => null]), flow_context())); + static::assertNull((new FunctionContext(flow_context()))->eval( + ref('v')->lessThan(lit(10)), + ['v' => null], + schema(str_schema('v', nullable: true)), + )); } public function test_less_than_returns_null_for_null_right(): void { - static::assertNull(ref('v')->lessThan(ref('other'))->eval(row(['v' => 10, 'other' => null]), flow_context())); + static::assertNull((new FunctionContext(flow_context()))->eval( + ref('v')->lessThan(ref('other')), + ['v' => 10, 'other' => null], + schema(int_schema('v'), str_schema('other', nullable: true)), + )); } public function test_less_than_returns_null_for_null_string(): void { - static::assertNull(ref('v')->lessThan(lit('a'))->eval(row(['v' => null]), flow_context())); + static::assertNull((new FunctionContext(flow_context()))->eval( + ref('v')->lessThan(lit('a')), + ['v' => null], + schema(str_schema('v', nullable: true)), + )); } public function test_less_than_returns_null_for_null_time_interval(): void { - static::assertNull(ref('v')->lessThan(lit(new DateInterval('PT1H')))->eval(row(['v' => null]), flow_context())); + static::assertNull((new FunctionContext(flow_context()))->eval( + ref('v')->lessThan(lit(new DateInterval('PT1H'))), + ['v' => null], + schema(time_schema('v', nullable: true)), + )); } public function test_less_than_strings(): void { $context = flow_context(); - static::assertTrue(ref('v')->lessThan(lit('banana'))->eval(row(['v' => 'apple']), $context)); - static::assertFalse(ref('v')->lessThan(lit('apple'))->eval(row(['v' => 'banana']), $context)); - static::assertFalse(ref('v')->lessThan(lit('apple'))->eval(row(['v' => 'apple']), $context)); + static::assertTrue((new FunctionContext($context))->eval( + ref('v')->lessThan(lit('banana')), + ['v' => 'apple'], + schema(str_schema('v')), + )); + static::assertFalse((new FunctionContext($context))->eval( + ref('v')->lessThan(lit('apple')), + ['v' => 'banana'], + schema(str_schema('v')), + )); + static::assertFalse((new FunctionContext($context))->eval( + ref('v')->lessThan(lit('apple')), + ['v' => 'apple'], + schema(str_schema('v')), + )); } public function test_less_than_throws_on_incompatible_types(): void @@ -116,14 +201,20 @@ public function test_less_than_time_intervals(): void { $context = flow_context(); - static::assertTrue( - ref('v')->lessThan(lit(new DateInterval('PT5H')))->eval(row(['v' => new DateInterval('PT1H')]), $context), - ); - static::assertFalse( - ref('v')->lessThan(lit(new DateInterval('PT1H')))->eval(row(['v' => new DateInterval('PT5H')]), $context), - ); - static::assertFalse( - ref('v')->lessThan(lit(new DateInterval('PT1H')))->eval(row(['v' => new DateInterval('PT1H')]), $context), - ); + static::assertTrue((new FunctionContext($context))->eval( + ref('v')->lessThan(lit(new DateInterval('PT5H'))), + ['v' => new DateInterval('PT1H')], + schema(time_schema('v')), + )); + static::assertFalse((new FunctionContext($context))->eval( + ref('v')->lessThan(lit(new DateInterval('PT1H'))), + ['v' => new DateInterval('PT5H')], + schema(time_schema('v')), + )); + static::assertFalse((new FunctionContext($context))->eval( + ref('v')->lessThan(lit(new DateInterval('PT1H'))), + ['v' => new DateInterval('PT1H')], + schema(time_schema('v')), + )); } } diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/ListSelectTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/ListSelectTest.php index c60de58b36..ca0c6bf3a5 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/ListSelectTest.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/ListSelectTest.php @@ -7,18 +7,19 @@ use Flow\ETL\Function\ListSelect; use Flow\ETL\Function\ReferenceResolver; use Flow\ETL\Function\ScalarFunction; +use Flow\ETL\Tests\Context\FunctionContext; use Flow\ETL\Tests\FlowTestCase; use function Flow\ETL\DSL\flow_context; use function Flow\ETL\DSL\list_schema; use function Flow\ETL\DSL\ref; -use function Flow\ETL\DSL\row; use function Flow\ETL\DSL\schema; use function Flow\Types\DSL\type_integer; use function Flow\Types\DSL\type_list; use function Flow\Types\DSL\type_map; use function Flow\Types\DSL\type_mixed; use function Flow\Types\DSL\type_string; +use function Flow\Types\DSL\type_structure; final class ListSelectTest extends FlowTestCase { @@ -57,73 +58,81 @@ public function test_selecting_from_a_list_of_scalars_declares_null_element_type public function test_selecting_non_existing_value_from_list_using_alias(): void { - $list = row([ - 'list' => [ - ['id' => 1, 'name' => 'test'], - ['id' => 2, 'name' => 'test2'], - ['id' => 3, 'name' => 'test3'], - ], - ]); - static::assertEquals( [ ['id' => 1, 'mail' => null], ['id' => 2, 'mail' => null], ['id' => 3, 'mail' => null], ], - (new ListSelect(ref('list'), ref('id'), ref('mail')))->eval($list, flow_context()), + (new FunctionContext(flow_context()))->eval( + new ListSelect(ref('list'), ref('id'), ref('mail')), + [ + 'list' => [ + ['id' => 1, 'name' => 'test'], + ['id' => 2, 'name' => 'test2'], + ['id' => 3, 'name' => 'test3'], + ], + ], + schema(list_schema( + 'list', + type_list(type_structure(['id' => type_integer(), 'name' => type_string()])), + )), + ), ); } public function test_selecting_value_from_list(): void { - $list = row([ - 'list' => [ - ['id' => 1, 'name' => 'test'], - ['id' => 2, 'name' => 'test2'], - ['id' => 3, 'name' => 'test3'], - ], - ]); - static::assertEquals( [ ['id' => 1], ['id' => 2], ['id' => 3], ], - (new ListSelect(ref('list'), 'id'))->eval($list, flow_context()), + (new FunctionContext(flow_context()))->eval( + new ListSelect(ref('list'), 'id'), + [ + 'list' => [ + ['id' => 1, 'name' => 'test'], + ['id' => 2, 'name' => 'test2'], + ['id' => 3, 'name' => 'test3'], + ], + ], + schema(list_schema( + 'list', + type_list(type_structure(['id' => type_integer(), 'name' => type_string()])), + )), + ), ); } public function test_selecting_value_from_list_using_alias(): void { - $list = row([ - 'list' => [ - ['id' => 1, 'name' => 'test'], - ['id' => 2, 'name' => 'test2'], - ['id' => 3, 'name' => 'test3'], - ], - ]); - static::assertEquals( [ ['new_id' => 1], ['new_id' => 2], ['new_id' => 3], ], - (new ListSelect(ref('list'), ref('id')->as('new_id')))->eval($list, flow_context()), + (new FunctionContext(flow_context()))->eval( + new ListSelect(ref('list'), ref('id')->as('new_id')), + [ + 'list' => [ + ['id' => 1, 'name' => 'test'], + ['id' => 2, 'name' => 'test2'], + ['id' => 3, 'name' => 'test3'], + ], + ], + schema(list_schema( + 'list', + type_list(type_structure(['id' => type_integer(), 'name' => type_string()])), + )), + ), ); } public function test_selecting_value_from_simple_list(): void { - $list = row(['list' => [ - 'a', - 'b', - 'c', - 'd', - ]]); - static::assertEquals( [ ['id' => null], @@ -131,7 +140,16 @@ public function test_selecting_value_from_simple_list(): void ['id' => null], ['id' => null], ], - (new ListSelect(ref('list'), ref('id')))->eval($list, flow_context()), + (new FunctionContext(flow_context()))->eval( + new ListSelect(ref('list'), ref('id')), + ['list' => [ + 'a', + 'b', + 'c', + 'd', + ]], + schema(list_schema('list', type_list(type_string()))), + ), ); } } diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/LiteralTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/LiteralTest.php index 9e5d09a1d1..ff93600f34 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/LiteralTest.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/LiteralTest.php @@ -6,7 +6,16 @@ use Flow\ETL\Exception\InvalidArgumentException; use Flow\ETL\Function\Literal; +use Flow\ETL\Tests\Double\SpyBackend; use Flow\ETL\Tests\FlowTestCase; +use Flow\ETL\Tests\Mother\RowsMother; + +use function Flow\ETL\DSL\array_to_rows; +use function Flow\ETL\DSL\config_builder; +use function Flow\ETL\DSL\flow_context; +use function Flow\ETL\DSL\int_schema; +use function Flow\ETL\DSL\lit; +use function Flow\ETL\DSL\schema; final class LiteralTest extends FlowTestCase { @@ -29,4 +38,59 @@ public function test_constructor_rejects_a_closure(): void new Literal(static fn(): int => 1); } + + public function test_a_batch_of_the_same_size_reuses_the_column(): void + { + $literal = lit(7); + $context = flow_context(); + $first = $literal->eval(array_to_rows([['id' => 1], ['id' => 2]], schema(int_schema('id'))), $context); + + static::assertSame($first, $literal->eval(array_to_rows([ + ['id' => 3], + ['id' => 4], + ], schema(int_schema('id'))), $context)); + static::assertSame([7, 7], $first->values()); + } + + public function test_a_batch_of_another_size_gets_a_column_of_its_size(): void + { + $literal = lit('x'); + $two = $literal->eval(array_to_rows([['id' => 1], ['id' => 2]], schema(int_schema('id'))), flow_context()); + $three = $literal->eval(array_to_rows([ + ['id' => 1], + ['id' => 2], + ['id' => 3], + ], schema(int_schema('id'))), flow_context()); + + static::assertSame(['x', 'x'], $two->values()); + static::assertSame(['x', 'x', 'x'], $three->values()); + static::assertSame( + ['x', 'x'], + $literal + ->eval(array_to_rows([['id' => 1], ['id' => 2]], schema(int_schema('id'))), flow_context()) + ->values(), + ); + } + + public function test_the_column_is_a_constant_of_the_configured_backend(): void + { + $backend = new SpyBackend(); + + lit(5)->eval(RowsMother::sequentialIds(3), flow_context(config_builder()->backend($backend)->build())); + + static::assertSame(1, $backend->constants()); + } + + public function test_a_batch_of_the_same_size_in_another_backend_gets_a_column_of_that_backend(): void + { + $literal = lit(5); + $first = new SpyBackend(); + $second = new SpyBackend(); + + $literal->eval(RowsMother::sequentialIds(3), flow_context(config_builder()->backend($first)->build())); + $literal->eval(RowsMother::sequentialIds(3), flow_context(config_builder()->backend($second)->build())); + + static::assertSame(1, $first->constants()); + static::assertSame(1, $second->constants()); + } } diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/LogicalFunctionsTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/LogicalFunctionsTest.php index e4dccbead4..759ed93312 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/LogicalFunctionsTest.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/LogicalFunctionsTest.php @@ -4,40 +4,38 @@ namespace Flow\ETL\Tests\Unit\Function; +use Flow\ETL\Tests\Context\FunctionContext; use Flow\ETL\Tests\FlowTestCase; use function Flow\ETL\DSL\flow_context; +use function Flow\ETL\DSL\int_schema; use function Flow\ETL\DSL\lit; use function Flow\ETL\DSL\ref; -use function Flow\ETL\DSL\row; +use function Flow\ETL\DSL\schema; final class LogicalFunctionsTest extends FlowTestCase { public function test_logical_operations(): void { - static::assertFalse( - ref('id') - ->isEven() - ->andNot(ref('id')->equals(lit(1))) - ->eval(row(['id' => 1]), flow_context()), - ); - static::assertTrue( - ref('id') - ->isOdd() - ->and(ref('id')->equals(lit(1))) - ->eval(row(['id' => 1]), flow_context()), - ); - static::assertTrue( - ref('id') - ->isEven() - ->or(ref('id')->equals(lit(1))) - ->eval(row(['id' => 1]), flow_context()), - ); - static::assertFalse( - ref('id') - ->isOdd() - ->andNot(ref('id')->equals(lit(1))) - ->eval(row(['id' => 1]), flow_context()), - ); + static::assertFalse((new FunctionContext(flow_context()))->eval( + ref('id')->isEven()->andNot(ref('id')->equals(lit(1))), + ['id' => 1], + schema(int_schema('id')), + )); + static::assertTrue((new FunctionContext(flow_context()))->eval( + ref('id')->isOdd()->and(ref('id')->equals(lit(1))), + ['id' => 1], + schema(int_schema('id')), + )); + static::assertTrue((new FunctionContext(flow_context()))->eval( + ref('id')->isEven()->or(ref('id')->equals(lit(1))), + ['id' => 1], + schema(int_schema('id')), + )); + static::assertFalse((new FunctionContext(flow_context()))->eval( + ref('id')->isOdd()->andNot(ref('id')->equals(lit(1))), + ['id' => 1], + schema(int_schema('id')), + )); } } diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/MatchCasesTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/MatchCasesTest.php index 44724e3b76..6ccc2af473 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/MatchCasesTest.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/MatchCasesTest.php @@ -5,15 +5,22 @@ namespace Flow\ETL\Tests\Unit\Function; use Flow\ETL\Exception\InvalidArgumentException; +use Flow\ETL\Function\ReferenceResolver; +use Flow\ETL\Tests\Context\FunctionContext; +use Flow\ETL\Tests\Double\FailingOnValuesFunction; use Flow\ETL\Tests\FlowTestCase; +use function Flow\ETL\DSL\array_to_rows; +use function Flow\ETL\DSL\bool_schema; use function Flow\ETL\DSL\config; use function Flow\ETL\DSL\flow_context; +use function Flow\ETL\DSL\int_schema; use function Flow\ETL\DSL\lit; use function Flow\ETL\DSL\match_cases; use function Flow\ETL\DSL\match_condition; use function Flow\ETL\DSL\ref; -use function Flow\ETL\DSL\row; +use function Flow\ETL\DSL\schema; +use function Flow\ETL\DSL\str_schema; final class MatchCasesTest extends FlowTestCase { @@ -22,9 +29,13 @@ public function test_no_matching_case_without_a_default_still_throws(): void $this->expectException(InvalidArgumentException::class); $this->expectExceptionMessage('Not a single case matches row, consider using default parameter'); - match_cases([ - match_condition(ref('string')->contains('_'), ref('string')->strReplace('_', ' ')), - ])->eval(row(['string' => 'weirdstring']), flow_context()); + (new FunctionContext(flow_context()))->eval( + match_cases([ + match_condition(ref('string')->contains('_'), ref('string')->strReplace('_', ' ')), + ]), + ['string' => 'weirdstring'], + schema(str_schema('string')), + ); } public function test_case_match(): void @@ -34,8 +45,20 @@ public function test_case_match(): void match_condition(ref('string')->contains('-'), ref('string')->strReplace('-', ' ')), ]); - static::assertSame('this is slug', $match->eval(row(['string' => 'this-is-slug']), flow_context())); - static::assertSame('this is slug', $match->eval(row(['string' => 'this_is_slug']), flow_context())); + static::assertSame('this is slug', (new FunctionContext(flow_context()))->eval( + $match, + [ + 'string' => 'this-is-slug', + ], + schema(str_schema('string')), + )); + static::assertSame('this is slug', (new FunctionContext(flow_context()))->eval( + $match, + [ + 'string' => 'this_is_slug', + ], + schema(str_schema('string')), + )); } public function test_not_matching_anything_in_strict_mode(): void @@ -51,7 +74,7 @@ public function test_not_matching_anything_in_strict_mode(): void ]); $context = flow_context(config()); - $match->eval(row(['string' => 'weirdstring']), $context); + (new FunctionContext($context))->eval($match, ['string' => 'weirdstring'], schema(str_schema('string'))); } public function test_not_matching_anything_with_default(): void @@ -61,6 +84,31 @@ public function test_not_matching_anything_with_default(): void match_condition(ref('string')->contains('-'), ref('string')->strReplace('-', ' ')), ], default: lit('normal string')); - static::assertEquals('normal string', $match->eval(row(['string' => 'weirdstring']), flow_context())); + static::assertEquals('normal string', (new FunctionContext(flow_context()))->eval( + $match, + [ + 'string' => 'weirdstring', + ], + schema(str_schema('string')), + )); + } + + public function test_a_later_case_runs_only_on_the_rows_no_earlier_case_took(): void + { + $rows = array_to_rows( + [['c' => true, 'v' => 99], ['c' => false, 'v' => 2]], + schema(bool_schema('c'), int_schema('v')), + ); + + static::assertSame( + [0, 2], + (new ReferenceResolver()) + ->resolve(match_cases([ + match_condition(ref('c'), lit(0)), + match_condition((new FailingOnValuesFunction(ref('v'), [99]))->isNotNull(), ref('v')), + ]), $rows->schema()) + ->eval($rows, flow_context()) + ->values(), + ); } } diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/MathTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/MathTest.php index a2396cde0c..9cd2706547 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/MathTest.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/MathTest.php @@ -11,70 +11,87 @@ use Flow\ETL\Function\Plus; use Flow\ETL\Function\Power; use Flow\ETL\Function\Round; +use Flow\ETL\Tests\Context\FunctionContext; use Flow\ETL\Tests\FlowTestCase; +use function Flow\ETL\DSL\float_schema; use function Flow\ETL\DSL\flow_context; +use function Flow\ETL\DSL\int_schema; use function Flow\ETL\DSL\lit; use function Flow\ETL\DSL\ref; -use function Flow\ETL\DSL\row; +use function Flow\ETL\DSL\schema; final class MathTest extends FlowTestCase { public function test_divide(): void { - $row = row(['a' => 100, 'b' => 10]); - - static::assertSame(10.0, (new Divide(ref('a'), ref('b')))->eval($row, flow_context())); + static::assertSame(10.0, (new FunctionContext(flow_context()))->eval( + new Divide(ref('a'), ref('b')), + ['a' => 100, 'b' => 10], + schema(int_schema('a'), int_schema('b')), + )); } public function test_minus(): void { - $row = row(['a' => 100, 'b' => 100]); - - static::assertSame(0, (new Minus(ref('a'), ref('b')))->eval($row, flow_context())); + static::assertSame(0, (new FunctionContext(flow_context()))->eval( + new Minus(ref('a'), ref('b')), + ['a' => 100, 'b' => 100], + schema(int_schema('a'), int_schema('b')), + )); } public function test_modulo(): void { - $row = row(['a' => 110, 'b' => 100]); - - static::assertSame(10, (new Mod(ref('a'), ref('b')))->eval($row, flow_context())); + static::assertSame(10, (new FunctionContext(flow_context()))->eval( + new Mod(ref('a'), ref('b')), + ['a' => 110, 'b' => 100], + schema(int_schema('a'), int_schema('b')), + )); } public function test_multiple_operations(): void { - static::assertSame(200, ref('a') - ->plus(lit(100)) - ->plus(lit(100)) - ->minus(ref('b')) - ->eval(row(['a' => 100, 'b' => 100]), flow_context())); + static::assertSame(200, (new FunctionContext(flow_context()))->eval( + ref('a')->plus(lit(100))->plus(lit(100))->minus(ref('b')), + ['a' => 100, 'b' => 100], + schema(int_schema('a'), int_schema('b')), + )); } public function test_multiply(): void { - $row = row(['a' => 100, 'b' => 100]); - - static::assertSame(10_000, (new Multiply(ref('a'), ref('b')))->eval($row, flow_context())); + static::assertSame(10_000, (new FunctionContext(flow_context()))->eval( + new Multiply(ref('a'), ref('b')), + ['a' => 100, 'b' => 100], + schema(int_schema('a'), int_schema('b')), + )); } public function test_plus(): void { - $row = row(['a' => 100, 'b' => 100]); - - static::assertSame(200, (new Plus(ref('a'), ref('b')))->eval($row, flow_context())); + static::assertSame(200, (new FunctionContext(flow_context()))->eval( + new Plus(ref('a'), ref('b')), + ['a' => 100, 'b' => 100], + schema(int_schema('a'), int_schema('b')), + )); } public function test_power(): void { - $row = row(['a' => 1, 'b' => 2]); - - static::assertSame(1, (new Power(ref('a'), ref('b')))->eval($row, flow_context())); + static::assertSame(1, (new FunctionContext(flow_context()))->eval( + new Power(ref('a'), ref('b')), + ['a' => 1, 'b' => 2], + schema(int_schema('a'), int_schema('b')), + )); } public function test_round(): void { - $row = row(['a' => 1.009, 'b' => 2]); - - static::assertSame(1.01, (new Round(ref('a'), ref('b')))->eval($row, flow_context())); + static::assertSame(1.01, (new FunctionContext(flow_context()))->eval( + new Round(ref('a'), ref('b')), + ['a' => 1.009, 'b' => 2], + schema(float_schema('a'), int_schema('b')), + )); } } diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/MaxTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/MaxTest.php index 1402e0b125..b32ff2c343 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/MaxTest.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/MaxTest.php @@ -5,15 +5,21 @@ namespace Flow\ETL\Tests\Unit\Function; use DateTimeImmutable; +use Flow\ETL\Exception\InvalidArgumentException; use Flow\ETL\Function\ReferenceResolver; use Flow\ETL\Tests\FlowTestCase; +use function array_map; +use function Flow\ETL\DSL\array_to_rows; +use function Flow\ETL\DSL\datetime_schema; +use function Flow\ETL\DSL\float_schema; use function Flow\ETL\DSL\flow_context; use function Flow\ETL\DSL\int_schema; use function Flow\ETL\DSL\max; +use function Flow\ETL\DSL\min; use function Flow\ETL\DSL\ref; -use function Flow\ETL\DSL\row; use function Flow\ETL\DSL\schema; +use function Flow\ETL\DSL\str_schema; use function Flow\Types\DSL\type_datetime; final class MaxTest extends FlowTestCase @@ -27,11 +33,17 @@ public function test_aggregation_max_from_numeric_values(): void { $aggregator = max(ref('int')); - $aggregator->aggregate(row(['int' => '10']), flow_context()); - $aggregator->aggregate(row(['int' => '20']), flow_context()); - $aggregator->aggregate(row(['int' => '55']), flow_context()); - $aggregator->aggregate(row(['int' => '25']), flow_context()); - $aggregator->aggregate(row(['not_int' => null]), flow_context()); + $aggregator->aggregate(array_to_rows([['int' => '10']], schema(str_schema('int'))), [0], flow_context()); + $aggregator->aggregate(array_to_rows([['int' => '20']], schema(str_schema('int'))), [0], flow_context()); + $aggregator->aggregate(array_to_rows([['int' => '55']], schema(str_schema('int'))), [0], flow_context()); + $aggregator->aggregate(array_to_rows([['int' => '25']], schema(str_schema('int'))), [0], flow_context()); + $aggregator->aggregate( + array_to_rows([[ + 'not_int' => null, + ]], schema(str_schema('not_int', nullable: true))), + [0], + flow_context(), + ); static::assertSame(55.0, $aggregator->value()); } @@ -40,10 +52,16 @@ public function test_aggregation_max_including_null_value(): void { $aggregator = max(ref('int')); - $aggregator->aggregate(row(['int' => 10]), flow_context()); - $aggregator->aggregate(row(['int' => 20]), flow_context()); - $aggregator->aggregate(row(['int' => 30]), flow_context()); - $aggregator->aggregate(row(['int' => null]), flow_context()); + $aggregator->aggregate(array_to_rows([['int' => 10]], schema(int_schema('int'))), [0], flow_context()); + $aggregator->aggregate(array_to_rows([['int' => 20]], schema(int_schema('int'))), [0], flow_context()); + $aggregator->aggregate(array_to_rows([['int' => 30]], schema(int_schema('int'))), [0], flow_context()); + $aggregator->aggregate( + array_to_rows([[ + 'int' => null, + ]], schema(int_schema('int', nullable: true))), + [0], + flow_context(), + ); static::assertSame(30.0, $aggregator->value()); } @@ -52,10 +70,34 @@ public function test_aggregation_max_with_datetime_values(): void { $aggregator = max(ref('datetime')); - $aggregator->aggregate(row(['datetime' => type_datetime()->cast('2021-01-01 00:00:00')]), flow_context()); - $aggregator->aggregate(row(['datetime' => type_datetime()->cast('2021-01-02 00:00:00')]), flow_context()); - $aggregator->aggregate(row(['datetime' => type_datetime()->cast('2021-01-03 00:00:00')]), flow_context()); - $aggregator->aggregate(row(['datetime' => type_datetime()->cast('2021-01-04 00:00:00')]), flow_context()); + $aggregator->aggregate( + array_to_rows([[ + 'datetime' => type_datetime()->cast('2021-01-01 00:00:00'), + ]], schema(datetime_schema('datetime'))), + [0], + flow_context(), + ); + $aggregator->aggregate( + array_to_rows([[ + 'datetime' => type_datetime()->cast('2021-01-02 00:00:00'), + ]], schema(datetime_schema('datetime'))), + [0], + flow_context(), + ); + $aggregator->aggregate( + array_to_rows([[ + 'datetime' => type_datetime()->cast('2021-01-03 00:00:00'), + ]], schema(datetime_schema('datetime'))), + [0], + flow_context(), + ); + $aggregator->aggregate( + array_to_rows([[ + 'datetime' => type_datetime()->cast('2021-01-04 00:00:00'), + ]], schema(datetime_schema('datetime'))), + [0], + flow_context(), + ); static::assertEquals(new DateTimeImmutable('2021-01-04 00:00:00'), $aggregator->value()); } @@ -64,10 +106,10 @@ public function test_aggregation_max_with_float_result(): void { $aggregator = max(ref('int')); - $aggregator->aggregate(row(['int' => 10]), flow_context()); - $aggregator->aggregate(row(['int' => 20]), flow_context()); - $aggregator->aggregate(row(['int' => 30.5]), flow_context()); - $aggregator->aggregate(row(['int' => 25]), flow_context()); + $aggregator->aggregate(array_to_rows([['int' => 10]], schema(int_schema('int'))), [0], flow_context()); + $aggregator->aggregate(array_to_rows([['int' => 20]], schema(int_schema('int'))), [0], flow_context()); + $aggregator->aggregate(array_to_rows([['int' => 30.5]], schema(float_schema('int'))), [0], flow_context()); + $aggregator->aggregate(array_to_rows([['int' => 25]], schema(int_schema('int'))), [0], flow_context()); static::assertSame(30.5, $aggregator->value()); } @@ -76,10 +118,10 @@ public function test_aggregation_max_with_integer_result(): void { $aggregator = max(ref('int')); - $aggregator->aggregate(row(['int' => 10]), flow_context()); - $aggregator->aggregate(row(['int' => 20]), flow_context()); - $aggregator->aggregate(row(['int' => 30]), flow_context()); - $aggregator->aggregate(row(['int' => 40]), flow_context()); + $aggregator->aggregate(array_to_rows([['int' => 10]], schema(int_schema('int'))), [0], flow_context()); + $aggregator->aggregate(array_to_rows([['int' => 20]], schema(int_schema('int'))), [0], flow_context()); + $aggregator->aggregate(array_to_rows([['int' => 30]], schema(int_schema('int'))), [0], flow_context()); + $aggregator->aggregate(array_to_rows([['int' => 40]], schema(int_schema('int'))), [0], flow_context()); static::assertSame(40.0, $aggregator->value()); } @@ -111,4 +153,59 @@ public function test_max_over_an_int_column_declares_optional_integer(): void ->toString(), ); } + + public function test_aggregate_reads_only_given_indices(): void + { + $aggregator = max(ref('v')); + $aggregator->aggregate( + array_to_rows(array_map(static fn(mixed $v): array => ['v' => $v], [ + 0.5, + 1.25, + 2.0, + 3.0, + ]), schema(float_schema('v'))), + [0, 2], + flow_context(), + ); + + static::assertSame(2.0, $aggregator->value()); + } + + public function test_merge_of_two_halves_equals_one_pass(): void + { + $rows = array_to_rows(array_map(static fn(mixed $v): array => ['v' => $v], [ + 0.5, + 1.25, + 2.0, + 3.0, + ]), schema(float_schema('v'))); + $onePass = max(ref('v')); + $onePass->aggregate($rows, [0, 1, 2, 3], flow_context()); + $left = max(ref('v')); + $left->aggregate($rows, [0, 1], flow_context()); + $right = max(ref('v')); + $right->aggregate($rows, [2, 3], flow_context()); + + $left->merge($right, flow_context()); + + static::assertSame($onePass->value(), $left->value()); + } + + public function test_merge_refuses_another_class(): void + { + $this->expectException(InvalidArgumentException::class); + + max(ref('v'))->merge(min(ref('v')), flow_context()); + } + + public function test_merge_into_an_empty_half_takes_the_other_value(): void + { + $right = max(ref('v')); + $right->aggregate(array_to_rows([['v' => 1.5]], schema(float_schema('v'))), [0], flow_context()); + $left = max(ref('v')); + $left->merge(max(ref('v')), flow_context()); + $left->merge($right, flow_context()); + + static::assertSame(1.5, $left->value()); + } } diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/MinTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/MinTest.php index 3898729388..a585969211 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/MinTest.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/MinTest.php @@ -5,15 +5,21 @@ namespace Flow\ETL\Tests\Unit\Function; use DateTimeImmutable; +use Flow\ETL\Exception\InvalidArgumentException; use Flow\ETL\Function\ReferenceResolver; use Flow\ETL\Tests\FlowTestCase; +use function array_map; +use function Flow\ETL\DSL\array_to_rows; +use function Flow\ETL\DSL\datetime_schema; +use function Flow\ETL\DSL\float_schema; use function Flow\ETL\DSL\flow_context; use function Flow\ETL\DSL\int_schema; +use function Flow\ETL\DSL\max; use function Flow\ETL\DSL\min; use function Flow\ETL\DSL\ref; -use function Flow\ETL\DSL\row; use function Flow\ETL\DSL\schema; +use function Flow\ETL\DSL\str_schema; use function Flow\Types\DSL\type_datetime; final class MinTest extends FlowTestCase @@ -27,7 +33,7 @@ public function test_the_output_suffix_survives_an_immutable_reference(): void { $aggregator = min(ref('int')); - $aggregator->aggregate(row(['int' => 10]), flow_context()); + $aggregator->aggregate(array_to_rows([['int' => 10]], schema(int_schema('int'))), [0], flow_context()); static::assertSame('int_min', $aggregator->outputName()); } @@ -36,11 +42,17 @@ public function test_aggregation_min_from_numeric_values(): void { $aggregator = min(ref('int')); - $aggregator->aggregate(row(['int' => '10']), flow_context()); - $aggregator->aggregate(row(['int' => '20']), flow_context()); - $aggregator->aggregate(row(['int' => '55']), flow_context()); - $aggregator->aggregate(row(['int' => '25']), flow_context()); - $aggregator->aggregate(row(['not_int' => null]), flow_context()); + $aggregator->aggregate(array_to_rows([['int' => '10']], schema(str_schema('int'))), [0], flow_context()); + $aggregator->aggregate(array_to_rows([['int' => '20']], schema(str_schema('int'))), [0], flow_context()); + $aggregator->aggregate(array_to_rows([['int' => '55']], schema(str_schema('int'))), [0], flow_context()); + $aggregator->aggregate(array_to_rows([['int' => '25']], schema(str_schema('int'))), [0], flow_context()); + $aggregator->aggregate( + array_to_rows([[ + 'not_int' => null, + ]], schema(str_schema('not_int', nullable: true))), + [0], + flow_context(), + ); static::assertSame(10.0, $aggregator->value()); } @@ -49,10 +61,16 @@ public function test_aggregation_min_including_null_value(): void { $aggregator = min(ref('int')); - $aggregator->aggregate(row(['int' => 10]), flow_context()); - $aggregator->aggregate(row(['int' => 20]), flow_context()); - $aggregator->aggregate(row(['int' => 30]), flow_context()); - $aggregator->aggregate(row(['int' => null]), flow_context()); + $aggregator->aggregate(array_to_rows([['int' => 10]], schema(int_schema('int'))), [0], flow_context()); + $aggregator->aggregate(array_to_rows([['int' => 20]], schema(int_schema('int'))), [0], flow_context()); + $aggregator->aggregate(array_to_rows([['int' => 30]], schema(int_schema('int'))), [0], flow_context()); + $aggregator->aggregate( + array_to_rows([[ + 'int' => null, + ]], schema(int_schema('int', nullable: true))), + [0], + flow_context(), + ); static::assertSame(10.0, $aggregator->value()); } @@ -61,10 +79,34 @@ public function test_aggregation_min_with_datetime_values(): void { $aggregator = min(ref('datetime')); - $aggregator->aggregate(row(['datetime' => type_datetime()->cast('2021-01-01 00:00:00')]), flow_context()); - $aggregator->aggregate(row(['datetime' => type_datetime()->cast('2021-01-02 00:00:00')]), flow_context()); - $aggregator->aggregate(row(['datetime' => type_datetime()->cast('2021-01-03 00:00:00')]), flow_context()); - $aggregator->aggregate(row(['datetime' => type_datetime()->cast('2021-01-04 00:00:00')]), flow_context()); + $aggregator->aggregate( + array_to_rows([[ + 'datetime' => type_datetime()->cast('2021-01-01 00:00:00'), + ]], schema(datetime_schema('datetime'))), + [0], + flow_context(), + ); + $aggregator->aggregate( + array_to_rows([[ + 'datetime' => type_datetime()->cast('2021-01-02 00:00:00'), + ]], schema(datetime_schema('datetime'))), + [0], + flow_context(), + ); + $aggregator->aggregate( + array_to_rows([[ + 'datetime' => type_datetime()->cast('2021-01-03 00:00:00'), + ]], schema(datetime_schema('datetime'))), + [0], + flow_context(), + ); + $aggregator->aggregate( + array_to_rows([[ + 'datetime' => type_datetime()->cast('2021-01-04 00:00:00'), + ]], schema(datetime_schema('datetime'))), + [0], + flow_context(), + ); static::assertEquals(new DateTimeImmutable('2021-01-01 00:00:00'), $aggregator->value()); } @@ -73,10 +115,10 @@ public function test_aggregation_min_with_float_result(): void { $aggregator = min(ref('int')); - $aggregator->aggregate(row(['int' => 10.25]), flow_context()); - $aggregator->aggregate(row(['int' => 20]), flow_context()); - $aggregator->aggregate(row(['int' => 305]), flow_context()); - $aggregator->aggregate(row(['int' => 25]), flow_context()); + $aggregator->aggregate(array_to_rows([['int' => 10.25]], schema(float_schema('int'))), [0], flow_context()); + $aggregator->aggregate(array_to_rows([['int' => 20]], schema(int_schema('int'))), [0], flow_context()); + $aggregator->aggregate(array_to_rows([['int' => 305]], schema(int_schema('int'))), [0], flow_context()); + $aggregator->aggregate(array_to_rows([['int' => 25]], schema(int_schema('int'))), [0], flow_context()); static::assertSame(10.25, $aggregator->value()); } @@ -85,10 +127,10 @@ public function test_aggregation_min_with_integer_result(): void { $aggregator = min(ref('int')); - $aggregator->aggregate(row(['int' => 10]), flow_context()); - $aggregator->aggregate(row(['int' => 20]), flow_context()); - $aggregator->aggregate(row(['int' => 30]), flow_context()); - $aggregator->aggregate(row(['int' => 40]), flow_context()); + $aggregator->aggregate(array_to_rows([['int' => 10]], schema(int_schema('int'))), [0], flow_context()); + $aggregator->aggregate(array_to_rows([['int' => 20]], schema(int_schema('int'))), [0], flow_context()); + $aggregator->aggregate(array_to_rows([['int' => 30]], schema(int_schema('int'))), [0], flow_context()); + $aggregator->aggregate(array_to_rows([['int' => 40]], schema(int_schema('int'))), [0], flow_context()); static::assertSame(10.0, $aggregator->value()); } @@ -120,4 +162,59 @@ public function test_min_over_an_int_column_declares_optional_integer(): void ->toString(), ); } + + public function test_aggregate_reads_only_given_indices(): void + { + $aggregator = min(ref('v')); + $aggregator->aggregate( + array_to_rows(array_map(static fn(mixed $v): array => ['v' => $v], [ + 0.5, + 1.25, + 2.0, + 3.0, + ]), schema(float_schema('v'))), + [1, 3], + flow_context(), + ); + + static::assertSame(1.25, $aggregator->value()); + } + + public function test_merge_of_two_halves_equals_one_pass(): void + { + $rows = array_to_rows(array_map(static fn(mixed $v): array => ['v' => $v], [ + 0.5, + 1.25, + 2.0, + 3.0, + ]), schema(float_schema('v'))); + $onePass = min(ref('v')); + $onePass->aggregate($rows, [0, 1, 2, 3], flow_context()); + $left = min(ref('v')); + $left->aggregate($rows, [0, 1], flow_context()); + $right = min(ref('v')); + $right->aggregate($rows, [2, 3], flow_context()); + + $left->merge($right, flow_context()); + + static::assertSame($onePass->value(), $left->value()); + } + + public function test_merge_refuses_another_class(): void + { + $this->expectException(InvalidArgumentException::class); + + min(ref('v'))->merge(max(ref('v')), flow_context()); + } + + public function test_merge_into_an_empty_half_takes_the_other_value(): void + { + $right = min(ref('v')); + $right->aggregate(array_to_rows([['v' => 1.5]], schema(float_schema('v'))), [0], flow_context()); + $left = min(ref('v')); + $left->merge(min(ref('v')), flow_context()); + $left->merge($right, flow_context()); + + static::assertSame(1.5, $left->value()); + } } diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/MinusTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/MinusTest.php new file mode 100644 index 0000000000..11ff622580 --- /dev/null +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/MinusTest.php @@ -0,0 +1,57 @@ +eval( + ref('a')->minus(ref('b'), exact: true), + ['a' => 0.3, 'b' => 0.1], + schema(float_schema('a'), float_schema('b')), + )); + } + + public function test_floats_use_ieee_arithmetic(): void + { + static::assertSame(0.19999999999999998, (new FunctionContext(flow_context()))->eval( + ref('a')->minus(ref('b')), + ['a' => 0.3, 'b' => 0.1], + schema(float_schema('a'), float_schema('b')), + )); + } + + public function test_integers_stay_integers(): void + { + static::assertSame(2, (new FunctionContext(flow_context()))->eval( + ref('a')->minus(ref('b')), + ['a' => 5, 'b' => 3], + schema(int_schema('a'), int_schema('b')), + )); + } + + public function test_integer_overflow_names_its_row(): void + { + $this->expectException(EvaluationException::class); + $this->expectExceptionMessage('Minus function integer overflow (row 0)'); + + (new FunctionContext(flow_context()))->eval( + ref('a')->minus(ref('b')), + ['a' => PHP_INT_MIN, 'b' => 1], + schema(int_schema('a'), int_schema('b')), + ); + } +} diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/ModifyDateTimeTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/ModifyDateTimeTest.php index 6b29d26ff5..455cf2e9e7 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/ModifyDateTimeTest.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/ModifyDateTimeTest.php @@ -7,13 +7,14 @@ use DateTimeImmutable; use Flow\ETL\Exception\InvalidArgumentException; use Flow\ETL\Function\ReferenceResolver; +use Flow\ETL\Tests\Context\FunctionContext; use Flow\ETL\Tests\FlowTestCase; +use function Flow\ETL\DSL\date_schema; use function Flow\ETL\DSL\datetime_schema; use function Flow\ETL\DSL\flow_context; use function Flow\ETL\DSL\lit; use function Flow\ETL\DSL\ref; -use function Flow\ETL\DSL\row; use function Flow\ETL\DSL\schema; use function Flow\ETL\DSL\str_schema; use function Flow\Types\DSL\type_date; @@ -25,9 +26,13 @@ public function test_modify_date(): void { static::assertEquals( new DateTimeImmutable('2025-01-01 12:00:00 +00:00'), - ref('datetime') - ->modifyDateTime('noon') - ->eval(row(['datetime' => type_date()->cast('2025-01-01')]), flow_context()), + (new FunctionContext(flow_context()))->eval( + ref('datetime')->modifyDateTime('noon'), + [ + 'datetime' => type_date()->cast('2025-01-01'), + ], + schema(date_schema('datetime')), + ), ); } @@ -35,9 +40,13 @@ public function test_modify_datetime(): void { static::assertEquals( new DateTimeImmutable('2025-01-01 00:00:00 +00:00'), - ref('datetime') - ->modifyDateTime('midnight') - ->eval(row(['datetime' => type_datetime()->cast('2025-01-01 10:00:23 +00:00')]), flow_context()), + (new FunctionContext(flow_context()))->eval( + ref('datetime')->modifyDateTime('midnight'), + [ + 'datetime' => type_datetime()->cast('2025-01-01 10:00:23 +00:00'), + ], + schema(datetime_schema('datetime')), + ), ); } @@ -46,9 +55,13 @@ public function test_modify_using_invalid_modifier(): void $this->expectException(InvalidArgumentException::class); $this->expectExceptionMessage('Expected type "string", got "integer".'); - ref('datetime') - ->modifyDateTime(lit(1)) - ->eval(row(['datetime' => type_datetime()->cast('2025-01-01 10:00:23 +00:00')]), flow_context()); + (new FunctionContext(flow_context()))->eval( + ref('datetime')->modifyDateTime(lit(1)), + [ + 'datetime' => type_datetime()->cast('2025-01-01 10:00:23 +00:00'), + ], + schema(datetime_schema('datetime')), + ); } public function test_zone_of_the_reference_is_kept(): void diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/MultiplyTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/MultiplyTest.php new file mode 100644 index 0000000000..7e41d8ea04 --- /dev/null +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/MultiplyTest.php @@ -0,0 +1,57 @@ +eval( + ref('a')->multiply(ref('b'), exact: true), + ['a' => 0.1, 'b' => 3], + schema(float_schema('a'), int_schema('b')), + )); + } + + public function test_floats_use_ieee_arithmetic(): void + { + static::assertSame(0.30000000000000004, (new FunctionContext(flow_context()))->eval( + ref('a')->multiply(ref('b')), + ['a' => 0.1, 'b' => 3], + schema(float_schema('a'), int_schema('b')), + )); + } + + public function test_integers_stay_integers(): void + { + static::assertSame(15, (new FunctionContext(flow_context()))->eval( + ref('a')->multiply(ref('b')), + ['a' => 5, 'b' => 3], + schema(int_schema('a'), int_schema('b')), + )); + } + + public function test_integer_overflow_names_its_row(): void + { + $this->expectException(EvaluationException::class); + $this->expectExceptionMessage('Multiply function integer overflow (row 0)'); + + (new FunctionContext(flow_context()))->eval( + ref('a')->multiply(ref('b')), + ['a' => PHP_INT_MAX, 'b' => 2], + schema(int_schema('a'), int_schema('b')), + ); + } +} diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/NotTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/NotTest.php index 9f0810d1d8..1ca34d9b7f 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/NotTest.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/NotTest.php @@ -4,16 +4,21 @@ namespace Flow\ETL\Tests\Unit\Function; +use Flow\ETL\Tests\Context\FunctionContext; use Flow\ETL\Tests\FlowTestCase; use function Flow\ETL\DSL\df; use function Flow\ETL\DSL\flow_context; use function Flow\ETL\DSL\from_array; +use function Flow\ETL\DSL\int_schema; +use function Flow\ETL\DSL\list_schema; use function Flow\ETL\DSL\lit; use function Flow\ETL\DSL\not; use function Flow\ETL\DSL\ref; -use function Flow\ETL\DSL\row; +use function Flow\ETL\DSL\schema; +use function Flow\ETL\DSL\str_schema; use function Flow\Types\DSL\type_integer; +use function Flow\Types\DSL\type_list; final class NotTest extends FlowTestCase { @@ -47,38 +52,44 @@ public function test_not_over_a_null_predicate_drops_the_row(): void public function test_not_expression_on_array_true_value(): void { - static::assertFalse(not(lit([1, 2, 3]))->eval(row([]), flow_context())); + static::assertFalse((new FunctionContext(flow_context()))->eval(not(lit([1, 2, 3])), [], schema())); } public function test_not_expression_on_boolean_true_value(): void { - static::assertFalse(not(lit(true))->eval(row([]), flow_context())); + static::assertFalse((new FunctionContext(flow_context()))->eval(not(lit(true)), [], schema())); } public function test_not_expression_on_is_in_expression(): void { - static::assertTrue( - not(ref('value')->isIn(ref('array')))->eval(row(['array' => [1, 2, 3], 'value' => 10]), flow_context()), - ); + static::assertTrue((new FunctionContext(flow_context()))->eval( + not(ref('value')->isIn(ref('array'))), + ['array' => [1, 2, 3], 'value' => 10], + schema(list_schema('array', type_list(type_integer())), int_schema('value')), + )); } public function test_not_expression_with_and_operator(): void { - static::assertTrue( - not(ref('value')->isNull()->or(ref('value')->isType(type_integer()))) - ->eval(row(['value' => '10']), flow_context()), - ); - static::assertFalse( - not(ref('value')->isNull()->or(ref('value')->isType(type_integer()))) - ->eval(row(['value' => null]), flow_context()), - ); - static::assertTrue( - not(ref('value')->isNull()->and(ref('value')->size()->between(1, 10))) - ->eval(row(['value' => 'abcd']), flow_context()), - ); - static::assertTrue( - not(ref('value')->isNull()->or(ref('value')->size()->equals(1))) - ->eval(row(['value' => 'abcd']), flow_context()), - ); + static::assertTrue((new FunctionContext(flow_context()))->eval( + not(ref('value')->isNull()->or(ref('value')->isType(type_integer()))), + ['value' => '10'], + schema(str_schema('value')), + )); + static::assertFalse((new FunctionContext(flow_context()))->eval( + not(ref('value')->isNull()->or(ref('value')->isType(type_integer()))), + ['value' => null], + schema(str_schema('value', nullable: true)), + )); + static::assertTrue((new FunctionContext(flow_context()))->eval( + not(ref('value')->isNull()->and(ref('value')->size()->between(1, 10))), + ['value' => 'abcd'], + schema(str_schema('value')), + )); + static::assertTrue((new FunctionContext(flow_context()))->eval( + not(ref('value')->isNull()->or(ref('value')->size()->equals(1))), + ['value' => 'abcd'], + schema(str_schema('value')), + )); } } diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/NowTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/NowTest.php index 42a169cd74..c87293ab65 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/NowTest.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/NowTest.php @@ -6,12 +6,15 @@ use DateTimeZone; use Flow\ETL\Exception\InvalidArgumentException; +use Flow\ETL\Tests\Context\FunctionContext; use Flow\ETL\Tests\FlowTestCase; use function Flow\ETL\DSL\flow_context; use function Flow\ETL\DSL\now; use function Flow\ETL\DSL\ref; -use function Flow\ETL\DSL\row; +use function Flow\ETL\DSL\schema; +use function Flow\ETL\DSL\str_schema; +use function Flow\ETL\DSL\time_zone_schema; use function Flow\Types\DSL\type_datetime; final class NowTest extends FlowTestCase @@ -31,7 +34,11 @@ public function test_value_is_in_the_zone(): void static::assertSame( 'Europe/Warsaw', type_datetime() - ->assert(now(new DateTimeZone('Europe/Warsaw'))->eval(row([]), flow_context())) + ->assert((new FunctionContext(flow_context()))->eval( + now(new DateTimeZone('Europe/Warsaw')), + [], + schema(), + )) ->getTimezone() ->getName(), ); @@ -42,7 +49,13 @@ public function test_function_zone_value_is_in_the_declared_zone(): void static::assertSame( 'UTC', type_datetime() - ->assert(now(ref('tz'))->eval(row(['tz' => new DateTimeZone('Europe/Warsaw')]), flow_context())) + ->assert((new FunctionContext(flow_context()))->eval( + now(ref('tz')), + [ + 'tz' => new DateTimeZone('Europe/Warsaw'), + ], + schema(time_zone_schema('tz')), + )) ->getTimezone() ->getName(), ); @@ -53,7 +66,11 @@ public function test_function_zone_null_is_refused(): void $this->expectException(InvalidArgumentException::class); $this->expectExceptionMessage('Now function requires valid DateTimeZone'); - now(ref('tz'))->eval(row(['tz' => null]), flow_context()); + (new FunctionContext(flow_context()))->eval( + now(ref('tz')), + ['tz' => null], + schema(str_schema('tz', nullable: true)), + ); } public function test_utc_alias_zone_lands_in_utc(): void @@ -61,7 +78,7 @@ public function test_utc_alias_zone_lands_in_utc(): void static::assertSame( 'UTC', type_datetime() - ->assert(now(new DateTimeZone('GMT'))->eval(row([]), flow_context())) + ->assert((new FunctionContext(flow_context()))->eval(now(new DateTimeZone('GMT')), [], schema())) ->getTimezone() ->getName(), ); diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/NumberFormatTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/NumberFormatTest.php index 81f083c847..eb3ba39a0b 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/NumberFormatTest.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/NumberFormatTest.php @@ -6,13 +6,17 @@ use Flow\ETL\Exception\InvalidArgumentException; use Flow\ETL\Function\NumberFormat; +use Flow\ETL\Tests\Context\FunctionContext; use Flow\ETL\Tests\FlowTestCase; +use function Flow\ETL\DSL\float_schema; use function Flow\ETL\DSL\flow_context; +use function Flow\ETL\DSL\int_schema; use function Flow\ETL\DSL\lit; use function Flow\ETL\DSL\number_format; use function Flow\ETL\DSL\ref; -use function Flow\ETL\DSL\row; +use function Flow\ETL\DSL\schema; +use function Flow\ETL\DSL\str_schema; final class NumberFormatTest extends FlowTestCase { @@ -25,19 +29,34 @@ public function test_number_format(): void ref('thousands_separator'), ); - static::assertSame('1,234.57', $expression->eval(row([ - 'value' => 1234.5678, - 'decimals' => 2, - 'decimal_separator' => '.', - 'thousands_separator' => ',', - ]), flow_context())); + static::assertSame('1,234.57', (new FunctionContext(flow_context()))->eval( + $expression, + [ + 'value' => 1234.5678, + 'decimals' => 2, + 'decimal_separator' => '.', + 'thousands_separator' => ',', + ], + schema( + float_schema('value'), + int_schema('decimals'), + str_schema('decimal_separator'), + str_schema('thousands_separator'), + ), + )); } public function test_number_format_dsl(): void { $expression = number_format(ref('value'), lit(2), lit('.'), lit(',')); - static::assertSame('1,234.57', $expression->eval(row(['value' => 1234.5678]), flow_context())); + static::assertSame('1,234.57', (new FunctionContext(flow_context()))->eval( + $expression, + [ + 'value' => 1234.5678, + ], + schema(float_schema('value')), + )); } public function test_number_format_on_decimals_that_are_not_integer(): void @@ -52,12 +71,21 @@ public function test_number_format_on_decimals_that_are_not_integer(): void ref('thousands_separator'), ); - $expression->eval(row([ - 'value' => 1234.5678, - 'decimals' => 2.5, - 'decimal_separator' => '.', - 'thousands_separator' => ',', - ]), flow_context()); + (new FunctionContext(flow_context()))->eval( + $expression, + [ + 'value' => 1234.5678, + 'decimals' => 2.5, + 'decimal_separator' => '.', + 'thousands_separator' => ',', + ], + schema( + float_schema('value'), + float_schema('decimals'), + str_schema('decimal_separator'), + str_schema('thousands_separator'), + ), + ); } public function test_number_format_on_non_int_entry(): void @@ -72,11 +100,20 @@ public function test_number_format_on_non_int_entry(): void ref('thousands_separator'), ); - $expression->eval(row([ - 'value' => 'test', - 'decimals' => 2, - 'decimal_separator' => '.', - 'thousands_separator' => ',', - ]), flow_context()); + (new FunctionContext(flow_context()))->eval( + $expression, + [ + 'value' => 'test', + 'decimals' => 2, + 'decimal_separator' => '.', + 'thousands_separator' => ',', + ], + schema( + str_schema('value'), + int_schema('decimals'), + str_schema('decimal_separator'), + str_schema('thousands_separator'), + ), + ); } } diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/OnEachTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/OnEachTest.php index 379afb4b76..ce7c73f868 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/OnEachTest.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/OnEachTest.php @@ -11,19 +11,25 @@ use Flow\ETL\Function\OnEachElementSchema; use Flow\ETL\Function\ReferenceResolver; use Flow\ETL\Function\ScalarFunction; +use Flow\ETL\Tests\Context\FunctionContext; use Flow\ETL\Tests\Double\CountingReturnsFunction; +use Flow\ETL\Tests\Double\FailingOnValuesFunction; +use Flow\ETL\Tests\Double\SpyBackend; use Flow\ETL\Tests\FlowTestCase; use Generator; use PHPUnit\Framework\Attributes\DataProvider; +use function Flow\ETL\DSL\array_to_rows; +use function Flow\ETL\DSL\config_builder; use function Flow\ETL\DSL\definition_from_type; use function Flow\ETL\DSL\flow_context; +use function Flow\ETL\DSL\int_schema; use function Flow\ETL\DSL\list_schema; use function Flow\ETL\DSL\lit; use function Flow\ETL\DSL\map_schema; use function Flow\ETL\DSL\ref; -use function Flow\ETL\DSL\row; use function Flow\ETL\DSL\schema; +use function Flow\ETL\DSL\str_schema; use function Flow\ETL\DSL\structure_schema; use function Flow\Types\DSL\structure_element; use function Flow\Types\DSL\type_array; @@ -32,6 +38,7 @@ use function Flow\Types\DSL\type_integer; use function Flow\Types\DSL\type_list; use function Flow\Types\DSL\type_map; +use function Flow\Types\DSL\type_optional; use function Flow\Types\DSL\type_string; use function Flow\Types\DSL\type_structure; use function serialize; @@ -44,36 +51,55 @@ public function test_a_throwing_body_is_not_swallowed(): void $this->expectException(InvalidArgumentException::class); $this->expectExceptionMessage('Expected type "string", got "integer"'); - (new ReferenceResolver()) - ->resolve( + (new FunctionContext(flow_context()))->eval( + (new ReferenceResolver())->resolve( ref('array')->onEach(ref('element')->upper()), schema(list_schema('array', type_list(type_integer()))), - ) - ->eval(row(['array' => [1, 2, 3]]), flow_context()); + ), + ['array' => [ + 1, + 2, + 3, + ]], + schema(list_schema('array', type_list(type_integer()))), + ); } public function test_an_element_the_declared_type_refuses_is_not_swallowed(): void { $this->expectException(SchemaMismatchException::class); - (new ReferenceResolver()) - ->resolve( + (new FunctionContext(flow_context()))->eval( + (new ReferenceResolver())->resolve( ref('array')->onEach(ref('element')->upper()), schema(list_schema('array', type_list(type_string()))), - ) - ->eval(row(['array' => ['a', null]]), flow_context()); + ), + ['array' => [ + 'a', + null, + ]], + schema(list_schema('array', type_list(type_optional(type_string())))), + ); } public function test_executing_function_on_each_value_from_array(): void { static::assertSame( ['1', '2', '3', '4', '5'], - (new ReferenceResolver()) - ->resolve( + (new FunctionContext(flow_context()))->eval( + (new ReferenceResolver())->resolve( ref('array')->onEach(ref('element')->cast(type_string())), schema(list_schema('array', type_list(type_integer()))), - ) - ->eval(row(['array' => [1, 2, 3, 4, 5]]), flow_context()), + ), + ['array' => [ + 1, + 2, + 3, + 4, + 5, + ]], + schema(list_schema('array', type_list(type_integer()))), + ), ); } @@ -81,12 +107,14 @@ public function test_executing_function_on_each_value_from_empty_array(): void { static::assertSame( [], - (new ReferenceResolver()) - ->resolve( + (new FunctionContext(flow_context()))->eval( + (new ReferenceResolver())->resolve( ref('array')->onEach(ref('element')->cast(type_string())), schema(list_schema('array', type_list(type_integer()))), - ) - ->eval(row(['array' => []]), flow_context()), + ), + ['array' => []], + schema(list_schema('array', type_list(type_integer()))), + ), ); } @@ -94,12 +122,20 @@ public function test_executing_function_on_each_value_with_preserving_keys(): vo { static::assertSame( ['a' => '1', 'b' => '2', 'c' => '3', 'd' => '4', 'e' => '5'], - (new ReferenceResolver()) - ->resolve( + (new FunctionContext(flow_context()))->eval( + (new ReferenceResolver())->resolve( ref('array')->onEach(ref('element')->cast(type_string()), true), schema(map_schema('array', type_map(type_string(), type_integer()))), - ) - ->eval(row(['array' => ['a' => 1, 'b' => 2, 'c' => 3, 'd' => 4, 'e' => 5]]), flow_context()), + ), + ['array' => [ + 'a' => 1, + 'b' => 2, + 'c' => 3, + 'd' => 4, + 'e' => 5, + ]], + schema(map_schema('array', type_map(type_string(), type_integer()))), + ), ); } @@ -107,12 +143,20 @@ public function test_executing_function_on_each_value_without_preserving_keys(): { static::assertSame( ['1', '2', '3', '4', '5'], - (new ReferenceResolver()) - ->resolve( + (new FunctionContext(flow_context()))->eval( + (new ReferenceResolver())->resolve( ref('array')->onEach(ref('element')->cast(type_string()), false), schema(map_schema('array', type_map(type_string(), type_integer()))), - ) - ->eval(row(['array' => ['a' => 1, 'b' => 2, 'c' => 3, 'd' => 4, 'e' => 5]]), flow_context()), + ), + ['array' => [ + 'a' => 1, + 'b' => 2, + 'c' => 3, + 'd' => 4, + 'e' => 5, + ]], + schema(map_schema('array', type_map(type_string(), type_integer()))), + ), ); } @@ -123,19 +167,28 @@ public function test_on_each_over_an_untyped_array_operand_is_refused(): void $this->expectException(SchemaNotDerivableException::class); $this->expectExceptionMessage('the array operand declares "json", which has no element type'); - (new ReferenceResolver()) - ->resolve( + (new FunctionContext(flow_context()))->eval( + (new ReferenceResolver())->resolve( ref('array')->onEach(ref('element')->cast(type_string())), schema(definition_from_type('array', type_array())), - ) - ->eval(row(['array' => [1, 2]]), flow_context()); + ), + ['array' => [ + 1, + 2, + ]], + schema(list_schema('array', type_list(type_integer()))), + ); } public function test_the_element_schema_is_built_once_per_eval_not_once_per_element(): void { $operand = new CountingReturnsFunction(lit([1, 2, 3, 4, 5]), type_list(type_integer())); - (new OnEach($operand, ref('element')->cast(type_string())))->eval(row([]), flow_context()); + (new FunctionContext(flow_context()))->eval( + new OnEach($operand, ref('element')->cast(type_string())), + [], + schema(), + ); static::assertSame(1, $operand->returnsCalls); } @@ -224,7 +277,11 @@ public function test_a_null_array_is_refused(): void $this->expectException(InvalidArgumentException::class); $this->expectExceptionMessage('OnEach requires non-null array'); - ref('tags')->onEach(ref('element'))->eval(row(['tags' => null]), flow_context()); + (new FunctionContext(flow_context()))->eval( + ref('tags')->onEach(ref('element')), + ['tags' => null], + schema(str_schema('tags', nullable: true)), + ); } public function test_with_children_keeps_the_element_schema(): void @@ -235,4 +292,49 @@ public function test_with_children_keeps_the_element_schema(): void // two stateless instances are equal, but serialize() names an instance it already wrote only by back-reference static::assertSame(1, substr_count(serialize([$elementSchema, $rebuilt]), 'OnEachElementSchema')); } + + public function test_the_body_is_bound_once_per_node_not_once_per_batch(): void + { + $operand = new CountingReturnsFunction(lit([1, 2]), type_list(type_integer())); + $onEach = new OnEach($operand, ref('element')->cast(type_string())); + $rows = array_to_rows([['id' => 1]], schema(int_schema('id'))); + + static::assertSame([['1', '2']], $onEach->eval($rows, flow_context())->values()); + static::assertSame([['1', '2']], $onEach->eval($rows, flow_context())->values()); + static::assertSame(1, $operand->returnsCalls); + } + + public function test_the_element_batch_is_built_by_the_configured_backend(): void + { + $backend = new SpyBackend(); + $rows = array_to_rows([['l' => [1, 2]]], schema(list_schema('l', type_list(type_integer())))); + + static::assertSame( + [['1', '2']], + (new ReferenceResolver()) + ->resolve(new OnEach(ref('l'), ref('element')->cast(type_string())), $rows->schema()) + ->eval($rows, flow_context(config_builder()->backend($backend)->build())) + ->values(), + ); + static::assertGreaterThanOrEqual(1, $backend->builders()); + } + + public function test_the_function_evaluates_once_per_batch(): void + { + $spy = new FailingOnValuesFunction(ref('element')); + $rows = array_to_rows([ + ['l' => [1, 2]], + ['l' => [3]], + ['l' => []], + ], schema(list_schema('l', type_list(type_integer())))); + + static::assertSame( + [[1, 2], [3], []], + (new ReferenceResolver()) + ->resolve(new OnEach(ref('l'), $spy), $rows->schema()) + ->eval($rows, flow_context()) + ->values(), + ); + static::assertSame(1, $spy->evals); + } } diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/OptionalTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/OptionalTest.php index 5201813d9c..a3f52b90e7 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/OptionalTest.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/OptionalTest.php @@ -4,12 +4,24 @@ namespace Flow\ETL\Tests\Unit\Function; +use Error; +use Flow\ETL\Function\ReferenceResolver; +use Flow\ETL\Tests\Context\FunctionContext; +use Flow\ETL\Tests\Double\FailingOnValuesFunction; use Flow\ETL\Tests\FlowTestCase; +use Generator; +use PHPUnit\Framework\Attributes\DataProvider; +use function array_map; +use function Flow\ETL\DSL\array_to_rows; use function Flow\ETL\DSL\flow_context; +use function Flow\ETL\DSL\int_schema; use function Flow\ETL\DSL\optional; use function Flow\ETL\DSL\ref; -use function Flow\ETL\DSL\row; +use function Flow\ETL\DSL\schema; +use function Flow\ETL\DSL\str_schema; +use function in_array; +use function range; final class OptionalTest extends FlowTestCase { @@ -20,6 +32,63 @@ public function test_optional_declares_the_inner_type_widened_to_optional(): voi public function test_optional_returns_null_when_the_inner_function_throws(): void { - static::assertNull(optional(ref('name')->upper())->eval(row(['other' => 'flow']), flow_context())); + static::assertNull((new FunctionContext(flow_context()))->eval( + optional(ref('name')->upper()), + ['other' => 'flow'], + schema(str_schema('other')), + )); + } + + /** + * @return Generator, string, int}> + */ + public static function failing_rows(): Generator + { + yield 'one failing row: one retry' => [[3], FailingOnValuesFunction::WITH_ROW, 2]; + yield 'three failing rows: three retries' => [[2, 4, 6], FailingOnValuesFunction::WITH_ROW, 4]; + yield 'twelve failing rows: eight batch evals, then each remaining row alone' => [ + [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12], + FailingOnValuesFunction::WITH_ROW, + 8 + 6, + ]; + yield 'a failure without a row: each row alone' => [[3], FailingOnValuesFunction::WITHOUT_ROW, 1 + 14]; + } + + /** + * @param list $failing + */ + #[DataProvider('failing_rows')] + public function test_only_the_failing_rows_are_null(array $failing, string $failure, int $evals): void + { + $spy = new FailingOnValuesFunction(ref('v'), $failing, $failure); + $rows = array_to_rows( + array_map(static fn(int $v): array => ['v' => $v], range(1, 14)), + schema(int_schema('v')), + ); + + $values = (new ReferenceResolver()) + ->resolve(optional($spy), $rows->schema()) + ->eval($rows, flow_context()) + ->values(); + + static::assertSame( + array_map(static fn(int $v): ?int => in_array($v, $failing, true) ? null : $v, range(1, 14)), + $values, + ); + static::assertSame($evals, $spy->evals); + } + + public function test_an_error_propagates(): void + { + $rows = array_to_rows([['v' => 1]], schema(int_schema('v'))); + + $this->expectException(Error::class); + + (new ReferenceResolver()) + ->resolve( + optional(new FailingOnValuesFunction(ref('v'), [1], FailingOnValuesFunction::ERROR)), + $rows->schema(), + ) + ->eval($rows, flow_context()); } } diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/ParameterTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/ParameterTest.php index df1677d7b5..c19f40c875 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/ParameterTest.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/ParameterTest.php @@ -7,18 +7,28 @@ use DateTimeImmutable; use DateTimeInterface; use Flow\Doctrine\Bulk\SQLParametersStyle; +use Flow\ETL\Exception\EvaluationException; use Flow\ETL\Exception\InvalidArgumentException; use Flow\ETL\Function\Parameter; +use Flow\ETL\Function\ReferenceResolver; use Flow\ETL\String\StringStyles; use Flow\ETL\Tests\FlowTestCase; +use Flow\ETL\Tests\Mother\RowsMother; use Generator; use PHPUnit\Framework\Attributes\DataProvider; use stdClass; +use function Flow\ETL\DSL\array_to_rows; +use function Flow\ETL\DSL\bool_schema; +use function Flow\ETL\DSL\datetime_schema; +use function Flow\ETL\DSL\float_schema; use function Flow\ETL\DSL\flow_context; +use function Flow\ETL\DSL\int_schema; +use function Flow\ETL\DSL\json_schema; use function Flow\ETL\DSL\lit; use function Flow\ETL\DSL\ref; -use function Flow\ETL\DSL\row; +use function Flow\ETL\DSL\schema; +use function Flow\ETL\DSL\str_schema; use function Flow\Types\DSL\type_boolean; use function Flow\Types\DSL\type_integer; use function Flow\Types\DSL\type_string; @@ -42,14 +52,6 @@ public static function boolean_data_provider(): Generator yield 'null propagates' => [null, null]; } - public static function float_data_provider(): Generator - { - yield 'valid float' => [3.14, 3.14]; - yield 'zero float' => [0.0, 0.0]; - yield 'negative float' => [-2.5, -2.5]; - yield 'null propagates' => [null, null]; - } - public static function int_data_provider(): Generator { yield 'valid integer' => [42, null, 42]; @@ -85,113 +87,96 @@ public static function string_data_provider(): Generator public function test_as_array_with_empty_array(): void { - $parameter = new Parameter(lit([])); - static::assertSame([], $parameter->asArray(row([]), flow_context())); - } - - public function test_as_boolean_throws_on_a_malformed_value(): void - { - $this->expectException(InvalidArgumentException::class); - - (new Parameter(lit(['value'])))->asBoolean(row([]), flow_context()); - } - - public function test_as_boolean_propagates_null(): void - { - static::assertNull((new Parameter(lit(null)))->asBoolean(row([]), flow_context())); - } - - public function test_as_float_throws_on_a_malformed_value(): void - { - $this->expectException(InvalidArgumentException::class); - - (new Parameter(lit('3.14')))->asFloat(row([]), flow_context()); - } - - public function test_as_float_propagates_null(): void - { - static::assertNull((new Parameter(lit(null)))->asFloat(row([]), flow_context())); + static::assertSame([[]], (new Parameter(lit([])))->asArrays(RowsMother::sequentialIds(1), flow_context())); } - public function test_as_int_throws_on_a_malformed_value(): void - { - // A default never applies to a malformed value - only to a NULL input. - $this->expectException(InvalidArgumentException::class); - - (new Parameter(lit(3.14)))->asInt(row([]), flow_context(), 99); - } - - public function test_as_int_propagates_null(): void - { - static::assertNull((new Parameter(lit(null)))->asInt(row([]), flow_context())); - } - - public function test_as_number_throws_on_a_malformed_value(): void - { - $this->expectException(InvalidArgumentException::class); - - (new Parameter(lit('not numeric')))->asNumber(row([]), flow_context(), 99); - } - - public function test_as_number_propagates_null(): void - { - static::assertNull((new Parameter(lit(null)))->asNumber(row([]), flow_context())); - } - - public function test_as_string_throws_on_a_malformed_value(): void + public function test_as_array_with_valid_array(): void { - $this->expectException(InvalidArgumentException::class); - - (new Parameter(lit(42)))->asString(row([]), flow_context(), 'default'); + static::assertSame( + [['key' => 'value', 'number' => 42]], + (new Parameter(lit(['key' => 'value', 'number' => 42])))->asArrays( + RowsMother::sequentialIds(1), + flow_context(), + ), + ); } - public function test_as_string_propagates_null(): void + public function test_as_arrays_reads_a_json_column_as_arrays(): void { - static::assertNull((new Parameter(lit(null)))->asString(row([]), flow_context())); + static::assertSame( + [['a' => 1], null], + (new Parameter((new ReferenceResolver())->resolve( + ref('j'), + array_to_rows([ + ['j' => '{"a":1}'], + ['j' => null], + ], schema(json_schema('j', nullable: true)))->schema(), + )))->asArrays(array_to_rows([ + ['j' => '{"a":1}'], + ['j' => null], + ], schema(json_schema('j', nullable: true))), flow_context()), + ); } public function test_as_array_throws_on_a_malformed_value(): void { $this->expectException(InvalidArgumentException::class); - (new Parameter(lit('not an array')))->asArray(row([]), flow_context()); + (new Parameter(lit('not an array')))->asArrays(RowsMother::sequentialIds(1), flow_context()); } public function test_as_array_propagates_null(): void { - static::assertNull((new Parameter(lit(null)))->asArray(row([]), flow_context())); - } - - public function test_as_array_with_valid_array(): void - { - $parameter = new Parameter(lit(['key' => 'value', 'number' => 42])); - $result = $parameter->asArray(row([]), flow_context()); - - static::assertSame(['key' => 'value', 'number' => 42], $result); + static::assertSame([null], (new Parameter(lit(null)))->asArrays(RowsMother::sequentialIds(1), flow_context())); } + /** + * @param null|bool $expected + */ #[DataProvider('boolean_data_provider')] public function test_as_boolean(mixed $input, ?bool $expected): void { - $parameter = new Parameter(lit($input)); - static::assertSame($expected, $parameter->asBoolean(row([]), flow_context())); + static::assertSame( + [$expected], + (new Parameter(lit($input)))->asBooleans(RowsMother::sequentialIds(1), flow_context()), + ); } - public function test_as_value_with_literal(): void + public function test_as_booleans_reads_a_boolean_column_as_is(): void { - static::assertNull((new Parameter(lit('literal_value')))->asValue(row([]))); + static::assertSame( + [true, null, false], + (new Parameter((new ReferenceResolver())->resolve( + ref('b'), + array_to_rows([ + ['b' => true], + ['b' => null], + ['b' => false], + ], schema(bool_schema('b', nullable: true)))->schema(), + )))->asBooleans(array_to_rows([ + ['b' => true], + ['b' => null], + ['b' => false], + ], schema(bool_schema('b', nullable: true))), flow_context()), + ); } - public function test_as_value_with_missing_reference(): void + public function test_as_boolean_throws_on_a_malformed_value(): void { - static::assertNull((new Parameter(ref('missing_column')))->asValue(row(['other_column' => 'test_value']))); + $this->expectException(InvalidArgumentException::class); + + (new Parameter(lit(['value'])))->asBooleans(RowsMother::sequentialIds(1), flow_context()); } - public function test_as_value_with_reference(): void + public function test_as_enum_propagates_null(): void { static::assertSame( - 'test_value', - (new Parameter(ref('test_column')))->asValue(row(['test_column' => 'test_value'])), + [null], + (new Parameter(lit(null)))->asEnums( + RowsMother::sequentialIds(1), + flow_context(), + SQLParametersStyle::class, + ), ); } @@ -199,81 +184,138 @@ public function test_as_enum_throws_on_a_malformed_value(): void { $this->expectException(InvalidArgumentException::class); - (new Parameter(lit('not an enum')))->asEnum(row([]), flow_context(), SQLParametersStyle::class); - } - - public function test_as_enum_propagates_null(): void - { - static::assertNull((new Parameter(lit(null)))->asEnum(row([]), flow_context(), SQLParametersStyle::class)); + (new Parameter(lit('not an enum')))->asEnums( + RowsMother::sequentialIds(1), + flow_context(), + SQLParametersStyle::class, + ); } public function test_as_enum_with_valid_enum(): void { - $parameter = new Parameter(lit(SQLParametersStyle::NAMED)); - $result = $parameter->asEnum(row([]), flow_context(), SQLParametersStyle::class); - - static::assertSame(SQLParametersStyle::NAMED, $result); + static::assertSame( + [SQLParametersStyle::NAMED], + (new Parameter(lit(SQLParametersStyle::NAMED)))->asEnums( + RowsMother::sequentialIds(1), + flow_context(), + SQLParametersStyle::class, + ), + ); } public function test_as_enum_with_wrong_enum_class(): void { $this->expectException(InvalidArgumentException::class); - (new Parameter(lit(SQLParametersStyle::NAMED)))->asEnum(row([]), flow_context(), StringStyles::class); + (new Parameter(lit(SQLParametersStyle::NAMED)))->asEnums( + RowsMother::sequentialIds(1), + flow_context(), + StringStyles::class, + ); } - #[DataProvider('float_data_provider')] - public function test_as_float(mixed $input, ?float $expected): void + public function test_as_instance_of_propagates_null(): void { - $parameter = new Parameter(lit($input)); - static::assertSame($expected, $parameter->asFloat(row([]), flow_context())); + static::assertSame( + [null], + (new Parameter(lit(null)))->asInstancesOf( + RowsMother::sequentialIds(1), + flow_context(), + DateTimeImmutable::class, + ), + ); } public function test_as_instance_of_throws_on_a_malformed_value(): void { $this->expectException(InvalidArgumentException::class); - (new Parameter(lit('not an object')))->asInstanceOf(row([]), flow_context(), DateTimeImmutable::class); - } - - public function test_as_instance_of_propagates_null(): void - { - static::assertNull((new Parameter(lit(null)))->asInstanceOf(row([]), flow_context(), DateTimeImmutable::class)); + (new Parameter(lit('not an object')))->asInstancesOf( + RowsMother::sequentialIds(1), + flow_context(), + DateTimeImmutable::class, + ); } public function test_as_instance_of_with_valid_object(): void { $dateTime = new DateTimeImmutable('2023-01-01'); - $parameter = new Parameter(lit($dateTime)); - - $result = $parameter->asInstanceOf(row([]), flow_context(), DateTimeImmutable::class); - static::assertSame($dateTime, $result); - - $result = $parameter->asInstanceOf(row([]), flow_context(), DateTimeInterface::class); - static::assertSame($dateTime, $result); + $rows = array_to_rows([['at' => $dateTime]], schema(datetime_schema('at'))); + + static::assertEquals( + [$dateTime], + (new Parameter((new ReferenceResolver())->resolve(ref('at'), $rows->schema())))->asInstancesOf( + $rows, + flow_context(), + DateTimeImmutable::class, + ), + ); + static::assertEquals( + [$dateTime], + (new Parameter((new ReferenceResolver())->resolve(ref('at'), $rows->schema())))->asInstancesOf( + $rows, + flow_context(), + DateTimeInterface::class, + ), + ); } #[DataProvider('int_data_provider')] public function test_as_int(mixed $input, ?int $default, ?int $expected): void { - $parameter = new Parameter(lit($input)); - static::assertSame($expected, $parameter->asInt(row([]), flow_context(), $default)); + static::assertSame( + [$expected], + (new Parameter(lit($input)))->asInts(RowsMother::sequentialIds(1), flow_context(), $default), + ); } - public function test_as_list_of_objects_with_empty_array(): void + public function test_as_ints_reads_an_integer_column_as_is_and_defaults_its_nulls(): void { - $parameter = new Parameter(lit([])); - $result = $parameter->asListOfObjects(row([]), flow_context(), DateTimeImmutable::class); + $rows = array_to_rows([['n' => 1], ['n' => null]], schema(int_schema('n', nullable: true))); - static::assertSame([], $result); + static::assertSame( + [1, null], + (new Parameter((new ReferenceResolver())->resolve(ref('n'), $rows->schema())))->asInts( + $rows, + flow_context(), + ), + ); + static::assertSame( + [1, 7], + (new Parameter((new ReferenceResolver())->resolve(ref('n'), $rows->schema())))->asInts( + $rows, + flow_context(), + 7, + ), + ); + } + + public function test_as_int_throws_on_a_malformed_value(): void + { + // A default never applies to a malformed value - only to a NULL input. + $this->expectException(InvalidArgumentException::class); + + (new Parameter(lit(3.14)))->asInts(RowsMother::sequentialIds(1), flow_context(), 99); + } + + public function test_as_list_of_objects_propagates_null(): void + { + static::assertSame( + [null], + (new Parameter(lit(null)))->asListsOfObjects( + RowsMother::sequentialIds(1), + flow_context(), + DateTimeImmutable::class, + ), + ); } public function test_as_list_of_objects_throws_on_a_malformed_element(): void { $this->expectException(InvalidArgumentException::class); - (new Parameter(lit([new DateTimeImmutable('2023-01-01'), 'not an object'])))->asListOfObjects( - row([]), + (new Parameter(lit([new DateTimeImmutable('2023-01-01'), 'not an object'])))->asListsOfObjects( + RowsMother::sequentialIds(1), flow_context(), DateTimeImmutable::class, ); @@ -283,34 +325,46 @@ public function test_as_list_of_objects_throws_on_a_non_array(): void { $this->expectException(InvalidArgumentException::class); - (new Parameter(lit('not an array')))->asListOfObjects(row([]), flow_context(), DateTimeImmutable::class); + (new Parameter(lit('not an array')))->asListsOfObjects( + RowsMother::sequentialIds(1), + flow_context(), + DateTimeImmutable::class, + ); } - public function test_as_list_of_objects_propagates_null(): void + public function test_as_list_of_objects_with_empty_array(): void { - static::assertNull((new Parameter(lit(null)))->asListOfObjects( - row([]), - flow_context(), - DateTimeImmutable::class, - )); + static::assertSame( + [[]], + (new Parameter(lit([])))->asListsOfObjects( + RowsMother::sequentialIds(1), + flow_context(), + DateTimeImmutable::class, + ), + ); } public function test_as_list_of_objects_with_valid_array(): void { $date1 = new DateTimeImmutable('2023-01-01'); $date2 = new DateTimeImmutable('2023-01-02'); - $parameter = new Parameter(lit([$date1, $date2])); - $result = $parameter->asListOfObjects(row([]), flow_context(), DateTimeImmutable::class); - static::assertSame([$date1, $date2], $result); + static::assertEquals( + [[$date1, $date2]], + (new Parameter(lit([$date1, $date2])))->asListsOfObjects( + RowsMother::sequentialIds(1), + flow_context(), + DateTimeImmutable::class, + ), + ); } public function test_as_list_of_objects_with_wrong_object_type(): void { $this->expectException(InvalidArgumentException::class); - (new Parameter(lit([new DateTimeImmutable('2023-01-01'), new stdClass()])))->asListOfObjects( - row([]), + (new Parameter(lit([new DateTimeImmutable('2023-01-01'), new stdClass()])))->asListsOfObjects( + RowsMother::sequentialIds(1), flow_context(), DateTimeImmutable::class, ); @@ -319,103 +373,173 @@ public function test_as_list_of_objects_with_wrong_object_type(): void #[DataProvider('number_data_provider')] public function test_as_number(mixed $input, int|float|null $default, int|float|null $expected): void { - $parameter = new Parameter(lit($input)); - static::assertSame($expected, $parameter->asNumber(row([]), flow_context(), $default)); + static::assertSame( + [$expected], + (new Parameter(lit($input)))->asNumbers(RowsMother::sequentialIds(1), flow_context(), $default), + ); } - public function test_as_object_throws_on_a_malformed_value(): void + public function test_as_numbers_reads_numeric_columns_as_is_and_defaults_their_nulls(): void { - $this->expectException(InvalidArgumentException::class); + $rows = array_to_rows([['f' => 1.5], ['f' => null]], schema(float_schema('f', nullable: true))); - (new Parameter(lit('not an object')))->asObject(row([]), flow_context()); + static::assertSame( + [1.5, null], + (new Parameter((new ReferenceResolver())->resolve(ref('f'), $rows->schema())))->asNumbers( + $rows, + flow_context(), + ), + ); + static::assertSame( + [1.5, 0], + (new Parameter((new ReferenceResolver())->resolve(ref('f'), $rows->schema())))->asNumbers( + $rows, + flow_context(), + 0, + ), + ); } - public function test_as_object_propagates_null(): void + public function test_as_number_throws_on_a_malformed_value(): void { - static::assertNull((new Parameter(lit(null)))->asObject(row([]), flow_context())); + $this->expectException(InvalidArgumentException::class); + + (new Parameter(lit('not numeric')))->asNumbers(RowsMother::sequentialIds(1), flow_context(), 99); } - public function test_as_object_with_valid_object(): void + #[DataProvider('string_data_provider')] + public function test_as_string(mixed $input, ?string $default, ?string $expected): void { - $object = new stdClass(); - $object->property = 'value'; - $parameter = new Parameter(lit($object)); - - static::assertSame($object, $parameter->asObject(row([]), flow_context())); + static::assertSame( + [$expected], + (new Parameter(lit($input)))->asStrings(RowsMother::sequentialIds(1), flow_context(), $default), + ); } - public function test_as_one_of(): void + public function test_as_strings_reads_a_string_column_as_is_and_defaults_its_nulls(): void { - static::assertSame('42', (new Parameter(ref('value')))->as( - row(['value' => '42']), - flow_context(), - type_string(), - type_integer(), - )); + $rows = array_to_rows([['s' => 'a'], ['s' => null]], schema(str_schema('s', nullable: true))); + + static::assertSame( + ['a', null], + (new Parameter((new ReferenceResolver())->resolve(ref('s'), $rows->schema())))->asStrings( + $rows, + flow_context(), + ), + ); + static::assertSame( + ['a', 'x'], + (new Parameter((new ReferenceResolver())->resolve(ref('s'), $rows->schema())))->asStrings( + $rows, + flow_context(), + 'x', + ), + ); } - public function test_as_throws_when_no_type_matches(): void + public function test_as_string_throws_on_a_malformed_value(): void { $this->expectException(InvalidArgumentException::class); - (new Parameter(ref('value')))->as(row(['value' => '42']), flow_context(), type_integer(), type_boolean()); + (new Parameter(lit(42)))->asStrings(RowsMother::sequentialIds(1), flow_context(), 'default'); } - public function test_as_propagates_null(): void + public function test_as_types_accepts_the_first_type_that_matches(): void { - static::assertNull((new Parameter(lit(null)))->as(row([]), flow_context(), type_integer())); + static::assertSame( + ['42'], + (new Parameter((new ReferenceResolver())->resolve( + ref('value'), + array_to_rows([['value' => '42']], schema(str_schema('value')))->schema(), + )))->asTypes( + array_to_rows([['value' => '42']], schema(str_schema('value'))), + flow_context(), + type_string(), + type_integer(), + ), + ); } - public function test_as_scalar(): void + public function test_as_types_propagates_null(): void { - static::assertSame('42', (new Parameter(ref('value')))->as( - row(['value' => '42']), - flow_context(), - type_string(), - )); + static::assertSame( + [null], + (new Parameter(lit(null)))->asTypes(RowsMother::sequentialIds(1), flow_context(), type_integer()), + ); } - #[DataProvider('string_data_provider')] - public function test_as_string(mixed $input, ?string $default, ?string $expected): void + public function test_as_types_throws_when_no_type_matches(): void { - $parameter = new Parameter(lit($input)); - static::assertSame($expected, $parameter->asString(row([]), flow_context(), $default)); + $this->expectException(InvalidArgumentException::class); + $this->expectExceptionMessage('Expected one of "integer", "boolean", got "string". (row 0)'); + + (new Parameter((new ReferenceResolver())->resolve( + ref('value'), + array_to_rows([['value' => '42']], schema(str_schema('value')))->schema(), + )))->asTypes( + array_to_rows([['value' => '42']], schema(str_schema('value'))), + flow_context(), + type_integer(), + type_boolean(), + ); } - public function test_constructor_with_mixed_value(): void + public function test_a_refused_value_names_its_row(): void { - $parameter = new Parameter('direct_string'); - static::assertSame('direct_string', $parameter->eval(row([]), flow_context())); - - $parameter = new Parameter(123); - static::assertSame(123, $parameter->eval(row([]), flow_context())); - - $parameter = new Parameter(true); - static::assertTrue($parameter->eval(row([]), flow_context())); + try { + (new Parameter((new ReferenceResolver())->resolve( + ref('value'), + array_to_rows([ + ['value' => '1'], + ['value' => '2'], + ['value' => 'x'], + ], schema(str_schema('value')))->schema(), + )))->asInts(array_to_rows([ + ['value' => '1'], + ['value' => '2'], + ['value' => 'x'], + ], schema(str_schema('value'))), flow_context()); + static::fail('expected an EvaluationException'); + } catch (EvaluationException $e) { + static::assertSame(0, $e->rowIndex); + static::assertInstanceOf(InvalidArgumentException::class, $e->getPrevious()); + } } - public function test_constructor_with_scalar_function(): void + public function test_column_evaluates_the_function(): void { - $scalarFunction = lit('test'); - $parameter = new Parameter($scalarFunction); + $rows = array_to_rows([['column' => 'ref_value']], schema(str_schema('column'))); - static::assertSame('test', $parameter->eval(row([]), flow_context())); + static::assertSame( + ['ref_value'], + (new Parameter((new ReferenceResolver())->resolve(ref('column'), $rows->schema()))) + ->column($rows, flow_context()) + ->values(), + ); } - public function test_eval_with_direct_value(): void + public function test_constructor_with_mixed_value(): void { - $parameter = new Parameter(lit('direct_value')); - static::assertSame('direct_value', $parameter->eval(row([]), flow_context())); - - $parameter = new Parameter(lit(42)); - static::assertSame(42, $parameter->eval(row([]), flow_context())); + static::assertSame( + ['direct_string'], + (new Parameter('direct_string'))->values(RowsMother::sequentialIds(1), flow_context()), + ); + static::assertSame([123], (new Parameter(123))->values(RowsMother::sequentialIds(1), flow_context())); + static::assertSame([true], (new Parameter(true))->values(RowsMother::sequentialIds(1), flow_context())); } - public function test_eval_with_reference(): void + public function test_values_read_an_array_result_back_as_arrays(): void { - $parameter = new Parameter(ref('column')); - $result = $parameter->eval(row(['column' => 'ref_value']), flow_context()); - - static::assertSame('ref_value', $result); + static::assertSame( + [['name' => 'a']], + (new Parameter((new ReferenceResolver())->resolve( + ref('j')->jsonDecode(), + array_to_rows([[ + 'j' => '{"name":"a"}', + ]], schema(json_schema('j')))->schema(), + )))->values(array_to_rows([[ + 'j' => '{"name":"a"}', + ]], schema(json_schema('j'))), flow_context()), + ); } } diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/PlusTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/PlusTest.php new file mode 100644 index 0000000000..c2b31b87ed --- /dev/null +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/PlusTest.php @@ -0,0 +1,57 @@ +eval( + ref('a')->plus(ref('b'), exact: true), + ['a' => 0.1, 'b' => 0.2], + schema(float_schema('a'), float_schema('b')), + )); + } + + public function test_floats_use_ieee_arithmetic(): void + { + static::assertSame(0.30000000000000004, (new FunctionContext(flow_context()))->eval( + ref('a')->plus(ref('b')), + ['a' => 0.1, 'b' => 0.2], + schema(float_schema('a'), float_schema('b')), + )); + } + + public function test_integers_stay_integers(): void + { + static::assertSame(8, (new FunctionContext(flow_context()))->eval( + ref('a')->plus(ref('b')), + ['a' => 5, 'b' => 3], + schema(int_schema('a'), int_schema('b')), + )); + } + + public function test_integer_overflow_names_its_row(): void + { + $this->expectException(EvaluationException::class); + $this->expectExceptionMessage('Plus function integer overflow (row 0)'); + + (new FunctionContext(flow_context()))->eval( + ref('a')->plus(ref('b')), + ['a' => PHP_INT_MAX, 'b' => 1], + schema(int_schema('a'), int_schema('b')), + ); + } +} diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/PowerTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/PowerTest.php index dac7637e58..789b841234 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/PowerTest.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/PowerTest.php @@ -4,13 +4,18 @@ namespace Flow\ETL\Tests\Unit\Function; +use Flow\ETL\Exception\EvaluationException; use Flow\ETL\Exception\InvalidArgumentException; +use Flow\ETL\Tests\Context\FunctionContext; use Flow\ETL\Tests\FlowTestCase; +use function Flow\ETL\DSL\float_schema; use function Flow\ETL\DSL\flow_context; +use function Flow\ETL\DSL\int_schema; use function Flow\ETL\DSL\lit; use function Flow\ETL\DSL\ref; -use function Flow\ETL\DSL\row; +use function Flow\ETL\DSL\schema; +use function Flow\ETL\DSL\str_schema; final class PowerTest extends FlowTestCase { @@ -19,12 +24,66 @@ public function test_power_non_numeric_values(): void $this->expectException(InvalidArgumentException::class); $this->expectExceptionMessage('Expected type "integer", got "string".'); - ref('int')->power(lit('non numeric'))->eval(row(['int' => 10]), flow_context()); - ref('str')->power(lit(2))->eval(row(['str' => 'abc']), flow_context()); + (new FunctionContext(flow_context()))->eval( + ref('int')->power(lit('non numeric')), + ['int' => 10], + schema(int_schema('int')), + ); + (new FunctionContext(flow_context()))->eval( + ref('str')->power(lit(2)), + ['str' => 'abc'], + schema(str_schema('str')), + ); } public function test_power_two_numeric_values(): void { - static::assertSame(100, ref('int')->power(lit(2))->eval(row(['int' => 10]), flow_context())); + static::assertSame(100, (new FunctionContext(flow_context()))->eval( + ref('int')->power(lit(2)), + ['int' => 10], + schema(int_schema('int')), + )); + } + + public function test_exact_keeps_decimal_arithmetic(): void + { + static::assertSame(1.21, (new FunctionContext(flow_context()))->eval( + ref('a')->power(ref('b'), exact: true), + ['a' => 1.1, 'b' => 2], + schema(float_schema('a'), int_schema('b')), + )); + } + + public function test_floats_use_ieee_arithmetic(): void + { + static::assertSame(1.2100000000000002, (new FunctionContext(flow_context()))->eval( + ref('a')->power(ref('b')), + ['a' => 1.1, 'b' => 2], + schema(float_schema('a'), int_schema('b')), + )); + } + + public function test_a_negative_exponent_of_an_integer_is_refused(): void + { + $this->expectException(EvaluationException::class); + $this->expectExceptionMessage('Power function of an integer requires a non-negative exponent (row 0)'); + + (new FunctionContext(flow_context()))->eval( + ref('a')->power(ref('b')), + ['a' => 2, 'b' => -1], + schema(int_schema('a'), int_schema('b')), + ); + } + + public function test_integer_overflow_names_its_row(): void + { + $this->expectException(EvaluationException::class); + $this->expectExceptionMessage('Power function integer overflow (row 0)'); + + (new FunctionContext(flow_context()))->eval( + ref('a')->power(ref('b')), + ['a' => 10, 'b' => 30], + schema(int_schema('a'), int_schema('b')), + ); } } diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/PregReplaceTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/PregReplaceTest.php index 246d0c81f1..c28ab749df 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/PregReplaceTest.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/PregReplaceTest.php @@ -5,12 +5,13 @@ namespace Flow\ETL\Tests\Unit\Function; use Flow\ETL\Exception\InvalidArgumentException; +use Flow\ETL\Tests\Context\FunctionContext; use Flow\ETL\Tests\FlowTestCase; use function Flow\ETL\DSL\flow_context; use function Flow\ETL\DSL\lit; use function Flow\ETL\DSL\regex_replace; -use function Flow\ETL\DSL\row; +use function Flow\ETL\DSL\schema; final class PregReplaceTest extends FlowTestCase { @@ -21,7 +22,7 @@ public function test_preg_replace_expression_on_invalid_pattern(): void $pregReplace = regex_replace(lit(1), lit('bar'), lit('foo is awesome')); - $pregReplace->eval(row([]), flow_context()); + (new FunctionContext(flow_context()))->eval($pregReplace, [], schema()); } public function test_preg_replace_expression_on_invalid_replacement(): void @@ -31,7 +32,7 @@ public function test_preg_replace_expression_on_invalid_replacement(): void $pregReplace = regex_replace(lit('/(foo)/'), lit(2), lit('foo is awesome')); - $pregReplace->eval(row([]), flow_context()); + (new FunctionContext(flow_context()))->eval($pregReplace, [], schema()); } public function test_preg_replace_expression_on_invalid_subject(): void @@ -41,13 +42,13 @@ public function test_preg_replace_expression_on_invalid_subject(): void $pregReplace = regex_replace(lit('/(foo)/'), lit('bar'), lit(3)); - $pregReplace->eval(row([]), flow_context()); + (new FunctionContext(flow_context()))->eval($pregReplace, [], schema()); } public function test_preg_replace_expression_on_valid_strings(): void { $pregReplace = regex_replace(lit('/(foo)/'), lit('bar'), lit('foo is awesome')); - static::assertSame('bar is awesome', $pregReplace->eval(row([]), flow_context())); + static::assertSame('bar is awesome', (new FunctionContext(flow_context()))->eval($pregReplace, [], schema())); } } diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/PrependTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/PrependTest.php index bb5f455498..ad9468f5fa 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/PrependTest.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/PrependTest.php @@ -5,40 +5,50 @@ namespace Flow\ETL\Tests\Unit\Function; use Flow\ETL\Exception\InvalidArgumentException; +use Flow\ETL\Tests\Context\FunctionContext; use Flow\ETL\Tests\FlowTestCase; use function Flow\ETL\DSL\flow_context; use function Flow\ETL\DSL\ref; -use function Flow\ETL\DSL\row; +use function Flow\ETL\DSL\schema; +use function Flow\ETL\DSL\str_schema; final class PrependTest extends FlowTestCase { public function test_prepend_empty_string_to_content(): void { - $result = ref('str')->prepend('')->eval(row(['str' => 'hello']), flow_context()); - - static::assertEquals('hello', $result); + static::assertEquals('hello', (new FunctionContext(flow_context()))->eval( + ref('str')->prepend(''), + ['str' => 'hello'], + schema(str_schema('str')), + )); } public function test_prepend_to_empty_string(): void { - $result = ref('str')->prepend('hello')->eval(row(['str' => '']), flow_context()); - - static::assertEquals('hello', $result); + static::assertEquals('hello', (new FunctionContext(flow_context()))->eval( + ref('str')->prepend('hello'), + ['str' => ''], + schema(str_schema('str')), + )); } public function test_prepend_to_non_empty_string(): void { - $result = ref('str')->prepend('Hello ')->eval(row(['str' => 'world']), flow_context()); - - static::assertEquals('Hello world', $result); + static::assertEquals('Hello world', (new FunctionContext(flow_context()))->eval( + ref('str')->prepend('Hello '), + ['str' => 'world'], + schema(str_schema('str')), + )); } public function test_prepend_with_null_prefix(): void { - $result = ref('str')->prepend(ref('prefix'))->eval(row(['str' => 'world', 'prefix' => null]), flow_context()); - - static::assertEquals('world', $result); + static::assertEquals('world', (new FunctionContext(flow_context()))->eval( + ref('str')->prepend(ref('prefix')), + ['str' => 'world', 'prefix' => null], + schema(str_schema('str'), str_schema('prefix', nullable: true)), + )); } public function test_prepend_with_null_value(): void @@ -46,17 +56,19 @@ public function test_prepend_with_null_value(): void $this->expectException(InvalidArgumentException::class); $this->expectExceptionMessage('Prepend function requires non-null value'); - $result = ref('str')->prepend('Hello ')->eval(row(['str' => null]), flow_context()); - - static::assertNull($result); + static::assertNull((new FunctionContext(flow_context()))->eval( + ref('str')->prepend('Hello '), + ['str' => null], + schema(str_schema('str', nullable: true)), + )); } public function test_prepend_with_scalar_function_parameter(): void { - $result = ref('str') - ->prepend(ref('prefix')) - ->eval(row(['str' => 'world', 'prefix' => 'Hello ']), flow_context()); - - static::assertEquals('Hello world', $result); + static::assertEquals('Hello world', (new FunctionContext(flow_context()))->eval( + ref('str')->prepend(ref('prefix')), + ['str' => 'world', 'prefix' => 'Hello '], + schema(str_schema('str'), str_schema('prefix')), + )); } } diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/RankTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/RankTest.php index e9e638519d..1146da01bd 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/RankTest.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/RankTest.php @@ -8,11 +8,10 @@ use Flow\ETL\Tests\FlowTestCase; use Flow\ETL\Tests\Mother\WindowContextMother; +use function Flow\ETL\DSL\array_to_rows; use function Flow\ETL\DSL\int_schema; use function Flow\ETL\DSL\rank; use function Flow\ETL\DSL\ref; -use function Flow\ETL\DSL\row; -use function Flow\ETL\DSL\rows; use function Flow\ETL\DSL\schema; use function Flow\ETL\DSL\window; @@ -69,7 +68,7 @@ public function test_rank_requires_an_order_by(): void { $this->expectExceptionMessage('Rank window function requires to be ordered by one column'); - rank()->over(window())->rankPartition(rows(schema(int_schema('salary')), row(['salary' => 6000]))); + rank()->over(window())->rankPartition(array_to_rows([['salary' => 6000]], schema(int_schema('salary')))); } public function test_rank_without_over_clause(): void diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/RegexAllTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/RegexAllTest.php index 0399de8a8b..7e250ac621 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/RegexAllTest.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/RegexAllTest.php @@ -5,12 +5,13 @@ namespace Flow\ETL\Tests\Unit\Function; use Flow\ETL\Exception\InvalidArgumentException; +use Flow\ETL\Tests\Context\FunctionContext; use Flow\ETL\Tests\FlowTestCase; use function Flow\ETL\DSL\flow_context; use function Flow\ETL\DSL\lit; use function Flow\ETL\DSL\regex_all; -use function Flow\ETL\DSL\row; +use function Flow\ETL\DSL\schema; final class RegexAllTest extends FlowTestCase { @@ -21,14 +22,14 @@ public function test_regex_all_expression_on_invalid_subject(): void $pregMatch = regex_all(lit('/\d+/'), lit(2)); - $pregMatch->eval(row([]), flow_context()); + (new FunctionContext(flow_context()))->eval($pregMatch, [], schema()); } public function test_regex_all_expression_on_no_match(): void { $pregMatch = regex_all(lit('/\d+/'), lit('apples and oranges')); - static::assertNull($pregMatch->eval(row([]), flow_context())); + static::assertNull((new FunctionContext(flow_context()))->eval($pregMatch, [], schema())); } public function test_regex_all_expression_on_valid_strings(): void @@ -41,7 +42,7 @@ public function test_regex_all_expression_on_valid_strings(): void ['124.23', '12', '45'], ['EUR', 'USD', 'PLN'], ], - $pregMatch->eval(row([]), flow_context()), + (new FunctionContext(flow_context()))->eval($pregMatch, [], schema()), ); } @@ -52,6 +53,6 @@ public function test_regex_expression_on_invalid_pattern(): void $pregMatch = regex_all(lit(1), lit('12 apples and 45 oranges')); - $pregMatch->eval(row([]), flow_context()); + (new FunctionContext(flow_context()))->eval($pregMatch, [], schema()); } } diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/RegexMatchAllTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/RegexMatchAllTest.php index 393b2f5f84..85a7b0c14d 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/RegexMatchAllTest.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/RegexMatchAllTest.php @@ -5,12 +5,13 @@ namespace Flow\ETL\Tests\Unit\Function; use Flow\ETL\Exception\InvalidArgumentException; +use Flow\ETL\Tests\Context\FunctionContext; use Flow\ETL\Tests\FlowTestCase; use function Flow\ETL\DSL\flow_context; use function Flow\ETL\DSL\lit; use function Flow\ETL\DSL\regex_match_all; -use function Flow\ETL\DSL\row; +use function Flow\ETL\DSL\schema; final class RegexMatchAllTest extends FlowTestCase { @@ -21,7 +22,7 @@ public function test_regex_match_all_expression_on_invalid_flags(): void $pregMatchAll = regex_match_all(lit('/\d+/'), lit('12 apples and 45 oranges'), lit('invalid')); - $pregMatchAll->eval(row([]), flow_context()); + (new FunctionContext(flow_context()))->eval($pregMatchAll, [], schema()); } public function test_regex_match_all_expression_on_invalid_pattern(): void @@ -31,7 +32,7 @@ public function test_regex_match_all_expression_on_invalid_pattern(): void $pregMatchAll = regex_match_all(lit(1), lit('12 apples and 45 oranges')); - $pregMatchAll->eval(row([]), flow_context()); + (new FunctionContext(flow_context()))->eval($pregMatchAll, [], schema()); } public function test_regex_match_all_expression_on_invalid_subject(): void @@ -41,14 +42,14 @@ public function test_regex_match_all_expression_on_invalid_subject(): void $pregMatchAll = regex_match_all(lit('/\d+/'), lit(2)); - $pregMatchAll->eval(row([]), flow_context()); + (new FunctionContext(flow_context()))->eval($pregMatchAll, [], schema()); } public function test_regex_match_all_expression_on_valid_strings(): void { $pregMatchAll = regex_match_all(lit('/\d+/'), lit('12 apples and 45 oranges')); - static::assertTrue($pregMatchAll->eval(row([]), flow_context())); + static::assertTrue((new FunctionContext(flow_context()))->eval($pregMatchAll, [], schema())); } public function test_regex_match_all_expression_on_valid_strings_with_flags(): void @@ -59,6 +60,6 @@ public function test_regex_match_all_expression_on_valid_strings_with_flags(): v lit(PREG_PATTERN_ORDER), ); - static::assertTrue($pregMatchAll->eval(row([]), flow_context())); + static::assertTrue((new FunctionContext(flow_context()))->eval($pregMatchAll, [], schema())); } } diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/RegexMatchTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/RegexMatchTest.php index 54c248e09d..67419212cb 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/RegexMatchTest.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/RegexMatchTest.php @@ -5,12 +5,13 @@ namespace Flow\ETL\Tests\Unit\Function; use Flow\ETL\Exception\InvalidArgumentException; +use Flow\ETL\Tests\Context\FunctionContext; use Flow\ETL\Tests\FlowTestCase; use function Flow\ETL\DSL\flow_context; use function Flow\ETL\DSL\lit; use function Flow\ETL\DSL\regex_match; -use function Flow\ETL\DSL\row; +use function Flow\ETL\DSL\schema; final class RegexMatchTest extends FlowTestCase { @@ -21,7 +22,7 @@ public function test_regex_match_expression_on_invalid_pattern(): void $pregMatch = regex_match(lit(1), lit('12 apples and 45 oranges')); - $pregMatch->eval(row([]), flow_context()); + (new FunctionContext(flow_context()))->eval($pregMatch, [], schema()); } public function test_regex_match_expression_on_invalid_subject(): void @@ -31,20 +32,20 @@ public function test_regex_match_expression_on_invalid_subject(): void $pregMatch = regex_match(lit('/\d+/'), lit(2)); - $pregMatch->eval(row([]), flow_context()); + (new FunctionContext(flow_context()))->eval($pregMatch, [], schema()); } public function test_regex_match_expression_on_no_match(): void { $pregMatch = regex_match(lit('/\d+/'), lit('apples and oranges')); - static::assertFalse($pregMatch->eval(row([]), flow_context())); + static::assertFalse((new FunctionContext(flow_context()))->eval($pregMatch, [], schema())); } public function test_regex_match_expression_on_valid_strings(): void { $pregMatch = regex_match(lit('/\d+/'), lit('12 apples and 45 oranges')); - static::assertTrue($pregMatch->eval(row([]), flow_context())); + static::assertTrue((new FunctionContext(flow_context()))->eval($pregMatch, [], schema())); } } diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/RegexReplaceTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/RegexReplaceTest.php index 70d5ab9566..6d7bce3633 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/RegexReplaceTest.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/RegexReplaceTest.php @@ -4,27 +4,37 @@ namespace Flow\ETL\Tests\Unit\Function; +use Flow\ETL\Tests\Context\FunctionContext; use Flow\ETL\Tests\FlowTestCase; use function Flow\ETL\DSL\flow_context; use function Flow\ETL\DSL\lit; use function Flow\ETL\DSL\ref; use function Flow\ETL\DSL\regex_replace; -use function Flow\ETL\DSL\row; +use function Flow\ETL\DSL\schema; +use function Flow\ETL\DSL\str_schema; final class RegexReplaceTest extends FlowTestCase { public function test_a_null_limit_still_means_unlimited(): void { - static::assertSame('x x x', regex_replace(lit('/a/'), lit('x'), ref('text'))->eval(row([ - 'text' => 'a a a', - ]), flow_context())); + static::assertSame('x x x', (new FunctionContext(flow_context()))->eval( + regex_replace(lit('/a/'), lit('x'), ref('text')), + [ + 'text' => 'a a a', + ], + schema(str_schema('text')), + )); } public function test_a_limit_caps_the_replacements(): void { - static::assertSame('x a a', regex_replace(lit('/a/'), lit('x'), ref('text'), lit(1))->eval(row([ - 'text' => 'a a a', - ]), flow_context())); + static::assertSame('x a a', (new FunctionContext(flow_context()))->eval( + regex_replace(lit('/a/'), lit('x'), ref('text'), lit(1)), + [ + 'text' => 'a a a', + ], + schema(str_schema('text')), + )); } } diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/RegexTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/RegexTest.php index f5917e1f24..bb3557bcc0 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/RegexTest.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/RegexTest.php @@ -5,13 +5,14 @@ namespace Flow\ETL\Tests\Unit\Function; use Flow\ETL\Exception\InvalidArgumentException; +use Flow\ETL\Tests\Context\FunctionContext; use Flow\ETL\Tests\FlowTestCase; use function Flow\ETL\DSL\flow_context; use function Flow\ETL\DSL\lit; use function Flow\ETL\DSL\regex; use function Flow\ETL\DSL\regex_match_all; -use function Flow\ETL\DSL\row; +use function Flow\ETL\DSL\schema; final class RegexTest extends FlowTestCase { @@ -22,7 +23,7 @@ public function test_regex_expression_on_invalid_pattern(): void $pregMatch = regex(lit(1), lit('12 apples and 45 oranges')); - $pregMatch->eval(row([]), flow_context()); + (new FunctionContext(flow_context()))->eval($pregMatch, [], schema()); } public function test_regex_expression_on_invalid_subject(): void @@ -32,20 +33,20 @@ public function test_regex_expression_on_invalid_subject(): void $pregMatch = regex(lit('/\d+/'), lit(2)); - $pregMatch->eval(row([]), flow_context()); + (new FunctionContext(flow_context()))->eval($pregMatch, [], schema()); } public function test_regex_expression_on_no_match(): void { $pregMatch = regex(lit('/\d+/'), lit('apples and oranges')); - static::assertNull($pregMatch->eval(row([]), flow_context())); + static::assertNull((new FunctionContext(flow_context()))->eval($pregMatch, [], schema())); } public function test_regex_expression_on_valid_strings(): void { $pregMatch = regex_match_all(lit('/\d+/'), lit('12 apples and 45 oranges')); - static::assertTrue($pregMatch->eval(row([]), flow_context())); + static::assertTrue((new FunctionContext(flow_context()))->eval($pregMatch, [], schema())); } } diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/RepeatTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/RepeatTest.php index e2d83a7f45..c947204b18 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/RepeatTest.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/RepeatTest.php @@ -5,17 +5,24 @@ namespace Flow\ETL\Tests\Unit\Function; use Flow\ETL\Exception\InvalidArgumentException; +use Flow\ETL\Tests\Context\FunctionContext; use Flow\ETL\Tests\FlowTestCase; use function Flow\ETL\DSL\flow_context; +use function Flow\ETL\DSL\int_schema; use function Flow\ETL\DSL\ref; -use function Flow\ETL\DSL\row; +use function Flow\ETL\DSL\schema; +use function Flow\ETL\DSL\str_schema; final class RepeatTest extends FlowTestCase { public function test_repeat_empty_string(): void { - static::assertSame('', ref('str')->repeat(3)->eval(row(['str' => '']), flow_context())); + static::assertSame('', (new FunctionContext(flow_context()))->eval( + ref('str')->repeat(3), + ['str' => ''], + schema(str_schema('str')), + )); } public function test_repeat_negative_times(): void @@ -23,7 +30,11 @@ public function test_repeat_negative_times(): void $this->expectException(InvalidArgumentException::class); $this->expectExceptionMessage('Repeat function requires non-null, positive times'); - ref('str')->repeat(-1)->eval(row(['str' => 'hello']), flow_context()); + (new FunctionContext(flow_context()))->eval( + ref('str')->repeat(-1), + ['str' => 'hello'], + schema(str_schema('str')), + ); } public function test_repeat_null_input(): void @@ -31,12 +42,20 @@ public function test_repeat_null_input(): void $this->expectException(InvalidArgumentException::class); $this->expectExceptionMessage('Repeat function requires non-null value'); - ref('str')->repeat(3)->eval(row(['str' => null]), flow_context()); + (new FunctionContext(flow_context()))->eval( + ref('str')->repeat(3), + ['str' => null], + schema(str_schema('str', nullable: true)), + ); } public function test_repeat_string_multiple_times(): void { - static::assertSame('hellohellohello', ref('str')->repeat(3)->eval(row(['str' => 'hello']), flow_context())); + static::assertSame('hellohellohello', (new FunctionContext(flow_context()))->eval( + ref('str')->repeat(3), + ['str' => 'hello'], + schema(str_schema('str')), + )); } public function test_repeat_with_null_times(): void @@ -44,14 +63,20 @@ public function test_repeat_with_null_times(): void $this->expectException(InvalidArgumentException::class); $this->expectExceptionMessage('Repeat function requires non-null, positive times'); - ref('str')->repeat(ref('times'))->eval(row(['str' => 'hello', 'times' => null]), flow_context()); + (new FunctionContext(flow_context()))->eval( + ref('str')->repeat(ref('times')), + ['str' => 'hello', 'times' => null], + schema(str_schema('str'), str_schema('times', nullable: true)), + ); } public function test_repeat_with_scalar_function_times(): void { - static::assertSame('hellohello', ref('str') - ->repeat(ref('times')) - ->eval(row(['str' => 'hello', 'times' => 2]), flow_context())); + static::assertSame('hellohello', (new FunctionContext(flow_context()))->eval( + ref('str')->repeat(ref('times')), + ['str' => 'hello', 'times' => 2], + schema(str_schema('str'), int_schema('times')), + )); } public function test_repeat_zero_times(): void @@ -59,6 +84,10 @@ public function test_repeat_zero_times(): void $this->expectException(InvalidArgumentException::class); $this->expectExceptionMessage('Repeat function requires non-null, positive times'); - ref('str')->repeat(0)->eval(row(['str' => 'hello']), flow_context()); + (new FunctionContext(flow_context()))->eval( + ref('str')->repeat(0), + ['str' => 'hello'], + schema(str_schema('str')), + ); } } diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/ReverseTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/ReverseTest.php index daffef3795..dde38a5ebf 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/ReverseTest.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/ReverseTest.php @@ -5,22 +5,32 @@ namespace Flow\ETL\Tests\Unit\Function; use Flow\ETL\Exception\InvalidArgumentException; +use Flow\ETL\Tests\Context\FunctionContext; use Flow\ETL\Tests\FlowTestCase; use function Flow\ETL\DSL\flow_context; use function Flow\ETL\DSL\ref; -use function Flow\ETL\DSL\row; +use function Flow\ETL\DSL\schema; +use function Flow\ETL\DSL\str_schema; final class ReverseTest extends FlowTestCase { public function test_reverse_ascii_string(): void { - static::assertSame('olleh', ref('str')->reverse()->eval(row(['str' => 'hello']), flow_context())); + static::assertSame('olleh', (new FunctionContext(flow_context()))->eval( + ref('str')->reverse(), + ['str' => 'hello'], + schema(str_schema('str')), + )); } public function test_reverse_empty_string(): void { - static::assertSame('', ref('str')->reverse()->eval(row(['str' => '']), flow_context())); + static::assertSame('', (new FunctionContext(flow_context()))->eval( + ref('str')->reverse(), + ['str' => ''], + schema(str_schema('str')), + )); } public function test_reverse_throws_on_null_input(): void @@ -28,6 +38,10 @@ public function test_reverse_throws_on_null_input(): void $this->expectException(InvalidArgumentException::class); $this->expectExceptionMessage('Reverse function requires non-null value'); - ref('str')->reverse()->eval(row(['str' => null]), flow_context()); + (new FunctionContext(flow_context()))->eval( + ref('str')->reverse(), + ['str' => null], + schema(str_schema('str', nullable: true)), + ); } } diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/RoundTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/RoundTest.php index 7b62ca6668..f0aad8479f 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/RoundTest.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/RoundTest.php @@ -5,32 +5,58 @@ namespace Flow\ETL\Tests\Unit\Function; use Flow\ETL\Function\Round; +use Flow\ETL\Tests\Context\FunctionContext; use Flow\ETL\Tests\FlowTestCase; +use function Flow\ETL\DSL\float_schema; use function Flow\ETL\DSL\flow_context; use function Flow\ETL\DSL\lit; use function Flow\ETL\DSL\ref; -use function Flow\ETL\DSL\row; +use function Flow\ETL\DSL\schema; final class RoundTest extends FlowTestCase { public function test_round_float(): void { - static::assertEquals(10.12, ref('float')->round(lit(2))->eval(row(['float' => 10.123]), flow_context())); - - static::assertIsFloat(ref('float')->round(lit(2))->eval(row(['float' => 10.123]), flow_context())); + static::assertEquals(10.12, (new FunctionContext(flow_context()))->eval( + ref('float')->round(lit(2)), + ['float' => 10.123], + schema(float_schema('float')), + )); + + static::assertIsFloat((new FunctionContext(flow_context()))->eval( + ref('float')->round(lit(2)), + ['float' => 10.123], + schema(float_schema('float')), + )); } public function test_round_with_precision_0(): void { - static::assertSame(10.0, ref('float')->round(lit(0))->eval(row(['float' => 10.123]), flow_context())); + static::assertSame(10.0, (new FunctionContext(flow_context()))->eval( + ref('float')->round(lit(0)), + ['float' => 10.123], + schema(float_schema('float')), + )); } public function test_constructor_default_matches_the_dsl_default(): void { - $row = row(['float' => 10.12345]); - - static::assertSame(round(10.12345, 2), (new Round(ref('float')))->eval($row, flow_context())); - static::assertSame(round(10.12345, 2), ref('float')->round()->eval($row, flow_context())); + static::assertSame( + round(10.12345, 2), + (new FunctionContext(flow_context()))->eval( + new Round(ref('float')), + ['float' => 10.12345], + schema(float_schema('float')), + ), + ); + static::assertSame( + round(10.12345, 2), + (new FunctionContext(flow_context()))->eval( + ref('float')->round(), + ['float' => 10.12345], + schema(float_schema('float')), + ), + ); } } diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/RowNumberTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/RowNumberTest.php index 6a96fa357b..a47f1b35c3 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/RowNumberTest.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/RowNumberTest.php @@ -7,11 +7,10 @@ use Flow\ETL\Tests\FlowTestCase; use Flow\ETL\Tests\Mother\WindowContextMother; +use function Flow\ETL\DSL\array_to_rows; use function Flow\ETL\DSL\int_schema; use function Flow\ETL\DSL\ref; -use function Flow\ETL\DSL\row; use function Flow\ETL\DSL\row_number; -use function Flow\ETL\DSL\rows; use function Flow\ETL\DSL\schema; use function Flow\ETL\DSL\window; @@ -19,32 +18,33 @@ final class RowNumberTest extends FlowTestCase { public function test_row_number_is_the_position_in_the_ordered_partition(): void { - $rows = rows( + $rows = array_to_rows( + [ + ['id' => 5, 'value' => 1], + ['id' => 4, 'value' => 1], + ['id' => 3, 'value' => 1], + ['id' => 2, 'value' => 1], + ['id' => 1, 'value' => 1], + ], schema(int_schema('id'), int_schema('value')), - row(['id' => 5, 'value' => 1]), - row(['id' => 4, 'value' => 1]), - row(['id' => 3, 'value' => 1]), - row(['id' => 2, 'value' => 1]), - $row1 = row(['id' => 1, 'value' => 1]), ); $rowNumber = row_number()->over(window()->partitionBy(ref('value'))->orderBy(ref('id')->desc())); - static::assertSame(5, $rowNumber->apply(WindowContextMother::forRow($row1, $rows, 4))); + static::assertSame(5, $rowNumber->apply(WindowContextMother::atIndex($rows, 4))); } public function test_row_number_numbers_duplicated_rows_separately(): void { - $rows = rows( + $rows = array_to_rows( + [['id' => 1, 'value' => 100], ['id' => 1, 'value' => 100]], schema(int_schema('id'), int_schema('value')), - $row1 = row(['id' => 1, 'value' => 100]), - $row2 = row(['id' => 1, 'value' => 100]), ); $rowNumber = row_number()->over(window()->orderBy(ref('value'))); - static::assertSame(1, $rowNumber->apply(WindowContextMother::forRow($row1, $rows, 0))); - static::assertSame(2, $rowNumber->apply(WindowContextMother::forRow($row2, $rows, 1))); + static::assertSame(1, $rowNumber->apply(WindowContextMother::atIndex($rows, 0))); + static::assertSame(2, $rowNumber->apply(WindowContextMother::atIndex($rows, 1))); } public function test_with_children_returns_the_same_leaf(): void diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/SameTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/SameTest.php index b586dfe240..66d7d9a189 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/SameTest.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/SameTest.php @@ -4,19 +4,90 @@ namespace Flow\ETL\Tests\Unit\Function; +use DateInterval; +use DateTimeImmutable; +use DateTimeZone; +use Flow\ETL\Tests\Context\FunctionContext; +use Flow\ETL\Tests\Fixtures\Enum\BackedStringEnum; use Flow\ETL\Tests\FlowTestCase; +use Flow\Types\Type; +use Generator; +use PHPUnit\Framework\Attributes\DataProvider; +use function Flow\ETL\DSL\definition_from_type; use function Flow\ETL\DSL\flow_context; use function Flow\ETL\DSL\lit; use function Flow\ETL\DSL\ref; -use function Flow\ETL\DSL\row; +use function Flow\ETL\DSL\schema; +use function Flow\ETL\DSL\str_schema; +use function Flow\Types\DSL\type_date; +use function Flow\Types\DSL\type_datetime; +use function Flow\Types\DSL\type_enum; +use function Flow\Types\DSL\type_json; +use function Flow\Types\DSL\type_time; +use function Flow\Types\DSL\type_uuid; final class SameTest extends FlowTestCase { public function test_null_is_identical_to_null(): void { // PHP identity, not SQL equality - ->same() is the migration path for both-null checks. - static::assertTrue(ref('a')->same(lit(null))->eval(row(['a' => null]), flow_context())); - static::assertFalse(ref('a')->same(lit(1))->eval(row(['a' => null]), flow_context())); + static::assertTrue((new FunctionContext(flow_context()))->eval( + ref('a')->same(lit(null)), + ['a' => null], + schema(str_schema('a', nullable: true)), + )); + static::assertFalse((new FunctionContext(flow_context()))->eval( + ref('a')->same(lit(1)), + ['a' => null], + schema(str_schema('a', nullable: true)), + )); + } + + /** + * @return Generator, mixed, mixed, bool}> + */ + public static function values_of_one_type(): Generator + { + yield 'uuid' => [ + type_uuid(), + '00000000-0000-4000-8000-000000000001', + '00000000-0000-4000-8000-000000000001', + true, + ]; + yield 'other uuid' => [ + type_uuid(), + '00000000-0000-4000-8000-000000000001', + '00000000-0000-4000-8000-000000000002', + false, + ]; + yield 'json' => [type_json(), '{"a":1,"b":2}', '{"a":1,"b":2}', true]; + yield 'json compares its stored text' => [type_json(), '{"a":1,"b":2}', '{"b":2,"a":1}', false]; + yield 'enum' => [type_enum(BackedStringEnum::class), BackedStringEnum::one, BackedStringEnum::one, true]; + yield 'datetime of the same instant in another zone' => [ + type_datetime(), + new DateTimeImmutable('2024-01-01 12:00:00', new DateTimeZone('UTC')), + new DateTimeImmutable('2024-01-01 13:00:00', new DateTimeZone('Europe/Warsaw')), + true, + ]; + yield 'date' => [type_date(), new DateTimeImmutable('2024-01-01'), new DateTimeImmutable('2024-01-01'), true]; + yield 'time' => [type_time(), new DateInterval('PT60M'), new DateInterval('PT1H'), true]; + } + + /** + * @param Type $type + */ + #[DataProvider('values_of_one_type')] + public function test_values_of_one_type_are_the_same_by_value( + Type $type, + mixed $left, + mixed $right, + bool $same, + ): void { + static::assertSame($same, (new FunctionContext(flow_context()))->eval( + ref('a')->same(ref('b')), + ['a' => $left, 'b' => $right], + schema(definition_from_type('a', $type), definition_from_type('b', $type)), + )); } } diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/SanitizeTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/SanitizeTest.php index 76e81a8ca7..88d66e43b9 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/SanitizeTest.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/SanitizeTest.php @@ -5,12 +5,15 @@ namespace Flow\ETL\Tests\Unit\Function; use Flow\ETL\Exception\InvalidArgumentException; +use Flow\ETL\Tests\Context\FunctionContext; use Flow\ETL\Tests\FlowTestCase; use function Flow\ETL\DSL\flow_context; +use function Flow\ETL\DSL\int_schema; use function Flow\ETL\DSL\lit; use function Flow\ETL\DSL\ref; -use function Flow\ETL\DSL\row; +use function Flow\ETL\DSL\schema; +use function Flow\ETL\DSL\str_schema; final class SanitizeTest extends FlowTestCase { @@ -19,30 +22,46 @@ public function test_sanitize_on_non_string_value(): void $this->expectException(InvalidArgumentException::class); $this->expectExceptionMessage('Expected type "string", got "integer".'); - ref('value')->sanitize()->eval(row(['value' => 1000]), flow_context()); + (new FunctionContext(flow_context()))->eval( + ref('value')->sanitize(), + ['value' => 1000], + schema(int_schema('value')), + ); } public function test_sanitize_on_valid_string(): void { - static::assertSame('****', ref('value')->sanitize()->eval(row(['value' => 'test']), flow_context())); + static::assertSame('****', (new FunctionContext(flow_context()))->eval( + ref('value')->sanitize(), + ['value' => 'test'], + schema(str_schema('value')), + )); } public function test_sanitize_on_valid_string_with_left_characters(): void { - static::assertSame('te**', ref('value') - ->sanitize(skipCharacters: lit(2)) - ->eval(row(['value' => 'test']), flow_context())); + static::assertSame('te**', (new FunctionContext(flow_context()))->eval( + ref('value')->sanitize(skipCharacters: lit(2)), + ['value' => 'test'], + schema(str_schema('value')), + )); } public function test_sanitize_on_valid_string_with_left_characters_longer_than_string(): void { - static::assertSame('****', ref('value') - ->sanitize(skipCharacters: lit(5)) - ->eval(row(['value' => 'test']), flow_context())); + static::assertSame('****', (new FunctionContext(flow_context()))->eval( + ref('value')->sanitize(skipCharacters: lit(5)), + ['value' => 'test'], + schema(str_schema('value')), + )); } public function test_sanitize_on_valid_string_with_placeholder(): void { - static::assertSame('----', ref('value')->sanitize(lit('-'))->eval(row(['value' => 'test']), flow_context())); + static::assertSame('----', (new FunctionContext(flow_context()))->eval( + ref('value')->sanitize(lit('-')), + ['value' => 'test'], + schema(str_schema('value')), + )); } } diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/SizeTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/SizeTest.php index 396a294d4c..2fdba80d9d 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/SizeTest.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/SizeTest.php @@ -4,27 +4,32 @@ namespace Flow\ETL\Tests\Unit\Function; +use Flow\ETL\Tests\Context\FunctionContext; use Flow\ETL\Tests\FlowTestCase; use function Flow\ETL\DSL\flow_context; use function Flow\ETL\DSL\lit; -use function Flow\ETL\DSL\row; +use function Flow\ETL\DSL\schema; use function Flow\ETL\DSL\size; final class SizeTest extends FlowTestCase { public function test_size_expression_on_array_value(): void { - static::assertSame(3, size(lit(['foo', 'bar', 'baz']))->eval(row([]), flow_context())); + static::assertSame(3, (new FunctionContext(flow_context()))->eval( + size(lit(['foo', 'bar', 'baz'])), + [], + schema(), + )); } public function test_size_expression_on_integer_value(): void { - static::assertNull(size(lit(1))->eval(row([]), flow_context())); + static::assertNull((new FunctionContext(flow_context()))->eval(size(lit(1)), [], schema())); } public function test_size_expression_on_string_value(): void { - static::assertSame(3, size(lit('foo'))->eval(row([]), flow_context())); + static::assertSame(3, (new FunctionContext(flow_context()))->eval(size(lit('foo')), [], schema())); } } diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/SlugTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/SlugTest.php index 4a08d27b13..515bc4afd1 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/SlugTest.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/SlugTest.php @@ -5,12 +5,14 @@ namespace Flow\ETL\Tests\Unit\Function; use Flow\ETL\Exception\InvalidArgumentException; +use Flow\ETL\Tests\Context\FunctionContext; use Flow\ETL\Tests\FlowTestCase; use function Flow\ETL\DSL\flow_context; use function Flow\ETL\DSL\lit; use function Flow\ETL\DSL\ref; -use function Flow\ETL\DSL\row; +use function Flow\ETL\DSL\schema; +use function Flow\ETL\DSL\str_schema; final class SlugTest extends FlowTestCase { @@ -18,7 +20,11 @@ public function test_a_malformed_separator_operand_throws(): void { $this->expectException(InvalidArgumentException::class); - ref('value')->slug(lit(123))->eval(row(['value' => 'hello world']), flow_context()); + (new FunctionContext(flow_context()))->eval( + ref('value')->slug(lit(123)), + ['value' => 'hello world'], + schema(str_schema('value')), + ); } public function test_ascii_on_null(): void @@ -26,16 +32,28 @@ public function test_ascii_on_null(): void $this->expectException(InvalidArgumentException::class); $this->expectExceptionMessage('Slug function requires non-null value'); - ref('str')->slug()->eval(row(['str' => null]), flow_context()); + (new FunctionContext(flow_context()))->eval( + ref('str')->slug(), + ['str' => null], + schema(str_schema('str', nullable: true)), + ); } public function test_slug(): void { - static::assertSame('azcz', ref('str')->slug()->eval(row(['str' => 'ąźćż']), flow_context())); + static::assertSame('azcz', (new FunctionContext(flow_context()))->eval( + ref('str')->slug(), + ['str' => 'ąźćż'], + schema(str_schema('str')), + )); } public function test_slug_separator(): void { - static::assertSame('Some_Text', ref('str')->slug('_')->eval(row(['str' => 'Some Text']), flow_context())); + static::assertSame('Some_Text', (new FunctionContext(flow_context()))->eval( + ref('str')->slug('_'), + ['str' => 'Some Text'], + schema(str_schema('str')), + )); } } diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/SplitTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/SplitTest.php index 3e025ae451..a87af464f1 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/SplitTest.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/SplitTest.php @@ -5,11 +5,12 @@ namespace Flow\ETL\Tests\Unit\Function; use Flow\ETL\Exception\InvalidArgumentException; +use Flow\ETL\Tests\Context\FunctionContext; use Flow\ETL\Tests\FlowTestCase; use function Flow\ETL\DSL\flow_context; use function Flow\ETL\DSL\lit; -use function Flow\ETL\DSL\row; +use function Flow\ETL\DSL\schema; use function Flow\ETL\DSL\split; final class SplitTest extends FlowTestCase @@ -19,16 +20,22 @@ public function test_split_not_string(): void $this->expectException(InvalidArgumentException::class); $this->expectExceptionMessage('Expected type "string", got "integer".'); - split(lit(123), ',')->eval(row([]), flow_context()); + (new FunctionContext(flow_context()))->eval(split(lit(123), ','), [], schema()); } public function test_split_string(): void { - static::assertSame(['foo', 'bar', 'baz'], split(lit('foo,bar,baz'), ',')->eval(row([]), flow_context())); + static::assertSame( + ['foo', 'bar', 'baz'], + (new FunctionContext(flow_context()))->eval(split(lit('foo,bar,baz'), ','), [], schema()), + ); } public function test_split_string_with_limit(): void { - static::assertSame(['foo', 'bar,baz'], split(lit('foo,bar,baz'), ',', 2)->eval(row([]), flow_context())); + static::assertSame( + ['foo', 'bar,baz'], + (new FunctionContext(flow_context()))->eval(split(lit('foo,bar,baz'), ',', 2), [], schema()), + ); } } diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/SprintfTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/SprintfTest.php index d73c7b7e25..6e7aafacd2 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/SprintfTest.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/SprintfTest.php @@ -5,11 +5,12 @@ namespace Flow\ETL\Tests\Unit\Function; use Flow\ETL\Exception\InvalidArgumentException; +use Flow\ETL\Tests\Context\FunctionContext; use Flow\ETL\Tests\FlowTestCase; use function Flow\ETL\DSL\flow_context; use function Flow\ETL\DSL\lit; -use function Flow\ETL\DSL\row; +use function Flow\ETL\DSL\schema; use function Flow\ETL\DSL\sprintf; final class SprintfTest extends FlowTestCase @@ -21,13 +22,17 @@ public function test_sprintf_expression_on_invalid_format(): void $sprintf = sprintf(lit(1), lit('John'), lit(25)); - $sprintf->eval(row([]), flow_context()); + (new FunctionContext(flow_context()))->eval($sprintf, [], schema()); } public function test_sprintf_expression_on_valid_format_and_args(): void { $sprintf = sprintf(lit('Hello, %s! Your age is %d.'), lit('John'), lit(25)); - static::assertSame('Hello, John! Your age is 25.', $sprintf->eval(row([]), flow_context())); + static::assertSame('Hello, John! Your age is 25.', (new FunctionContext(flow_context()))->eval( + $sprintf, + [], + schema(), + )); } } diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/StrPadTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/StrPadTest.php index 30e39558a7..ab01aabbf5 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/StrPadTest.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/StrPadTest.php @@ -4,11 +4,13 @@ namespace Flow\ETL\Tests\Unit\Function; +use Flow\ETL\Tests\Context\FunctionContext; use Flow\ETL\Tests\FlowTestCase; use function Flow\ETL\DSL\flow_context; use function Flow\ETL\DSL\ref; -use function Flow\ETL\DSL\row; +use function Flow\ETL\DSL\schema; +use function Flow\ETL\DSL\str_schema; use const STR_PAD_LEFT; @@ -16,8 +18,10 @@ final class StrPadTest extends FlowTestCase { public function test_str_pad_on_valid_string(): void { - static::assertSame('----N', ref('value') - ->strPad(5, '-', STR_PAD_LEFT) - ->eval(row(['value' => 'N']), flow_context())); + static::assertSame('----N', (new FunctionContext(flow_context()))->eval( + ref('value')->strPad(5, '-', STR_PAD_LEFT), + ['value' => 'N'], + schema(str_schema('value')), + )); } } diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/StrReplaceTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/StrReplaceTest.php index 406f437022..448444a9ed 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/StrReplaceTest.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/StrReplaceTest.php @@ -4,23 +4,31 @@ namespace Flow\ETL\Tests\Unit\Function; +use Flow\ETL\Tests\Context\FunctionContext; use Flow\ETL\Tests\FlowTestCase; use function Flow\ETL\DSL\flow_context; use function Flow\ETL\DSL\ref; -use function Flow\ETL\DSL\row; +use function Flow\ETL\DSL\schema; +use function Flow\ETL\DSL\str_schema; final class StrReplaceTest extends FlowTestCase { public function test_str_replace_on_valid_string(): void { - static::assertSame('1', ref('value')->strReplace('test', '1')->eval(row(['value' => 'test']), flow_context())); + static::assertSame('1', (new FunctionContext(flow_context()))->eval( + ref('value')->strReplace('test', '1'), + ['value' => 'test'], + schema(str_schema('value')), + )); } public function test_str_replace_on_valid_string_with_array_of_replacements(): void { - static::assertSame('test was successful', ref('value') - ->strReplace(['is', 'broken'], ['was', 'successful']) - ->eval(row(['value' => 'test is broken']), flow_context())); + static::assertSame('test was successful', (new FunctionContext(flow_context()))->eval( + ref('value')->strReplace(['is', 'broken'], ['was', 'successful']), + ['value' => 'test is broken'], + schema(str_schema('value')), + )); } } diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/StringAfterLastTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/StringAfterLastTest.php index 302593965c..d7d387fa36 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/StringAfterLastTest.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/StringAfterLastTest.php @@ -5,26 +5,32 @@ namespace Flow\ETL\Tests\Unit\Function; use Flow\ETL\Exception\InvalidArgumentException; +use Flow\ETL\Tests\Context\FunctionContext; use Flow\ETL\Tests\FlowTestCase; use function Flow\ETL\DSL\flow_context; use function Flow\ETL\DSL\ref; -use function Flow\ETL\DSL\row; +use function Flow\ETL\DSL\schema; +use function Flow\ETL\DSL\str_schema; final class StringAfterLastTest extends FlowTestCase { public function test_string_after_last(): void { - static::assertSame('rld', ref('str') - ->stringAfterLast(ref('needle')) - ->eval(row(['str' => 'hello world', 'needle' => 'o']), flow_context())); + static::assertSame('rld', (new FunctionContext(flow_context()))->eval( + ref('str')->stringAfterLast(ref('needle')), + ['str' => 'hello world', 'needle' => 'o'], + schema(str_schema('str'), str_schema('needle')), + )); } public function test_string_after_last_including_needle(): void { - static::assertSame('orld', ref('str') - ->stringAfterLast(ref('needle'), includeNeedle: true) - ->eval(row(['str' => 'hello world', 'needle' => 'o']), flow_context())); + static::assertSame('orld', (new FunctionContext(flow_context()))->eval( + ref('str')->stringAfterLast(ref('needle'), includeNeedle: true), + ['str' => 'hello world', 'needle' => 'o'], + schema(str_schema('str'), str_schema('needle')), + )); } public function test_string_after_last_throws_on_null_input(): void @@ -32,6 +38,10 @@ public function test_string_after_last_throws_on_null_input(): void $this->expectException(InvalidArgumentException::class); $this->expectExceptionMessage('StringAfterLast function requires non-null value'); - ref('str')->stringAfterLast('x')->eval(row(['str' => null]), flow_context()); + (new FunctionContext(flow_context()))->eval( + ref('str')->stringAfterLast('x'), + ['str' => null], + schema(str_schema('str', nullable: true)), + ); } } diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/StringAfterTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/StringAfterTest.php index 83b7711ede..4477421804 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/StringAfterTest.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/StringAfterTest.php @@ -5,30 +5,38 @@ namespace Flow\ETL\Tests\Unit\Function; use Flow\ETL\Exception\InvalidArgumentException; +use Flow\ETL\Tests\Context\FunctionContext; use Flow\ETL\Tests\FlowTestCase; use function Flow\ETL\DSL\flow_context; use function Flow\ETL\DSL\ref; -use function Flow\ETL\DSL\row; +use function Flow\ETL\DSL\schema; +use function Flow\ETL\DSL\str_schema; final class StringAfterTest extends FlowTestCase { public function test_string_after(): void { - static::assertSame(' world', ref('str') - ->stringAfter(ref('needle')) - ->eval(row(['str' => 'hello world', 'needle' => 'hello']), flow_context())); - - static::assertSame(' world', ref('str') - ->stringAfter(ref('needle')) - ->eval(row(['str' => 'hello world', 'needle' => 'o']), flow_context())); + static::assertSame(' world', (new FunctionContext(flow_context()))->eval( + ref('str')->stringAfter(ref('needle')), + ['str' => 'hello world', 'needle' => 'hello'], + schema(str_schema('str'), str_schema('needle')), + )); + + static::assertSame(' world', (new FunctionContext(flow_context()))->eval( + ref('str')->stringAfter(ref('needle')), + ['str' => 'hello world', 'needle' => 'o'], + schema(str_schema('str'), str_schema('needle')), + )); } public function test_string_after_including_needle(): void { - static::assertSame('o world', ref('str') - ->stringAfter(ref('needle'), includeNeedle: true) - ->eval(row(['str' => 'hello world', 'needle' => 'o']), flow_context())); + static::assertSame('o world', (new FunctionContext(flow_context()))->eval( + ref('str')->stringAfter(ref('needle'), includeNeedle: true), + ['str' => 'hello world', 'needle' => 'o'], + schema(str_schema('str'), str_schema('needle')), + )); } public function test_string_after_throws_on_null_input(): void @@ -36,6 +44,10 @@ public function test_string_after_throws_on_null_input(): void $this->expectException(InvalidArgumentException::class); $this->expectExceptionMessage('StringAfter function requires non-null value'); - ref('str')->stringAfter('x')->eval(row(['str' => null]), flow_context()); + (new FunctionContext(flow_context()))->eval( + ref('str')->stringAfter('x'), + ['str' => null], + schema(str_schema('str', nullable: true)), + ); } } diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/StringAggregateTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/StringAggregateTest.php index 25feab7dae..4a2651092c 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/StringAggregateTest.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/StringAggregateTest.php @@ -4,12 +4,19 @@ namespace Flow\ETL\Tests\Unit\Function; +use Flow\ETL\Exception\InvalidArgumentException; use Flow\ETL\Row\SortOrder; use Flow\ETL\Tests\FlowTestCase; +use PHPUnit\Framework\Attributes\TestWith; +use function array_map; +use function Flow\ETL\DSL\array_to_rows; +use function Flow\ETL\DSL\collect; use function Flow\ETL\DSL\flow_context; +use function Flow\ETL\DSL\int_schema; use function Flow\ETL\DSL\ref; -use function Flow\ETL\DSL\row; +use function Flow\ETL\DSL\schema; +use function Flow\ETL\DSL\str_schema; use function Flow\ETL\DSL\string_agg; final class StringAggregateTest extends FlowTestCase @@ -23,10 +30,10 @@ public function test_string_agg(): void { $aggregator = string_agg(ref('data')); - $aggregator->aggregate(row(['data' => 'a']), flow_context()); - $aggregator->aggregate(row(['data' => 'b']), flow_context()); - $aggregator->aggregate(row(['data' => 'b']), flow_context()); - $aggregator->aggregate(row(['data' => 'c']), flow_context()); + $aggregator->aggregate(array_to_rows([['data' => 'a']], schema(str_schema('data'))), [0], flow_context()); + $aggregator->aggregate(array_to_rows([['data' => 'b']], schema(str_schema('data'))), [0], flow_context()); + $aggregator->aggregate(array_to_rows([['data' => 'b']], schema(str_schema('data'))), [0], flow_context()); + $aggregator->aggregate(array_to_rows([['data' => 'c']], schema(str_schema('data'))), [0], flow_context()); static::assertSame('a, b, b, c', $aggregator->value()); static::assertSame('data_str_agg', $aggregator->outputName()); @@ -43,10 +50,10 @@ public function test_string_agg_on_non_string(): void { $aggregator = string_agg(ref('data'), sort: SortOrder::DESC); - $aggregator->aggregate(row(['data' => 'a']), flow_context()); - $aggregator->aggregate(row(['data' => 1]), flow_context()); - $aggregator->aggregate(row(['data' => 'b']), flow_context()); - $aggregator->aggregate(row(['data' => 'c']), flow_context()); + $aggregator->aggregate(array_to_rows([['data' => 'a']], schema(str_schema('data'))), [0], flow_context()); + $aggregator->aggregate(array_to_rows([['data' => 1]], schema(int_schema('data'))), [0], flow_context()); + $aggregator->aggregate(array_to_rows([['data' => 'b']], schema(str_schema('data'))), [0], flow_context()); + $aggregator->aggregate(array_to_rows([['data' => 'c']], schema(str_schema('data'))), [0], flow_context()); static::assertSame('c, b, a', $aggregator->value()); } @@ -55,10 +62,10 @@ public function test_string_agg_with_alias(): void { $aggregator = string_agg(ref('data')->as('string')); - $aggregator->aggregate(row(['data' => 'a']), flow_context()); - $aggregator->aggregate(row(['data' => 'b']), flow_context()); - $aggregator->aggregate(row(['data' => 'b']), flow_context()); - $aggregator->aggregate(row(['data' => 'c']), flow_context()); + $aggregator->aggregate(array_to_rows([['data' => 'a']], schema(str_schema('data'))), [0], flow_context()); + $aggregator->aggregate(array_to_rows([['data' => 'b']], schema(str_schema('data'))), [0], flow_context()); + $aggregator->aggregate(array_to_rows([['data' => 'b']], schema(str_schema('data'))), [0], flow_context()); + $aggregator->aggregate(array_to_rows([['data' => 'c']], schema(str_schema('data'))), [0], flow_context()); static::assertSame('a, b, b, c', $aggregator->value()); static::assertSame('string', $aggregator->outputName()); @@ -68,14 +75,33 @@ public function test_string_agg_with_order(): void { $aggregator = string_agg(ref('data'), sort: SortOrder::DESC); - $aggregator->aggregate(row(['data' => 'a']), flow_context()); - $aggregator->aggregate(row(['data' => 'b']), flow_context()); - $aggregator->aggregate(row(['data' => 'b']), flow_context()); - $aggregator->aggregate(row(['data' => 'c']), flow_context()); + $aggregator->aggregate(array_to_rows([['data' => 'a']], schema(str_schema('data'))), [0], flow_context()); + $aggregator->aggregate(array_to_rows([['data' => 'b']], schema(str_schema('data'))), [0], flow_context()); + $aggregator->aggregate(array_to_rows([['data' => 'b']], schema(str_schema('data'))), [0], flow_context()); + $aggregator->aggregate(array_to_rows([['data' => 'c']], schema(str_schema('data'))), [0], flow_context()); static::assertSame('c, b, b, a', $aggregator->value()); } + #[TestWith([SortOrder::ASC, '10,100,9,B'])] + #[TestWith([SortOrder::DESC, 'B,9,100,10'])] + public function test_string_agg_orders_numeric_looking_strings_by_bytes(SortOrder $order, string $expected): void + { + $aggregator = string_agg(ref('data'), ',', $order); + $aggregator->aggregate( + array_to_rows([ + ['data' => '9'], + ['data' => '10'], + ['data' => '100'], + ['data' => 'B'], + ], schema(str_schema('data'))), + [0, 1, 2, 3], + flow_context(), + ); + + static::assertSame($expected, $aggregator->value()); + } + public function test_with_children_rebuilds_the_aggregate_with_the_given_reference(): void { $aggregate = string_agg(ref('a')); @@ -87,4 +113,48 @@ public function test_with_children_rebuilds_the_aggregate_with_the_given_referen static::assertNotSame($aggregate, $rebuilt); static::assertEquals([ref('b')], $rebuilt->references()); } + + public function test_aggregate_reads_only_given_indices(): void + { + $aggregator = string_agg(ref('v')); + $aggregator->aggregate( + array_to_rows(array_map(static fn(mixed $v): array => ['v' => $v], [ + 'a', + 'b', + 'c', + 'd', + ]), schema(str_schema('v'))), + [1, 3], + flow_context(), + ); + + static::assertSame('b, d', $aggregator->value()); + } + + public function test_merge_of_two_halves_equals_one_pass(): void + { + $rows = array_to_rows(array_map(static fn(mixed $v): array => ['v' => $v], [ + 'a', + 'b', + 'c', + 'd', + ]), schema(str_schema('v'))); + $onePass = string_agg(ref('v')); + $onePass->aggregate($rows, [0, 1, 2, 3], flow_context()); + $left = string_agg(ref('v')); + $left->aggregate($rows, [0, 1], flow_context()); + $right = string_agg(ref('v')); + $right->aggregate($rows, [2, 3], flow_context()); + + $left->merge($right, flow_context()); + + static::assertSame($onePass->value(), $left->value()); + } + + public function test_merge_refuses_another_class(): void + { + $this->expectException(InvalidArgumentException::class); + + string_agg(ref('v'))->merge(collect(ref('v')), flow_context()); + } } diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/StringBeforeLastTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/StringBeforeLastTest.php index 2a815400fb..acc98e2273 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/StringBeforeLastTest.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/StringBeforeLastTest.php @@ -5,33 +5,41 @@ namespace Flow\ETL\Tests\Unit\Function; use Flow\ETL\Exception\InvalidArgumentException; +use Flow\ETL\Tests\Context\FunctionContext; use Flow\ETL\Tests\FlowTestCase; use function Flow\ETL\DSL\flow_context; use function Flow\ETL\DSL\ref; -use function Flow\ETL\DSL\row; +use function Flow\ETL\DSL\schema; +use function Flow\ETL\DSL\str_schema; final class StringBeforeLastTest extends FlowTestCase { public function test_string_before_last(): void { - static::assertSame('hello w', ref('str') - ->stringBeforeLast(ref('needle')) - ->eval(row(['str' => 'hello world', 'needle' => 'o']), flow_context())); + static::assertSame('hello w', (new FunctionContext(flow_context()))->eval( + ref('str')->stringBeforeLast(ref('needle')), + ['str' => 'hello world', 'needle' => 'o'], + schema(str_schema('str'), str_schema('needle')), + )); } public function test_string_before_last_including_needle(): void { - static::assertSame('hello wo', ref('str') - ->stringBeforeLast(ref('needle'), includeNeedle: true) - ->eval(row(['str' => 'hello world', 'needle' => 'o']), flow_context())); + static::assertSame('hello wo', (new FunctionContext(flow_context()))->eval( + ref('str')->stringBeforeLast(ref('needle'), includeNeedle: true), + ['str' => 'hello world', 'needle' => 'o'], + schema(str_schema('str'), str_schema('needle')), + )); } public function test_string_before_last_returns_empty_string(): void { - static::assertSame('', ref('str') - ->stringBeforeLast(ref('needle')) - ->eval(row(['str' => '', 'needle' => 'o']), flow_context())); + static::assertSame('', (new FunctionContext(flow_context()))->eval( + ref('str')->stringBeforeLast(ref('needle')), + ['str' => '', 'needle' => 'o'], + schema(str_schema('str'), str_schema('needle')), + )); } public function test_string_before_last_throws_on_null_input(): void @@ -39,6 +47,10 @@ public function test_string_before_last_throws_on_null_input(): void $this->expectException(InvalidArgumentException::class); $this->expectExceptionMessage('StringBeforeLast function requires non-null value'); - ref('str')->stringBeforeLast(ref('needle'))->eval(row(['str' => null, 'needle' => 'o']), flow_context()); + (new FunctionContext(flow_context()))->eval( + ref('str')->stringBeforeLast(ref('needle')), + ['str' => null, 'needle' => 'o'], + schema(str_schema('str', nullable: true), str_schema('needle')), + ); } } diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/StringBeforeTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/StringBeforeTest.php index 8f6a31ef23..5c268ce87a 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/StringBeforeTest.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/StringBeforeTest.php @@ -5,37 +5,47 @@ namespace Flow\ETL\Tests\Unit\Function; use Flow\ETL\Exception\InvalidArgumentException; +use Flow\ETL\Tests\Context\FunctionContext; use Flow\ETL\Tests\FlowTestCase; use function Flow\ETL\DSL\flow_context; use function Flow\ETL\DSL\ref; -use function Flow\ETL\DSL\row; +use function Flow\ETL\DSL\schema; +use function Flow\ETL\DSL\str_schema; final class StringBeforeTest extends FlowTestCase { public function test_string_before(): void { - static::assertSame('hello ', ref('str') - ->stringBefore(ref('needle')) - ->eval(row(['str' => 'hello world', 'needle' => 'world']), flow_context())); - - static::assertSame('hell', ref('str') - ->stringBefore(ref('needle')) - ->eval(row(['str' => 'hello world', 'needle' => 'o']), flow_context())); + static::assertSame('hello ', (new FunctionContext(flow_context()))->eval( + ref('str')->stringBefore(ref('needle')), + ['str' => 'hello world', 'needle' => 'world'], + schema(str_schema('str'), str_schema('needle')), + )); + + static::assertSame('hell', (new FunctionContext(flow_context()))->eval( + ref('str')->stringBefore(ref('needle')), + ['str' => 'hello world', 'needle' => 'o'], + schema(str_schema('str'), str_schema('needle')), + )); } public function test_string_before_including_needle(): void { - static::assertSame('hello', ref('str') - ->stringBefore(ref('needle'), includeNeedle: true) - ->eval(row(['str' => 'hello world', 'needle' => 'o']), flow_context())); + static::assertSame('hello', (new FunctionContext(flow_context()))->eval( + ref('str')->stringBefore(ref('needle'), includeNeedle: true), + ['str' => 'hello world', 'needle' => 'o'], + schema(str_schema('str'), str_schema('needle')), + )); } public function test_string_before_returns_empty_string(): void { - static::assertSame('', ref('str') - ->stringBefore(ref('needle')) - ->eval(row(['str' => '', 'needle' => 'o']), flow_context())); + static::assertSame('', (new FunctionContext(flow_context()))->eval( + ref('str')->stringBefore(ref('needle')), + ['str' => '', 'needle' => 'o'], + schema(str_schema('str'), str_schema('needle')), + )); } public function test_string_before_throws_on_null_input(): void @@ -43,6 +53,10 @@ public function test_string_before_throws_on_null_input(): void $this->expectException(InvalidArgumentException::class); $this->expectExceptionMessage('StringBefore function requires non-null value'); - ref('str')->stringBefore(ref('needle'))->eval(row(['str' => null, 'needle' => 'o']), flow_context()); + (new FunctionContext(flow_context()))->eval( + ref('str')->stringBefore(ref('needle')), + ['str' => null, 'needle' => 'o'], + schema(str_schema('str', nullable: true), str_schema('needle')), + ); } } diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/StringContainsAnyTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/StringContainsAnyTest.php index 22eff5b9aa..afd6d9f1cc 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/StringContainsAnyTest.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/StringContainsAnyTest.php @@ -5,12 +5,17 @@ namespace Flow\ETL\Tests\Unit\Function; use Flow\ETL\Exception\InvalidArgumentException; +use Flow\ETL\Tests\Context\FunctionContext; use Flow\ETL\Tests\FlowTestCase; use function Flow\ETL\DSL\config; use function Flow\ETL\DSL\flow_context; +use function Flow\ETL\DSL\list_schema; use function Flow\ETL\DSL\ref; -use function Flow\ETL\DSL\row; +use function Flow\ETL\DSL\schema; +use function Flow\ETL\DSL\str_schema; +use function Flow\Types\DSL\type_list; +use function Flow\Types\DSL\type_string; final class StringContainsAnyTest extends FlowTestCase { @@ -19,51 +24,65 @@ public function test_contains_any_empty_needles_array(): void $this->expectException(InvalidArgumentException::class); $this->expectExceptionMessage('StringContainsAny function requires a non-empty needles array'); - ref('str')->stringContainsAny([])->eval(row(['str' => 'hello world']), flow_context()); + (new FunctionContext(flow_context()))->eval( + ref('str')->stringContainsAny([]), + ['str' => 'hello world'], + schema(str_schema('str')), + ); } public function test_contains_any_multiple_needles_one_found(): void { - static::assertTrue( - ref('str')->stringContainsAny(['foo', 'world', 'bar'])->eval(row(['str' => 'hello world']), flow_context()), - ); + static::assertTrue((new FunctionContext(flow_context()))->eval( + ref('str')->stringContainsAny(['foo', 'world', 'bar']), + ['str' => 'hello world'], + schema(str_schema('str')), + )); } public function test_contains_any_no_needles_found(): void { - static::assertFalse( - ref('str')->stringContainsAny(['foo', 'bar', 'baz'])->eval(row(['str' => 'hello world']), flow_context()), - ); + static::assertFalse((new FunctionContext(flow_context()))->eval( + ref('str')->stringContainsAny(['foo', 'bar', 'baz']), + ['str' => 'hello world'], + schema(str_schema('str')), + )); } public function test_contains_any_null_needles(): void { - static::assertNull( - ref('str') - ->stringContainsAny(ref('needles')) - ->eval(row(['str' => 'hello world', 'needles' => null]), flow_context()), - ); + static::assertNull((new FunctionContext(flow_context()))->eval( + ref('str')->stringContainsAny(ref('needles')), + ['str' => 'hello world', 'needles' => null], + schema(str_schema('str'), str_schema('needles', nullable: true)), + )); } public function test_contains_any_null_string(): void { - static::assertNull( - ref('str')->stringContainsAny(['hello', 'world'])->eval(row(['str' => null]), flow_context()), - ); + static::assertNull((new FunctionContext(flow_context()))->eval( + ref('str')->stringContainsAny(['hello', 'world']), + ['str' => null], + schema(str_schema('str', nullable: true)), + )); } public function test_contains_any_null_string_in_strict_mode(): void { $context = flow_context(config()); - static::assertNull(ref('str')->stringContainsAny(['hello', 'world'])->eval(row(['str' => null]), $context)); + static::assertNull((new FunctionContext($context))->eval( + ref('str')->stringContainsAny(['hello', 'world']), + ['str' => null], + schema(str_schema('str', nullable: true)), + )); } public function test_contains_any_with_scalar_function_parameter(): void { - static::assertTrue( - ref('str') - ->stringContainsAny(ref('needles')) - ->eval(row(['str' => 'hello world', 'needles' => ['world', 'foo']]), flow_context()), - ); + static::assertTrue((new FunctionContext(flow_context()))->eval( + ref('str')->stringContainsAny(ref('needles')), + ['str' => 'hello world', 'needles' => ['world', 'foo']], + schema(str_schema('str'), list_schema('needles', type_list(type_string()))), + )); } } diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/StringEqualsToTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/StringEqualsToTest.php index 55cb9d7bfb..c24dc19ecf 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/StringEqualsToTest.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/StringEqualsToTest.php @@ -4,49 +4,67 @@ namespace Flow\ETL\Tests\Unit\Function; +use Flow\ETL\Tests\Context\FunctionContext; use Flow\ETL\Tests\FlowTestCase; use function Flow\ETL\DSL\flow_context; use function Flow\ETL\DSL\ref; -use function Flow\ETL\DSL\row; +use function Flow\ETL\DSL\schema; +use function Flow\ETL\DSL\str_schema; final class StringEqualsToTest extends FlowTestCase { public function test_equals_to_empty_strings(): void { - static::assertTrue(ref('str')->stringEqualsTo('')->eval(row(['str' => '']), flow_context())); + static::assertTrue((new FunctionContext(flow_context()))->eval( + ref('str')->stringEqualsTo(''), + ['str' => ''], + schema(str_schema('str')), + )); } public function test_equals_to_exact_match(): void { - static::assertTrue(ref('str')->stringEqualsTo('hello')->eval(row(['str' => 'hello']), flow_context())); + static::assertTrue((new FunctionContext(flow_context()))->eval( + ref('str')->stringEqualsTo('hello'), + ['str' => 'hello'], + schema(str_schema('str')), + )); } public function test_equals_to_no_match(): void { - static::assertFalse(ref('str')->stringEqualsTo('world')->eval(row(['str' => 'hello']), flow_context())); + static::assertFalse((new FunctionContext(flow_context()))->eval( + ref('str')->stringEqualsTo('world'), + ['str' => 'hello'], + schema(str_schema('str')), + )); } public function test_equals_to_null_comparison_string_returns_null(): void { - static::assertNull( - ref('str') - ->stringEqualsTo(ref('compare')) - ->eval(row(['str' => 'hello', 'compare' => null]), flow_context()), - ); + static::assertNull((new FunctionContext(flow_context()))->eval( + ref('str')->stringEqualsTo(ref('compare')), + ['str' => 'hello', 'compare' => null], + schema(str_schema('str'), str_schema('compare', nullable: true)), + )); } public function test_equals_to_null_string_returns_null(): void { - static::assertNull(ref('str')->stringEqualsTo('hello')->eval(row(['str' => null]), flow_context())); + static::assertNull((new FunctionContext(flow_context()))->eval( + ref('str')->stringEqualsTo('hello'), + ['str' => null], + schema(str_schema('str', nullable: true)), + )); } public function test_equals_to_with_scalar_function_parameter(): void { - static::assertTrue( - ref('str') - ->stringEqualsTo(ref('compare')) - ->eval(row(['str' => 'hello', 'compare' => 'hello']), flow_context()), - ); + static::assertTrue((new FunctionContext(flow_context()))->eval( + ref('str')->stringEqualsTo(ref('compare')), + ['str' => 'hello', 'compare' => 'hello'], + schema(str_schema('str'), str_schema('compare')), + )); } } diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/StringFoldTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/StringFoldTest.php index f90d574f09..a2f3ab0fee 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/StringFoldTest.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/StringFoldTest.php @@ -5,19 +5,23 @@ namespace Flow\ETL\Tests\Unit\Function; use Flow\ETL\Exception\InvalidArgumentException; +use Flow\ETL\Tests\Context\FunctionContext; use Flow\ETL\Tests\FlowTestCase; use function Flow\ETL\DSL\flow_context; use function Flow\ETL\DSL\ref; -use function Flow\ETL\DSL\row; +use function Flow\ETL\DSL\schema; +use function Flow\ETL\DSL\str_schema; final class StringFoldTest extends FlowTestCase { public function test_string_folded(): void { - static::assertSame("die o'brian strasse", ref('str') - ->stringFold() - ->eval(row(['str' => "Die O'Brian Straße"]), flow_context())); + static::assertSame("die o'brian strasse", (new FunctionContext(flow_context()))->eval( + ref('str')->stringFold(), + ['str' => "Die O'Brian Straße"], + schema(str_schema('str')), + )); } public function test_string_fold_throws_on_null_input(): void @@ -25,6 +29,10 @@ public function test_string_fold_throws_on_null_input(): void $this->expectException(InvalidArgumentException::class); $this->expectExceptionMessage('StringFold function requires non-null value'); - ref('str')->stringFold()->eval(row(['str' => null]), flow_context()); + (new FunctionContext(flow_context()))->eval( + ref('str')->stringFold(), + ['str' => null], + schema(str_schema('str', nullable: true)), + ); } } diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/StringMatchAllTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/StringMatchAllTest.php index 6f37097796..b2f5fc097f 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/StringMatchAllTest.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/StringMatchAllTest.php @@ -5,33 +5,50 @@ namespace Flow\ETL\Tests\Unit\Function; use Flow\ETL\Exception\InvalidArgumentException; +use Flow\ETL\Tests\Context\FunctionContext; use Flow\ETL\Tests\FlowTestCase; use function Flow\ETL\DSL\flow_context; use function Flow\ETL\DSL\ref; -use function Flow\ETL\DSL\row; +use function Flow\ETL\DSL\schema; +use function Flow\ETL\DSL\str_schema; final class StringMatchAllTest extends FlowTestCase { public function test_empty_haystack_string(): void { - $result = ref('str')->stringMatchAll('/hello/')->eval(row(['str' => '']), flow_context()); - - static::assertEquals([], $result); + static::assertEquals( + [], + (new FunctionContext(flow_context()))->eval( + ref('str')->stringMatchAll('/hello/'), + ['str' => ''], + schema(str_schema('str')), + ), + ); } public function test_multiple_successful_pattern_matches(): void { - $result = ref('str')->stringMatchAll('/\d+/')->eval(row(['str' => 'test 123 and 456 and 789']), flow_context()); - - static::assertEquals([['123'], ['456'], ['789']], $result); + static::assertEquals( + [['123'], ['456'], ['789']], + (new FunctionContext(flow_context()))->eval( + ref('str')->stringMatchAll('/\d+/'), + ['str' => 'test 123 and 456 and 789'], + schema(str_schema('str')), + ), + ); } public function test_no_matches_found(): void { - $result = ref('str')->stringMatchAll('/foo/')->eval(row(['str' => 'hello world']), flow_context()); - - static::assertEquals([], $result); + static::assertEquals( + [], + (new FunctionContext(flow_context()))->eval( + ref('str')->stringMatchAll('/foo/'), + ['str' => 'hello world'], + schema(str_schema('str')), + ), + ); } public function test_null_haystack(): void @@ -39,7 +56,11 @@ public function test_null_haystack(): void $this->expectException(InvalidArgumentException::class); $this->expectExceptionMessage('StringMatchAll function requires non-null haystack'); - ref('str')->stringMatchAll('/hello/')->eval(row(['str' => null]), flow_context()); + (new FunctionContext(flow_context()))->eval( + ref('str')->stringMatchAll('/hello/'), + ['str' => null], + schema(str_schema('str', nullable: true)), + ); } public function test_null_pattern(): void @@ -47,17 +68,22 @@ public function test_null_pattern(): void $this->expectException(InvalidArgumentException::class); $this->expectExceptionMessage('StringMatchAll function requires non-null pattern'); - ref('str') - ->stringMatchAll(ref('pattern')) - ->eval(row(['str' => 'hello world', 'pattern' => null]), flow_context()); + (new FunctionContext(flow_context()))->eval( + ref('str')->stringMatchAll(ref('pattern')), + ['str' => 'hello world', 'pattern' => null], + schema(str_schema('str'), str_schema('pattern', nullable: true)), + ); } public function test_with_scalar_function_parameter(): void { - $result = ref('str') - ->stringMatchAll(ref('pattern')) - ->eval(row(['str' => 'test 123 and 456', 'pattern' => '/\d+/']), flow_context()); - - static::assertEquals([['123'], ['456']], $result); + static::assertEquals( + [['123'], ['456']], + (new FunctionContext(flow_context()))->eval( + ref('str')->stringMatchAll(ref('pattern')), + ['str' => 'test 123 and 456', 'pattern' => '/\d+/'], + schema(str_schema('str'), str_schema('pattern')), + ), + ); } } diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/StringMatchTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/StringMatchTest.php index 5fdfef9484..02a7192367 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/StringMatchTest.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/StringMatchTest.php @@ -4,20 +4,26 @@ namespace Flow\ETL\Tests\Unit\Function; +use Flow\ETL\Exception\EvaluationException; use Flow\ETL\Exception\InvalidArgumentException; +use Flow\ETL\Tests\Context\FunctionContext; use Flow\ETL\Tests\FlowTestCase; use function Flow\ETL\DSL\flow_context; +use function Flow\ETL\DSL\lit; use function Flow\ETL\DSL\ref; -use function Flow\ETL\DSL\row; +use function Flow\ETL\DSL\schema; +use function Flow\ETL\DSL\str_schema; final class StringMatchTest extends FlowTestCase { public function test_no_matches_found(): void { - $result = ref('str')->stringMatch('/foo/')->eval(row(['str' => 'hello world']), flow_context()); - - static::assertNull($result); + static::assertNull((new FunctionContext(flow_context()))->eval( + ref('str')->stringMatch('/foo/'), + ['str' => 'hello world'], + schema(str_schema('str')), + )); } public function test_null_haystack(): void @@ -25,9 +31,11 @@ public function test_null_haystack(): void $this->expectException(InvalidArgumentException::class); $this->expectExceptionMessage('StringMatch function requires non-null haystack'); - $result = ref('str')->stringMatch('/hello/')->eval(row(['str' => null]), flow_context()); - - static::assertNull($result); + static::assertNull((new FunctionContext(flow_context()))->eval( + ref('str')->stringMatch('/hello/'), + ['str' => null], + schema(str_schema('str', nullable: true)), + )); } public function test_null_pattern(): void @@ -35,26 +43,49 @@ public function test_null_pattern(): void $this->expectException(InvalidArgumentException::class); $this->expectExceptionMessage('StringMatch function requires non-null pattern'); - $result = ref('str') - ->stringMatch(ref('pattern')) - ->eval(row(['str' => 'hello world', 'pattern' => null]), flow_context()); + static::assertNull((new FunctionContext(flow_context()))->eval( + ref('str')->stringMatch(ref('pattern')), + ['str' => 'hello world', 'pattern' => null], + schema(str_schema('str'), str_schema('pattern', nullable: true)), + )); + } - static::assertNull($result); + public function test_an_invalid_pattern_keeps_the_regex_error_as_the_cause(): void + { + try { + (new FunctionContext(flow_context()))->eval( + ref('value')->stringMatch(lit('/(/')), + ['value' => 'abc'], + schema(str_schema('value')), + ); + static::fail('An invalid pattern must be refused.'); + } catch (EvaluationException $e) { + static::assertStringStartsWith('StringMatch error: ', (string) $e->getPrevious()?->getMessage()); + static::assertNotNull($e->getPrevious()?->getPrevious()); + } } public function test_successful_pattern_match(): void { - $result = ref('str')->stringMatch('/hello/')->eval(row(['str' => 'hello world']), flow_context()); - - static::assertEquals(['hello'], $result); + static::assertEquals( + ['hello'], + (new FunctionContext(flow_context()))->eval( + ref('str')->stringMatch('/hello/'), + ['str' => 'hello world'], + schema(str_schema('str')), + ), + ); } public function test_with_scalar_function_parameter(): void { - $result = ref('str') - ->stringMatch(ref('pattern')) - ->eval(row(['str' => 'hello world', 'pattern' => '/world/']), flow_context()); - - static::assertEquals(['world'], $result); + static::assertEquals( + ['world'], + (new FunctionContext(flow_context()))->eval( + ref('str')->stringMatch(ref('pattern')), + ['str' => 'hello world', 'pattern' => '/world/'], + schema(str_schema('str'), str_schema('pattern')), + ), + ); } } diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/StringNormalizeTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/StringNormalizeTest.php index 7164b7fbab..33ffd192f3 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/StringNormalizeTest.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/StringNormalizeTest.php @@ -5,13 +5,16 @@ namespace Flow\ETL\Tests\Unit\Function; use Flow\ETL\Exception\InvalidArgumentException; +use Flow\ETL\Tests\Context\FunctionContext; use Flow\ETL\Tests\FlowTestCase; use Normalizer; use function Flow\ETL\DSL\flow_context; +use function Flow\ETL\DSL\int_schema; use function Flow\ETL\DSL\lit; use function Flow\ETL\DSL\ref; -use function Flow\ETL\DSL\row; +use function Flow\ETL\DSL\schema; +use function Flow\ETL\DSL\str_schema; final class StringNormalizeTest extends FlowTestCase { @@ -19,38 +22,56 @@ public function test_a_malformed_form_operand_throws(): void { $this->expectException(InvalidArgumentException::class); - ref('value')->stringNormalize(lit('not-a-form'))->eval(row(['value' => 'abc']), flow_context()); + (new FunctionContext(flow_context()))->eval( + ref('value')->stringNormalize(lit('not-a-form')), + ['value' => 'abc'], + schema(str_schema('value')), + ); } public function test_normalize_already_normalized(): void { - static::assertSame('hello', ref('str') - ->stringNormalize(Normalizer::NFC) - ->eval(row(['str' => 'hello']), flow_context())); + static::assertSame('hello', (new FunctionContext(flow_context()))->eval( + ref('str')->stringNormalize(Normalizer::NFC), + ['str' => 'hello'], + schema(str_schema('str')), + )); } public function test_normalize_empty_string(): void { - static::assertSame('', ref('str')->stringNormalize()->eval(row(['str' => '']), flow_context())); + static::assertSame('', (new FunctionContext(flow_context()))->eval( + ref('str')->stringNormalize(), + ['str' => ''], + schema(str_schema('str')), + )); } public function test_normalize_nfc_default(): void { - static::assertSame('é', ref('str')->stringNormalize()->eval(row(['str' => "e\u{0301}"]), flow_context())); + static::assertSame('é', (new FunctionContext(flow_context()))->eval( + ref('str')->stringNormalize(), + ['str' => "e\u{0301}"], + schema(str_schema('str')), + )); } public function test_normalize_nfc_explicit(): void { - static::assertSame('é', ref('str') - ->stringNormalize(Normalizer::NFC) - ->eval(row(['str' => "e\u{0301}"]), flow_context())); + static::assertSame('é', (new FunctionContext(flow_context()))->eval( + ref('str')->stringNormalize(Normalizer::NFC), + ['str' => "e\u{0301}"], + schema(str_schema('str')), + )); } public function test_normalize_nfd(): void { - static::assertSame("e\u{0301}", ref('str') - ->stringNormalize(Normalizer::NFD) - ->eval(row(['str' => 'é']), flow_context())); + static::assertSame("e\u{0301}", (new FunctionContext(flow_context()))->eval( + ref('str')->stringNormalize(Normalizer::NFD), + ['str' => 'é'], + schema(str_schema('str')), + )); } public function test_normalize_throws_on_null_input(): void @@ -58,15 +79,19 @@ public function test_normalize_throws_on_null_input(): void $this->expectException(InvalidArgumentException::class); $this->expectExceptionMessage('StringNormalize function requires non-null value'); - ref('str')->stringNormalize()->eval(row(['str' => null]), flow_context()); + (new FunctionContext(flow_context()))->eval( + ref('str')->stringNormalize(), + ['str' => null], + schema(str_schema('str', nullable: true)), + ); } public function test_normalize_with_scalar_function_form(): void { - $normalized = ref('str') - ->stringNormalize(ref('form')) - ->eval(row(['str' => "e\u{0301}", 'form' => Normalizer::NFC]), flow_context()); - - static::assertSame('é', $normalized); + static::assertSame('é', (new FunctionContext(flow_context()))->eval( + ref('str')->stringNormalize(ref('form')), + ['str' => "e\u{0301}", 'form' => Normalizer::NFC], + schema(str_schema('str'), int_schema('form')), + )); } } diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/StringStyleTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/StringStyleTest.php index 6c5420a708..0359a1f841 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/StringStyleTest.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/StringStyleTest.php @@ -6,12 +6,14 @@ use Flow\ETL\Exception\InvalidArgumentException; use Flow\ETL\String\StringStyles; +use Flow\ETL\Tests\Context\FunctionContext; use Flow\ETL\Tests\FlowTestCase; use PHPUnit\Framework\Attributes\DataProvider; use function Flow\ETL\DSL\flow_context; use function Flow\ETL\DSL\ref; -use function Flow\ETL\DSL\row; +use function Flow\ETL\DSL\schema; +use function Flow\ETL\DSL\str_schema; final class StringStyleTest extends FlowTestCase { @@ -20,7 +22,11 @@ public function test_a_null_value_throws(): void $this->expectException(InvalidArgumentException::class); $this->expectExceptionMessage('StringStyle function requires non-null value'); - ref('value')->stringStyle(StringStyles::LOWER)->eval(row(['value' => null]), flow_context()); + (new FunctionContext(flow_context()))->eval( + ref('value')->stringStyle(StringStyles::LOWER), + ['value' => null], + schema(str_schema('value', nullable: true)), + ); } /** @@ -55,28 +61,38 @@ public static function provideStringStyles(): iterable public function test_string_style_camel(): void { - static::assertSame('fooBarBaz', ref('str') - ->stringStyle(ref('style')) - ->eval(row(['str' => 'Foo: Bar-baz.', 'style' => 'camel']), flow_context())); + static::assertSame('fooBarBaz', (new FunctionContext(flow_context()))->eval( + ref('str')->stringStyle(ref('style')), + ['str' => 'Foo: Bar-baz.', 'style' => 'camel'], + schema(str_schema('str'), str_schema('style')), + )); } public function test_string_style_kebab(): void { - static::assertSame('foo-bar-baz', ref('str') - ->stringStyle('kebab') - ->eval(row(['str' => 'Foo: Bar-baz.']), flow_context())); + static::assertSame('foo-bar-baz', (new FunctionContext(flow_context()))->eval( + ref('str')->stringStyle('kebab'), + ['str' => 'Foo: Bar-baz.'], + schema(str_schema('str')), + )); } public function test_string_style_lower(): void { - static::assertSame('foo bar bri̇an', ref('str') - ->stringStyle('lower') - ->eval(row(['str' => 'FOO Bar Brİan']), flow_context())); + static::assertSame('foo bar bri̇an', (new FunctionContext(flow_context()))->eval( + ref('str')->stringStyle('lower'), + ['str' => 'FOO Bar Brİan'], + schema(str_schema('str')), + )); } #[DataProvider('provideStringStyles')] public function test_string_styles(StringStyles $style, ?string $value, ?string $expected): void { - static::assertSame($expected, ref('str')->stringStyle($style)->eval(row(['str' => $value]), flow_context())); + static::assertSame($expected, (new FunctionContext(flow_context()))->eval( + ref('str')->stringStyle($style), + ['str' => $value], + schema(str_schema('str', nullable: true)), + )); } } diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/StringTitleTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/StringTitleTest.php index ae59f88bba..7d722a5fab 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/StringTitleTest.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/StringTitleTest.php @@ -5,24 +5,32 @@ namespace Flow\ETL\Tests\Unit\Function; use Flow\ETL\Exception\InvalidArgumentException; +use Flow\ETL\Tests\Context\FunctionContext; use Flow\ETL\Tests\FlowTestCase; use function Flow\ETL\DSL\flow_context; use function Flow\ETL\DSL\ref; -use function Flow\ETL\DSL\row; +use function Flow\ETL\DSL\schema; +use function Flow\ETL\DSL\str_schema; final class StringTitleTest extends FlowTestCase { public function test_string_title(): void { - static::assertSame('Foo ijssel', ref('str')->stringTitle()->eval(row(['str' => 'foo ijssel']), flow_context())); + static::assertSame('Foo ijssel', (new FunctionContext(flow_context()))->eval( + ref('str')->stringTitle(), + ['str' => 'foo ijssel'], + schema(str_schema('str')), + )); } public function test_string_title_allwords(): void { - static::assertSame('Foo Ijssel', ref('str') - ->stringTitle(allWords: true) - ->eval(row(['str' => 'foo ijssel']), flow_context())); + static::assertSame('Foo Ijssel', (new FunctionContext(flow_context()))->eval( + ref('str')->stringTitle(allWords: true), + ['str' => 'foo ijssel'], + schema(str_schema('str')), + )); } public function test_string_title_throws_on_null_input(): void @@ -30,6 +38,10 @@ public function test_string_title_throws_on_null_input(): void $this->expectException(InvalidArgumentException::class); $this->expectExceptionMessage('StringTitle function requires non-null value'); - ref('str')->stringTitle()->eval(row(['str' => null]), flow_context()); + (new FunctionContext(flow_context()))->eval( + ref('str')->stringTitle(), + ['str' => null], + schema(str_schema('str', nullable: true)), + ); } } diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/StringWidthTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/StringWidthTest.php index bd3d4dedf3..3c1f8ebb3e 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/StringWidthTest.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/StringWidthTest.php @@ -5,22 +5,32 @@ namespace Flow\ETL\Tests\Unit\Function; use Flow\ETL\Exception\InvalidArgumentException; +use Flow\ETL\Tests\Context\FunctionContext; use Flow\ETL\Tests\FlowTestCase; use function Flow\ETL\DSL\flow_context; use function Flow\ETL\DSL\ref; -use function Flow\ETL\DSL\row; +use function Flow\ETL\DSL\schema; +use function Flow\ETL\DSL\str_schema; final class StringWidthTest extends FlowTestCase { public function test_width_ascii_string(): void { - static::assertSame(5, ref('str')->stringWidth()->eval(row(['str' => 'hello']), flow_context())); + static::assertSame(5, (new FunctionContext(flow_context()))->eval( + ref('str')->stringWidth(), + ['str' => 'hello'], + schema(str_schema('str')), + )); } public function test_width_empty_string(): void { - static::assertSame(0, ref('str')->stringWidth()->eval(row(['str' => '']), flow_context())); + static::assertSame(0, (new FunctionContext(flow_context()))->eval( + ref('str')->stringWidth(), + ['str' => ''], + schema(str_schema('str')), + )); } public function test_width_throws_on_null_input(): void @@ -28,11 +38,19 @@ public function test_width_throws_on_null_input(): void $this->expectException(InvalidArgumentException::class); $this->expectExceptionMessage('StringWidth function requires non-null value'); - ref('str')->stringWidth()->eval(row(['str' => null]), flow_context()); + (new FunctionContext(flow_context()))->eval( + ref('str')->stringWidth(), + ['str' => null], + schema(str_schema('str', nullable: true)), + ); } public function test_width_single_character(): void { - static::assertSame(1, ref('str')->stringWidth()->eval(row(['str' => 'a']), flow_context())); + static::assertSame(1, (new FunctionContext(flow_context()))->eval( + ref('str')->stringWidth(), + ['str' => 'a'], + schema(str_schema('str')), + )); } } diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/StructureSelectTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/StructureSelectTest.php index 1d39ea81f9..ebddc163c0 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/StructureSelectTest.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/StructureSelectTest.php @@ -8,15 +8,17 @@ use Flow\ETL\Function\ReferenceResolver; use Flow\ETL\Function\ScalarFunction; use Flow\ETL\Function\StructureSelect; +use Flow\ETL\Tests\Context\FunctionContext; use Flow\ETL\Tests\FlowTestCase; use function Flow\ETL\DSL\flow_context; use function Flow\ETL\DSL\list_schema; use function Flow\ETL\DSL\ref; -use function Flow\ETL\DSL\row; use function Flow\ETL\DSL\schema; +use function Flow\ETL\DSL\structure_schema; use function Flow\Types\DSL\type_integer; use function Flow\Types\DSL\type_list; +use function Flow\Types\DSL\type_optional; use function Flow\Types\DSL\type_string; use function Flow\Types\DSL\type_structure; @@ -26,10 +28,14 @@ public function test_selecting_multiple_values_from_structure(): void { static::assertEquals( ['id' => 1, 'name' => 'test'], - (new StructureSelect(ref('struct'), ref('id'), ref('name')))->eval(row(['struct' => [ - 'id' => 1, - 'name' => 'test', - ]]), flow_context()), + (new FunctionContext(flow_context()))->eval( + new StructureSelect(ref('struct'), ref('id'), ref('name')), + ['struct' => [ + 'id' => 1, + 'name' => 'test', + ]], + schema(structure_schema('struct', type_structure(['id' => type_integer(), 'name' => type_string()]))), + ), ); } @@ -37,10 +43,14 @@ public function test_selecting_single_value_from_structure(): void { static::assertEquals( ['id' => 1], - (new StructureSelect(ref('struct'), 'id'))->eval(row(['struct' => [ - 'id' => 1, - 'name' => 'test', - ]]), flow_context()), + (new FunctionContext(flow_context()))->eval( + new StructureSelect(ref('struct'), 'id'), + ['struct' => [ + 'id' => 1, + 'name' => 'test', + ]], + schema(structure_schema('struct', type_structure(['id' => type_integer(), 'name' => type_string()]))), + ), ); } @@ -48,10 +58,14 @@ public function test_selecting_single_value_from_structure_with_alias(): void { static::assertEquals( ['new_id' => 1], - (new StructureSelect(ref('struct'), ref('id')->as('new_id')))->eval(row(['struct' => [ - 'id' => 1, - 'name' => 'test', - ]]), flow_context()), + (new FunctionContext(flow_context()))->eval( + new StructureSelect(ref('struct'), ref('id')->as('new_id')), + ['struct' => [ + 'id' => 1, + 'name' => 'test', + ]], + schema(structure_schema('struct', type_structure(['id' => type_integer(), 'name' => type_string()]))), + ), ); } @@ -59,11 +73,19 @@ public function test_selecting_values_from_empty_structure(): void { static::assertEquals( ['new_id' => null], - (new StructureSelect(ref('struct'), ref('id')->as('new_id')))->eval(row(['struct' => [ - 'id' => null, - 'email' => 'email@email.com', - 'name' => null, - ]]), flow_context()), + (new FunctionContext(flow_context()))->eval( + new StructureSelect(ref('struct'), ref('id')->as('new_id')), + ['struct' => [ + 'id' => null, + 'email' => 'email@email.com', + 'name' => null, + ]], + schema(structure_schema('struct', type_structure([ + 'id' => type_optional(type_integer()), + 'email' => type_string(), + 'name' => type_optional(type_string()), + ]))), + ), ); } diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/StructureTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/StructureTest.php index 1bd6fc9edd..4f7eb866d3 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/StructureTest.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/StructureTest.php @@ -7,13 +7,13 @@ use Flow\ETL\Exception\InvalidArgumentException; use Flow\ETL\Exception\InvalidLogicException; use Flow\ETL\Function\ReferenceResolver; +use Flow\ETL\Tests\Context\FunctionContext; use Flow\ETL\Tests\FlowTestCase; use function Flow\ETL\DSL\flow_context; use function Flow\ETL\DSL\int_schema; use function Flow\ETL\DSL\lit; use function Flow\ETL\DSL\ref; -use function Flow\ETL\DSL\row; use function Flow\ETL\DSL\schema; use function Flow\ETL\DSL\str_schema; use function Flow\ETL\DSL\structure; @@ -25,11 +25,15 @@ public function test_eval_preserves_key_order(): void { static::assertSame( ['second' => 'x', 'first' => 1], - structure(['second' => ref('b'), 'first' => ref('a')])->eval(row([ - 'a' => 1, - 'b' => 'x', - 'q' => null, - ]), flow_context()), + (new FunctionContext(flow_context()))->eval( + structure(['second' => ref('b'), 'first' => ref('a')]), + [ + 'a' => 1, + 'b' => 'x', + 'q' => null, + ], + schema(int_schema('a'), str_schema('b'), str_schema('q', nullable: true)), + ), ); } @@ -37,11 +41,15 @@ public function test_eval_keeps_null_under_its_key(): void { static::assertSame( ['id' => 'x', 'quantity' => null], - structure(['id' => ref('b'), 'quantity' => ref('q')])->eval(row([ - 'a' => 1, - 'b' => 'x', - 'q' => null, - ]), flow_context()), + (new FunctionContext(flow_context()))->eval( + structure(['id' => ref('b'), 'quantity' => ref('q')]), + [ + 'a' => 1, + 'b' => 'x', + 'q' => null, + ], + schema(int_schema('a'), str_schema('b'), str_schema('q', nullable: true)), + ), ); } @@ -49,7 +57,11 @@ public function test_eval_with_numeric_keys(): void { static::assertSame( [5 => 1, 7 => 'x'], - structure([5 => ref('a'), 7 => ref('b')])->eval(row(['a' => 1, 'b' => 'x', 'q' => null]), flow_context()), + (new FunctionContext(flow_context()))->eval( + structure([5 => ref('a'), 7 => ref('b')]), + ['a' => 1, 'b' => 'x', 'q' => null], + schema(int_schema('a'), str_schema('b'), str_schema('q', nullable: true)), + ), ); } @@ -67,11 +79,15 @@ public function test_eval_nested_structure(): void { static::assertSame( ['user' => ['id' => 1]], - structure(['user' => structure(['id' => ref('a')])])->eval(row([ - 'a' => 1, - 'b' => 'x', - 'q' => null, - ]), flow_context()), + (new FunctionContext(flow_context()))->eval( + structure(['user' => structure(['id' => ref('a')])]), + [ + 'a' => 1, + 'b' => 'x', + 'q' => null, + ], + schema(int_schema('a'), str_schema('b'), str_schema('q', nullable: true)), + ), ); } @@ -125,9 +141,11 @@ public function test_with_children_keeps_keys(): void { static::assertSame( ['id' => 1, 'name' => 'x'], - structure(['id' => ref('a'), 'name' => ref('b')]) - ->withChildren([lit(1), lit('x')]) - ->eval(row(['a' => 1, 'b' => 'x', 'q' => null]), flow_context()), + (new FunctionContext(flow_context()))->eval( + structure(['id' => ref('a'), 'name' => ref('b')])->withChildren([lit(1), lit('x')]), + ['a' => 1, 'b' => 'x', 'q' => null], + schema(int_schema('a'), str_schema('b'), str_schema('q', nullable: true)), + ), ); } diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/SumTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/SumTest.php index d111edbe5f..40c5b5d3fa 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/SumTest.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/SumTest.php @@ -8,26 +8,22 @@ use Flow\ETL\Tests\FlowTestCase; use Flow\ETL\Tests\Mother\WindowContextMother; -use function Flow\ETL\DSL\bool_schema; +use function array_map; +use function Flow\ETL\DSL\array_to_rows; +use function Flow\ETL\DSL\average; use function Flow\ETL\DSL\config; use function Flow\ETL\DSL\float_schema; use function Flow\ETL\DSL\flow_context; use function Flow\ETL\DSL\int_schema; -use function Flow\ETL\DSL\lit; use function Flow\ETL\DSL\ref; -use function Flow\ETL\DSL\row; -use function Flow\ETL\DSL\rows; use function Flow\ETL\DSL\schema; +use function Flow\ETL\DSL\str_schema; use function Flow\ETL\DSL\sum; use function Flow\ETL\DSL\window; +use function range; final class SumTest extends FlowTestCase { - public function test_references_is_null_when_exact_is_a_scalar_function(): void - { - static::assertNull(sum(ref('value'), exact: ref('flag'))->references()); - } - public function test_references_returns_the_aggregated_reference(): void { static::assertEquals([ref('value')], sum(ref('value'))->references()); @@ -38,11 +34,17 @@ public function test_aggregation_sum_from_numeric_values(): void { $aggregator = sum(ref('int')); - $aggregator->aggregate(row(['int' => '10']), flow_context()); - $aggregator->aggregate(row(['int' => '20']), flow_context()); - $aggregator->aggregate(row(['int' => '55']), flow_context()); - $aggregator->aggregate(row(['int' => '25']), flow_context()); - $aggregator->aggregate(row(['not_int' => null]), flow_context()); + $aggregator->aggregate(array_to_rows([['int' => '10']], schema(str_schema('int'))), [0], flow_context()); + $aggregator->aggregate(array_to_rows([['int' => '20']], schema(str_schema('int'))), [0], flow_context()); + $aggregator->aggregate(array_to_rows([['int' => '55']], schema(str_schema('int'))), [0], flow_context()); + $aggregator->aggregate(array_to_rows([['int' => '25']], schema(str_schema('int'))), [0], flow_context()); + $aggregator->aggregate( + array_to_rows([[ + 'not_int' => null, + ]], schema(str_schema('not_int', nullable: true))), + [0], + flow_context(), + ); static::assertSame(110.0, $aggregator->value()); } @@ -51,10 +53,16 @@ public function test_aggregation_sum_including_null_value(): void { $aggregator = sum(ref('int')); - $aggregator->aggregate(row(['int' => 10]), flow_context()); - $aggregator->aggregate(row(['int' => 20]), flow_context()); - $aggregator->aggregate(row(['int' => 30]), flow_context()); - $aggregator->aggregate(row(['int' => null]), flow_context()); + $aggregator->aggregate(array_to_rows([['int' => 10]], schema(int_schema('int'))), [0], flow_context()); + $aggregator->aggregate(array_to_rows([['int' => 20]], schema(int_schema('int'))), [0], flow_context()); + $aggregator->aggregate(array_to_rows([['int' => 30]], schema(int_schema('int'))), [0], flow_context()); + $aggregator->aggregate( + array_to_rows([[ + 'int' => null, + ]], schema(int_schema('int', nullable: true))), + [0], + flow_context(), + ); static::assertSame(60.0, $aggregator->value()); } @@ -63,10 +71,10 @@ public function test_aggregation_sum_with_float_result(): void { $aggregator = sum(ref('int')); - $aggregator->aggregate(row(['int' => 10.25]), flow_context()); - $aggregator->aggregate(row(['int' => 20]), flow_context()); - $aggregator->aggregate(row(['int' => 305]), flow_context()); - $aggregator->aggregate(row(['int' => 25]), flow_context()); + $aggregator->aggregate(array_to_rows([['int' => 10.25]], schema(float_schema('int'))), [0], flow_context()); + $aggregator->aggregate(array_to_rows([['int' => 20]], schema(int_schema('int'))), [0], flow_context()); + $aggregator->aggregate(array_to_rows([['int' => 305]], schema(int_schema('int'))), [0], flow_context()); + $aggregator->aggregate(array_to_rows([['int' => 25]], schema(int_schema('int'))), [0], flow_context()); static::assertSame(360.25, $aggregator->value()); } @@ -75,30 +83,47 @@ public function test_aggregation_sum_of_decimal_fractions(): void { $aggregator = sum(ref('value')); - $aggregator->aggregate(row(['value' => 0.1]), flow_context()); - $aggregator->aggregate(row(['value' => 0.2]), flow_context()); + $aggregator->aggregate(array_to_rows([['value' => 0.1]], schema(float_schema('value'))), [0], flow_context()); + $aggregator->aggregate(array_to_rows([['value' => 0.2]], schema(float_schema('value'))), [0], flow_context()); static::assertSame(0.30000000000000004, $aggregator->value()); } + public function test_ten_tenths_sum_to_one_on_the_ieee_path(): void + { + $aggregator = sum(ref('value')); + $aggregator->aggregate(array_to_rows(array_fill(0, 10, [ + 'value' => 0.1, + ]), schema(float_schema('value'))), range(0, 9), flow_context()); + + // plain IEEE addition drifts to 0.9999999999999999 + static::assertSame(1.0, $aggregator->value()); + } + public function test_window_function_sum_of_decimal_fractions_uses_float_arithmetic_by_default(): void { - $rows = rows(schema(int_schema('id'), float_schema('value')), $row1 = row(['id' => 1, 'value' => 0.1]), row([ - 'id' => 2, - 'value' => 0.2, - ])); + $rows = array_to_rows( + [ + ['id' => 1, 'value' => 0.1], + [ + 'id' => 2, + 'value' => 0.2, + ], + ], + schema(int_schema('id'), float_schema('value')), + ); $sum = sum(ref('value'))->over(window()->orderBy(ref('id')->desc())); - static::assertSame(0.1 + 0.2, $sum->apply(WindowContextMother::forRow($row1, $rows))); + static::assertSame(0.1 + 0.2, $sum->apply(WindowContextMother::atIndex($rows, 0))); } public function test_aggregation_sum_of_floats_stays_float_when_sum_is_whole(): void { $aggregator = sum(ref('value')); - $aggregator->aggregate(row(['value' => 2.5]), flow_context()); - $aggregator->aggregate(row(['value' => 2.5]), flow_context()); + $aggregator->aggregate(array_to_rows([['value' => 2.5]], schema(float_schema('value'))), [0], flow_context()); + $aggregator->aggregate(array_to_rows([['value' => 2.5]], schema(float_schema('value'))), [0], flow_context()); static::assertSame(5.0, $aggregator->value()); static::assertSame('?float', $aggregator->returns()->toString()); @@ -108,90 +133,59 @@ public function test_sum_over_an_int_column_declares_float(): void { $aggregator = sum(ref('value')); - $aggregator->aggregate(row(['value' => 1]), flow_context()); - $aggregator->aggregate(row(['value' => 2]), flow_context()); - $aggregator->aggregate(row(['value' => 3]), flow_context()); - $aggregator->aggregate(row(['value' => 4]), flow_context()); + $aggregator->aggregate(array_to_rows([['value' => 1]], schema(int_schema('value'))), [0], flow_context()); + $aggregator->aggregate(array_to_rows([['value' => 2]], schema(int_schema('value'))), [0], flow_context()); + $aggregator->aggregate(array_to_rows([['value' => 3]], schema(int_schema('value'))), [0], flow_context()); + $aggregator->aggregate(array_to_rows([['value' => 4]], schema(int_schema('value'))), [0], flow_context()); static::assertSame(10.0, $aggregator->value()); static::assertSame('?float', $aggregator->returns()->toString()); } - public function test_a_malformed_exact_operand_throws_wrapped_as_a_sum_error(): void - { - $aggregator = sum(ref('value'), exact: ref('exact')); - - $this->expectException(InvalidArgumentException::class); - $this->expectExceptionMessage('Sum error:'); - - $aggregator->aggregate(row(['value' => 0.1, 'exact' => ['not' => 'a boolean']]), flow_context()); - } - public function test_exact_aggregation_sum_of_decimal_fractions(): void { $aggregator = sum(ref('value'), exact: true); - $aggregator->aggregate(row(['value' => 0.1]), flow_context()); - $aggregator->aggregate(row(['value' => 0.2]), flow_context()); + $aggregator->aggregate(array_to_rows([['value' => 0.1]], schema(float_schema('value'))), [0], flow_context()); + $aggregator->aggregate(array_to_rows([['value' => 0.2]], schema(float_schema('value'))), [0], flow_context()); static::assertSame(0.3, $aggregator->value()); } public function test_window_function_sum_on_partitioned_rows(): void { - $rows = rows( + $rows = array_to_rows( + [ + ['id' => 1, 'value' => 1], + ['id' => 2, 'value' => 1], + ['id' => 3, 'value' => 1], + ['id' => 4, 'value' => 1], + ['id' => 5, 'value' => 1], + ], schema(int_schema('id'), int_schema('value')), - $row1 = row(['id' => 1, 'value' => 1]), - row(['id' => 2, 'value' => 1]), - row(['id' => 3, 'value' => 1]), - row(['id' => 4, 'value' => 1]), - row(['id' => 5, 'value' => 1]), ); $sum = sum(ref('id'))->over(window()->orderBy(ref('id')->desc())); - static::assertSame(15, $sum->apply(WindowContextMother::forRow($row1, $rows))); + static::assertSame(15, $sum->apply(WindowContextMother::atIndex($rows, 0))); } public function test_window_function_sum_of_decimal_fractions_in_exact_mode(): void { - $rows = rows(schema(int_schema('id'), float_schema('value')), $row1 = row(['id' => 1, 'value' => 0.1]), row([ - 'id' => 2, - 'value' => 0.2, - ])); - - $sum = sum(ref('value'), exact: true)->over(window()->orderBy(ref('id')->desc())); - - static::assertSame(0.3, $sum->apply(WindowContextMother::forRow($row1, $rows))); - } - - public function test_window_function_sum_with_exact_mode_from_column(): void - { - $rows = rows( - schema(int_schema('id'), float_schema('value'), bool_schema('is_exact')), - $row1 = row(['id' => 1, 'value' => 0.1, 'is_exact' => true]), - row([ - 'id' => 2, - 'value' => 0.2, - 'is_exact' => true, - ]), + $rows = array_to_rows( + [ + ['id' => 1, 'value' => 0.1], + [ + 'id' => 2, + 'value' => 0.2, + ], + ], + schema(int_schema('id'), float_schema('value')), ); - $sum = sum(ref('value'), exact: ref('is_exact'))->over(window()->orderBy(ref('id')->desc())); - - static::assertSame(0.3, $sum->apply(WindowContextMother::forRow($row1, $rows))); - } - - public function test_window_function_sum_with_exact_mode_from_literal(): void - { - $rows = rows(schema(int_schema('id'), float_schema('value')), $row1 = row(['id' => 1, 'value' => 0.1]), row([ - 'id' => 2, - 'value' => 0.2, - ])); - - $sum = sum(ref('value'), exact: lit(true))->over(window()->orderBy(ref('id')->desc())); + $sum = sum(ref('value'), exact: true)->over(window()->orderBy(ref('id')->desc())); - static::assertSame(0.3, $sum->apply(WindowContextMother::forRow($row1, $rows))); + static::assertSame(0.3, $sum->apply(WindowContextMother::atIndex($rows, 0))); } public function test_window_function_sum_with_missing_reference_in_strict_mode(): void @@ -199,12 +193,12 @@ public function test_window_function_sum_with_missing_reference_in_strict_mode() $this->expectException(InvalidArgumentException::class); $this->expectExceptionMessage('Sum window function error:'); - $rows = rows(schema(int_schema('id')), $row1 = row(['id' => 1]), row(['id' => 2])); + $rows = array_to_rows([['id' => 1], ['id' => 2]], schema(int_schema('id'))); $sum = sum(ref('missing_column'))->over(window()->orderBy(ref('id'))); $context = flow_context(config()); - $sum->apply(WindowContextMother::forRow($row1, $rows, context: $context)); + $sum->apply(WindowContextMother::atIndex($rows, 0, context: $context)); } public function test_with_children_rebuilds_the_aggregate_with_the_given_reference(): void @@ -219,14 +213,6 @@ public function test_with_children_rebuilds_the_aggregate_with_the_given_referen static::assertEquals([ref('b')], $rebuilt->references()); } - public function test_with_children_carries_the_exact_operand_as_a_child(): void - { - $aggregate = sum(ref('a'), lit(true)); - - static::assertCount(2, $aggregate->children()); - static::assertNull($aggregate->withChildren([ref('b'), lit(false)])->references()); - } - public function test_output_name_is_not_affected_by_a_shared_reference(): void { $shared = ref('value'); @@ -242,4 +228,69 @@ public function test_sum_of_nothing_is_null(): void { static::assertNull(sum(ref('value'))->value()); } + + public function test_aggregate_reads_only_given_indices(): void + { + $aggregator = sum(ref('v')); + $aggregator->aggregate( + array_to_rows(array_map(static fn(mixed $v): array => ['v' => $v], [ + 0.5, + 1.25, + 2.0, + 3.0, + ]), schema(float_schema('v'))), + [1, 3], + flow_context(), + ); + + static::assertSame(4.25, $aggregator->value()); + } + + public function test_merge_of_two_halves_equals_one_pass(): void + { + $rows = array_to_rows(array_map(static fn(mixed $v): array => ['v' => $v], [ + 0.5, + 1.25, + 2.0, + 3.0, + ]), schema(float_schema('v'))); + $onePass = sum(ref('v')); + $onePass->aggregate($rows, [0, 1, 2, 3], flow_context()); + $left = sum(ref('v')); + $left->aggregate($rows, [0, 1], flow_context()); + $right = sum(ref('v')); + $right->aggregate($rows, [2, 3], flow_context()); + + $left->merge($right, flow_context()); + + static::assertSame($onePass->value(), $left->value()); + } + + public function test_merge_refuses_another_class(): void + { + $this->expectException(InvalidArgumentException::class); + + sum(ref('v'))->merge(average(ref('v')), flow_context()); + } + + public function test_merge_of_an_empty_half_keeps_null(): void + { + $left = sum(ref('v')); + $left->merge(sum(ref('v')), flow_context()); + + static::assertNull($left->value()); + } + + public function test_exact_merge_of_decimal_fractions(): void + { + $rows = array_to_rows([['v' => 0.1], ['v' => 0.2]], schema(float_schema('v'))); + $left = sum(ref('v'), exact: true); + $left->aggregate($rows, [0], flow_context()); + $right = sum(ref('v'), exact: true); + $right->aggregate($rows, [1], flow_context()); + + $left->merge($right, flow_context()); + + static::assertSame(0.3, $left->value()); + } } diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/ToDateTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/ToDateTest.php index 71aede9daa..e6d613fed1 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/ToDateTest.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/ToDateTest.php @@ -6,11 +6,15 @@ use DateTimeImmutable; use DateTimeZone; +use Flow\ETL\Tests\Context\FunctionContext; use Flow\ETL\Tests\FlowTestCase; +use function Flow\ETL\DSL\datetime_schema; use function Flow\ETL\DSL\flow_context; +use function Flow\ETL\DSL\int_schema; use function Flow\ETL\DSL\ref; -use function Flow\ETL\DSL\row; +use function Flow\ETL\DSL\schema; +use function Flow\ETL\DSL\str_schema; use function Flow\ETL\DSL\to_date; final class ToDateTest extends FlowTestCase @@ -19,11 +23,11 @@ public function test_date_time_to_date(): void { static::assertEquals( new DateTimeImmutable('2020-01-01 00:00:00', new DateTimeZone('UTC')), - to_date(ref('date_time')) - ->eval(row(['date_time' => new DateTimeImmutable( - '2020-01-01 12:43:23', - new DateTimeZone('UTC'), - )]), flow_context()), + (new FunctionContext(flow_context()))->eval( + to_date(ref('date_time')), + ['date_time' => new DateTimeImmutable('2020-01-01 12:43:23', new DateTimeZone('UTC'))], + schema(datetime_schema('date_time')), + ), ); } @@ -31,10 +35,13 @@ public function test_int_to_date_time(): void { static::assertEquals( new DateTimeImmutable('2020-01-01 00:00:00', new DateTimeZone('UTC')), - to_date(ref('int')) - ->eval(row([ + (new FunctionContext(flow_context()))->eval( + to_date(ref('int')), + [ 'int' => (int) (new DateTimeImmutable('2020-01-01 10:11:11', new DateTimeZone('UTC')))->format('U'), - ]), flow_context()), + ], + schema(int_schema('int')), + ), ); } @@ -42,7 +49,13 @@ public function test_string_to_date_time(): void { static::assertEquals( new DateTimeImmutable('2020-01-01 00:00:00', new DateTimeZone('UTC')), - to_date(ref('string'), 'Y-m-d H:i:s')->eval(row(['string' => '2020-01-01 10:08:00']), flow_context()), + (new FunctionContext(flow_context()))->eval( + to_date(ref('string'), 'Y-m-d H:i:s'), + [ + 'string' => '2020-01-01 10:08:00', + ], + schema(str_schema('string')), + ), ); } } diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/ToDateTimeTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/ToDateTimeTest.php index e69b488498..ce1f97b3db 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/ToDateTimeTest.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/ToDateTimeTest.php @@ -8,15 +8,19 @@ use DateTimeImmutable; use DateTimeZone; use Flow\ETL\Exception\InvalidArgumentException; +use Flow\ETL\Tests\Context\FunctionContext; use Flow\ETL\Tests\FlowTestCase; use stdClass; use function array_slice; +use function Flow\ETL\DSL\datetime_schema; use function Flow\ETL\DSL\flow_context; +use function Flow\ETL\DSL\int_schema; use function Flow\ETL\DSL\lit; use function Flow\ETL\DSL\random_string; use function Flow\ETL\DSL\ref; -use function Flow\ETL\DSL\row; +use function Flow\ETL\DSL\schema; +use function Flow\ETL\DSL\str_schema; use function Flow\ETL\DSL\to_date_time; use function Flow\Types\DSL\type_datetime; @@ -68,11 +72,11 @@ public function test_date_time_to_date_time(): void { static::assertEquals( new DateTimeImmutable('2020-01-01 00:00:00', new DateTimeZone('UTC')), - to_date_time(ref('date_time')) - ->eval(row(['date_time' => new DateTimeImmutable( - '2020-01-01 00:00:00', - new DateTimeZone('UTC'), - )]), flow_context()), + (new FunctionContext(flow_context()))->eval( + to_date_time(ref('date_time')), + ['date_time' => new DateTimeImmutable('2020-01-01 00:00:00', new DateTimeZone('UTC'))], + schema(datetime_schema('date_time')), + ), ); } @@ -80,10 +84,13 @@ public function test_int_to_date_time(): void { static::assertEquals( new DateTimeImmutable('2020-01-01 00:00:00', new DateTimeZone('UTC')), - to_date_time(ref('int')) - ->eval(row([ + (new FunctionContext(flow_context()))->eval( + to_date_time(ref('int')), + [ 'int' => (int) (new DateTimeImmutable('2020-01-01 00:00:00', new DateTimeZone('UTC')))->format('U'), - ]), flow_context()), + ], + schema(int_schema('int')), + ), ); } @@ -91,15 +98,25 @@ public function test_string_to_date_time(): void { static::assertEquals( new DateTimeImmutable('2020-01-01 00:00:00', new DateTimeZone('UTC')), - to_date_time(ref('string'), 'Y-m-d H:i:s')->eval(row(['string' => '2020-01-01 00:00:00']), flow_context()), + (new FunctionContext(flow_context()))->eval( + to_date_time(ref('string'), 'Y-m-d H:i:s'), + [ + 'string' => '2020-01-01 00:00:00', + ], + schema(str_schema('string')), + ), ); } public function test_unparseable_string_to_date_time_is_null(): void { - static::assertNull(to_date_time(ref('string'), 'Y-m-d H:i:s')->eval(row([ - 'string' => 'not a datetime', - ]), flow_context())); + static::assertNull((new FunctionContext(flow_context()))->eval( + to_date_time(ref('string'), 'Y-m-d H:i:s'), + [ + 'string' => 'not a datetime', + ], + schema(str_schema('string')), + )); } public function test_constant_zone_returns_a_zoned_type(): void @@ -115,13 +132,11 @@ public function test_object_keeps_its_time(): void static::assertSame( '10:30:00', type_datetime() - ->assert( - to_date_time(ref('at')) - ->eval(row(['at' => new DateTimeImmutable( - '2020-01-01 10:30:00', - new DateTimeZone('UTC'), - )]), flow_context()), - ) + ->assert((new FunctionContext(flow_context()))->eval( + to_date_time(ref('at')), + ['at' => new DateTimeImmutable('2020-01-01 10:30:00', new DateTimeZone('UTC'))], + schema(datetime_schema('at')), + )) ->format('H:i:s'), ); } @@ -131,9 +146,13 @@ public function test_int_lands_in_the_constant_zone(): void static::assertSame( '2020-01-01 11:30:00 Europe/Warsaw', type_datetime() - ->assert(to_date_time(ref('at'), 'Y-m-d H:i:s', new DateTimeZone('Europe/Warsaw'))->eval(row([ - 'at' => 1577874600, - ]), flow_context())) + ->assert((new FunctionContext(flow_context()))->eval( + to_date_time(ref('at'), 'Y-m-d H:i:s', new DateTimeZone('Europe/Warsaw')), + [ + 'at' => 1577874600, + ], + schema(int_schema('at')), + )) ->format('Y-m-d H:i:s e'), ); } @@ -144,9 +163,13 @@ public function test_mutable_datetime_is_converted_without_mutation(): void static::assertEquals( new DateTimeImmutable('2020-01-01 11:30:00', new DateTimeZone('Europe/Warsaw')), - to_date_time(ref('at'), 'Y-m-d H:i:s', new DateTimeZone('Europe/Warsaw'))->eval(row([ - 'at' => $value, - ]), flow_context()), + (new FunctionContext(flow_context()))->eval( + to_date_time(ref('at'), 'Y-m-d H:i:s', new DateTimeZone('Europe/Warsaw')), + [ + 'at' => $value, + ], + schema(datetime_schema('at')), + ), ); static::assertSame('2020-01-01 10:30:00 UTC', $value->format('Y-m-d H:i:s e')); } @@ -156,7 +179,7 @@ public function test_non_datetime_object_is_refused(): void $this->expectException(InvalidArgumentException::class); $this->expectExceptionMessage('ToDateTime function requires DateTimeInterface object'); - to_date_time(ref('at'))->eval(row(['at' => new stdClass()]), flow_context()); + (new FunctionContext(flow_context()))->eval(to_date_time(lit(new stdClass())), [], schema()); } public function test_string_in_a_utc_alias_zone_lands_in_utc(): void @@ -164,9 +187,13 @@ public function test_string_in_a_utc_alias_zone_lands_in_utc(): void static::assertSame( '2020-01-01 10:30:00 UTC', type_datetime() - ->assert(to_date_time(ref('at'), 'Y-m-d H:i:s', new DateTimeZone('GMT'))->eval(row([ - 'at' => '2020-01-01 10:30:00', - ]), flow_context())) + ->assert((new FunctionContext(flow_context()))->eval( + to_date_time(ref('at'), 'Y-m-d H:i:s', new DateTimeZone('GMT')), + [ + 'at' => '2020-01-01 10:30:00', + ], + schema(str_schema('at')), + )) ->format('Y-m-d H:i:s e'), ); } diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/ToLowerTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/ToLowerTest.php index 48f645394c..fe0f29196c 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/ToLowerTest.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/ToLowerTest.php @@ -4,17 +4,18 @@ namespace Flow\ETL\Tests\Unit\Function; +use Flow\ETL\Tests\Context\FunctionContext; use Flow\ETL\Tests\FlowTestCase; use function Flow\ETL\DSL\flow_context; use function Flow\ETL\DSL\lit; use function Flow\ETL\DSL\lower; -use function Flow\ETL\DSL\row; +use function Flow\ETL\DSL\schema; final class ToLowerTest extends FlowTestCase { public function test_string_to_lower(): void { - static::assertSame('lower', lower(lit('LOWER'))->eval(row([]), flow_context())); + static::assertSame('lower', (new FunctionContext(flow_context()))->eval(lower(lit('LOWER')), [], schema())); } } diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/ToTimeZoneTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/ToTimeZoneTest.php index 3f5f95d00e..1b75c509fb 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/ToTimeZoneTest.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/ToTimeZoneTest.php @@ -9,13 +9,15 @@ use DateTimeInterface; use DateTimeZone; use Flow\ETL\Exception\InvalidArgumentException; +use Flow\ETL\Tests\Context\FunctionContext; use Flow\ETL\Tests\FlowTestCase; use Flow\Types\Exception\InvalidArgumentException as TypesInvalidArgumentException; use function Flow\ETL\DSL\flow_context; use function Flow\ETL\DSL\lit; use function Flow\ETL\DSL\ref; -use function Flow\ETL\DSL\row; +use function Flow\ETL\DSL\schema; +use function Flow\ETL\DSL\str_schema; use function Flow\ETL\DSL\to_timezone; final class ToTimeZoneTest extends FlowTestCase @@ -33,10 +35,14 @@ public function test_abbreviation_zone_is_refused(): void public function test_casting_date_time_pst_to_utc_time_zone(): void { // @mago-ignore analysis:mixed-assignment - $result = to_timezone( - lit(new DateTimeImmutable('2020-01-01 00:00:00', new DateTimeZone('PST'))), - new DateTimeZone('UTC'), - )->eval(row([]), flow_context()); + $result = (new FunctionContext(flow_context()))->eval( + to_timezone( + lit(new DateTimeImmutable('2020-01-01 00:00:00', new DateTimeZone('PST'))), + new DateTimeZone('UTC'), + ), + [], + schema(), + ); static::assertInstanceOf(DateTimeInterface::class, $result); static::assertSame('2020-01-01 08:00:00.000000', $result->format('Y-m-d H:i:s.u')); } @@ -44,9 +50,10 @@ public function test_casting_date_time_pst_to_utc_time_zone(): void public function test_casting_date_time_pst_to_utc_time_zone_from_string_tz(): void { // @mago-ignore analysis:mixed-assignment - $result = to_timezone(lit(new DateTimeImmutable('2020-01-01 00:00:00', new DateTimeZone('PST'))), 'UTC')->eval( - row([]), - flow_context(), + $result = (new FunctionContext(flow_context()))->eval( + to_timezone(lit(new DateTimeImmutable('2020-01-01 00:00:00', new DateTimeZone('PST'))), 'UTC'), + [], + schema(), ); static::assertInstanceOf(DateTimeInterface::class, $result); static::assertSame('2020-01-01 08:00:00.000000', $result->format('Y-m-d H:i:s.u')); @@ -56,7 +63,7 @@ public function test_does_not_mutate_a_datetime(): void { $value = new DateTime('2020-01-01 00:00:00 UTC'); - to_timezone(lit($value), 'Europe/Warsaw')->eval(row([]), flow_context()); + (new FunctionContext(flow_context()))->eval(to_timezone(lit($value), 'Europe/Warsaw'), [], schema()); static::assertSame('UTC', $value->getTimezone()->getName()); } @@ -76,6 +83,12 @@ public function test_null_value_is_refused(): void $this->expectException(InvalidArgumentException::class); $this->expectExceptionMessage('ToTimeZone function requires non-null values'); - to_timezone(ref('at'), 'UTC')->eval(row(['at' => null]), flow_context()); + (new FunctionContext(flow_context()))->eval( + to_timezone(ref('at'), 'UTC'), + [ + 'at' => null, + ], + schema(str_schema('at', nullable: true)), + ); } } diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/ToUpperTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/ToUpperTest.php index f733e5298a..1bb0d48735 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/ToUpperTest.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/ToUpperTest.php @@ -4,17 +4,18 @@ namespace Flow\ETL\Tests\Unit\Function; +use Flow\ETL\Tests\Context\FunctionContext; use Flow\ETL\Tests\FlowTestCase; use function Flow\ETL\DSL\flow_context; use function Flow\ETL\DSL\lit; -use function Flow\ETL\DSL\row; +use function Flow\ETL\DSL\schema; use function Flow\ETL\DSL\upper; final class ToUpperTest extends FlowTestCase { public function test_string_to_upper(): void { - static::assertSame('UPPER', upper(lit('upper'))->eval(row([]), flow_context())); + static::assertSame('UPPER', (new FunctionContext(flow_context()))->eval(upper(lit('upper')), [], schema())); } } diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/TrimTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/TrimTest.php index c5fbf7ed16..2abcfd8a54 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/TrimTest.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/TrimTest.php @@ -5,30 +5,40 @@ namespace Flow\ETL\Tests\Unit\Function; use Flow\ETL\Function\Trim\Type; +use Flow\ETL\Tests\Context\FunctionContext; use Flow\ETL\Tests\FlowTestCase; use function Flow\ETL\DSL\flow_context; use function Flow\ETL\DSL\ref; -use function Flow\ETL\DSL\row; +use function Flow\ETL\DSL\schema; +use function Flow\ETL\DSL\str_schema; final class TrimTest extends FlowTestCase { public function test_trim_both_valid_string(): void { - static::assertSame('value', ref('string')->trim()->eval(row(['string' => ' value']), flow_context())); + static::assertSame('value', (new FunctionContext(flow_context()))->eval( + ref('string')->trim(), + ['string' => ' value'], + schema(str_schema('string')), + )); } public function test_trim_left_valid_string(): void { - static::assertSame('value ', ref('string') - ->trim(Type::LEFT) - ->eval(row(['string' => ' value ']), flow_context())); + static::assertSame('value ', (new FunctionContext(flow_context()))->eval( + ref('string')->trim(Type::LEFT), + ['string' => ' value '], + schema(str_schema('string')), + )); } public function test_trim_right_valid_string(): void { - static::assertSame(' value', ref('string') - ->trim(Type::RIGHT) - ->eval(row(['string' => ' value ']), flow_context())); + static::assertSame(' value', (new FunctionContext(flow_context()))->eval( + ref('string')->trim(Type::RIGHT), + ['string' => ' value '], + schema(str_schema('string')), + )); } } diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/TruncateTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/TruncateTest.php index 226fd77964..9994dd29e2 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/TruncateTest.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/TruncateTest.php @@ -5,29 +5,42 @@ namespace Flow\ETL\Tests\Unit\Function; use Flow\ETL\Exception\InvalidArgumentException; +use Flow\ETL\Tests\Context\FunctionContext; use Flow\ETL\Tests\FlowTestCase; use function Flow\ETL\DSL\flow_context; +use function Flow\ETL\DSL\int_schema; use function Flow\ETL\DSL\ref; -use function Flow\ETL\DSL\row; +use function Flow\ETL\DSL\schema; +use function Flow\ETL\DSL\str_schema; final class TruncateTest extends FlowTestCase { public function test_truncate_ellipsis_longer_than_limit(): void { - static::assertSame('hello', ref('str') - ->truncate(5, 'verylongellipsis') - ->eval(row(['str' => 'hello world']), flow_context())); + static::assertSame('hello', (new FunctionContext(flow_context()))->eval( + ref('str')->truncate(5, 'verylongellipsis'), + ['str' => 'hello world'], + schema(str_schema('str')), + )); } public function test_truncate_empty_string(): void { - static::assertSame('', ref('str')->truncate(10)->eval(row(['str' => '']), flow_context())); + static::assertSame('', (new FunctionContext(flow_context()))->eval( + ref('str')->truncate(10), + ['str' => ''], + schema(str_schema('str')), + )); } public function test_truncate_exact_length(): void { - static::assertSame('hello', ref('str')->truncate(5)->eval(row(['str' => 'hello']), flow_context())); + static::assertSame('hello', (new FunctionContext(flow_context()))->eval( + ref('str')->truncate(5), + ['str' => 'hello'], + schema(str_schema('str')), + )); } public function test_truncate_throws_on_null_input(): void @@ -35,59 +48,91 @@ public function test_truncate_throws_on_null_input(): void $this->expectException(InvalidArgumentException::class); $this->expectExceptionMessage('Truncate function requires non-null value'); - ref('str')->truncate(10)->eval(row(['str' => null]), flow_context()); + (new FunctionContext(flow_context()))->eval( + ref('str')->truncate(10), + ['str' => null], + schema(str_schema('str', nullable: true)), + ); } public function test_truncate_string_longer_than_limit(): void { - static::assertSame('he...', ref('str')->truncate(5)->eval(row(['str' => 'hello world']), flow_context())); + static::assertSame('he...', (new FunctionContext(flow_context()))->eval( + ref('str')->truncate(5), + ['str' => 'hello world'], + schema(str_schema('str')), + )); } public function test_truncate_string_shorter_than_limit(): void { - static::assertSame('hello', ref('str')->truncate(10)->eval(row(['str' => 'hello']), flow_context())); + static::assertSame('hello', (new FunctionContext(flow_context()))->eval( + ref('str')->truncate(10), + ['str' => 'hello'], + schema(str_schema('str')), + )); } public function test_truncate_with_custom_ellipsis(): void { - static::assertSame('he***', ref('str') - ->truncate(5, '***') - ->eval(row(['str' => 'hello world']), flow_context())); + static::assertSame('he***', (new FunctionContext(flow_context()))->eval( + ref('str')->truncate(5, '***'), + ['str' => 'hello world'], + schema(str_schema('str')), + )); } public function test_truncate_with_length_one(): void { - static::assertSame('h', ref('str')->truncate(1)->eval(row(['str' => 'hello']), flow_context())); + static::assertSame('h', (new FunctionContext(flow_context()))->eval( + ref('str')->truncate(1), + ['str' => 'hello'], + schema(str_schema('str')), + )); } public function test_truncate_with_length_zero(): void { - static::assertSame('', ref('str')->truncate(0)->eval(row(['str' => 'hello']), flow_context())); + static::assertSame('', (new FunctionContext(flow_context()))->eval( + ref('str')->truncate(0), + ['str' => 'hello'], + schema(str_schema('str')), + )); } public function test_truncate_with_negative_length(): void { - static::assertSame('hell', ref('str')->truncate(-1)->eval(row(['str' => 'hello']), flow_context())); + static::assertSame('hell', (new FunctionContext(flow_context()))->eval( + ref('str')->truncate(-1), + ['str' => 'hello'], + schema(str_schema('str')), + )); } public function test_truncate_with_null_length(): void { - static::assertSame('hello world', ref('str') - ->truncate(ref('length')) - ->eval(row(['str' => 'hello world', 'length' => null]), flow_context())); + static::assertSame('hello world', (new FunctionContext(flow_context()))->eval( + ref('str')->truncate(ref('length')), + ['str' => 'hello world', 'length' => null], + schema(str_schema('str'), str_schema('length', nullable: true)), + )); } public function test_truncate_with_scalar_function_ellipsis(): void { - static::assertSame('hel>>', ref('str') - ->truncate(5, ref('ellipsis')) - ->eval(row(['str' => 'hello world', 'ellipsis' => '>>']), flow_context())); + static::assertSame('hel>>', (new FunctionContext(flow_context()))->eval( + ref('str')->truncate(5, ref('ellipsis')), + ['str' => 'hello world', 'ellipsis' => '>>'], + schema(str_schema('str'), str_schema('ellipsis')), + )); } public function test_truncate_with_scalar_function_length(): void { - static::assertSame('he...', ref('str') - ->truncate(ref('length')) - ->eval(row(['str' => 'hello world', 'length' => 5]), flow_context())); + static::assertSame('he...', (new FunctionContext(flow_context()))->eval( + ref('str')->truncate(ref('length')), + ['str' => 'hello world', 'length' => 5], + schema(str_schema('str'), int_schema('length')), + )); } } diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/UlidTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/UlidTest.php index 6d8d1e4def..dd9dc19b06 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/UlidTest.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/UlidTest.php @@ -5,12 +5,13 @@ namespace Flow\ETL\Tests\Unit\Function; use Flow\ETL\Exception\InvalidArgumentException; +use Flow\ETL\Tests\Context\FunctionContext; use Flow\ETL\Tests\FlowTestCase; use Symfony\Component\Uid\Ulid; use function Flow\ETL\DSL\flow_context; use function Flow\ETL\DSL\lit; -use function Flow\ETL\DSL\row; +use function Flow\ETL\DSL\schema; use function Flow\ETL\DSL\ulid; final class UlidTest extends FlowTestCase @@ -18,18 +19,25 @@ final class UlidTest extends FlowTestCase public function test_ulid_produces_a_string_column(): void { $expression = ulid(); - $result = $expression->eval(row([]), flow_context()); + // @mago-ignore analysis:mixed-assignment + $result = (new FunctionContext(flow_context()))->eval($expression, [], schema()); static::assertIsString($result); static::assertTrue(Ulid::isValid($result)); - static::assertNotSame($expression->eval(row([]), flow_context()), $expression->eval(row([]), flow_context())); + static::assertNotSame( + (new FunctionContext(flow_context()))->eval($expression, [], schema()), + (new FunctionContext(flow_context()))->eval($expression, [], schema()), + ); } public function test_ulid_is_unique(): void { $expression = ulid(); - static::assertNotEquals($expression->eval(row([]), flow_context()), $expression->eval(row([]), flow_context())); + static::assertNotEquals( + (new FunctionContext(flow_context()))->eval($expression, [], schema()), + (new FunctionContext(flow_context()))->eval($expression, [], schema()), + ); } public function test_ulid_with_invalid_value_throws(): void @@ -37,6 +45,6 @@ public function test_ulid_with_invalid_value_throws(): void $this->expectException(InvalidArgumentException::class); $this->expectExceptionMessage('Ulid requires valid ULID string: Invalid ULID'); - ulid(lit(''))->eval(row([]), flow_context()); + (new FunctionContext(flow_context()))->eval(ulid(lit('')), [], schema()); } } diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/UnicodeLengthTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/UnicodeLengthTest.php index cb4d41d372..e6ebcb229c 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/UnicodeLengthTest.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/UnicodeLengthTest.php @@ -5,22 +5,32 @@ namespace Flow\ETL\Tests\Unit\Function; use Flow\ETL\Exception\InvalidArgumentException; +use Flow\ETL\Tests\Context\FunctionContext; use Flow\ETL\Tests\FlowTestCase; use function Flow\ETL\DSL\flow_context; use function Flow\ETL\DSL\ref; -use function Flow\ETL\DSL\row; +use function Flow\ETL\DSL\schema; +use function Flow\ETL\DSL\str_schema; final class UnicodeLengthTest extends FlowTestCase { public function test_unicode_length_ascii_string(): void { - static::assertSame(5, ref('str')->unicodeLength()->eval(row(['str' => 'hello']), flow_context())); + static::assertSame(5, (new FunctionContext(flow_context()))->eval( + ref('str')->unicodeLength(), + ['str' => 'hello'], + schema(str_schema('str')), + )); } public function test_unicode_length_empty_string(): void { - static::assertSame(0, ref('str')->unicodeLength()->eval(row(['str' => '']), flow_context())); + static::assertSame(0, (new FunctionContext(flow_context()))->eval( + ref('str')->unicodeLength(), + ['str' => ''], + schema(str_schema('str')), + )); } public function test_unicode_length_throws_on_null_input(): void @@ -28,6 +38,10 @@ public function test_unicode_length_throws_on_null_input(): void $this->expectException(InvalidArgumentException::class); $this->expectExceptionMessage('UnicodeLength function requires non-null value'); - ref('str')->unicodeLength()->eval(row(['str' => null]), flow_context()); + (new FunctionContext(flow_context()))->eval( + ref('str')->unicodeLength(), + ['str' => null], + schema(str_schema('str', nullable: true)), + ); } } diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/UuidTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/UuidTest.php index 51b1e8e845..71d46ec092 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/UuidTest.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/UuidTest.php @@ -7,6 +7,7 @@ use DateTimeImmutable; use DateTimeZone; use Flow\ETL\Exception\InvalidArgumentException; +use Flow\ETL\Tests\Context\FunctionContext; use Flow\ETL\Tests\FlowTestCase; use Flow\Types\Value\Uuid as FlowUuid; use Ramsey\Uuid\Uuid; @@ -15,7 +16,7 @@ use function class_exists; use function Flow\ETL\DSL\flow_context; use function Flow\ETL\DSL\lit; -use function Flow\ETL\DSL\row; +use function Flow\ETL\DSL\schema; use function Flow\ETL\DSL\uuid_v4; use function Flow\ETL\DSL\uuid_v7; @@ -35,17 +36,24 @@ public function test_uuid4(): void } $expression = uuid_v4(); - $result = $expression->eval(row([]), flow_context()); + // @mago-ignore analysis:mixed-assignment + $result = (new FunctionContext(flow_context()))->eval($expression, [], schema()); static::assertInstanceOf(FlowUuid::class, $result); static::assertTrue(Uuid::isValid($result->toString())); - static::assertNotSame($expression->eval(row([]), flow_context()), $expression->eval(row([]), flow_context())); + static::assertNotSame( + (new FunctionContext(flow_context()))->eval($expression, [], schema()), + (new FunctionContext(flow_context()))->eval($expression, [], schema()), + ); } public function test_uuid4_is_unique(): void { $expression = uuid_v4(); - static::assertNotEquals($expression->eval(row([]), flow_context()), $expression->eval(row([]), flow_context())); + static::assertNotEquals( + (new FunctionContext(flow_context()))->eval($expression, [], schema()), + (new FunctionContext(flow_context()))->eval($expression, [], schema()), + ); } public function test_uuid7(): void @@ -54,8 +62,12 @@ public function test_uuid7(): void static::markTestSkipped("Package 'ramsey/uuid' is required for this test."); } - $result = uuid_v7(lit(new DateTimeImmutable('2020-01-01 00:00:00', new DateTimeZone('UTC')))) - ->eval(row([]), flow_context()); + // @mago-ignore analysis:mixed-assignment + $result = (new FunctionContext(flow_context()))->eval( + uuid_v7(lit(new DateTimeImmutable('2020-01-01 00:00:00', new DateTimeZone('UTC')))), + [], + schema(), + ); static::assertInstanceOf(FlowUuid::class, $result); static::assertTrue(Uuid::isValid($result->toString())); } @@ -64,8 +76,8 @@ public function test_uuid7_is_unique(): void { $dateTime = lit(new DateTimeImmutable('2020-01-01 00:00:00', new DateTimeZone('UTC'))); static::assertNotEquals( - uuid_v7($dateTime)->eval(row([]), flow_context()), - uuid_v7($dateTime)->eval(row([]), flow_context()), + (new FunctionContext(flow_context()))->eval(uuid_v7($dateTime), [], schema()), + (new FunctionContext(flow_context()))->eval(uuid_v7($dateTime), [], schema()), ); } @@ -74,6 +86,6 @@ public function test_uuid_v7_requires_a_date_time(): void $this->expectException(InvalidArgumentException::class); $this->expectExceptionMessage('Uuid uuid7 function requires a DateTimeInterface value'); - uuid_v7(lit(''))->eval(row([]), flow_context()); + (new FunctionContext(flow_context()))->eval(uuid_v7(lit('')), [], schema()); } } diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/WhenTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/WhenTest.php index 8e9fe97a25..ea91a492dd 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/WhenTest.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/WhenTest.php @@ -4,45 +4,168 @@ namespace Flow\ETL\Tests\Unit\Function; +use Flow\ETL\Exception\EvaluationException; use Flow\ETL\Function\Literal; +use Flow\ETL\Function\ReferenceResolver; use Flow\ETL\Function\When; +use Flow\ETL\Tests\Context\FunctionContext; +use Flow\ETL\Tests\Double\FailingOnValuesFunction; use Flow\ETL\Tests\FlowTestCase; +use function Flow\ETL\DSL\array_to_rows; +use function Flow\ETL\DSL\bool_schema; use function Flow\ETL\DSL\flow_context; +use function Flow\ETL\DSL\int_schema; use function Flow\ETL\DSL\lit; use function Flow\ETL\DSL\ref; -use function Flow\ETL\DSL\row; +use function Flow\ETL\DSL\schema; use function Flow\ETL\DSL\when; final class WhenTest extends FlowTestCase { public function test_a_falsy_else_branch_is_returned(): void { - static::assertFalse(when(ref('active'), lit('yes'), false)->eval(row(['active' => false]), flow_context())); - static::assertSame(0, when(ref('active'), lit('yes'), 0)->eval(row(['active' => false]), flow_context())); - static::assertSame('', when(ref('active'), lit('yes'), '')->eval(row(['active' => false]), flow_context())); - static::assertSame([], when(ref('active'), lit('yes'), [])->eval(row(['active' => false]), flow_context())); - static::assertNull(when(ref('active'), lit('yes'))->eval(row(['active' => false]), flow_context())); + static::assertFalse((new FunctionContext(flow_context()))->eval( + when(ref('active'), lit(true), false), + [ + 'active' => false, + ], + schema(bool_schema('active')), + )); + static::assertSame(0, (new FunctionContext(flow_context()))->eval( + when(ref('active'), lit(1), 0), + [ + 'active' => false, + ], + schema(bool_schema('active')), + )); + static::assertSame('', (new FunctionContext(flow_context()))->eval( + when(ref('active'), lit('yes'), ''), + [ + 'active' => false, + ], + schema(bool_schema('active')), + )); + static::assertSame( + [], + (new FunctionContext(flow_context()))->eval( + when(ref('active'), lit(['yes']), []), + [ + 'active' => false, + ], + schema(bool_schema('active')), + ), + ); + static::assertNull((new FunctionContext(flow_context()))->eval( + when(ref('active'), lit('yes')), + [ + 'active' => false, + ], + schema(bool_schema('active')), + )); } public function test_condition_not_satisfied_without_else(): void { - static::assertSame(1, (new When(ref('id')->equals(lit(2)), new Literal('then'), ref('id')))->eval(row([ - 'id' => 1, - ]), flow_context())); + // when() of string and integer branches declares string: the value is cast to it + static::assertSame('1', (new FunctionContext(flow_context()))->eval( + new When(ref('id')->equals(lit(2)), new Literal('then'), ref('id')), + [ + 'id' => 1, + ], + schema(int_schema('id')), + )); } public function test_else(): void { - static::assertSame('else', (new When(new Literal(false), new Literal('then'), new Literal('else')))->eval(row([ - 'id' => 1, - ]), flow_context())); + static::assertSame('else', (new FunctionContext(flow_context()))->eval( + new When(new Literal(false), new Literal('then'), new Literal('else')), + [ + 'id' => 1, + ], + schema(int_schema('id')), + )); } public function test_when(): void { - static::assertSame('then', (new When(new Literal(true), new Literal('then')))->eval(row([ - 'id' => 1, - ]), flow_context())); + static::assertSame('then', (new FunctionContext(flow_context()))->eval( + new When(new Literal(true), new Literal('then')), + [ + 'id' => 1, + ], + schema(int_schema('id')), + )); + } + + public function test_then_branch_error_names_the_batch_row(): void + { + $rows = array_to_rows( + [ + ['c' => true, 'v' => 1], + ['c' => false, 'v' => 99], + ['c' => true, 'v' => 3], + ['c' => false, 'v' => 99], + ['c' => false, 'v' => 99], + ['c' => true, 'v' => 50], + ], + schema(bool_schema('c'), int_schema('v')), + ); + + try { + (new ReferenceResolver()) + ->resolve(when(ref('c'), new FailingOnValuesFunction(ref('v'), [50]), lit(0)), $rows->schema()) + ->eval($rows, flow_context()); + static::fail('expected an EvaluationException'); + } catch (EvaluationException $e) { + static::assertSame(5, $e->rowIndex); + } + } + + public function test_else_branch_error_names_the_batch_row(): void + { + $rows = array_to_rows( + [ + ['c' => false, 'v' => 1], + ['c' => true, 'v' => 99], + ['c' => true, 'v' => 99], + ['c' => false, 'v' => 50], + ['c' => true, 'v' => 99], + ], + schema(bool_schema('c'), int_schema('v')), + ); + + try { + (new ReferenceResolver()) + ->resolve(when(ref('c'), lit(0), new FailingOnValuesFunction(ref('v'), [50])), $rows->schema()) + ->eval($rows, flow_context()); + static::fail('expected an EvaluationException'); + } catch (EvaluationException $e) { + static::assertSame(3, $e->rowIndex); + } + } + + public function test_a_branch_runs_only_on_the_rows_that_reach_it(): void + { + $rows = array_to_rows( + [ + ['c' => true, 'v' => 1], + ['c' => false, 'v' => 99], + ['c' => true, 'v' => 3], + ['c' => false, 'v' => 99], + ['c' => false, 'v' => 99], + ['c' => true, 'v' => 50], + ], + schema(bool_schema('c'), int_schema('v')), + ); + + static::assertSame( + [1, 0, 3, 0, 0, 50], + (new ReferenceResolver()) + ->resolve(when(ref('c'), new FailingOnValuesFunction(ref('v'), [99]), lit(0)), $rows->schema()) + ->eval($rows, flow_context()) + ->values(), + ); } } diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/WordwrapTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/WordwrapTest.php index b892f22fdd..4828179f3f 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/WordwrapTest.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/WordwrapTest.php @@ -5,35 +5,43 @@ namespace Flow\ETL\Tests\Unit\Function; use Flow\ETL\Exception\InvalidArgumentException; +use Flow\ETL\Tests\Context\FunctionContext; use Flow\ETL\Tests\FlowTestCase; +use function Flow\ETL\DSL\bool_schema; use function Flow\ETL\DSL\flow_context; +use function Flow\ETL\DSL\int_schema; use function Flow\ETL\DSL\ref; -use function Flow\ETL\DSL\row; +use function Flow\ETL\DSL\schema; +use function Flow\ETL\DSL\str_schema; final class WordwrapTest extends FlowTestCase { public function test_empty_string(): void { - $result = ref('str')->wordwrap(10)->eval(row(['str' => '']), flow_context()); - - static::assertEquals('', $result); + static::assertEquals('', (new FunctionContext(flow_context()))->eval( + ref('str')->wordwrap(10), + ['str' => ''], + schema(str_schema('str')), + )); } public function test_normal_word_wrapping(): void { - $result = ref('str')->wordwrap(10)->eval(row(['str' => 'The quick brown fox jumps']), flow_context()); - - static::assertEquals("The quick\nbrown fox\njumps", $result); + static::assertEquals("The quick\nbrown fox\njumps", (new FunctionContext(flow_context()))->eval( + ref('str')->wordwrap(10), + ['str' => 'The quick brown fox jumps'], + schema(str_schema('str')), + )); } public function test_null_break_character(): void { - $result = ref('str') - ->wordwrap(10, ref('break')) - ->eval(row(['str' => 'Hello World Test', 'break' => null]), flow_context()); - - static::assertEquals("Hello\nWorld Test", $result); + static::assertEquals("Hello\nWorld Test", (new FunctionContext(flow_context()))->eval( + ref('str')->wordwrap(10, ref('break')), + ['str' => 'Hello World Test', 'break' => null], + schema(str_schema('str'), str_schema('break', nullable: true)), + )); } public function test_null_value(): void @@ -41,49 +49,55 @@ public function test_null_value(): void $this->expectException(InvalidArgumentException::class); $this->expectExceptionMessage('Wordwrap function requires non-null value'); - $result = ref('str')->wordwrap(10)->eval(row(['str' => null]), flow_context()); - - static::assertNull($result); + static::assertNull((new FunctionContext(flow_context()))->eval( + ref('str')->wordwrap(10), + ['str' => null], + schema(str_schema('str', nullable: true)), + )); } public function test_text_shorter_than_width(): void { - $result = ref('str')->wordwrap(20)->eval(row(['str' => 'Hello']), flow_context()); - - static::assertEquals('Hello', $result); + static::assertEquals('Hello', (new FunctionContext(flow_context()))->eval( + ref('str')->wordwrap(20), + ['str' => 'Hello'], + schema(str_schema('str')), + )); } public function test_width_zero(): void { - $result = ref('str')->wordwrap(0)->eval(row(['str' => 'Hello World']), flow_context()); - - static::assertEquals('Hello World', $result); + static::assertEquals('Hello World', (new FunctionContext(flow_context()))->eval( + ref('str')->wordwrap(0), + ['str' => 'Hello World'], + schema(str_schema('str')), + )); } public function test_with_scalar_function_break(): void { - $result = ref('str') - ->wordwrap(10, ref('break')) - ->eval(row(['str' => 'Hello World Test', 'break' => ' | ']), flow_context()); - - static::assertEquals('Hello | World Test', $result); + static::assertEquals('Hello | World Test', (new FunctionContext(flow_context()))->eval( + ref('str')->wordwrap(10, ref('break')), + ['str' => 'Hello World Test', 'break' => ' | '], + schema(str_schema('str'), str_schema('break')), + )); } public function test_with_scalar_function_cut(): void { - $result = ref('str') - ->wordwrap(3, "\n", ref('cut')) - ->eval(row(['str' => 'Hello', 'cut' => true]), flow_context()); - - static::assertEquals("Hel\nlo", $result); + static::assertEquals("Hel\nlo", (new FunctionContext(flow_context()))->eval( + ref('str')->wordwrap(3, "\n", ref('cut')), + ['str' => 'Hello', 'cut' => true], + schema(str_schema('str'), bool_schema('cut')), + )); } public function test_with_scalar_function_width(): void { - $result = ref('str') - ->wordwrap(ref('width')) - ->eval(row(['str' => 'Hello World Test', 'width' => 8]), flow_context()); - - static::assertEquals("Hello\nWorld\nTest", $result); + static::assertEquals("Hello\nWorld\nTest", (new FunctionContext(flow_context()))->eval( + ref('str')->wordwrap(ref('width')), + ['str' => 'Hello World Test', 'width' => 8], + schema(str_schema('str'), int_schema('width')), + )); } } diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/XPathTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/XPathTest.php index 2ef26aa17f..3f9229413f 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/XPathTest.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Function/XPathTest.php @@ -6,11 +6,13 @@ use DOMDocument; use DOMElement; +use Flow\ETL\Tests\Context\FunctionContext; use Flow\ETL\Tests\FlowTestCase; use function Flow\ETL\DSL\flow_context; use function Flow\ETL\DSL\ref; -use function Flow\ETL\DSL\row; +use function Flow\ETL\DSL\schema; +use function Flow\ETL\DSL\xml_schema; final class XPathTest extends FlowTestCase { @@ -22,7 +24,11 @@ public function test_xpath_on_simple_xml_with_only_one_node_returned(): void static::assertInstanceOf(DOMElement::class, $xml->documentElement); static::assertEquals( [$xml->documentElement->firstChild], - ref('value')->xpath('/root/foo')->eval(row(['value' => $xml]), flow_context()), + (new FunctionContext(flow_context()))->eval( + ref('value')->xpath('/root/foo'), + ['value' => $xml], + schema(xml_schema('value')), + ), ); } @@ -37,7 +43,11 @@ public function test_xpath_when_there_are_more_than_one_elements_under_given_pat $xml->documentElement->firstChild, $xml->documentElement->lastChild, ], - ref('value')->xpath('/root/foo')->eval(row(['value' => $xml]), flow_context()), + (new FunctionContext(flow_context()))->eval( + ref('value')->xpath('/root/foo'), + ['value' => $xml], + schema(xml_schema('value')), + ), ); } @@ -46,7 +56,11 @@ public function test_xpath_with_invalid_path_syntax(): void $xml = new DOMDocument(); $xml->loadXML('bar'); - static::assertNull(ref('value')->xpath('/root/foo/asa')->eval(row(['value' => $xml]), flow_context())); + static::assertNull((new FunctionContext(flow_context()))->eval( + ref('value')->xpath('/root/foo/asa'), + ['value' => $xml], + schema(xml_schema('value')), + )); } public function test_xpath_with_non_existing_path(): void @@ -54,7 +68,11 @@ public function test_xpath_with_non_existing_path(): void $xml = new DOMDocument(); $xml->loadXML('bar'); - static::assertNull(ref('value')->xpath('/root/bar')->eval(row(['value' => $xml]), flow_context())); + static::assertNull((new FunctionContext(flow_context()))->eval( + ref('value')->xpath('/root/bar'), + ['value' => $xml], + schema(xml_schema('value')), + )); } public function test_xpath_selecting_non_element_nodes_returns_null(): void @@ -62,7 +80,15 @@ public function test_xpath_selecting_non_element_nodes_returns_null(): void $xml = new DOMDocument(); $xml->loadXML('bar'); - static::assertNull(ref('value')->xpath('/root/foo/text()')->eval(row(['value' => $xml]), flow_context())); - static::assertNull(ref('value')->xpath('/root/foo/@baz')->eval(row(['value' => $xml]), flow_context())); + static::assertNull((new FunctionContext(flow_context()))->eval( + ref('value')->xpath('/root/foo/text()'), + ['value' => $xml], + schema(xml_schema('value')), + )); + static::assertNull((new FunctionContext(flow_context()))->eval( + ref('value')->xpath('/root/foo/@baz'), + ['value' => $xml], + schema(xml_schema('value')), + )); } } diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/GroupBy/AggregatedGroupsTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/GroupBy/AggregatedGroupsTest.php index 20563bbfb4..2a498538e8 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Unit/GroupBy/AggregatedGroupsTest.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/GroupBy/AggregatedGroupsTest.php @@ -11,13 +11,12 @@ use Flow\ETL\Tests\FlowTestCase; use function array_map; +use function Flow\ETL\DSL\array_to_rows; use function Flow\ETL\DSL\config; use function Flow\ETL\DSL\float_schema; use function Flow\ETL\DSL\flow_context; use function Flow\ETL\DSL\int_schema; use function Flow\ETL\DSL\ref; -use function Flow\ETL\DSL\row; -use function Flow\ETL\DSL\rows; use function Flow\ETL\DSL\schema; use function Flow\ETL\DSL\str_schema; use function Flow\ETL\DSL\sum; @@ -32,14 +31,14 @@ public function test_it_accumulates_rows_of_the_same_key_into_one_group(): void $input = schema(str_schema('country'), int_schema('amount')); $groups = new AggregatedGroups($groupBy, GroupByShape::of($groupBy, $input)); - $groups->accumulate( - rows($input, row(['country' => 'PL', 'amount' => 10]), row(['country' => 'DE', 'amount' => 20])), - flow_context(config()), - ); - $groups->accumulate(rows($input, row(['country' => 'PL', 'amount' => 30])), flow_context(config())); + $groups->accumulate(array_to_rows([ + ['country' => 'PL', 'amount' => 10], + ['country' => 'DE', 'amount' => 20], + ], $input), flow_context(config())); + $groups->accumulate(array_to_rows([['country' => 'PL', 'amount' => 30]], $input), flow_context(config())); /** @var list $batches */ - $batches = iterator_to_array($groups->flush(1000)); + $batches = iterator_to_array($groups->flush(1000, flow_context(config()))); static::assertCount(1, $batches); static::assertSame( @@ -60,10 +59,10 @@ public function test_the_flushed_batch_carries_the_shape_output_schema(): void $shape = GroupByShape::of($groupBy, $input); $groups = new AggregatedGroups($groupBy, $shape); - $groups->accumulate(rows($input, row(['country' => 'PL', 'amount' => 10])), flow_context(config())); + $groups->accumulate(array_to_rows([['country' => 'PL', 'amount' => 10]], $input), flow_context(config())); /** @var list $batches */ - $batches = iterator_to_array($groups->flush(1000)); + $batches = iterator_to_array($groups->flush(1000, flow_context(config()))); static::assertEquals($shape->output, $batches[0]->schema()); static::assertEquals(schema(str_schema('country'), float_schema('amount_sum', nullable: true)), $shape->output); @@ -77,19 +76,18 @@ public function test_it_splits_the_flushed_groups_into_batches_of_the_requested_ $input = schema(str_schema('country'), int_schema('amount')); $groups = new AggregatedGroups($groupBy, GroupByShape::of($groupBy, $input)); - $groups->accumulate( - rows( - $input, - row(['country' => 'PL', 'amount' => 10]), - row(['country' => 'DE', 'amount' => 20]), - row(['country' => 'FR', 'amount' => 30]), - ), - flow_context(config()), - ); + $groups->accumulate(array_to_rows([ + ['country' => 'PL', 'amount' => 10], + ['country' => 'DE', 'amount' => 20], + ['country' => 'FR', 'amount' => 30], + ], $input), flow_context(config())); static::assertSame( [2, 1], - array_map(static fn(Rows $batch): int => $batch->count(), iterator_to_array($groups->flush(2))), + array_map( + static fn(Rows $batch): int => $batch->count(), + iterator_to_array($groups->flush(2, flow_context(config()))), + ), ); } @@ -103,7 +101,7 @@ public function test_flushing_without_accumulating_yields_nothing(): void int_schema('amount'), ))); - static::assertSame([], iterator_to_array($groups->flush(1000))); + static::assertSame([], iterator_to_array($groups->flush(1000, flow_context(config())))); } public function test_a_row_missing_a_nullable_group_key_lands_in_the_null_group(): void @@ -114,13 +112,14 @@ public function test_a_row_missing_a_nullable_group_key_lands_in_the_null_group( $input = schema(str_schema('country', nullable: true), int_schema('amount')); $groups = new AggregatedGroups($groupBy, GroupByShape::of($groupBy, $input)); - $groups->accumulate( - rows($input, row(['amount' => 10]), row(['country' => 'PL', 'amount' => 20]), row(['amount' => 5])), - flow_context(config()), - ); + $groups->accumulate(array_to_rows([ + ['amount' => 10], + ['country' => 'PL', 'amount' => 20], + ['amount' => 5], + ], $input), flow_context(config())); /** @var list $batches */ - $batches = iterator_to_array($groups->flush(1000)); + $batches = iterator_to_array($groups->flush(1000, flow_context(config()))); static::assertSame( [ diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/GroupBy/AggregatorsTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/GroupBy/AggregatorsTest.php index 50e95f33c5..9e9aeb6c9f 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Unit/GroupBy/AggregatorsTest.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/GroupBy/AggregatorsTest.php @@ -6,6 +6,7 @@ use Flow\ETL\Exception\InvalidArgumentException; use Flow\ETL\GroupBy\Aggregators; +use Flow\ETL\Tests\Double\UnenumerableSum; use Flow\ETL\Tests\FlowTestCase; use Flow\ETL\Tests\Mother\ListColumnsMother; @@ -49,7 +50,7 @@ public function test_references_union_all_aggregators(): void public function test_references_are_null_when_any_aggregator_cannot_enumerate_them(): void { static::assertNull( - (new Aggregators(count(ref('order_id')), sum(ref('discount'), exact: ref('flag'))))->references(), + (new Aggregators(count(ref('order_id')), new UnenumerableSum(ref('discount'))))->references(), ); } @@ -60,7 +61,7 @@ public function test_resolved_refuses_an_expand_in_an_aggregate(): void 'aggregate() cannot contain array_expand(), it turns one row into many rows. Expand with withEntry() first, then use the new column.', ); - (new Aggregators(sum(ref('n'), ref('flags')->expand()->equals(lit(true)))))->resolved( + (new Aggregators(new UnenumerableSum(ref('n'), ref('flags')->expand()->equals(lit(true)))))->resolved( ListColumnsMother::numberAndFlagsSchema(), ); } diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/GroupBy/BucketAggregationTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/GroupBy/BucketAggregationTest.php deleted file mode 100644 index ae0474f3b2..0000000000 --- a/src/core/etl/tests/Flow/ETL/Tests/Unit/GroupBy/BucketAggregationTest.php +++ /dev/null @@ -1,224 +0,0 @@ -aggregate(sum(ref('v'))); - - $batches = (static function () { - yield rows( - schema(str_schema('k'), int_schema('v')), - row(['k' => 'a', 'v' => 1]), - row(['k' => 'b', 'v' => 10]), - ); - })(); - - $result = iterator_to_array( - (new BucketAggregation(batchSize: 1))->aggregate($batches, flow_context(), $groupBy), - preserve_keys: false, - ); - - static::assertCount(2, $result); - - foreach ($result as $batch) { - static::assertCount(1, $batch); - } - } - - public function test_throws_when_batch_size_below_one(): void - { - $this->expectException(InvalidArgumentException::class); - $this->expectExceptionMessage('Batch size must be greater than 0, given: 0'); - - // @mago-ignore analysis:invalid-argument - new BucketAggregation(0); - } - - public function test_folds_groups_across_multiple_batches(): void - { - $context = flow_context(); - - $groupBy = new GroupBy(ref('k')); - $groupBy->aggregate(sum(ref('v'))); - - $batches = (static function () { - yield rows( - schema(str_schema('k'), int_schema('v')), - row(['k' => 'a', 'v' => 1]), - row(['k' => 'b', 'v' => 10]), - ); - yield rows( - schema(str_schema('k'), int_schema('v')), - row(['k' => 'a', 'v' => 2]), - row(['k' => 'b', 'v' => 20]), - ); - })(); - - $sums = []; - - foreach ((new BucketAggregation())->aggregate($batches, $context, $groupBy) as $resultRows) { - foreach ($resultRows as $resultRow) { - $key = $resultRow->get(ref('k')); - assert(is_string($key)); - - $sums[$key] = $resultRow->get(ref('v_sum')); - } - } - - static::assertSame(['a' => 3.0, 'b' => 30.0], $sums); - } - - public function test_first_preserves_input_order_across_batches(): void - { - $context = flow_context(); - - $groupBy = new GroupBy(ref('k')); - $groupBy->aggregate(first(ref('v'))); - - $batches = (static function () { - yield rows(schema(str_schema('k'), int_schema('v')), row(['k' => 'a', 'v' => 100])); - yield rows(schema(str_schema('k'), int_schema('v')), row(['k' => 'a', 'v' => 200])); - })(); - - $firsts = []; - - foreach ((new BucketAggregation())->aggregate($batches, $context, $groupBy) as $resultRows) { - foreach ($resultRows as $resultRow) { - $key = $resultRow->get(ref('k')); - assert(is_string($key)); - - $firsts[$key] = $resultRow->get(ref('v_first')); - } - } - - static::assertSame(['a' => 100], $firsts); - } - - public function test_last_keeps_the_latest_across_batches(): void - { - $context = flow_context(); - - $groupBy = new GroupBy(ref('k')); - $groupBy->aggregate(last(ref('v'))); - - $batches = (static function () { - yield rows(schema(str_schema('k'), int_schema('v')), row(['k' => 'a', 'v' => 100])); - yield rows(schema(str_schema('k'), int_schema('v')), row(['k' => 'a', 'v' => 200])); - })(); - - $lasts = []; - - foreach ((new BucketAggregation())->aggregate($batches, $context, $groupBy) as $resultRows) { - foreach ($resultRows as $resultRow) { - $key = $resultRow->get(ref('k')); - assert(is_string($key)); - - $lasts[$key] = $resultRow->get(ref('v_last')); - } - } - - static::assertSame(['a' => 200], $lasts); - } - - public function test_a_leading_empty_batch_defers_the_bind_to_the_first_data_batch(): void - { - $groupBy = new GroupBy(ref('k')); - $groupBy->aggregate(sum(ref('v'))); - - $batches = (static function () { - yield rows(schema()); - yield rows( - schema(str_schema('k'), int_schema('v')), - row(['k' => 'a', 'v' => 1]), - row(['k' => 'a', 'v' => 2]), - ); - })(); - - $result = iterator_to_array( - (new BucketAggregation())->aggregate($batches, flow_context(), $groupBy), - preserve_keys: false, - ); - - static::assertCount(1, $result); - static::assertSame([['k' => 'a', 'v_sum' => 3.0]], $result[0]->toArray()); - } - - public function test_groups_in_one_run_share_one_output_definition(): void - { - $groupBy = new GroupBy(ref('k')); - $groupBy->aggregate(min(ref('v'))); - - $batches = (static function () { - yield rows( - schema(str_schema('k'), float_schema('v')), - row(['k' => 'a', 'v' => 10.0]), - row(['k' => 'b', 'v' => 0.5]), - ); - })(); - - $definitions = []; - $mins = []; - - foreach ((new BucketAggregation())->aggregate($batches, flow_context(), $groupBy) as $batch) { - $definitions[] = $batch->schema()->get('v_min'); - - foreach ($batch as $row) { - $mins[] = $row->get('v_min'); - } - } - - static::assertCount(2, $mins); - static::assertEquals([float_schema('v_min', true)], array_unique($definitions, SORT_REGULAR)); - } - - /** - * A group that accumulates nothing is the reachable twin of the never-aggregated instance: the - * rows carry the key but omit the aggregated column, so min() emits null under its nullable - * definition instead of the definition being widened after the fact. - */ - public function test_a_group_whose_aggregated_column_is_absent_emits_null_under_a_nullable_definition(): void - { - $groupBy = new GroupBy(ref('k')); - $groupBy->aggregate(min(ref('v'))); - - $batches = (static function () { - yield rows( - schema(str_schema('k'), float_schema('v', nullable: true)), - row(['k' => 'a']), - row(['k' => 'a']), - ); - })(); - - $result = iterator_to_array( - (new BucketAggregation())->aggregate($batches, flow_context(), $groupBy), - preserve_keys: false, - ); - - static::assertSame([['k' => 'a', 'v_min' => null]], $result[0]->toArray()); - static::assertEquals(float_schema('v_min', true), $result[0]->schema()->get('v_min')); - } -} diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/GroupBy/DiscoveredPivotValuesTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/GroupBy/DiscoveredPivotValuesTest.php index b547b2a19c..995f837462 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Unit/GroupBy/DiscoveredPivotValuesTest.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/GroupBy/DiscoveredPivotValuesTest.php @@ -9,6 +9,7 @@ use Flow\ETL\GroupBy\DiscoveredPivotValues; use Flow\ETL\Tests\Double\EmptyExtractor; use Flow\ETL\Tests\FlowTestCase; +use Flow\Types\Exception\InvalidTypeException; use function Flow\ETL\DSL\concat; use function Flow\ETL\DSL\df; @@ -75,6 +76,36 @@ public function test_it_refuses_more_distinct_values_than_the_bound_allows(): vo ); } + public function test_numeric_looking_strings_are_ordered_by_bytes_as_sort_by_does(): void + { + static::assertSame( + ['10', '100', '9', 'B'], + (new DiscoveredPivotValues())->resolve( + df()->read(from_array([['code' => '9'], ['code' => '10'], ['code' => 'B'], ['code' => '100']])), + ref('code'), + )->all(), + ); + } + + public function test_only_string_and_integer_values_can_become_pivot_columns(): void + { + $this->expectException(InvalidTypeException::class); + $this->expectExceptionMessage('Expected type "integer|string", got "float".'); + + (new DiscoveredPivotValues())->resolve(df()->read(from_array([['code' => 1.5]])), ref('code')); + } + + public function test_integers_are_ordered_numerically(): void + { + static::assertSame( + [9, 10, 100], + (new DiscoveredPivotValues())->resolve( + df()->read(from_array([['code' => 100], ['code' => 9], ['code' => 10]])), + ref('code'), + )->all(), + ); + } + public function test_the_discovered_values_are_distinct_null_free_and_sorted(): void { static::assertSame( diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/GroupBy/GroupByStepsTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/GroupBy/GroupByStepsTest.php index e2d1f0da4d..9f26823e39 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Unit/GroupBy/GroupByStepsTest.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/GroupBy/GroupByStepsTest.php @@ -5,16 +5,18 @@ namespace Flow\ETL\Tests\Unit\GroupBy; use Flow\ETL\Bucketing\Storage\MemoryBuckets; +use Flow\ETL\Dataset\Memory\Unit; use Flow\ETL\GroupBy; use Flow\ETL\GroupBy\GroupBySteps; -use Flow\ETL\Processor\BucketingProcessor; use Flow\ETL\Processor\GroupByAggregationProcessor; use Flow\ETL\Processor\PivotProcessor; use Flow\ETL\Tests\Context\GroupByContext; use Flow\ETL\Tests\Double\SpyBucketsStorage; +use Flow\ETL\Tests\Double\UnenumerableSum; use Flow\ETL\Tests\FlowTestCase; use Flow\ETL\Transformer\PruneEntriesTransformer; +use function Flow\ETL\DSL\array_to_rows; use function Flow\ETL\DSL\bool_schema; use function Flow\ETL\DSL\config; use function Flow\ETL\DSL\config_builder; @@ -25,8 +27,6 @@ use function Flow\ETL\DSL\int_schema; use function Flow\ETL\DSL\pivot_values; use function Flow\ETL\DSL\ref; -use function Flow\ETL\DSL\row; -use function Flow\ETL\DSL\rows; use function Flow\ETL\DSL\schema; use function Flow\ETL\DSL\str_schema; use function Flow\ETL\DSL\sum; @@ -52,22 +52,20 @@ public function test_enumerable_references_add_a_pruning_transformer(): void $steps = GroupBySteps::of($groupBy, config()); - static::assertCount(3, $steps); + static::assertCount(2, $steps); static::assertInstanceOf(PruneEntriesTransformer::class, $steps[0]); - static::assertInstanceOf(BucketingProcessor::class, $steps[1]); - static::assertInstanceOf(GroupByAggregationProcessor::class, $steps[2]); + static::assertInstanceOf(GroupByAggregationProcessor::class, $steps[1]); } public function test_non_enumerable_references_skip_the_pruning_transformer(): void { $groupBy = new GroupBy(ref('category')); - $groupBy->aggregate(sum(ref('amount'), exact: ref('flag'))); + $groupBy->aggregate(new UnenumerableSum(ref('amount'))); $steps = GroupBySteps::of($groupBy, config()); - static::assertCount(2, $steps); - static::assertInstanceOf(BucketingProcessor::class, $steps[0]); - static::assertInstanceOf(GroupByAggregationProcessor::class, $steps[1]); + static::assertCount(1, $steps); + static::assertInstanceOf(GroupByAggregationProcessor::class, $steps[0]); } public function test_groups_across_multiple_batches(): void @@ -78,9 +76,9 @@ public function test_groups_across_multiple_batches(): void $result = GroupByContext::aggregate( $groupBy, flow_context(config_builder()->groupBy(hash_group_by()->storage(new MemoryBuckets()))->build()), - rows(schema(str_schema('category'), int_schema('amount')), row(['category' => 'a', 'amount' => 10])), - rows(schema(str_schema('category'), int_schema('amount')), row(['category' => 'a', 'amount' => 20])), - rows(schema(str_schema('category'), int_schema('amount')), row(['category' => 'b', 'amount' => 15])), + array_to_rows([['category' => 'a', 'amount' => 10]], schema(str_schema('category'), int_schema('amount'))), + array_to_rows([['category' => 'a', 'amount' => 20]], schema(str_schema('category'), int_schema('amount'))), + array_to_rows([['category' => 'b', 'amount' => 15]], schema(str_schema('category'), int_schema('amount'))), ); $aggregated = []; @@ -106,11 +104,13 @@ public function test_aggregates_each_bucket_separately_with_single_bucket(): voi flow_context( config_builder()->groupBy(hash_group_by()->storage(new MemoryBuckets())->bucketsCount(1))->build(), ), - rows( + array_to_rows( + [ + ['category' => 'a', 'amount' => 10], + ['category' => 'b', 'amount' => 15], + ['category' => 'a', 'amount' => 20], + ], schema(str_schema('category'), int_schema('amount')), - row(['category' => 'a', 'amount' => 10]), - row(['category' => 'b', 'amount' => 15]), - row(['category' => 'a', 'amount' => 20]), ), ); @@ -136,17 +136,20 @@ public function test_aggregates_many_buckets_and_flushes_per_bucket(): void $batches = []; for ($id = 0; $id < 20; $id++) { - $batches[] = rows( + $batches[] = array_to_rows( + [['id' => $id, 'amount' => 1], ['id' => $id, 'amount' => 2]], schema(int_schema('id'), int_schema('amount')), - row(['id' => $id, 'amount' => 1]), - row(['id' => $id, 'amount' => 2]), ); } $result = GroupByContext::aggregate( $groupBy, flow_context( - config_builder()->groupBy(hash_group_by()->storage(new MemoryBuckets())->bucketsCount(4))->build(), + config_builder() + ->groupBy( + hash_group_by()->storage(new MemoryBuckets())->memoryLimit(Unit::fromBytes(1))->bucketsCount(4), + ) + ->build(), ), ...$batches, ); @@ -175,12 +178,13 @@ public function test_removes_storage_buckets_after_last_bucket(): void GroupByContext::aggregate( $groupBy, - flow_context(config_builder()->groupBy(hash_group_by()->storage($storage))->build()), - rows( - schema(str_schema('category'), int_schema('amount')), - row(['category' => 'a', 'amount' => 10]), - row(['category' => 'b', 'amount' => 15]), + flow_context( + config_builder() + ->groupBy(hash_group_by()->storage($storage)->memoryLimit(Unit::fromBytes(1))->batchSize(1)) + ->build(), ), + array_to_rows([['category' => 'a', 'amount' => 10]], schema(str_schema('category'), int_schema('amount'))), + array_to_rows([['category' => 'b', 'amount' => 15]], schema(str_schema('category'), int_schema('amount'))), ); static::assertNotSame([], $storage->readBucketIds()); @@ -195,11 +199,9 @@ public function test_groups_rows_missing_group_column_under_null(): void $result = GroupByContext::aggregate( $groupBy, flow_context(config_builder()->groupBy(hash_group_by()->storage(new MemoryBuckets()))->build()), - rows( + array_to_rows( + [['category' => 'a', 'amount' => 10], ['amount' => 15], ['amount' => 20]], schema(str_schema('category', nullable: true), int_schema('amount')), - row(['category' => 'a', 'amount' => 10]), - row(['amount' => 15]), - row(['amount' => 20]), ), ); @@ -225,18 +227,27 @@ public function test_spills_only_grouping_and_aggregation_columns(): void GroupByContext::aggregate( $groupBy, - flow_context(config_builder()->groupBy(hash_group_by()->storage($storage))->build()), - rows( + flow_context( + config_builder() + ->groupBy(hash_group_by()->storage($storage)->memoryLimit(Unit::fromBytes(1))->batchSize(1)) + ->build(), + ), + array_to_rows( + [['category' => 'a', 'amount' => 10, 'noise' => 'x', 'id' => 1]], + schema(str_schema('category'), int_schema('amount'), str_schema('noise'), int_schema('id')), + ), + array_to_rows( + [['category' => 'b', 'amount' => 15, 'noise' => 'y', 'id' => 2]], schema(str_schema('category'), int_schema('amount'), str_schema('noise'), int_schema('id')), - row(['category' => 'a', 'amount' => 10, 'noise' => 'x', 'id' => 1]), - row(['category' => 'b', 'amount' => 15, 'noise' => 'y', 'id' => 2]), ), ); + static::assertNotSame([], $storage->appendedRows()); + foreach ($storage->appendedRows() as $batches) { foreach ($batches as $batch) { - foreach ($batch as $spilledRow) { - static::assertSame(['category', 'amount'], $spilledRow->names()); + foreach ($batch->toArray() as $spilledRow) { + static::assertSame(['category', 'amount'], array_keys($spilledRow)); } } } @@ -245,29 +256,38 @@ public function test_spills_only_grouping_and_aggregation_columns(): void public function test_spills_all_columns_when_aggregator_references_cannot_be_enumerated(): void { $groupBy = new GroupBy(ref('category')); - $groupBy->aggregate(sum(ref('amount'), exact: ref('flag'))); + $groupBy->aggregate(new UnenumerableSum(ref('amount'))); $storage = new SpyBucketsStorage(new MemoryBuckets()); $result = GroupByContext::aggregate( $groupBy, - flow_context(config_builder()->groupBy(hash_group_by()->storage($storage))->build()), - rows( + flow_context( + config_builder() + ->groupBy(hash_group_by()->storage($storage)->memoryLimit(Unit::fromBytes(1))->batchSize(1)) + ->build(), + ), + array_to_rows( + [['category' => 'a', 'amount' => 0.5, 'flag' => true]], + schema(str_schema('category'), float_schema('amount'), bool_schema('flag')), + ), + array_to_rows( + [['category' => 'a', 'amount' => 0.25, 'flag' => true]], schema(str_schema('category'), float_schema('amount'), bool_schema('flag')), - row(['category' => 'a', 'amount' => 0.1, 'flag' => true]), - row(['category' => 'a', 'amount' => 0.2, 'flag' => true]), ), ); + static::assertNotSame([], $storage->appendedRows()); + foreach ($storage->appendedRows() as $batches) { foreach ($batches as $batch) { - foreach ($batch as $spilledRow) { - static::assertSame(['category', 'amount', 'flag'], $spilledRow->names()); + foreach ($batch->toArray() as $spilledRow) { + static::assertSame(['category', 'amount', 'flag'], array_keys($spilledRow)); } } } - static::assertSame(0.3, $result[0]->first()->get('amount_sum')); + static::assertSame(0.75, $result[0]->column('amount_sum')->value(0)); } public function test_missing_aggregated_column_is_treated_as_null_in_strict_mode(): void @@ -279,14 +299,13 @@ public function test_missing_aggregated_column_is_treated_as_null_in_strict_mode $result = GroupByContext::aggregate( $groupBy, $context, - rows( + array_to_rows( + [['category' => 'a', 'amount' => 10], ['category' => 'a']], schema(str_schema('category'), int_schema('amount', nullable: true)), - row(['category' => 'a', 'amount' => 10]), - row(['category' => 'a']), ), ); - static::assertSame(10.0, $result[0]->first()->get('amount_sum')); + static::assertSame(10.0, $result[0]->column('amount_sum')->value(0)); } public function test_handles_empty_input(): void diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/GroupBy/PivotAggregationTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/GroupBy/PivotAggregationTest.php index cb68d9f35b..993fa70c54 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Unit/GroupBy/PivotAggregationTest.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/GroupBy/PivotAggregationTest.php @@ -12,14 +12,13 @@ use Flow\ETL\Tests\FlowTestCase; use Generator; +use function Flow\ETL\DSL\array_to_rows; use function Flow\ETL\DSL\config; use function Flow\ETL\DSL\flow_context; use function Flow\ETL\DSL\int_schema; use function Flow\ETL\DSL\min; use function Flow\ETL\DSL\pivot_values; use function Flow\ETL\DSL\ref; -use function Flow\ETL\DSL\row; -use function Flow\ETL\DSL\rows; use function Flow\ETL\DSL\schema; use function Flow\ETL\DSL\str_schema; use function Flow\ETL\DSL\sum; @@ -46,8 +45,8 @@ public function test_a_bound_aggregation_pivots_without_reading_a_schema_from_th /** @var list $batches */ $batches = iterator_to_array((new PivotAggregation())->aggregateBound( (static function () use ($input): Generator { - yield rows($input, row(['product' => 'Banana', 'country' => 'USA', 'amount' => 30])); - yield rows($input, row(['product' => 'Banana', 'country' => 'USA', 'amount' => 10])); + yield array_to_rows([['product' => 'Banana', 'country' => 'USA', 'amount' => 30]], $input); + yield array_to_rows([['product' => 'Banana', 'country' => 'USA', 'amount' => 10]], $input); })(), flow_context(config()), $groupBy, @@ -86,12 +85,11 @@ public function test_it_splits_the_result_into_batches_of_the_requested_size(): 2, iterator_to_array((new PivotAggregation(2))->aggregateBound( (static function () use ($input): Generator { - yield rows( - $input, - row(['product' => 'Banana', 'country' => 'USA', 'amount' => 1]), - row(['product' => 'Apple', 'country' => 'USA', 'amount' => 2]), - row(['product' => 'Cherry', 'country' => 'USA', 'amount' => 3]), - ); + yield array_to_rows([ + ['product' => 'Banana', 'country' => 'USA', 'amount' => 1], + ['product' => 'Apple', 'country' => 'USA', 'amount' => 2], + ['product' => 'Cherry', 'country' => 'USA', 'amount' => 3], + ], $input); })(), flow_context(config()), $groupBy, diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/GroupBy/PivotedTableTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/GroupBy/PivotedTableTest.php index 17e1213942..a363c59075 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Unit/GroupBy/PivotedTableTest.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/GroupBy/PivotedTableTest.php @@ -10,12 +10,11 @@ use Flow\ETL\Tests\Mother\PivotedTableMother; use function array_map; +use function Flow\ETL\DSL\array_to_rows; use function Flow\ETL\DSL\config; use function Flow\ETL\DSL\flow_context; use function Flow\ETL\DSL\min; use function Flow\ETL\DSL\ref; -use function Flow\ETL\DSL\row; -use function Flow\ETL\DSL\rows; use function Flow\ETL\DSL\sum; final class PivotedTableTest extends FlowTestCase @@ -24,18 +23,15 @@ public function test_it_accumulates_rows_of_the_same_key_across_batches(): void { $table = PivotedTableMother::of(sum(ref('amount')), 'USA', 'PL'); - $table->accumulate( - rows( - PivotedTableMother::schema(), - row(['product' => 'Banana', 'country' => 'USA', 'amount' => 10]), - row(['product' => 'Banana', 'country' => 'PL', 'amount' => 20]), - ), - flow_context(config()), - ); - $table->accumulate( - rows(PivotedTableMother::schema(), row(['product' => 'Banana', 'country' => 'USA', 'amount' => 5])), - flow_context(config()), - ); + $table->accumulate(array_to_rows([ + ['product' => 'Banana', 'country' => 'USA', 'amount' => 10], + ['product' => 'Banana', 'country' => 'PL', 'amount' => 20], + ], PivotedTableMother::schema()), flow_context(config())); + $table->accumulate(array_to_rows([[ + 'product' => 'Banana', + 'country' => 'USA', + 'amount' => 5, + ]], PivotedTableMother::schema()), flow_context(config())); /** @var list $batches */ $batches = iterator_to_array($table->flush(1000, flow_context(config()))); @@ -48,10 +44,11 @@ public function test_a_pivot_value_never_seen_becomes_null(): void { $table = PivotedTableMother::of(sum(ref('amount')), 'USA', 'PL'); - $table->accumulate( - rows(PivotedTableMother::schema(), row(['product' => 'Banana', 'country' => 'USA', 'amount' => 10])), - flow_context(config()), - ); + $table->accumulate(array_to_rows([[ + 'product' => 'Banana', + 'country' => 'USA', + 'amount' => 10, + ]], PivotedTableMother::schema()), flow_context(config())); /** @var list $batches */ $batches = iterator_to_array($table->flush(1000, flow_context(config()))); @@ -64,10 +61,11 @@ public function test_a_null_pivot_value_keeps_the_group_row_and_feeds_no_column( $input = PivotedTableMother::schema(nullableCountry: true); $table = PivotedTableMother::boundTo($input, sum(ref('amount')), 'USA'); - $table->accumulate( - rows($input, row(['product' => 'Banana', 'country' => null, 'amount' => 10])), - flow_context(config()), - ); + $table->accumulate(array_to_rows([[ + 'product' => 'Banana', + 'country' => null, + 'amount' => 10, + ]], $input), flow_context(config())); /** @var list $batches */ $batches = iterator_to_array($table->flush(1000, flow_context(config()))); @@ -79,14 +77,10 @@ public function test_a_pivot_value_outside_the_declared_values_is_discarded(): v { $table = PivotedTableMother::of(sum(ref('amount')), 'USA'); - $table->accumulate( - rows( - PivotedTableMother::schema(), - row(['product' => 'Banana', 'country' => 'USA', 'amount' => 30]), - row(['product' => 'Banana', 'country' => 'DE', 'amount' => 99]), - ), - flow_context(config()), - ); + $table->accumulate(array_to_rows([ + ['product' => 'Banana', 'country' => 'USA', 'amount' => 30], + ['product' => 'Banana', 'country' => 'DE', 'amount' => 99], + ], PivotedTableMother::schema()), flow_context(config())); /** @var list $batches */ $batches = iterator_to_array($table->flush(1000, flow_context(config()))); @@ -99,15 +93,11 @@ public function test_it_splits_the_flushed_rows_into_batches_of_the_requested_si { $table = PivotedTableMother::of(sum(ref('amount')), 'USA'); - $table->accumulate( - rows( - PivotedTableMother::schema(), - row(['product' => 'Banana', 'country' => 'USA', 'amount' => 10]), - row(['product' => 'Apple', 'country' => 'USA', 'amount' => 20]), - row(['product' => 'Cherry', 'country' => 'USA', 'amount' => 30]), - ), - flow_context(config()), - ); + $table->accumulate(array_to_rows([ + ['product' => 'Banana', 'country' => 'USA', 'amount' => 10], + ['product' => 'Apple', 'country' => 'USA', 'amount' => 20], + ['product' => 'Cherry', 'country' => 'USA', 'amount' => 30], + ], PivotedTableMother::schema()), flow_context(config())); static::assertSame( [2, 1], @@ -139,14 +129,10 @@ public function test_an_argument_typed_aggregate_keeps_its_resolved_column_type( { $table = PivotedTableMother::of(min(ref('amount')), 'USA'); - $table->accumulate( - rows( - PivotedTableMother::schema(), - row(['product' => 'Banana', 'country' => 'USA', 'amount' => 30]), - row(['product' => 'Banana', 'country' => 'USA', 'amount' => 10]), - ), - flow_context(config()), - ); + $table->accumulate(array_to_rows([ + ['product' => 'Banana', 'country' => 'USA', 'amount' => 30], + ['product' => 'Banana', 'country' => 'USA', 'amount' => 10], + ], PivotedTableMother::schema()), flow_context(config())); /** @var list $batches */ $batches = iterator_to_array($table->flush(1000, flow_context(config()))); diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/GroupByTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/GroupByTest.php index 28b472bb46..4a9e5e947e 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Unit/GroupByTest.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/GroupByTest.php @@ -4,29 +4,39 @@ namespace Flow\ETL\Tests\Unit; +use DateTimeImmutable; use Flow\ETL\Exception\InvalidArgumentException; use Flow\ETL\Exception\RuntimeException; use Flow\ETL\Exception\SchemaDefinitionNotFoundException; use Flow\ETL\GroupBy; +use Flow\ETL\GroupBy\GroupKey; use Flow\ETL\GroupBy\PivotAggregation; use Flow\ETL\GroupBy\PivotShape; use Flow\ETL\Tests\Context\GroupByContext; use Flow\ETL\Tests\FlowTestCase; +use Flow\Types\Value\Json; +use Flow\Types\Value\Uuid; use Generator; +use function array_map; +use function Flow\ETL\DSL\array_to_rows; use function Flow\ETL\DSL\config; use function Flow\ETL\DSL\count; +use function Flow\ETL\DSL\date_schema; +use function Flow\ETL\DSL\datetime_schema; use function Flow\ETL\DSL\float_schema; use function Flow\ETL\DSL\flow_context; use function Flow\ETL\DSL\int_schema; +use function Flow\ETL\DSL\json_schema; use function Flow\ETL\DSL\min; use function Flow\ETL\DSL\pivot_values; use function Flow\ETL\DSL\ref; -use function Flow\ETL\DSL\row; use function Flow\ETL\DSL\rows; use function Flow\ETL\DSL\schema; use function Flow\ETL\DSL\str_schema; use function Flow\ETL\DSL\sum; +use function Flow\ETL\DSL\uuid_schema; +use function iterator_to_array; final class GroupByTest extends FlowTestCase { @@ -38,11 +48,9 @@ public function test_group_by_missing_entry(): void $result = GroupByContext::aggregated( $groupBy, flow_context(config()), - rows( + array_to_rows( + [['type' => 'a'], ['not-type' => 'b'], ['type' => 'c']], schema(str_schema('type', nullable: true), str_schema('not-type', nullable: true)), - row(['type' => 'a']), - row(['not-type' => 'b']), - row(['type' => 'c']), ), ); @@ -65,13 +73,15 @@ public function test_group_by_with_aggregation(): void $result = GroupByContext::aggregated( $group, flow_context(config()), - rows( + array_to_rows( + [ + ['id' => 1, 'type' => 'a'], + ['id' => 2, 'type' => 'b'], + ['id' => 3, 'type' => 'c'], + ['id' => 4, 'type' => 'a'], + ['id' => 5, 'type' => 'd'], + ], schema(int_schema('id'), str_schema('type')), - row(['id' => 1, 'type' => 'a']), - row(['id' => 2, 'type' => 'b']), - row(['id' => 3, 'type' => 'c']), - row(['id' => 4, 'type' => 'a']), - row(['id' => 5, 'type' => 'd']), ), ); @@ -116,9 +126,9 @@ public function test_a_pivot_declares_its_output_schema(): void $schema = iterator_to_array((new PivotAggregation())->aggregate( (static function (): Generator { - yield rows( + yield array_to_rows( + [['product' => 'Banana', 'country' => 'USA', 'amount' => 1000]], schema(str_schema('product'), str_schema('country'), int_schema('amount')), - row(['product' => 'Banana', 'country' => 'USA', 'amount' => 1000]), ); })(), flow_context(config()), @@ -198,10 +208,12 @@ public function test_an_unbound_pivot_resolves_an_argument_typed_aggregate(): vo $result = iterator_to_array((new PivotAggregation())->aggregate( (static function (): Generator { - yield rows( + yield array_to_rows( + [ + ['product' => 'Banana', 'country' => 'USA', 'amount' => 30], + ['product' => 'Banana', 'country' => 'USA', 'amount' => 10], + ], schema(str_schema('product'), str_schema('country'), int_schema('amount')), - row(['product' => 'Banana', 'country' => 'USA', 'amount' => 30]), - row(['product' => 'Banana', 'country' => 'USA', 'amount' => 10]), ); })(), flow_context(config()), @@ -230,10 +242,12 @@ public function test_a_falsy_pivot_value_keeps_its_column(): void $result = iterator_to_array((new PivotAggregation())->aggregate( (static function (): Generator { - yield rows( + yield array_to_rows( + [ + ['product' => 'Banana', 'country' => '0', 'amount' => 7], + ['product' => 'Banana', 'country' => 'USA', 'amount' => 3], + ], schema(str_schema('product'), str_schema('country'), int_schema('amount')), - row(['product' => 'Banana', 'country' => '0', 'amount' => 7]), - row(['product' => 'Banana', 'country' => 'USA', 'amount' => 3]), ); })(), flow_context(config()), @@ -251,7 +265,13 @@ public function test_group_by_with_pivoting(): void $group->pivot(ref('country'), pivot_values('Canada', 'China', 'Mexico', 'USA')); static::assertEquals( - rows( + array_to_rows( + [ + ['product' => 'Banana', 'Canada' => 2000.0, 'China' => 400.0, 'Mexico' => null, 'USA' => 1000.0], + ['product' => 'Beans', 'Canada' => null, 'China' => 1500.0, 'Mexico' => 2000.0, 'USA' => 1600.0], + ['product' => 'Carrots', 'Canada' => 2000.0, 'China' => 1200.0, 'Mexico' => null, 'USA' => 1500.0], + ['product' => 'Orange', 'Canada' => null, 'China' => 4000.0, 'Mexico' => null, 'USA' => 4000.0], + ], schema( str_schema('product'), // PivotSchema declares every pivot column optional: the yield loop writes null @@ -261,28 +281,26 @@ public function test_group_by_with_pivoting(): void float_schema('Mexico', nullable: true), float_schema('USA', nullable: true), ), - row(['product' => 'Banana', 'Canada' => 2000.0, 'China' => 400.0, 'Mexico' => null, 'USA' => 1000.0]), - row(['product' => 'Beans', 'Canada' => null, 'China' => 1500.0, 'Mexico' => 2000.0, 'USA' => 1600.0]), - row(['product' => 'Carrots', 'Canada' => 2000.0, 'China' => 1200.0, 'Mexico' => null, 'USA' => 1500.0]), - row(['product' => 'Orange', 'Canada' => null, 'China' => 4000.0, 'Mexico' => null, 'USA' => 4000.0]), ), GroupByContext::aggregated( $group, flow_context(config()), - rows( + array_to_rows( + [ + ['product' => 'Banana', 'amount' => 1000, 'country' => 'USA'], + ['product' => 'Carrots', 'amount' => 1500, 'country' => 'USA'], + ['product' => 'Beans', 'amount' => 1600, 'country' => 'USA'], + ['product' => 'Orange', 'amount' => 2000, 'country' => 'USA'], + ['product' => 'Orange', 'amount' => 2000, 'country' => 'USA'], + ['product' => 'Banana', 'amount' => 400, 'country' => 'China'], + ['product' => 'Carrots', 'amount' => 1200, 'country' => 'China'], + ['product' => 'Beans', 'amount' => 1500, 'country' => 'China'], + ['product' => 'Orange', 'amount' => 4000, 'country' => 'China'], + ['product' => 'Banana', 'amount' => 2000, 'country' => 'Canada'], + ['product' => 'Carrots', 'amount' => 2000, 'country' => 'Canada'], + ['product' => 'Beans', 'amount' => 2000, 'country' => 'Mexico'], + ], schema(str_schema('product'), int_schema('amount'), str_schema('country')), - row(['product' => 'Banana', 'amount' => 1000, 'country' => 'USA']), - row(['product' => 'Carrots', 'amount' => 1500, 'country' => 'USA']), - row(['product' => 'Beans', 'amount' => 1600, 'country' => 'USA']), - row(['product' => 'Orange', 'amount' => 2000, 'country' => 'USA']), - row(['product' => 'Orange', 'amount' => 2000, 'country' => 'USA']), - row(['product' => 'Banana', 'amount' => 400, 'country' => 'China']), - row(['product' => 'Carrots', 'amount' => 1200, 'country' => 'China']), - row(['product' => 'Beans', 'amount' => 1500, 'country' => 'China']), - row(['product' => 'Orange', 'amount' => 4000, 'country' => 'China']), - row(['product' => 'Banana', 'amount' => 2000, 'country' => 'Canada']), - row(['product' => 'Carrots', 'amount' => 2000, 'country' => 'Canada']), - row(['product' => 'Beans', 'amount' => 2000, 'country' => 'Mexico']), ), )->sortBy(ref('product')), ); @@ -295,18 +313,19 @@ public function test_group_by_with_pivoting_with_null_pivot_column(): void $group->pivot(ref('country'), pivot_values('USA')); static::assertEquals( - rows( + array_to_rows( + [['product' => 'Apple', 'USA' => null], ['product' => 'Banana', 'USA' => 1000.0]], schema(str_schema('product'), float_schema('USA', nullable: true)), - row(['product' => 'Apple', 'USA' => null]), - row(['product' => 'Banana', 'USA' => 1000.0]), ), GroupByContext::aggregated( $group, flow_context(config()), - rows( + array_to_rows( + [ + ['product' => 'Banana', 'country' => 'USA', 'amount' => 1000], + ['product' => 'Apple', 'country' => null, 'amount' => 400], + ], schema(str_schema('product'), str_schema('country', nullable: true), int_schema('amount')), - row(['product' => 'Banana', 'country' => 'USA', 'amount' => 1000]), - row(['product' => 'Apple', 'country' => null, 'amount' => 400]), ), )->sortBy(ref('product')), ); @@ -327,7 +346,7 @@ public function test_a_group_column_absent_from_the_schema_is_refused_at_bind(): GroupByContext::aggregated( $groupBy, flow_context(config()), - rows(schema(str_schema('type')), row(['type' => 'a']), row(['type' => 'b'])), + array_to_rows([['type' => 'a'], ['type' => 'b']], schema(str_schema('type'))), ); } @@ -357,22 +376,114 @@ public function test_a_nullable_group_key_stays_nullable_in_the_output_schema(): ); } - public function test_key_values_throws_when_a_row_lacks_a_not_null_key(): void + public function test_keys_throws_when_a_row_lacks_a_not_null_key(): void { $this->expectException(InvalidArgumentException::class); $this->expectExceptionMessage('Column "country" does not exist.'); - (new GroupBy('country'))->keyValues(row(['age' => 20]), schema(str_schema('country'), int_schema('age'))); + (new GroupBy('country'))->keys( + array_to_rows([['age' => 20]], schema(int_schema('age'))), + schema(str_schema('country'), int_schema('age')), + ); } - public function test_key_values_substitutes_null_when_a_row_lacks_a_nullable_key(): void + public function test_keys_substitutes_null_when_a_row_lacks_a_nullable_key(): void { + $keys = (new GroupBy('country'))->keys( + array_to_rows([['age' => 20], ['age' => 30]], schema(int_schema('age'))), + schema(str_schema('country', nullable: true), int_schema('age')), + ); + static::assertSame( - ['country' => null], - iterator_to_array((new GroupBy('country'))->keyValues( - row(['age' => 20]), - schema(str_schema('country', nullable: true), int_schema('age')), - )), + [['country' => null], ['country' => null]], + array_map(static fn(GroupKey $key): array => iterator_to_array($key), $keys), + ); + } + + public function test_keys_reads_one_key_per_row(): void + { + $keys = (new GroupBy('country', 'age'))->keys( + array_to_rows( + [['country' => 'PL', 'age' => 20], ['country' => 'DE', 'age' => 30]], + schema(str_schema('country'), int_schema('age')), + ), + schema(str_schema('country'), int_schema('age')), + ); + + static::assertSame( + [['country' => 'PL', 'age' => 20], ['country' => 'DE', 'age' => 30]], + array_map(static fn(GroupKey $key): array => iterator_to_array($key), $keys), ); } + + public function test_keys_of_an_empty_batch_are_empty(): void + { + static::assertSame( + [], + (new GroupBy('country'))->keys(rows(schema(int_schema('age'))), schema(str_schema('country'))), + ); + } + + public function test_equal_datetime_instants_in_other_zones_are_one_group(): void + { + $rows = array_to_rows([ + ['at' => new DateTimeImmutable('2026-01-02 10:00:00 UTC')], + ['at' => new DateTimeImmutable('2026-01-02 11:00:00 +01:00')], + ], schema(datetime_schema('at'))); + $keys = (new GroupBy('at'))->keys($rows, $rows->schema()); + + static::assertSame((string) $keys[0], (string) $keys[1]); + } + + public function test_a_date_and_an_equal_datetime_instant_share_a_key(): void + { + $dates = array_to_rows([['at' => new DateTimeImmutable('2026-01-02')]], schema(date_schema('at'))); + $datetimes = array_to_rows([[ + 'at' => new DateTimeImmutable('2026-01-02 00:00:00 UTC'), + ]], schema(datetime_schema('at'))); + + static::assertSame( + (string) (new GroupBy('at'))->keys($dates, $dates->schema())[0], + (string) (new GroupBy('at'))->keys($datetimes, $datetimes->schema())[0], + ); + } + + public function test_uuid_and_json_keys_group_equal_values_and_keep_their_values(): void + { + $rows = array_to_rows( + [ + ['id' => 'f47ac10b-58cc-4372-a567-0e02b2c3d479', 'doc' => '{"a":1}'], + ['id' => 'f47ac10b-58cc-4372-a567-0e02b2c3d479', 'doc' => '{"a":1}'], + ['id' => '00000000-0000-4000-8000-000000000000', 'doc' => '{"a":1}'], + ], + schema(uuid_schema('id'), json_schema('doc')), + ); + $keys = (new GroupBy('id', 'doc'))->keys($rows, $rows->schema()); + + static::assertSame((string) $keys[0], (string) $keys[1]); + static::assertNotSame((string) $keys[0], (string) $keys[2]); + static::assertEquals( + ['id' => new Uuid('f47ac10b-58cc-4372-a567-0e02b2c3d479'), 'doc' => new Json('{"a":1}')], + iterator_to_array($keys[0]), + ); + } + + public function test_keys_of_a_batch_that_lacks_a_nullable_key_match_a_present_null(): void + { + $input = schema(str_schema('a'), str_schema('b', nullable: true)); + $lacking = (new GroupBy('a', 'b'))->keys(array_to_rows([['a' => 'x']], schema(str_schema('a'))), $input); + $present = (new GroupBy('a', 'b'))->keys(array_to_rows([['a' => 'x', 'b' => null]], $input), $input); + + static::assertSame(['a' => 'x', 'b' => null], iterator_to_array($lacking[0])); + static::assertSame((string) $present[0], (string) $lacking[0]); + } + + public function test_keys_of_swapped_absences_do_not_collide(): void + { + $input = schema(str_schema('a', nullable: true), str_schema('b', nullable: true)); + $lacksB = (new GroupBy('a', 'b'))->keys(array_to_rows([['a' => 'x']], schema(str_schema('a'))), $input); + $lacksA = (new GroupBy('a', 'b'))->keys(array_to_rows([['b' => 'x']], schema(str_schema('b'))), $input); + + static::assertNotSame((string) $lacksB[0], (string) $lacksA[0]); + } } diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Join/Comparison/AllTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Join/Comparison/AllTest.php index 20cd70e31d..f53230595a 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Unit/Join/Comparison/AllTest.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Join/Comparison/AllTest.php @@ -4,33 +4,61 @@ namespace Flow\ETL\Tests\Unit\Join\Comparison; -use Flow\ETL\Join\Comparison; use Flow\ETL\Join\Comparison\All; +use Flow\ETL\Join\Comparison\Equal; +use Flow\ETL\Tests\Double\ConstantComparison; use Flow\ETL\Tests\FlowTestCase; -use function Flow\ETL\DSL\row; +use function Flow\ETL\DSL\array_to_rows; +use function Flow\ETL\DSL\int_schema; +use function Flow\ETL\DSL\schema; final class AllTest extends FlowTestCase { public function test_failure(): void { - $comparison1 = self::createStub(Comparison::class); - $comparison1->method('compare')->willReturn(true); + $comparison1 = new ConstantComparison(true); - $comparison2 = self::createStub(Comparison::class); - $comparison2->method('compare')->willReturn(false); + $comparison2 = new ConstantComparison(false); - static::assertFalse((new All($comparison1, $comparison2))->compare(row(['id' => 1]), row(['id' => 2]))); + static::assertSame( + [false], + (new All($comparison1, $comparison2))->compare( + array_to_rows([['id' => 1]], schema(int_schema('id'))), + array_to_rows([['id' => 2]], schema(int_schema('id'))), + ), + ); } public function test_success(): void { - $comparison1 = self::createStub(Comparison::class); - $comparison1->method('compare')->willReturn(true); + $comparison1 = new ConstantComparison(true); - $comparison2 = self::createStub(Comparison::class); - $comparison2->method('compare')->willReturn(true); + $comparison2 = new ConstantComparison(true); - static::assertTrue((new All($comparison1, $comparison2))->compare(row(['id' => 1]), row(['id' => 2]))); + static::assertSame( + [true], + (new All($comparison1, $comparison2))->compare( + array_to_rows([['id' => 1]], schema(int_schema('id'))), + array_to_rows([['id' => 2]], schema(int_schema('id'))), + ), + ); + } + + public function test_combines_children_element_wise(): void + { + static::assertSame( + [true, false, false], + (new All(new Equal('a', 'a'), new Equal('b', 'b')))->compare( + array_to_rows( + [['a' => 1, 'b' => 1], ['a' => 1, 'b' => 2], ['a' => 3, 'b' => 3]], + schema(int_schema('a'), int_schema('b')), + ), + array_to_rows( + [['a' => 1, 'b' => 1], ['a' => 1, 'b' => 9], ['a' => 9, 'b' => 9]], + schema(int_schema('a'), int_schema('b')), + ), + ), + ); } } diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Join/Comparison/AnyTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Join/Comparison/AnyTest.php index 1808dd0a2f..567acd952d 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Unit/Join/Comparison/AnyTest.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Join/Comparison/AnyTest.php @@ -4,33 +4,61 @@ namespace Flow\ETL\Tests\Unit\Join\Comparison; -use Flow\ETL\Join\Comparison; use Flow\ETL\Join\Comparison\Any; +use Flow\ETL\Join\Comparison\Equal; +use Flow\ETL\Tests\Double\ConstantComparison; use Flow\ETL\Tests\FlowTestCase; -use function Flow\ETL\DSL\row; +use function Flow\ETL\DSL\array_to_rows; +use function Flow\ETL\DSL\int_schema; +use function Flow\ETL\DSL\schema; final class AnyTest extends FlowTestCase { public function test_failure(): void { - $comparison1 = self::createStub(Comparison::class); - $comparison1->method('compare')->willReturn(false); + $comparison1 = new ConstantComparison(false); - $comparison2 = self::createStub(Comparison::class); - $comparison2->method('compare')->willReturn(false); + $comparison2 = new ConstantComparison(false); - static::assertFalse((new Any($comparison1, $comparison2))->compare(row(['id' => 1]), row(['id' => 2]))); + static::assertSame( + [false], + (new Any($comparison1, $comparison2))->compare( + array_to_rows([['id' => 1]], schema(int_schema('id'))), + array_to_rows([['id' => 2]], schema(int_schema('id'))), + ), + ); } public function test_success(): void { - $comparison1 = self::createStub(Comparison::class); - $comparison1->method('compare')->willReturn(true); + $comparison1 = new ConstantComparison(true); - $comparison2 = self::createStub(Comparison::class); - $comparison2->method('compare')->willReturn(false); + $comparison2 = new ConstantComparison(false); - static::assertTrue((new Any($comparison1, $comparison2))->compare(row(['id' => 1]), row(['id' => 2]))); + static::assertSame( + [true], + (new Any($comparison1, $comparison2))->compare( + array_to_rows([['id' => 1]], schema(int_schema('id'))), + array_to_rows([['id' => 2]], schema(int_schema('id'))), + ), + ); + } + + public function test_combines_children_element_wise(): void + { + static::assertSame( + [true, true, false], + (new Any(new Equal('a', 'a'), new Equal('b', 'b')))->compare( + array_to_rows( + [['a' => 1, 'b' => 1], ['a' => 1, 'b' => 2], ['a' => 3, 'b' => 3]], + schema(int_schema('a'), int_schema('b')), + ), + array_to_rows( + [['a' => 1, 'b' => 1], ['a' => 1, 'b' => 9], ['a' => 9, 'b' => 9]], + schema(int_schema('a'), int_schema('b')), + ), + ), + ); } } diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Join/Comparison/EqualTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Join/Comparison/EqualTest.php index d0a32ae599..0b6b7a8418 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Unit/Join/Comparison/EqualTest.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Join/Comparison/EqualTest.php @@ -8,58 +8,118 @@ use Flow\ETL\Join\Comparison\Equal; use Flow\ETL\Tests\FlowTestCase; -use function Flow\ETL\DSL\row; +use function Flow\ETL\DSL\array_to_rows; +use function Flow\ETL\DSL\datetime_schema; +use function Flow\ETL\DSL\float_schema; +use function Flow\ETL\DSL\int_schema; +use function Flow\ETL\DSL\schema; +use function Flow\ETL\DSL\str_schema; final class EqualTest extends FlowTestCase { public function test_failure(): void { - static::assertFalse((new Equal('datetime', 'datetime'))->compare( - row(['datetime' => new DateTimeImmutable('2022-10-01 00:00:00')]), - row(['datetime' => new DateTimeImmutable('2022-10-01 01:00:00')]), - )); + static::assertSame( + [false], + (new Equal('datetime', 'datetime'))->compare( + array_to_rows([[ + 'datetime' => new DateTimeImmutable('2022-10-01 00:00:00'), + ]], schema(datetime_schema('datetime'))), + array_to_rows([[ + 'datetime' => new DateTimeImmutable('2022-10-01 01:00:00'), + ]], schema(datetime_schema('datetime'))), + ), + ); } public function test_null_is_not_equal_to_null(): void { - static::assertFalse((new Equal('id', 'id'))->compare(row(['id' => null]), row(['id' => null]))); + static::assertSame( + [false], + (new Equal('id', 'id'))->compare( + array_to_rows([['id' => null]], schema(str_schema('id', nullable: true))), + array_to_rows([['id' => null]], schema(str_schema('id', nullable: true))), + ), + ); } public function test_null_is_not_equal_to_value(): void { - static::assertFalse((new Equal('id', 'id'))->compare(row(['id' => null]), row(['id' => 1]))); - static::assertFalse((new Equal('id', 'id'))->compare(row(['id' => 1]), row(['id' => null]))); + static::assertSame( + [false], + (new Equal('id', 'id'))->compare( + array_to_rows([['id' => null]], schema(int_schema('id', nullable: true))), + array_to_rows([['id' => 1]], schema(int_schema('id'))), + ), + ); + static::assertSame( + [false], + (new Equal('id', 'id'))->compare( + array_to_rows([['id' => 1]], schema(int_schema('id'))), + array_to_rows([['id' => null]], schema(int_schema('id', nullable: true))), + ), + ); } public function test_object_and_scalar_are_not_equal(): void { - static::assertFalse((new Equal('id', 'id'))->compare( - row(['id' => 'f47ac10b-58cc-4372-a567-0e02b2c3d479']), - row(['id' => 1]), - )); + static::assertSame( + [false], + (new Equal('id', 'id'))->compare( + array_to_rows([['id' => 'f47ac10b-58cc-4372-a567-0e02b2c3d479']], schema(str_schema('id'))), + array_to_rows([['id' => 1]], schema(int_schema('id'))), + ), + ); } public function test_uuid_objects_with_different_values_are_not_equal(): void { - static::assertFalse((new Equal('id', 'id'))->compare( - row(['id' => 'f47ac10b-58cc-4372-a567-0e02b2c3d479']), - row(['id' => '00000000-0000-4000-8000-000000000000']), - )); + static::assertSame( + [false], + (new Equal('id', 'id'))->compare( + array_to_rows([['id' => 'f47ac10b-58cc-4372-a567-0e02b2c3d479']], schema(str_schema('id'))), + array_to_rows([['id' => '00000000-0000-4000-8000-000000000000']], schema(str_schema('id'))), + ), + ); } public function test_uuid_objects_with_same_value_are_equal(): void { - static::assertTrue((new Equal('id', 'id'))->compare( - row(['id' => 'f47ac10b-58cc-4372-a567-0e02b2c3d479']), - row(['id' => 'f47ac10b-58cc-4372-a567-0e02b2c3d479']), - )); + static::assertSame( + [true], + (new Equal('id', 'id'))->compare( + array_to_rows([['id' => 'f47ac10b-58cc-4372-a567-0e02b2c3d479']], schema(str_schema('id'))), + array_to_rows([['id' => 'f47ac10b-58cc-4372-a567-0e02b2c3d479']], schema(str_schema('id'))), + ), + ); } public function test_success(): void { - static::assertTrue((new Equal('datetime', 'datetime'))->compare( - row(['datetime' => $datetime = new DateTimeImmutable('2022-10-01 00:00:00')]), - row(['datetime' => $datetime]), - )); + static::assertSame( + [true], + (new Equal('datetime', 'datetime'))->compare( + array_to_rows([[ + 'datetime' => $datetime = new DateTimeImmutable('2022-10-01 00:00:00'), + ]], schema(datetime_schema('datetime'))), + array_to_rows([['datetime' => $datetime]], schema(datetime_schema('datetime'))), + ), + ); + } + + public function test_compares_pair_aligned_batches(): void + { + static::assertSame( + [true, false, false, true], + (new Equal('a', 'b'))->compare( + array_to_rows([ + ['a' => 1], + ['a' => 2], + ['a' => null], + ['a' => 4], + ], schema(int_schema('a', nullable: true))), + array_to_rows([['b' => 1.0], ['b' => 3.0], ['b' => 1.0], ['b' => 4.0]], schema(float_schema('b'))), + ), + ); } } diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Join/Comparison/IdenticalTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Join/Comparison/IdenticalTest.php index dc06266645..b6f2823330 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Unit/Join/Comparison/IdenticalTest.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Join/Comparison/IdenticalTest.php @@ -4,31 +4,117 @@ namespace Flow\ETL\Tests\Unit\Join\Comparison; +use DateTimeImmutable; use Flow\ETL\Join\Comparison\Identical; use Flow\ETL\Tests\FlowTestCase; -use function Flow\ETL\DSL\row; +use function Flow\ETL\DSL\array_to_rows; +use function Flow\ETL\DSL\datetime_schema; +use function Flow\ETL\DSL\int_schema; +use function Flow\ETL\DSL\schema; +use function Flow\ETL\DSL\str_schema; +use function Flow\ETL\DSL\uuid_schema; final class IdenticalTest extends FlowTestCase { public function test_failure(): void { - static::assertFalse((new Identical('id', 'id'))->compare(row(['id' => 1]), row(['id' => 2]))); + static::assertSame( + [false], + (new Identical('id', 'id'))->compare( + array_to_rows([['id' => 1]], schema(int_schema('id'))), + array_to_rows([['id' => 2]], schema(int_schema('id'))), + ), + ); } public function test_null_is_not_identical_to_null(): void { - static::assertFalse((new Identical('id', 'id'))->compare(row(['id' => null]), row(['id' => null]))); + static::assertSame( + [false], + (new Identical('id', 'id'))->compare( + array_to_rows([['id' => null]], schema(str_schema('id', nullable: true))), + array_to_rows([['id' => null]], schema(str_schema('id', nullable: true))), + ), + ); } public function test_null_is_not_identical_to_value(): void { - static::assertFalse((new Identical('id', 'id'))->compare(row(['id' => null]), row(['id' => 1]))); - static::assertFalse((new Identical('id', 'id'))->compare(row(['id' => 1]), row(['id' => null]))); + static::assertSame( + [false], + (new Identical('id', 'id'))->compare( + array_to_rows([['id' => null]], schema(int_schema('id', nullable: true))), + array_to_rows([['id' => 1]], schema(int_schema('id'))), + ), + ); + static::assertSame( + [false], + (new Identical('id', 'id'))->compare( + array_to_rows([['id' => 1]], schema(int_schema('id'))), + array_to_rows([['id' => null]], schema(int_schema('id', nullable: true))), + ), + ); } public function test_success(): void { - static::assertTrue((new Identical('id', 'id'))->compare(row(['id' => 1]), row(['id' => 1]))); + static::assertSame( + [true], + (new Identical('id', 'id'))->compare( + array_to_rows([['id' => 1]], schema(int_schema('id'))), + array_to_rows([['id' => 1]], schema(int_schema('id'))), + ), + ); + } + + public function test_compares_pair_aligned_batches(): void + { + static::assertSame( + [true, false, false], + (new Identical('id', 'id'))->compare( + array_to_rows([['id' => 1], ['id' => 2], ['id' => null]], schema(int_schema('id', nullable: true))), + array_to_rows([['id' => 1], ['id' => 3], ['id' => null]], schema(int_schema('id', nullable: true))), + ), + ); + } + + public function test_equal_datetimes_are_identical(): void + { + static::assertSame( + [true], + (new Identical('at', 'at'))->compare( + array_to_rows([['at' => new DateTimeImmutable('2024-01-01 00:00:00')]], schema(datetime_schema('at'))), + array_to_rows([['at' => new DateTimeImmutable('2024-01-01 00:00:00')]], schema(datetime_schema('at'))), + ), + ); + } + + public function test_equal_uuids_are_identical(): void + { + static::assertSame( + [true, false], + (new Identical('id', 'id'))->compare( + array_to_rows([ + ['id' => '00000000-0000-4000-8000-000000000001'], + ['id' => '00000000-0000-4000-8000-000000000001'], + ], schema(uuid_schema('id'))), + array_to_rows([ + ['id' => '00000000-0000-4000-8000-000000000001'], + ['id' => '00000000-0000-4000-8000-000000000002'], + ], schema(uuid_schema('id'))), + ), + ); + } + + public function test_values_of_different_types_are_not_identical(): void + { + static::assertSame( + [false], + (new Identical('id', 'id'))->compare( + array_to_rows([['id' => 1]], schema(int_schema('id'))), + array_to_rows([['id' => '1']], schema(str_schema('id'))), + ), + ); } } diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Join/ExpressionTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Join/ExpressionTest.php index cb675c6b74..50a583400c 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Unit/Join/ExpressionTest.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Join/ExpressionTest.php @@ -8,8 +8,10 @@ use Flow\ETL\Join\Expression; use Flow\ETL\Tests\FlowTestCase; +use function Flow\ETL\DSL\array_to_rows; use function Flow\ETL\DSL\col; -use function Flow\ETL\DSL\row; +use function Flow\ETL\DSL\int_schema; +use function Flow\ETL\DSL\schema; final class ExpressionTest extends FlowTestCase { @@ -26,6 +28,12 @@ public function test_expression_comparison(): void { $expression = Expression::on(new Equal('id', 'id'), '_'); - static::assertTrue($expression->meet(row(['id' => 1]), row(['id' => 1]))); + static::assertSame( + [true], + $expression->meet( + array_to_rows([['id' => 1]], schema(int_schema('id'))), + array_to_rows([['id' => 1]], schema(int_schema('id'))), + ), + ); } } diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Join/HashJoin/HashTableTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Join/HashJoin/HashTableTest.php index 64c6b168b3..8a444ebb42 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Unit/Join/HashJoin/HashTableTest.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Join/HashJoin/HashTableTest.php @@ -7,46 +7,41 @@ use Flow\ETL\Join\HashJoin\HashTable; use Flow\ETL\Tests\FlowTestCase; -use function Flow\ETL\DSL\row; -use function iterator_to_array; - final class HashTableTest extends FlowTestCase { public function test_candidates_for_matching_hash(): void { $hashTable = new HashTable(); - $hashTable->add('hash-a', row(['user_id' => 1, 'name' => 'Alice'])); - $hashTable->add('hash-b', row(['user_id' => 2, 'name' => 'Bob'])); - - $candidates = $hashTable->candidatesFor('hash-a'); + $hashTable->add('hash-a', 0); + $hashTable->add('hash-b', 1); + $hashTable->add('hash-a', 2); - static::assertCount(1, $candidates); - static::assertSame('Alice', $candidates[0]->get('name')); + static::assertSame([0, 2], $hashTable->candidatesFor('hash-a')); + static::assertSame([1], $hashTable->candidatesFor('hash-b')); } public function test_constant_hash_makes_every_row_a_candidate(): void { $hashTable = new HashTable(); - $hashTable->add('0', row(['user_id' => 1])); - $hashTable->add('0', row(['user_id' => 2])); + $hashTable->add('0', 0); + $hashTable->add('0', 1); - static::assertCount(2, $hashTable->candidatesFor('0')); + static::assertSame([0, 1], $hashTable->candidatesFor('0')); } - public function test_duplicated_rows_are_preserved(): void + public function test_count_is_every_added_row(): void { $hashTable = new HashTable(); - $hashTable->add('hash-a', row(['user_id' => 1, 'name' => 'Alice'])); - $hashTable->add('hash-a', row(['user_id' => 1, 'name' => 'Alice'])); + $hashTable->add('hash-a', 0); + $hashTable->add('hash-a', 1); static::assertSame(2, $hashTable->count()); - static::assertCount(2, $hashTable->candidatesFor('hash-a')); } public function test_no_candidates_for_unknown_hash(): void { $hashTable = new HashTable(); - $hashTable->add('hash-a', row(['user_id' => 1])); + $hashTable->add('hash-a', 0); static::assertSame([], $hashTable->candidatesFor('hash-404')); } @@ -54,24 +49,20 @@ public function test_no_candidates_for_unknown_hash(): void public function test_unmatched_rows_tracking(): void { $hashTable = new HashTable(trackUnmatched: true); - $hashTable->add('hash-a', row(['user_id' => 1, 'name' => 'Alice'])); - $hashTable->add('hash-b', row(['user_id' => 2, 'name' => 'Bob'])); - $hashTable->add('hash-c', row(['user_id' => 3, 'name' => 'Cid'])); + $hashTable->add('hash-a', 0); + $hashTable->add('hash-b', 1); + $hashTable->add('hash-c', 2); $hashTable->matched(1); - $unmatched = iterator_to_array($hashTable->unmatchedRows(), false); - - static::assertCount(2, $unmatched); - static::assertSame('Alice', $unmatched[0]->get('name')); - static::assertSame('Cid', $unmatched[1]->get('name')); + static::assertSame([0, 2], $hashTable->unmatched()); } public function test_unmatched_rows_without_tracking_are_empty(): void { $hashTable = new HashTable(); - $hashTable->add('hash-a', row(['user_id' => 1])); + $hashTable->add('hash-a', 0); - static::assertSame([], iterator_to_array($hashTable->unmatchedRows(), false)); + static::assertSame([], $hashTable->unmatched()); } } diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Join/HashJoin/JoinerTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Join/HashJoin/JoinerTest.php index 575ac1af13..8ee649f3e0 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Unit/Join/HashJoin/JoinerTest.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Join/HashJoin/JoinerTest.php @@ -4,6 +4,7 @@ namespace Flow\ETL\Tests\Unit\Join\HashJoin; +use Flow\ETL\Column\AdaptiveBackend; use Flow\ETL\Exception\SchemaDefinitionNotUniqueException; use Flow\ETL\Join\Comparison\All; use Flow\ETL\Join\Comparison\Any; @@ -17,10 +18,9 @@ use Flow\ETL\Tests\FlowTestCase; use Generator; +use function Flow\ETL\DSL\array_to_rows; use function Flow\ETL\DSL\int_schema; use function Flow\ETL\DSL\join_on; -use function Flow\ETL\DSL\row; -use function Flow\ETL\DSL\rows; use function Flow\ETL\DSL\schema; use function Flow\ETL\DSL\str_schema; @@ -37,18 +37,20 @@ public function test_any_comparison_falls_back_to_verifying_every_pair(): void $joiner = new Joiner( Expression::on(new Any(new Equal('id', 'user_id'), new Equal('email', 'contact'))), Join::inner, + new AdaptiveBackend(), ); foreach ($joiner->join( - JoinSide::of($batches(rows( + JoinSide::of($batches(array_to_rows( + [['id' => 1, 'email' => 'alice@flow.php'], ['id' => 2, 'email' => 'bob@flow.php']], schema(int_schema('id'), str_schema('email')), - row(['id' => 1, 'email' => 'alice@flow.php']), - row(['id' => 2, 'email' => 'bob@flow.php']), ))), - JoinSide::of($batches(rows( + JoinSide::of($batches(array_to_rows( + [ + ['user_id' => 5, 'contact' => 'alice@flow.php', 'name' => 'Alice'], + ['user_id' => 2, 'contact' => 'other@flow.php', 'name' => 'Bob'], + ], schema(int_schema('user_id'), str_schema('contact'), str_schema('name')), - row(['user_id' => 5, 'contact' => 'alice@flow.php', 'name' => 'Alice']), - row(['user_id' => 2, 'contact' => 'other@flow.php', 'name' => 'Bob']), ))), ) as $batch) { foreach ($batch->toArray() as $rowData) { @@ -87,18 +89,23 @@ public function test_all_with_any_hashes_by_the_equality_and_verifies_the_rest() ), ), Join::inner, + new AdaptiveBackend(), ); foreach ($joiner->join( - JoinSide::of($batches(rows( + JoinSide::of($batches(array_to_rows( + [ + ['id' => 1, 'email' => 'alice@flow.php', 'phone' => '111'], + ['id' => 2, 'email' => 'bob@flow.php', 'phone' => '222'], + ], schema(int_schema('id'), str_schema('email'), str_schema('phone')), - row(['id' => 1, 'email' => 'alice@flow.php', 'phone' => '111']), - row(['id' => 2, 'email' => 'bob@flow.php', 'phone' => '222']), ))), - JoinSide::of($batches(rows( + JoinSide::of($batches(array_to_rows( + [ + ['user_id' => 1, 'contact' => 'alice@flow.php', 'phone' => '999'], + ['user_id' => 2, 'contact' => 'other@flow.php', 'phone' => '999'], + ], schema(int_schema('user_id'), str_schema('contact'), str_schema('phone')), - row(['user_id' => 1, 'contact' => 'alice@flow.php', 'phone' => '999']), - row(['user_id' => 2, 'contact' => 'other@flow.php', 'phone' => '999']), ))), ) as $batch) { foreach ($batch->toArray() as $rowData) { @@ -129,14 +136,13 @@ public function test_duplicated_right_rows_are_all_joined(): void $joined = []; - $joiner = new Joiner(join_on(['id' => 'user_id']), Join::inner); + $joiner = new Joiner(join_on(['id' => 'user_id']), Join::inner, new AdaptiveBackend()); foreach ($joiner->join( - JoinSide::of($batches(rows(schema(int_schema('id')), row(['id' => 1])))), - JoinSide::of($batches(rows( + JoinSide::of($batches(array_to_rows([['id' => 1]], schema(int_schema('id'))))), + JoinSide::of($batches(array_to_rows( + [['user_id' => 1, 'name' => 'Alice'], ['user_id' => 1, 'name' => 'Alice']], schema(int_schema('user_id'), str_schema('name')), - row(['user_id' => 1, 'name' => 'Alice']), - row(['user_id' => 1, 'name' => 'Alice']), ))), ) as $batch) { foreach ($batch->toArray() as $rowData) { @@ -155,18 +161,20 @@ public function test_inner_join_emits_every_matching_right_row(): void $joined = []; - $joiner = new Joiner(join_on(['id' => 'user_id']), Join::inner); + $joiner = new Joiner(join_on(['id' => 'user_id']), Join::inner, new AdaptiveBackend()); foreach ($joiner->join( - JoinSide::of($batches(rows( + JoinSide::of($batches(array_to_rows( + [['id' => 1, 'amount' => 100]], schema(int_schema('id'), int_schema('amount')), - row(['id' => 1, 'amount' => 100]), ))), - JoinSide::of($batches(rows( + JoinSide::of($batches(array_to_rows( + [ + ['user_id' => 1, 'role' => 'admin'], + ['user_id' => 1, 'role' => 'writer'], + ['user_id' => 2, 'role' => 'reader'], + ], schema(int_schema('user_id'), str_schema('role')), - row(['user_id' => 1, 'role' => 'admin']), - row(['user_id' => 1, 'role' => 'writer']), - row(['user_id' => 2, 'role' => 'reader']), ))), ) as $batch) { foreach ($batch->toArray() as $rowData) { @@ -191,16 +199,16 @@ public function test_inner_join_drops_duplicated_right_join_columns(): void $joined = []; - $joiner = new Joiner(join_on(['id' => 'id']), Join::inner); + $joiner = new Joiner(join_on(['id' => 'id']), Join::inner, new AdaptiveBackend()); foreach ($joiner->join( - JoinSide::of($batches(rows( + JoinSide::of($batches(array_to_rows( + [['id' => 1, 'amount' => 100]], schema(int_schema('id'), int_schema('amount')), - row(['id' => 1, 'amount' => 100]), ))), - JoinSide::of($batches(rows( + JoinSide::of($batches(array_to_rows( + [['id' => 1, 'name' => 'Alice']], schema(int_schema('id'), str_schema('name')), - row(['id' => 1, 'name' => 'Alice']), ))), ) as $batch) { foreach ($batch->toArray() as $rowData) { @@ -220,15 +228,15 @@ public function test_left_anti_join_keeps_left_row_on_hash_hit_without_match(): $joined = []; // int 1 and string "1" hash into the same bucket but are not identical - $joiner = new Joiner(Expression::on(new Identical('id', 'user_id')), Join::left_anti); + $joiner = new Joiner(Expression::on(new Identical('id', 'user_id')), Join::left_anti, new AdaptiveBackend()); $right = static function (): Generator { - yield rows(schema(str_schema('user_id')), row(['user_id' => '2'])); - yield rows(schema(str_schema('user_id')), row(['user_id' => '9'])); + yield array_to_rows([['user_id' => '2']], schema(str_schema('user_id'))); + yield array_to_rows([['user_id' => '9']], schema(str_schema('user_id'))); }; foreach ($joiner->join( - JoinSide::of($batches(rows(schema(str_schema('id')), row(['id' => '1']), row(['id' => '2'])))), + JoinSide::of($batches(array_to_rows([['id' => '1'], ['id' => '2']], schema(str_schema('id'))))), JoinSide::of($right()), ) as $batch) { foreach ($batch->toArray() as $rowData) { @@ -249,16 +257,16 @@ public function test_left_join_keeps_left_row_on_hash_hit_without_match(): void $joined = []; // int 1 and string "1" hash into the same bucket but are not identical - $joiner = new Joiner(Expression::on(new Identical('id', 'user_id')), Join::left); + $joiner = new Joiner(Expression::on(new Identical('id', 'user_id')), Join::left, new AdaptiveBackend()); foreach ($joiner->join( - JoinSide::of($batches(rows( + JoinSide::of($batches(array_to_rows( + [['id' => 1, 'amount' => 100]], schema(int_schema('id'), int_schema('amount')), - row(['id' => 1, 'amount' => 100]), ))), - JoinSide::of($batches(rows( + JoinSide::of($batches(array_to_rows( + [['user_id' => '1', 'name' => 'Alice']], schema(str_schema('user_id'), str_schema('name')), - row(['user_id' => '1', 'name' => 'Alice']), ))), ) as $batch) { foreach ($batch->toArray() as $rowData) { @@ -277,17 +285,16 @@ public function test_left_join_pads_unmatched_left_rows_with_nulls(): void $joined = []; - $joiner = new Joiner(join_on(['id' => 'user_id']), Join::left); + $joiner = new Joiner(join_on(['id' => 'user_id']), Join::left, new AdaptiveBackend()); foreach ($joiner->join( - JoinSide::of($batches(rows( + JoinSide::of($batches(array_to_rows( + [['id' => 1, 'amount' => 100], ['id' => 404, 'amount' => 200]], schema(int_schema('id'), int_schema('amount')), - row(['id' => 1, 'amount' => 100]), - row(['id' => 404, 'amount' => 200]), ))), - JoinSide::of($batches(rows( + JoinSide::of($batches(array_to_rows( + [['user_id' => 1, 'name' => 'Alice']], schema(int_schema('user_id'), str_schema('name')), - row(['user_id' => 1, 'name' => 'Alice']), ))), ) as $batch) { foreach ($batch->toArray() as $rowData) { @@ -313,16 +320,16 @@ public function test_prefix_collision_with_left_column_throws(): void $this->expectException(SchemaDefinitionNotUniqueException::class); $this->expectExceptionMessage('Entry definitions must be unique, duplicated entries: [left_name]'); - $joiner = new Joiner(join_on(['id' => 'user_id'], join_prefix: 'left_'), Join::inner); + $joiner = new Joiner(join_on(['id' => 'user_id'], join_prefix: 'left_'), Join::inner, new AdaptiveBackend()); foreach ($joiner->join( - JoinSide::of($batches(rows( + JoinSide::of($batches(array_to_rows( + [['id' => 1, 'left_name' => 'collision']], schema(int_schema('id'), str_schema('left_name')), - row(['id' => 1, 'left_name' => 'collision']), ))), - JoinSide::of($batches(rows( + JoinSide::of($batches(array_to_rows( + [['user_id' => 1, 'name' => 'Alice']], schema(int_schema('user_id'), str_schema('name')), - row(['user_id' => 1, 'name' => 'Alice']), ))), ) as $batch) { $batch->count(); @@ -337,18 +344,20 @@ public function test_swapped_inner_join_emits_the_same_rows(): void $joined = []; - $joiner = new Joiner(join_on(['id' => 'user_id']), Join::inner); + $joiner = new Joiner(join_on(['id' => 'user_id']), Join::inner, new AdaptiveBackend()); foreach ($joiner->join( - JoinSide::of($batches(rows( + JoinSide::of($batches(array_to_rows( + [['id' => 1, 'amount' => 100]], schema(int_schema('id'), int_schema('amount')), - row(['id' => 1, 'amount' => 100]), ))), - JoinSide::of($batches(rows( + JoinSide::of($batches(array_to_rows( + [ + ['user_id' => 1, 'role' => 'admin'], + ['user_id' => 1, 'role' => 'writer'], + ['user_id' => 2, 'role' => 'reader'], + ], schema(int_schema('user_id'), str_schema('role')), - row(['user_id' => 1, 'role' => 'admin']), - row(['user_id' => 1, 'role' => 'writer']), - row(['user_id' => 2, 'role' => 'reader']), ))), buildLeft: true, ) as $batch) { @@ -374,11 +383,11 @@ public function test_swapped_left_anti_join_emits_unmatched_left_rows_at_the_end $joined = []; - $joiner = new Joiner(join_on(['id' => 'user_id']), Join::left_anti); + $joiner = new Joiner(join_on(['id' => 'user_id']), Join::left_anti, new AdaptiveBackend()); foreach ($joiner->join( - JoinSide::of($batches(rows(schema(int_schema('id')), row(['id' => 1]), row(['id' => 2])))), - JoinSide::of($batches(rows(schema(int_schema('user_id')), row(['user_id' => 1])))), + JoinSide::of($batches(array_to_rows([['id' => 1], ['id' => 2]], schema(int_schema('id'))))), + JoinSide::of($batches(array_to_rows([['user_id' => 1]], schema(int_schema('user_id'))))), buildLeft: true, ) as $batch) { foreach ($batch->toArray() as $rowData) { @@ -397,17 +406,16 @@ public function test_swapped_left_join_pads_unmatched_left_rows_at_the_end(): vo $joined = []; - $joiner = new Joiner(join_on(['id' => 'user_id']), Join::left); + $joiner = new Joiner(join_on(['id' => 'user_id']), Join::left, new AdaptiveBackend()); foreach ($joiner->join( - JoinSide::of($batches(rows( + JoinSide::of($batches(array_to_rows( + [['id' => 1, 'amount' => 100], ['id' => 404, 'amount' => 200]], schema(int_schema('id'), int_schema('amount')), - row(['id' => 1, 'amount' => 100]), - row(['id' => 404, 'amount' => 200]), ))), - JoinSide::of($batches(rows( + JoinSide::of($batches(array_to_rows( + [['user_id' => 1, 'name' => 'Alice']], schema(int_schema('user_id'), str_schema('name')), - row(['user_id' => 1, 'name' => 'Alice']), ))), buildLeft: true, ) as $batch) { @@ -437,12 +445,12 @@ public function test_swapped_left_join_with_empty_right_side_pads_every_left_row $joined = []; - $joiner = new Joiner(join_on(['id' => 'user_id']), Join::left); + $joiner = new Joiner(join_on(['id' => 'user_id']), Join::left, new AdaptiveBackend()); foreach ($joiner->join( - JoinSide::of($batches(rows( + JoinSide::of($batches(array_to_rows( + [['id' => 1, 'amount' => 100]], schema(int_schema('id'), int_schema('amount')), - row(['id' => 1, 'amount' => 100]), ))), JoinSide::of($empty()), buildLeft: true, @@ -463,17 +471,16 @@ public function test_swapped_right_join_pads_unmatched_right_rows_inline(): void $joined = []; - $joiner = new Joiner(join_on(['id' => 'user_id']), Join::right); + $joiner = new Joiner(join_on(['id' => 'user_id']), Join::right, new AdaptiveBackend()); foreach ($joiner->join( - JoinSide::of($batches(rows( + JoinSide::of($batches(array_to_rows( + [['id' => 1, 'amount' => 100]], schema(int_schema('id'), int_schema('amount')), - row(['id' => 1, 'amount' => 100]), ))), - JoinSide::of($batches(rows( + JoinSide::of($batches(array_to_rows( + [['user_id' => 1, 'name' => 'Alice'], ['user_id' => 2, 'name' => 'Bob']], schema(int_schema('user_id'), str_schema('name')), - row(['user_id' => 1, 'name' => 'Alice']), - row(['user_id' => 2, 'name' => 'Bob']), ))), buildLeft: true, ) as $batch) { @@ -499,17 +506,16 @@ public function test_right_join_emits_unmatched_right_rows_with_null_left_entrie $joined = []; - $joiner = new Joiner(join_on(['id' => 'user_id']), Join::right); + $joiner = new Joiner(join_on(['id' => 'user_id']), Join::right, new AdaptiveBackend()); foreach ($joiner->join( - JoinSide::of($batches(rows( + JoinSide::of($batches(array_to_rows( + [['id' => 1, 'amount' => 100]], schema(int_schema('id'), int_schema('amount')), - row(['id' => 1, 'amount' => 100]), ))), - JoinSide::of($batches(rows( + JoinSide::of($batches(array_to_rows( + [['user_id' => 1, 'name' => 'Alice'], ['user_id' => 2, 'name' => 'Bob']], schema(int_schema('user_id'), str_schema('name')), - row(['user_id' => 1, 'name' => 'Alice']), - row(['user_id' => 2, 'name' => 'Bob']), ))), ) as $batch) { foreach ($batch->toArray() as $rowData) { @@ -532,18 +538,17 @@ public function test_left_join_emits_a_schema_declaring_the_right_side_nullable( yield $rows; }; - $joiner = new Joiner(join_on(['id' => 'user_id']), Join::left); + $joiner = new Joiner(join_on(['id' => 'user_id']), Join::left, new AdaptiveBackend()); $emitted = iterator_to_array( $joiner->join( - JoinSide::of($batches(rows( + JoinSide::of($batches(array_to_rows( + [['id' => 1, 'amount' => 100], ['id' => 404, 'amount' => 200]], schema(int_schema('id'), int_schema('amount')), - row(['id' => 1, 'amount' => 100]), - row(['id' => 404, 'amount' => 200]), ))), - JoinSide::of($batches(rows( + JoinSide::of($batches(array_to_rows( + [['user_id' => 1, 'name' => 'Alice']], schema(int_schema('user_id'), str_schema('name')), - row(['user_id' => 1, 'name' => 'Alice']), ))), ), preserve_keys: false, @@ -566,18 +571,17 @@ public function test_right_join_emits_a_schema_declaring_the_left_side_nullable( yield $rows; }; - $joiner = new Joiner(join_on(['id' => 'user_id']), Join::right); + $joiner = new Joiner(join_on(['id' => 'user_id']), Join::right, new AdaptiveBackend()); $emitted = iterator_to_array( $joiner->join( - JoinSide::of($batches(rows( + JoinSide::of($batches(array_to_rows( + [['id' => 1, 'amount' => 100]], schema(int_schema('id'), int_schema('amount')), - row(['id' => 1, 'amount' => 100]), ))), - JoinSide::of($batches(rows( + JoinSide::of($batches(array_to_rows( + [['user_id' => 1, 'name' => 'Alice'], ['user_id' => 404, 'name' => 'Bob']], schema(int_schema('user_id'), str_schema('name')), - row(['user_id' => 1, 'name' => 'Alice']), - row(['user_id' => 404, 'name' => 'Bob']), ))), ), preserve_keys: false, @@ -596,21 +600,27 @@ public function test_right_join_emits_a_schema_declaring_the_left_side_nullable( public function test_every_batch_of_a_multi_batch_left_side_shares_one_output_schema(): void { - $joiner = new Joiner(join_on(['id' => 'user_id']), Join::left); + $joiner = new Joiner(join_on(['id' => 'user_id']), Join::left, new AdaptiveBackend()); $emitted = iterator_to_array( $joiner->join( JoinSide::of( (static function (): Generator { - yield rows(schema(int_schema('id'), int_schema('amount')), row(['id' => 1, 'amount' => 100])); - yield rows(schema(int_schema('id'), int_schema('amount')), row(['id' => 404, 'amount' => 200])); + yield array_to_rows( + [['id' => 1, 'amount' => 100]], + schema(int_schema('id'), int_schema('amount')), + ); + yield array_to_rows( + [['id' => 404, 'amount' => 200]], + schema(int_schema('id'), int_schema('amount')), + ); })(), ), JoinSide::of( (static function (): Generator { - yield rows( + yield array_to_rows( + [['user_id' => 1, 'name' => 'Alice']], schema(int_schema('user_id'), str_schema('name')), - row(['user_id' => 1, 'name' => 'Alice']), ); })(), ), @@ -628,15 +638,15 @@ public function test_a_padded_row_covers_a_right_column_no_source_row_carried(): yield $rows; }; - $joiner = new Joiner(join_on(['id' => 'user_id']), Join::left); + $joiner = new Joiner(join_on(['id' => 'user_id']), Join::left, new AdaptiveBackend()); $joined = []; foreach ($joiner->join( - JoinSide::of($batches(rows(schema(int_schema('id')), row(['id' => 1]), row(['id' => 404])))), - JoinSide::of($batches(rows( + JoinSide::of($batches(array_to_rows([['id' => 1], ['id' => 404]], schema(int_schema('id'))))), + JoinSide::of($batches(array_to_rows( + [['user_id' => 1, 'name' => 'Alice']], schema(int_schema('user_id'), str_schema('name'), str_schema('nickname', nullable: true)), - row(['user_id' => 1, 'name' => 'Alice']), ))), ) as $batch) { foreach ($batch->toArray() as $rowData) { @@ -663,19 +673,18 @@ public function test_a_row_omitting_a_nullable_join_key_is_unmatched_rather_than yield $rows; }; - $joiner = new Joiner(join_on(['id' => 'user_id']), Join::left); + $joiner = new Joiner(join_on(['id' => 'user_id']), Join::left, new AdaptiveBackend()); $joined = []; foreach ($joiner->join( - JoinSide::of($batches(rows( + JoinSide::of($batches(array_to_rows( + [['id' => 1, 'l' => 'has-key'], ['l' => 'no-key']], schema(int_schema('id', nullable: true), str_schema('l')), - row(['id' => 1, 'l' => 'has-key']), - row(['l' => 'no-key']), ))), - JoinSide::of($batches(rows( + JoinSide::of($batches(array_to_rows( + [['user_id' => 1, 'r' => 'Alice']], schema(int_schema('user_id'), str_schema('r')), - row(['user_id' => 1, 'r' => 'Alice']), ))), ) as $batch) { foreach ($batch->toArray() as $rowData) { @@ -698,17 +707,20 @@ public function test_a_left_batch_wider_than_the_output_schema_is_projected_not_ yield $rows; }; - $joiner = new Joiner(join_on(['id' => 'user_id']), Join::left); + $joiner = new Joiner(join_on(['id' => 'user_id']), Join::left, new AdaptiveBackend()); $joined = []; foreach ($joiner->join( JoinSide::of( - $batches(rows(schema(int_schema('id'), str_schema('extra')), row(['id' => 404, 'extra' => 'drop-me']))), + $batches(array_to_rows( + [['id' => 404, 'extra' => 'drop-me']], + schema(int_schema('id'), str_schema('extra')), + )), null, schema(int_schema('id')), ), - JoinSide::of($batches(rows(schema(int_schema('user_id')), row(['user_id' => 1])))), + JoinSide::of($batches(array_to_rows([['user_id' => 1]], schema(int_schema('user_id'))))), ) as $batch) { foreach ($batch->toArray() as $rowData) { $joined[] = $rowData; @@ -717,4 +729,99 @@ public function test_a_left_batch_wider_than_the_output_schema_is_projected_not_ static::assertSame([['id' => 404, 'user_id' => null]], $joined); } + + public function test_a_build_side_spanning_batches_is_joined_as_one(): void + { + $right = static function (): Generator { + yield array_to_rows( + [['user_id' => 1, 'name' => 'Alice']], + schema(int_schema('user_id'), str_schema('name')), + ); + yield array_to_rows([], schema(int_schema('user_id'), str_schema('name'))); + yield array_to_rows([['user_id' => 2, 'name' => 'Bob']], schema(int_schema('user_id'), str_schema('name'))); + }; + $left = static function (): Generator { + yield array_to_rows([['id' => 2], ['id' => 1], ['id' => 3]], schema(int_schema('id'))); + }; + + $joined = []; + + foreach ((new Joiner(join_on(['id' => 'user_id']), Join::left, new AdaptiveBackend()))->join( + JoinSide::of($left()), + JoinSide::of($right()), + ) as $batch) { + $joined = [...$joined, ...$batch->toArray()]; + } + + static::assertSame( + [ + ['id' => 2, 'user_id' => 2, 'name' => 'Bob'], + ['id' => 1, 'user_id' => 1, 'name' => 'Alice'], + ['id' => 3, 'user_id' => null, 'name' => null], + ], + $joined, + ); + } + + public function test_swapped_unmatched_build_rows_are_emitted_in_chunks_of_batch_size(): void + { + $left = static function (): Generator { + yield array_to_rows([ + ['id' => 1], + ['id' => 2], + ['id' => 3], + ['id' => 4], + ['id' => 5], + ], schema(int_schema('id'))); + }; + $right = static function (): Generator { + yield array_to_rows([['user_id' => 3]], schema(int_schema('user_id'))); + }; + + $batches = []; + + foreach ((new Joiner(join_on(['id' => 'user_id']), Join::left, new AdaptiveBackend(), batchSize: 2))->join( + JoinSide::of($left()), + JoinSide::of($right()), + buildLeft: true, + ) as $batch) { + $batches[] = $batch->column('id')->values(); + } + + static::assertSame([[3], [1, 2], [4, 5]], $batches); + } + + public function test_unmatched_build_rows_are_emitted_in_chunks_of_batch_size(): void + { + $left = static function (): Generator { + yield array_to_rows([['id' => 3]], schema(int_schema('id'))); + }; + $right = static function (): Generator { + yield array_to_rows([ + ['user_id' => 1], + ['user_id' => 2], + ['user_id' => 3], + ['user_id' => 4], + ['user_id' => 5], + ], schema(int_schema('user_id'))); + }; + + $batches = []; + + foreach ((new Joiner(join_on(['id' => 'user_id']), Join::right, new AdaptiveBackend(), batchSize: 2))->join( + JoinSide::of($left()), + JoinSide::of($right()), + ) as $batch) { + $batches[] = $batch->toArray(); + } + + static::assertSame( + [ + [['id' => 3, 'user_id' => 3]], + [['id' => null, 'user_id' => 1], ['id' => null, 'user_id' => 2]], + [['id' => null, 'user_id' => 4], ['id' => null, 'user_id' => 5]], + ], + $batches, + ); + } } diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Join/HashJoin/NullRowBuilderTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Join/HashJoin/NullRowBuilderTest.php index c029e825c0..debbd17a90 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Unit/Join/HashJoin/NullRowBuilderTest.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Join/HashJoin/NullRowBuilderTest.php @@ -4,33 +4,52 @@ namespace Flow\ETL\Tests\Unit\Join\HashJoin; +use Flow\ETL\Column\AdaptiveBackend; use Flow\ETL\Join\HashJoin\NullRowBuilder; use Flow\ETL\Tests\FlowTestCase; +use function array_keys; use function Flow\ETL\DSL\int_schema; use function Flow\ETL\DSL\schema; use function Flow\ETL\DSL\str_schema; final class NullRowBuilderTest extends FlowTestCase { - public function test_empty_schema_produces_empty_row(): void + public function test_empty_schema_produces_one_row_without_columns(): void { - static::assertSame([], (new NullRowBuilder(schema()))->row()->toArray()); + $rows = (new NullRowBuilder(schema(), new AdaptiveBackend()))->rows(); + + static::assertSame(1, $rows->count()); + static::assertSame([], $rows->columns()); } public function test_row_follows_schema_column_order(): void { static::assertSame( ['name', 'id'], - (new NullRowBuilder(schema(str_schema('name'), int_schema('id'))))->row()->names(), + array_keys( + (new NullRowBuilder(schema(str_schema('name'), int_schema('id')), new AdaptiveBackend())) + ->rows() + ->columns(), + ), ); } - public function test_row_nulls_every_schema_column(): void + public function test_row_nulls_every_schema_column_under_a_nullable_schema(): void { - static::assertSame( - ['id' => null, 'name' => null, 'country' => null], - (new NullRowBuilder(schema(int_schema('id'), str_schema('name'), str_schema('country'))))->row()->toArray(), + $rows = (new NullRowBuilder( + schema(int_schema('id'), str_schema('name'), str_schema('country')), + new AdaptiveBackend(), + ))->rows(); + + static::assertSame([['id' => null, 'name' => null, 'country' => null]], $rows->toArray()); + static::assertEquals( + schema( + int_schema('id', nullable: true), + str_schema('name', nullable: true), + str_schema('country', nullable: true), + ), + $rows->schema(), ); } } diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Join/HashJoin/RowMergerTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Join/HashJoin/RowMergerTest.php index 14b9d22333..4b9165cd5f 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Unit/Join/HashJoin/RowMergerTest.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Join/HashJoin/RowMergerTest.php @@ -8,7 +8,10 @@ use Flow\ETL\Join\HashJoin\RowMerger; use Flow\ETL\Tests\FlowTestCase; -use function Flow\ETL\DSL\row; +use function Flow\ETL\DSL\array_to_rows; +use function Flow\ETL\DSL\int_schema; +use function Flow\ETL\DSL\schema; +use function Flow\ETL\DSL\str_schema; final class RowMergerTest extends FlowTestCase { @@ -17,7 +20,12 @@ public function test_collision_between_left_and_right_names_throws(): void $this->expectException(DuplicatedEntriesException::class); $this->expectExceptionMessage('Merged entries names must be unique'); - (new RowMerger())->merge(row(['id' => 1, 'name' => 'left']), row(['name' => 'right'])); + (new RowMerger()) + ->merge( + array_to_rows([['id' => 1, 'name' => 'left']], schema(int_schema('id'), str_schema('name'))), + array_to_rows([['name' => 'right']], schema(str_schema('name'))), + ) + ->values(0); } public function test_drop_left_skips_duplicated_join_columns(): void @@ -26,7 +34,12 @@ public function test_drop_left_skips_duplicated_join_columns(): void static::assertSame( ['amount' => 100, 'id' => 1, 'name' => 'Alice'], - $merger->merge(row(['id' => 1, 'amount' => 100]), row(['id' => 1, 'name' => 'Alice']))->toArray(), + $merger + ->merge( + array_to_rows([['id' => 1, 'amount' => 100]], schema(int_schema('id'), int_schema('amount'))), + array_to_rows([['id' => 1, 'name' => 'Alice']], schema(int_schema('id'), str_schema('name'))), + ) + ->values(0), ); } @@ -36,7 +49,12 @@ public function test_drop_right_skips_duplicated_join_columns(): void static::assertSame( ['id' => 1, 'amount' => 100, 'name' => 'Alice'], - $merger->merge(row(['id' => 1, 'amount' => 100]), row(['id' => 1, 'name' => 'Alice']))->toArray(), + $merger + ->merge( + array_to_rows([['id' => 1, 'amount' => 100]], schema(int_schema('id'), int_schema('amount'))), + array_to_rows([['id' => 1, 'name' => 'Alice']], schema(int_schema('id'), str_schema('name'))), + ) + ->values(0), ); } @@ -45,8 +63,11 @@ public function test_merge_without_prefix_keeps_right_names(): void static::assertSame( ['id' => 1, 'name' => 'Alice'], (new RowMerger()) - ->merge(row(['id' => 1]), row(['name' => 'Alice'])) - ->toArray(), + ->merge( + array_to_rows([['id' => 1]], schema(int_schema('id'))), + array_to_rows([['name' => 'Alice']], schema(str_schema('name'))), + ) + ->values(0), ); } @@ -54,7 +75,12 @@ public function test_prefix_collision_with_left_name_throws(): void { $this->expectException(DuplicatedEntriesException::class); - (new RowMerger('left_'))->merge(row(['left_name' => 'left']), row(['name' => 'right'])); + (new RowMerger('left_')) + ->merge( + array_to_rows([['left_name' => 'left']], schema(str_schema('left_name'))), + array_to_rows([['name' => 'right']], schema(str_schema('name'))), + ) + ->values(0); } public function test_prefix_renames_every_right_entry(): void @@ -62,25 +88,48 @@ public function test_prefix_renames_every_right_entry(): void static::assertSame( ['id' => 1, 'right_id' => 1, 'right_name' => 'Alice'], (new RowMerger('right_')) - ->merge(row(['id' => 1]), row(['id' => 1, 'name' => 'Alice'])) - ->toArray(), + ->merge( + array_to_rows([['id' => 1]], schema(int_schema('id'))), + array_to_rows([['id' => 1, 'name' => 'Alice']], schema(int_schema('id'), str_schema('name'))), + ) + ->values(0), ); } public function test_renamed_entries_carry_renamed_names(): void { - $merged = (new RowMerger('r_'))->merge(row(['id' => 1]), row(['name' => 'Alice'])); - - static::assertSame(['id', 'r_name'], $merged->names()); - static::assertSame('Alice', $merged->get('r_name')); + $merged = (new RowMerger('r_')) + ->merge( + array_to_rows([['id' => 1]], schema(int_schema('id'))), + array_to_rows([['name' => 'Alice']], schema(str_schema('name'))), + ) + ->values(0); + + static::assertSame(['id' => 1, 'r_name' => 'Alice'], $merged); } public function test_reused_plan_renames_rows_with_identical_name_sets(): void { $merger = new RowMerger('r_'); - static::assertSame(['id' => 1, 'r_x' => 'a'], $merger->merge(row(['id' => 1]), row(['x' => 'a']))->toArray()); - static::assertSame(['id' => 2, 'r_x' => 5], $merger->merge(row(['id' => 2]), row(['x' => 5]))->toArray()); + static::assertSame( + ['id' => 1, 'r_x' => 'a'], + $merger + ->merge( + array_to_rows([['id' => 1]], schema(int_schema('id'))), + array_to_rows([['x' => 'a']], schema(str_schema('x'))), + ) + ->values(0), + ); + static::assertSame( + ['id' => 2, 'r_x' => 5], + $merger + ->merge( + array_to_rows([['id' => 2]], schema(int_schema('id'))), + array_to_rows([['x' => 5]], schema(int_schema('x'))), + ) + ->values(0), + ); } public function test_reused_merger_handles_rows_with_different_entry_sets(): void @@ -89,15 +138,50 @@ public function test_reused_merger_handles_rows_with_different_entry_sets(): voi static::assertSame( ['id' => 1, 'name' => 'Alice'], - $merger->merge(row(['id' => 1]), row(['name' => 'Alice']))->toArray(), + $merger + ->merge( + array_to_rows([['id' => 1]], schema(int_schema('id'))), + array_to_rows([['name' => 'Alice']], schema(str_schema('name'))), + ) + ->values(0), ); static::assertSame( ['id' => 2, 'name' => 'Bob', 'age' => 30], - $merger->merge(row(['id' => 2]), row(['name' => 'Bob', 'age' => 30]))->toArray(), + $merger + ->merge( + array_to_rows([['id' => 2]], schema(int_schema('id'))), + array_to_rows([['name' => 'Bob', 'age' => 30]], schema(str_schema('name'), int_schema('age'))), + ) + ->values(0), ); static::assertSame( ['id' => 3, 'name' => 'Cid'], - $merger->merge(row(['id' => 3]), row(['name' => 'Cid']))->toArray(), + $merger + ->merge( + array_to_rows([['id' => 3]], schema(int_schema('id'))), + array_to_rows([['name' => 'Cid']], schema(str_schema('name'))), + ) + ->values(0), + ); + } + + public function test_merges_pair_aligned_batches_with_renamed_definitions(): void + { + $merged = (new RowMerger('r_'))->merge( + array_to_rows([['id' => 1], ['id' => 2]], schema(int_schema('id'))), + array_to_rows( + [['id' => 10, 'name' => null], ['id' => 20, 'name' => 'b']], + schema(int_schema('id'), str_schema('name', nullable: true)), + ), + ); + + static::assertEquals( + schema(int_schema('id'), int_schema('r_id'), str_schema('r_name', nullable: true)), + $merged->schema(), + ); + static::assertSame( + [['id' => 1, 'r_id' => 10, 'r_name' => null], ['id' => 2, 'r_id' => 20, 'r_name' => 'b']], + $merged->toArray(), ); } } diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Join/JoinSchemaTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Join/JoinSchemaTest.php index 1f8d52e90a..0ad49db36a 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Unit/Join/JoinSchemaTest.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Join/JoinSchemaTest.php @@ -4,6 +4,7 @@ namespace Flow\ETL\Tests\Unit\Join; +use Flow\ETL\Exception\SchemaDefinitionNotUniqueException; use Flow\ETL\Join\Join; use Flow\ETL\Join\JoinSchema; use Flow\ETL\Tests\FlowTestCase; @@ -120,4 +121,24 @@ public function test_right_makes_the_left_side_nullable(): void ), ); } + + public function test_a_cross_name_collision_suggests_a_prefix(): void + { + $this->expectException(SchemaDefinitionNotUniqueException::class); + $this->expectExceptionMessageMatches( + '/Entry definitions must be unique.*\. Please consider using join prefix option$/', + ); + + (new JoinSchema())->cross(schema(int_schema('id')), schema(int_schema('id'))); + } + + public function test_a_keyed_join_name_collision_suggests_a_prefix(): void + { + $this->expectException(SchemaDefinitionNotUniqueException::class); + $this->expectExceptionMessageMatches( + '/Entry definitions must be unique.*\. Please consider using join prefix option$/', + ); + + (new JoinSchema())->of(Join::left, schema(int_schema('id'), str_schema('name')), schema(str_schema('name'))); + } } diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Join/JoinShapeTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Join/JoinShapeTest.php index a847100c5c..c40f0d691f 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Unit/Join/JoinShapeTest.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Join/JoinShapeTest.php @@ -8,9 +8,9 @@ use Flow\ETL\Join\JoinShape; use Flow\ETL\Tests\FlowTestCase; +use function Flow\ETL\DSL\array_to_rows; use function Flow\ETL\DSL\int_schema; use function Flow\ETL\DSL\join_on; -use function Flow\ETL\DSL\row; use function Flow\ETL\DSL\schema; use function Flow\ETL\DSL\str_schema; @@ -107,8 +107,11 @@ public function test_the_merger_drops_the_same_column_the_schema_does(): void ['id' => 1, 'country' => 'PL', 'name' => 'Norbert'], JoinShape::of(join_on(['id' => 'id']), Join::inner) ->merger() - ->merge(row(['id' => 1, 'country' => 'PL']), row(['id' => 1, 'name' => 'Norbert'])) - ->toArray(), + ->merge( + array_to_rows([['id' => 1, 'country' => 'PL']], schema(int_schema('id'), str_schema('country'))), + array_to_rows([['id' => 1, 'name' => 'Norbert']], schema(int_schema('id'), str_schema('name'))), + ) + ->values(0), ); } @@ -118,8 +121,11 @@ public function test_the_merger_of_a_right_join_keeps_the_right_side_value(): vo ['country' => 'PL', 'id' => 2, 'name' => 'Norbert'], JoinShape::of(join_on(['id' => 'id']), Join::right) ->merger() - ->merge(row(['id' => 1, 'country' => 'PL']), row(['id' => 2, 'name' => 'Norbert'])) - ->toArray(), + ->merge( + array_to_rows([['id' => 1, 'country' => 'PL']], schema(int_schema('id'), str_schema('country'))), + array_to_rows([['id' => 2, 'name' => 'Norbert']], schema(int_schema('id'), str_schema('name'))), + ) + ->values(0), ); } } diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Join/JoinStepsTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Join/JoinStepsTest.php index 8cf0433d2d..3cf2a3e7e9 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Unit/Join/JoinStepsTest.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Join/JoinStepsTest.php @@ -11,10 +11,10 @@ use Flow\ETL\Tests\FlowTestCase; use Flow\ETL\Tests\Mother\PhysicalPlanMother; +use function Flow\ETL\DSL\array_to_rows; use function Flow\ETL\DSL\config_builder; use function Flow\ETL\DSL\from_rows; use function Flow\ETL\DSL\int_schema; -use function Flow\ETL\DSL\row; use function Flow\ETL\DSL\rows; use function Flow\ETL\DSL\schema; @@ -35,7 +35,7 @@ public function test_builds_a_hash_join_processor(): void public function test_of_takes_a_frame_output_as_its_right_side(): void { - $right = PhysicalPlanMother::reading(from_rows(rows(schema(int_schema('id')), row(['id' => 1])))); + $right = PhysicalPlanMother::reading(from_rows(array_to_rows([['id' => 1]], schema(int_schema('id'))))); $steps = JoinSteps::of($right, Expression::on(['id' => 'id'], 'r_'), Join::inner, config_builder()->build()); diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Join/RowsJoinTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Join/RowsJoinTest.php new file mode 100644 index 0000000000..973d1fa315 --- /dev/null +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Join/RowsJoinTest.php @@ -0,0 +1,711 @@ + 1, 'country' => 'PL'], + ['id' => 2, 'country' => 'PL'], + ['id' => 3, 'country' => 'US'], + ['id' => 4, 'country' => 'FR'], + ], + schema(int_schema('id'), str_schema('country')), + ); + + $joined = (new RowsJoin(new AdaptiveBackend()))->cross($left, array_to_rows( + [['num' => 1, 'active' => true], ['num' => 2, 'active' => false]], + schema(int_schema('num'), bool_schema('active')), + )); + + static::assertEquals( + [ + ['id' => 1, 'country' => 'PL', 'joined_num' => 1, 'joined_active' => true], + ['id' => 1, 'country' => 'PL', 'joined_num' => 2, 'joined_active' => false], + ['id' => 2, 'country' => 'PL', 'joined_num' => 1, 'joined_active' => true], + ['id' => 2, 'country' => 'PL', 'joined_num' => 2, 'joined_active' => false], + ['id' => 3, 'country' => 'US', 'joined_num' => 1, 'joined_active' => true], + ['id' => 3, 'country' => 'US', 'joined_num' => 2, 'joined_active' => false], + ['id' => 4, 'country' => 'FR', 'joined_num' => 1, 'joined_active' => true], + ['id' => 4, 'country' => 'FR', 'joined_num' => 2, 'joined_active' => false], + ], + $joined->toArray(), + ); + } + + public function test_cross_join_empty(): void + { + $left = array_to_rows( + [ + ['id' => 1, 'country' => 'PL'], + ['id' => 2, 'country' => 'PL'], + ['id' => 3, 'country' => 'US'], + ['id' => 4, 'country' => 'FR'], + ], + schema(int_schema('id'), str_schema('country')), + ); + + // n x 0 = 0, and a zero-row batch fits the cross schema vacuously + static::assertSame( + [], + (new RowsJoin(new AdaptiveBackend())) + ->cross($left, rows(schema())) + ->toArray(), + ); + } + + public function test_cross_join_with_an_empty_side_yields_no_rows_under_the_cross_schema(): void + { + // a cross join with an empty side produces zero rows, and the batch still declares both + // sides' columns - the schema describes the plan, not the cardinality + $emptyRight = (new RowsJoin(new AdaptiveBackend()))->cross( + array_to_rows([['id' => 1, 'country' => 'PL']], schema(int_schema('id'), str_schema('country'))), + rows(schema(str_schema('code'))), + ); + + static::assertSame(['id', 'country', 'joined_code'], $emptyRight->schema()->references()->names()); + static::assertSame([], $emptyRight->toArray()); + + $emptyLeft = (new RowsJoin(new AdaptiveBackend()))->cross( + rows(schema(int_schema('id'))), + array_to_rows([['code' => 'PL']], schema(str_schema('code'))), + ); + + static::assertSame(['id', 'joined_code'], $emptyLeft->schema()->references()->names()); + static::assertSame([], $emptyLeft->toArray()); + } + + public function test_cross_join_left_empty(): void + { + $left = rows(schema()); + + $joined = (new RowsJoin(new AdaptiveBackend()))->cross($left, array_to_rows( + [ + ['id' => 1, 'country' => 'PL'], + ['id' => 2, 'country' => 'PL'], + ['id' => 3, 'country' => 'US'], + ['id' => 4, 'country' => 'FR'], + ], + schema(int_schema('id'), str_schema('country')), + )); + + static::assertSame([], $joined->toArray()); + } + + public function test_cross_join_left_with_name_conflict(): void + { + $this->expectException(SchemaDefinitionNotUniqueException::class); + $this->expectExceptionMessage( + 'Entry definitions must be unique, duplicated entries: [active], all: [id, country, active, active]', + ); + + $left = array_to_rows( + [ + ['id' => 1, 'country' => 'PL', 'active' => false], + ['id' => 2, 'country' => 'PL', 'active' => false], + ['id' => 3, 'country' => 'US', 'active' => false], + ['id' => 4, 'country' => 'FR', 'active' => false], + ], + schema(int_schema('id'), str_schema('country'), bool_schema('active')), + ); + + (new RowsJoin(new AdaptiveBackend()))->cross( + $left, + array_to_rows([['active' => true]], schema(bool_schema('active'))), + '', + ); + } + + public function test_cross_join_left_with_name_conflict_with_prefix(): void + { + $left = array_to_rows( + [ + ['id' => 1, 'country' => 'PL', 'active' => false], + ['id' => 2, 'country' => 'PL', 'active' => false], + ['id' => 3, 'country' => 'US', 'active' => false], + ['id' => 4, 'country' => 'FR', 'active' => false], + ], + schema(int_schema('id'), str_schema('country'), bool_schema('active')), + ); + + $joined = (new RowsJoin(new AdaptiveBackend()))->cross( + $left, + array_to_rows([['active' => true]], schema(bool_schema('active'))), + '_', + ); + + static::assertEquals( + [ + ['id' => 1, 'country' => 'PL', 'active' => false, '_active' => true], + ['id' => 2, 'country' => 'PL', 'active' => false, '_active' => true], + ['id' => 3, 'country' => 'US', 'active' => false, '_active' => true], + ['id' => 4, 'country' => 'FR', 'active' => false, '_active' => true], + ], + $joined->toArray(), + ); + } + + public function test_inner_empty(): void + { + $left = array_to_rows( + [ + ['id' => 1, 'country' => 'PL'], + ['id' => 2, 'country' => 'PL'], + ['id' => 3, 'country' => 'US'], + ['id' => 4, 'country' => 'FR'], + ], + schema(int_schema('id'), str_schema('country')), + ); + + $joined = (new RowsJoin(new AdaptiveBackend()))->inner( + $left, + rows(schema()), + Expression::on(['country' => 'code']), + ); + + static::assertEquals(rows(schema(int_schema('id'), str_schema('country'))), $joined); + } + + public function test_inner_join(): void + { + $left = array_to_rows( + [ + ['id' => 1, 'country' => 'PL'], + ['id' => 2, 'country' => 'PL'], + ['id' => 3, 'country' => 'US'], + ['id' => 4, 'country' => 'FR'], + ], + schema(int_schema('id'), str_schema('country')), + ); + + $joined = (new RowsJoin(new AdaptiveBackend()))->inner( + $left, + array_to_rows( + [ + ['code' => 'PL', 'name' => 'Poland'], + ['code' => 'US', 'name' => 'United States'], + ['code' => 'GB', 'name' => 'Great Britain'], + ], + schema(str_schema('code'), str_schema('name')), + ), + join_on(['country' => 'code'], 'joined_'), + ); + + static::assertEquals( + [ + ['id' => 1, 'country' => 'PL', 'joined_code' => 'PL', 'joined_name' => 'Poland'], + ['id' => 2, 'country' => 'PL', 'joined_code' => 'PL', 'joined_name' => 'Poland'], + ['id' => 3, 'country' => 'US', 'joined_code' => 'US', 'joined_name' => 'United States'], + ], + $joined->toArray(), + ); + } + + public function test_inner_join_emits_every_matching_right_row(): void + { + $joined = (new RowsJoin(new AdaptiveBackend()))->inner( + array_to_rows([['id' => 1, 'country' => 'PL']], schema(int_schema('id'), str_schema('country'))), + array_to_rows( + [['code' => 1, 'city' => 'Warsaw'], ['code' => 1, 'city' => 'Cracow']], + schema(int_schema('code'), str_schema('city')), + ), + join_on(['id' => 'code']), + ); + + static::assertSame( + [ + ['id' => 1, 'country' => 'PL', 'code' => 1, 'city' => 'Warsaw'], + ['id' => 1, 'country' => 'PL', 'code' => 1, 'city' => 'Cracow'], + ], + $joined->toArray(), + ); + } + + public function test_inner_join_into_empty(): void + { + $left = rows(schema()); + + $joined = (new RowsJoin(new AdaptiveBackend()))->inner( + $left, + array_to_rows( + [ + ['code' => 'PL', 'name' => 'Poland'], + ['code' => 'US', 'name' => 'United States'], + ['code' => 'GB', 'name' => 'Great Britain'], + ], + schema(str_schema('code'), str_schema('name')), + ), + Expression::on(['country' => 'code']), + ); + + static::assertEquals(rows(schema(str_schema('code'), str_schema('name'))), $joined); + } + + public function test_inner_join_with_duplicated_entries(): void + { + $this->expectException(SchemaDefinitionNotUniqueException::class); + $this->expectExceptionMessage( + 'Entry definitions must be unique, duplicated entries: [id], all: [id, country, id, code, name]', + ); + + $left = array_to_rows( + [ + ['id' => 1, 'country' => 'PL'], + ['id' => 2, 'country' => 'PL'], + ['id' => 3, 'country' => 'US'], + ['id' => 4, 'country' => 'FR'], + ], + schema(int_schema('id'), str_schema('country')), + ); + + (new RowsJoin(new AdaptiveBackend()))->inner( + $left, + array_to_rows( + [ + ['id' => 101, 'code' => 'PL', 'name' => 'Poland'], + ['id' => 102, 'code' => 'US', 'name' => 'United States'], + ['id' => 103, 'code' => 'GB', 'name' => 'Great Britain'], + ], + schema(int_schema('id'), str_schema('code'), str_schema('name')), + ), + Expression::on(['country' => 'code'], joinPrefix: ''), + ); + } + + public function test_inner_join_without_prefix(): void + { + $left = array_to_rows( + [ + ['id' => 1, 'country_code' => 'PL'], + ['id' => 2, 'country_code' => 'PL'], + ['id' => 3, 'country_code' => 'US'], + ['id' => 4, 'country_code' => 'FR'], + ], + schema(int_schema('id'), str_schema('country_code')), + ); + + $joined = (new RowsJoin(new AdaptiveBackend()))->inner( + $left, + array_to_rows( + [ + ['country_code' => 'PL', 'name' => 'Poland'], + ['country_code' => 'US', 'name' => 'United States'], + ['country_code' => 'GB', 'name' => 'Great Britain'], + ], + schema(str_schema('country_code'), str_schema('name')), + ), + join_on(['country_code' => 'country_code']), + ); + + static::assertEquals( + [ + ['id' => 1, 'country_code' => 'PL', 'name' => 'Poland'], + ['id' => 2, 'country_code' => 'PL', 'name' => 'Poland'], + ['id' => 3, 'country_code' => 'US', 'name' => 'United States'], + ], + $joined->toArray(), + ); + } + + public function test_left_anti_join(): void + { + $left = array_to_rows( + [['id' => 1, 'country' => 'PL'], ['id' => 2, 'country' => 'US'], ['id' => 3, 'country' => 'FR']], + schema(int_schema('id'), str_schema('country')), + ); + + $joined = (new RowsJoin(new AdaptiveBackend()))->leftAnti( + $left, + array_to_rows( + [['code' => 'US', 'name' => 'United States'], ['code' => 'FR', 'name' => 'France']], + schema(str_schema('code'), str_schema('name')), + ), + Expression::on(['country' => 'code']), + ); + + static::assertEquals( + array_to_rows([['id' => 1, 'country' => 'PL']], schema(int_schema('id'), str_schema('country'))), + $joined, + ); + } + + public function test_left_anti_join_on_empty(): void + { + $left = array_to_rows( + [['id' => 1, 'country' => 'PL'], ['id' => 2, 'country' => 'US'], ['id' => 3, 'country' => 'FR']], + schema(int_schema('id'), str_schema('country')), + ); + + $joined = (new RowsJoin(new AdaptiveBackend()))->leftAnti( + $left, + rows(schema()), + Expression::on(['country' => 'code']), + ); + + static::assertEquals($left, $joined); + } + + public function test_left_anti_join_without_prefix(): void + { + $left = array_to_rows( + [ + ['id' => 1, 'country_code' => 'PL'], + ['id' => 2, 'country_code' => 'US'], + ['id' => 3, 'country_code' => 'FR'], + ], + schema(int_schema('id'), str_schema('country_code')), + ); + + $joined = (new RowsJoin(new AdaptiveBackend()))->leftAnti( + $left, + array_to_rows( + [['country_code' => 'US', 'name' => 'United States'], ['country_code' => 'FR', 'name' => 'France']], + schema(str_schema('country_code'), str_schema('name')), + ), + Expression::on(['country_code' => 'country_code']), + ); + + static::assertEquals( + array_to_rows([['id' => 1, 'country_code' => 'PL']], schema(int_schema('id'), str_schema('country_code'))), + $joined, + ); + } + + public function test_left_join(): void + { + $left = array_to_rows( + [['id' => 1, 'country' => 'PL'], ['id' => 2, 'country' => 'US'], ['id' => 3, 'country' => 'FR']], + schema(int_schema('id'), str_schema('country')), + ); + + $joined = (new RowsJoin(new AdaptiveBackend()))->left( + $left, + array_to_rows( + [ + ['code' => 'PL', 'name' => 'Poland'], + ['code' => 'US', 'name' => 'United States'], + ['code' => 'GB', 'name' => 'Great Britain'], + ], + schema(str_schema('code'), str_schema('name')), + ), + Expression::on(['country' => 'code'], 'joined_'), + ); + + static::assertEquals( + [ + ['id' => 1, 'country' => 'PL', 'joined_code' => 'PL', 'joined_name' => 'Poland'], + ['id' => 2, 'country' => 'US', 'joined_code' => 'US', 'joined_name' => 'United States'], + ['id' => 3, 'country' => 'FR', 'joined_code' => null, 'joined_name' => null], + ], + $joined->toArray(), + ); + } + + public function test_left_join_empty(): void + { + $left = array_to_rows( + [['id' => 1, 'country' => 'PL'], ['id' => 2, 'country' => 'US'], ['id' => 3, 'country' => 'FR']], + schema(int_schema('id'), str_schema('country')), + ); + + $joined = (new RowsJoin(new AdaptiveBackend()))->left( + $left, + rows(schema()), + Expression::on(['country' => 'code']), + ); + + static::assertEquals( + array_to_rows( + [['id' => 1, 'country' => 'PL'], ['id' => 2, 'country' => 'US'], ['id' => 3, 'country' => 'FR']], + schema(int_schema('id'), str_schema('country')), + ), + $joined, + ); + } + + public function test_left_join_empty_without_prefix(): void + { + $left = array_to_rows( + [ + ['id' => 1, 'country_code' => 'PL'], + ['id' => 2, 'country_code' => 'US'], + ['id' => 3, 'country_code' => 'FR'], + ], + schema(int_schema('id'), str_schema('country_code')), + ); + + $joined = (new RowsJoin(new AdaptiveBackend()))->left( + $left, + rows(schema()), + Expression::on(['country_code' => 'country_code']), + ); + + static::assertEquals( + array_to_rows( + [ + ['id' => 1, 'country_code' => 'PL'], + ['id' => 2, 'country_code' => 'US'], + ['id' => 3, 'country_code' => 'FR'], + ], + schema(int_schema('id'), str_schema('country_code')), + ), + $joined, + ); + } + + public function test_left_join_to_empty(): void + { + $left = rows(schema()); + + $joined = (new RowsJoin(new AdaptiveBackend()))->left( + $left, + array_to_rows( + [ + ['code' => 'PL', 'name' => 'Poland'], + ['code' => 'US', 'name' => 'United States'], + ['code' => 'GB', 'name' => 'Great Britain'], + ], + schema(str_schema('code'), str_schema('name')), + ), + Expression::on(['country' => 'code']), + ); + + static::assertEquals( + rows(schema(str_schema('code', nullable: true), str_schema('name', nullable: true))), + $joined, + ); + } + + public function test_left_join_with_the_duplicated_columns(): void + { + $this->expectException(SchemaDefinitionNotUniqueException::class); + $this->expectExceptionMessage( + 'Entry definitions must be unique, duplicated entries: [id], all: [id, country, id, code, name]', + ); + + $left = array_to_rows( + [['id' => 1, 'country' => 'PL'], ['id' => 2, 'country' => 'US'], ['id' => 3, 'country' => 'FR']], + schema(int_schema('id'), str_schema('country')), + ); + + (new RowsJoin(new AdaptiveBackend()))->left( + $left, + array_to_rows( + [ + ['id' => 100, 'code' => 'PL', 'name' => 'Poland'], + ['id' => 101, 'code' => 'US', 'name' => 'United States'], + ['id' => 102, 'code' => 'GB', 'name' => 'Great Britain'], + ], + schema(int_schema('id'), str_schema('code'), str_schema('name')), + ), + Expression::on(['country' => 'code'], ''), + ); + } + + public function test_left_join_without_prefix(): void + { + $left = array_to_rows( + [ + ['id' => 1, 'country_code' => 'PL'], + ['id' => 2, 'country_code' => 'US'], + ['id' => 3, 'country_code' => 'FR'], + ], + schema(int_schema('id'), str_schema('country_code')), + ); + + $joined = (new RowsJoin(new AdaptiveBackend()))->left( + $left, + array_to_rows( + [ + ['country_code' => 'PL', 'name' => 'Poland'], + ['country_code' => 'US', 'name' => 'United States'], + ['country_code' => 'GB', 'name' => 'Great Britain'], + ], + schema(str_schema('country_code'), str_schema('name')), + ), + Expression::on(['country_code' => 'country_code']), + ); + + static::assertEquals( + [ + ['id' => 1, 'country_code' => 'PL', 'name' => 'Poland'], + ['id' => 2, 'country_code' => 'US', 'name' => 'United States'], + ['id' => 3, 'country_code' => 'FR', 'name' => null], + ], + $joined->toArray(), + ); + } + + public function test_right_join(): void + { + $left = array_to_rows( + [ + ['id' => 1, 'country' => 'PL'], + ['id' => 2, 'country' => 'PL'], + ['id' => 3, 'country' => 'US'], + ['id' => 4, 'country' => 'FR'], + ], + schema(int_schema('id'), str_schema('country')), + ); + + $joined = (new RowsJoin(new AdaptiveBackend()))->right( + $left, + array_to_rows( + [ + ['code' => 'PL', 'name' => 'Poland'], + ['code' => 'US', 'name' => 'United States'], + ['code' => 'GB', 'name' => 'Great Britain'], + ], + schema(str_schema('code'), str_schema('name')), + ), + Expression::on(['country' => 'code'], 'joined_'), + ); + + static::assertEquals( + [ + ['id' => 1, 'country' => 'PL', 'joined_code' => 'PL', 'joined_name' => 'Poland'], + ['id' => 2, 'country' => 'PL', 'joined_code' => 'PL', 'joined_name' => 'Poland'], + ['id' => 3, 'country' => 'US', 'joined_code' => 'US', 'joined_name' => 'United States'], + ['id' => null, 'country' => null, 'joined_code' => 'GB', 'joined_name' => 'Great Britain'], + ], + $joined->toArray(), + ); + } + + public function test_right_join_empty(): void + { + $left = array_to_rows( + [ + ['id' => 1, 'country' => 'PL'], + ['id' => 2, 'country' => 'PL'], + ['id' => 3, 'country' => 'US'], + ['id' => 4, 'country' => 'FR'], + ], + schema(int_schema('id'), str_schema('country')), + ); + + $joined = (new RowsJoin(new AdaptiveBackend()))->right( + $left, + rows(schema()), + Expression::on(['country' => 'code']), + ); + + static::assertEquals( + rows(schema(int_schema('id', nullable: true), str_schema('country', nullable: true))), + $joined, + ); + } + + public function test_right_join_to_empty(): void + { + $left = rows(schema()); + + $joined = (new RowsJoin(new AdaptiveBackend()))->right( + $left, + array_to_rows( + [ + ['code' => 'PL', 'name' => 'Poland'], + ['code' => 'US', 'name' => 'United States'], + ['code' => 'GB', 'name' => 'Great Britain'], + ], + schema(str_schema('code'), str_schema('name')), + ), + Expression::on(['country' => 'code'], 'joined_'), + ); + + static::assertEquals( + [ + ['joined_code' => 'PL', 'joined_name' => 'Poland'], + ['joined_code' => 'US', 'joined_name' => 'United States'], + ['joined_code' => 'GB', 'joined_name' => 'Great Britain'], + ], + $joined->toArray(), + ); + } + + public function test_right_join_with_duplicated_entry_names(): void + { + $this->expectException(SchemaDefinitionNotUniqueException::class); + $this->expectExceptionMessage( + 'Entry definitions must be unique, duplicated entries: [id], all: [id, country, id, code, name]', + ); + + $left = array_to_rows( + [ + ['id' => 1, 'country' => 'PL'], + ['id' => 2, 'country' => 'PL'], + ['id' => 3, 'country' => 'US'], + ['id' => 4, 'country' => 'FR'], + ], + schema(int_schema('id'), str_schema('country')), + ); + + (new RowsJoin(new AdaptiveBackend()))->right( + $left, + array_to_rows( + [ + ['id' => 101, 'code' => 'PL', 'name' => 'Poland'], + ['id' => 102, 'code' => 'US', 'name' => 'United States'], + ['id' => 103, 'code' => 'GB', 'name' => 'Great Britain'], + ], + schema(int_schema('id'), str_schema('code'), str_schema('name')), + ), + Expression::on(['country' => 'code'], ''), + ); + } + + public function test_right_join_without_prefix(): void + { + $left = array_to_rows( + [ + ['id' => 1, 'country_code' => 'PL'], + ['id' => 2, 'country_code' => 'PL'], + ['id' => 3, 'country_code' => 'US'], + ['id' => 4, 'country_code' => 'FR'], + ], + schema(int_schema('id'), str_schema('country_code')), + ); + + $joined = (new RowsJoin(new AdaptiveBackend()))->right( + $left, + array_to_rows( + [ + ['country_code' => 'PL', 'name' => 'Poland'], + ['country_code' => 'US', 'name' => 'United States'], + ['country_code' => 'GB', 'name' => 'Great Britain'], + ], + schema(str_schema('country_code'), str_schema('name')), + ), + Expression::on(['country_code' => 'country_code']), + ); + + static::assertEquals( + [ + ['id' => 1, 'country_code' => 'PL', 'name' => 'Poland'], + ['id' => 2, 'country_code' => 'PL', 'name' => 'Poland'], + ['id' => 3, 'country_code' => 'US', 'name' => 'United States'], + ['id' => null, 'country_code' => 'GB', 'name' => 'Great Britain'], + ], + $joined->toArray(), + ); + } +} diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Loader/ArrayLoaderTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Loader/ArrayLoaderTest.php index 81bda654ac..9cb9922bb6 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Unit/Loader/ArrayLoaderTest.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Loader/ArrayLoaderTest.php @@ -6,10 +6,9 @@ use Flow\ETL\Tests\FlowTestCase; +use function Flow\ETL\DSL\array_to_rows; use function Flow\ETL\DSL\flow_context; use function Flow\ETL\DSL\int_schema; -use function Flow\ETL\DSL\row; -use function Flow\ETL\DSL\rows; use function Flow\ETL\DSL\schema; use function Flow\ETL\DSL\str_schema; use function Flow\ETL\DSL\to_array; @@ -18,24 +17,34 @@ final class ArrayLoaderTest extends FlowTestCase { public function test_loads_rows_data_into_memory(): void { - $rows1 = rows( + $rows1 = array_to_rows( + [['number' => 1, 'name' => 'one'], ['number' => 2, 'name' => 'two']], schema(int_schema('number'), str_schema('name')), - row(['number' => 1, 'name' => 'one']), - row(['number' => 2, 'name' => 'two']), ); - $rows2 = rows( + $rows2 = array_to_rows( + [['number' => 3, 'name' => 'three'], ['number' => 4, 'name' => 'four']], schema(int_schema('number'), str_schema('name')), - row(['number' => 3, 'name' => 'three']), - row(['number' => 4, 'name' => 'four']), ); + $rows3 = array_to_rows([['number' => 5, 'name' => 'five']], schema(int_schema('number'), str_schema('name'))); + $array = []; $loader = to_array($array); $loader->load($rows1, flow_context()); $loader->load($rows2, flow_context()); - - static::assertEquals($rows1->merge($rows2)->toArray(), $array); + $loader->load($rows3, flow_context()); + + static::assertSame( + [ + ['number' => 1, 'name' => 'one'], + ['number' => 2, 'name' => 'two'], + ['number' => 3, 'name' => 'three'], + ['number' => 4, 'name' => 'four'], + ['number' => 5, 'name' => 'five'], + ], + $array, + ); } } diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Loader/File/FileWriteFrameTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Loader/File/FileWriteFrameTest.php new file mode 100644 index 0000000000..dc82bc2526 --- /dev/null +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Loader/File/FileWriteFrameTest.php @@ -0,0 +1,156 @@ +write( + array_to_rows([['id' => 1, 'group' => 'a'], ['id' => 2, 'group' => 'b']], $schema), + flow_context(), + to_array($output), + ); + $frame->write(array_to_rows([['id' => 3, 'group' => 'a']], $schema), flow_context(), to_array($output)); + $frame->closure(); + + static::assertSame(['memory://out/group=a/file.txt', 'memory://out/group=b/file.txt'], $sinks->opened); + static::assertSame( + 'rows:1;rows:1;closed;', + $filesystem->readFrom(path('memory://out/group=a/file.txt'))->content(), + ); + static::assertSame('rows:1;closed;', $filesystem->readFrom(path('memory://out/group=b/file.txt'))->content()); + } + + public function test_closure_closes_every_sink_before_it_publishes(): void + { + $filesystem = memory_filesystem(); + $frame = new FileWriteFrame( + $filesystem, + path('memory://out/file.txt'), + SaveMode::ExceptionIfExists, + new PartitionRouter(Partitioning::none()), + new SpyFileSinks(), + ); + $output = []; + + $frame->write(array_to_rows([['id' => 1]], schema(int_schema('id'))), flow_context(), to_array($output)); + $frame->closure(); + + static::assertSame('rows:1;closed;', $filesystem->readFrom(path('memory://out/file.txt'))->content()); + } + + public function test_discard_closes_every_sink_and_leaves_no_file(): void + { + $filesystem = memory_filesystem(); + $sinks = new SpyFileSinks(); + $frame = new FileWriteFrame( + $filesystem, + path('memory://out/file.txt'), + SaveMode::ExceptionIfExists, + new PartitionRouter(Partitioning::none()), + $sinks, + ); + $output = []; + + $frame->write(array_to_rows([['id' => 1]], schema(int_schema('id'))), flow_context(), to_array($output)); + $frame->discard(); + + static::assertSame(['memory://out/file.txt'], $sinks->closed); + static::assertNull($filesystem->status(path('memory://out/file.txt'))); + } + + public function test_a_throwing_sink_reports_the_load_as_failed(): void + { + $telemetry = new MemoryTelemetryContext(telemetry_options(trace_loading: true)); + $telemetry->flowContext->telemetry()->dataFrameStarted($telemetry->flowContext); + $frame = new FileWriteFrame( + memory_filesystem(), + path('memory://out/file.txt'), + SaveMode::ExceptionIfExists, + new PartitionRouter(Partitioning::none()), + new SpyFileSinks(failing: true), + ); + $output = []; + + try { + $frame->write( + array_to_rows([['id' => 1]], schema(int_schema('id'))), + $telemetry->flowContext, + to_array($output), + ); + static::fail('the sink failure must reach the caller'); + } catch (RuntimeException $e) { + static::assertSame('sink refused the batch', $e->getMessage()); + } + + $failed = $telemetry->spans->endedSpans(); + static::assertCount(1, $failed); + static::assertTrue($failed[0]->status()?->isError() ?? false); + } + + public function test_closure_closes_every_sink_then_rethrows_the_first_close_failure(): void + { + $sinks = new SpyFileSinks(failingClose: true); + $frame = new FileWriteFrame( + memory_filesystem(), + path('memory://out/file.txt'), + SaveMode::ExceptionIfExists, + new PartitionRouter(partition_by('group')), + $sinks, + ); + $output = []; + + $frame->write( + array_to_rows( + [['id' => 1, 'group' => 'a'], ['id' => 2, 'group' => 'b']], + schema(int_schema('id'), str_schema('group')), + ), + flow_context(), + to_array($output), + ); + + try { + $frame->closure(); + static::fail('a failing close must reach the caller'); + } catch (RuntimeException $e) { + static::assertSame('close of memory://out/group=a/file.txt failed', $e->getMessage()); + } + + static::assertSame(['memory://out/group=a/file.txt', 'memory://out/group=b/file.txt'], $sinks->closed); + } +} diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Loader/File/FilesSinkTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Loader/File/FilesSinkTest.php new file mode 100644 index 0000000000..e1564d19e7 --- /dev/null +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Loader/File/FilesSinkTest.php @@ -0,0 +1,65 @@ +writeTo()->append('half written'); + + $files->abandon(); + + // a format writer flushes its footer on close, so the file must not be removed before that happens + static::assertLessThan( + array_search('rm', $filesystem->calls, true), + array_search('close', $filesystem->calls, true), + ); + } + + public function test_a_destination_without_an_extension_is_refused(): void + { + $this->expectException(RuntimeException::class); + $this->expectExceptionMessage('Stream path must have an extension'); + + (new FilesSink(memory_filesystem(), path('memory://out'), exception_if_exists()))->writeTo(); + } + + public function test_a_pattern_destination_is_refused(): void + { + $this->expectException(RuntimeException::class); + $this->expectExceptionMessage("Destination path can't be pattern"); + + (new FilesSink(memory_filesystem(), path('memory://out/*.csv'), exception_if_exists()))->writeTo(); + } + + public function test_a_placeholder_destination_without_partitions_is_refused(): void + { + $this->expectException(RuntimeException::class); + $this->expectExceptionMessage('contains partition placeholders but rows are not partitioned'); + + (new FilesSink(memory_filesystem(), path('memory://out/{name}.csv'), exception_if_exists()))->writeTo(); + } + + public function test_the_same_destination_returns_the_same_stream(): void + { + $files = new FilesSink(memory_filesystem(), path('memory://out.csv'), exception_if_exists()); + + static::assertSame($files->writeTo(), $files->writeTo()); + } +} diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Loader/File/PartitionRouterTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Loader/File/PartitionRouterTest.php new file mode 100644 index 0000000000..2cd3624daf --- /dev/null +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Loader/File/PartitionRouterTest.php @@ -0,0 +1,132 @@ + 1, 'region' => 'eu'], ['id' => 2, 'region' => 'us'], ['id' => 3, 'region' => 'eu']], + schema(int_schema('id'), str_schema('region')), + ), + ); + + static::assertSame( + [['eu', [1, 3]], ['us', [2]]], + array_map(static fn(array $group): array => [ + $group[0]->get('region')->value, + $group[1]->reduceToArray(ref('id')), + ], $groups), + ); + } + + public function test_partition_columns_are_stripped_from_the_body_by_default(): void + { + $groups = PartitionRoutingContext::route( + Partitioning::by(ref('region')), + array_to_rows([['id' => 1, 'region' => 'eu']], schema(int_schema('id'), str_schema('region'))), + ); + + static::assertSame(['id'], $groups[0][1]->schema()->references()->names()); + static::assertSame([['id' => 1]], $groups[0][1]->toArray()); + } + + public function test_write_columns_keeps_the_partition_column_in_the_body(): void + { + $groups = PartitionRoutingContext::route( + Partitioning::by(ref('region'))->writeColumns(), + array_to_rows([['id' => 1, 'region' => 'eu']], schema(int_schema('id'), str_schema('region'))), + ); + + static::assertSame(['id', 'region'], $groups[0][1]->schema()->references()->names()); + } + + public function test_a_null_partition_value_becomes_the_hive_sentinel_on_the_path(): void + { + $groups = PartitionRoutingContext::route( + Partitioning::by(ref('region')), + array_to_rows( + [['id' => 1, 'region' => null]], + schema(int_schema('id'), str_schema('region', nullable: true)), + ), + ); + + static::assertNull($groups[0][0]->get('region')->value); + static::assertSame('region=__HIVE_DEFAULT_PARTITION__', $groups[0][0]->get('region')->segment()); + } + + public function test_partitions_keep_declaration_order_not_name_order(): void + { + $groups = PartitionRoutingContext::route( + Partitioning::by(ref('year'), ref('day'), ref('month')), + array_to_rows( + [['id' => 1, 'year' => '2024', 'month' => '03', 'day' => '01']], + schema(int_schema('id'), str_schema('year'), str_schema('month'), str_schema('day')), + ), + ); + + static::assertSame( + ['year', 'day', 'month'], + array_map(static fn($partition): string => $partition->name, $groups[0][0]->toArray()), + ); + } + + public function test_partitioning_by_every_column_leaves_nothing_to_write(): void + { + $this->expectException(InvalidArgumentException::class); + $this->expectExceptionMessage('No column left to write, every column is a partition column: "region"'); + + PartitionRoutingContext::route( + Partitioning::by('region'), + array_to_rows([['region' => 'eu']], schema(str_schema('region'))), + ); + } + + public function test_partitioning_by_every_column_is_allowed_with_write_columns(): void + { + static::assertSame( + [['region' => 'eu']], + PartitionRoutingContext::route( + Partitioning::by('region')->writeColumns(), + array_to_rows([['region' => 'eu']], schema(str_schema('region'))), + )[0][1]->toArray(), + ); + } + + public function test_equal_datetime_instants_route_to_one_partition_and_keep_the_path_name(): void + { + $groups = PartitionRoutingContext::route( + Partitioning::by(ref('at')), + array_to_rows( + [ + ['id' => 1, 'at' => new DateTimeImmutable('2026-01-02 10:00:00 UTC')], + ['id' => 2, 'at' => new DateTimeImmutable('2026-01-02 10:00:00 UTC')], + ], + schema(int_schema('id'), datetime_schema('at')), + ), + ); + + static::assertCount(1, $groups); + static::assertSame([1, 2], $groups[0][1]->reduceToArray(ref('id'))); + static::assertSame('2026-01-02', $groups[0][0]->get('at')->value); + } +} diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Loader/MemoryLoaderTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Loader/MemoryLoaderTest.php index d4646c0ab6..f2502e8a01 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Unit/Loader/MemoryLoaderTest.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Loader/MemoryLoaderTest.php @@ -7,11 +7,10 @@ use Flow\ETL\Memory\ArrayMemory; use Flow\ETL\Tests\FlowTestCase; +use function Flow\ETL\DSL\array_to_rows; use function Flow\ETL\DSL\config; use function Flow\ETL\DSL\flow_context; use function Flow\ETL\DSL\int_schema; -use function Flow\ETL\DSL\row; -use function Flow\ETL\DSL\rows; use function Flow\ETL\DSL\schema; use function Flow\ETL\DSL\str_schema; use function Flow\ETL\DSL\to_memory; @@ -20,10 +19,9 @@ final class MemoryLoaderTest extends FlowTestCase { public function test_loads_rows_data_into_memory(): void { - $rows = rows( + $rows = array_to_rows( + [['number' => 1, 'name' => 'one'], ['number' => 2, 'name' => 'two']], schema(int_schema('number'), str_schema('name')), - row(['number' => 1, 'name' => 'one']), - row(['number' => 2, 'name' => 'two']), ); $memory = new ArrayMemory(); diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Loader/PartitionRouterTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Loader/PartitionRouterTest.php deleted file mode 100644 index 0fc1227884..0000000000 --- a/src/core/etl/tests/Flow/ETL/Tests/Unit/Loader/PartitionRouterTest.php +++ /dev/null @@ -1,112 +0,0 @@ - 1, 'region' => 'eu']), - row(['id' => 2, 'region' => 'us']), - row(['id' => 3, 'region' => 'eu']), - ), - ); - - static::assertSame( - [['eu', [1, 3]], ['us', [2]]], - array_map(static fn(array $group): array => [ - $group[0]->get('region')->value, - $group[1]->reduceToArray(ref('id')), - ], $groups), - ); - } - - public function test_partition_columns_are_stripped_from_the_body_by_default(): void - { - $groups = PartitionRoutingContext::route( - Partitioning::by(ref('region')), - rows(schema(int_schema('id'), str_schema('region')), row(['id' => 1, 'region' => 'eu'])), - ); - - static::assertSame(['id'], $groups[0][1]->schema()->references()->names()); - static::assertSame([['id' => 1]], $groups[0][1]->toArray()); - } - - public function test_write_columns_keeps_the_partition_column_in_the_body(): void - { - $groups = PartitionRoutingContext::route( - Partitioning::by(ref('region'))->writeColumns(), - rows(schema(int_schema('id'), str_schema('region')), row(['id' => 1, 'region' => 'eu'])), - ); - - static::assertSame(['id', 'region'], $groups[0][1]->schema()->references()->names()); - } - - public function test_a_null_partition_value_becomes_the_hive_sentinel_on_the_path(): void - { - $groups = PartitionRoutingContext::route( - Partitioning::by(ref('region')), - rows(schema(int_schema('id'), str_schema('region', nullable: true)), row(['id' => 1, 'region' => null])), - ); - - static::assertNull($groups[0][0]->get('region')->value); - static::assertSame('region=__HIVE_DEFAULT_PARTITION__', $groups[0][0]->get('region')->segment()); - } - - public function test_partitions_keep_declaration_order_not_name_order(): void - { - $groups = PartitionRoutingContext::route( - Partitioning::by(ref('year'), ref('day'), ref('month')), - rows( - schema(int_schema('id'), str_schema('year'), str_schema('month'), str_schema('day')), - row(['id' => 1, 'year' => '2024', 'month' => '03', 'day' => '01']), - ), - ); - - static::assertSame( - ['year', 'day', 'month'], - array_map(static fn($partition): string => $partition->name, $groups[0][0]->toArray()), - ); - } - - public function test_partitioning_by_every_column_leaves_nothing_to_write(): void - { - $this->expectException(InvalidArgumentException::class); - $this->expectExceptionMessage('No column left to write, every column is a partition column: "region"'); - - PartitionRoutingContext::route( - Partitioning::by('region'), - rows(schema(str_schema('region')), row(['region' => 'eu'])), - ); - } - - public function test_partitioning_by_every_column_is_allowed_with_write_columns(): void - { - static::assertSame( - [['region' => 'eu']], - PartitionRoutingContext::route( - Partitioning::by('region')->writeColumns(), - rows(schema(str_schema('region')), row(['region' => 'eu'])), - )[0][1]->toArray(), - ); - } -} diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Loader/RowPartitionsTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Loader/RowPartitionsTest.php index 55c95d55a0..368ae9965f 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Unit/Loader/RowPartitionsTest.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Loader/RowPartitionsTest.php @@ -11,6 +11,7 @@ use Flow\Filesystem\Exception\InvalidArgumentException; use PHPUnit\Framework\Attributes\DataProvider; +use function Flow\ETL\DSL\array_to_rows; use function Flow\ETL\DSL\bool_schema; use function Flow\ETL\DSL\datetime_schema; use function Flow\ETL\DSL\float_schema; @@ -18,7 +19,6 @@ use function Flow\ETL\DSL\list_schema; use function Flow\ETL\DSL\ref; use function Flow\ETL\DSL\refs; -use function Flow\ETL\DSL\row; use function Flow\ETL\DSL\schema; use function Flow\ETL\DSL\str_schema; use function Flow\ETL\DSL\structure_schema; @@ -55,15 +55,28 @@ public function test_a_scalar_column_becomes_its_formatted_value( ): void { static::assertSame( $expected, - (new RowPartitions(refs(ref('v'))))->of(row(['v' => $value]), schema($definition))->get('v')->value, + (new RowPartitions(refs(ref('v'))))->of(array_to_rows([['v' => $value]], schema($definition)), 0)->get( + 'v', + )->value, + ); + } + + public function test_an_aliased_partition_reference_reads_its_base_column(): void + { + static::assertSame( + '7', + (new RowPartitions(refs(ref('v')->as('alias'))))->of( + array_to_rows([['v' => 7, 'alias' => 9]], schema(int_schema('v'), int_schema('alias'))), + 0, + )->get('alias')->value, ); } public function test_a_null_value_stays_null(): void { - static::assertNull((new RowPartitions(refs(ref('v'))))->of(row([ + static::assertNull((new RowPartitions(refs(ref('v'))))->of(array_to_rows([[ 'v' => null, - ]), schema(str_schema('v', nullable: true)))->get('v')->value); + ]], schema(str_schema('v', nullable: true))), 0)->get('v')->value); } public function test_a_list_column_is_refused(): void @@ -71,10 +84,10 @@ public function test_a_list_column_is_refused(): void $this->expectException(InvalidArgumentException::class); $this->expectExceptionMessage('can\'t be used as a partition'); - (new RowPartitions(refs(ref('v'))))->of(row(['v' => [ + (new RowPartitions(refs(ref('v'))))->of(array_to_rows([['v' => [ 1, 2, - ]]), schema(list_schema('v', type_list(type_integer())))); + ]]], schema(list_schema('v', type_list(type_integer())))), 0); } public function test_a_structure_column_is_refused(): void @@ -84,8 +97,8 @@ public function test_a_structure_column_is_refused(): void $structure = type_structure(['a' => type_string()]); - (new RowPartitions(refs(ref('v'))))->of(row(['v' => $structure->cast([ + (new RowPartitions(refs(ref('v'))))->of(array_to_rows([['v' => $structure->cast([ 'a' => 'x', - ])]), schema(structure_schema('v', $structure))); + ])]], schema(structure_schema('v', $structure))), 0); } } diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Loader/SchemaValidationLoaderTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Loader/SchemaValidationLoaderTest.php index cc6e3569ce..a3acbfcbf8 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Unit/Loader/SchemaValidationLoaderTest.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Loader/SchemaValidationLoaderTest.php @@ -9,14 +9,13 @@ use Flow\ETL\Schema\Validator\StrictValidator; use Flow\ETL\Tests\FlowTestCase; +use function Flow\ETL\DSL\array_to_rows; use function Flow\ETL\DSL\config; use function Flow\ETL\DSL\datetime_schema; use function Flow\ETL\DSL\flow_context; use function Flow\ETL\DSL\int_schema; use function Flow\ETL\DSL\integer_schema; use function Flow\ETL\DSL\null_schema; -use function Flow\ETL\DSL\row; -use function Flow\ETL\DSL\rows; use function Flow\ETL\DSL\schema; use function Flow\ETL\DSL\str_schema; use function Flow\ETL\DSL\string_schema; @@ -34,7 +33,7 @@ public function test_schema_validation_failed_by_mismatching(): void $loader = new SchemaValidationLoader(schema(integer_schema('id')), new StrictValidator()); - $loader->load(rows(schema(str_schema('id')), row(['id' => '1'])), flow_context(config())); + $loader->load(array_to_rows([['id' => '1']], schema(str_schema('id'))), flow_context(config())); } public function test_schema_validation_failed_by_unexpected(): void @@ -50,7 +49,7 @@ public function test_schema_validation_failed_by_unexpected(): void $loader = new SchemaValidationLoader(schema(integer_schema('id')), new StrictValidator()); - $loader->load(rows(schema(str_schema('name')), row(['name' => '1'])), flow_context(config())); + $loader->load(array_to_rows([['name' => '1']], schema(str_schema('name'))), flow_context(config())); } public function test_schema_validation_failure_message_lists_only_definitions_rejected_by_validator(): void @@ -62,7 +61,7 @@ public function test_schema_validation_failure_message_lists_only_definitions_re try { $loader->load( - rows(schema(int_schema('id'), null_schema('deleted_at')), row(['id' => 1, 'deleted_at' => null])), + array_to_rows([['id' => 1, 'deleted_at' => null]], schema(int_schema('id'), null_schema('deleted_at'))), flow_context(config()), ); static::fail('SchemaValidationException was not thrown'); @@ -76,7 +75,7 @@ public function test_schema_validation_succeed(): void { $loader = new SchemaValidationLoader(schema(integer_schema('id')), new StrictValidator()); - $loader->load(rows(schema(int_schema('id')), row(['id' => 1])), flow_context(config())); + $loader->load(array_to_rows([['id' => 1]], schema(int_schema('id'))), flow_context(config())); // validate that error wasn't thrown $this->addToAssertionCount(1); diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Loader/StreamLoaderTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Loader/StreamLoaderTest.php index d8e712b7ac..14fead110e 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Unit/Loader/StreamLoaderTest.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Loader/StreamLoaderTest.php @@ -11,11 +11,10 @@ use Flow\ETL\Tests\FlowTestCase; use Flow\Filesystem\Stream\Mode; +use function Flow\ETL\DSL\array_to_rows; use function Flow\ETL\DSL\config; use function Flow\ETL\DSL\flow_context; use function Flow\ETL\DSL\int_schema; -use function Flow\ETL\DSL\row; -use function Flow\ETL\DSL\rows; use function Flow\ETL\DSL\schema; use function Flow\ETL\DSL\str_schema; use function Flow\ETL\DSL\to_output; @@ -35,11 +34,9 @@ public function test_columns_count_to_php_output_stream(): void ob_start(); $loader->load( - rows( + array_to_rows( + [['id' => 1, 'name' => 'id_1'], ['id' => 2, 'name' => 'id_2'], ['id' => 3, 'name' => 'id_3']], schema(int_schema('id'), str_schema('name')), - row(['id' => 1, 'name' => 'id_1']), - row(['id' => 2, 'name' => 'id_2']), - row(['id' => 3, 'name' => 'id_3']), ), flow_context(config()), ); @@ -60,11 +57,9 @@ public function test_loading_data_into_invalid_stream(): void $loader = to_stream('php://qweqweqw', 0); $loader->load( - rows( + array_to_rows( + [['id' => 1, 'name' => 'id_1'], ['id' => 2, 'name' => 'id_2'], ['id' => 3, 'name' => 'id_3']], schema(int_schema('id'), str_schema('name')), - row(['id' => 1, 'name' => 'id_1']), - row(['id' => 2, 'name' => 'id_2']), - row(['id' => 3, 'name' => 'id_3']), ), flow_context(config()), ); @@ -77,11 +72,9 @@ public function test_loading_rows_and_schema_into_output_stream(): void ob_start(); $loader->load( - rows( + array_to_rows( + [['id' => 1, 'name' => 'id_1'], ['id' => 2, 'name' => 'id_2'], ['id' => 3, 'name' => 'id_3']], schema(int_schema('id'), str_schema('name')), - row(['id' => 1, 'name' => 'id_1']), - row(['id' => 2, 'name' => 'id_2']), - row(['id' => 3, 'name' => 'id_3']), ), flow_context(config()), ); @@ -112,11 +105,9 @@ public function test_loading_rows_into_php_output_stream(): void ob_start(); $loader->load( - rows( + array_to_rows( + [['id' => 1, 'name' => 'id_1'], ['id' => 2, 'name' => 'id_2'], ['id' => 3, 'name' => 'id_3']], schema(int_schema('id'), str_schema('name')), - row(['id' => 1, 'name' => 'id_1']), - row(['id' => 2, 'name' => 'id_2']), - row(['id' => 3, 'name' => 'id_3']), ), flow_context(config()), ); @@ -142,11 +133,9 @@ public function test_loading_schema_into_php_output_stream(): void ob_start(); $loader->load( - rows( + array_to_rows( + [['id' => 1, 'name' => 'id_1'], ['id' => 2, 'name' => 'id_2'], ['id' => 3, 'name' => 'id_3']], schema(int_schema('id'), str_schema('name')), - row(['id' => 1, 'name' => 'id_1']), - row(['id' => 2, 'name' => 'id_2']), - row(['id' => 3, 'name' => 'id_3']), ), flow_context(config()), ); @@ -168,11 +157,9 @@ public function test_rows_and_columns_count_to_php_output_stream(): void ob_start(); $loader->load( - rows( + array_to_rows( + [['id' => 1, 'name' => 'id_1'], ['id' => 2, 'name' => 'id_2'], ['id' => 3, 'name' => 'id_3']], schema(int_schema('id'), str_schema('name')), - row(['id' => 1, 'name' => 'id_1']), - row(['id' => 2, 'name' => 'id_2']), - row(['id' => 3, 'name' => 'id_3']), ), flow_context(config()), ); @@ -192,11 +179,9 @@ public function test_rows_count_to_php_output_stream(): void ob_start(); $loader->load( - rows( + array_to_rows( + [['id' => 1, 'name' => 'id_1'], ['id' => 2, 'name' => 'id_2'], ['id' => 3, 'name' => 'id_3']], schema(int_schema('id'), str_schema('name')), - row(['id' => 1, 'name' => 'id_1']), - row(['id' => 2, 'name' => 'id_2']), - row(['id' => 3, 'name' => 'id_3']), ), flow_context(config()), ); diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Memory/ArrayMemoryTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Memory/ArrayMemoryTest.php index 8ce1950ef8..4977f6a44e 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Unit/Memory/ArrayMemoryTest.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Memory/ArrayMemoryTest.php @@ -70,8 +70,9 @@ public function test_saving_multiple_entries_into_memory(): void $memory = new ArrayMemory(); $memory->save([['id' => 1], ['id' => 2]]); $memory->save([['id' => 3], ['id' => 4]]); + $memory->save([['id' => 5]]); - static::assertSame([['id' => 1], ['id' => 2], ['id' => 3], ['id' => 4]], $memory->dump()); - static::assertcount(4, $memory); + static::assertSame([['id' => 1], ['id' => 2], ['id' => 3], ['id' => 4], ['id' => 5]], $memory->dump()); + static::assertcount(5, $memory); } } diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Optimizer/Rule/CombineSortAndLimitTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Optimizer/Rule/CombineSortAndLimitTest.php index 89c76c22f8..40b51a92fe 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Unit/Optimizer/Rule/CombineSortAndLimitTest.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Optimizer/Rule/CombineSortAndLimitTest.php @@ -7,7 +7,6 @@ use Flow\ETL\Memory\ArrayMemory; use Flow\ETL\Optimizer\Rule\CombineSortAndLimit; use Flow\ETL\Plan\LogicalPlan; -use Flow\ETL\Plan\Node\Limit; use Flow\ETL\Plan\Node\Outputs; use Flow\ETL\Plan\Node\Result; use Flow\ETL\Plan\Node\Sort; @@ -16,8 +15,6 @@ use Flow\ETL\Tests\FlowTestCase; use Flow\ETL\Tests\Mother\NodeMother; -use function Flow\ETL\DSL\config_builder; -use function Flow\ETL\DSL\external_sort; use function Flow\ETL\DSL\ref; use function Flow\ETL\DSL\refs; use function Flow\ETL\DSL\to_memory; @@ -48,19 +45,6 @@ public function test_a_limit_over_anything_else_is_left_alone(): void static::assertSame($plan->root, (new CombineSortAndLimit())->apply($plan, NodeMother::context())->root); } - public function test_the_rule_uses_the_context_it_is_given(): void - { - $context = NodeMother::context(config_builder()->sort(external_sort()->runSize(2))->build()); - $plan = NodeMother::plan(NodeMother::limit(NodeMother::sort(NodeMother::read()), 3)); - - static::assertInstanceOf( - Limit::class, - (new CombineSortAndLimit()) - ->apply($plan, $context) - ->spine(), - ); - } - public function test_a_sort_another_consumer_reads_stays_for_that_consumer(): void { $sort = NodeMother::sort(NodeMother::read()); diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Optimizer/Rule/CountFromStatisticsTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Optimizer/Rule/CountFromStatisticsTest.php index de5434b921..1c8079b83b 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Unit/Optimizer/Rule/CountFromStatisticsTest.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Optimizer/Rule/CountFromStatisticsTest.php @@ -19,6 +19,7 @@ use function Flow\ETL\Adapter\Parquet\from_parquet; use function Flow\ETL\Adapter\Parquet\to_parquet; +use function Flow\ETL\DSL\array_to_rows; use function Flow\ETL\DSL\df; use function Flow\ETL\DSL\from_array; use function Flow\ETL\DSL\from_rows; @@ -27,8 +28,6 @@ use function Flow\ETL\DSL\join_on; use function Flow\ETL\DSL\lit; use function Flow\ETL\DSL\ref; -use function Flow\ETL\DSL\row; -use function Flow\ETL\DSL\rows; use function Flow\ETL\DSL\schema; use function Flow\ETL\DSL\to_memory; use function Flow\Filesystem\DSL\memory_filesystem; @@ -46,7 +45,7 @@ public function test_a_count_over_a_plain_array_read_reads_the_number_instead(): ); static::assertEquals( - from_rows(rows(schema(int_schema('count')), row(['count' => 3]))), + from_rows(array_to_rows([['count' => 3]], schema(int_schema('count')))), $plan->source()->extractor(), ); } @@ -59,7 +58,7 @@ public function test_a_count_over_a_plain_sequence_read_reads_the_number_instead ); static::assertEquals( - from_rows(rows(schema(int_schema('count')), row(['count' => 10]))), + from_rows(array_to_rows([['count' => 10]], schema(int_schema('count')))), $plan->source()->extractor(), ); } @@ -81,7 +80,7 @@ public function test_a_count_over_a_plain_parquet_read_reads_the_number_instead( ); static::assertEquals( - from_rows(rows(schema(int_schema('count')), row(['count' => 25]))), + from_rows(array_to_rows([['count' => 25]], schema(int_schema('count')))), $plan->source()->extractor(), ); } @@ -103,7 +102,7 @@ public function test_a_count_over_a_plain_floe_read_reads_the_number_instead(): ); static::assertEquals( - from_rows(rows(schema(int_schema('count')), row(['count' => 25]))), + from_rows(array_to_rows([['count' => 25]], schema(int_schema('count')))), $plan->source()->extractor(), ); } diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Optimizer/Rule/PushFilterIntoSourceTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Optimizer/Rule/PushFilterIntoSourceTest.php index 193b22f18d..5c0c37cb02 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Unit/Optimizer/Rule/PushFilterIntoSourceTest.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Optimizer/Rule/PushFilterIntoSourceTest.php @@ -23,6 +23,7 @@ use Flow\Filesystem\Path\Filter\OnlyFiles; use Flow\Types\Exception\InvalidArgumentException; +use function Flow\ETL\DSL\array_to_rows; use function Flow\ETL\DSL\df; use function Flow\ETL\DSL\from_array; use function Flow\ETL\DSL\int_schema; @@ -30,8 +31,6 @@ use function Flow\ETL\DSL\lit; use function Flow\ETL\DSL\random_string; use function Flow\ETL\DSL\ref; -use function Flow\ETL\DSL\row; -use function Flow\ETL\DSL\rows; use function Flow\ETL\DSL\schema; use function Flow\ETL\DSL\to_memory; use function Flow\Types\DSL\type_boolean; @@ -300,7 +299,7 @@ public function test_a_source_without_partition_columns_is_never_pushed_into(): { $extractor = new RecordingFileExtractor( schema(int_schema('year')), - rows(schema(int_schema('year')), row(['year' => 2023])), + array_to_rows([['year' => 2023]], schema(int_schema('year'))), ); df() @@ -384,11 +383,11 @@ public function test_the_extractor_instance_is_never_mutated(): void $frame->fetch(); static::assertStringEndsWith( - "Extractor: RecordingFileExtractor\n Source: file://dev/null", + "Extractor: RecordingFileExtractor\n Source: file:///dev/null", $frame->explain()->toString(Stage::unoptimized), ); static::assertStringEndsWith( - "Extractor: RecordingFileExtractor\n Source: file://dev/null\n Files: Filters", + "Extractor: RecordingFileExtractor\n Source: file:///dev/null\n Files: Filters", $frame->explain()->toString(), ); } diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Optimizer/TopNRewriteTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Optimizer/TopNRewriteTest.php index 6e1232c096..053bafae5b 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Unit/Optimizer/TopNRewriteTest.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Optimizer/TopNRewriteTest.php @@ -4,16 +4,14 @@ namespace Flow\ETL\Tests\Unit\Optimizer; +use Flow\ETL\Dataset\Memory\Unit; use Flow\ETL\Optimizer\TopNRewrite; -use Flow\ETL\Plan\Node\Limit; use Flow\ETL\Plan\Node\Sort; use Flow\ETL\Plan\Node\TopN; use Flow\ETL\Tests\FlowTestCase; use Flow\ETL\Tests\Mother\NodeMother; -use function Flow\ETL\DSL\config_builder; use function Flow\ETL\DSL\external_sort; -use function Flow\ETL\DSL\memory_sort; use function Flow\ETL\DSL\ref; use function Flow\ETL\DSL\refs; @@ -23,14 +21,14 @@ public function test_a_node_that_is_not_a_limit_is_returned_as_is(): void { $sort = NodeMother::sort(NodeMother::read()); - static::assertSame($sort, (new TopNRewrite(NodeMother::context()))->of($sort)); + static::assertSame($sort, (new TopNRewrite())->of($sort)); } public function test_a_limit_over_anything_but_a_sort_is_returned_as_is(): void { $limit = NodeMother::limit(NodeMother::select(NodeMother::sort(NodeMother::read())), 5); - static::assertSame($limit, (new TopNRewrite(NodeMother::context()))->of($limit)); + static::assertSame($limit, (new TopNRewrite())->of($limit)); } public function test_a_limit_over_a_sort_becomes_a_top_n_over_the_sorts_input_refs_and_limit(): void @@ -38,7 +36,7 @@ public function test_a_limit_over_a_sort_becomes_a_top_n_over_the_sorts_input_re $read = NodeMother::read(); $refs = refs(ref('id')); - $topN = (new TopNRewrite(NodeMother::context()))->of(NodeMother::limit(new Sort($read, $refs), 5)); + $topN = (new TopNRewrite())->of(NodeMother::limit(new Sort($read, $refs), 5)); static::assertInstanceOf(TopN::class, $topN); static::assertSame([$read], $topN->children()); @@ -46,31 +44,16 @@ public function test_a_limit_over_a_sort_becomes_a_top_n_over_the_sorts_input_re static::assertSame(5, $topN->limit); } - public function test_an_external_sort_is_rewritten_up_to_its_run_size_and_kept_beyond_it(): void + public function test_a_limit_over_any_sort_is_rewritten_and_keeps_the_pinned_algorithm(): void { - $rewrite = new TopNRewrite(NodeMother::context()); - $sort = new Sort(NodeMother::read(), refs(ref('id')), external_sort()->runSize(10)); + $algorithm = external_sort()->memoryLimit(Unit::fromMb(1)); - static::assertInstanceOf(TopN::class, $rewrite->of(NodeMother::limit($sort, 10))); - static::assertInstanceOf(Limit::class, $rewrite->of(NodeMother::limit($sort, 11))); - } + $topN = (new TopNRewrite())->of(NodeMother::limit( + new Sort(NodeMother::read(), refs(ref('id')), $algorithm), + 1_000_000, + )); - public function test_the_configured_sort_decides_when_the_sort_pins_none(): void - { - $rewrite = new TopNRewrite(NodeMother::context(config_builder()->sort(external_sort()->runSize(2))->build())); - $sort = NodeMother::sort(NodeMother::read()); - - static::assertInstanceOf(TopN::class, $rewrite->of(NodeMother::limit($sort, 2))); - static::assertInstanceOf(Limit::class, $rewrite->of(NodeMother::limit($sort, 3))); - } - - public function test_a_sort_that_pins_its_algorithm_wins_over_the_configured_one(): void - { - $rewrite = new TopNRewrite(NodeMother::context(config_builder()->sort(external_sort()->runSize(2))->build())); - - static::assertInstanceOf( - TopN::class, - $rewrite->of(NodeMother::limit(new Sort(NodeMother::read(), refs(ref('id')), memory_sort()), 1_000_000)), - ); + static::assertInstanceOf(TopN::class, $topN); + static::assertSame($algorithm, $topN->algorithm); } } diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Plan/Explain/PhysicalOutlineTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Plan/Explain/PhysicalOutlineTest.php index bce4212b75..d8d612ddeb 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Unit/Plan/Explain/PhysicalOutlineTest.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Plan/Explain/PhysicalOutlineTest.php @@ -4,6 +4,7 @@ namespace Flow\ETL\Tests\Unit\Plan\Explain; +use Flow\ETL\Dataset\Memory\Unit; use Flow\ETL\Plan\Explain\PhysicalOutline; use Flow\ETL\Plan\Explain\TreeLayout; use Flow\ETL\Tests\Double\UndescribableRowLessExtractor; @@ -12,6 +13,7 @@ use function Flow\ETL\DSL\data_frame; use function Flow\ETL\DSL\from_array; +use function Flow\ETL\DSL\hash_join; use function Flow\ETL\DSL\join_on; use function Flow\ETL\DSL\to_output; @@ -72,6 +74,7 @@ public function test_a_joined_frame_is_a_plan_of_its_own_under_the_step_that_rea │ On: id = id │ Prefix: joined_ │ Storage: FilesystemBuckets + │ Memory: 64 MB │ Buckets: 64 │ Batch: 1000 └─ Right side: Pipeline #0 @@ -84,6 +87,7 @@ public function test_a_joined_frame_is_a_plan_of_its_own_under_the_step_that_rea ->join( data_frame()->read(from_array([['id' => 1]])), join_on(['id' => 'id'], join_prefix: 'joined_'), + algorithm: hash_join()->memoryLimit(Unit::fromMb(64)), ) ->collect() ->write(to_output(truncate: false)), diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Plan/Explain/StepDetailsTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Plan/Explain/StepDetailsTest.php index 1515448c22..872856ea41 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Unit/Plan/Explain/StepDetailsTest.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Plan/Explain/StepDetailsTest.php @@ -4,12 +4,10 @@ namespace Flow\ETL\Tests\Unit\Plan\Explain; -use Flow\ETL\Bucketing\Buckets; -use Flow\ETL\Bucketing\HashBucketing; -use Flow\ETL\Bucketing\NativeHasher; use Flow\ETL\Bucketing\Storage\MemoryBuckets; use Flow\ETL\Cardinality; use Flow\ETL\Constraint\UniqueConstraint; +use Flow\ETL\Dataset\Memory\Unit; use Flow\ETL\Executor; use Flow\ETL\Extractor\Statistics; use Flow\ETL\GroupBy; @@ -17,27 +15,27 @@ use Flow\ETL\Join\Comparison\Equal; use Flow\ETL\Join\Expression; use Flow\ETL\Join\Join; -use Flow\ETL\NativePHPRandomValueGenerator; use Flow\ETL\Plan\Explain\StepDetails; use Flow\ETL\Processor\BatchingByProcessor; use Flow\ETL\Processor\BatchingProcessor; -use Flow\ETL\Processor\BucketingProcessor; use Flow\ETL\Processor\CachingProcessor; use Flow\ETL\Processor\CollectingProcessor; use Flow\ETL\Processor\ConstrainedProcessor; -use Flow\ETL\Processor\GroupByAggregationProcessor; use Flow\ETL\Processor\MemorySortProcessor; use Flow\ETL\Processor\MergeSortProcessor; use Flow\ETL\Processor\OffsetProcessor; use Flow\ETL\Processor\PivotProcessor; -use Flow\ETL\Processor\RepartitionProcessor; use Flow\ETL\Processor\TopNProcessor; use Flow\ETL\Processor\VoidProcessor; use Flow\ETL\Processor\WindowProcessor; +use Flow\ETL\Row\NullsOrder; use Flow\ETL\Tests\Double\DeclaringExtractor; use Flow\ETL\Tests\FlowTestCase; +use Flow\ETL\Tests\Mother\ExternalSortMother; +use Flow\ETL\Tests\Mother\GroupByAggregationProcessorMother; use Flow\ETL\Tests\Mother\HashJoinProcessorMother; use Flow\ETL\Tests\Mother\PhysicalPlanMother; +use Flow\ETL\Tests\Mother\RepartitionProcessorMother; use Flow\ETL\Transformer\CrossJoinRowsTransformer; use function Flow\ETL\DSL\from_array; @@ -110,19 +108,23 @@ public function test_sorting_steps_list_their_columns_with_the_direction(): void $details->settings(new MemorySortProcessor(refs(ref('id'), ref('name')->desc()))), ); static::assertSame( - ['Sort: id desc', 'Spill: MemoryBuckets', 'Merge: 4 ways', 'Batch: 50'], - $details->settings( - new MergeSortProcessor( - refs(ref('id')->desc()), - new Buckets(new MemoryBuckets()), - new Buckets(new MemoryBuckets()), - new NativePHPRandomValueGenerator(), - 4, - 50, - ), - ), + ['Sort: id desc', 'Memory: 64 MB', 'Spill: MemoryBuckets', 'Merge: 4 ways', 'Batch: 50'], + $details->settings(new MergeSortProcessor(ExternalSortMother::with( + refs(ref('id')->desc()), + new MemoryBuckets(), + Unit::fromMb(64), + 4, + 50, + ))), ); static::assertSame(['Top: 3', 'Sort: id asc'], $details->settings(new TopNProcessor(refs(ref('id')), 3))); + static::assertSame( + ['Sort: id asc nulls last, name desc nulls first'], + $details->settings(new MemorySortProcessor(refs( + ref('id')->asc(NullsOrder::LAST), + ref('name')->desc(NullsOrder::FIRST), + ))), + ); } public function test_a_group_by_lists_its_columns_and_aggregations(): void @@ -131,10 +133,13 @@ public function test_a_group_by_lists_its_columns_and_aggregations(): void $groupBy->aggregate(sum(ref('id'))); static::assertSame( - ['Group by: name', 'Aggregations: Sum', 'Storage: MemoryBuckets', 'Batch: 25'], - (new StepDetails())->settings( - new GroupByAggregationProcessor($groupBy, new Buckets(new MemoryBuckets()), 25), - ), + ['Group by: name', 'Aggregations: Sum', 'Memory: 64 MB', 'Storage: MemoryBuckets', 'Batch: 25'], + (new StepDetails())->settings(GroupByAggregationProcessorMother::with( + $groupBy, + new MemoryBuckets(), + Unit::fromMb(64), + batchSize: 25, + )), ); } @@ -159,10 +164,12 @@ public function test_a_pivot_lists_the_column_it_pivots_on(): void public function test_a_repartition_lists_its_columns_hasher_and_storage(): void { static::assertSame( - ['By: id', 'Hasher: NativeHasher', 'Storage: MemoryBuckets'], - (new StepDetails())->settings( - new RepartitionProcessor(refs(ref('id')), new Buckets(new MemoryBuckets()), new NativeHasher()), - ), + ['By: id', 'Hasher: NativeHasher', 'Memory: 64 MB', 'Storage: MemoryBuckets'], + (new StepDetails())->settings(RepartitionProcessorMother::with( + refs(ref('id')), + new MemoryBuckets(), + Unit::fromMb(64), + )), ); } @@ -235,17 +242,4 @@ public function test_a_source_that_declares_nothing_lists_no_statistics(): void { static::assertSame([], (new StepDetails())->settings(new DeclaringExtractor(new Statistics()))); } - - public function test_bucketing_lists_its_strategy_and_storage(): void - { - static::assertSame( - ['Strategy: HashBucketing', 'Storage: MemoryBuckets'], - (new StepDetails())->settings( - new BucketingProcessor( - new HashBucketing([ref('id')], 4, new NativeHasher(), new NativePHPRandomValueGenerator(), 'test'), - new Buckets(new MemoryBuckets()), - ), - ), - ); - } } diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Planner/NodeTranslatorTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Planner/NodeTranslatorTest.php index be3b1624e9..08272e2083 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Unit/Planner/NodeTranslatorTest.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Planner/NodeTranslatorTest.php @@ -44,7 +44,6 @@ use Flow\ETL\Planner\NodeTranslator; use Flow\ETL\Processor\BatchingByProcessor; use Flow\ETL\Processor\BatchingProcessor; -use Flow\ETL\Processor\BucketingProcessor; use Flow\ETL\Processor\CachingProcessor; use Flow\ETL\Processor\CollectingProcessor; use Flow\ETL\Processor\ConstrainedProcessor; @@ -85,6 +84,7 @@ use PHPUnit\Framework\Attributes\TestWith; use function array_map; +use function Flow\ETL\DSL\array_to_rows; use function Flow\ETL\DSL\df; use function Flow\ETL\DSL\flow_context; use function Flow\ETL\DSL\from_array; @@ -97,8 +97,6 @@ use function Flow\ETL\DSL\rank; use function Flow\ETL\DSL\ref; use function Flow\ETL\DSL\refs; -use function Flow\ETL\DSL\row; -use function Flow\ETL\DSL\rows; use function Flow\ETL\DSL\schema; use function Flow\ETL\DSL\to_memory; use function Flow\ETL\DSL\window; @@ -108,7 +106,7 @@ final class NodeTranslatorTest extends FlowTestCase public function test_aggregate_steps_are_the_exact_list_in_order(): void { static::assertSame( - [PruneEntriesTransformer::class, BucketingProcessor::class, GroupByAggregationProcessor::class], + [PruneEntriesTransformer::class, GroupByAggregationProcessor::class], array_map( static fn($step) => $step::class, NodeTranslator::toSteps( @@ -376,7 +374,7 @@ public function test_rename_steps_are_the_exact_list_in_order(): void public function test_repartition_steps_are_the_exact_list_in_order(): void { static::assertSame( - [BucketingProcessor::class, RepartitionProcessor::class], + [RepartitionProcessor::class], array_map( static fn($step) => $step::class, NodeTranslator::toSteps(new Repartition(NodeMother::read(), refs('id')), NodeMother::context(), []), @@ -438,7 +436,7 @@ public function test_transform_the_instance_the_node_holds_is_the_step(): void public function test_transform_a_stateful_transformer_runs_as_its_fresh_instance(): void { $transformer = new AddRowIndexTransformer('idx', StartFrom::ZERO); - $transformer->transform(rows(schema(int_schema('id')), row(['id' => 1])), flow_context()); + $transformer->transform(array_to_rows([['id' => 1]], schema(int_schema('id'))), flow_context()); $steps = NodeTranslator::toSteps(new Transform(NodeMother::read(), $transformer), NodeMother::context(), []); @@ -447,7 +445,7 @@ public function test_transform_a_stateful_transformer_runs_as_its_fresh_instance static::assertNotSame($transformer, $steps[0]); static::assertSame( [['id' => 1, 'idx' => 0]], - $steps[0]->transform(rows(schema(int_schema('id')), row(['id' => 1])), flow_context())->toArray(), + $steps[0]->transform(array_to_rows([['id' => 1]], schema(int_schema('id'))), flow_context())->toArray(), ); } @@ -499,7 +497,7 @@ public function test_window_column_steps_are_the_exact_list_in_order(): void public function test_window_column_a_partitioned_window_translates_to_repartition_steps_then_the_window_processor(): void { static::assertSame( - [BucketingProcessor::class, RepartitionProcessor::class, WindowProcessor::class], + [RepartitionProcessor::class, WindowProcessor::class], array_map( static fn($step) => $step::class, NodeTranslator::toSteps( diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Planner/SinkFeedFactoryTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Planner/SinkFeedFactoryTest.php index a0c2081d13..d3fce7bd52 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Unit/Planner/SinkFeedFactoryTest.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Planner/SinkFeedFactoryTest.php @@ -19,10 +19,9 @@ use Flow\ETL\Tests\Mother\NodeMother; use RuntimeException; +use function Flow\ETL\DSL\array_to_rows; use function Flow\ETL\DSL\from_array; use function Flow\ETL\DSL\int_schema; -use function Flow\ETL\DSL\row; -use function Flow\ETL\DSL\rows; use function Flow\ETL\DSL\schema; final class SinkFeedFactoryTest extends FlowTestCase @@ -43,7 +42,7 @@ public function test_rows_fed_to_the_pipeline_reach_the_tail_loader(): void new SinkOffers($context->errorHandler()), 7, ); - $feed->load(rows(schema(int_schema('id')), row(['id' => 1]), row(['id' => 2])), $context); + $feed->load(array_to_rows([['id' => 1], ['id' => 2]], schema(int_schema('id'))), $context); $feed->closure($context); static::assertSame([2], $spy->loadedRowCounts()); @@ -65,7 +64,7 @@ public function test_a_host_without_a_schema_feeds_an_empty_schema(): void new SinkOffers($context->errorHandler()), 0, ); - $feed->load(rows(schema(int_schema('id')), row(['id' => 1])), $context); + $feed->load(array_to_rows([['id' => 1]], schema(int_schema('id'))), $context); $feed->closure($context); static::assertNotNull($planned->refusal()); @@ -90,6 +89,6 @@ public function test_a_throw_only_handler_surfaces_a_failing_step(): void $this->expectException(SinkFailure::class); - $feed->load(rows(schema(int_schema('id')), row(['id' => 1])), $context); + $feed->load(array_to_rows([['id' => 1]], schema(int_schema('id'))), $context); } } diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/PlannerTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/PlannerTest.php index 95c21713d2..191d1d00aa 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Unit/PlannerTest.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/PlannerTest.php @@ -54,6 +54,7 @@ use function array_map; use function Flow\ETL\Adapter\CSV\from_csv; use function Flow\ETL\Adapter\CSV\to_csv; +use function Flow\ETL\DSL\array_to_rows; use function Flow\ETL\DSL\config; use function Flow\ETL\DSL\df; use function Flow\ETL\DSL\from_array; @@ -63,8 +64,6 @@ use function Flow\ETL\DSL\memory_sort; use function Flow\ETL\DSL\ref; use function Flow\ETL\DSL\refs; -use function Flow\ETL\DSL\row; -use function Flow\ETL\DSL\rows; use function Flow\ETL\DSL\schema; use function Flow\ETL\DSL\str_schema; use function Flow\ETL\DSL\to_array; @@ -535,7 +534,10 @@ public function test_a_planned_node_with_a_null_schema_is_distinct_from_a_node_n public function test_planning_reads_no_row(): void { - $extractor = new CountingExtractor(schema(int_schema('id')), rows(schema(int_schema('id')), row(['id' => 1]))); + $extractor = new CountingExtractor( + schema(int_schema('id')), + array_to_rows([['id' => 1]], schema(int_schema('id'))), + ); $planned = new PlannedNodes(); (new Planner())->node(NodeMother::select(new Read($extractor)), NodeMother::context(), $planned); diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Processor/BatchingByProcessorTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Processor/BatchingByProcessorTest.php index 7f353db7ec..6c487677ed 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Unit/Processor/BatchingByProcessorTest.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Processor/BatchingByProcessorTest.php @@ -4,6 +4,7 @@ namespace Flow\ETL\Tests\Unit\Processor; +use DateTimeImmutable; use Flow\ETL\Exception\InvalidArgumentException; use Flow\ETL\Extractor\Signal; use Flow\ETL\Processor\BatchingByProcessor; @@ -11,14 +12,17 @@ use Flow\ETL\Tests\Double\CountingExtractor; use Flow\ETL\Tests\FlowTestCase; use Flow\ETL\Tests\Mother\RowsMother; +use Flow\Types\Value\Uuid; +use function array_map; +use function Flow\ETL\DSL\array_to_rows; +use function Flow\ETL\DSL\datetime_schema; use function Flow\ETL\DSL\flow_context; use function Flow\ETL\DSL\int_schema; use function Flow\ETL\DSL\ref; -use function Flow\ETL\DSL\row; -use function Flow\ETL\DSL\rows; use function Flow\ETL\DSL\schema; use function Flow\ETL\DSL\str_schema; +use function Flow\ETL\DSL\uuid_schema; use function iterator_to_array; final class BatchingByProcessorTest extends FlowTestCase @@ -47,12 +51,14 @@ public function test_groups_rows_by_column_value(): void { $processor = new BatchingByProcessor(ref('group')); $generator = (static function () { - yield rows( + yield array_to_rows( + [ + ['group' => 'a', 'id' => 1], + ['group' => 'a', 'id' => 2], + ['group' => 'b', 'id' => 3], + ['group' => 'b', 'id' => 4], + ], schema(str_schema('group'), int_schema('id')), - row(['group' => 'a', 'id' => 1]), - row(['group' => 'a', 'id' => 2]), - row(['group' => 'b', 'id' => 3]), - row(['group' => 'b', 'id' => 4]), ); })(); $result = iterator_to_array($processor->process($generator, flow_context())); @@ -61,8 +67,20 @@ public function test_groups_rows_by_column_value(): void static::assertInstanceOf(Rows::class, $result[1]); static::assertCount(2, $result[0]); static::assertCount(2, $result[1]); - static::assertEquals('a', $result[0]->first()->get('group')); - static::assertEquals('b', $result[1]->first()->get('group')); + static::assertEquals('a', $result[0]->column('group')->value(0)); + static::assertEquals('b', $result[1]->column('group')->value(0)); + } + + public function test_an_empty_batch_is_skipped_without_reading_the_key(): void + { + $generator = (static function () { + yield array_to_rows([], schema(int_schema('id'))); + })(); + + static::assertSame( + [], + iterator_to_array((new BatchingByProcessor(ref('group')))->process($generator, flow_context())), + ); } public function test_handles_empty_input(): void @@ -79,10 +97,9 @@ public function test_handles_single_group(): void { $processor = new BatchingByProcessor(ref('group')); $generator = (static function () { - yield rows( + yield array_to_rows( + [['group' => 'a', 'id' => 1], ['group' => 'a', 'id' => 2]], schema(str_schema('group'), int_schema('id')), - row(['group' => 'a', 'id' => 1]), - row(['group' => 'a', 'id' => 2]), ); })(); $result = iterator_to_array($processor->process($generator, flow_context())); @@ -94,12 +111,14 @@ public function test_respects_min_size(): void { $processor = new BatchingByProcessor(ref('group'), minSize: 3); $generator = (static function () { - yield rows( + yield array_to_rows( + [ + ['group' => 'a', 'id' => 1], + ['group' => 'a', 'id' => 2], + ['group' => 'b', 'id' => 3], + ['group' => 'b', 'id' => 4], + ], schema(str_schema('group'), int_schema('id')), - row(['group' => 'a', 'id' => 1]), - row(['group' => 'a', 'id' => 2]), - row(['group' => 'b', 'id' => 3]), - row(['group' => 'b', 'id' => 4]), ); })(); $result = iterator_to_array($processor->process($generator, flow_context())); @@ -114,4 +133,62 @@ public function test_throws_exception_for_invalid_min_size(): void // @mago-ignore analysis:invalid-argument new BatchingByProcessor(ref('group'), minSize: 0); } + + public function test_equal_datetimes_stay_in_one_batch(): void + { + $generator = (static function () { + yield array_to_rows( + [ + ['dt' => new DateTimeImmutable('2024-01-01 00:00:00'), 'id' => 1], + ['dt' => new DateTimeImmutable('2024-01-01 00:00:00'), 'id' => 2], + ['dt' => new DateTimeImmutable('2024-01-02 00:00:00'), 'id' => 3], + ], + schema(datetime_schema('dt'), int_schema('id')), + ); + yield array_to_rows( + [['dt' => new DateTimeImmutable('2024-01-02 00:00:00'), 'id' => 4]], + schema(datetime_schema('dt'), int_schema('id')), + ); + })(); + + $batches = iterator_to_array((new BatchingByProcessor(ref('dt')))->process($generator, flow_context()), false); + + static::assertSame( + [[1, 2], [3, 4]], + array_map(static fn(Rows $rows): array => $rows->column('id')->values(), $batches), + ); + } + + public function test_equal_uuids_stay_in_one_batch(): void + { + $generator = (static function () { + yield array_to_rows( + [ + ['u' => new Uuid('00000000-0000-4000-8000-000000000001'), 'id' => 1], + ['u' => new Uuid('00000000-0000-4000-8000-000000000001'), 'id' => 2], + ['u' => new Uuid('00000000-0000-4000-8000-000000000002'), 'id' => 3], + ], + schema(uuid_schema('u'), int_schema('id')), + ); + })(); + + $batches = iterator_to_array((new BatchingByProcessor(ref('u')))->process($generator, flow_context()), false); + + static::assertSame( + [[1, 2], [3]], + array_map(static fn(Rows $rows): array => $rows->column('id')->values(), $batches), + ); + } + + public function test_a_batch_under_another_schema_is_conformed_to_the_first(): void + { + $generator = (static function () { + yield array_to_rows([['g' => 'a', 'name' => 'x']], schema(str_schema('g'), str_schema('name', true))); + yield array_to_rows([['g' => 'a']], schema(str_schema('g'))); + })(); + + $batches = iterator_to_array((new BatchingByProcessor(ref('g')))->process($generator, flow_context()), false); + + static::assertSame([['g' => 'a', 'name' => 'x'], ['g' => 'a', 'name' => null]], $batches[0]->toArray()); + } } diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Processor/BatchingProcessorTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Processor/BatchingProcessorTest.php index 58785a6e58..d4440f604b 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Unit/Processor/BatchingProcessorTest.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Processor/BatchingProcessorTest.php @@ -13,10 +13,9 @@ use Flow\ETL\Tests\Mother\RowsMother; use PHPUnit\Framework\Attributes\TestWith; +use function Flow\ETL\DSL\array_to_rows; use function Flow\ETL\DSL\flow_context; use function Flow\ETL\DSL\int_schema; -use function Flow\ETL\DSL\row; -use function Flow\ETL\DSL\rows; use function Flow\ETL\DSL\schema; use function Flow\ETL\DSL\str_schema; @@ -30,8 +29,8 @@ final class BatchingProcessorTest extends FlowTestCase public function test_a_batch_under_another_schema_is_conformed_to_the_first(int $size): void { $generator = (static function () { - yield rows(schema(int_schema('id'), str_schema('name', true)), row(['id' => 1, 'name' => 'a'])); - yield rows(schema(int_schema('id')), row(['id' => 2])); + yield array_to_rows([['id' => 1, 'name' => 'a']], schema(int_schema('id'), str_schema('name', true))); + yield array_to_rows([['id' => 2]], schema(int_schema('id'))); })(); /** @var list $result */ @@ -74,13 +73,7 @@ public function test_handles_exact_batch_size_multiple(): void { $processor = new BatchingProcessor(2); $generator = (static function () { - yield rows( - schema(int_schema('id')), - row(['id' => 1]), - row(['id' => 2]), - row(['id' => 3]), - row(['id' => 4]), - ); + yield array_to_rows([['id' => 1], ['id' => 2], ['id' => 3], ['id' => 4]], schema(int_schema('id'))); })(); /** @var list $result */ $result = iterator_to_array($processor->process($generator, flow_context())); @@ -93,14 +86,13 @@ public function test_handles_single_large_batch(): void { $processor = new BatchingProcessor(3); $generator = (static function () { - yield rows( - schema(int_schema('id')), - row(['id' => 1]), - row(['id' => 2]), - row(['id' => 3]), - row(['id' => 4]), - row(['id' => 5]), - ); + yield array_to_rows([ + ['id' => 1], + ['id' => 2], + ['id' => 3], + ['id' => 4], + ['id' => 5], + ], schema(int_schema('id'))); })(); /** @var list $result */ $result = iterator_to_array($processor->process($generator, flow_context())); @@ -113,11 +105,11 @@ public function test_rebatches_rows_into_fixed_size(): void { $processor = new BatchingProcessor(2); $generator = (static function () { - yield rows(schema(int_schema('id')), row(['id' => 1])); - yield rows(schema(int_schema('id')), row(['id' => 2])); - yield rows(schema(int_schema('id')), row(['id' => 3])); - yield rows(schema(int_schema('id')), row(['id' => 4])); - yield rows(schema(int_schema('id')), row(['id' => 5])); + yield array_to_rows([['id' => 1]], schema(int_schema('id'))); + yield array_to_rows([['id' => 2]], schema(int_schema('id'))); + yield array_to_rows([['id' => 3]], schema(int_schema('id'))); + yield array_to_rows([['id' => 4]], schema(int_schema('id'))); + yield array_to_rows([['id' => 5]], schema(int_schema('id'))); })(); /** @var list $result */ $result = iterator_to_array($processor->process($generator, flow_context())); diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Processor/BucketingProcessorTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Processor/BucketingProcessorTest.php deleted file mode 100644 index defe41df25..0000000000 --- a/src/core/etl/tests/Flow/ETL/Tests/Unit/Processor/BucketingProcessorTest.php +++ /dev/null @@ -1,116 +0,0 @@ -bind($input)->output); - } - - public function test_empty_input_registers_no_buckets_and_yields_nothing(): void - { - $buckets = new Buckets(new MemoryBuckets()); - $processor = new BucketingProcessor( - new HashBucketing([ref('id')], 2, new NativeHasher(), new NativePHPRandomValueGenerator()), - $buckets, - ); - - $generator = (static function () { - yield from []; - })(); - - static::assertCount(0, iterator_to_array($processor->process($generator, flow_context(config())))); - static::assertCount(0, $buckets->all()); - } - - public function test_registers_every_yielded_bucket_in_the_repository(): void - { - $buckets = new Buckets(new MemoryBuckets()); - $processor = new BucketingProcessor( - new HashBucketing([ref('id')], 2, new NativeHasher(), new NativePHPRandomValueGenerator()), - $buckets, - ); - - $generator = (static function () { - yield rows( - schema(int_schema('id')), - row(['id' => 1]), - row(['id' => 2]), - row(['id' => 3]), - row(['id' => 4]), - ); - })(); - - /** @var list $result */ - $result = iterator_to_array($processor->process($generator, flow_context(config()))); - - static::assertCount(count($buckets->all()), $result); - - $registeredIds = array_map(static fn(Bucket $bucket): string => $bucket->id, $buckets->all()); - - foreach ($result as $batch) { - static::assertContains($batch->first()->get(BucketShape::id->value), $registeredIds); - } - } - - public function test_yields_one_metadata_rows_batch_per_bucket(): void - { - $buckets = new Buckets(new MemoryBuckets()); - $processor = new BucketingProcessor( - new SortedRunBucketing([ref('id')], 2, new NativePHPRandomValueGenerator()), - $buckets, - ); - - $generator = (static function () { - yield rows(schema(int_schema('id')), row(['id' => 3]), row(['id' => 1]), row(['id' => 2])); - })(); - - /** @var list $result */ - $result = iterator_to_array($processor->process($generator, flow_context(config()))); - - static::assertCount(2, $result); - - $totalRows = 0; - - foreach ($result as $batch) { - static::assertCount(1, $batch); - $totalRows += (int) $batch->first()->get(BucketShape::totalRows->value); - } - - static::assertSame(3, $totalRows); - } -} diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Processor/CachingProcessorTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Processor/CachingProcessorTest.php index 9208c0412d..590a6db035 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Unit/Processor/CachingProcessorTest.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Processor/CachingProcessorTest.php @@ -7,6 +7,7 @@ use Flow\ETL\Cache\CacheIndex; use Flow\ETL\Cache\Implementation\InMemoryCache; use Flow\ETL\Cardinality; +use Flow\ETL\Column\PhpBackend; use Flow\ETL\Extractor\Signal; use Flow\ETL\Processor\CachingProcessor; use Flow\ETL\Rows; @@ -14,11 +15,10 @@ use Flow\ETL\Tests\FlowTestCase; use Flow\ETL\Tests\Mother\RowsMother; +use function Flow\ETL\DSL\array_to_rows; use function Flow\ETL\DSL\config_builder; use function Flow\ETL\DSL\flow_context; use function Flow\ETL\DSL\int_schema; -use function Flow\ETL\DSL\row; -use function Flow\ETL\DSL\rows; use function Flow\ETL\DSL\schema; use function Flow\ETL\DSL\str_schema; @@ -62,8 +62,8 @@ public function test_caches_batches_and_yields_them(): void $processor = new CachingProcessor('test-cache'); $generator = (static function () { - yield rows(schema(int_schema('id')), row(['id' => 1])); - yield rows(schema(int_schema('id')), row(['id' => 2])); + yield array_to_rows([['id' => 1]], schema(int_schema('id'))); + yield array_to_rows([['id' => 2]], schema(int_schema('id'))); })(); $result = iterator_to_array($processor->process($generator, $context)); @@ -106,12 +106,12 @@ public function test_passes_through_when_cache_exists(): void $cache = new InMemoryCache(); $context = flow_context(config_builder()->cache($cache)->build()); - $cache->set('test-cache', (new CacheIndex('test-cache'))->toRows()); + $cache->set('test-cache', (new CacheIndex('test-cache'))->toRows(new PhpBackend())); $processor = new CachingProcessor('test-cache'); $generator = (static function () { - yield rows(schema(int_schema('id')), row(['id' => 1])); + yield array_to_rows([['id' => 1]], schema(int_schema('id'))); })(); $result = iterator_to_array($processor->process($generator, $context)); @@ -127,7 +127,7 @@ public function test_uses_config_id_when_no_id_provided(): void $processor = new CachingProcessor(); $generator = (static function () { - yield rows(schema(int_schema('id')), row(['id' => 1])); + yield array_to_rows([['id' => 1]], schema(int_schema('id'))); })(); iterator_to_array($processor->process($generator, $context)); diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Processor/CollectingProcessorTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Processor/CollectingProcessorTest.php index c36ad1324b..0e0f60e4aa 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Unit/Processor/CollectingProcessorTest.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Processor/CollectingProcessorTest.php @@ -8,10 +8,9 @@ use Flow\ETL\Rows; use Flow\ETL\Tests\FlowTestCase; +use function Flow\ETL\DSL\array_to_rows; use function Flow\ETL\DSL\flow_context; use function Flow\ETL\DSL\int_schema; -use function Flow\ETL\DSL\row; -use function Flow\ETL\DSL\rows; use function Flow\ETL\DSL\schema; use function Flow\ETL\DSL\str_schema; @@ -46,9 +45,9 @@ public function test_collects_all_rows_into_single_batch(): void $processor = new CollectingProcessor(); $generator = (static function () { - yield rows(schema(int_schema('id')), row(['id' => 1]), row(['id' => 2])); - yield rows(schema(int_schema('id')), row(['id' => 3])); - yield rows(schema(int_schema('id')), row(['id' => 4]), row(['id' => 5])); + yield array_to_rows([['id' => 1], ['id' => 2]], schema(int_schema('id'))); + yield array_to_rows([['id' => 3]], schema(int_schema('id'))); + yield array_to_rows([['id' => 4], ['id' => 5]], schema(int_schema('id'))); })(); /** @var list $result */ @@ -88,7 +87,7 @@ public function test_handles_single_batch(): void $processor = new CollectingProcessor(); $generator = (static function () { - yield rows(schema(int_schema('id')), row(['id' => 1]), row(['id' => 2])); + yield array_to_rows([['id' => 1], ['id' => 2]], schema(int_schema('id'))); })(); /** @var list $result */ diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Processor/ConstrainedProcessorTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Processor/ConstrainedProcessorTest.php index e0e7095c3e..105e15eb99 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Unit/Processor/ConstrainedProcessorTest.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Processor/ConstrainedProcessorTest.php @@ -9,18 +9,19 @@ use Flow\ETL\Exception\InvalidArgumentException; use Flow\ETL\Extractor\Signal; use Flow\ETL\Processor\ConstrainedProcessor; -use Flow\ETL\Row; use Flow\ETL\Rows; -use Flow\ETL\Schema; use Flow\ETL\Tests\Double\CountingExtractor; use Flow\ETL\Tests\FlowTestCase; use Flow\ETL\Tests\Mother\RowsMother; +use function Flow\ETL\DSL\array_to_rows; +use function Flow\ETL\DSL\constraint_sorted_by; +use function Flow\ETL\DSL\constraint_unique; use function Flow\ETL\DSL\flow_context; use function Flow\ETL\DSL\int_schema; -use function Flow\ETL\DSL\row; -use function Flow\ETL\DSL\rows; +use function Flow\ETL\DSL\ref; use function Flow\ETL\DSL\schema; +use function is_int; final class ConstrainedProcessorTest extends FlowTestCase { @@ -48,7 +49,7 @@ public function test_handles_empty_constraints(): void { $processor = new ConstrainedProcessor([]); $generator = (static function () { - yield rows(schema(int_schema('id')), row(['id' => 1])); + yield array_to_rows([['id' => 1]], schema(int_schema('id'))); })(); $result = iterator_to_array($processor->process($generator, flow_context())); static::assertCount(1, $result); @@ -57,9 +58,9 @@ public function test_handles_empty_constraints(): void public function test_handles_empty_input(): void { $constraint = new class implements Constraint { - public function isSatisfiedBy(Row $row, Schema $schema): bool + public function firstViolation(Rows $rows): ?int { - return true; + return null; } public function toString(): string @@ -67,7 +68,7 @@ public function toString(): string return 'always'; } - public function violation(Row $row, Schema $schema): string + public function violation(Rows $rows, int $index): string { return ''; } @@ -83,10 +84,16 @@ public function violation(Row $row, Schema $schema): string public function test_passes_rows_when_all_constraints_satisfied(): void { $constraint = new class implements Constraint { - public function isSatisfiedBy(Row $row, Schema $schema): bool + public function firstViolation(Rows $rows): ?int { - // @mago-ignore analysis:possibly-null-operand,possibly-invalid-operand - return $row->get('id') > 0; + // @mago-ignore analysis:mixed-assignment + foreach ($rows->column('id')->values() as $i => $id) { + if (!is_int($id) || $id <= 0) { + return $i; + } + } + + return null; } public function toString(): string @@ -94,14 +101,14 @@ public function toString(): string return 'id > 0'; } - public function violation(Row $row, Schema $schema): string + public function violation(Rows $rows, int $index): string { return 'id must be greater than 0'; } }; $processor = new ConstrainedProcessor([$constraint]); $generator = (static function () { - yield rows(schema(int_schema('id')), row(['id' => 1]), row(['id' => 2])); + yield array_to_rows([['id' => 1], ['id' => 2]], schema(int_schema('id'))); })(); /** @var list $result */ $result = iterator_to_array($processor->process($generator, flow_context())); @@ -120,10 +127,16 @@ public function test_throws_exception_for_invalid_constraint_type(): void public function test_throws_exception_when_constraint_violated(): void { $constraint = new class implements Constraint { - public function isSatisfiedBy(Row $row, Schema $schema): bool + public function firstViolation(Rows $rows): ?int { - // @mago-ignore analysis:possibly-null-operand,possibly-invalid-operand - return $row->get('id') > 0; + // @mago-ignore analysis:mixed-assignment + foreach ($rows->column('id')->values() as $i => $id) { + if (!is_int($id) || $id <= 0) { + return $i; + } + } + + return null; } public function toString(): string @@ -131,16 +144,43 @@ public function toString(): string return 'id > 0'; } - public function violation(Row $row, Schema $schema): string + public function violation(Rows $rows, int $index): string { return 'id must be greater than 0'; } }; $processor = new ConstrainedProcessor([$constraint]); $generator = (static function () { - yield rows(schema(int_schema('id')), row(['id' => 1]), row(['id' => -1])); + yield array_to_rows([['id' => 1], ['id' => -1]], schema(int_schema('id'))); })(); $this->expectException(ConstraintViolationException::class); iterator_to_array($processor->process($generator, flow_context())); } + + public function test_the_earliest_violation_wins_and_names_the_stream_row(): void + { + $processor = new ConstrainedProcessor([ + constraint_unique('id'), + constraint_sorted_by(ref('n')), + ]); + $generator = (static function () { + yield array_to_rows( + [['id' => 1, 'n' => 1], ['id' => 2, 'n' => 2]], + schema(int_schema('id'), int_schema('n')), + ); + yield array_to_rows( + [['id' => 3, 'n' => 3], ['id' => 4, 'n' => 0], ['id' => 3, 'n' => 5]], + schema(int_schema('id'), int_schema('n')), + ); + })(); + + try { + iterator_to_array($processor->process($generator, flow_context())); + static::fail('expected a ConstraintViolationException'); + } catch (ConstraintViolationException $e) { + static::assertStringContainsString('Sorted constraint on [n ASC]', $e->getMessage()); + static::assertStringContainsString('current: 0, previous: 3', $e->getMessage()); + static::assertStringEndsWith('in row: 3', $e->getMessage()); + } + } } diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Processor/CountingProcessorTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Processor/CountingProcessorTest.php index 0c33f90c24..9439d94826 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Unit/Processor/CountingProcessorTest.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Processor/CountingProcessorTest.php @@ -5,11 +5,13 @@ namespace Flow\ETL\Tests\Unit\Processor; use Flow\ETL\Processor\CountingProcessor; +use Flow\ETL\Tests\Double\SpyBackend; use Flow\ETL\Tests\FlowTestCase; +use function Flow\ETL\DSL\array_to_rows; +use function Flow\ETL\DSL\config_builder; use function Flow\ETL\DSL\flow_context; use function Flow\ETL\DSL\int_schema; -use function Flow\ETL\DSL\row; use function Flow\ETL\DSL\rows; use function Flow\ETL\DSL\schema; use function Flow\ETL\DSL\str_schema; @@ -20,13 +22,13 @@ final class CountingProcessorTest extends FlowTestCase public function test_every_batch_is_counted_into_one_row(): void { $batches = (static function () { - yield rows(schema(int_schema('id')), row(['id' => 1]), row(['id' => 2])); + yield array_to_rows([['id' => 1], ['id' => 2]], schema(int_schema('id'))); yield rows(schema(int_schema('id'))); - yield rows(schema(int_schema('id')), row(['id' => 3])); + yield array_to_rows([['id' => 3]], schema(int_schema('id'))); })(); static::assertEquals( - [rows(schema(int_schema('count')), row(['count' => 3]))], + [array_to_rows([['count' => 3]], schema(int_schema('count')))], iterator_to_array((new CountingProcessor())->process($batches, flow_context())), ); } @@ -38,7 +40,7 @@ public function test_no_batch_counts_to_zero(): void })(); static::assertEquals( - [rows(schema(int_schema('count')), row(['count' => 0]))], + [array_to_rows([['count' => 0]], schema(int_schema('count')))], iterator_to_array((new CountingProcessor())->process($batches, flow_context())), ); } @@ -50,4 +52,19 @@ public function test_bind_outputs_the_count_column_whatever_the_input(): void static::assertEquals(schema(int_schema('count')), $bound->output); static::assertEquals(new CountingProcessor(), $bound->step); } + + public function test_count_builds_with_the_configured_backend(): void + { + $backend = new SpyBackend(); + $batches = (static function () { + yield array_to_rows([['id' => 1]], schema(int_schema('id'))); + })(); + + iterator_to_array((new CountingProcessor())->process( + $batches, + flow_context(config_builder()->backend($backend)->build()), + )); + + static::assertGreaterThan(0, $backend->builders()); + } } diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Processor/GroupByAggregationProcessorTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Processor/GroupByAggregationProcessorTest.php index 21d9f4249c..577b94e520 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Unit/Processor/GroupByAggregationProcessorTest.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Processor/GroupByAggregationProcessorTest.php @@ -4,138 +4,177 @@ namespace Flow\ETL\Tests\Unit\Processor; -use Flow\ETL\Bucketing\Bucket; -use Flow\ETL\Bucketing\Buckets; -use Flow\ETL\Bucketing\HashBucketing; -use Flow\ETL\Bucketing\NativeHasher; use Flow\ETL\Bucketing\Storage\MemoryBuckets; +use Flow\ETL\Dataset\Memory\Unit; use Flow\ETL\Exception\InvalidArgumentException; use Flow\ETL\Exception\SchemaDefinitionNotFoundException; use Flow\ETL\GroupBy; -use Flow\ETL\NativePHPRandomValueGenerator; use Flow\ETL\Processor\GroupByAggregationProcessor; -use Flow\ETL\Tests\Context\GroupByContext; +use Flow\ETL\Tests\Context\GroupedRows; +use Flow\ETL\Tests\Double\ReportedMemoryBackend; +use Flow\ETL\Tests\Double\SpyBackend; use Flow\ETL\Tests\Double\SpyBucketsStorage; use Flow\ETL\Tests\FlowTestCase; +use Flow\ETL\Tests\Mother\GroupByAggregationProcessorMother; use Generator; +use PHPUnit\Framework\Attributes\DataProvider; +use function Flow\ETL\DSL\array_to_rows; +use function Flow\ETL\DSL\collect; +use function Flow\ETL\DSL\config_builder; use function Flow\ETL\DSL\count; +use function Flow\ETL\DSL\first; use function Flow\ETL\DSL\float_schema; use function Flow\ETL\DSL\flow_context; use function Flow\ETL\DSL\int_schema; +use function Flow\ETL\DSL\last; +use function Flow\ETL\DSL\max; +use function Flow\ETL\DSL\min; use function Flow\ETL\DSL\ref; -use function Flow\ETL\DSL\row; -use function Flow\ETL\DSL\rows; use function Flow\ETL\DSL\schema; use function Flow\ETL\DSL\str_schema; use function Flow\ETL\DSL\sum; +use function iterator_to_array; +use function memory_get_usage; +use function mt_rand; +use function mt_srand; +use function serialize; final class GroupByAggregationProcessorTest extends FlowTestCase { - public function test_aggregates_every_bucket_from_the_metadata_stream(): void + public static function random_datasets(): Generator + { + for ($seed = 1; $seed <= 50; $seed++) { + yield "seed {$seed}" => [$seed]; + } + } + + public function test_under_the_limit_it_aggregates_in_one_pass_without_touching_the_buckets(): void { $groupBy = new GroupBy(ref('category')); $groupBy->aggregate(sum(ref('amount'))); + $storage = new SpyBucketsStorage(new MemoryBuckets()); - $buckets = new Buckets(new MemoryBuckets()); - $strategy = new HashBucketing( - [ref('category')], - 4, - new NativeHasher(), - new NativePHPRandomValueGenerator(), - 'group-by', - ); - - $input = (static function (): Generator { - yield rows( - schema(str_schema('category'), int_schema('amount')), - row(['category' => 'a', 'amount' => 10]), - row(['category' => 'b', 'amount' => 15]), - row(['category' => 'a', 'amount' => 20]), - ); - })(); - - $metadata = []; - - foreach ($strategy->bucketize($input, $buckets->storage()) as $bucket) { - $buckets->add($bucket); - $metadata[] = rows(Bucket::schema(), $bucket->toRow()); - } - - $result = iterator_to_array( - (new GroupByAggregationProcessor($groupBy, $buckets))->process( - (static function () use ($metadata): Generator { - yield from $metadata; - })(), + static::assertSame( + ['a' => 30.0, 'b' => 15.0], + GroupedRows::sums(GroupByAggregationProcessorMother::inMemory($groupBy, $storage)->process( + GroupedRows::categories(), flow_context(), - ), - preserve_keys: false, + )), ); - - $aggregated = []; - - foreach ($result as $batch) { - foreach ($batch->toArray() as $groupRow) { - $aggregated[$groupRow['category']] = $groupRow['amount_sum']; - } - } - - ksort($aggregated); - - static::assertSame(['a' => 30.0, 'b' => 15.0], $aggregated); + static::assertSame([], $storage->appendedRows()); } - public function test_aggregates_every_bucket_when_metadata_rows_arrive_in_one_batch(): void + public function test_past_the_limit_the_rest_is_partitioned_and_the_result_is_the_same(): void { $groupBy = new GroupBy(ref('category')); $groupBy->aggregate(sum(ref('amount'))); + $storage = new SpyBucketsStorage(new MemoryBuckets()); - $buckets = new Buckets(new MemoryBuckets()); - $strategy = new HashBucketing( - [ref('category')], - 4, - new NativeHasher(), - new NativePHPRandomValueGenerator(), - 'group-by', + static::assertSame( + ['a' => 30.0, 'b' => 15.0], + GroupedRows::sums(GroupByAggregationProcessorMother::with( + $groupBy, + $storage, + Unit::fromBytes(1), + batchSize: 1, + )->process(GroupedRows::categories(), flow_context())), ); + static::assertNotSame([], $storage->appendedRows()); + static::assertSame([], $storage->liveBucketIds()); + } - $input = (static function (): Generator { - yield rows( - schema(str_schema('category'), int_schema('amount')), - row(['category' => 'a', 'amount' => 10]), - row(['category' => 'b', 'amount' => 15]), - row(['category' => 'a', 'amount' => 20]), - ); - })(); + /** + * Any batch but the last can be the one that passes the limit: aggregation before it and partitioned aggregation + * after it give the result one in-memory pass gives - many keys, nulls, order-dependent aggregates included. + */ + #[DataProvider('random_datasets')] + public function test_crossing_the_limit_at_any_batch_equals_one_in_memory_pass(int $seed): void + { + mt_srand($seed); + $schema = schema(int_schema('k', nullable: true), int_schema('v')); + $batches = []; + + for ($b = 0, $count = mt_rand(3, 8); $b < $count; $b++) { + $rows = []; - $metadataRows = []; + for ($r = 0, $size = mt_rand(1, 20); $r < $size; $r++) { + $rows[] = ['k' => mt_rand(0, 6) === 0 ? null : mt_rand(0, 40), 'v' => mt_rand(-50, 50)]; + } - foreach ($strategy->bucketize($input, $buckets->storage()) as $bucket) { - $buckets->add($bucket); - $metadataRows[] = $bucket->toRow(); + $batches[] = array_to_rows($rows, $schema); } - $result = iterator_to_array( - (new GroupByAggregationProcessor($groupBy, $buckets))->process( - (static function () use ($metadataRows): Generator { - yield rows(Bucket::schema(), ...$metadataRows); - })(), - flow_context(), - ), - preserve_keys: false, + $crossAfter = mt_rand(0, $count - 2); + $groupBy = new GroupBy(ref('k')); + $groupBy->aggregate( + sum(ref('v')), + count(ref('v')), + min(ref('v')), + max(ref('v')), + first(ref('v')), + last(ref('v')), + collect(ref('v')), ); - $aggregated = []; + $backend = new ReportedMemoryBackend(); + $input = (static function () use ($batches, $crossAfter, $backend): Generator { + foreach ($batches as $position => $batch) { + if ($position === $crossAfter) { + $backend->reported = 1_000_000_000_000; + } - foreach ($result as $batch) { - foreach ($batch->toArray() as $groupRow) { - $aggregated[$groupRow['category']] = $groupRow['amount_sum']; + yield $batch; } - } + })(); - ksort($aggregated); + $storage = new SpyBucketsStorage(new MemoryBuckets()); + $spilled = GroupByAggregationProcessorMother::with( + $groupBy, + $storage, + Unit::fromBytes(memory_get_usage(false) + 500_000_000), + batchSize: 3, + )->process($input, flow_context(config_builder()->backend($backend)->build())); + $inMemory = GroupByAggregationProcessorMother::inMemory($groupBy)->process( + (static fn(): Generator => yield from $batches)(), + flow_context(), + ); + + static::assertSame(serialize(GroupedRows::byKey($inMemory, 'k')), serialize(GroupedRows::byKey($spilled, 'k'))); + static::assertNotSame([], $storage->appendedRows()); + } + + public function test_a_key_aggregated_before_the_limit_is_merged_with_its_rows_after_it(): void + { + $schema = schema(str_schema('k'), int_schema('v')); + $backend = new ReportedMemoryBackend(); + $input = (static function () use ($schema, $backend): Generator { + yield array_to_rows([['k' => 'a', 'v' => 1], ['k' => 'b', 'v' => 2]], $schema); + $backend->reported = 1_000_000_000_000; + + yield array_to_rows([['k' => 'a', 'v' => 3], ['k' => 'c', 'v' => 4]], $schema); + yield array_to_rows([['k' => 'a', 'v' => 5]], $schema); + })(); + $groupBy = new GroupBy(ref('k')); + $groupBy->aggregate(first(ref('v')), last(ref('v')), collect(ref('v')), sum(ref('v'))); + $storage = new SpyBucketsStorage(new MemoryBuckets()); - static::assertSame(['a' => 30.0, 'b' => 15.0], $aggregated); + static::assertSame( + [ + 's:1:"a";' => ['k' => 'a', 'v_first' => 1, 'v_last' => 5, 'v_collection' => [1, 3, 5], 'v_sum' => 9.0], + 's:1:"b";' => ['k' => 'b', 'v_first' => 2, 'v_last' => 2, 'v_collection' => [2], 'v_sum' => 2.0], + 's:1:"c";' => ['k' => 'c', 'v_first' => 4, 'v_last' => 4, 'v_collection' => [4], 'v_sum' => 4.0], + ], + GroupedRows::byKey( + GroupByAggregationProcessorMother::with( + $groupBy, + $storage, + Unit::fromBytes(memory_get_usage(false) + 500_000_000), + )->process($input, flow_context(config_builder()->backend($backend)->build())), + 'k', + ), + ); + static::assertNotSame([], $storage->appendedRows()); } public function test_bind_derives_the_group_by_columns_followed_by_the_aggregations(): void @@ -145,7 +184,7 @@ public function test_bind_derives_the_group_by_columns_followed_by_the_aggregati static::assertEquals( schema(str_schema('category'), float_schema('amount_sum', nullable: true)), - (new GroupByAggregationProcessor($groupBy, new Buckets(new MemoryBuckets())))->bind(schema( + GroupByAggregationProcessorMother::inMemory($groupBy)->bind(schema( str_schema('category'), int_schema('amount'), str_schema('dropped'), @@ -160,53 +199,7 @@ public function test_bind_refuses_a_group_by_column_missing_from_the_input(): vo $this->expectException(SchemaDefinitionNotFoundException::class); - (new GroupByAggregationProcessor($groupBy, new Buckets(new MemoryBuckets())))->bind(schema(int_schema( - 'amount', - ))); - } - - public function test_clears_storage_after_the_last_bucket(): void - { - $groupBy = new GroupBy(ref('category')); - $groupBy->aggregate(sum(ref('amount'))); - - $storage = new SpyBucketsStorage(new MemoryBuckets()); - $buckets = new Buckets($storage); - $strategy = new HashBucketing( - [ref('category')], - 4, - new NativeHasher(), - new NativePHPRandomValueGenerator(), - 'group-by', - ); - - $input = (static function (): Generator { - yield rows( - schema(str_schema('category'), int_schema('amount')), - row(['category' => 'a', 'amount' => 10]), - row(['category' => 'b', 'amount' => 15]), - ); - })(); - - $metadata = []; - - foreach ($strategy->bucketize($input, $buckets->storage()) as $bucket) { - $buckets->add($bucket); - $metadata[] = rows(Bucket::schema(), $bucket->toRow()); - } - - iterator_to_array( - (new GroupByAggregationProcessor($groupBy, $buckets))->process( - (static function () use ($metadata): Generator { - yield from $metadata; - })(), - flow_context(), - ), - preserve_keys: false, - ); - - static::assertNotSame([], $storage->readBucketIds()); - static::assertSame([], $storage->liveBucketIds()); + GroupByAggregationProcessorMother::inMemory($groupBy)->bind(schema(int_schema('amount'))); } public function test_throws_when_batch_size_below_one(): void @@ -215,25 +208,30 @@ public function test_throws_when_batch_size_below_one(): void $this->expectExceptionMessage('Batch size must be greater than 0, given: 0'); // @mago-ignore analysis:invalid-argument - new GroupByAggregationProcessor(new GroupBy(ref('category')), new Buckets(new MemoryBuckets()), 0); + GroupByAggregationProcessorMother::with( + new GroupBy(ref('category')), + new MemoryBuckets(), + Unit::fromGb(1), + 4, + 0, + ); } - public function test_handles_an_empty_metadata_stream(): void + public function test_handles_an_empty_stream(): void { $groupBy = new GroupBy(ref('category')); $groupBy->aggregate(sum(ref('amount'))); - $result = iterator_to_array( - (new GroupByAggregationProcessor($groupBy, new Buckets(new MemoryBuckets())))->process( - (static function (): Generator { - yield from []; - })(), - flow_context(), + static::assertSame( + [], + iterator_to_array( + GroupByAggregationProcessorMother::inMemory($groupBy)->process( + (static fn(): Generator => yield from [])(), + flow_context(), + ), + false, ), - preserve_keys: false, ); - - static::assertSame([], $result); } public function test_a_bound_global_aggregate_over_an_empty_stream_emits_one_row_of_defaults(): void @@ -241,20 +239,13 @@ public function test_a_bound_global_aggregate_over_an_empty_stream_emits_one_row $groupBy = new GroupBy(); $groupBy->aggregate(sum(ref('amount')), count(ref('amount'))); - $bound = (new GroupByAggregationProcessor($groupBy, new Buckets(new MemoryBuckets())))->bind(schema(int_schema( - 'amount', - ))); + $bound = GroupByAggregationProcessorMother::inMemory($groupBy)->bind(schema(int_schema('amount'))); static::assertInstanceOf(GroupByAggregationProcessor::class, $bound->step); $result = iterator_to_array( - $bound->step->process( - (static function (): Generator { - yield from []; - })(), - flow_context(), - ), - preserve_keys: false, + $bound->step->process((static fn(): Generator => yield from [])(), flow_context()), + false, ); static::assertCount(1, $result); @@ -267,7 +258,7 @@ public function test_a_bound_keyed_aggregate_over_an_empty_stream_emits_nothing( $groupBy = new GroupBy(ref('category')); $groupBy->aggregate(sum(ref('amount'))); - $bound = (new GroupByAggregationProcessor($groupBy, new Buckets(new MemoryBuckets())))->bind(schema( + $bound = GroupByAggregationProcessorMother::inMemory($groupBy)->bind(schema( str_schema('category'), int_schema('amount'), )); @@ -277,13 +268,8 @@ public function test_a_bound_keyed_aggregate_over_an_empty_stream_emits_nothing( static::assertSame( [], iterator_to_array( - $bound->step->process( - (static function (): Generator { - yield from []; - })(), - flow_context(), - ), - preserve_keys: false, + $bound->step->process((static fn(): Generator => yield from [])(), flow_context()), + false, ), ); } @@ -296,13 +282,11 @@ public function test_an_unbound_global_aggregate_over_an_empty_stream_emits_noth static::assertSame( [], iterator_to_array( - (new GroupByAggregationProcessor($groupBy, new Buckets(new MemoryBuckets())))->process( - (static function (): Generator { - yield from []; - })(), + GroupByAggregationProcessorMother::inMemory($groupBy)->process( + (static fn(): Generator => yield from [])(), flow_context(), ), - preserve_keys: false, + false, ), ); } @@ -312,28 +296,39 @@ public function test_a_bound_global_aggregate_with_input_emits_no_extra_row(): v $groupBy = new GroupBy(); $groupBy->aggregate(sum(ref('amount')), count(ref('amount'))); - $buckets = new Buckets(new MemoryBuckets()); - $metadata = GroupByContext::bucketMetadata( - $buckets, - [], - rows(schema(int_schema('amount')), row(['amount' => 10]), row(['amount' => 20])), - ); - - $bound = (new GroupByAggregationProcessor($groupBy, $buckets))->bind(schema(int_schema('amount'))); + $bound = GroupByAggregationProcessorMother::inMemory($groupBy)->bind(schema(int_schema('amount'))); static::assertInstanceOf(GroupByAggregationProcessor::class, $bound->step); $result = iterator_to_array( $bound->step->process( - (static function () use ($metadata): Generator { - yield from $metadata; - })(), + (static fn(): Generator => yield array_to_rows([ + ['amount' => 10], + ['amount' => 20], + ], schema(int_schema('amount'))))(), flow_context(), ), - preserve_keys: false, + false, ); static::assertCount(1, $result); static::assertSame([['amount_sum' => 30.0, 'amount_count' => 2]], $result[0]->toArray()); } + + public function test_output_builds_with_the_configured_backend(): void + { + $backend = new SpyBackend(); + $groupBy = new GroupBy(ref('category')); + $groupBy->aggregate(sum(ref('amount'))); + + iterator_to_array( + GroupByAggregationProcessorMother::inMemory($groupBy)->process( + GroupedRows::categories(), + flow_context(config_builder()->backend($backend)->build()), + ), + false, + ); + + static::assertGreaterThan(0, $backend->builders()); + } } diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Processor/HashJoinProcessorTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Processor/HashJoinProcessorTest.php index 216cb51983..23829a8c54 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Unit/Processor/HashJoinProcessorTest.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Processor/HashJoinProcessorTest.php @@ -5,6 +5,7 @@ namespace Flow\ETL\Tests\Unit\Processor; use Flow\ETL\Bucketing\Storage\MemoryBuckets; +use Flow\ETL\Dataset\Memory\Unit; use Flow\ETL\Exception\JoinException; use Flow\ETL\Extractor\Signal; use Flow\ETL\Join\Comparison\Any; @@ -12,16 +13,18 @@ use Flow\ETL\Join\Expression; use Flow\ETL\Join\Join; use Flow\ETL\Tests\Double\CountingExtractor; +use Flow\ETL\Tests\Double\SpyBackend; use Flow\ETL\Tests\Double\SpyBucketsStorage; use Flow\ETL\Tests\FlowTestCase; use Flow\ETL\Tests\Mother\HashJoinProcessorMother; use Flow\ETL\Tests\Mother\PhysicalPlanMother; use Flow\ETL\Tests\Mother\RowsMother; +use function Flow\ETL\DSL\array_to_rows; +use function Flow\ETL\DSL\config_builder; use function Flow\ETL\DSL\flow_context; use function Flow\ETL\DSL\from_rows; use function Flow\ETL\DSL\int_schema; -use function Flow\ETL\DSL\row; use function Flow\ETL\DSL\rows; use function Flow\ETL\DSL\schema; use function Flow\ETL\DSL\str_schema; @@ -34,9 +37,9 @@ final class HashJoinProcessorTest extends FlowTestCase public function test_bind_derives_the_joined_schema_from_both_sides(): void { $processor = HashJoinProcessorMother::resident( - PhysicalPlanMother::reading(from_rows(rows( + PhysicalPlanMother::reading(from_rows(array_to_rows( + [['id' => 1, 'name' => 'Alice']], schema(int_schema('id'), str_schema('name')), - row(['id' => 1, 'name' => 'Alice']), ))), Expression::on(['id' => 'id']), Join::inner, @@ -50,14 +53,19 @@ public function test_bind_derives_the_joined_schema_from_both_sides(): void public function test_builds_hash_table_from_the_smaller_side_without_changing_results(): void { - $bigger = rows( + $bigger = array_to_rows( + [ + ['id' => 1, 'amount' => 100], + ['id' => 1, 'amount' => 150], + ['id' => 2, 'amount' => 200], + ['id' => 3, 'amount' => 300], + ], schema(int_schema('id'), int_schema('amount')), - row(['id' => 1, 'amount' => 100]), - row(['id' => 1, 'amount' => 150]), - row(['id' => 2, 'amount' => 200]), - row(['id' => 3, 'amount' => 300]), ); - $smaller = rows(schema(int_schema('user_id'), str_schema('name')), row(['user_id' => 1, 'name' => 'Alice'])); + $smaller = array_to_rows( + [['user_id' => 1, 'name' => 'Alice']], + schema(int_schema('user_id'), str_schema('name')), + ); $leftBiggerJoined = []; @@ -95,17 +103,13 @@ public function test_builds_hash_table_from_the_smaller_side_without_changing_re public function test_grace_and_resident_storages_produce_the_same_rows(): void { - $leftRows = rows( + $leftRows = array_to_rows( + [['id' => 1, 'amount' => 100], ['id' => 2, 'amount' => 200], ['id' => 404, 'amount' => 300]], schema(int_schema('id'), int_schema('amount')), - row(['id' => 1, 'amount' => 100]), - row(['id' => 2, 'amount' => 200]), - row(['id' => 404, 'amount' => 300]), ); - $rightRows = rows( + $rightRows = array_to_rows( + [['user_id' => 1, 'name' => 'Alice'], ['user_id' => 2, 'name' => 'Bob'], ['user_id' => 3, 'name' => 'Cid']], schema(int_schema('user_id'), str_schema('name')), - row(['user_id' => 1, 'name' => 'Alice']), - row(['user_id' => 2, 'name' => 'Bob']), - row(['user_id' => 3, 'name' => 'Cid']), ); $results = []; @@ -152,12 +156,76 @@ public function test_grace_and_resident_storages_produce_the_same_rows(): void static::assertSame($results[0], $results[1]); } + public function test_a_build_side_under_the_memory_limit_joins_in_memory_without_bucket_writes(): void + { + $storage = new SpyBucketsStorage(new MemoryBuckets()); + $processor = HashJoinProcessorMother::with( + PhysicalPlanMother::reading(from_rows(array_to_rows( + [['user_id' => 1, 'name' => 'Alice'], ['user_id' => 2, 'name' => 'Bob']], + schema(int_schema('user_id'), str_schema('name')), + ))), + Expression::on(['id' => 'user_id']), + Join::left, + $storage, + memoryLimit: Unit::fromGb(64), + ); + $generator = (static function () { + yield array_to_rows( + [['id' => 2, 'amount' => 20], ['id' => 404, 'amount' => 40]], + schema(int_schema('id'), int_schema('amount')), + ); + })(); + + static::assertSame( + [ + ['id' => 2, 'amount' => 20, 'user_id' => 2, 'name' => 'Bob'], + ['id' => 404, 'amount' => 40, 'user_id' => null, 'name' => null], + ], + iterator_to_array($processor->process($generator, flow_context()), false)[0]->toArray(), + ); + static::assertSame([], $storage->appendedRows()); + } + + public function test_a_build_side_past_the_memory_limit_spills_what_it_read_and_the_rest(): void + { + $storage = new SpyBucketsStorage(new MemoryBuckets()); + $processor = HashJoinProcessorMother::with( + PhysicalPlanMother::reading(from_rows( + array_to_rows([['user_id' => 1, 'name' => 'Alice']], schema(int_schema('user_id'), str_schema('name'))), + array_to_rows([['user_id' => 2, 'name' => 'Bob']], schema(int_schema('user_id'), str_schema('name'))), + )), + Expression::on(['id' => 'user_id']), + Join::inner, + $storage, + memoryLimit: Unit::fromBytes(1), + ); + $generator = (static function () { + yield array_to_rows([['id' => 1], ['id' => 2]], schema(int_schema('id'))); + })(); + + $joined = []; + + foreach ($processor->process($generator, flow_context()) as $batch) { + foreach ($batch->toArray() as $row) { + $joined[] = $row; + } + } + + usort($joined, static fn(array $left, array $right): int => (int) $left['id'] <=> (int) $right['id']); + + static::assertSame( + [['id' => 1, 'user_id' => 1, 'name' => 'Alice'], ['id' => 2, 'user_id' => 2, 'name' => 'Bob']], + $joined, + ); + static::assertNotSame([], $storage->appendedRows()); + } + public function test_handles_empty_left_side(): void { $processor = HashJoinProcessorMother::grace( - PhysicalPlanMother::reading(from_rows(rows( + PhysicalPlanMother::reading(from_rows(array_to_rows( + [['user_id' => 1, 'name' => 'Alice']], schema(int_schema('user_id'), str_schema('name')), - row(['user_id' => 1, 'name' => 'Alice']), ))), Expression::on(['id' => 'user_id']), Join::inner, @@ -179,7 +247,7 @@ public function test_handles_empty_right_side(): void ); $generator = (static function () { - yield rows(schema(int_schema('id'), int_schema('amount')), row(['id' => 1, 'amount' => 100])); + yield array_to_rows([['id' => 1, 'amount' => 100]], schema(int_schema('id'), int_schema('amount'))); })(); $result = iterator_to_array($processor->process($generator, flow_context()), false); @@ -197,21 +265,18 @@ public function test_handles_empty_right_side(): void public function test_inner_join(): void { $processor = HashJoinProcessorMother::grace( - PhysicalPlanMother::reading(from_rows(rows( + PhysicalPlanMother::reading(from_rows(array_to_rows( + [['user_id' => 1, 'name' => 'Alice'], ['user_id' => 2, 'name' => 'Bob']], schema(int_schema('user_id'), str_schema('name')), - row(['user_id' => 1, 'name' => 'Alice']), - row(['user_id' => 2, 'name' => 'Bob']), ))), Expression::on(['id' => 'user_id']), Join::inner, ); $generator = (static function () { - yield rows( + yield array_to_rows( + [['id' => 1, 'amount' => 100], ['id' => 2, 'amount' => 200], ['id' => 3, 'amount' => 300]], schema(int_schema('id'), int_schema('amount')), - row(['id' => 1, 'amount' => 100]), - row(['id' => 2, 'amount' => 200]), - row(['id' => 3, 'amount' => 300]), ); })(); @@ -239,9 +304,9 @@ public function test_inner_join_drops_null_key_rows_without_touching_storage(): $storage = new SpyBucketsStorage(new MemoryBuckets()); $processor = HashJoinProcessorMother::grace( - PhysicalPlanMother::reading(from_rows(rows( + PhysicalPlanMother::reading(from_rows(array_to_rows( + [['user_id' => 1, 'name' => 'Alice']], schema(int_schema('user_id'), str_schema('name')), - row(['user_id' => 1, 'name' => 'Alice']), ))), Expression::on(['id' => 'user_id']), Join::inner, @@ -249,7 +314,7 @@ public function test_inner_join_drops_null_key_rows_without_touching_storage(): ); $generator = (static function () { - yield rows(schema(int_schema('id', nullable: true)), row(['id' => null]), row(['id' => null])); + yield array_to_rows([['id' => null], ['id' => null]], schema(int_schema('id', nullable: true))); })(); static::assertCount(0, iterator_to_array($processor->process($generator, flow_context()), false)); @@ -259,17 +324,16 @@ public function test_inner_join_drops_null_key_rows_without_touching_storage(): public function test_inner_join_emits_every_matching_right_row(): void { $processor = HashJoinProcessorMother::grace( - PhysicalPlanMother::reading(from_rows(rows( + PhysicalPlanMother::reading(from_rows(array_to_rows( + [['user_id' => 1, 'role' => 'admin'], ['user_id' => 1, 'role' => 'writer']], schema(int_schema('user_id'), str_schema('role')), - row(['user_id' => 1, 'role' => 'admin']), - row(['user_id' => 1, 'role' => 'writer']), ))), Expression::on(['id' => 'user_id']), Join::inner, ); $generator = (static function () { - yield rows(schema(int_schema('id')), row(['id' => 1])); + yield array_to_rows([['id' => 1]], schema(int_schema('id'))); })(); $joined = []; @@ -294,13 +358,13 @@ public function test_inner_join_emits_every_matching_right_row(): void public function test_left_anti_join(): void { $processor = HashJoinProcessorMother::grace( - PhysicalPlanMother::reading(from_rows(rows(schema(int_schema('user_id')), row(['user_id' => 1])))), + PhysicalPlanMother::reading(from_rows(array_to_rows([['user_id' => 1]], schema(int_schema('user_id'))))), Expression::on(['id' => 'user_id']), Join::left_anti, ); $generator = (static function () { - yield rows(schema(int_schema('id')), row(['id' => 1]), row(['id' => 2])); + yield array_to_rows([['id' => 1], ['id' => 2]], schema(int_schema('id'))); })(); $joined = []; @@ -319,13 +383,13 @@ public function test_left_anti_join(): void public function test_left_anti_join_keeps_null_key_left_rows(): void { $processor = HashJoinProcessorMother::grace( - PhysicalPlanMother::reading(from_rows(rows(schema(int_schema('user_id')), row(['user_id' => 1])))), + PhysicalPlanMother::reading(from_rows(array_to_rows([['user_id' => 1]], schema(int_schema('user_id'))))), Expression::on(['id' => 'user_id']), Join::left_anti, ); $generator = (static function () { - yield rows(schema(int_schema('id', nullable: true)), row(['id' => 1]), row(['id' => null])); + yield array_to_rows([['id' => 1], ['id' => null]], schema(int_schema('id', nullable: true))); })(); $joined = []; @@ -344,19 +408,18 @@ public function test_left_anti_join_keeps_null_key_left_rows(): void public function test_left_join(): void { $processor = HashJoinProcessorMother::grace( - PhysicalPlanMother::reading(from_rows(rows( + PhysicalPlanMother::reading(from_rows(array_to_rows( + [['user_id' => 1, 'name' => 'Alice']], schema(int_schema('user_id'), str_schema('name')), - row(['user_id' => 1, 'name' => 'Alice']), ))), Expression::on(['id' => 'user_id']), Join::left, ); $generator = (static function () { - yield rows( + yield array_to_rows( + [['id' => 1, 'amount' => 100], ['id' => 2, 'amount' => 200]], schema(int_schema('id'), int_schema('amount')), - row(['id' => 1, 'amount' => 100]), - row(['id' => 2, 'amount' => 200]), ); })(); @@ -380,16 +443,16 @@ public function test_left_join(): void public function test_left_join_pads_null_key_left_rows(): void { $processor = HashJoinProcessorMother::grace( - PhysicalPlanMother::reading(from_rows(rows( + PhysicalPlanMother::reading(from_rows(array_to_rows( + [['user_id' => 1, 'name' => 'Alice']], schema(int_schema('user_id'), str_schema('name')), - row(['user_id' => 1, 'name' => 'Alice']), ))), Expression::on(['id' => 'user_id']), Join::left, ); $generator = (static function () { - yield rows(schema(int_schema('id', nullable: true)), row(['id' => 1]), row(['id' => null])); + yield array_to_rows([['id' => 1], ['id' => null]], schema(int_schema('id', nullable: true))); })(); $joined = []; @@ -416,17 +479,16 @@ public function test_left_join_pads_null_key_left_rows(): void public function test_non_equality_join_falls_back_to_a_single_bucket(): void { $processor = HashJoinProcessorMother::grace( - PhysicalPlanMother::reading(from_rows(rows( + PhysicalPlanMother::reading(from_rows(array_to_rows( + [['user_id' => 1, 'name' => 'Alice'], ['user_id' => 2, 'name' => 'Bob']], schema(int_schema('user_id'), str_schema('name')), - row(['user_id' => 1, 'name' => 'Alice']), - row(['user_id' => 2, 'name' => 'Bob']), ))), Expression::on(new Any(new Equal('id', 'user_id'))), Join::inner, ); $generator = (static function () { - yield rows(schema(int_schema('id')), row(['id' => 1]), row(['id' => 3])); + yield array_to_rows([['id' => 1], ['id' => 3]], schema(int_schema('id'))); })(); $joined = []; @@ -445,19 +507,21 @@ public function test_non_equality_join_falls_back_to_a_single_bucket(): void public function test_resident_storage_preserves_left_row_order(): void { $processor = HashJoinProcessorMother::resident( - PhysicalPlanMother::reading(from_rows(rows( + PhysicalPlanMother::reading(from_rows(array_to_rows( + [ + ['user_id' => 1, 'name' => 'Alice'], + ['user_id' => 2, 'name' => 'Bob'], + ['user_id' => 3, 'name' => 'Cid'], + ], schema(int_schema('user_id'), str_schema('name')), - row(['user_id' => 1, 'name' => 'Alice']), - row(['user_id' => 2, 'name' => 'Bob']), - row(['user_id' => 3, 'name' => 'Cid']), ))), Expression::on(['id' => 'user_id']), Join::inner, ); $generator = (static function () { - yield rows(schema(int_schema('id')), row(['id' => 3]), row(['id' => 1])); - yield rows(schema(int_schema('id')), row(['id' => 2])); + yield array_to_rows([['id' => 3], ['id' => 1]], schema(int_schema('id'))); + yield array_to_rows([['id' => 2]], schema(int_schema('id'))); })(); $joined = []; @@ -483,17 +547,16 @@ public function test_resident_storage_preserves_left_row_order(): void public function test_right_join(): void { $processor = HashJoinProcessorMother::grace( - PhysicalPlanMother::reading(from_rows(rows( + PhysicalPlanMother::reading(from_rows(array_to_rows( + [['user_id' => 1, 'name' => 'Alice'], ['user_id' => 2, 'name' => 'Bob']], schema(int_schema('user_id'), str_schema('name')), - row(['user_id' => 1, 'name' => 'Alice']), - row(['user_id' => 2, 'name' => 'Bob']), ))), Expression::on(['id' => 'user_id']), Join::right, ); $generator = (static function () { - yield rows(schema(int_schema('id'), int_schema('amount')), row(['id' => 1, 'amount' => 100])); + yield array_to_rows([['id' => 1, 'amount' => 100]], schema(int_schema('id'), int_schema('amount'))); })(); $joined = []; @@ -520,17 +583,16 @@ public function test_right_join(): void public function test_right_join_pads_null_key_right_rows(): void { $processor = HashJoinProcessorMother::grace( - PhysicalPlanMother::reading(from_rows(rows( + PhysicalPlanMother::reading(from_rows(array_to_rows( + [['user_id' => 1, 'name' => 'Alice'], ['user_id' => null, 'name' => 'Bob']], schema(int_schema('user_id', nullable: true), str_schema('name')), - row(['user_id' => 1, 'name' => 'Alice']), - row(['user_id' => null, 'name' => 'Bob']), ))), Expression::on(['id' => 'user_id']), Join::right, ); $generator = (static function () { - yield rows(schema(int_schema('id', nullable: true)), row(['id' => 1]), row(['id' => null])); + yield array_to_rows([['id' => 1], ['id' => null]], schema(int_schema('id', nullable: true))); })(); $joined = []; @@ -559,9 +621,9 @@ public function test_storages_are_cleared_after_the_join(): void $storage = new SpyBucketsStorage(new MemoryBuckets()); $processor = HashJoinProcessorMother::grace( - PhysicalPlanMother::reading(from_rows(rows( + PhysicalPlanMother::reading(from_rows(array_to_rows( + [['user_id' => 1, 'name' => 'Alice']], schema(int_schema('user_id'), str_schema('name')), - row(['user_id' => 1, 'name' => 'Alice']), ))), Expression::on(['id' => 'user_id']), Join::left, @@ -569,7 +631,7 @@ public function test_storages_are_cleared_after_the_join(): void ); $generator = (static function () { - yield rows(schema(int_schema('id')), row(['id' => 1]), row(['id' => 2])); + yield array_to_rows([['id' => 1], ['id' => 2]], schema(int_schema('id'))); })(); iterator_to_array($processor->process($generator, flow_context()), false); @@ -581,15 +643,16 @@ public function test_pairs_with_a_missing_right_side_read_only_left_buckets(): v { $storage = new SpyBucketsStorage(new MemoryBuckets()); + // the right side's only key hashes to a bucket no left row lands in, so no pair has a right bucket $processor = HashJoinProcessorMother::grace( - PhysicalPlanMother::reading(from_rows(rows(schema()))), + PhysicalPlanMother::reading(from_rows(array_to_rows([['user_id' => 3]], schema(int_schema('user_id'))))), Expression::on(['id' => 'user_id']), Join::left, $storage, ); $generator = (static function () { - yield rows(schema(int_schema('id')), row(['id' => 1]), row(['id' => 2])); + yield array_to_rows([['id' => 1], ['id' => 2]], schema(int_schema('id'))); })(); iterator_to_array($processor->process($generator, flow_context()), false); @@ -625,16 +688,16 @@ public function test_resident_join_forwards_stop_to_its_left_upstream(): void public function test_duplicated_entries_outside_join_columns_throw_join_exception(): void { $processor = HashJoinProcessorMother::grace( - PhysicalPlanMother::reading(from_rows(rows( + PhysicalPlanMother::reading(from_rows(array_to_rows( + [['user_id' => 1, 'name' => 'Alice']], schema(int_schema('user_id'), str_schema('name')), - row(['user_id' => 1, 'name' => 'Alice']), ))), Expression::on(['id' => 'user_id']), Join::inner, ); $generator = (static function () { - yield rows(schema(int_schema('id'), str_schema('name')), row(['id' => 1, 'name' => 'Norbert'])); + yield array_to_rows([['id' => 1, 'name' => 'Norbert']], schema(int_schema('id'), str_schema('name'))); })(); $this->expectException(JoinException::class); @@ -647,18 +710,42 @@ public function test_duplicated_entries_outside_join_columns_throw_join_exceptio public function test_the_right_side_is_pulled_through_a_frame_output(): void { - $right = PhysicalPlanMother::reading(from_rows(rows( + $right = PhysicalPlanMother::reading(from_rows(array_to_rows( + [['id' => 1, 'name' => 'Alice']], schema(int_schema('id'), str_schema('name')), - row(['id' => 1, 'name' => 'Alice']), ))); $processor = HashJoinProcessorMother::resident($right, Expression::on(['id' => 'id'], 'r_'), Join::inner); $generator = (static function () { - yield rows(schema(int_schema('id')), row(['id' => 1]), row(['id' => 2])); + yield array_to_rows([['id' => 1], ['id' => 2]], schema(int_schema('id'))); })(); $batches = iterator_to_array($processor->process($generator, flow_context()), false); static::assertSame([['id' => 1, 'r_id' => 1, 'r_name' => 'Alice']], $batches[0]->toArray()); } + + public function test_join_output_gathers_the_input_columns_without_building(): void + { + $backend = new SpyBackend(); + $processor = HashJoinProcessorMother::grace( + PhysicalPlanMother::reading(from_rows(array_to_rows( + [['user_id' => 1, 'name' => 'Alice']], + schema(int_schema('user_id'), str_schema('name')), + ))), + Expression::on(['id' => 'user_id']), + Join::inner, + ); + $generator = (static function () { + yield array_to_rows([['id' => 1, 'amount' => 100]], schema(int_schema('id'), int_schema('amount'))); + })(); + + $joined = iterator_to_array( + $processor->process($generator, flow_context(config_builder()->backend($backend)->build())), + false, + ); + + static::assertSame([['id' => 1, 'amount' => 100, 'user_id' => 1, 'name' => 'Alice']], $joined[0]->toArray()); + static::assertSame(0, $backend->builders()); + } } diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Processor/MemorySortProcessorTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Processor/MemorySortProcessorTest.php index dee0c9e0ac..97271be500 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Unit/Processor/MemorySortProcessorTest.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Processor/MemorySortProcessorTest.php @@ -8,12 +8,11 @@ use Flow\ETL\Rows; use Flow\ETL\Tests\FlowTestCase; +use function Flow\ETL\DSL\array_to_rows; use function Flow\ETL\DSL\flow_context; use function Flow\ETL\DSL\int_schema; use function Flow\ETL\DSL\ref; use function Flow\ETL\DSL\refs; -use function Flow\ETL\DSL\row; -use function Flow\ETL\DSL\rows; use function Flow\ETL\DSL\schema; final class MemorySortProcessorTest extends FlowTestCase @@ -37,7 +36,7 @@ public function test_the_declared_schema_wins_over_the_batch_schema(): void $declared = schema(int_schema('id', nullable: true)); $generator = (static function () { - yield rows(schema(int_schema('id')), row(['id' => 2]), row(['id' => 1])); + yield array_to_rows([['id' => 2], ['id' => 1]], schema(int_schema('id'))); })(); /** @var list $result */ @@ -66,9 +65,9 @@ public function test_sorts_across_multiple_batches(): void $processor = new MemorySortProcessor(refs(ref('id'))); $generator = (static function () { - yield rows(schema(int_schema('id')), row(['id' => 3])); - yield rows(schema(int_schema('id')), row(['id' => 1])); - yield rows(schema(int_schema('id')), row(['id' => 2])); + yield array_to_rows([['id' => 3]], schema(int_schema('id'))); + yield array_to_rows([['id' => 1]], schema(int_schema('id'))); + yield array_to_rows([['id' => 2]], schema(int_schema('id'))); })(); /** @var list $result */ @@ -89,7 +88,7 @@ public function test_sorts_rows_ascending(): void $processor = new MemorySortProcessor(refs(ref('id'))); $generator = (static function () { - yield rows(schema(int_schema('id')), row(['id' => 3]), row(['id' => 1]), row(['id' => 2])); + yield array_to_rows([['id' => 3], ['id' => 1], ['id' => 2]], schema(int_schema('id'))); })(); /** @var list $result */ @@ -110,7 +109,7 @@ public function test_sorts_rows_descending(): void $processor = new MemorySortProcessor(refs(ref('id')->desc())); $generator = (static function () { - yield rows(schema(int_schema('id')), row(['id' => 1]), row(['id' => 3]), row(['id' => 2])); + yield array_to_rows([['id' => 1], ['id' => 3], ['id' => 2]], schema(int_schema('id'))); })(); /** @var list $result */ diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Processor/MergeSortProcessorTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Processor/MergeSortProcessorTest.php index 7c4b935896..fa07cde75a 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Unit/Processor/MergeSortProcessorTest.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Processor/MergeSortProcessorTest.php @@ -4,311 +4,56 @@ namespace Flow\ETL\Tests\Unit\Processor; -use Flow\ETL\Bucketing\Bucket; -use Flow\ETL\Bucketing\Buckets; -use Flow\ETL\Bucketing\SortedRunBucketing; use Flow\ETL\Bucketing\Storage\MemoryBuckets; -use Flow\ETL\Exception\InvalidArgumentException; -use Flow\ETL\Exception\RuntimeException; -use Flow\ETL\NativePHPRandomValueGenerator; -use Flow\ETL\Processor\BucketingProcessor; use Flow\ETL\Processor\MergeSortProcessor; -use Flow\ETL\Row; use Flow\ETL\Rows; -use Flow\ETL\Tests\Double\SpyBucketsStorage; -use Flow\ETL\Tests\Double\ThrowingRemoveBucketsStorage; +use Flow\ETL\Tests\Double\RecordingBucketsStorage; use Flow\ETL\Tests\FlowTestCase; +use Flow\ETL\Tests\Mother\ExternalSortMother; use Generator; use function array_map; use function array_merge; +use function Flow\ETL\DSL\array_to_rows; use function Flow\ETL\DSL\config; use function Flow\ETL\DSL\flow_context; use function Flow\ETL\DSL\int_schema; -use function Flow\ETL\DSL\ref; use function Flow\ETL\DSL\refs; -use function Flow\ETL\DSL\row; -use function Flow\ETL\DSL\rows; use function Flow\ETL\DSL\schema; use function iterator_to_array; -use function range; final class MergeSortProcessorTest extends FlowTestCase { public function test_bind_returns_the_input_schema(): void { - $storage = new MemoryBuckets(); $input = schema(int_schema('id')); - $processor = new MergeSortProcessor( - refs(ref('id')), - new Buckets($storage), - new Buckets($storage), - new NativePHPRandomValueGenerator(), - ); - - static::assertEquals($input, $processor->bind($input)->output); - } - - public function test_descending_single_column(): void - { - $storage = new SpyBucketsStorage(new MemoryBuckets()); - $buckets = new Buckets($storage); - $context = flow_context(config()); - - $bucketing = new BucketingProcessor( - new SortedRunBucketing([ref('id')->desc()], 2, new NativePHPRandomValueGenerator()), - $buckets, - ); - $merge = new MergeSortProcessor( - refs(ref('id')->desc()), - $buckets, - new Buckets($storage), - new NativePHPRandomValueGenerator(), - 10, - 1000, - ); - - $input = (static function () { - yield rows( - schema(int_schema('id')), - row(['id' => 3]), - row(['id' => 1]), - row(['id' => 4]), - row(['id' => 2]), - ); - })(); - - $result = iterator_to_array($merge->process($bucketing->process($input, $context), $context), false); - static::assertSame( - [4, 3, 2, 1], - array_merge(...array_map(static fn(Rows $r): array => $r->reduceToArray('id'), $result)), + static::assertEquals( + $input, + (new MergeSortProcessor(ExternalSortMother::inMemory(refs('id'))))->bind($input)->output, ); - static::assertSame([], $storage->liveBucketIds()); } - public function test_empty_input_yields_nothing_and_leaves_storage_empty(): void + public function test_process_sorts_through_the_external_sort(): void { - $storage = new SpyBucketsStorage(new MemoryBuckets()); - $buckets = new Buckets($storage); - $context = flow_context(config()); - - $bucketing = new BucketingProcessor( - new SortedRunBucketing([ref('id')], 2, new NativePHPRandomValueGenerator()), - $buckets, - ); - $merge = new MergeSortProcessor( - refs('id'), - $buckets, - new Buckets($storage), - new NativePHPRandomValueGenerator(), - 10, - 1000, - ); - - $input = (static function () { - yield from []; + $storage = new RecordingBucketsStorage(new MemoryBuckets()); + $input = (static function (): Generator { + yield array_to_rows([['id' => 3], ['id' => 1]], schema(int_schema('id'))); + yield array_to_rows([['id' => 2]], schema(int_schema('id'))); })(); - static::assertCount(0, iterator_to_array( - $merge->process($bucketing->process($input, $context), $context), + $result = iterator_to_array( + (new MergeSortProcessor(ExternalSortMother::spilling(refs('id'), $storage)))->process( + $input, + flow_context(config()), + ), false, - )); - static::assertSame([], $storage->liveBucketIds()); - } - - public function test_heap_merges_overlapping_runs_without_reduction(): void - { - $storage = new SpyBucketsStorage(new MemoryBuckets()); - $buckets = new Buckets($storage); - $context = flow_context(config()); - - $bucketing = new BucketingProcessor( - new SortedRunBucketing([ref('id')], 2, new NativePHPRandomValueGenerator()), - $buckets, - ); - $merge = new MergeSortProcessor( - refs('id'), - $buckets, - new Buckets($storage), - new NativePHPRandomValueGenerator(), - 10, - 1000, - ); - - $input = (static function () { - yield rows(schema(int_schema('id')), ...array_map(static fn(int $i): Row => row(['id' => $i]), [ - 0, - 10, - 1, - 11, - 2, - 12, - 3, - 13, - ])); - })(); - - $result = iterator_to_array($merge->process($bucketing->process($input, $context), $context), false); - - static::assertSame( - [0, 1, 2, 3, 10, 11, 12, 13], - array_merge(...array_map(static fn(Rows $r): array => $r->reduceToArray('id'), $result)), ); - static::assertSame([], $storage->liveBucketIds()); - } - - public function test_sorts_by_multiple_columns(): void - { - $storage = new SpyBucketsStorage(new MemoryBuckets()); - $buckets = new Buckets($storage); - $context = flow_context(config()); - - $bucketing = new BucketingProcessor( - new SortedRunBucketing([ref('a'), ref('b')], 2, new NativePHPRandomValueGenerator()), - $buckets, - ); - $merge = new MergeSortProcessor( - refs('a', 'b'), - $buckets, - new Buckets($storage), - new NativePHPRandomValueGenerator(), - 10, - 1000, - ); - - $input = (static function () { - yield rows( - schema(int_schema('a'), int_schema('b')), - row(['a' => 1, 'b' => 2]), - row(['a' => 1, 'b' => 1]), - row(['a' => 0, 'b' => 5]), - row(['a' => 0, 'b' => 3]), - ); - })(); - - $result = iterator_to_array($merge->process($bucketing->process($input, $context), $context), false); static::assertSame( - [ - ['a' => 0, 'b' => 3], - ['a' => 0, 'b' => 5], - ['a' => 1, 'b' => 1], - ['a' => 1, 'b' => 2], - ], - array_merge(...array_map(static fn(Rows $r): array => $r->toArray(), $result)), - ); - static::assertSame([], $storage->liveBucketIds()); - } - - public function test_multi_pass_reduction_produces_sorted_output(): void - { - $storage = new SpyBucketsStorage(new MemoryBuckets()); - $buckets = new Buckets($storage); - $context = flow_context(config()); - - $bucketing = new BucketingProcessor( - new SortedRunBucketing([ref('id')], 2, new NativePHPRandomValueGenerator()), - $buckets, - ); - $merge = new MergeSortProcessor( - refs('id'), - $buckets, - new Buckets($storage), - new NativePHPRandomValueGenerator(), - 2, - 1000, - ); - - $input = (static function () { - yield rows( - schema(int_schema('id')), - ...array_map(static fn(int $i): Row => row(['id' => $i]), range(19, 0)), - ); - })(); - - $result = iterator_to_array($merge->process($bucketing->process($input, $context), $context), false); - - static::assertSame( - range(0, 19), - array_merge(...array_map(static fn(Rows $r): array => $r->reduceToArray('id'), $result)), - ); - static::assertSame([], $storage->liveBucketIds()); - } - - public function test_both_storages_are_cleared_when_the_first_clear_throws(): void - { - $spillStorage = new SpyBucketsStorage(new MemoryBuckets()); - $mergeStorage = new SpyBucketsStorage(new MemoryBuckets()); - $spill = new Buckets(new ThrowingRemoveBucketsStorage($spillStorage)); - $merge = new Buckets($mergeStorage); - $context = flow_context(config()); - - // the upstream throws during the drain, so reduce() never writes a merged run - seed one, or the - // merge-side assertion is true whether or not the nested finally ran - $mergeStorage->append('pre-existing', rows(schema(int_schema('id')), row(['id' => 1]))); - $merge->add(new Bucket('pre-existing', 1, 0)); - - $bucketing = new BucketingProcessor( - new SortedRunBucketing([ref('id')], 2, new NativePHPRandomValueGenerator()), - $spill, - ); - $processor = new MergeSortProcessor(refs('id'), $spill, $merge, new NativePHPRandomValueGenerator(), 2, 1000); - - $upstream = (static function (): Generator { - yield rows( - schema(int_schema('id')), - ...array_map(static fn(int $i): Row => row(['id' => $i]), range(9, 0)), - ); - - throw new RuntimeException('upstream failed'); - })(); - - try { - iterator_to_array($processor->process($bucketing->process($upstream, $context), $context), false); - static::fail('The upstream failure must escape.'); - } catch (RuntimeException $escaped) { - static::assertSame('spill clear failed', $escaped->getMessage()); - - $previous = $escaped->getPrevious(); - - static::assertInstanceOf(RuntimeException::class, $previous); - static::assertSame('upstream failed', $previous->getMessage()); - } - - // the nested finally runs the merge clear even though the spill clear threw - static::assertSame([], $mergeStorage->liveBucketIds()); - } - - public function test_throws_when_batch_size_below_one(): void - { - $this->expectException(InvalidArgumentException::class); - $this->expectExceptionMessage('Batch size must be greater than 0, given: 0'); - - // @mago-ignore analysis:invalid-argument - new MergeSortProcessor( - refs('id'), - new Buckets(new MemoryBuckets()), - new Buckets(new MemoryBuckets()), - new NativePHPRandomValueGenerator(), - 10, - 0, - ); - } - - public function test_throws_when_merge_fan_in_below_one(): void - { - $this->expectException(InvalidArgumentException::class); - $this->expectExceptionMessage('Merge fan-in must be greater than 0, given: 0'); - - // @mago-ignore analysis:invalid-argument - new MergeSortProcessor( - refs('id'), - new Buckets(new MemoryBuckets()), - new Buckets(new MemoryBuckets()), - new NativePHPRandomValueGenerator(), - 0, - 1000, + [1, 2, 3], + array_merge(...array_map(static fn(Rows $rows): array => $rows->reduceToArray('id'), $result)), ); + static::assertNotSame([], $storage->appended); } } diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Processor/OffsetProcessorTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Processor/OffsetProcessorTest.php index 93660215cc..46c7279427 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Unit/Processor/OffsetProcessorTest.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Processor/OffsetProcessorTest.php @@ -11,10 +11,9 @@ use Flow\ETL\Tests\FlowTestCase; use Flow\ETL\Tests\Mother\RowsMother; +use function Flow\ETL\DSL\array_to_rows; use function Flow\ETL\DSL\flow_context; use function Flow\ETL\DSL\int_schema; -use function Flow\ETL\DSL\row; -use function Flow\ETL\DSL\rows; use function Flow\ETL\DSL\schema; final class OffsetProcessorTest extends FlowTestCase @@ -43,7 +42,7 @@ public function test_offset_greater_than_total_rows_yields_nothing(): void { $processor = new OffsetProcessor(10); $generator = (static function () { - yield rows(schema(int_schema('id')), row(['id' => 1]), row(['id' => 2])); + yield array_to_rows([['id' => 1], ['id' => 2]], schema(int_schema('id'))); })(); $result = iterator_to_array($processor->process($generator, flow_context())); $totalRows = 0; @@ -59,7 +58,7 @@ public function test_offset_within_single_batch(): void { $processor = new OffsetProcessor(1); $generator = (static function () { - yield rows(schema(int_schema('id')), row(['id' => 1]), row(['id' => 2]), row(['id' => 3])); + yield array_to_rows([['id' => 1], ['id' => 2], ['id' => 3]], schema(int_schema('id'))); })(); $result = iterator_to_array($processor->process($generator, flow_context())); $allRows = []; @@ -83,7 +82,7 @@ public function test_offset_zero_yields_all_rows(): void { $processor = new OffsetProcessor(0); $generator = (static function () { - yield rows(schema(int_schema('id')), row(['id' => 1]), row(['id' => 2])); + yield array_to_rows([['id' => 1], ['id' => 2]], schema(int_schema('id'))); })(); $result = iterator_to_array($processor->process($generator, flow_context())); $totalRows = 0; @@ -99,8 +98,8 @@ public function test_skips_first_n_rows(): void { $processor = new OffsetProcessor(2); $generator = (static function () { - yield rows(schema(int_schema('id')), row(['id' => 1]), row(['id' => 2])); - yield rows(schema(int_schema('id')), row(['id' => 3]), row(['id' => 4])); + yield array_to_rows([['id' => 1], ['id' => 2]], schema(int_schema('id'))); + yield array_to_rows([['id' => 3], ['id' => 4]], schema(int_schema('id'))); })(); $result = iterator_to_array($processor->process($generator, flow_context())); $allRows = []; diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Processor/PivotProcessorTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Processor/PivotProcessorTest.php index 904603a91a..ee0ae3c829 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Unit/Processor/PivotProcessorTest.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Processor/PivotProcessorTest.php @@ -12,6 +12,7 @@ use Generator; use PHPUnit\Framework\Attributes\DataProvider; +use function Flow\ETL\DSL\array_to_rows; use function Flow\ETL\DSL\average; use function Flow\ETL\DSL\collect; use function Flow\ETL\DSL\collect_unique; @@ -25,8 +26,6 @@ use function Flow\ETL\DSL\min; use function Flow\ETL\DSL\pivot_values; use function Flow\ETL\DSL\ref; -use function Flow\ETL\DSL\row; -use function Flow\ETL\DSL\rows; use function Flow\ETL\DSL\schema; use function Flow\ETL\DSL\str_schema; use function Flow\ETL\DSL\string_agg; @@ -144,12 +143,11 @@ public function test_a_bound_processor_pivots_grouped_rows(): void $result = iterator_to_array( $bound->step->process( (static function () use ($input): Generator { - yield rows( - $input, - row(['date' => '2024-01-01', 'user' => 'norbert', 'contributions' => 2]), - row(['date' => '2024-01-01', 'user' => 'stloyd', 'contributions' => 3]), - row(['date' => '2024-01-02', 'user' => 'norbert', 'contributions' => 5]), - ); + yield array_to_rows([ + ['date' => '2024-01-01', 'user' => 'norbert', 'contributions' => 2], + ['date' => '2024-01-01', 'user' => 'stloyd', 'contributions' => 3], + ['date' => '2024-01-02', 'user' => 'norbert', 'contributions' => 5], + ], $input); })(), flow_context(), ), @@ -174,11 +172,13 @@ public function test_pivots_grouped_rows(): void $groupBy->aggregate(sum(ref('contributions'))); $input = (static function (): Generator { - yield rows( + yield array_to_rows( + [ + ['date' => '2024-01-01', 'user' => 'norbert', 'contributions' => 2], + ['date' => '2024-01-01', 'user' => 'stloyd', 'contributions' => 3], + ['date' => '2024-01-02', 'user' => 'norbert', 'contributions' => 5], + ], schema(str_schema('date'), str_schema('user'), int_schema('contributions')), - row(['date' => '2024-01-01', 'user' => 'norbert', 'contributions' => 2]), - row(['date' => '2024-01-01', 'user' => 'stloyd', 'contributions' => 3]), - row(['date' => '2024-01-02', 'user' => 'norbert', 'contributions' => 5]), ); })(); diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Processor/RepartitionProcessorTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Processor/RepartitionProcessorTest.php index 5979beed99..a0313581c0 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Unit/Processor/RepartitionProcessorTest.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Processor/RepartitionProcessorTest.php @@ -4,127 +4,152 @@ namespace Flow\ETL\Tests\Unit\Processor; -use Flow\ETL\Bucketing\Bucket; -use Flow\ETL\Bucketing\Buckets; -use Flow\ETL\Bucketing\HashBucketing; -use Flow\ETL\Bucketing\NativeHasher; use Flow\ETL\Bucketing\Storage\MemoryBuckets; -use Flow\ETL\NativePHPRandomValueGenerator; -use Flow\ETL\Processor\RepartitionProcessor; +use Flow\ETL\Dataset\Memory\Unit; use Flow\ETL\Rows; +use Flow\ETL\Tests\Context\GroupedRows; +use Flow\ETL\Tests\Double\ReportedMemoryBackend; +use Flow\ETL\Tests\Double\SpyBucketsStorage; use Flow\ETL\Tests\FlowTestCase; +use Flow\ETL\Tests\Mother\RepartitionProcessorMother; use Generator; +use PHPUnit\Framework\Attributes\DataProvider; use PHPUnit\Framework\Attributes\TestWith; use function array_map; +use function Flow\ETL\DSL\array_to_rows; +use function Flow\ETL\DSL\config_builder; use function Flow\ETL\DSL\flow_context; use function Flow\ETL\DSL\int_schema; use function Flow\ETL\DSL\ref; use function Flow\ETL\DSL\refs; -use function Flow\ETL\DSL\row; -use function Flow\ETL\DSL\rows; use function Flow\ETL\DSL\schema; use function Flow\ETL\DSL\str_schema; use function iterator_to_array; -use function usort; +use function memory_get_usage; +use function mt_rand; +use function mt_srand; +use function sort; final class RepartitionProcessorTest extends FlowTestCase { + public static function random_datasets(): Generator + { + for ($seed = 1; $seed <= 50; $seed++) { + yield "seed {$seed}" => [$seed]; + } + } + public function test_bind_returns_the_input_schema(): void { $input = schema(str_schema('k'), int_schema('v')); - $processor = new RepartitionProcessor(refs(ref('k')), new Buckets(new MemoryBuckets())); - static::assertEquals($input, $processor->bind($input)->output); + static::assertEquals($input, RepartitionProcessorMother::inMemory(refs(ref('k')))->bind($input)->output); + } + + public function test_under_the_limit_it_groups_in_one_pass_without_touching_the_buckets(): void + { + $storage = new SpyBucketsStorage(new MemoryBuckets()); + $schema = schema(str_schema('k'), int_schema('v')); + + $groups = iterator_to_array( + RepartitionProcessorMother::inMemory(refs(ref('k')), $storage)->process( + (static function () use ($schema): Generator { + yield array_to_rows([['k' => 'a', 'v' => 1], ['k' => 'b', 'v' => 2]], $schema); + yield array_to_rows([['k' => 'a', 'v' => 3], ['k' => 'b', 'v' => 4]], $schema); + })(), + flow_context(), + ), + false, + ); + + static::assertSame( + [[1, 3], [2, 4]], + array_map(static fn(Rows $group): array => $group->reduceToArray('v'), $groups), + ); + static::assertSame([], $storage->appendedRows()); } /** - * One bucket proves the grouping is the processor's own work; 64 proves a key is never split + * Past the limit: one bucket proves the grouping is the processor's own work; 64 prove a key is never split * across buckets, which is what makes the downstream window contiguous. * * @param int<1, max> $bucketsCount */ #[TestWith([1])] #[TestWith([64])] - public function test_every_row_sharing_a_key_arrives_in_one_batch(int $bucketsCount): void + public function test_past_the_limit_every_row_sharing_a_key_still_arrives_in_one_batch(int $bucketsCount): void { + $storage = new SpyBucketsStorage(new MemoryBuckets()); $schema = schema(str_schema('k'), int_schema('v')); - $buckets = new Buckets(new MemoryBuckets()); - $strategy = new HashBucketing( - [ref('k')], - $bucketsCount, - new NativeHasher(), - new NativePHPRandomValueGenerator(), - 'repartition', - ); - - $input = (static function () use ($schema): Generator { - yield rows($schema, row(['k' => 'a', 'v' => 1]), row(['k' => 'b', 'v' => 2])); - yield rows($schema, row(['k' => 'a', 'v' => 3]), row(['k' => 'b', 'v' => 4])); - })(); - - $metadata = []; - - foreach ($strategy->bucketize($input, $buckets->storage()) as $bucket) { - $buckets->add($bucket); - $metadata[] = rows(Bucket::schema(), $bucket->toRow()); - } $groups = array_map( - static fn(Rows $group): array => array_map( - static fn(array $values): int => (int) $values['v'], - $group->toArray(), - ), + static fn(Rows $group): array => $group->reduceToArray('v'), iterator_to_array( - (new RepartitionProcessor(refs(ref('k')), $buckets))->process( - (static function () use ($metadata): Generator { - yield from $metadata; + RepartitionProcessorMother::with(refs(ref('k')), $storage, Unit::fromBytes(1), $bucketsCount)->process( + (static function () use ($schema): Generator { + yield array_to_rows([['k' => 'a', 'v' => 1], ['k' => 'b', 'v' => 2]], $schema); + yield array_to_rows([['k' => 'a', 'v' => 3], ['k' => 'b', 'v' => 4]], $schema); })(), flow_context(), ), - preserve_keys: false, + false, ), ); - - usort($groups, static fn(array $a, array $b): int => $a[0] <=> $b[0]); + sort($groups); static::assertSame([[1, 3], [2, 4]], $groups); + static::assertNotSame([], $storage->appendedRows()); + static::assertSame([], $storage->liveBucketIds()); } - public function test_buckets_are_cleared_when_the_stream_is_done(): void + #[DataProvider('random_datasets')] + public function test_crossing_the_limit_at_any_batch_gives_the_groups_one_in_memory_pass_gives(int $seed): void { - $schema = schema(str_schema('k'), int_schema('v')); - $buckets = new Buckets(new MemoryBuckets()); - $strategy = new HashBucketing( - [ref('k')], - 4, - new NativeHasher(), - new NativePHPRandomValueGenerator(), - 'repartition', - ); + mt_srand($seed); + $schema = schema(int_schema('k', nullable: true), int_schema('v')); + $batches = []; + $id = 0; - $metadata = []; + for ($b = 0, $count = mt_rand(1, 8); $b < $count; $b++) { + $rows = []; - foreach ($strategy->bucketize( - (static function () use ($schema): Generator { - yield rows($schema, row(['k' => 'a', 'v' => 1])); - })(), - $buckets->storage(), - ) as $bucket) { - $buckets->add($bucket); - $metadata[] = rows(Bucket::schema(), $bucket->toRow()); + for ($r = 0, $size = mt_rand(1, 20); $r < $size; $r++) { + $rows[] = ['k' => mt_rand(0, 6) === 0 ? null : mt_rand(0, 40), 'v' => $id++]; + } + + $batches[] = array_to_rows($rows, $schema); } - iterator_to_array( - (new RepartitionProcessor(refs(ref('k')), $buckets))->process( - (static function () use ($metadata): Generator { - yield from $metadata; - })(), + $crossAt = mt_rand(0, $count); + $backend = new ReportedMemoryBackend(); + $input = (static function () use ($batches, $crossAt, $backend): Generator { + foreach ($batches as $position => $batch) { + if ($position === $crossAt) { + $backend->reported = 1_000_000_000_000; + } + + yield $batch; + } + })(); + + $storage = new SpyBucketsStorage(new MemoryBuckets()); + + static::assertSame( + GroupedRows::batchesAsSet(RepartitionProcessorMother::inMemory(refs(ref('k')))->process( + (static fn(): Generator => yield from $batches)(), flow_context(), - ), - preserve_keys: false, + )), + GroupedRows::batchesAsSet(RepartitionProcessorMother::with( + refs(ref('k')), + $storage, + Unit::fromBytes(memory_get_usage(false) + 500_000_000), + )->process($input, flow_context(config_builder()->backend($backend)->build()))), ); - static::assertSame([], $buckets->all()); + // the batch at $crossAt is read past the limit: it and every later batch are partitioned + if ($crossAt < $count) { + static::assertNotSame([], $storage->appendedRows()); + } } } diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Processor/TopNProcessorTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Processor/TopNProcessorTest.php index b8f98755d5..e66f3b20c7 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Unit/Processor/TopNProcessorTest.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Processor/TopNProcessorTest.php @@ -4,23 +4,129 @@ namespace Flow\ETL\Tests\Unit\Processor; +use Flow\ETL\Bucketing\Storage\MemoryBuckets; +use Flow\ETL\Column\AdaptiveBackend; use Flow\ETL\Exception\InvalidArgumentException; use Flow\ETL\Processor\MemorySortProcessor; use Flow\ETL\Processor\TopNProcessor; use Flow\ETL\Tests\Context\ScoredRows; +use Flow\ETL\Tests\Double\RecordingBucketsStorage; use Flow\ETL\Tests\FlowTestCase; +use Flow\ETL\Tests\Mother\ExternalSortMother; +use Flow\ETL\Tests\Mother\SortDatasetMother; use Generator; +use PHPUnit\Framework\Attributes\DataProvider; +use function array_merge; +use function Flow\ETL\DSL\array_to_rows; use function Flow\ETL\DSL\flow_context; use function Flow\ETL\DSL\int_schema; use function Flow\ETL\DSL\ref; use function Flow\ETL\DSL\refs; +use function Flow\ETL\DSL\rows; use function Flow\ETL\DSL\schema; use function Flow\ETL\DSL\str_schema; use function iterator_to_array; +use function serialize; final class TopNProcessorTest extends FlowTestCase { + public static function random_datasets(): Generator + { + for ($seed = 1; $seed <= 200; $seed++) { + yield "seed {$seed}" => [$seed]; + } + } + + public function test_under_the_memory_limit_it_trims_in_memory_with_zero_spills(): void + { + $storage = new RecordingBucketsStorage(new MemoryBuckets()); + $refs = refs(ref('score')); + + $top = ScoredRows::merged((new TopNProcessor($refs, 2, ExternalSortMother::inMemory($refs, $storage)))->process( + ScoredRows::batches([[3, 'c'], [1, 'a']], [[2, 'b'], [0, 'z']]), + flow_context(), + )); + + static::assertSame([['score' => 0, 'name' => 'z'], ['score' => 1, 'name' => 'a']], $top); + static::assertSame([], $storage->appended); + } + + public function test_past_the_memory_limit_it_falls_back_to_the_spilling_sort(): void + { + $storage = new RecordingBucketsStorage(new MemoryBuckets()); + $refs = refs(ref('score')); + + $top = ScoredRows::merged((new TopNProcessor($refs, 2, ExternalSortMother::spilling($refs, $storage)))->process( + ScoredRows::batches([[3, 'c'], [1, 'a']], [[2, 'b'], [0, 'z']], [[1, 'y']]), + flow_context(), + )); + + static::assertSame([['score' => 0, 'name' => 'z'], ['score' => 1, 'name' => 'a']], $top); + static::assertNotSame([], $storage->appended); + } + + public function test_a_fallback_on_the_last_batch_emits_the_kept_rows(): void + { + $refs = refs(ref('score')); + + static::assertSame( + [['score' => 1, 'name' => 'a']], + ScoredRows::merged((new TopNProcessor($refs, 1, ExternalSortMother::spilling($refs)))->process( + ScoredRows::batches([[3, 'c'], [1, 'a']]), + flow_context(), + )), + ); + } + + #[DataProvider('random_datasets')] + public function test_the_fallback_equals_a_sort_followed_by_a_limit(int $seed): void + { + $dataset = SortDatasetMother::random($seed); + $limit = ($seed % 7) + 1; + $refs = refs(...$dataset['references']); + $input = (static function () use ($dataset): Generator { + foreach ($dataset['runs'] as $rows) { + yield array_to_rows($rows, $dataset['schema']); + } + })(); + + $top = rows($dataset['schema']); + $storage = new RecordingBucketsStorage(new MemoryBuckets()); + + foreach ((new TopNProcessor( + $refs, + $limit, + ExternalSortMother::spilling($refs, $storage, mergeFanIn: 2), + ))->process($input, flow_context()) as $batch) { + $top = $top->concat(new AdaptiveBackend(), $batch->matchTo($dataset['schema'], new AdaptiveBackend())); + } + + // serialized, so NaN and -0.0 compare by what they are + static::assertSame( + serialize( + array_to_rows(array_merge(...$dataset['runs']), $dataset['schema']) + ->sortBy(...$dataset['references']) + ->take($limit) + ->toArray(), + ), + serialize($top->toArray()), + ); + + if (array_merge(...$dataset['runs']) !== []) { + static::assertNotSame([], $storage->appended, 'the fallback must have spilled'); + } + } + + public function test_bind_keeps_the_fallback(): void + { + $fallback = ExternalSortMother::inMemory(refs(ref('score'))); + $bound = (new TopNProcessor(refs(ref('score')), 3, $fallback))->bind(schema(int_schema('score')))->step; + + static::assertInstanceOf(TopNProcessor::class, $bound); + static::assertSame($fallback, $bound->fallback); + } + public function test_it_emits_exactly_what_a_sort_followed_by_a_limit_emits(): void { $refs = refs(ref('score')->desc(), ref('name')); @@ -97,4 +203,27 @@ public function test_a_limit_below_one_is_refused(): void new TopNProcessor(refs(ref('score')), 0); } + + public function test_top_sorts_and_keeps_the_limit(): void + { + static::assertSame( + [['score' => 9, 'name' => 'a'], ['score' => 7, 'name' => 'b']], + (new TopNProcessor(refs(ref('score')->desc()), 2))->top(array_to_rows( + [['score' => 3, 'name' => 'c'], ['score' => 9, 'name' => 'a'], ['score' => 7, 'name' => 'b']], + schema(int_schema('score'), str_schema('name')), + ))->toArray(), + ); + } + + public function test_batches_are_conformed_to_the_declared_schema(): void + { + $declared = schema(int_schema('score', nullable: true), str_schema('name')); + $bound = (new TopNProcessor(refs(ref('score')), 5))->bind($declared)->step; + static::assertInstanceOf(TopNProcessor::class, $bound); + + $out = iterator_to_array($bound->process(ScoredRows::batches([[2, 'b'], [1, 'a']]), flow_context()), false); + + static::assertEquals($declared, $out[0]->schema()); + static::assertSame([['score' => 1, 'name' => 'a'], ['score' => 2, 'name' => 'b']], $out[0]->toArray()); + } } diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Processor/VoidProcessorTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Processor/VoidProcessorTest.php index 79396bfb79..e4da3a8279 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Unit/Processor/VoidProcessorTest.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Processor/VoidProcessorTest.php @@ -8,10 +8,9 @@ use Flow\ETL\Rows; use Flow\ETL\Tests\FlowTestCase; +use function Flow\ETL\DSL\array_to_rows; use function Flow\ETL\DSL\flow_context; use function Flow\ETL\DSL\int_schema; -use function Flow\ETL\DSL\row; -use function Flow\ETL\DSL\rows; use function Flow\ETL\DSL\schema; use function Flow\ETL\DSL\str_schema; @@ -46,8 +45,8 @@ public function test_discards_all_rows_and_yields_empty_batch(): void $processor = new VoidProcessor(); $generator = (static function () { - yield rows(schema(int_schema('id')), row(['id' => 1]), row(['id' => 2])); - yield rows(schema(int_schema('id')), row(['id' => 3]), row(['id' => 4])); + yield array_to_rows([['id' => 1], ['id' => 2]], schema(int_schema('id'))); + yield array_to_rows([['id' => 3], ['id' => 4]], schema(int_schema('id'))); })(); /** @var list $result */ diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Processor/WindowProcessorTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Processor/WindowProcessorTest.php index a84b7c23e3..cf0f85cd0e 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Unit/Processor/WindowProcessorTest.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Processor/WindowProcessorTest.php @@ -12,11 +12,13 @@ use Flow\ETL\Rows; use Flow\ETL\Tests\Context\WindowProcessorContext; use Flow\ETL\Tests\Double\CountingFrameAccumulating; +use Flow\ETL\Tests\Double\WindowFunctionOverOperand; use Flow\ETL\Tests\FlowTestCase; use Flow\ETL\Tests\Mother\ListColumnsMother; use Generator; use PHPUnit\Framework\Attributes\DataProvider; +use function Flow\ETL\DSL\array_to_rows; use function Flow\ETL\DSL\average; use function Flow\ETL\DSL\count; use function Flow\ETL\DSL\current_row; @@ -28,7 +30,6 @@ use function Flow\ETL\DSL\preceding; use function Flow\ETL\DSL\rank; use function Flow\ETL\DSL\ref; -use function Flow\ETL\DSL\row; use function Flow\ETL\DSL\row_number; use function Flow\ETL\DSL\rows; use function Flow\ETL\DSL\schema; @@ -80,12 +81,14 @@ public function test_constant_frame_accumulates_once_per_partition(): void WindowProcessorContext::values( 'total', $function->over(window()->partitionBy(ref('group'))), - rows( + array_to_rows( + [ + ['group' => 'a', 'value' => 10], + ['group' => 'a', 'value' => 20], + ['group' => 'a', 'value' => 30], + ['group' => 'a', 'value' => 40], + ], schema(str_schema('group'), int_schema('value')), - row(['group' => 'a', 'value' => 10]), - row(['group' => 'a', 'value' => 20]), - row(['group' => 'a', 'value' => 30]), - row(['group' => 'a', 'value' => 40]), ), ), ); @@ -106,13 +109,12 @@ public function test_growing_frame_accumulates_each_row_once(): void WindowProcessorContext::values( 'total', $function->over(window()->orderBy(ref('value'))), - rows( - schema(int_schema('value')), - row(['value' => 10]), - row(['value' => 20]), - row(['value' => 30]), - row(['value' => 40]), - ), + array_to_rows([ + ['value' => 10], + ['value' => 20], + ['value' => 30], + ['value' => 40], + ], schema(int_schema('value'))), ), ); static::assertSame(1, $function->accumulatorCalls); @@ -131,13 +133,12 @@ public function test_peer_group_reuses_the_value_within_a_group(): void WindowProcessorContext::values( 'total', $function->over(window()->orderBy(ref('value'))), - rows( - schema(int_schema('value')), - row(['value' => 10]), - row(['value' => 10]), - row(['value' => 20]), - row(['value' => 30]), - ), + array_to_rows([ + ['value' => 10], + ['value' => 10], + ['value' => 20], + ['value' => 30], + ], schema(int_schema('value'))), ), ); static::assertSame(1, $function->accumulatorCalls); @@ -156,13 +157,12 @@ public function test_sliding_frame_recomputes_per_row(): void WindowProcessorContext::values( 'total', $function->over(window()->orderBy(ref('value'))->rowsBetween(preceding(1), current_row())), - rows( - schema(int_schema('value')), - row(['value' => 10]), - row(['value' => 20]), - row(['value' => 30]), - row(['value' => 40]), - ), + array_to_rows([ + ['value' => 10], + ['value' => 20], + ['value' => 30], + ['value' => 40], + ], schema(int_schema('value'))), ), ); static::assertSame(3, $function->accumulatorCalls); @@ -178,18 +178,17 @@ public function test_definition_entry_keeps_its_type_and_metadata(): void $batches = WindowProcessorContext::batches( int_schema('total')->addMetadata('origin', 'window'), sum(ref('value'))->over(window()->partitionBy(ref('group'))), - rows( + array_to_rows( + [['group' => 'a', 'value' => 40], ['group' => 'a', 'value' => 60]], schema(str_schema('group'), int_schema('value')), - row(['group' => 'a', 'value' => 40]), - row(['group' => 'a', 'value' => 60]), ), ); static::assertCount(1, $batches); static::assertEquals(int_schema('total')->addMetadata('origin', 'window'), $batches[0]->schema()->get('total')); - foreach ($batches[0] as $row) { - static::assertSame(100, $row->get('total')); + foreach ($batches[0]->toArray() as $row) { + static::assertSame(100, $row['total']); } } @@ -201,7 +200,7 @@ public function test_an_unknown_partition_reference_is_refused_at_bind(): void WindowProcessorContext::values( 'row_number', row_number()->over(window()->partitionBy(ref('missing'))->orderBy(ref('value'))), - rows(schema(int_schema('value')), row(['value' => 10]), row(['value' => 20]), row(['value' => 30])), + array_to_rows([['value' => 10], ['value' => 20], ['value' => 30]], schema(int_schema('value'))), ); } @@ -213,7 +212,7 @@ public function test_an_unknown_order_reference_is_refused_at_bind(): void WindowProcessorContext::values( 'row_number', row_number()->over(window()->orderBy(ref('missing'))), - rows(schema(int_schema('value')), row(['value' => 10]), row(['value' => 20]), row(['value' => 30])), + array_to_rows([['value' => 10], ['value' => 20], ['value' => 30]], schema(int_schema('value'))), ); } @@ -224,7 +223,7 @@ public function test_empty_frame_yields_the_accumulator_empty_value(): void WindowProcessorContext::values( 'total', sum(ref('value'))->over(window()->orderBy(ref('value'))->rowsBetween(following(10), following(20))), - rows(schema(int_schema('value')), row(['value' => 10]), row(['value' => 20]), row(['value' => 30])), + array_to_rows([['value' => 10], ['value' => 20], ['value' => 30]], schema(int_schema('value'))), ), ); } @@ -236,12 +235,14 @@ public function test_empty_frame_yields_the_accumulator_empty_value(): void */ public function test_row_number_is_not_memoised_by_a_constant_frame(): void { - $partition = rows( + $partition = array_to_rows( + [ + ['group' => 'a', 'value' => 10], + ['group' => 'a', 'value' => 20], + ['group' => 'a', 'value' => 30], + ['group' => 'a', 'value' => 40], + ], schema(str_schema('group'), int_schema('value')), - row(['group' => 'a', 'value' => 10]), - row(['group' => 'a', 'value' => 20]), - row(['group' => 'a', 'value' => 30]), - row(['group' => 'a', 'value' => 40]), ); $frame = window()->partitionBy(ref('group'))->frame(); @@ -263,12 +264,14 @@ public function test_whole_partition_frame_gives_every_row_the_partition_total() WindowProcessorContext::values( 'total', sum(ref('value'))->over(window()->partitionBy(ref('group'))), - rows( + array_to_rows( + [ + ['group' => 'a', 'value' => 10], + ['group' => 'a', 'value' => 20], + ['group' => 'a', 'value' => 30], + ['group' => 'a', 'value' => 40], + ], schema(str_schema('group'), int_schema('value')), - row(['group' => 'a', 'value' => 10]), - row(['group' => 'a', 'value' => 20]), - row(['group' => 'a', 'value' => 30]), - row(['group' => 'a', 'value' => 40]), ), ), ); @@ -281,11 +284,13 @@ public function test_applies_window_function(): void $processor = new WindowProcessor('rank', $windowFunction); $generator = (static function () { - yield rows( + yield array_to_rows( + [ + ['category' => 'a', 'amount' => 100], + ['category' => 'a', 'amount' => 200], + ['category' => 'b', 'amount' => 150], + ], schema(str_schema('category'), int_schema('amount')), - row(['category' => 'a', 'amount' => 100]), - row(['category' => 'a', 'amount' => 200]), - row(['category' => 'b', 'amount' => 150]), ); })(); @@ -326,12 +331,11 @@ public function test_handles_single_partition(): void $processor = new WindowProcessor('rank', $windowFunction); $generator = (static function () { - yield rows( - schema(int_schema('amount')), - row(['amount' => 300]), - row(['amount' => 100]), - row(['amount' => 200]), - ); + yield array_to_rows([ + ['amount' => 300], + ['amount' => 100], + ['amount' => 200], + ], schema(int_schema('amount'))); })(); /** @var list $result */ @@ -391,7 +395,7 @@ public function test_a_running_sum_over_mixed_numerics_yields_one_definition(): $batches = WindowProcessorContext::batches( 'total', sum(ref('value'))->over(window()->orderBy(ref('value'))), - rows(schema(float_schema('value')), row(['value' => 1.0]), row(['value' => 2.5]), row(['value' => 3.0])), + array_to_rows([['value' => 1.0], ['value' => 2.5], ['value' => 3.0]], schema(float_schema('value'))), ); $definitions = []; @@ -400,8 +404,8 @@ public function test_a_running_sum_over_mixed_numerics_yields_one_definition(): foreach ($batches as $batch) { $definitions[] = $batch->schema()->get('total'); - foreach ($batch as $row) { - $values[] = $row->get('total'); + foreach ($batch->toArray() as $row) { + $values[] = $row['total']; } } @@ -416,12 +420,14 @@ public function test_processes_multiple_partitions(): void $processor = new WindowProcessor('rank', $windowFunction); $generator = (static function () { - yield rows( + yield array_to_rows( + [ + ['group' => 'a', 'value' => 10], + ['group' => 'a', 'value' => 20], + ['group' => 'b', 'value' => 5], + ['group' => 'b', 'value' => 15], + ], schema(str_schema('group'), int_schema('value')), - row(['group' => 'a', 'value' => 10]), - row(['group' => 'a', 'value' => 20]), - row(['group' => 'b', 'value' => 5]), - row(['group' => 'b', 'value' => 15]), ); })(); @@ -450,14 +456,14 @@ public function test_a_leading_empty_batch_is_yielded_under_the_bound_schema(): $generator = (static function () { yield rows(schema(int_schema('value'))); - yield rows(schema(int_schema('value')), row(['value' => 10]), row(['value' => 20])); + yield array_to_rows([['value' => 10], ['value' => 20]], schema(int_schema('value'))); })(); $batches = iterator_to_array($processor->process($generator, flow_context()), preserve_keys: false); $values = []; - foreach ($batches[1] as $row) { - $values[] = $row->get('row_number'); + foreach ($batches[1]->toArray() as $row) { + $values[] = $row['row_number']; } static::assertCount(2, $batches); @@ -475,7 +481,9 @@ public function test_bind_refuses_an_expand_in_the_window_function(): void (new WindowProcessor( 's', - sum(ref('n'), ref('flags')->expand()->equals(lit(true)))->over(window()->partitionBy(ref('n'))), + (new WindowFunctionOverOperand(ref('flags')->expand()->equals(lit(true))))->over(window()->partitionBy(ref( + 'n', + ))), ))->bind(ListColumnsMother::numberAndFlagsSchema()); } } diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Repartition/RepartitionStepsTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Repartition/RepartitionStepsTest.php index bfbbab6b55..30d65a0a51 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Unit/Repartition/RepartitionStepsTest.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Repartition/RepartitionStepsTest.php @@ -5,7 +5,6 @@ namespace Flow\ETL\Tests\Unit\Repartition; use Flow\ETL\Bucketing\Storage\MemoryBuckets; -use Flow\ETL\Processor\BucketingProcessor; use Flow\ETL\Processor\RepartitionProcessor; use Flow\ETL\Repartition\RepartitionSteps; use Flow\ETL\Tests\FlowTestCase; @@ -25,9 +24,8 @@ public function test_an_algorithm_builder_pins_the_algorithm_for_this_operation( hash_repartition()->storage(new MemoryBuckets())->bucketsCount(8), ); - static::assertCount(2, $steps); - static::assertInstanceOf(BucketingProcessor::class, $steps[0]); - static::assertInstanceOf(RepartitionProcessor::class, $steps[1]); + static::assertCount(1, $steps); + static::assertInstanceOf(RepartitionProcessor::class, $steps[0]); } public function test_config_supplies_the_algorithm_when_none_is_pinned(): void @@ -37,8 +35,7 @@ public function test_config_supplies_the_algorithm_when_none_is_pinned(): void config_builder()->repartition(hash_repartition()->storage(new MemoryBuckets()))->build(), ); - static::assertCount(2, $steps); - static::assertInstanceOf(BucketingProcessor::class, $steps[0]); - static::assertInstanceOf(RepartitionProcessor::class, $steps[1]); + static::assertCount(1, $steps); + static::assertInstanceOf(RepartitionProcessor::class, $steps[0]); } } diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Row/AdaptiveRowHydratorTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Row/AdaptiveRowHydratorTest.php deleted file mode 100644 index 07bb3601d2..0000000000 --- a/src/core/etl/tests/Flow/ETL/Tests/Unit/Row/AdaptiveRowHydratorTest.php +++ /dev/null @@ -1,62 +0,0 @@ -hydrate( - [new RawRowValues(['id' => 1, 'name' => 'flow']), new RawRowValues(['id' => 2, 'name' => null])], - schema(int_schema('id'), str_schema('name', nullable: true)), - ); - - static::assertSame(2, $rows->count()); - static::assertSame(1, $rows->first()->get('id')); - static::assertNull($rows->all()[1]->get('name')); - } - - public function test_dehydrate_turns_rows_into_typed_values(): void - { - $dehydrated = (new AdaptiveRowHydrator())->dehydrate(rows( - schema(int_schema('id'), str_schema('name')), - row(['id' => 1, 'name' => 'flow']), - )); - - static::assertCount(1, $dehydrated); - static::assertSame(['id' => 1, 'name' => 'flow'], $dehydrated[0]->values); - } - - public function test_hydrate_throws_on_missing_required_structure_element(): void - { - $this->expectException(SchemaMismatchException::class); - $this->expectExceptionMessage('Rows do not match their schema: column "data" (row 0)'); - - (new AdaptiveRowHydrator())->hydrate([new RawRowValues(['data' => [ - 'id' => 1, - ]])], schema(structure_schema('data', type_structure(['id' => type_integer(), 'name' => type_string()])))); - } - - public function test_is_native_when_the_native_hydrator_is_supported(): void - { - static::assertSame(NativeRowHydrator::isSupported(), (new AdaptiveRowHydrator())->isNative()); - } -} diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Row/ArrayToRowTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Row/ArrayToRowTest.php deleted file mode 100644 index 3a15848672..0000000000 --- a/src/core/etl/tests/Flow/ETL/Tests/Unit/Row/ArrayToRowTest.php +++ /dev/null @@ -1,96 +0,0 @@ - ['a', 'b', 'c', 'd']], - schema(list_schema('data', type_list(type_string()))), - flow_context(config())->hydrator(), - ); - - static::assertEquals(row(['data' => ['a', 'b', 'c', 'd']]), $row); - } - - public function test_building_single_row_from_array_with_rows_fails(): void - { - $row = array_to_row( - [ - ['id' => 1234, 'deleted' => false, 'phase' => null], - ['id' => 4321, 'deleted' => true, 'phase' => 'launch'], - ], - schema( - map_schema('e00', type_map(type_string(), type_mixed()), nullable: true), - map_schema('e01', type_map(type_string(), type_mixed()), nullable: true), - ), - flow_context(config())->hydrator(), - ); - - static::assertEquals( - row([ - 'e00' => ['id' => 1234, 'deleted' => false, 'phase' => null], - 'e01' => ['id' => 4321, 'deleted' => true, 'phase' => 'launch'], - ]), - $row, - ); - } - - public function test_building_single_row_from_array_with_schema_and_additional_fields_not_covered_by_schema(): void - { - $row = array_to_row( - ['id' => 1234, 'deleted' => false, 'phase' => null], - schema(int_schema('id'), bool_schema('deleted')), - flow_context(config())->hydrator(), - ); - - static::assertEquals(row(['id' => 1234, 'deleted' => false]), $row); - } - - public function test_building_single_row_from_array_with_schema_but_entries_not_available_in_rows(): void - { - $row = array_to_row( - ['id' => 1234, 'deleted' => false], - schema(int_schema('id'), bool_schema('deleted'), str_schema('phase', true)), - flow_context(config())->hydrator(), - ); - - static::assertEquals(row(['id' => 1234, 'deleted' => false, 'phase' => null]), $row); - } - - public function test_building_single_row_from_flat_array(): void - { - $row = array_to_row( - [ - 'id' => 1234, - 'deleted' => false, - 'phase' => null, - ], - schema(int_schema('id'), bool_schema('deleted'), str_schema('phase', true)), - flow_context(config())->hydrator(), - ); - - static::assertEquals(row(['id' => 1234, 'deleted' => false, 'phase' => null]), $row); - } -} diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Row/Comparator/NativeComparatorTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Row/Comparator/NativeComparatorTest.php deleted file mode 100644 index e206d1502d..0000000000 --- a/src/core/etl/tests/Flow/ETL/Tests/Unit/Row/Comparator/NativeComparatorTest.php +++ /dev/null @@ -1,42 +0,0 @@ -equals( - row(['test' => 'test']), - row(['test' => 'test']), - schema(str_schema('test')), - )); - } - - public function test_row_comparison_for_different_rows(): void - { - static::assertFalse((new NativeComparator())->equals( - row(['test' => 'test']), - row(['test' => 'other']), - schema(str_schema('test')), - )); - } - - public function test_row_comparison_for_rows_with_different_columns(): void - { - static::assertFalse((new NativeComparator())->equals( - row(['test' => 'test']), - row(['other' => 'test']), - schema(str_schema('test')), - )); - } -} diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Row/HydratedBatchTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Row/HydratedBatchTest.php deleted file mode 100644 index 4173e8556d..0000000000 --- a/src/core/etl/tests/Flow/ETL/Tests/Unit/Row/HydratedBatchTest.php +++ /dev/null @@ -1,163 +0,0 @@ - 1, 'name' => null])), - (new HydratedBatch())->of( - [new RawRowValues(['id' => 1])], - schema(int_schema('id'), str_schema('name', nullable: true)), - ), - ); - } - - public function test_a_missing_nullable_column_is_padded_by_the_batch_door(): void - { - static::assertSame( - ['id', 'name'], - (new HydratedBatch()) - ->of([new RawRowValues(['id' => 1])], schema(int_schema('id'), str_schema('name', nullable: true))) - ->first() - ->names(), - ); - } - - public function test_a_refused_value_in_the_first_row_reports_row_zero(): void - { - $this->expectException(SchemaMismatchException::class); - $this->expectExceptionMessage( - 'Rows do not match their schema: column "i" (row 0): could not convert \'abc\' (string) to integer', - ); - - (new HydratedBatch())->of([new RawRowValues(['i' => 'abc'])], schema(int_schema('i'))); - } - - public function test_a_refusal_in_the_second_row_reports_row_one(): void - { - $this->expectException(SchemaMismatchException::class); - $this->expectExceptionMessage( - 'Rows do not match their schema: column "i" (row 1): could not convert \'abc\' (string) to integer', - ); - - (new HydratedBatch())->of([ - new RawRowValues(['i' => 1]), - new RawRowValues(['i' => 'abc']), - ], schema(int_schema('i'))); - } - - public function test_a_refused_value_is_reported_with_its_column_and_row(): void - { - $this->expectException(SchemaMismatchException::class); - $this->expectExceptionMessage( - 'Rows do not match their schema: column "i" (row 1): could not convert \'n/a\' (string) to integer', - ); - - (new HydratedBatch())->of([ - new RawRowValues(['i' => 1]), - new RawRowValues(['i' => 'n/a']), - new RawRowValues(['i' => 3]), - ], schema(int_schema('i'))); - } - - public function test_a_null_on_a_not_null_column_is_reported_as_a_value_that_does_not_match(): void - { - $this->expectException(SchemaMismatchException::class); - $this->expectExceptionMessage( - 'Rows do not match their schema: column "i" (row 0): could not convert null to integer, column is not nullable', - ); - - (new HydratedBatch())->of([new RawRowValues(['i' => null])], schema(int_schema('i'))); - } - - public function test_a_value_the_schema_does_not_declare_is_dropped(): void - { - static::assertSame( - ['id'], - (new HydratedBatch()) - ->of([new RawRowValues(['id' => 1, 'undeclared' => 'x'])], schema(int_schema('id'))) - ->first() - ->names(), - ); - } - - public function test_metadata_is_folded_onto_the_column_with_last_write_winning(): void - { - static::assertSame( - ['k' => 'v2'], - (new HydratedBatch()) - ->of([ - new RawRowValues(['id' => 1], ['id' => Metadata::fromArray(['k' => 'v1'])]), - new RawRowValues(['id' => 2], ['id' => Metadata::fromArray(['k' => 'v2'])]), - ], schema(int_schema('id'))) - ->schema() - ->get('id') - ->metadata() - ->normalize(), - ); - } - - public function test_metadata_is_folded_onto_a_numeric_column_name(): void - { - static::assertSame( - ['k' => 'v'], - (new HydratedBatch()) - ->of([new RawRowValues([], ['0' => Metadata::fromArray([ - 'k' => 'v', - ])])], schema(int_schema('0', nullable: true))) - ->schema() - ->get('0') - ->metadata() - ->normalize(), - ); - } - - public function test_metadata_for_an_undeclared_column_is_ignored(): void - { - static::assertEquals( - schema(int_schema('id')), - (new HydratedBatch())->of([new RawRowValues(['id' => 1], ['nope' => Metadata::fromArray([ - 'k' => 'v', - ])])], schema(int_schema('id')))->schema(), - ); - } - - public function test_rows_follow_the_schema_column_order(): void - { - static::assertSame( - ['id', 'name'], - (new HydratedBatch()) - ->of([new RawRowValues(['name' => 'a', 'id' => 1])], schema(int_schema('id'), str_schema('name'))) - ->first() - ->names(), - ); - } - - public function test_every_value_is_cast_against_its_own_column(): void - { - static::assertEquals( - rows(schema(int_schema('id'), str_schema('name')), row(['id' => 1, 'name' => '2'])), - (new HydratedBatch())->of( - [new RawRowValues(['id' => '1', 'name' => 2])], - schema(int_schema('id'), str_schema('name')), - ), - ); - } -} diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Row/InferredBatchTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Row/InferredBatchTest.php deleted file mode 100644 index 894983b80b..0000000000 --- a/src/core/etl/tests/Flow/ETL/Tests/Unit/Row/InferredBatchTest.php +++ /dev/null @@ -1,101 +0,0 @@ -of([ - new RawRowValues(['id' => 1, 'name' => 'a']), - new RawRowValues(['id' => 2]), - ])->schema(), - ); - } - - public function test_an_empty_batch_infers_an_empty_schema(): void - { - static::assertEquals(rows(schema()), (new InferredBatch())->of([])); - } - - public function test_an_explicit_null_makes_the_column_nullable(): void - { - static::assertEquals( - schema(int_schema('id'), str_schema('name', nullable: true)), - (new InferredBatch())->of([ - new RawRowValues(['id' => 1, 'name' => 'a']), - new RawRowValues(['id' => 2, 'name' => null]), - ])->schema(), - ); - } - - public function test_a_numeric_column_name_is_a_string_column(): void - { - // PHP hands a numeric array key back as an int, and definition_from_type() takes - // Reference|string - a pivot names its columns by their own values, so this is reachable - $values = []; - - foreach ([0, 7] as $pivotValue) { - $values[(string) $pivotValue] = $pivotValue + 1; - } - - // PHP cannot hold '0' as a string key, which is precisely the shape under test - $inferred = (new InferredBatch())->of([new RawRowValues($values)]); - - static::assertSame(['0', '7'], $inferred->schema()->references()->names()); - static::assertSame([['0' => 1, '7' => 8]], $inferred->toArray()); - } - - public function test_metadata_travels_from_the_raw_values_onto_the_definition(): void - { - static::assertSame( - ['origin' => 'csv'], - (new InferredBatch()) - ->of([ - new RawRowValues(['id' => 1], ['id' => Metadata::fromArray(['origin' => 'csv'])]), - ]) - ->schema() - ->get('id') - ->metadata() - ->normalize(), - ); - } - - public function test_rows_keep_their_values_verbatim(): void - { - static::assertEquals( - rows(schema(int_schema('id'), str_schema('name')), row(['id' => 1, 'name' => 'a'])), - (new InferredBatch())->of([new RawRowValues(['id' => 1, 'name' => 'a'])]), - ); - } - - /** - * Widening derives one column type from values of two, so the values follow the schema it just - * derived - otherwise the batch would contradict its own declaration at the door. - */ - public function test_widening_folds_int_and_string_into_one_column_and_converts_the_values(): void - { - $rows = (new InferredBatch())->of([ - new RawRowValues(['v' => 1]), - new RawRowValues(['v' => 'text']), - ]); - - static::assertSame('string', $rows->schema()->get('v')->type()->toString()); - static::assertSame([['v' => '1'], ['v' => 'text']], $rows->toArray()); - } -} diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Row/NativeRowHydratorTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Row/NativeRowHydratorTest.php deleted file mode 100644 index ce00a48721..0000000000 --- a/src/core/etl/tests/Flow/ETL/Tests/Unit/Row/NativeRowHydratorTest.php +++ /dev/null @@ -1,742 +0,0 @@ -}> - */ - public static function serializable_datasets(): Generator - { - yield 'scalars over multiple rows' => [ - schema(int_schema('id'), str_schema('name', nullable: true), float_schema('p'), bool_schema('a')), - [ - new RawRowValues(['id' => 1, 'name' => 'x', 'p' => 1.5, 'a' => true]), - new RawRowValues(['id' => 2, 'name' => null, 'p' => -0.25, 'a' => false]), - new RawRowValues(['id' => 3, 'name' => 'z', 'p' => 0.0, 'a' => true]), - ], - ]; - - yield 'null values across rows' => [ - schema(int_schema('id'), str_schema('name', nullable: true)), - [ - new RawRowValues(['id' => 1, 'name' => null]), - new RawRowValues(['id' => 2, 'name' => null]), - ], - ]; - - yield 'absent columns are skipped' => [ - schema(int_schema('id'), str_schema('name', nullable: true), bool_schema('flag', nullable: true)), - [ - new RawRowValues(['id' => 1]), - new RawRowValues(['id' => 2, 'name' => 'y']), - ], - ]; - - yield 'per-value metadata divergence across rows' => [ - schema(int_schema('id')), - [ - new RawRowValues(['id' => 1], ['id' => Metadata::fromArray(['k' => 'v1'])]), - new RawRowValues(['id' => 2], ['id' => Metadata::fromArray(['k' => 'v2'])]), - ], - ]; - - yield 'per-value metadata together with a null value' => [ - schema(int_schema('id', nullable: true)), - [new RawRowValues(['id' => null], ['id' => Metadata::fromArray(['k' => 'v'])])], - ]; - - yield 'temporal and uuid values' => [ - schema(datetime_schema('at'), date_schema('d'), time_schema('t'), uuid_schema('u')), - [new RawRowValues([ - 'at' => new DateTimeImmutable('2025-01-01 12:00:00.123456', new DateTimeZone('Europe/Warsaw')), - 'd' => new DateTimeImmutable('2025-03-01'), - 't' => new DateInterval('PT2H30M5S'), - 'u' => new Uuid('01234567-89ab-4def-8123-456789abcdef'), - ])], - ]; - - yield 'containers and json' => [ - schema( - list_schema('l', type_list(type_integer())), - map_schema('m', type_map(type_string(), type_integer())), - structure_schema('st', type_structure(['a' => type_integer(), 'b' => type_string()])), - list_schema('opt', type_list(type_optional(type_integer()))), - json_schema('j'), - ), - [new RawRowValues([ - 'l' => [1, 2, 3], - 'm' => ['a' => 1], - 'st' => ['a' => 1, 'b' => 'kept'], - 'opt' => [1, null], - 'j' => Json::fromArray(['x' => 1]), - ])], - ]; - - yield 'enum values including a null variant' => [ - schema(enum_schema('s', ParitySuit::class, nullable: true)), - [ - new RawRowValues(['s' => ParitySuit::Hearts]), - new RawRowValues(['s' => ParitySuit::Spades]), - new RawRowValues(['s' => null]), - ], - ]; - - yield 'null definition column' => [ - schema(null_schema('n'), int_schema('id', nullable: true)), - [ - new RawRowValues(['n' => null, 'id' => null]), - new RawRowValues(['n' => null, 'id' => 5]), - ], - ]; - - yield 'empty batch' => [schema(int_schema('id')), []]; - } - - /** - * @return \Generator}> - */ - public static function hydratable_datasets(): Generator - { - yield 'scalar columns from raw strings and coercion edges' => [ - schema(int_schema('id'), float_schema('price'), bool_schema('active'), str_schema('name', nullable: true)), - [ - new RawRowValues(['id' => '42', 'price' => '3.14', 'active' => 'yes', 'name' => 7]), - new RawRowValues(['id' => ' 7', 'price' => '1e3', 'active' => 'OFF', 'name' => 1.5]), - new RawRowValues(['id' => '007', 'price' => '.5', 'active' => 'ON', 'name' => true]), - new RawRowValues(['id' => '1e2', 'price' => true, 'active' => 0, 'name' => false]), - new RawRowValues(['id' => '9223372036854775807', 'price' => 7, 'active' => 3.5, 'name' => null]), - new RawRowValues(['id' => 5, 'price' => 2.5, 'active' => true, 'name' => 'text']), - ], - ]; - - yield 'nested optional datetimes and structures move into the declared shape' => [ - schema( - list_schema('at', type_list(type_optional(type_datetime('Europe/Warsaw')))), - structure_schema('s', type_structure([ - 'at' => type_optional(type_datetime('Europe/Warsaw')), - 'inner' => type_optional(type_structure(['a' => type_integer(), 'b' => type_string()])), - ])), - ), - [ - new RawRowValues([ - 'at' => [new DateTimeImmutable('2026-01-02 03:04:05', new DateTimeZone('UTC')), null], - 's' => [ - 'at' => new DateTimeImmutable('2026-01-02 03:04:05', new DateTimeZone('UTC')), - 'inner' => ['b' => 'x', 'a' => 1], - ], - ]), - ], - ]; - - yield 'nested positive integer and string family elements' => [ - schema( - list_schema('counts', type_list(type_positive_integer())), - structure_schema('labels', type_structure(['label' => type_non_empty_string()])), - structure_schema('codes', type_structure(['code' => type_numeric_string()])), - ), - [ - new RawRowValues([ - 'counts' => [5, 7], - 'labels' => ['label' => 'x'], - 'codes' => ['code' => '42'], - ]), - new RawRowValues([ - 'counts' => ['5', 8], - 'labels' => ['label' => 9], - 'codes' => ['code' => 42], - ]), - new RawRowValues([ - 'counts' => [], - 'labels' => ['label' => 1.5], - 'codes' => ['code' => '3.14'], - ]), - new RawRowValues([ - 'counts' => [1], - 'labels' => ['label' => true], - 'codes' => ['code' => 42.5], - ]), - ], - ]; - - $shared = new DateTimeImmutable('2025-01-01 12:00:00.123456', new DateTimeZone('Europe/Warsaw')); - - yield 'temporal columns preserving object identity topology' => [ - schema( - datetime_schema('at', zone: 'Europe/Warsaw'), - datetime_schema('at2', zone: 'Europe/Warsaw'), - date_schema('d'), - ), - [ - new RawRowValues(['at' => $shared, 'at2' => $shared, 'd' => $shared]), - new RawRowValues([ - 'at' => $shared, - 'at2' => '2024-03-01 10:20:30', - 'd' => new DateTimeImmutable('2025-03-01'), - ]), - new RawRowValues(['at' => 1700000000, 'at2' => 1700000000.5, 'd' => '2024-03-05 08:30:00']), - new RawRowValues(['at' => '2024-02-29T12:00:00+02:00', 'at2' => '2024-06-01', 'd' => 1700000000]), - ], - ]; - - yield 'uuid and json columns from raw values' => [ - schema(uuid_schema('u', nullable: true), json_schema('j')), - [ - new RawRowValues(['u' => '01234567-89ab-4def-8123-456789abcdef', 'j' => '["a","b"]']), - new RawRowValues(['u' => new Uuid('01234567-89ab-4def-8123-456789abcdef'), 'j' => '{"a":1}']), - new RawRowValues(['u' => null, 'j' => ['a' => 1]]), - new RawRowValues(['u' => '11234567-89ab-4def-8123-456789abcdef', 'j' => []]), - new RawRowValues(['u' => '21234567-89ab-4def-8123-456789abcdef', 'j' => Json::fromArray(['x' => 1])]), - ], - ]; - - yield 'containers from raw values' => [ - schema( - list_schema('l', type_list(type_integer())), - list_schema('ll', type_list(type_list(type_integer()))), - map_schema('m', type_map(type_string(), type_integer())), - map_schema('mi', type_map(type_integer(), type_string())), - list_schema('lo', type_list(type_optional(type_integer()))), - structure_schema('st', type_structure([ - 'a' => type_integer(), - 'b' => structure_element('b', type_string(), optional: true), - ])), - ), - [ - new RawRowValues([ - 'l' => ['1', 2, '3'], - 'll' => [['1', '2'], [3]], - 'm' => ['a' => '1', 'b' => 2], - 'mi' => [0 => 'x', 5 => 7], - 'lo' => ['1', null, 3], - 'st' => ['a' => '5', 'extra' => 'dropped'], - ]), - new RawRowValues([ - 'l' => [], - 'll' => [], - 'm' => [], - 'mi' => [], - 'lo' => [], - 'st' => ['a' => 1, 'b' => 'present'], - ]), - ], - ]; - - yield 'exotic columns cast through the per-value fallback' => [ - schema(enum_schema('s', ParitySuit::class), xml_schema('x'), time_schema('t')), - [ - new RawRowValues([ - 's' => ParitySuit::Hearts, - 'x' => 'v', - 't' => new DateInterval('PT2H30M5S'), - ]), - new RawRowValues(['s' => 'h', 'x' => '', 't' => 'PT2H']), - ], - ]; - - yield 'metadata variants fill-missing and extra raw keys' => [ - schema( - int_schema('id', nullable: true), - str_schema('name', nullable: true), - bool_schema('flag', nullable: true), - ), - [ - new RawRowValues(['id' => '1'], ['id' => Metadata::fromArray(['k' => 'v1'])]), - new RawRowValues([]), - new RawRowValues(['id' => 2, 'name' => null, 'unknown' => 'dropped', 'flag' => 'on']), - new RawRowValues(['id' => null], ['id' => Metadata::fromArray(['k' => 'v2'])]), - ], - ]; - - yield 'empty cast batch' => [schema(int_schema('id')), []]; - - yield 'all-optional structure with no matching keys' => [ - schema(structure_schema('st', type_structure(['b' => structure_element( - 'b', - type_string(), - optional: true, - )]))), - [new RawRowValues(['st' => ['other' => 1]])], - ]; - - yield 'interleaved structure' => [ - schema(structure_schema('st', type_structure([ - 'z' => type_integer(), - 'a' => structure_element('a', type_string(), optional: true), - 'b' => type_string(), - ]))), - [ - new RawRowValues(['st' => ['b' => 'x', 'z' => '5']]), - new RawRowValues(['st' => ['z' => 1, 'a' => 'present', 'b' => 'y']]), - ], - ]; - } - - /** - * @return \Generator, class-string<\Throwable>, string}> - */ - public static function refusing_datasets(): Generator - { - yield 'null in a not null column' => [ - schema(int_schema('id'), str_schema('name')), - [new RawRowValues(['id' => 1, 'name' => null])], - SchemaMismatchException::class, - 'Rows do not match their schema: column "name" (row 0): could not convert null to string, column is not nullable', - ]; - - yield 'non numeric string in an integer column' => [ - schema(int_schema('id')), - [new RawRowValues(['id' => 'abc'])], - SchemaMismatchException::class, - 'Rows do not match their schema: column "id" (row 0): could not convert \'abc\' (string) to integer', - ]; - - yield 'trailing garbage after digits in an integer column' => [ - schema(int_schema('id')), - [new RawRowValues(['id' => '12abc'])], - SchemaMismatchException::class, - 'Rows do not match their schema: column "id" (row 0): could not convert \'12abc\' (string) to integer', - ]; - - yield 'hex string in a float column' => [ - schema(float_schema('price')), - [new RawRowValues(['price' => '0x1A'])], - SchemaMismatchException::class, - 'Rows do not match their schema: column "price" (row 0): could not convert \'0x1A\' (string) to float', - ]; - - yield 'unrecognised word in a boolean column' => [ - schema(bool_schema('active')), - [new RawRowValues(['active' => 'weird'])], - SchemaMismatchException::class, - 'Rows do not match their schema: column "active" (row 0): could not convert \'weird\' (string) to boolean', - ]; - - yield 'array in an integer column' => [ - schema(int_schema('id')), - [new RawRowValues(['id' => [1, 2, 3]])], - SchemaMismatchException::class, - 'Rows do not match their schema: column "id" (row 0): could not convert array (list) to integer', - ]; - - yield 'invalid uuid string' => [ - schema(uuid_schema('u')), - [new RawRowValues(['u' => 'not-a-uuid'])], - SchemaMismatchException::class, - 'Rows do not match their schema: column "u" (row 0): could not convert \'not-a-uuid\' (string) to uuid', - ]; - - yield 'uppercase uuid string' => [ - schema(uuid_schema('u')), - [new RawRowValues(['u' => '01234567-89AB-4DEF-8123-456789ABCDEF'])], - SchemaMismatchException::class, - 'Rows do not match their schema: column "u" (row 0): could not convert \'01234567-89AB-4DEF-8123-456789AB...\' (string) to uuid', - ]; - - yield 'json from a scalar' => [ - schema(json_schema('j')), - [new RawRowValues(['j' => 5])], - SchemaMismatchException::class, - 'Rows do not match their schema: column "j" (row 0): could not convert 5 (integer) to json', - ]; - - yield 'json from an invalid json string' => [ - schema(json_schema('j')), - [new RawRowValues(['j' => '{oops'])], - SchemaMismatchException::class, - 'Rows do not match their schema: column "j" (row 0): could not convert \'{oops\' (string) to json', - ]; - - yield 'json from a plain string' => [ - schema(json_schema('j')), - [new RawRowValues(['j' => 'plain'])], - SchemaMismatchException::class, - 'Rows do not match their schema: column "j" (row 0): could not convert \'plain\' (string) to json', - ]; - - yield 'datetime from garbage' => [ - schema(datetime_schema('at')), - [new RawRowValues(['at' => 'not-a-date'])], - SchemaMismatchException::class, - 'Rows do not match their schema: column "at" (row 0): could not convert \'not-a-date\' (string) to datetime', - ]; - - yield 'datetime from an array' => [ - schema(datetime_schema('at')), - [new RawRowValues(['at' => ['nope']])], - SchemaMismatchException::class, - 'Rows do not match their schema: column "at" (row 0): could not convert array (list) to datetime', - ]; - - yield 'date from garbage' => [ - schema(date_schema('d')), - [new RawRowValues(['d' => 'not-a-date'])], - SchemaMismatchException::class, - 'Rows do not match their schema: column "d" (row 0): could not convert \'not-a-date\' (string) to date', - ]; - - yield 'string map with integer keys' => [ - schema(map_schema('m', type_map(type_string(), type_integer()))), - [new RawRowValues(['m' => [5 => 1]])], - SchemaMismatchException::class, - 'Rows do not match their schema: column "m" (row 0): could not convert array (map) to map', - ]; - - yield 'list with non-sequential keys' => [ - schema(list_schema('l', type_list(type_integer()))), - [new RawRowValues(['l' => [1 => 'x']])], - SchemaMismatchException::class, - 'Rows do not match their schema: column "l" (row 0): could not convert array (map) to list', - ]; - - yield 'positive integer list from a non-numeric string' => [ - schema(list_schema('l', type_list(type_positive_integer()))), - [new RawRowValues(['l' => ['abc']])], - SchemaMismatchException::class, - 'Rows do not match their schema: column "l" (row 0): could not convert array (list) to list', - ]; - - yield 'positive integer list from a negative int' => [ - schema(list_schema('l', type_list(type_positive_integer()))), - [new RawRowValues(['l' => [-3]])], - SchemaMismatchException::class, - 'Rows do not match their schema: column "l" (row 0): could not convert array (list) to list', - ]; - - yield 'structure missing required element' => [ - schema(structure_schema('data', type_structure(['id' => type_integer(), 'name' => type_string()]))), - [new RawRowValues(['data' => ['id' => 1]])], - SchemaMismatchException::class, - 'Rows do not match their schema: column "data" (row 0): could not convert array (structure{id: integer}) to structure{id: integer, name: string}', - ]; - - yield 'structure present-null required element' => [ - schema(structure_schema('data', type_structure(['id' => type_integer(), 'name' => type_string()]))), - [new RawRowValues(['data' => ['id' => 1, 'name' => null]])], - SchemaMismatchException::class, - 'Rows do not match their schema: column "data" (row 0): could not convert array (structure{id: integer, name: null}) to structure{id: integer, name: string}', - ]; - - yield 'structure present-null optional element' => [ - schema(structure_schema('data', type_structure([ - 'id' => type_integer(), - 'name' => structure_element('name', type_string(), optional: true), - ]))), - [new RawRowValues(['data' => ['id' => 1, 'name' => null]])], - SchemaMismatchException::class, - 'Rows do not match their schema: column "data" (row 0): could not convert array (structure{id: integer, name: null}) to structure{id: integer, name?: string}', - ]; - - // the refusal is placed at the batch position, so a constant row index would diverge here - yield 'non numeric string in the second row of an integer column' => [ - schema(int_schema('id')), - [new RawRowValues(['id' => '1']), new RawRowValues(['id' => 'x'])], - SchemaMismatchException::class, - 'Rows do not match their schema: column "id" (row 1): could not convert \'x\' (string) to integer', - ]; - - yield 'a not null column the row does not carry' => [ - schema(int_schema('id'), str_schema('name')), - [new RawRowValues(['id' => 1])], - SchemaMismatchException::class, - 'Rows do not match their schema: column "name" (row 0) declared by the schema is missing from the row', - ]; - - yield 'integer overflow' => [ - schema(int_schema('id')), - [new RawRowValues(['id' => '9223372036854775808'])], - SchemaMismatchException::class, - 'Rows do not match their schema: column "id" (row 0): could not convert \'9223372036854775808\' (string) to integer', - ]; - - yield 'date from an empty string' => [ - schema(date_schema('d')), - [new RawRowValues(['d' => ''])], - SchemaMismatchException::class, - 'Rows do not match their schema: column "d" (row 0): could not convert \'\' (string) to date', - ]; - - yield 'datetime from a relative word' => [ - schema(datetime_schema('at')), - [new RawRowValues(['at' => 'now'])], - SchemaMismatchException::class, - 'Rows do not match their schema: column "at" (row 0): could not convert \'now\' (string) to datetime', - ]; - - yield 'string from a null element inside a list' => [ - schema(list_schema('l', type_list(type_string()))), - [new RawRowValues(['l' => ['a', null]])], - SchemaMismatchException::class, - 'Rows do not match their schema: column "l" (row 0): could not convert array (list) to list', - ]; - - yield 'list from a scalar' => [ - schema(list_schema('l', type_list(type_integer()))), - [new RawRowValues(['l' => 5])], - SchemaMismatchException::class, - 'Rows do not match their schema: column "l" (row 0): could not convert 5 (integer) to list', - ]; - - // ordering: an absence must not pre-empt a refusal that PHP reports first, or the two - // hydrators name different columns - and different ROWS - for the same batch - yield 'an absent column before a cast refusal in the same row' => [ - schema(int_schema('a'), int_schema('b')), - [new RawRowValues(['b' => 'abc'])], - SchemaMismatchException::class, - 'Rows do not match their schema: column "b" (row 0): could not convert \'abc\' (string) to integer', - ]; - - yield 'an absent column in row 0 before a cast refusal in row 1' => [ - schema(int_schema('a'), int_schema('b')), - [new RawRowValues(['a' => 1]), new RawRowValues(['a' => 1, 'b' => 'abc'])], - SchemaMismatchException::class, - 'Rows do not match their schema: column "b" (row 1): could not convert \'abc\' (string) to integer', - ]; - - yield 'an absent column in row 0 before a present null in row 1' => [ - schema(int_schema('a'), int_schema('b')), - [new RawRowValues(['a' => 1]), new RawRowValues(['a' => 1, 'b' => null])], - SchemaMismatchException::class, - 'Rows do not match their schema: column "b" (row 1): could not convert null to integer, column is not nullable', - ]; - - // an exception the types package did not raise is not a refusal, so neither engine wraps it - yield 'a type raising an exception outside the types package' => [ - schema( - new ForeignTypeDefinition('a', new ThrowingType(new LogicException('stub type refuses everything'))), - ), - [new RawRowValues(['a' => [1, 2]])], - LogicException::class, - 'stub type refuses everything', - ]; - } - - /** - * @param list $batch - */ - #[DataProvider('serializable_datasets')] - public function test_native_hydrate_of_typed_values_is_serialize_identical_to_php( - Schema $schema, - array $batch, - ): void { - if (!NativeRowHydrator::isSupported()) { - static::markTestSkipped('flow_php extension with the native hydrator is not loaded.'); - } - - static::assertSame( - serialize((new PhpRowHydrator())->hydrate($batch, $schema)), - serialize((new NativeRowHydrator())->hydrate($batch, $schema)), - ); - } - - /** - * @param list $batch - */ - #[DataProvider('serializable_datasets')] - public function test_native_dehydrate_is_serialize_identical_to_php(Schema $schema, array $batch): void - { - if (!NativeRowHydrator::isSupported()) { - static::markTestSkipped('flow_php extension with the native hydrator is not loaded.'); - } - - $php = new PhpRowHydrator(); - $rows = $php->hydrate($batch, $schema); - - static::assertSame(serialize($php->dehydrate($rows)), serialize((new NativeRowHydrator())->dehydrate($rows))); - } - - /** - * @param list $batch - */ - #[DataProvider('hydratable_datasets')] - public function test_native_hydrate_of_raw_values_is_serialize_identical_to_php(Schema $schema, array $batch): void - { - if (!NativeRowHydrator::isSupported()) { - static::markTestSkipped('flow_php extension with the native hydrator is not loaded.'); - } - - static::assertSame( - serialize((new PhpRowHydrator())->hydrate($batch, $schema)), - serialize((new NativeRowHydrator())->hydrate($batch, $schema)), - ); - } - - /** - * The expected class and message are pinned literally rather than compared between the two - * hydrators, so a drift they share still reddens. - * - * @param list $batch - * @param class-string<\Throwable> $exception - */ - #[DataProvider('refusing_datasets')] - public function test_php_and_native_refuse_identically( - Schema $schema, - array $batch, - string $exception, - string $message, - ): void { - $refusal = static function (Hydrator $hydrator) use ($batch, $schema): Throwable { - try { - $hydrator->hydrate($batch, $schema); - } catch (Throwable $e) { - return $e; - } - - throw new LogicException('the batch was accepted, but the schema declares it a refusal'); - }; - - $php = $refusal(new PhpRowHydrator()); - - static::assertSame($exception, $php::class); - static::assertSame($message, $php->getMessage()); - - if (!NativeRowHydrator::isSupported()) { - return; - } - - $native = $refusal(new NativeRowHydrator()); - - static::assertSame($exception, $native::class); - static::assertSame($message, $native->getMessage()); - } - - /** - * @param list $batch - */ - #[DataProvider('hydratable_datasets')] - public function test_native_hydrate_is_idempotent_on_already_hydrated_values(Schema $schema, array $batch): void - { - if (!NativeRowHydrator::isSupported()) { - static::markTestSkipped('flow_php extension with the native hydrator is not loaded.'); - } - - $php = new PhpRowHydrator(); - $recastBatch = []; - - foreach ($php->hydrate($batch, $schema) as $row) { - $recastBatch[] = new RawRowValues($row->values()); - } - - static::assertSame( - serialize($php->hydrate($recastBatch, $schema)), - serialize((new NativeRowHydrator())->hydrate($recastBatch, $schema)), - ); - } - - public function test_native_hydrate_follows_schema_changes(): void - { - if (!NativeRowHydrator::isSupported()) { - static::markTestSkipped('flow_php extension with the native hydrator is not loaded.'); - } - - $schema = schema(int_schema('id'), str_schema('name')); - $php = new PhpRowHydrator(); - $native = new NativeRowHydrator(); - - $batch = [new RawRowValues(['id' => '1', 'name' => 7])]; - static::assertSame(serialize($php->hydrate($batch, $schema)), serialize($native->hydrate($batch, $schema))); - - $schema = $schema->add(bool_schema('active', nullable: true)); - - $batch = [new RawRowValues(['id' => '2', 'name' => 'b', 'active' => 'yes'])]; - static::assertSame(serialize($php->hydrate($batch, $schema)), serialize($native->hydrate($batch, $schema))); - - $schema = $schema->makeNullable(); - - $batch = [new RawRowValues(['id' => null, 'name' => null, 'active' => null])]; - static::assertSame(serialize($php->hydrate($batch, $schema)), serialize($native->hydrate($batch, $schema))); - } - - public function test_native_moves_markup_values_verbatim(): void - { - if (!NativeRowHydrator::isSupported()) { - static::markTestSkipped('flow_php extension with the native hydrator is not loaded.'); - } - - $document = new DOMDocument(); - $document->loadXML('v'); - - $rows = rows(schema(xml_schema('doc')), row(['doc' => $document])); - - $php = new PhpRowHydrator(); - $native = new NativeRowHydrator(); - - $phpDehydrated = $php->dehydrate($rows); - $nativeDehydrated = $native->dehydrate($rows); - - static::assertSame($phpDehydrated[0]->values['doc'], $nativeDehydrated[0]->values['doc']); - static::assertEquals($phpDehydrated[0]->types['doc'], $nativeDehydrated[0]->types['doc']); - } - - public function test_supports_extension_requires_the_native_class(): void - { - static::assertSame( - extension_loaded('flow_php') && class_exists(RustRowHydratorNative::class, false), - NativeRowHydrator::isSupported(), - ); - } -} - -enum ParitySuit: string -{ - case Hearts = 'h'; - case Spades = 's'; -} diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Row/NullsOrderTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Row/NullsOrderTest.php new file mode 100644 index 0000000000..e3cea66e42 --- /dev/null +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Row/NullsOrderTest.php @@ -0,0 +1,18 @@ +hydrate( - [new RawRowValues(['id' => '1'])], - schema(int_schema('id'), str_schema('name', nullable: true)), - ); - - static::assertTrue($rows->first()->has('name')); - static::assertNull($rows->first()->get('name')); - static::assertTrue($rows->schema()->get('name')->isNullable()); - static::assertSame(['id' => 1, 'name' => null], $rows->first()->toArray()); - } - - public function test_hydrate_does_not_wrap_an_exception_the_types_package_did_not_raise(): void - { - $this->expectException(LogicException::class); - $this->expectExceptionMessage('stub type refuses everything'); - - (new PhpRowHydrator())->hydrate([new RawRowValues(['a' => [ - 1, - 2, - ]])], schema(new ForeignTypeDefinition('a', new ThrowingType(new LogicException('stub type refuses everything'))))); - } - - public function test_hydrate_throws_on_missing_required_structure_element(): void - { - $this->expectException(SchemaMismatchException::class); - $this->expectExceptionMessage('Rows do not match their schema: column "data" (row 0)'); - - (new PhpRowHydrator())->hydrate([new RawRowValues(['data' => [ - 'id' => 1, - ]])], schema(structure_schema('data', type_structure(['id' => type_integer(), 'name' => type_string()])))); - } - - public function test_casts_datetime_and_uuid_strings(): void - { - $rows = (new PhpRowHydrator())->hydrate( - [new RawRowValues([ - 'created_at' => '2024-01-01 12:00:00 UTC', - 'uuid' => 'f47ac10b-58cc-4372-a567-0e02b2c3d479', - ])], - schema(datetime_schema('created_at'), uuid_schema('uuid')), - ); - - $uuid = $rows->first()->get('uuid'); - - static::assertInstanceOf(DateTimeImmutable::class, $rows->first()->get('created_at')); - static::assertInstanceOf(Uuid::class, $uuid); - static::assertSame('f47ac10b-58cc-4372-a567-0e02b2c3d479', $uuid->toString()); - } - - public function test_casts_raw_scalar_strings_to_schema_types(): void - { - $rows = (new PhpRowHydrator())->hydrate( - [new RawRowValues(['id' => '1', 'price' => '9.99', 'active' => 'true', 'name' => 'Alice'])], - schema(int_schema('id'), float_schema('price'), bool_schema('active'), str_schema('name')), - ); - - static::assertSame( - ['id' => 1, 'price' => 9.99, 'active' => true, 'name' => 'Alice'], - $rows->first()->toArray(), - ); - } - - public function test_columns_absent_from_the_schema_are_dropped(): void - { - $rows = (new PhpRowHydrator())->hydrate([new RawRowValues([ - 'id' => '1', - 'extra' => 'raw', - ])], schema(int_schema('id'))); - - static::assertFalse($rows->first()->has('extra')); - static::assertSame(['id' => 1], $rows->first()->toArray()); - } - - public function test_dehydrate_returns_typed_values_keyed_by_entry_name(): void - { - $hydrator = new PhpRowHydrator(); - $schema = schema(int_schema('id'), str_schema('name', nullable: true)); - - $batch = [ - new RawRowValues(['id' => 1, 'name' => 'Alice']), - new RawRowValues(['id' => 2, 'name' => null]), - ]; - - static::assertEquals( - [ - new TypedRowValues(['id' => 1, 'name' => 'Alice'], ['id' => type_integer(), 'name' => type_string()]), - new TypedRowValues(['id' => 2, 'name' => null], ['id' => type_integer(), 'name' => type_string()]), - ], - $hydrator->dehydrate($hydrator->hydrate($batch, $schema)), - ); - } - - public function test_empty_batch_produces_empty_rows(): void - { - static::assertCount(0, (new PhpRowHydrator())->hydrate([], schema(int_schema('id')))); - } - - public function test_hydrate_keeps_native_typed_values(): void - { - $rows = (new PhpRowHydrator())->hydrate( - [ - new RawRowValues(['id' => 1, 'name' => 'Alice']), - new RawRowValues(['id' => 2, 'name' => 'Bob']), - ], - schema(int_schema('id'), str_schema('name')), - ); - - static::assertSame( - [ - ['id' => 1, 'name' => 'Alice'], - ['id' => 2, 'name' => 'Bob'], - ], - $rows->toArray(), - ); - } - - public function test_hydrate_keeps_native_uuid_value_objects(): void - { - $rows = (new PhpRowHydrator())->hydrate([new RawRowValues([ - 'id' => new Uuid('f47ac10b-58cc-4372-a567-0e02b2c3d479'), - ])], schema(uuid_schema('id'))); - - $value = $rows->first()->get('id'); - - static::assertInstanceOf(Uuid::class, $value); - static::assertSame('f47ac10b-58cc-4372-a567-0e02b2c3d479', $value->toString()); - } - - public function test_null_value_in_a_non_nullable_column_is_refused(): void - { - $this->expectException(SchemaMismatchException::class); - $this->expectExceptionMessage('Rows do not match their schema: column "name" (row 0): could not convert null to string, ' - . 'column is not nullable'); - - (new PhpRowHydrator())->hydrate( - [new RawRowValues(['id' => 1, 'name' => null])], - schema(int_schema('id'), str_schema('name')), - ); - } - - public function test_null_value_keeps_the_schema_definition(): void - { - $rows = (new PhpRowHydrator())->hydrate( - [new RawRowValues(['id' => 1, 'name' => null])], - schema(int_schema('id'), str_schema('name', nullable: true)), - ); - - static::assertNull($rows->first()->get('name')); - static::assertTrue($rows->schema()->get('name')->isNullable()); - } - - public function test_plan_is_rebuilt_when_the_schema_changes(): void - { - $hydrator = new PhpRowHydrator(); - - $ids = $hydrator->hydrate([new RawRowValues(['id' => '1'])], schema(int_schema('id'))); - $prices = $hydrator->hydrate([new RawRowValues(['id' => '1'])], schema(float_schema('id'))); - - static::assertSame(['id' => 1], $ids->first()->toArray()); - static::assertSame(['id' => 1.0], $prices->first()->toArray()); - } - - public function test_rows_share_the_declared_schema_instance(): void - { - $schema = schema(int_schema('id')); - - $rows = (new PhpRowHydrator())->hydrate([ - new RawRowValues(['id' => 1]), - new RawRowValues(['id' => 2]), - ], $schema); - - static::assertSame($schema, $rows->schema()); - } - - public function test_hydrate_does_not_cast_native_values(): void - { - $createdAt = new DateTimeImmutable('2024-01-01 12:00:00 UTC'); - - $rows = (new PhpRowHydrator())->hydrate( - [new RawRowValues(['id' => 1, 'created_at' => $createdAt])], - schema(int_schema('id'), datetime_schema('created_at')), - ); - - static::assertSame(1, $rows->first()->get('id')); - static::assertSame($createdAt, $rows->first()->get('created_at')); - } - - public function test_hydrate_does_not_cast_nested_list_map_and_structure(): void - { - $rows = (new PhpRowHydrator())->hydrate( - [new RawRowValues([ - 'tags' => ['a', 'b'], - 'counts' => ['x' => 1, 'y' => 2], - 'address' => ['city' => 'NYC', 'zip' => 10001], - ])], - schema( - list_schema('tags', type_list(type_string())), - map_schema('counts', type_map(type_string(), type_integer())), - structure_schema('address', type_structure(['city' => type_string(), 'zip' => type_integer()])), - ), - ); - - static::assertInstanceOf(ListType::class, $rows->schema()->get('tags')->type()); - static::assertInstanceOf(MapType::class, $rows->schema()->get('counts')->type()); - static::assertInstanceOf(StructureType::class, $rows->schema()->get('address')->type()); - static::assertSame(['a', 'b'], $rows->first()->get('tags')); - static::assertSame(['x' => 1, 'y' => 2], $rows->first()->get('counts')); - static::assertSame(['city' => 'NYC', 'zip' => 10001], $rows->first()->get('address')); - } - - public function test_hydrate_pads_a_nullable_column_absent_from_the_values(): void - { - $rows = (new PhpRowHydrator())->hydrate( - [new RawRowValues(['id' => 1])], - schema(int_schema('id'), str_schema('name', nullable: true)), - ); - - static::assertTrue($rows->first()->has('id')); - static::assertTrue($rows->first()->has('name')); - static::assertNull($rows->first()->get('name')); - } - - public function test_hydrating_already_typed_values_is_idempotent(): void - { - $schema = schema(int_schema('id'), str_schema('name'), datetime_schema('created_at')); - $hydrator = new PhpRowHydrator(); - $rows = $hydrator->hydrate([new RawRowValues([ - 'id' => 1, - 'name' => 'Alice', - 'created_at' => new DateTimeImmutable('2024-01-01 00:00:00 UTC'), - ])], $schema); - - static::assertSame( - serialize($rows), - serialize($hydrator->hydrate([new RawRowValues($rows->first()->values())], $schema)), - ); - } -} diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Row/RawRowValuesTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Row/RawRowValuesTest.php deleted file mode 100644 index 1b14ffeb16..0000000000 --- a/src/core/etl/tests/Flow/ETL/Tests/Unit/Row/RawRowValuesTest.php +++ /dev/null @@ -1,29 +0,0 @@ - 1]); - - static::assertSame(['id' => 1], $rowValues->values); - static::assertSame([], $rowValues->metadata); - } - - public function test_carries_per_value_metadata(): void - { - $metadata = Metadata::with('source', 'decoder'); - $rowValues = new RawRowValues(['id' => null], ['id' => $metadata]); - - static::assertSame(['id' => null], $rowValues->values); - static::assertSame(['id' => $metadata], $rowValues->metadata); - } -} diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Row/ResolvedReferenceTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Row/ResolvedReferenceTest.php index 515813e61e..fe470a71c9 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Unit/Row/ResolvedReferenceTest.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Row/ResolvedReferenceTest.php @@ -4,7 +4,9 @@ namespace Flow\ETL\Tests\Unit\Row; +use Flow\ETL\Row\NullsOrder; use Flow\ETL\Row\ResolvedReference; +use Flow\ETL\Row\SortOrder; use Flow\ETL\Tests\FlowTestCase; use function Flow\ETL\DSL\int_schema; @@ -36,4 +38,15 @@ public function test_as_returns_a_copy_carrying_the_type(): void static::assertSame('b', $aliased->name()); static::assertSame('integer', $aliased->returns()->toString()); } + + public function test_it_carries_the_sort_and_nulls_order(): void + { + $resolved = ref('a')->desc(NullsOrder::FIRST)->resolve(int_schema('a')); + + static::assertSame(SortOrder::DESC, $resolved->sort()); + static::assertSame(NullsOrder::FIRST, $resolved->nulls()); + static::assertSame(NullsOrder::FIRST, $resolved->as('b')->nulls()); + static::assertSame(NullsOrder::LAST, $resolved->asc(NullsOrder::LAST)->nulls()); + static::assertSame(NullsOrder::LAST, $resolved->desc()->nulls()); + } } diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Row/RowRenamingTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Row/RowRenamingTest.php deleted file mode 100644 index ced421efc1..0000000000 --- a/src/core/etl/tests/Flow/ETL/Tests/Unit/Row/RowRenamingTest.php +++ /dev/null @@ -1,33 +0,0 @@ - 1]); - - static::assertSame($row, RowRenaming::of([])->apply($row)); - } - - public function test_it_keeps_positions_and_leaves_unlisted_columns_alone(): void - { - static::assertSame(['x', 'b'], RowRenaming::of(['a' => 'x'])->apply(row(['a' => 1, 'b' => 2]))->names()); - } - - public function test_it_maps_values_onto_their_new_names(): void - { - static::assertEquals( - row(['x' => 1, 'y' => 2]), - RowRenaming::of(['a' => 'x', 'b' => 'y'])->apply(row(['a' => 1, 'b' => 2])), - ); - } -} diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Row/RowsBufferTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Row/RowsBufferTest.php deleted file mode 100644 index 39d2329be1..0000000000 --- a/src/core/etl/tests/Flow/ETL/Tests/Unit/Row/RowsBufferTest.php +++ /dev/null @@ -1,80 +0,0 @@ - 'x']], $buffer->add(row(['id' => 'x']))?->toArray()); - } - - public function test_the_default_batch_checks_every_row(): void - { - $this->expectException(SchemaMismatchException::class); - - (new RowsBuffer(schema(int_schema('id')), 1))->add(row(['id' => 'x'])); - } - - public function test_add_returns_a_full_batch_and_resets(): void - { - $buffer = new RowsBuffer(schema(int_schema('id')), 2); - - static::assertNull($buffer->add(row(['id' => 1]))); - - $batch = $buffer->add(row(['id' => 2])); - - static::assertNotNull($batch); - static::assertSame([1, 2], $batch->reduceToArray('id')); - static::assertNull($buffer->add(row(['id' => 3]))); - } - - public function test_add_returns_null_until_size_is_reached(): void - { - $buffer = new RowsBuffer(schema(int_schema('id')), 3); - - static::assertNull($buffer->add(row(['id' => 1]))); - static::assertNull($buffer->add(row(['id' => 2]))); - } - - public function test_flush_returns_null_when_empty(): void - { - static::assertNull((new RowsBuffer(schema(int_schema('id')), 2))->flush()); - } - - public function test_flush_returns_the_remainder(): void - { - $buffer = new RowsBuffer(schema(int_schema('id')), 3); - $buffer->add(row(['id' => 1])); - - $batch = $buffer->flush(); - - static::assertNotNull($batch); - static::assertSame([1], $batch->reduceToArray('id')); - static::assertNull($buffer->flush()); - } - - public function test_throws_when_size_below_one(): void - { - $this->expectException(InvalidArgumentException::class); - $this->expectExceptionMessage('Buffer size must be greater than 0, given: 0'); - - // @mago-ignore analysis:invalid-argument - new RowsBuffer(schema(int_schema('id')), 0); - } -} diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Row/TypedRowValuesTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Row/TypedRowValuesTest.php deleted file mode 100644 index 9ae895ab45..0000000000 --- a/src/core/etl/tests/Flow/ETL/Tests/Unit/Row/TypedRowValuesTest.php +++ /dev/null @@ -1,33 +0,0 @@ - 1], ['id' => type_integer()]); - - static::assertSame(['id' => 1], $rowValues->values); - static::assertEquals(['id' => type_integer()], $rowValues->types); - static::assertSame([], $rowValues->metadata); - } - - public function test_carries_per_value_types_and_metadata(): void - { - $metadata = Metadata::with('source', 'decoder'); - $rowValues = new TypedRowValues(['id' => null], ['id' => type_integer()], ['id' => $metadata]); - - static::assertSame(['id' => null], $rowValues->values); - static::assertEquals(['id' => type_integer()], $rowValues->types); - static::assertSame(['id' => $metadata], $rowValues->metadata); - } -} diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Row/UnresolvedReferenceTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Row/UnresolvedReferenceTest.php index 602e86b337..6608c1490a 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Unit/Row/UnresolvedReferenceTest.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Row/UnresolvedReferenceTest.php @@ -4,12 +4,15 @@ namespace Flow\ETL\Tests\Unit\Row; +use Flow\ETL\Row\NullsOrder; use Flow\ETL\Row\SortOrder; +use Flow\ETL\Tests\Context\FunctionContext; use Flow\ETL\Tests\FlowTestCase; use function Flow\ETL\DSL\flow_context; +use function Flow\ETL\DSL\int_schema; use function Flow\ETL\DSL\ref; -use function Flow\ETL\DSL\row; +use function Flow\ETL\DSL\schema; final class UnresolvedReferenceTest extends FlowTestCase { @@ -40,31 +43,45 @@ public function test_executing_equals_expression(): void { $ref = ref('a')->equals(ref('b')); - static::assertTrue($ref->eval(row(['a' => 1, 'b' => 1]), flow_context())); + static::assertTrue((new FunctionContext(flow_context()))->eval( + $ref, + ['a' => 1, 'b' => 1], + schema(int_schema('a'), int_schema('b')), + )); } public function test_executing_expression(): void { $ref = ref('b')->literal(100); - static::assertSame(100, $ref->eval(row(['a' => 1]), flow_context())); + static::assertSame(100, (new FunctionContext(flow_context()))->eval($ref, ['a' => 1], schema(int_schema('a')))); } public function test_is_even(): void { $ref = ref('a')->isEven(); - static::assertFalse($ref->eval(row(['a' => 1]), flow_context())); + static::assertFalse((new FunctionContext(flow_context()))->eval($ref, ['a' => 1], schema(int_schema('a')))); - static::assertTrue($ref->eval(row(['a' => 2]), flow_context())); + static::assertTrue((new FunctionContext(flow_context()))->eval($ref, ['a' => 2], schema(int_schema('a')))); } public function test_is_odd(): void { $ref = ref('a')->isOdd(); - static::assertTrue($ref->eval(row(['a' => 1]), flow_context())); + static::assertTrue((new FunctionContext(flow_context()))->eval($ref, ['a' => 1], schema(int_schema('a')))); - static::assertFalse($ref->eval(row(['a' => 2]), flow_context())); + static::assertFalse((new FunctionContext(flow_context()))->eval($ref, ['a' => 2], schema(int_schema('a')))); + } + + public function test_nulls_default_to_the_smallest_value_and_can_be_moved(): void + { + static::assertSame(NullsOrder::FIRST, ref('a')->nulls()); + static::assertSame(NullsOrder::FIRST, ref('a')->asc()->nulls()); + static::assertSame(NullsOrder::LAST, ref('a')->desc()->nulls()); + static::assertSame(NullsOrder::LAST, ref('a')->asc(NullsOrder::LAST)->nulls()); + static::assertSame(NullsOrder::FIRST, ref('a')->desc(NullsOrder::FIRST)->nulls()); + static::assertSame(NullsOrder::LAST, ref('a')->asc(NullsOrder::LAST)->as('b')->nulls()); } } diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/RowTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/RowTest.php deleted file mode 100644 index ab1f58053b..0000000000 --- a/src/core/etl/tests/Flow/ETL/Tests/Unit/RowTest.php +++ /dev/null @@ -1,374 +0,0 @@ - 2, 'a' => 1], - row(['a' => 1, 'b' => 2])->conformTo(schema(int_schema('b'), int_schema('a')))->values(), - ); - } - - public function test_conform_to_a_schema_the_row_already_matches_returns_an_equal_row(): void - { - $row = row(['a' => 1, 'b' => 'x']); - - static::assertEquals($row, $row->conformTo(schema(int_schema('a'), str_schema('b')))); - } - - public function test_conform_to_a_schema_with_as_many_but_other_columns_refuses_the_row(): void - { - $this->expectException(ColumnMismatchException::class); - $this->expectExceptionMessage( - 'Row does not match its schema: column "b" declared by the schema is missing from the row', - ); - - row(['a' => 1, 'z' => 2])->conformTo(schema(int_schema('a'), int_schema('b'))); - } - - public function test_conform_to_does_not_validate_a_non_null_value(): void - { - // the caller produced the value by casting to the column's type - conformTo() checks the shape only - static::assertSame( - ['a' => 'not-an-integer'], - row(['a' => 'not-an-integer'])->conformTo(schema(int_schema('a')))->values(), - ); - } - - public function test_conform_to_carries_the_schema_order_and_pads_a_nullable_absence(): void - { - static::assertSame( - ['c' => 3, 'a' => 1, 'b' => null], - row(['a' => 1, 'c' => 3])->conformTo(schema( - int_schema('c'), - int_schema('a'), - str_schema('b', true), - ))->values(), - ); - } - - public function test_conform_to_refuses_a_null_under_not_null(): void - { - $this->expectException(ColumnMismatchException::class); - $this->expectExceptionMessage('column "a"'); - - row(['a' => null])->conformTo(schema(int_schema('a'))); - } - - public function test_conform_to_refuses_a_missing_not_null_column(): void - { - $this->expectException(ColumnMismatchException::class); - $this->expectExceptionMessage('column "b"'); - - row(['a' => 1])->conformTo(schema(int_schema('a'), int_schema('b'))); - } - - public function test_conform_to_refuses_an_unknown_column(): void - { - $this->expectException(ColumnMismatchException::class); - $this->expectExceptionMessage('column "z"'); - - row(['a' => 1, 'z' => 2])->conformTo(schema(int_schema('a'))); - } - - public function test_match_to_accepts_null_in_a_nullable_column(): void - { - static::assertSame(['a' => null], row(['a' => null])->matchTo(schema(str_schema('a', true)))->values()); - } - - public function test_match_to_carries_the_schema_order_into_the_row(): void - { - static::assertSame( - ['c', 'a', 'b'], - row(['a' => 1, 'b' => 2, 'c' => 3])->matchTo(schema( - int_schema('c'), - int_schema('a'), - int_schema('b'), - ))->names(), - ); - } - - public function test_match_to_leaves_a_matching_row_unchanged(): void - { - static::assertSame(['a' => 1], row(['a' => 1])->matchTo(schema(int_schema('a')))->values()); - } - - public function test_match_to_pads_a_declared_nullable_column_the_row_omits(): void - { - static::assertSame( - ['a' => 1, 'b' => null], - row(['a' => 1])->matchTo(schema(int_schema('a'), str_schema('b', true)))->values(), - ); - } - - public function test_match_to_rejects_a_column_the_schema_does_not_declare(): void - { - $this->expectException(ColumnMismatchException::class); - $this->expectExceptionMessage('Row does not match its schema: column "b" is not declared by the schema'); - - row(['a' => 1, 'b' => 2])->matchTo(schema(int_schema('a'))); - } - - public function test_match_to_rejects_a_row_against_an_empty_schema(): void - { - $this->expectException(ColumnMismatchException::class); - $this->expectExceptionMessage('Row does not match its schema: column "a" is not declared by the schema'); - - row(['a' => 1])->matchTo(schema()); - } - - public function test_match_to_rejects_a_value_of_the_wrong_type(): void - { - $this->expectException(ColumnMismatchException::class); - $this->expectExceptionMessage( - 'Row does not match its schema: column "code": could not convert 1000 (integer) to string', - ); - - row(['code' => 1000])->matchTo(schema(str_schema('code'))); - } - - public function test_match_to_rejects_null_in_a_column_that_is_not_nullable(): void - { - $this->expectException(ColumnMismatchException::class); - $this->expectExceptionMessage('column "a": could not convert null to string, column is not nullable'); - - row(['a' => null])->matchTo(schema(str_schema('a'))); - } - - public function test_match_to_rejects_when_a_declared_column_that_is_not_nullable_is_missing(): void - { - $this->expectException(ColumnMismatchException::class); - $this->expectExceptionMessage( - 'Row does not match its schema: column "b" declared by the schema is missing from the row', - ); - - row(['a' => 1])->matchTo(schema(int_schema('a'), int_schema('b'))); - } - - public function test_match_to_reports_no_row_coordinate_of_its_own(): void - { - // the row does not know its position - Rows attaches it, see RowsTest - $this->expectException(ColumnMismatchException::class); - $this->expectExceptionMessage('Row does not match its schema: column "a" is not declared by the schema'); - - row(['a' => 1])->matchTo(schema()); - } - - public function test_match_to_with_the_same_keys_still_validates_every_value(): void - { - $this->expectException(ColumnMismatchException::class); - $this->expectExceptionMessage( - 'Row does not match its schema: column "c": could not convert 3 (integer) to string', - ); - - row(['a' => 1, 'b' => 'x', 'c' => 3])->matchTo(schema(int_schema('a'), str_schema('b'), str_schema('c'))); - } - - public function test_get_throws_when_the_column_is_absent(): void - { - $this->expectException(InvalidArgumentException::class); - $this->expectExceptionMessage('Column "missing" does not exist'); - - row(['id' => 1])->get('missing'); - } - - public function test_has(): void - { - $row = row(['id' => 1, 'name' => 'one']); - - static::assertTrue($row->has('id')); - static::assertTrue($row->has('id', 'name')); - static::assertFalse($row->has('missing')); - static::assertFalse($row->has('id', 'missing')); - } - - /** - * The hash is a dedup key, so it stays order-independent: the Schema is sorted alphabetically - * before the values are folded, exactly as Entries::sort() did. - */ - public function test_hash_ignores_column_order(): void - { - $schema = schema(int_schema('id'), bool_schema('bool'), str_schema('string')); - - static::assertSame( - row(['id' => 1, 'string' => 'string', 'bool' => false])->hash($schema), - row(['bool' => false, 'id' => 1, 'string' => 'string'])->hash($schema), - ); - } - - public function test_hash_of_an_empty_row(): void - { - static::assertSame(row([])->hash(schema()), row([])->hash(schema())); - } - - public function test_hash_of_different_rows(): void - { - $schema = schema(list_schema('list', type_list(type_integer()))); - - static::assertNotSame(row(['list' => [1, 2, 3]])->hash($schema), row(['list' => [3, 2, 1]])->hash($schema)); - } - - public function test_names_and_values_keep_storage_order(): void - { - $row = row(['b' => 2, 'a' => 1]); - - static::assertSame(['b', 'a'], $row->names()); - static::assertSame(['b' => 2, 'a' => 1], $row->values()); - } - - public function test_project_carries_the_schema_order_into_the_row(): void - { - static::assertSame( - ['c', 'a'], - row(['a' => 1, 'c' => 3])->project(schema(int_schema('c'), int_schema('a')))->names(), - ); - } - - public function test_project_drops_the_columns_the_schema_does_not_declare(): void - { - static::assertSame(['a' => 1], row(['a' => 1, 'b' => 2])->project(schema(int_schema('a')))->values()); - } - - public function test_project_leaves_a_column_the_row_omits_absent_instead_of_padding_it(): void - { - static::assertSame( - ['a' => 1], - row(['a' => 1])->project(schema(int_schema('a'), str_schema('b', true)))->values(), - ); - } - - public function test_project_validates_nothing(): void - { - static::assertSame(['code' => 1000], row(['code' => 1000])->project(schema(str_schema('code')))->values()); - } - - public function test_transforms_row_to_array(): void - { - $row = row([ - 'id' => 1234, - 'deleted' => false, - 'created-at' => $createdAt = new DateTimeImmutable('2020-07-13 15:00'), - 'phase' => null, - 'items' => ['item-id' => 1, 'name' => 'one'], - 'statuses' => ['NEW', 'PENDING'], - ]); - - static::assertEquals( - [ - 'id' => 1234, - 'deleted' => false, - 'created-at' => $createdAt, - 'phase' => null, - 'items' => ['item-id' => 1, 'name' => 'one'], - 'statuses' => ['NEW', 'PENDING'], - ], - $row->toArray(), - ); - } - - public function test_transforms_row_to_array_without_keys(): void - { - static::assertSame([1, 'one'], row(['id' => 1, 'name' => 'one'])->toArray(withKeys: false)); - } - - /** - * The Schema names, types and orders the columns; the Row is name-keyed storage. - */ - public function test_values_are_read_by_the_schema(): void - { - $schema = schema( - int_schema('id'), - str_schema('name'), - datetime_schema('created-at'), - structure_schema('items', type_structure(['a' => type_integer()])), - map_schema('statuses', type_map(type_integer(), type_string())), - ); - $createdAt = new DateTimeImmutable('2020-07-13 15:00'); - $row = row([ - 'id' => 1, - 'name' => 'one', - 'created-at' => $createdAt, - 'items' => ['a' => 1], - 'statuses' => ['NEW'], - ]); - - $values = []; - - foreach ($schema->references()->names() as $name) { - $values[$name] = $row->get($name); - } - - static::assertSame( - [ - 'id' => 1, - 'name' => 'one', - 'created-at' => $createdAt, - 'items' => ['a' => 1], - 'statuses' => ['NEW'], - ], - $values, - ); - } - - public function test_match_to_hops_a_reordered_row_into_the_column_zone(): void - { - static::assertSame( - '2026-01-02 04:04:05 Europe/Warsaw', - type_datetime() - ->assert( - row(['at' => new DateTimeImmutable('2026-01-02 03:04:05', new DateTimeZone('UTC')), 'id' => 1]) - ->matchTo(schema(int_schema('id'), datetime_schema('at', zone: 'Europe/Warsaw'))) - ->get('at'), - ) - ->format('Y-m-d H:i:s e'), - ); - } - - public function test_match_to_refuses_a_reordered_row_with_a_wrong_typed_value(): void - { - $this->expectException(ColumnMismatchException::class); - - row(['b' => 'x', 'a' => 1])->matchTo(schema(int_schema('a'), int_schema('b'))); - } - - public function test_match_to_refuses_null_in_a_non_nullable_datetime_column(): void - { - $this->expectException(ColumnMismatchException::class); - - row(['at' => null])->matchTo(schema(datetime_schema('at'))); - } - - public function test_match_to_refuses_a_non_datetime_in_a_datetime_column(): void - { - $this->expectException(ColumnMismatchException::class); - - row(['at' => '2026-01-02 03:04:05'])->matchTo(schema(datetime_schema('at'))); - } -} diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Rows/ArrayToRowsTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Rows/ArrayToRowsTest.php index fcba4848c6..4a493bd823 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Unit/Rows/ArrayToRowsTest.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Rows/ArrayToRowsTest.php @@ -4,6 +4,7 @@ namespace Flow\ETL\Tests\Unit\Rows; +use Flow\ETL\Exception\SchemaMismatchException; use Flow\ETL\Tests\FlowTestCase; use function Flow\ETL\DSL\array_to_rows; @@ -12,8 +13,6 @@ use function Flow\ETL\DSL\flow_context; use function Flow\ETL\DSL\int_schema; use function Flow\ETL\DSL\list_schema; -use function Flow\ETL\DSL\row; -use function Flow\ETL\DSL\rows; use function Flow\ETL\DSL\schema; use function Flow\ETL\DSL\str_schema; use function Flow\Types\DSL\type_list; @@ -29,15 +28,14 @@ public function test_building_array_to_rows_with_entry_that_is_list_of_strings() ['data' => ['e', 'f', 'g', 'd']], ], schema(list_schema('data', type_list(type_string()))), - flow_context(config())->hydrator(), + flow_context(config())->backend(), ); static::assertEquals( - rows( - schema(list_schema('data', type_list(type_string()))), - row(['data' => ['a', 'b', 'c', 'd']]), - row(['data' => ['e', 'f', 'g', 'd']]), - ), + array_to_rows([ + ['data' => ['a', 'b', 'c', 'd']], + ['data' => ['e', 'f', 'g', 'd']], + ], schema(list_schema('data', type_list(type_string())))), $rows, ); } @@ -49,26 +47,26 @@ public function test_building_array_to_rows_with_entry_that_is_list_of_strings_w ['data' => ['e', 'f', 'g', 'd']], ], schema(list_schema('data', type_list(type_string()))), - flow_context(config())->hydrator(), + flow_context(config())->backend(), ); static::assertEquals( - rows(schema(list_schema('data', type_list(type_string()))), row(['data' => ['e', 'f', 'g', 'd']])), + array_to_rows([['data' => ['e', 'f', 'g', 'd']]], schema(list_schema('data', type_list(type_string())))), $rows, ); } - public function test_building_row_from_array_with_schema_and_additional_fields_not_covered_by_schema(): void + public function test_refuses_an_undeclared_key_of_a_single_flat_row(): void { - $rows = array_to_rows( - ['id' => 1234, 'deleted' => false, 'phase' => null], - schema(int_schema('id'), bool_schema('deleted')), - flow_context(config())->hydrator(), + $this->expectException(SchemaMismatchException::class); + $this->expectExceptionMessage( + 'Rows do not match their schema: column "phase" (row 0) is not declared by the schema', ); - static::assertEquals( - rows(schema(int_schema('id'), bool_schema('deleted')), row(['id' => 1234, 'deleted' => false])), - $rows, + array_to_rows( + ['id' => 1234, 'deleted' => false, 'phase' => null], + schema(int_schema('id'), bool_schema('deleted')), + flow_context(config())->backend(), ); } @@ -77,13 +75,13 @@ public function test_building_row_from_array_with_schema_but_entries_not_availab $rows = array_to_rows( ['id' => 1234, 'deleted' => false], schema(int_schema('id'), bool_schema('deleted'), str_schema('phase', true)), - flow_context(config())->hydrator(), + flow_context(config())->backend(), ); static::assertEquals( - rows( + array_to_rows( + [['id' => 1234, 'deleted' => false, 'phase' => null]], schema(int_schema('id'), bool_schema('deleted'), str_schema('phase', nullable: true)), - row(['id' => 1234, 'deleted' => false, 'phase' => null]), ), $rows, ); @@ -97,37 +95,35 @@ public function test_building_rows_from_array(): void ['id' => 4321, 'deleted' => true, 'phase' => 'launch'], ], schema(int_schema('id'), bool_schema('deleted'), str_schema('phase', nullable: true)), - flow_context(config())->hydrator(), + flow_context(config())->backend(), ); static::assertEquals( - rows( + array_to_rows( + [ + ['id' => 1234, 'deleted' => false, 'phase' => null], + ['id' => 4321, 'deleted' => true, 'phase' => 'launch'], + ], schema(int_schema('id'), bool_schema('deleted'), str_schema('phase', nullable: true)), - row(['id' => 1234, 'deleted' => false, 'phase' => null]), - row(['id' => 4321, 'deleted' => true, 'phase' => 'launch']), ), $rows, ); } - public function test_building_rows_from_array_with_schema_and_additional_fields_not_covered_by_schema(): void + public function test_refuses_an_undeclared_key(): void { - $rows = array_to_rows( - [ - ['id' => 1234, 'deleted' => false, 'phase' => null], - ['id' => 4321, 'deleted' => true, 'phase' => 'launch'], - ], - schema(int_schema('id'), bool_schema('deleted')), - flow_context(config())->hydrator(), + $this->expectException(SchemaMismatchException::class); + $this->expectExceptionMessage( + 'Rows do not match their schema: column "nmae" (row 1) is not declared by the schema', ); - static::assertEquals( - rows( - schema(int_schema('id'), bool_schema('deleted')), - row(['id' => 1234, 'deleted' => false]), - row(['id' => 4321, 'deleted' => true]), - ), - $rows, + array_to_rows( + [ + ['id' => 1234, 'name' => 'launch'], + ['id' => 4321, 'nmae' => 'landing'], + ], + schema(int_schema('id'), str_schema('name', nullable: true)), + flow_context(config())->backend(), ); } @@ -139,16 +135,40 @@ public function test_building_rows_from_array_with_schema_but_entries_not_availa ['id' => 4321, 'deleted' => true], ], schema(int_schema('id'), bool_schema('deleted'), str_schema('phase', true)), - flow_context(config())->hydrator(), + flow_context(config())->backend(), ); static::assertEquals( - rows( + array_to_rows( + [ + ['id' => 1234, 'deleted' => false, 'phase' => null], + ['id' => 4321, 'deleted' => true, 'phase' => null], + ], schema(int_schema('id'), bool_schema('deleted'), str_schema('phase', nullable: true)), - row(['id' => 1234, 'deleted' => false, 'phase' => null]), - row(['id' => 4321, 'deleted' => true, 'phase' => null]), ), $rows, ); } + + public function test_a_numeric_key_is_the_declared_column_of_that_name(): void + { + static::assertSame( + [['0' => 'x', 'e07' => 'y']], + array_to_rows( + [[0 => 'x', 7 => 'y']], + schema(str_schema('0'), str_schema('e07')), + flow_context(config())->backend(), + )->toArray(), + ); + } + + public function test_refuses_an_undeclared_numeric_key_under_its_positional_name(): void + { + $this->expectException(SchemaMismatchException::class); + $this->expectExceptionMessage( + 'Rows do not match their schema: column "e07" (row 0) is not declared by the schema', + ); + + array_to_rows([[0 => 'x', 7 => 'y']], schema(str_schema('0')), flow_context(config())->backend()); + } } diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Rows/ColumnRetypingTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Rows/ColumnRetypingTest.php new file mode 100644 index 0000000000..33ae9a8ea9 --- /dev/null +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Rows/ColumnRetypingTest.php @@ -0,0 +1,140 @@ +absent(int_schema('id', nullable: true), 2, new PhpBackend()) + ->values(), + ); + } + + public function test_an_absent_not_null_column_is_refused_when_there_is_a_row(): void + { + $this->expectException(SchemaMismatchException::class); + + (new ColumnRetyping())->absent(int_schema('id'), 1, new PhpBackend()); + } + + public function test_an_absent_not_null_column_of_no_rows_is_empty(): void + { + static::assertSame( + 0, + (new ColumnRetyping()) + ->absent(int_schema('id'), 0, new PhpBackend()) + ->count(), + ); + } + + public function test_the_same_definition_keeps_the_column(): void + { + $column = ColumnMother::of(int_schema('id'), [1, 2]); + + static::assertSame($column, (new ColumnRetyping())->restamped($column, int_schema('id'), int_schema('id'))); + } + + public function test_a_change_the_types_cannot_prove_is_not_restamped(): void + { + static::assertNull((new ColumnRetyping())->restamped( + ColumnMother::of(str_schema('id'), ['1']), + str_schema('id'), + int_schema('id'), + )); + } + + public function test_a_proved_change_is_restamped(): void + { + $retyped = (new ColumnRetyping())->restamped( + ColumnMother::of(int_schema('id'), [1, 2]), + int_schema('id'), + int_schema('id', nullable: true), + ); + + static::assertNotNull($retyped); + static::assertSame([1, 2], $retyped->values()); + } + + public function test_not_null_refuses_the_first_null_at_its_position(): void + { + try { + (new ColumnRetyping())->notNull( + int_schema('id'), + ColumnMother::of( + int_schema('id', nullable: true), + [ + 1, + null, + null, + ], + ), + ); + static::fail('a null under NOT NULL must be refused'); + } catch (SchemaMismatchException $e) { + static::assertSame(1, $e->rowIndex); + } + } + + public function test_cast_casts_what_the_types_cannot_prove(): void + { + static::assertSame( + [1, 2], + (new ColumnRetyping()) + ->cast( + ColumnMother::of(str_schema('id'), ['1', '2']), + str_schema('id'), + int_schema('id'), + new PhpBackend(), + ) + ->values(), + ); + } + + public function test_validate_refuses_the_lowest_bad_row_across_columns(): void + { + try { + (new ColumnRetyping())->validate([ + 'a' => [ + ColumnMother::of(list_schema('a', type_list(type_optional(type_integer()))), [[1], [2], [null]]), + list_schema('a', type_list(type_integer())), + ], + 'b' => [ + ColumnMother::of(list_schema('b', type_list(type_optional(type_integer()))), [[1], [null], [3]]), + list_schema('b', type_list(type_integer())), + ], + ], 3); + static::fail('a value the definition does not take must be refused'); + } catch (SchemaMismatchException $e) { + static::assertSame(1, $e->rowIndex); + static::assertStringContainsString('"b"', $e->getMessage()); + } + } + + public function test_a_proved_change_with_a_null_under_not_null_is_left_to_validate(): void + { + static::assertNull((new ColumnRetyping())->restamped( + ColumnMother::of(int_schema('id', nullable: true), [1, null]), + int_schema('id', nullable: true), + int_schema('id'), + )); + } +} diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Rows/RowHashesTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Rows/RowHashesTest.php new file mode 100644 index 0000000000..1452d743e5 --- /dev/null +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Rows/RowHashesTest.php @@ -0,0 +1,151 @@ + type_integer()])), + ); + $rows = array_to_rows([ + [ + 's' => 'a', + 'i' => 1, + 'f' => 1.5, + 'b' => true, + 'dt' => new DateTimeImmutable('2024-01-01 10:00:00.5'), + 'd' => new DateTimeImmutable('2024-01-01'), + 't' => 'PT1H', + 'u' => '00000000-0000-4000-8000-000000000001', + 'j' => '{"a":1}', + 'e' => BackedStringEnum::one, + 'x' => '
1', + 'tz' => 'UTC', + 'l' => [1, 2], + 'm' => ['k' => 1], + 'st' => ['a' => 1], + ], + [ + 's' => null, + 'i' => 2, + 'f' => 0.0, + 'b' => false, + 'dt' => new DateTimeImmutable('1969-12-31 23:59:59'), + 'd' => new DateTimeImmutable('1969-12-31'), + 't' => 'PT0S', + 'u' => '00000000-0000-4000-8000-000000000002', + 'j' => '[]', + 'e' => BackedStringEnum::two, + 'x' => '', + 'tz' => 'Europe/Warsaw', + 'l' => [], + 'm' => [], + 'st' => ['a' => 2], + ], + ], $schema); + + $hashes = (new RowHashes())->of($rows); + + static::assertSame( + [(new RowHashes())->of($rows->slice(0, 1))[0], (new RowHashes())->of($rows->slice(1, 1))[0]], + $hashes, + ); + static::assertNotSame($hashes[0], $hashes[1]); + } + + public function test_a_hash_ignores_column_order(): void + { + static::assertSame( + (new RowHashes())->of(array_to_rows( + [['id' => 1, 'string' => 'string', 'bool' => false]], + schema(int_schema('id'), str_schema('string'), bool_schema('bool')), + )), + (new RowHashes())->of(array_to_rows( + [['bool' => false, 'id' => 1, 'string' => 'string']], + schema(bool_schema('bool'), int_schema('id'), str_schema('string')), + )), + ); + } + + public function test_rows_differing_in_element_order_hash_differently(): void + { + $hashes = (new RowHashes())->of(array_to_rows([ + ['list' => [1, 2, 3]], + ['list' => [3, 2, 1]], + ], schema(list_schema('list', type_list(type_integer()))))); + + static::assertNotSame($hashes[0], $hashes[1]); + } + + public function test_a_row_of_a_batch_without_columns_hashes_the_empty_string(): void + { + static::assertSame( + [(new NativePHPHash())->hash('')], + (new RowHashes())->of(Rows::fromColumns(schema(), [], 1)), + ); + } + + public function test_the_algorithm_hashes_each_row(): void + { + $rows = array_to_rows([['id' => 1]], schema(int_schema('id'))); + + static::assertSame( + [(new NativePHPHash('md5'))->hash('id' . NativeHasher::normalize(1))], + (new RowHashes())->of($rows, new NativePHPHash('md5')), + ); + } + + public function test_an_empty_batch_has_no_hashes(): void + { + static::assertSame([], (new RowHashes())->of(array_to_rows([], schema(int_schema('id'))))); + } +} diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Rows/RowsBuilderTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Rows/RowsBuilderTest.php new file mode 100644 index 0000000000..e97eabe8e9 --- /dev/null +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Rows/RowsBuilderTest.php @@ -0,0 +1,843 @@ + + */ + public static function refused_views(): Generator + { + yield 'a not null column the source lacks' => [ + array_to_rows([['id' => 1]], schema(int_schema('id'))), + schema(int_schema('id'), str_schema('name')), + 'column "name" (row 0) declared by the schema is missing from the row', + ]; + yield 'null under not null' => [ + array_to_rows([['id' => null]], schema(int_schema('id', nullable: true))), + schema(int_schema('id')), + 'column "id" (row 0): could not convert null to integer, column is not nullable', + ]; + yield 'a value that matches only after a cast' => [ + array_to_rows([['id' => '2']], schema(str_schema('id'))), + schema(int_schema('id')), + 'column "id" (row 0): could not convert \'2\' (string) to integer', + ]; + yield 'a column the target does not declare' => [ + array_to_rows([['id' => 1, 'extra' => 2]], schema(int_schema('id'), int_schema('extra'))), + schema(int_schema('id')), + 'column "extra" (row 0) is not declared by the schema', + ]; + } + + /** + * @return \Generator>, class-string<\Throwable>, string}> + */ + public static function refused_rows(): Generator + { + yield 'null in a not null column' => [ + schema(int_schema('id'), str_schema('name')), + [['id' => 1, 'name' => null]], + SchemaMismatchException::class, + 'Rows do not match their schema: column "name" (row 0): could not convert null to string, column is not nullable', + ]; + + yield 'non numeric string in an integer column' => [ + schema(int_schema('id')), + [['id' => 'abc']], + SchemaMismatchException::class, + 'Rows do not match their schema: column "id" (row 0): could not convert \'abc\' (string) to integer', + ]; + + yield 'trailing garbage after digits in an integer column' => [ + schema(int_schema('id')), + [['id' => '12abc']], + SchemaMismatchException::class, + 'Rows do not match their schema: column "id" (row 0): could not convert \'12abc\' (string) to integer', + ]; + + yield 'hex string in a float column' => [ + schema(float_schema('price')), + [['price' => '0x1A']], + SchemaMismatchException::class, + 'Rows do not match their schema: column "price" (row 0): could not convert \'0x1A\' (string) to float', + ]; + + yield 'unrecognised word in a boolean column' => [ + schema(bool_schema('active')), + [['active' => 'weird']], + SchemaMismatchException::class, + 'Rows do not match their schema: column "active" (row 0): could not convert \'weird\' (string) to boolean', + ]; + + yield 'array in an integer column' => [ + schema(int_schema('id')), + [['id' => [1, 2, 3]]], + SchemaMismatchException::class, + 'Rows do not match their schema: column "id" (row 0): could not convert array (list) to integer', + ]; + + yield 'invalid uuid string' => [ + schema(uuid_schema('u')), + [['u' => 'not-a-uuid']], + SchemaMismatchException::class, + 'Rows do not match their schema: column "u" (row 0): could not convert \'not-a-uuid\' (string) to uuid', + ]; + + yield 'uppercase uuid string' => [ + schema(uuid_schema('u')), + [['u' => '01234567-89AB-4DEF-8123-456789ABCDEF']], + SchemaMismatchException::class, + 'Rows do not match their schema: column "u" (row 0): could not convert \'01234567-89AB-4DEF-8123-456789AB...\' (string) to uuid', + ]; + + yield 'json from a scalar' => [ + schema(json_schema('j')), + [['j' => 5]], + SchemaMismatchException::class, + 'Rows do not match their schema: column "j" (row 0): could not convert 5 (integer) to json', + ]; + + yield 'json from an invalid json string' => [ + schema(json_schema('j')), + [['j' => '{oops']], + SchemaMismatchException::class, + 'Rows do not match their schema: column "j" (row 0): could not convert \'{oops\' (string) to json', + ]; + + yield 'json from a plain string' => [ + schema(json_schema('j')), + [['j' => 'plain']], + SchemaMismatchException::class, + 'Rows do not match their schema: column "j" (row 0): could not convert \'plain\' (string) to json', + ]; + + yield 'datetime from garbage' => [ + schema(datetime_schema('at')), + [['at' => 'not-a-date']], + SchemaMismatchException::class, + 'Rows do not match their schema: column "at" (row 0): could not convert \'not-a-date\' (string) to datetime', + ]; + + yield 'datetime from an array' => [ + schema(datetime_schema('at')), + [['at' => ['nope']]], + SchemaMismatchException::class, + 'Rows do not match their schema: column "at" (row 0): could not convert array (list) to datetime', + ]; + + yield 'date from garbage' => [ + schema(date_schema('d')), + [['d' => 'not-a-date']], + SchemaMismatchException::class, + 'Rows do not match their schema: column "d" (row 0): could not convert \'not-a-date\' (string) to date', + ]; + + yield 'string map with integer keys' => [ + schema(map_schema('m', type_map(type_string(), type_integer()))), + [['m' => [5 => 1]]], + SchemaMismatchException::class, + 'Rows do not match their schema: column "m" (row 0): could not convert array (map) to map', + ]; + + yield 'list with non-sequential keys' => [ + schema(list_schema('l', type_list(type_integer()))), + [['l' => [1 => 'x']]], + SchemaMismatchException::class, + 'Rows do not match their schema: column "l" (row 0): could not convert array (map) to list', + ]; + + yield 'positive integer list from a non-numeric string' => [ + schema(list_schema('l', type_list(type_positive_integer()))), + [['l' => ['abc']]], + SchemaMismatchException::class, + 'Rows do not match their schema: column "l" (row 0): could not convert array (list) to list', + ]; + + yield 'positive integer list from a negative int' => [ + schema(list_schema('l', type_list(type_positive_integer()))), + [['l' => [-3]]], + SchemaMismatchException::class, + 'Rows do not match their schema: column "l" (row 0): could not convert array (list) to list', + ]; + + yield 'structure missing required element' => [ + schema(structure_schema('data', type_structure(['id' => type_integer(), 'name' => type_string()]))), + [['data' => ['id' => 1]]], + SchemaMismatchException::class, + 'Rows do not match their schema: column "data" (row 0): could not convert array (structure{id: integer}) to structure{id: integer, name: string}', + ]; + + yield 'structure present-null required element' => [ + schema(structure_schema('data', type_structure(['id' => type_integer(), 'name' => type_string()]))), + [['data' => ['id' => 1, 'name' => null]]], + SchemaMismatchException::class, + 'Rows do not match their schema: column "data" (row 0): could not convert array (structure{id: integer, name: null}) to structure{id: integer, name: string}', + ]; + + yield 'structure present-null optional element' => [ + schema(structure_schema('data', type_structure([ + 'id' => type_integer(), + 'name' => structure_element('name', type_string(), optional: true), + ]))), + [['data' => ['id' => 1, 'name' => null]]], + SchemaMismatchException::class, + 'Rows do not match their schema: column "data" (row 0): could not convert array (structure{id: integer, name: null}) to structure{id: integer, name?: string}', + ]; + + // the refusal is placed at the batch position, so a constant row index would diverge here + yield 'non numeric string in the second row of an integer column' => [ + schema(int_schema('id')), + [['id' => '1'], ['id' => 'x']], + SchemaMismatchException::class, + 'Rows do not match their schema: column "id" (row 1): could not convert \'x\' (string) to integer', + ]; + + yield 'a not null column the row does not carry' => [ + schema(int_schema('id'), str_schema('name')), + [['id' => 1]], + SchemaMismatchException::class, + 'Rows do not match their schema: column "name" (row 0) declared by the schema is missing from the row', + ]; + + yield 'integer overflow' => [ + schema(int_schema('id')), + [['id' => '9223372036854775808']], + SchemaMismatchException::class, + 'Rows do not match their schema: column "id" (row 0): could not convert \'9223372036854775808\' (string) to integer', + ]; + + yield 'date from an empty string' => [ + schema(date_schema('d')), + [['d' => '']], + SchemaMismatchException::class, + 'Rows do not match their schema: column "d" (row 0): could not convert \'\' (string) to date', + ]; + + yield 'datetime from a relative word' => [ + schema(datetime_schema('at')), + [['at' => 'now']], + SchemaMismatchException::class, + 'Rows do not match their schema: column "at" (row 0): could not convert \'now\' (string) to datetime', + ]; + + yield 'string from a null element inside a list' => [ + schema(list_schema('l', type_list(type_string()))), + [['l' => ['a', null]]], + SchemaMismatchException::class, + 'Rows do not match their schema: column "l" (row 0): could not convert array (list) to list', + ]; + + yield 'list from a scalar' => [ + schema(list_schema('l', type_list(type_integer()))), + [['l' => 5]], + SchemaMismatchException::class, + 'Rows do not match their schema: column "l" (row 0): could not convert 5 (integer) to list', + ]; + + // ordering: an absence must not pre-empt a refusal that PHP reports first, or the two + // hydrators name different columns - and different ROWS - for the same batch + yield 'an absent column before a cast refusal in the same row' => [ + schema(int_schema('a'), int_schema('b')), + [['b' => 'abc']], + SchemaMismatchException::class, + 'Rows do not match their schema: column "b" (row 0): could not convert \'abc\' (string) to integer', + ]; + + yield 'an absent column in row 0 before a cast refusal in row 1' => [ + schema(int_schema('a'), int_schema('b')), + [['a' => 1], ['a' => 1, 'b' => 'abc']], + SchemaMismatchException::class, + 'Rows do not match their schema: column "b" (row 1): could not convert \'abc\' (string) to integer', + ]; + + yield 'an absent column in row 0 before a present null in row 1' => [ + schema(int_schema('a'), int_schema('b')), + [['a' => 1], ['a' => 1, 'b' => null]], + SchemaMismatchException::class, + 'Rows do not match their schema: column "b" (row 1): could not convert null to integer, column is not nullable', + ]; + } + + public function test_a_refused_value_wins_over_an_earlier_absence(): void + { + $this->expectException(SchemaMismatchException::class); + $this->expectExceptionMessage('column "b" (row 1): could not convert null to integer, column is not nullable'); + + (new RowsBuilder(schema(int_schema('a'), int_schema('b')), new PhpBackend()))->appendRows([ + ['a' => 1], + ['a' => 1, 'b' => null], + ]); + } + + public function test_an_absence_is_placed_after_the_rows_already_appended(): void + { + $this->expectException(SchemaMismatchException::class); + $this->expectExceptionMessage('column "c" (row 2) declared by the schema is missing from the row'); + + (new RowsBuilder( + schema(int_schema('a'), int_schema('b', nullable: true), int_schema('c')), + new PhpBackend(), + ))->appendRows([['a' => 0, 'c' => 0]])->appendRows([['a' => 1, 'c' => 1], ['a' => 2]]); + } + + public function test_the_first_refused_value_in_row_order_across_columns(): void + { + $this->expectException(SchemaMismatchException::class); + $this->expectExceptionMessage('column "b" (row 1): could not convert \'x\' (string) to integer'); + + (new RowsBuilder(schema(int_schema('a'), int_schema('b')), new PhpBackend()))->appendRows([[ + 'a' => 0, + 'b' => 0, + ]])->appendRows([['a' => 1, 'b' => 'x'], ['a' => 'y', 'b' => 2]]); + } + + public function test_a_view_of_another_definition_keeps_a_value_that_matches_the_target(): void + { + static::assertSame( + [['id' => 5, 'name' => null]], + (new RowsBuilder(schema(int_schema('id'), str_schema('name', nullable: true)), new PhpBackend())) + ->appendFrom(array_to_rows([['id' => 5]], schema(int_schema('id', nullable: true))), 0) + ->finish() + ->toArray(), + ); + } + + public function test_a_refused_view_is_placed_at_its_output_row(): void + { + $this->expectException(SchemaMismatchException::class); + $this->expectExceptionMessage('column "id" (row 1): could not convert \'x\' (string) to integer'); + + (new RowsBuilder(schema(int_schema('id')), new PhpBackend())) + ->appendFrom(array_to_rows([['id' => 1]], schema(int_schema('id'))), 0) + ->appendFrom(array_to_rows([['id' => 'x']], schema(str_schema('id'))), 0); + } + + #[DataProvider('refused_views')] + public function test_refuses_a_view_that_does_not_fit_the_target( + Rows $source, + Schema $target, + string $message, + ): void { + $this->expectException(SchemaMismatchException::class); + $this->expectExceptionMessage($message); + + (new RowsBuilder($target, new PhpBackend()))->appendFrom($source, 0); + } + + public function test_a_missing_nullable_column_is_padded_with_null(): void + { + static::assertEquals( + array_to_rows([['id' => 1, 'name' => null]], schema(int_schema('id'), str_schema('name', nullable: true))), + (new RowsBuilder( + schema(int_schema('id'), str_schema('name', nullable: true)), + new PhpBackend(), + ))->appendRows([['id' => 1]])->finish(), + ); + } + + public function test_a_missing_not_null_column_is_refused_with_its_row(): void + { + $this->expectException(SchemaMismatchException::class); + $this->expectExceptionMessage( + 'Rows do not match their schema: column "name" (row 1) declared by the schema is missing from the row', + ); + + (new RowsBuilder(schema(int_schema('id'), str_schema('name')), new PhpBackend()))->appendRows([ + ['id' => 1, 'name' => 'a'], + ['id' => 2], + ]); + } + + public function test_a_refused_value_in_the_first_row_reports_row_zero(): void + { + $this->expectException(SchemaMismatchException::class); + $this->expectExceptionMessage( + 'Rows do not match their schema: column "i" (row 0): could not convert \'abc\' (string) to integer', + ); + + (new RowsBuilder(schema(int_schema('i')), new PhpBackend()))->appendRows([['i' => 'abc']]); + } + + public function test_a_refused_value_is_reported_with_its_column_and_row(): void + { + $this->expectException(SchemaMismatchException::class); + $this->expectExceptionMessage( + 'Rows do not match their schema: column "i" (row 1): could not convert \'n/a\' (string) to integer', + ); + + (new RowsBuilder(schema(int_schema('i')), new PhpBackend()))->appendRows([ + ['i' => 1], + ['i' => 'n/a'], + ['i' => 3], + ]); + } + + public function test_a_refusal_in_a_later_append_reports_its_row_in_the_whole_batch(): void + { + $builder = new RowsBuilder(schema(int_schema('i')), new PhpBackend()); + $builder->appendRows([['i' => 1], ['i' => 2]]); + + $this->expectException(SchemaMismatchException::class); + $this->expectExceptionMessage('column "i" (row 3): could not convert \'x\' (string) to integer'); + + $builder->appendRows([['i' => 3], ['i' => 'x']]); + } + + public function test_a_null_on_a_not_null_column_is_reported_as_a_value_that_does_not_match(): void + { + $this->expectException(SchemaMismatchException::class); + $this->expectExceptionMessage( + 'Rows do not match their schema: column "i" (row 0): could not convert null to integer, column is not nullable', + ); + + (new RowsBuilder(schema(int_schema('i')), new PhpBackend()))->append(['i' => null]); + } + + public function test_a_value_the_schema_does_not_declare_is_dropped(): void + { + static::assertSame( + ['id'], + array_keys( + (new RowsBuilder(schema(int_schema('id')), new PhpBackend())) + ->append(['id' => 1, 'undeclared' => 'x']) + ->finish() + ->values(0), + ), + ); + } + + public function test_rows_follow_the_schema_column_order(): void + { + static::assertSame( + [['id' => 1, 'name' => 'a']], + (new RowsBuilder(schema(int_schema('id'), str_schema('name')), new PhpBackend())) + ->append(['name' => 'a', 'id' => 1]) + ->finish() + ->toArray(), + ); + } + + public function test_every_value_is_cast_against_its_own_column(): void + { + static::assertEquals( + array_to_rows([['id' => 1, 'name' => '2']], schema(int_schema('id'), str_schema('name'))), + (new RowsBuilder(schema(int_schema('id'), str_schema('name')), new PhpBackend()))->append([ + 'id' => '1', + 'name' => 2, + ])->finish(), + ); + } + + public function test_a_schema_without_definitions_counts_rows(): void + { + $builder = new RowsBuilder(schema(), new PhpBackend()); + $builder->append([]); + $builder->append([]); + $builder->appendRows([[]]); + + static::assertSame(3, $builder->count()); + static::assertSame(3, $builder->finish()->count()); + } + + public function test_appending_no_rows_changes_nothing(): void + { + static::assertSame(0, (new RowsBuilder(schema(int_schema('id')), new PhpBackend()))->appendRows([])->count()); + } + + public function test_copies_physical_cells_from_another_batch(): void + { + $source = array_to_rows( + [ + ['id' => 1, 'name' => 'a'], + [ + 'id' => 2, + 'name' => 'b', + ], + ], + schema(int_schema('id'), str_schema('name')), + ); + + static::assertSame( + [['id' => 2, 'name' => 'b'], ['id' => 2, 'name' => 'b'], ['id' => 1, 'name' => 'a']], + (new RowsBuilder(schema(int_schema('id'), str_schema('name')), new PhpBackend())) + ->appendFrom($source, 1) + ->appendTake($source, [1, 0]) + ->finish() + ->toArray(), + ); + } + + public function test_feeds_one_column(): void + { + $builder = new RowsBuilder(schema(int_schema('id')), new PhpBackend()); + $builder->column('id')->appendMany([1, 2]); + + static::assertSame(2, $builder->count()); + static::assertSame([['id' => 1], ['id' => 2]], $builder->finish()->toArray()); + } + + public function test_an_unknown_column_is_refused(): void + { + $this->expectException(InvalidArgumentException::class); + $this->expectExceptionMessage('RowsBuilder has no column "nope"'); + + (new RowsBuilder(schema(int_schema('id')), new PhpBackend()))->column('nope'); + } + + public function test_ragged_columns_are_refused_at_finish(): void + { + $builder = new RowsBuilder(schema(int_schema('a'), int_schema('b')), new PhpBackend()); + $builder->column('a')->appendMany([1, 2]); + $builder->column('b')->append(1); + + $this->expectException(InvalidArgumentException::class); + $this->expectExceptionMessage('Ragged columns: 2 rows in [a], 1 rows in [b]'); + + $builder->finish(); + } + + /** + * @param list> $rows + * @param class-string<\Throwable> $exception + */ + #[DataProvider('refused_rows')] + public function test_append_rows_refuses(Schema $schema, array $rows, string $exception, string $message): void + { + $this->expectException($exception); + $this->expectExceptionMessage($message); + + (new RowsBuilder($schema, new PhpBackend())) + ->appendRows($rows) + ->finish(); + } + + public function test_append_rows_refuses_a_definition_outside_the_nineteen(): void + { + $this->expectException(ColumnMismatchException::class); + $this->expectExceptionMessage( + 'Row does not match its schema: column "a": throwing cannot be a batch column, only the 19 Flow definitions have a column kind', + ); + + (new RowsBuilder( + schema( + new ForeignTypeDefinition('a', new ThrowingType(new LogicException('stub type refuses everything'))), + ), + new PhpBackend(), + ))->appendRows([['a' => [1, 2]]]); + } + + public function test_append_rows_refuses_a_structure_missing_a_required_element(): void + { + $this->expectException(SchemaMismatchException::class); + $this->expectExceptionMessage('Rows do not match their schema: column "data" (row 0)'); + + (new RowsBuilder( + schema(structure_schema('data', type_structure(['id' => type_integer(), 'name' => type_string()]))), + new PhpBackend(), + ))->appendRows([['data' => ['id' => 1]]]); + } + + public function test_append_rows_casts_datetime_and_uuid_strings(): void + { + $rows = (new RowsBuilder( + schema(datetime_schema('created_at'), uuid_schema('uuid')), + new PhpBackend(), + ))->appendRows([[ + 'created_at' => '2024-01-01 12:00:00 UTC', + 'uuid' => 'f47ac10b-58cc-4372-a567-0e02b2c3d479', + ]])->finish(); + + // @mago-ignore analysis:mixed-assignment + $uuid = $rows->column('uuid')->value(0); + + static::assertInstanceOf(DateTimeImmutable::class, $rows->column('created_at')->value(0)); + static::assertInstanceOf(Uuid::class, $uuid); + static::assertSame('f47ac10b-58cc-4372-a567-0e02b2c3d479', $uuid->toString()); + } + + public function test_append_rows_casts_raw_scalar_strings_to_schema_types(): void + { + static::assertSame( + [['id' => 1, 'price' => 9.99, 'active' => true, 'name' => 'Alice']], + (new RowsBuilder( + schema(int_schema('id'), float_schema('price'), bool_schema('active'), str_schema('name')), + new PhpBackend(), + )) + ->appendRows([['id' => '1', 'price' => '9.99', 'active' => 'true', 'name' => 'Alice']]) + ->finish() + ->toArray(), + ); + } + + public function test_append_rows_keeps_native_typed_values(): void + { + static::assertSame( + [['id' => 1, 'name' => 'Alice'], ['id' => 2, 'name' => 'Bob']], + (new RowsBuilder(schema(int_schema('id'), str_schema('name')), new PhpBackend())) + ->appendRows([['id' => 1, 'name' => 'Alice'], ['id' => 2, 'name' => 'Bob']]) + ->finish() + ->toArray(), + ); + } + + public function test_append_rows_keeps_native_uuid_value_objects(): void + { + // @mago-ignore analysis:mixed-assignment + $value = (new RowsBuilder(schema(uuid_schema('id')), new PhpBackend())) + ->appendRows([['id' => new Uuid('f47ac10b-58cc-4372-a567-0e02b2c3d479')]]) + ->finish() + ->column('id') + ->value(0); + + static::assertInstanceOf(Uuid::class, $value); + static::assertSame('f47ac10b-58cc-4372-a567-0e02b2c3d479', $value->toString()); + } + + public function test_append_rows_keeps_the_schema_definition_of_a_null_value(): void + { + $rows = (new RowsBuilder( + schema(int_schema('id'), str_schema('name', nullable: true)), + new PhpBackend(), + ))->appendRows([['id' => 1, 'name' => null]])->finish(); + + static::assertNull($rows->column('name')->value(0)); + static::assertTrue($rows->schema()->get('name')->isNullable()); + } + + public function test_append_rows_shares_the_declared_schema_instance(): void + { + $schema = schema(int_schema('id')); + + static::assertSame( + $schema, + (new RowsBuilder($schema, new PhpBackend())) + ->appendRows([['id' => 1], ['id' => 2]]) + ->finish() + ->schema(), + ); + } + + public function test_append_rows_does_not_cast_native_values(): void + { + $createdAt = new DateTimeImmutable('2024-01-01 12:00:00 UTC'); + + $rows = (new RowsBuilder( + schema(int_schema('id'), datetime_schema('created_at')), + new PhpBackend(), + ))->appendRows([['id' => 1, 'created_at' => $createdAt]])->finish(); + + static::assertSame(1, $rows->column('id')->value(0)); + static::assertEquals($createdAt, $rows->column('created_at')->value(0)); + } + + public function test_append_rows_does_not_cast_nested_list_map_and_structure(): void + { + $rows = (new RowsBuilder( + schema( + list_schema('tags', type_list(type_string())), + map_schema('counts', type_map(type_string(), type_integer())), + structure_schema('address', type_structure(['city' => type_string(), 'zip' => type_integer()])), + ), + new PhpBackend(), + ))->appendRows([[ + 'tags' => ['a', 'b'], + 'counts' => ['x' => 1, 'y' => 2], + 'address' => ['city' => 'NYC', 'zip' => 10001], + ]])->finish(); + + static::assertInstanceOf(ListType::class, $rows->schema()->get('tags')->type()); + static::assertInstanceOf(MapType::class, $rows->schema()->get('counts')->type()); + static::assertInstanceOf(StructureType::class, $rows->schema()->get('address')->type()); + static::assertSame(['a', 'b'], $rows->column('tags')->value(0)); + static::assertSame(['x' => 1, 'y' => 2], $rows->column('counts')->value(0)); + static::assertSame(['city' => 'NYC', 'zip' => 10001], $rows->column('address')->value(0)); + } + + public function test_append_rows_of_already_typed_values_is_idempotent(): void + { + $schema = schema(int_schema('id'), str_schema('name'), datetime_schema('created_at')); + $rows = (new RowsBuilder($schema, new PhpBackend()))->appendRows([[ + 'id' => 1, + 'name' => 'Alice', + 'created_at' => new DateTimeImmutable('2024-01-01 00:00:00 UTC'), + ]])->finish(); + + static::assertEquals( + $rows, + (new RowsBuilder($schema, new PhpBackend()))->appendRows([$rows->values(0)])->finish(), + ); + } + + public function test_a_present_null_and_an_absent_key_are_both_null(): void + { + static::assertSame( + [['id' => 1, 'name' => null], ['id' => 2, 'name' => null], ['id' => 3, 'name' => 'c']], + (new RowsBuilder(schema(int_schema('id'), str_schema('name', nullable: true)), new PhpBackend())) + ->appendRows([['id' => 1, 'name' => null], ['id' => 2], ['id' => 3, 'name' => 'c']]) + ->finish() + ->toArray(), + ); + } + + public function test_a_refused_value_in_a_column_every_row_carries_is_placed_at_its_row(): void + { + $this->expectException(SchemaMismatchException::class); + $this->expectExceptionMessage('column "id" (row 3): could not convert \'x\' (string) to integer'); + + (new RowsBuilder(schema(int_schema('id')), new PhpBackend()))->appendRows([['id' => 1]])->appendRows([ + ['id' => 2], + ['id' => 3], + ['id' => 'x'], + ]); + } + + public function test_append_records_takes_a_single_record(): void + { + static::assertSame( + [['id' => 1]], + (new RowsBuilder(schema(int_schema('id')), new PhpBackend())) + ->appendRecords(['id' => 1]) + ->finish() + ->toArray(), + ); + } + + public function test_append_records_takes_a_list_of_records(): void + { + static::assertSame( + [['id' => 1], ['id' => 2]], + (new RowsBuilder(schema(int_schema('id')), new PhpBackend())) + ->appendRecords([['id' => 1], ['id' => 2]]) + ->finish() + ->toArray(), + ); + } + + public function test_append_records_matches_a_numeric_string_key_to_its_declared_name(): void + { + static::assertSame( + [1], + (new RowsBuilder(schema(int_schema('2024')), new PhpBackend())) + ->appendRecords([['2024' => 1]]) + ->finish() + ->column('2024') + ->values(), + ); + } + + public function test_append_records_refuses_an_undeclared_key_at_its_row(): void + { + try { + (new RowsBuilder(schema(int_schema('id')), new PhpBackend()))->appendRecords([ + ['id' => 1], + ['id' => 2, 'nmae' => 'two'], + ]); + static::fail('an undeclared key must be refused'); + } catch (SchemaMismatchException $e) { + static::assertSame(1, $e->rowIndex); + static::assertStringContainsString('nmae', $e->getMessage()); + } + } + + public function test_append_records_refuses_an_undeclared_key_at_its_row_after_a_prior_append(): void + { + try { + (new RowsBuilder(schema(int_schema('id')), new PhpBackend()))->appendRecords([ + ['id' => 1], + ['id' => 2], + ])->appendRecords([['id' => 3, 'nmae' => 'three']]); + static::fail('an undeclared key must be refused'); + } catch (SchemaMismatchException $e) { + static::assertSame(2, $e->rowIndex); + } + } + + public function test_append_projected_drops_an_undeclared_key(): void + { + static::assertSame( + [['id' => 1, 'name' => 'one'], ['id' => 2, 'name' => null]], + (new RowsBuilder(schema(int_schema('id'), str_schema('name', nullable: true)), new PhpBackend())) + ->appendProjected([['id' => 1, 'name' => 'one'], ['id' => 2, 'nmae' => 'two']]) + ->finish() + ->toArray(), + ); + } + + public function test_append_projected_keeps_an_int_key_whose_positional_name_is_declared(): void + { + static::assertSame( + [['e00' => 'a']], + (new RowsBuilder(schema(str_schema('e00')), new PhpBackend())) + ->appendProjected([['a', 'b']]) + ->finish() + ->toArray(), + ); + } + + public function test_append_projected_keeps_a_numeric_column_name_declared_as_is(): void + { + static::assertSame( + [1], + (new RowsBuilder(schema(int_schema('2024')), new PhpBackend())) + ->appendProjected([['2024' => 1, 'other' => 2]]) + ->finish() + ->column('2024') + ->values(), + ); + } + + public function test_append_projected_of_nothing_appends_nothing(): void + { + static::assertSame( + 0, + (new RowsBuilder(schema(int_schema('id')), new PhpBackend())) + ->appendProjected([]) + ->finish() + ->count(), + ); + } +} diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/RowsJoinTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/RowsJoinTest.php deleted file mode 100644 index c48b138ed8..0000000000 --- a/src/core/etl/tests/Flow/ETL/Tests/Unit/RowsJoinTest.php +++ /dev/null @@ -1,616 +0,0 @@ - 1, 'country' => 'PL']), - row(['id' => 2, 'country' => 'PL']), - row(['id' => 3, 'country' => 'US']), - row(['id' => 4, 'country' => 'FR']), - ); - - $joined = $left->joinCross(rows( - schema(int_schema('num'), bool_schema('active')), - row(['num' => 1, 'active' => true]), - row(['num' => 2, 'active' => false]), - )); - - static::assertEquals( - [ - ['id' => 1, 'country' => 'PL', 'joined_num' => 1, 'joined_active' => true], - ['id' => 1, 'country' => 'PL', 'joined_num' => 2, 'joined_active' => false], - ['id' => 2, 'country' => 'PL', 'joined_num' => 1, 'joined_active' => true], - ['id' => 2, 'country' => 'PL', 'joined_num' => 2, 'joined_active' => false], - ['id' => 3, 'country' => 'US', 'joined_num' => 1, 'joined_active' => true], - ['id' => 3, 'country' => 'US', 'joined_num' => 2, 'joined_active' => false], - ['id' => 4, 'country' => 'FR', 'joined_num' => 1, 'joined_active' => true], - ['id' => 4, 'country' => 'FR', 'joined_num' => 2, 'joined_active' => false], - ], - $joined->toArray(), - ); - } - - public function test_cross_join_empty(): void - { - $left = rows( - schema(int_schema('id'), str_schema('country')), - row(['id' => 1, 'country' => 'PL']), - row(['id' => 2, 'country' => 'PL']), - row(['id' => 3, 'country' => 'US']), - row(['id' => 4, 'country' => 'FR']), - ); - - // n x 0 = 0, and a zero-row batch fits the cross schema vacuously - static::assertSame([], $left->joinCross(rows(schema()))->toArray()); - } - - public function test_cross_join_with_an_empty_side_yields_no_rows_under_the_cross_schema(): void - { - // a cross join with an empty side produces zero rows, and the batch still declares both - // sides' columns - the schema describes the plan, not the cardinality - $emptyRight = rows( - schema(int_schema('id'), str_schema('country')), - row(['id' => 1, 'country' => 'PL']), - )->joinCross(rows(schema(str_schema('code')))); - - static::assertSame(['id', 'country', 'joined_code'], $emptyRight->schema()->references()->names()); - static::assertSame([], $emptyRight->toArray()); - - $emptyLeft = rows(schema(int_schema('id')))->joinCross(rows(schema(str_schema('code')), row(['code' => 'PL']))); - - static::assertSame(['id', 'joined_code'], $emptyLeft->schema()->references()->names()); - static::assertSame([], $emptyLeft->toArray()); - } - - public function test_cross_join_left_empty(): void - { - $left = rows(schema()); - - $joined = $left->joinCross(rows( - schema(int_schema('id'), str_schema('country')), - row(['id' => 1, 'country' => 'PL']), - row(['id' => 2, 'country' => 'PL']), - row(['id' => 3, 'country' => 'US']), - row(['id' => 4, 'country' => 'FR']), - )); - - static::assertSame([], $joined->toArray()); - } - - public function test_cross_join_left_with_name_conflict(): void - { - $this->expectException(SchemaDefinitionNotUniqueException::class); - $this->expectExceptionMessage( - 'Entry definitions must be unique, duplicated entries: [active], all: [id, country, active, active]', - ); - - $left = rows( - schema(int_schema('id'), str_schema('country'), bool_schema('active')), - row(['id' => 1, 'country' => 'PL', 'active' => false]), - row(['id' => 2, 'country' => 'PL', 'active' => false]), - row(['id' => 3, 'country' => 'US', 'active' => false]), - row(['id' => 4, 'country' => 'FR', 'active' => false]), - ); - - $left->joinCross(rows(schema(bool_schema('active')), row(['active' => true])), ''); - } - - public function test_cross_join_left_with_name_conflict_with_prefix(): void - { - $left = rows( - schema(int_schema('id'), str_schema('country'), bool_schema('active')), - row(['id' => 1, 'country' => 'PL', 'active' => false]), - row(['id' => 2, 'country' => 'PL', 'active' => false]), - row(['id' => 3, 'country' => 'US', 'active' => false]), - row(['id' => 4, 'country' => 'FR', 'active' => false]), - ); - - $joined = $left->joinCross(rows(schema(bool_schema('active')), row(['active' => true])), '_'); - - static::assertEquals( - [ - ['id' => 1, 'country' => 'PL', 'active' => false, '_active' => true], - ['id' => 2, 'country' => 'PL', 'active' => false, '_active' => true], - ['id' => 3, 'country' => 'US', 'active' => false, '_active' => true], - ['id' => 4, 'country' => 'FR', 'active' => false, '_active' => true], - ], - $joined->toArray(), - ); - } - - public function test_inner_empty(): void - { - $left = rows( - schema(int_schema('id'), str_schema('country')), - row(['id' => 1, 'country' => 'PL']), - row(['id' => 2, 'country' => 'PL']), - row(['id' => 3, 'country' => 'US']), - row(['id' => 4, 'country' => 'FR']), - ); - - $joined = $left->joinInner(rows(schema()), Expression::on(['country' => 'code'])); - - static::assertEquals(rows(schema(int_schema('id'), str_schema('country'))), $joined); - } - - public function test_inner_join(): void - { - $left = rows( - schema(int_schema('id'), str_schema('country')), - row(['id' => 1, 'country' => 'PL']), - row(['id' => 2, 'country' => 'PL']), - row(['id' => 3, 'country' => 'US']), - row(['id' => 4, 'country' => 'FR']), - ); - - $joined = $left->joinInner( - rows( - schema(str_schema('code'), str_schema('name')), - row(['code' => 'PL', 'name' => 'Poland']), - row(['code' => 'US', 'name' => 'United States']), - row(['code' => 'GB', 'name' => 'Great Britain']), - ), - join_on(['country' => 'code'], 'joined_'), - ); - - static::assertEquals( - [ - ['id' => 1, 'country' => 'PL', 'joined_code' => 'PL', 'joined_name' => 'Poland'], - ['id' => 2, 'country' => 'PL', 'joined_code' => 'PL', 'joined_name' => 'Poland'], - ['id' => 3, 'country' => 'US', 'joined_code' => 'US', 'joined_name' => 'United States'], - ], - $joined->toArray(), - ); - } - - public function test_inner_join_emits_every_matching_right_row(): void - { - $joined = rows(schema(int_schema('id'), str_schema('country')), row(['id' => 1, 'country' => 'PL']))->joinInner( - rows( - schema(int_schema('code'), str_schema('city')), - row(['code' => 1, 'city' => 'Warsaw']), - row(['code' => 1, 'city' => 'Cracow']), - ), - join_on(['id' => 'code']), - ); - - static::assertSame( - [ - ['id' => 1, 'country' => 'PL', 'code' => 1, 'city' => 'Warsaw'], - ['id' => 1, 'country' => 'PL', 'code' => 1, 'city' => 'Cracow'], - ], - $joined->toArray(), - ); - } - - public function test_inner_join_into_empty(): void - { - $left = rows(schema()); - - $joined = $left->joinInner( - rows( - schema(str_schema('code'), str_schema('name')), - row(['code' => 'PL', 'name' => 'Poland']), - row(['code' => 'US', 'name' => 'United States']), - row(['code' => 'GB', 'name' => 'Great Britain']), - ), - Expression::on(['country' => 'code']), - ); - - static::assertEquals(rows(schema(str_schema('code'), str_schema('name'))), $joined); - } - - public function test_inner_join_with_duplicated_entries(): void - { - $this->expectException(SchemaDefinitionNotUniqueException::class); - $this->expectExceptionMessage( - 'Entry definitions must be unique, duplicated entries: [id], all: [id, country, id, code, name]', - ); - - $left = rows( - schema(int_schema('id'), str_schema('country')), - row(['id' => 1, 'country' => 'PL']), - row(['id' => 2, 'country' => 'PL']), - row(['id' => 3, 'country' => 'US']), - row(['id' => 4, 'country' => 'FR']), - ); - - $left->joinInner( - rows( - schema(int_schema('id'), str_schema('code'), str_schema('name')), - row(['id' => 101, 'code' => 'PL', 'name' => 'Poland']), - row(['id' => 102, 'code' => 'US', 'name' => 'United States']), - row(['id' => 103, 'code' => 'GB', 'name' => 'Great Britain']), - ), - Expression::on(['country' => 'code'], joinPrefix: ''), - ); - } - - public function test_inner_join_without_prefix(): void - { - $left = rows( - schema(int_schema('id'), str_schema('country_code')), - row(['id' => 1, 'country_code' => 'PL']), - row(['id' => 2, 'country_code' => 'PL']), - row(['id' => 3, 'country_code' => 'US']), - row(['id' => 4, 'country_code' => 'FR']), - ); - - $joined = $left->joinInner( - rows( - schema(str_schema('country_code'), str_schema('name')), - row(['country_code' => 'PL', 'name' => 'Poland']), - row(['country_code' => 'US', 'name' => 'United States']), - row(['country_code' => 'GB', 'name' => 'Great Britain']), - ), - join_on(['country_code' => 'country_code']), - ); - - static::assertEquals( - [ - ['id' => 1, 'country_code' => 'PL', 'name' => 'Poland'], - ['id' => 2, 'country_code' => 'PL', 'name' => 'Poland'], - ['id' => 3, 'country_code' => 'US', 'name' => 'United States'], - ], - $joined->toArray(), - ); - } - - public function test_left_anti_join(): void - { - $left = rows( - schema(int_schema('id'), str_schema('country')), - row(['id' => 1, 'country' => 'PL']), - row(['id' => 2, 'country' => 'US']), - row(['id' => 3, 'country' => 'FR']), - ); - - $joined = $left->joinLeftAnti( - rows( - schema(str_schema('code'), str_schema('name')), - row(['code' => 'US', 'name' => 'United States']), - row(['code' => 'FR', 'name' => 'France']), - ), - Expression::on(['country' => 'code']), - ); - - static::assertEquals( - rows(schema(int_schema('id'), str_schema('country')), row(['id' => 1, 'country' => 'PL'])), - $joined, - ); - } - - public function test_left_anti_join_on_empty(): void - { - $left = rows( - schema(int_schema('id'), str_schema('country')), - row(['id' => 1, 'country' => 'PL']), - row(['id' => 2, 'country' => 'US']), - row(['id' => 3, 'country' => 'FR']), - ); - - $joined = $left->joinLeftAnti(rows(schema()), Expression::on(['country' => 'code'])); - - static::assertEquals($left, $joined); - } - - public function test_left_anti_join_without_prefix(): void - { - $left = rows( - schema(int_schema('id'), str_schema('country_code')), - row(['id' => 1, 'country_code' => 'PL']), - row(['id' => 2, 'country_code' => 'US']), - row(['id' => 3, 'country_code' => 'FR']), - ); - - $joined = $left->joinLeftAnti( - rows( - schema(str_schema('country_code'), str_schema('name')), - row(['country_code' => 'US', 'name' => 'United States']), - row(['country_code' => 'FR', 'name' => 'France']), - ), - Expression::on(['country_code' => 'country_code']), - ); - - static::assertEquals( - rows(schema(int_schema('id'), str_schema('country_code')), row(['id' => 1, 'country_code' => 'PL'])), - $joined, - ); - } - - public function test_left_join(): void - { - $left = rows( - schema(int_schema('id'), str_schema('country')), - row(['id' => 1, 'country' => 'PL']), - row(['id' => 2, 'country' => 'US']), - row(['id' => 3, 'country' => 'FR']), - ); - - $joined = $left->joinLeft( - rows( - schema(str_schema('code'), str_schema('name')), - row(['code' => 'PL', 'name' => 'Poland']), - row(['code' => 'US', 'name' => 'United States']), - row(['code' => 'GB', 'name' => 'Great Britain']), - ), - Expression::on(['country' => 'code'], 'joined_'), - ); - - static::assertEquals( - [ - ['id' => 1, 'country' => 'PL', 'joined_code' => 'PL', 'joined_name' => 'Poland'], - ['id' => 2, 'country' => 'US', 'joined_code' => 'US', 'joined_name' => 'United States'], - ['id' => 3, 'country' => 'FR', 'joined_code' => null, 'joined_name' => null], - ], - $joined->toArray(), - ); - } - - public function test_left_join_empty(): void - { - $left = rows( - schema(int_schema('id'), str_schema('country')), - row(['id' => 1, 'country' => 'PL']), - row(['id' => 2, 'country' => 'US']), - row(['id' => 3, 'country' => 'FR']), - ); - - $joined = $left->joinLeft(rows(schema()), Expression::on(['country' => 'code'])); - - static::assertEquals( - rows( - schema(int_schema('id'), str_schema('country')), - row(['id' => 1, 'country' => 'PL']), - row(['id' => 2, 'country' => 'US']), - row(['id' => 3, 'country' => 'FR']), - ), - $joined, - ); - } - - public function test_left_join_empty_without_prefix(): void - { - $left = rows( - schema(int_schema('id'), str_schema('country_code')), - row(['id' => 1, 'country_code' => 'PL']), - row(['id' => 2, 'country_code' => 'US']), - row(['id' => 3, 'country_code' => 'FR']), - ); - - $joined = $left->joinLeft(rows(schema()), Expression::on(['country_code' => 'country_code'])); - - static::assertEquals( - rows( - schema(int_schema('id'), str_schema('country_code')), - row(['id' => 1, 'country_code' => 'PL']), - row(['id' => 2, 'country_code' => 'US']), - row(['id' => 3, 'country_code' => 'FR']), - ), - $joined, - ); - } - - public function test_left_join_to_empty(): void - { - $left = rows(schema()); - - $joined = $left->joinLeft( - rows( - schema(str_schema('code'), str_schema('name')), - row(['code' => 'PL', 'name' => 'Poland']), - row(['code' => 'US', 'name' => 'United States']), - row(['code' => 'GB', 'name' => 'Great Britain']), - ), - Expression::on(['country' => 'code']), - ); - - static::assertEquals( - rows(schema(str_schema('code', nullable: true), str_schema('name', nullable: true))), - $joined, - ); - } - - public function test_left_join_with_the_duplicated_columns(): void - { - $this->expectException(SchemaDefinitionNotUniqueException::class); - $this->expectExceptionMessage( - 'Entry definitions must be unique, duplicated entries: [id], all: [id, country, id, code, name]', - ); - - $left = rows( - schema(int_schema('id'), str_schema('country')), - row(['id' => 1, 'country' => 'PL']), - row(['id' => 2, 'country' => 'US']), - row(['id' => 3, 'country' => 'FR']), - ); - - $left->joinLeft( - rows( - schema(int_schema('id'), str_schema('code'), str_schema('name')), - row(['id' => 100, 'code' => 'PL', 'name' => 'Poland']), - row(['id' => 101, 'code' => 'US', 'name' => 'United States']), - row(['id' => 102, 'code' => 'GB', 'name' => 'Great Britain']), - ), - Expression::on(['country' => 'code'], ''), - ); - } - - public function test_left_join_without_prefix(): void - { - $left = rows( - schema(int_schema('id'), str_schema('country_code')), - row(['id' => 1, 'country_code' => 'PL']), - row(['id' => 2, 'country_code' => 'US']), - row(['id' => 3, 'country_code' => 'FR']), - ); - - $joined = $left->joinLeft( - rows( - schema(str_schema('country_code'), str_schema('name')), - row(['country_code' => 'PL', 'name' => 'Poland']), - row(['country_code' => 'US', 'name' => 'United States']), - row(['country_code' => 'GB', 'name' => 'Great Britain']), - ), - Expression::on(['country_code' => 'country_code']), - ); - - static::assertEquals( - [ - ['id' => 1, 'country_code' => 'PL', 'name' => 'Poland'], - ['id' => 2, 'country_code' => 'US', 'name' => 'United States'], - ['id' => 3, 'country_code' => 'FR', 'name' => null], - ], - $joined->toArray(), - ); - } - - public function test_right_join(): void - { - $left = rows( - schema(int_schema('id'), str_schema('country')), - row(['id' => 1, 'country' => 'PL']), - row(['id' => 2, 'country' => 'PL']), - row(['id' => 3, 'country' => 'US']), - row(['id' => 4, 'country' => 'FR']), - ); - - $joined = $left->joinRight( - rows( - schema(str_schema('code'), str_schema('name')), - row(['code' => 'PL', 'name' => 'Poland']), - row(['code' => 'US', 'name' => 'United States']), - row(['code' => 'GB', 'name' => 'Great Britain']), - ), - Expression::on(['country' => 'code'], 'joined_'), - ); - - static::assertEquals( - [ - ['id' => 1, 'country' => 'PL', 'joined_code' => 'PL', 'joined_name' => 'Poland'], - ['id' => 2, 'country' => 'PL', 'joined_code' => 'PL', 'joined_name' => 'Poland'], - ['id' => 3, 'country' => 'US', 'joined_code' => 'US', 'joined_name' => 'United States'], - ['id' => null, 'country' => null, 'joined_code' => 'GB', 'joined_name' => 'Great Britain'], - ], - $joined->toArray(), - ); - } - - public function test_right_join_empty(): void - { - $left = rows( - schema(int_schema('id'), str_schema('country')), - row(['id' => 1, 'country' => 'PL']), - row(['id' => 2, 'country' => 'PL']), - row(['id' => 3, 'country' => 'US']), - row(['id' => 4, 'country' => 'FR']), - ); - - $joined = $left->joinRight(rows(schema()), Expression::on(['country' => 'code'])); - - static::assertEquals( - rows(schema(int_schema('id', nullable: true), str_schema('country', nullable: true))), - $joined, - ); - } - - public function test_right_join_to_empty(): void - { - $left = rows(schema()); - - $joined = $left->joinRight( - rows( - schema(str_schema('code'), str_schema('name')), - row(['code' => 'PL', 'name' => 'Poland']), - row(['code' => 'US', 'name' => 'United States']), - row(['code' => 'GB', 'name' => 'Great Britain']), - ), - Expression::on(['country' => 'code'], 'joined_'), - ); - - static::assertEquals( - [ - ['joined_code' => 'PL', 'joined_name' => 'Poland'], - ['joined_code' => 'US', 'joined_name' => 'United States'], - ['joined_code' => 'GB', 'joined_name' => 'Great Britain'], - ], - $joined->toArray(), - ); - } - - public function test_right_join_with_duplicated_entry_names(): void - { - $this->expectException(SchemaDefinitionNotUniqueException::class); - $this->expectExceptionMessage( - 'Entry definitions must be unique, duplicated entries: [id], all: [id, country, id, code, name]', - ); - - $left = rows( - schema(int_schema('id'), str_schema('country')), - row(['id' => 1, 'country' => 'PL']), - row(['id' => 2, 'country' => 'PL']), - row(['id' => 3, 'country' => 'US']), - row(['id' => 4, 'country' => 'FR']), - ); - - $left->joinRight( - rows( - schema(int_schema('id'), str_schema('code'), str_schema('name')), - row(['id' => 101, 'code' => 'PL', 'name' => 'Poland']), - row(['id' => 102, 'code' => 'US', 'name' => 'United States']), - row(['id' => 103, 'code' => 'GB', 'name' => 'Great Britain']), - ), - Expression::on(['country' => 'code'], ''), - ); - } - - public function test_right_join_without_prefix(): void - { - $left = rows( - schema(int_schema('id'), str_schema('country_code')), - row(['id' => 1, 'country_code' => 'PL']), - row(['id' => 2, 'country_code' => 'PL']), - row(['id' => 3, 'country_code' => 'US']), - row(['id' => 4, 'country_code' => 'FR']), - ); - - $joined = $left->joinRight( - rows( - schema(str_schema('country_code'), str_schema('name')), - row(['country_code' => 'PL', 'name' => 'Poland']), - row(['country_code' => 'US', 'name' => 'United States']), - row(['country_code' => 'GB', 'name' => 'Great Britain']), - ), - Expression::on(['country_code' => 'country_code']), - ); - - static::assertEquals( - [ - ['id' => 1, 'country_code' => 'PL', 'name' => 'Poland'], - ['id' => 2, 'country_code' => 'PL', 'name' => 'Poland'], - ['id' => 3, 'country_code' => 'US', 'name' => 'United States'], - ['id' => null, 'country_code' => 'GB', 'name' => 'Great Britain'], - ], - $joined->toArray(), - ); - } -} diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/RowsSerializationTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/RowsSerializationTest.php index 6848ced1d5..5ed2c1d30b 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Unit/RowsSerializationTest.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/RowsSerializationTest.php @@ -5,17 +5,39 @@ namespace Flow\ETL\Tests\Unit; use DOMDocument; +use DOMElement; +use Flow\ETL\Column\PhpBackend; +use Flow\ETL\Column\Physical\HtmlDocumentPhysical; +use Flow\ETL\Column\Physical\HtmlElementPhysical; use Flow\ETL\Rows; use Flow\ETL\Schema\Metadata; +use Flow\ETL\Tests\Context\RowsSerializationContext; use Flow\ETL\Tests\FlowTestCase; +use Flow\Floe\Exception\FloeException; +use PHPUnit\Framework\Attributes\RequiresPhp; use function assert; +use function Flow\ETL\DSL\array_to_rows; +use function Flow\ETL\DSL\html_element_schema; +use function Flow\ETL\DSL\html_schema; use function Flow\ETL\DSL\int_schema; -use function Flow\ETL\DSL\row; -use function Flow\ETL\DSL\rows; +use function Flow\ETL\DSL\list_schema; +use function Flow\ETL\DSL\map_schema; use function Flow\ETL\DSL\schema; use function Flow\ETL\DSL\str_schema; +use function Flow\ETL\DSL\structure_schema; +use function Flow\ETL\DSL\xml_element_schema; use function Flow\ETL\DSL\xml_schema; +use function Flow\Types\DSL\structure_element; +use function Flow\Types\DSL\type_instance_of; +use function Flow\Types\DSL\type_integer; +use function Flow\Types\DSL\type_list; +use function Flow\Types\DSL\type_map; +use function Flow\Types\DSL\type_object; +use function Flow\Types\DSL\type_optional; +use function Flow\Types\DSL\type_string; +use function Flow\Types\DSL\type_structure; +use function pack; use function serialize; use function unserialize; @@ -23,10 +45,9 @@ final class RowsSerializationTest extends FlowTestCase { public function test_native_columns_survive_a_serialize_round_trip(): void { - $rows = rows( + $rows = array_to_rows( + [['id' => 1, 'name' => 'a'], ['id' => 2, 'name' => null]], schema(int_schema('id'), str_schema('name', nullable: true)), - row(['id' => 1, 'name' => 'a']), - row(['id' => 2, 'name' => null]), ); /** @var Rows $restored */ @@ -41,18 +62,180 @@ public function test_xml_column_metadata_survives_a_serialize_round_trip(): void $document = new DOMDocument(); $document->loadXML('1'); - $rows = rows( - schema(xml_schema('x', metadata: Metadata::empty()->add('src', 'file.xml'))), - row(['x' => $document]), - ); + $rows = array_to_rows([[ + 'x' => $document, + ]], schema(xml_schema('x', metadata: Metadata::empty()->add('src', 'file.xml')))); /** @var Rows $restored */ $restored = unserialize(serialize($rows)); - $restoredDocument = $restored->first()->get('x'); + // @mago-ignore analysis:mixed-assignment + $restoredDocument = $restored->column('x')->value(0); assert($restoredDocument instanceof DOMDocument); static::assertSame(['src' => 'file.xml'], $restored->schema()->get('x')->metadata()->normalize()); static::assertSame($rows->schema()->get('x')->isNullable(), $restored->schema()->get('x')->isNullable()); static::assertSame($document->C14N(), $restoredDocument->C14N()); } + + public function test_xml_document_survives_a_round_trip(): void + { + $document = new DOMDocument(); + $document->loadXML('1'); + + $restored = type_instance_of(DOMDocument::class)->assert( + RowsSerializationContext::roundTrip(array_to_rows([[ + 'x' => $document, + ]], schema(xml_schema('x')))) + ->column('x') + ->value(0), + ); + + static::assertSame($document->C14N(), $restored->C14N()); + } + + public function test_xml_element_survives_a_round_trip(): void + { + $document = new DOMDocument(); + $document->loadXML('1'); + + $restored = type_instance_of(DOMElement::class)->assert( + RowsSerializationContext::roundTrip(array_to_rows([[ + 'x' => $document->documentElement, + ]], schema(xml_element_schema('x')))) + ->column('x') + ->value(0), + ); + + static::assertSame('1', $restored->C14N()); + } + + public function test_restored_xml_elements_do_not_canonicalize_to_an_empty_string(): void + { + $left = new DOMDocument(); + $left->loadXML('1'); + $right = new DOMDocument(); + $right->loadXML('999'); + + $restored = RowsSerializationContext::roundTrip(array_to_rows([ + ['x' => $left->documentElement], + ['x' => $right->documentElement], + ], schema(xml_element_schema('x')))); + $restoredLeft = type_instance_of(DOMElement::class)->assert($restored->column('x')->value(0)); + $restoredRight = type_instance_of(DOMElement::class)->assert($restored->column('x')->value(1)); + + static::assertNotSame('', $restoredLeft->C14N()); + static::assertNotSame($restoredLeft->C14N(), $restoredRight->C14N()); + } + + #[RequiresPhp('>= 8.4.0')] + public function test_html_document_survives_a_round_trip(): void + { + $document = type_object()->assert((new HtmlDocumentPhysical())->fromPhysical('

x

')); + + static::assertSame( + type_string()->cast($document), + type_string()->cast( + RowsSerializationContext::roundTrip(array_to_rows([[ + 'x' => $document, + ]], schema(html_schema('x')))) + ->column('x') + ->value(0), + ), + ); + } + + #[RequiresPhp('>= 8.4.0')] + public function test_html_element_survives_a_round_trip(): void + { + $element = type_object()->assert((new HtmlElementPhysical())->fromPhysical('

x

')); + + static::assertSame( + type_string()->cast($element), + type_string()->cast( + RowsSerializationContext::roundTrip(array_to_rows([[ + 'x' => $element, + ]], schema(html_element_schema('x')))) + ->column('x') + ->value(0), + ), + ); + } + + public function test_container_columns_survive_a_round_trip(): void + { + $rows = array_to_rows( + [ + ['l' => [1, null], 'm' => ['a' => 1], 's' => ['id' => 1, 'note' => 'x']], + ['l' => null, 'm' => [], 's' => ['id' => 2]], + ], + schema( + list_schema('l', type_list(type_optional(type_integer())), nullable: true), + map_schema('m', type_map(type_string(), type_integer())), + structure_schema('s', type_structure([ + 'id' => type_integer(), + 'note' => structure_element('note', type_string(), optional: true), + ])), + ), + ); + + static::assertEquals($rows, RowsSerializationContext::roundTrip($rows)); + } + + public function test_constant_columns_survive_a_round_trip(): void + { + $schema = schema(int_schema('id'), str_schema('tag', nullable: true)); + $backend = new PhpBackend(); + $rows = Rows::fromColumns( + $schema, + [ + 'id' => $backend->constant(int_schema('id'), 7, 3), + 'tag' => $backend->constant(str_schema('tag', nullable: true), null, 3), + ], + 3, + ); + + static::assertSame( + [['id' => 7, 'tag' => null], ['id' => 7, 'tag' => null], ['id' => 7, 'tag' => null]], + RowsSerializationContext::roundTrip($rows)->toArray(), + ); + } + + public function test_a_batch_without_columns_keeps_its_row_count(): void + { + $rows = array_to_rows([[], [], []], schema()); + + static::assertSame(3, RowsSerializationContext::roundTrip($rows)->count()); + } + + public function test_unserialize_refuses_a_frame_that_disagrees_with_the_schema(): void + { + $this->expectException(FloeException::class); + $this->expectExceptionMessage('Floe BATCH frame holds 2 nodes, the schema describes 1'); + + RowsSerializationContext::unserialize([ + 'schema' => schema(int_schema('id')), + 'frame' => array_to_rows( + [['id' => 1, 'other' => 2]], + schema(int_schema('id'), int_schema('other')), + )->encodeFrame(), + ]); + } + + public function test_unserialize_refuses_a_corrupt_buffer(): void + { + $this->expectException(FloeException::class); + $this->expectExceptionMessage('is malformed: Int64 values buffer of 2 bytes, expected 8 for 1 rows'); + + RowsSerializationContext::unserialize([ + 'schema' => schema(int_schema('id')), + 'frame' => + pack('VVV', 1, 1, 2) + . pack('VV', 1, 0) + . pack('VV', 0, 0) + . pack('VV', 0, 10) + . "\0\0\0\0" + . pack('P', -1) + . "\x01\x00\0\0\0\0\0\0", + ]); + } } diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/RowsTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/RowsTest.php index 11363a8618..d0043b79b1 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Unit/RowsTest.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/RowsTest.php @@ -4,25 +4,36 @@ namespace Flow\ETL\Tests\Unit; +use Closure; use DateTimeImmutable; use DateTimeZone; +use Flow\ETL\Column\AdaptiveBackend; +use Flow\ETL\Column\Php\ConstantColumn; +use Flow\ETL\Column\Php\ScalarColumn; +use Flow\ETL\Column\PhpBackend; +use Flow\ETL\Column\RustBackend; +use Flow\ETL\Column\ValueColumn; +use Flow\ETL\Exception\ColumnMismatchException; use Flow\ETL\Exception\InvalidArgumentException; -use Flow\ETL\Exception\RuntimeException; +use Flow\ETL\Exception\SchemaDefinitionNotFoundException; use Flow\ETL\Exception\SchemaMismatchException; -use Flow\ETL\Row; -use Flow\ETL\Row\Comparator; -use Flow\ETL\Row\Comparator\NativeComparator; use Flow\ETL\Rows; +use Flow\ETL\Schema; +use Flow\ETL\Tests\Double\SpyColumn; use Flow\ETL\Tests\FlowTestCase; use Generator; use PHPUnit\Framework\Attributes\DataProvider; +use PHPUnit\Framework\Attributes\RequiresPhpExtension; +use PHPUnit\Framework\Attributes\TestWith; +use function Flow\ETL\DSL\array_to_rows; use function Flow\ETL\DSL\bool_schema; +use function Flow\ETL\DSL\date_schema; use function Flow\ETL\DSL\datetime_schema; use function Flow\ETL\DSL\int_schema; use function Flow\ETL\DSL\list_schema; +use function Flow\ETL\DSL\map_schema; use function Flow\ETL\DSL\ref; -use function Flow\ETL\DSL\row; use function Flow\ETL\DSL\rows; use function Flow\ETL\DSL\schema; use function Flow\ETL\DSL\str_schema; @@ -31,121 +42,45 @@ use function Flow\Types\DSL\type_datetime; use function Flow\Types\DSL\type_integer; use function Flow\Types\DSL\type_list; +use function Flow\Types\DSL\type_map; use function Flow\Types\DSL\type_optional; use function Flow\Types\DSL\type_string; use function Flow\Types\DSL\type_structure; use function iterator_to_array; use function serialize; +use function sprintf; use function unserialize; final class RowsTest extends FlowTestCase { - public static function rows_diff_left_provider(): Generator + /** + * @return Generator + */ + public static function negative_sizes(): Generator { - yield 'one entry identical row' => [ - rows(schema()), - rows(schema(int_schema('number')), row(['number' => 1])), - rows(schema(int_schema('number')), row(['number' => 1])), - ]; - - yield 'one entry right different - missing entry' => [ - rows(schema(int_schema('number')), row(['number' => 1])), - rows(schema(int_schema('number')), row(['number' => 1])), - rows(schema()), - ]; - - yield 'one entry left different - missing entry' => [ - rows(schema()), - rows(schema()), - rows(schema(int_schema('number')), row(['number' => 1])), - ]; - - yield 'one entry right different - different entry' => [ - rows(schema(int_schema('number')), row(['number' => 1])), - rows(schema(int_schema('number')), row(['number' => 1])), - rows(schema(int_schema('number')), row(['number' => 2])), - ]; - - yield 'one entry left different - different entry' => [ - rows(schema(int_schema('number')), row(['number' => 2])), - rows(schema(int_schema('number')), row(['number' => 2])), - rows(schema(int_schema('number')), row(['number' => 1])), - ]; - } - - public static function rows_diff_right_provider(): Generator - { - yield 'one entry identical row' => [ - rows(schema()), - rows(schema(int_schema('number')), row(['number' => 1])), - rows(schema(int_schema('number')), row(['number' => 1])), - ]; - - yield 'one entry right different - missing entry' => [ - rows(schema()), - rows(schema(int_schema('number')), row(['number' => 1])), - rows(schema()), - ]; - - yield 'one entry left different - missing entry' => [ - rows(schema(int_schema('number')), row(['number' => 1])), - rows(schema()), - rows(schema(int_schema('number')), row(['number' => 1])), - ]; - - yield 'one entry right different - different entry' => [ - rows(schema(int_schema('number')), row(['number' => 2])), - rows(schema(int_schema('number')), row(['number' => 1])), - rows(schema(int_schema('number')), row(['number' => 2])), - ]; - - yield 'one entry left different - different entry' => [ - rows(schema(int_schema('number')), row(['number' => 1])), - rows(schema(int_schema('number')), row(['number' => 2])), - rows(schema(int_schema('number')), row(['number' => 1])), - ]; - } - - public static function unique_rows_provider(): Generator - { - yield 'simple identical rows' => [ - rows(schema(int_schema('number')), row(['number' => 1])), - rows(schema(int_schema('number')), row(['number' => 1]), row(['number' => 1])), - new NativeComparator(), - ]; - } - - public function test_add_checks_only_the_added_rows(): void - { - // the existing rows are not re-checked, so one that never passed a door survives add() - $rows = Rows::trusted(schema(int_schema('number')), [row(['number' => 'x'])])->add(row(['number' => 2])); - - static::assertSame([['number' => 'x'], ['number' => 2]], $rows->toArray()); + yield 'drop' => [static fn(Rows $rows): Rows => $rows->drop(-1)]; + yield 'dropRight' => [static fn(Rows $rows): Rows => $rows->dropRight(-1)]; + yield 'take' => [static fn(Rows $rows): Rows => $rows->take(-1)]; } - public function test_add_reports_the_violating_row_at_its_position_in_the_batch(): void + /** + * @param Closure(Rows): Rows $call + */ + #[DataProvider('negative_sizes')] + public function test_a_negative_size_is_refused(Closure $call): void { - $this->expectException(SchemaMismatchException::class); - $this->expectExceptionMessage('column "number" (row 1): could not convert \'x\' (string) to integer'); - - rows(schema(int_schema('number')), row(['number' => 1]))->add(row(['number' => 'x'])); - } - - public function test_adding_multiple_rows(): void - { - $one = row(['number' => 1, 'name' => 'one']); - $two = row(['number' => 2, 'name' => 'two']); - $schema = schema(int_schema('number'), str_schema('name')); + $this->expectException(InvalidArgumentException::class); + $this->expectExceptionMessage('Size must be greater than or equal to 0'); - static::assertEquals(rows($schema, $one, $two), rows($schema)->add($one, $two)); + $call(array_to_rows([['id' => 1]], schema(int_schema('id')))); } public function test_match_to_adopts_a_wider_schema_and_pads_the_new_nullable_column(): void { static::assertSame( [['id' => 1, 'name' => null]], - rows(schema(int_schema('id')), row(['id' => 1])) - ->matchTo(schema(int_schema('id'), str_schema('name', true))) + array_to_rows([['id' => 1]], schema(int_schema('id'))) + ->matchTo(schema(int_schema('id'), str_schema('name', true)), new AdaptiveBackend()) ->toArray(), ); } @@ -153,26 +88,31 @@ public function test_match_to_adopts_a_wider_schema_and_pads_the_new_nullable_co public function test_match_to_reports_the_violating_row_at_its_position_in_the_batch(): void { $this->expectException(SchemaMismatchException::class); - $this->expectExceptionMessage('column "id" (row 1) declared by the schema is missing from the row'); + $this->expectExceptionMessage('column "id" (row 0): could not convert 1 (integer) to string'); - Rows::trusted(schema(int_schema('id')), [ - row(['id' => 1]), - row(['other' => 2]), - ])->matchTo(schema(int_schema('id'))); + array_to_rows([['id' => 1], ['id' => 2]], schema(int_schema('id')))->matchTo( + schema(str_schema('id')), + new AdaptiveBackend(), + ); } public function test_project_drops_the_columns_the_schema_does_not_declare(): void { static::assertSame( [['id' => 1], ['id' => 2]], - rows( + array_to_rows( + [['id' => 1, 'name' => 'a'], ['id' => 2, 'name' => 'b']], schema(int_schema('id'), str_schema('name')), - row(['id' => 1, 'name' => 'a']), - row(['id' => 2, 'name' => 'b']), ) - ->project(schema(int_schema('id'))) + ->project(schema(int_schema('id')), new AdaptiveBackend()) ->toArray(), ); + static::assertSame( + ['name' => 'a', 'id' => 1], + array_to_rows([['id' => 1, 'name' => 'a']], schema(int_schema('id'), str_schema('name'))) + ->project(schema(str_schema('name'), int_schema('id')), new AdaptiveBackend()) + ->values(0), + ); } public function test_project_rejects_a_schema_that_widens_the_batch(): void @@ -181,7 +121,10 @@ public function test_project_rejects_a_schema_that_widens_the_batch(): void $this->expectException(SchemaMismatchException::class); $this->expectExceptionMessage('column "name" (row 0) declared by the schema is missing from the row'); - rows(schema(int_schema('id')), row(['id' => 1]))->project(schema(int_schema('id'), str_schema('name'))); + array_to_rows([['id' => 1]], schema(int_schema('id')))->project( + schema(int_schema('id'), str_schema('name')), + new AdaptiveBackend(), + ); } public function test_project_rejects_a_schema_that_retypes_a_column(): void @@ -189,50 +132,20 @@ public function test_project_rejects_a_schema_that_retypes_a_column(): void $this->expectException(SchemaMismatchException::class); $this->expectExceptionMessage('column "id" (row 0): could not convert 1 (integer) to string'); - rows(schema(int_schema('id'), str_schema('name')), row(['id' => 1, 'name' => 'a']))->project(schema(str_schema( - 'id', - ))); - } - - public function test_project_does_not_recheck_columns_that_keep_their_definition(): void - { - // the rows passed a door under these definitions, and a subset of them needs no second check - static::assertSame( - [['id' => 'not-an-integer']], - Rows::trusted(schema(int_schema('id'), str_schema('name')), [row([ - 'id' => 'not-an-integer', - 'name' => 'a', - ])]) - ->project(schema(int_schema('id'))) - ->toArray(), + array_to_rows([['id' => 1, 'name' => 'a']], schema(int_schema('id'), str_schema('name')))->project( + schema(str_schema('id')), + new AdaptiveBackend(), ); } - public function test_conformed_pads_the_shape_without_revalidating_values(): void - { - static::assertSame( - [['id' => 'not-an-integer', 'name' => null]], - Rows::conformed(schema(int_schema('id'), str_schema('name', true)), [row([ - 'id' => 'not-an-integer', - ])])->toArray(), - ); - } - - public function test_conformed_reports_the_violating_row_at_its_position_in_the_batch(): void - { - $this->expectException(SchemaMismatchException::class); - $this->expectExceptionMessage('column "id" (row 1)'); - - Rows::conformed(schema(int_schema('id')), [row(['id' => 1]), row(['id' => null])]); - } - public function test_construct_reorders_row_storage_into_schema_order(): void { static::assertSame( - ['c', 'a', 'b'], - rows(schema(int_schema('c'), int_schema('a'), int_schema('b')), row(['a' => 1, 'b' => 2, 'c' => 3])) - ->first() - ->names(), + ['c' => 3, 'a' => 1, 'b' => 2], + array_to_rows( + [['a' => 1, 'b' => 2, 'c' => 3]], + schema(int_schema('c'), int_schema('a'), int_schema('b')), + )->values(0), ); } @@ -241,84 +154,20 @@ public function test_construct_rejects_a_row_that_contradicts_the_schema(): void $this->expectException(SchemaMismatchException::class); $this->expectExceptionMessage('column "number" (row 0): could not convert \'x\' (string) to integer'); - rows(schema(int_schema('number')), row(['number' => 'x'])); - } - - public function test_the_epic_repro_is_unconstructible(): void - { - $this->expectException(SchemaMismatchException::class); - $this->expectExceptionMessage( - 'Rows do not match their schema: column "code" (row 1): could not convert 1000 (integer) to string', - ); - - rows(schema(str_schema('code')), row(['code' => 'AB-01']), row(['code' => 1000])); - } - - public function test_trusted_takes_a_matching_batch_verbatim(): void - { - static::assertSame([['number' => 1], ['number' => 2]], Rows::trusted(schema(int_schema('number')), [ - row(['number' => 1]), - row(['number' => 2]), - ])->toArray()); - } - - public function test_unserialize_rejects_a_payload_whose_rows_contradict_its_schema(): void - { - // a persisted payload is foreign input - cache reads and both serializers land here - $this->expectException(SchemaMismatchException::class); - $this->expectExceptionMessage('column "number" (row 0) declared by the schema is missing from the row'); - - rows(schema(int_schema('number')))->__unserialize([ - 'schema' => schema(int_schema('number')), - 'rows' => [['other' => 1]], - ]); - } - - public function test_array_access_exists(): void - { - $rows = rows(schema(int_schema('id')), row(['id' => 1]), row(['id' => 2]), row(['id' => 3])); - - static::assertTrue(isset($rows[0])); - static::assertFalse(isset($rows[3])); - } - - public function test_array_access_get(): void - { - $rows = rows(schema(int_schema('id')), row(['id' => 1]), row(['id' => 2]), row(['id' => 3])); - - static::assertSame(1, $rows[0]->get('id')); - static::assertSame(2, $rows[1]->get('id')); - static::assertSame(3, $rows[2]->get('id')); - } - - public function test_array_access_set(): void - { - $this->expectException(RuntimeException::class); - $this->expectExceptionMessage('In order to add new rows use Rows::add(Row $row) : self'); - $rows = rows(schema()); - $rows[0] = row(['id' => 1]); - } - - public function test_array_access_unset(): void - { - $this->expectException(RuntimeException::class); - $this->expectExceptionMessage('In order to remove rows use Rows::remove(int $offset) : self'); - $rows = rows(schema(int_schema('id')), row(['id' => 1])); - unset($rows[0]); + array_to_rows([['number' => 'x']], schema(int_schema('number'))); } public function test_chunks_with_less(): void { - $rows = rows( - schema(int_schema('id')), - row(['id' => 1]), - row(['id' => 2]), - row(['id' => 3]), - row(['id' => 4]), - row(['id' => 5]), - row(['id' => 6]), - row(['id' => 7]), - ); + $rows = array_to_rows([ + ['id' => 1], + ['id' => 2], + ['id' => 3], + ['id' => 4], + ['id' => 5], + ['id' => 6], + ['id' => 7], + ], schema(int_schema('id'))); $chunk = iterator_to_array($rows->chunks(10)); @@ -328,19 +177,18 @@ public function test_chunks_with_less(): void public function test_chunks_with_more_than_expected_in_chunk_rows(): void { - $rows = rows( - schema(int_schema('id')), - row(['id' => 1]), - row(['id' => 2]), - row(['id' => 3]), - row(['id' => 4]), - row(['id' => 5]), - row(['id' => 6]), - row(['id' => 7]), - row(['id' => 8]), - row(['id' => 9]), - row(['id' => 10]), - ); + $rows = array_to_rows([ + ['id' => 1], + ['id' => 2], + ['id' => 3], + ['id' => 4], + ['id' => 5], + ['id' => 6], + ['id' => 7], + ['id' => 8], + ['id' => 9], + ['id' => 10], + ], schema(int_schema('id'))); $chunk = iterator_to_array($rows->chunks(5)); @@ -351,18 +199,18 @@ public function test_chunks_with_more_than_expected_in_chunk_rows(): void public function test_drop(): void { - $rows = rows(schema(int_schema('id')), row(['id' => 1]), row(['id' => 2]), row(['id' => 3])); + $rows = array_to_rows([['id' => 1], ['id' => 2], ['id' => 3]], schema(int_schema('id'))); $rows = $rows->drop(1); static::assertCount(2, $rows); - static::assertSame(2, $rows[0]->get('id')); - static::assertSame(3, $rows[1]->get('id')); + static::assertSame(2, $rows->column('id')->value(0)); + static::assertSame(3, $rows->column('id')->value(1)); } public function test_drop_all(): void { - $rows = rows(schema(int_schema('id')), row(['id' => 1]), row(['id' => 2]), row(['id' => 3])); + $rows = array_to_rows([['id' => 1], ['id' => 2], ['id' => 3]], schema(int_schema('id'))); $rows = $rows->drop(3); @@ -371,7 +219,7 @@ public function test_drop_all(): void public function test_drop_more_than_exists(): void { - $rows = rows(schema(int_schema('id')), row(['id' => 1]), row(['id' => 2]), row(['id' => 3])); + $rows = array_to_rows([['id' => 1], ['id' => 2], ['id' => 3]], schema(int_schema('id'))); $rows = $rows->drop(4); @@ -380,18 +228,18 @@ public function test_drop_more_than_exists(): void public function test_drop_right(): void { - $rows = rows(schema(int_schema('id')), row(['id' => 1]), row(['id' => 2]), row(['id' => 3])); + $rows = array_to_rows([['id' => 1], ['id' => 2], ['id' => 3]], schema(int_schema('id'))); $rows = $rows->dropRight(1); static::assertCount(2, $rows); - static::assertSame(1, $rows[0]->get('id')); - static::assertSame(2, $rows[1]->get('id')); + static::assertSame(1, $rows->column('id')->value(0)); + static::assertSame(2, $rows->column('id')->value(1)); } public function test_drop_right_all(): void { - $rows = rows(schema(int_schema('id')), row(['id' => 1]), row(['id' => 2]), row(['id' => 3])); + $rows = array_to_rows([['id' => 1], ['id' => 2], ['id' => 3]], schema(int_schema('id'))); $rows = $rows->dropRight(3); @@ -400,7 +248,7 @@ public function test_drop_right_all(): void public function test_drop_right_more_than_available(): void { - $rows = rows(schema(int_schema('id')), row(['id' => 1]), row(['id' => 2]), row(['id' => 3])); + $rows = array_to_rows([['id' => 1], ['id' => 2], ['id' => 3]], schema(int_schema('id'))); $rows = $rows->dropRight(5); @@ -409,7 +257,7 @@ public function test_drop_right_more_than_available(): void public function test_drop_right_more_than_exists(): void { - $rows = rows(schema(int_schema('id')), row(['id' => 1]), row(['id' => 2]), row(['id' => 3])); + $rows = array_to_rows([['id' => 1], ['id' => 2], ['id' => 3]], schema(int_schema('id'))); $rows = $rows->dropRight(4); @@ -418,688 +266,998 @@ public function test_drop_right_more_than_exists(): void public function test_empty_rows(): void { - static::assertTrue(rows(schema())->empty()); - static::assertFalse(rows(schema(int_schema('id')), row(['id' => 1]))->empty()); + static::assertTrue(rows(schema())->isEmpty()); + static::assertFalse(array_to_rows([['id' => 1]], schema(int_schema('id')))->isEmpty()); } - public function test_first_on_empty_rows(): void + public function test_concat_throws_when_schemas_disagree(): void { - $this->expectException(RuntimeException::class); + $this->expectException(InvalidArgumentException::class); + $this->expectExceptionMessage( + 'Cannot concat Rows with different schemas: [id: integer] and [id: integer, name: string]', + ); - rows(schema())->first(); + array_to_rows([['id' => 1]], schema(int_schema('id')))->concat( + new PhpBackend(), + array_to_rows([['id' => 2, 'name' => 'x']], schema(int_schema('id'), str_schema('name'))), + ); } - public function test_hash(): void + public function test_concat_joins_batches_in_order(): void { - $rows = rows( - schema(int_schema('id'), bool_schema('bool')), - row(['id' => 1, 'bool' => false]), - row(['id' => 2, 'bool' => false]), - row(['id' => 3, 'bool' => false]), - row(['id' => 4, 'bool' => false]), - ); + $rowsOne = array_to_rows([['id' => 1], ['id' => 2]], schema(int_schema('id'))); + $rowsTwo = array_to_rows([['id' => 3], ['id' => 4], ['id' => 5]], schema(int_schema('id'))); - static::assertSame( - $rows->hash(), - rows( - schema(bool_schema('bool'), int_schema('id')), - row(['bool' => false, 'id' => 1]), - row(['bool' => false, 'id' => 2]), - row(['bool' => false, 'id' => 3]), - row(['bool' => false, 'id' => 4]), - )->hash(), + $rowsThree = array_to_rows([['id' => 6], ['id' => 7]], schema(int_schema('id'))); + + $merged = $rowsOne->concat(new PhpBackend(), $rowsTwo)->concat(new PhpBackend(), $rowsThree); + + static::assertEquals( + array_to_rows([ + ['id' => 1], + ['id' => 2], + ['id' => 3], + ['id' => 4], + ['id' => 5], + ['id' => 6], + ['id' => 7], + ], schema(int_schema('id'))), + $merged, ); } - public function test_hash_empty_rows(): void + public function test_remove(): void { - static::assertSame(rows(schema())->hash(), rows(schema())->hash()); + $rows = array_to_rows([['id' => 1], ['id' => 2], ['id' => 3]], schema(int_schema('id'))); + + $rows = $rows->remove(1); + + static::assertCount(2, $rows); + static::assertSame(1, $rows->column('id')->value(0)); + static::assertSame(3, $rows->column('id')->value(1)); } - public function test_hash_rows_with_different_columns(): void + public function test_remove_on_empty_rows(): void { - $rows = rows( - schema(int_schema('id'), bool_schema('bool')), - row(['id' => 1, 'bool' => false]), - row(['id' => 3, 'bool' => false]), - row(['id' => 2, 'bool' => false]), - row(['id' => 4, 'bool' => false]), - ); + $this->expectException(InvalidArgumentException::class); - static::assertNotSame( - $rows->hash(), - rows( - schema(bool_schema('bool')), - row(['bool' => false]), - row(['bool' => false]), - row(['bool' => false]), - row(['bool' => false]), - )->hash(), - ); + rows(schema())->remove(1); } - public function test_hash_rows_with_different_order(): void + public function test_rows_schema(): void { - $rows = rows( - schema(int_schema('id'), bool_schema('bool')), - row(['id' => 1, 'bool' => false]), - row(['id' => 3, 'bool' => false]), - row(['id' => 2, 'bool' => false]), - row(['id' => 4, 'bool' => false]), + $rows = array_to_rows( + [ + ['id' => 1, 'name' => 'foo'], + ['id' => 1, 'name' => null, 'list' => [1, 2]], + ['id' => 1, 'name' => 'bar', 'tags' => ['a', 'b']], + ['id' => 1, 'name' => 'baz'], + ], + schema( + int_schema('id'), + str_schema('name', nullable: true), + list_schema('list', type_list(type_integer()), nullable: true), + list_schema('tags', type_list(type_string()), nullable: true), + ), ); - static::assertNotSame( - $rows->hash(), - rows( - schema(bool_schema('bool'), int_schema('id')), - row(['bool' => false, 'id' => 1]), - row(['bool' => false, 'id' => 2]), - row(['bool' => false, 'id' => 3]), - row(['bool' => false, 'id' => 4]), - )->hash(), + static::assertEquals( + schema( + int_schema('id'), + str_schema('name', nullable: true), + list_schema('list', type_list(type_integer()), nullable: true), + list_schema('tags', type_list(type_string()), nullable: true), + ), + $rows->schema(), ); } - public function test_head(): void + public function test_rows_reject_a_row_whose_list_contradicts_the_declared_element_type(): void { - $rows = rows( - schema(int_schema('id')), - row(['id' => 1]), - row(['id' => 2]), - row(['id' => 3]), - row(['id' => 4]), - row(['id' => 5]), + $this->expectException(SchemaMismatchException::class); + $this->expectExceptionMessage( + 'Rows do not match their schema: column "list" (row 1): could not convert array (list) to list', ); - $head = $rows->head(3); - - static::assertCount(3, $head); - static::assertSame(1, $head[0]->get('id')); - static::assertSame(2, $head[1]->get('id')); - static::assertSame(3, $head[2]->get('id')); + array_to_rows([ + ['list' => []], + ['list' => [1, 2]], + ], schema(list_schema('list', type_list(type_integer()))))->matchTo( + schema(list_schema('list', type_list(type_string()))), + new AdaptiveBackend(), + ); } - public function test_head_on_empty_rows(): void + public function test_rows_serialization(): void { - $head = rows(schema())->head(5); + $rows = array_to_rows([['id' => 1], ['id' => 2], ['id' => 3]], schema(int_schema('id'))); - static::assertCount(0, $head); - } - - public function test_head_with_count_larger_than_available(): void - { - $rows = rows(schema(int_schema('id')), row(['id' => 1]), row(['id' => 2]), row(['id' => 3])); + $serialized = serialize($rows); - $head = $rows->head(10); + /** @var Rows $unserialized */ + $unserialized = unserialize($serialized); - static::assertCount(3, $head); - static::assertSame(1, $head[0]->get('id')); - static::assertSame(2, $head[1]->get('id')); - static::assertSame(3, $head[2]->get('id')); + static::assertSame($rows->toArray(), $unserialized->toArray()); + static::assertTrue($rows->schema()->isSame($unserialized->schema())); } - public function test_head_with_negative_count(): void + public function test_sort_rows_by_not_existing_column(): void { $this->expectException(InvalidArgumentException::class); - $this->expectExceptionMessage('Count must be greater than or equal to 0'); + $this->expectExceptionMessage('Column "c" does not exist'); - $rows = rows(schema(int_schema('id')), row(['id' => 1]), row(['id' => 2]), row(['id' => 3])); + $rows = array_to_rows( + [ + ['a' => 3, 'b' => 2], + ['a' => 1, 'b' => 5], + ['a' => 1, 'b' => 4], + ['a' => 2, 'b' => 7], + ['a' => 3, 'b' => 10], + ['a' => 2, 'b' => 4], + ], + schema(int_schema('a'), int_schema('b')), + ); - $rows->head(-1); + $rows->sortBy(ref('c'), ref('b')->desc()); } - public function test_head_with_zero_count(): void + public function test_sort_rows_by_two_columns(): void { - $rows = rows(schema(int_schema('id')), row(['id' => 1]), row(['id' => 2]), row(['id' => 3])); + $rows = array_to_rows( + [ + ['a' => 3, 'b' => 2], + ['a' => 1, 'b' => 5], + ['a' => 1, 'b' => 4], + ['a' => 2, 'b' => 7], + ['a' => 3, 'b' => 10], + ['a' => 2, 'b' => 4], + ], + schema(int_schema('a'), int_schema('b')), + ); - $head = $rows->head(0); + $ascending = $rows->sortBy(ref('a'), ref('b')->desc()); + $descending = $rows->sortBy(ref('a')->desc(), ref('b')); - static::assertCount(0, $head); + static::assertSame( + [ + ['a' => 1, 'b' => 5], + ['a' => 1, 'b' => 4], + ['a' => 2, 'b' => 7], + ['a' => 2, 'b' => 4], + ['a' => 3, 'b' => 10], + ['a' => 3, 'b' => 2], + ], + $ascending->toArray(), + ); + static::assertSame( + [ + ['a' => 3, 'b' => 2], + ['a' => 3, 'b' => 10], + ['a' => 2, 'b' => 4], + ['a' => 2, 'b' => 7], + ['a' => 1, 'b' => 4], + ['a' => 1, 'b' => 5], + ], + $descending->toArray(), + ); } - public function test_last(): void + public function test_sort_rows_without_changing_original_collection(): void { - $rows = rows(schema(int_schema('id')), row(['id' => 1]), row(['id' => 2]), row(['id' => 3])); + $rows = array_to_rows( + [ + ['number' => 3, 'name' => 'three'], + ['number' => 1, 'name' => 'one'], + ['number' => 5, 'name' => 'five'], + ['number' => 2, 'name' => 'two'], + ['number' => 4, 'name' => 'four'], + ], + schema(int_schema('number'), str_schema('name')), + ); - $lastRow = $rows->last(); + $ascending = $rows->sortBy(ref('number')); + $descending = $rows->sortBy(ref('number')->desc()); - static::assertNotNull($lastRow); - static::assertSame(3, $lastRow->get('id')); + static::assertSame([1, 2, 3, 4, 5], $ascending->column('number')->values()); + static::assertSame([5, 4, 3, 2, 1], $descending->column('number')->values()); + static::assertSame([3, 1, 5, 2, 4], $rows->column('number')->values()); } - public function test_last_on_empty_rows(): void + public function test_take(): void { - $lastRow = rows(schema())->last(); + $rows = array_to_rows([['id' => 1], ['id' => 2], ['id' => 3]], schema(int_schema('id'))); + + $rows = $rows->take(1); - static::assertNull($lastRow); + static::assertCount(1, $rows); + static::assertSame(1, $rows->column('id')->value(0)); } - public function test_last_on_single_row(): void + public function test_take_all(): void { - $rows = rows(schema(int_schema('id')), row(['id' => 42])); + $rows = array_to_rows([['id' => 1], ['id' => 2], ['id' => 3]], schema(int_schema('id'))); - $lastRow = $rows->last(); + $rows = $rows->take(3); - static::assertNotNull($lastRow); - static::assertSame(42, $lastRow->get('id')); + static::assertCount(3, $rows); + static::assertSame(1, $rows->column('id')->value(0)); + static::assertSame(2, $rows->column('id')->value(1)); + static::assertSame(3, $rows->column('id')->value(2)); } - public function test_merge_accepts_an_empty_side_with_any_schema(): void + public function test_take_more_than_exists(): void { - $rows = rows(schema(int_schema('id')), row(['id' => 1])); + $rows = array_to_rows([['id' => 1], ['id' => 2], ['id' => 3]], schema(int_schema('id'))); + + $rows = $rows->take(4); - // an empty batch has nothing to disagree about, so it short-circuits before the schema check - static::assertEquals($rows, $rows->merge(rows(schema(str_schema('unrelated'))))); - static::assertEquals($rows, rows(schema(str_schema('unrelated')))->merge($rows)); + static::assertCount(3, $rows); + static::assertSame(1, $rows->column('id')->value(0)); + static::assertSame(2, $rows->column('id')->value(1)); + static::assertSame(3, $rows->column('id')->value(2)); } - public function test_merge_throws_when_schemas_disagree(): void + public function test_transforms_rows_to_array(): void { - $this->expectException(InvalidArgumentException::class); - $this->expectExceptionMessage( - 'Cannot merge Rows with different schemas: [id: integer] and [id: integer, name: string]', + $rows = array_to_rows( + [ + ['id' => 1234, 'deleted' => false, 'phase' => null], + ['id' => 4321, 'deleted' => true, 'phase' => 'launch'], + ], + schema(int_schema('id'), bool_schema('deleted'), string_schema('phase', nullable: true)), ); - rows(schema(int_schema('id')), row(['id' => 1]))->merge(rows( - schema(int_schema('id'), str_schema('name')), - row(['id' => 2, 'name' => 'x']), - )); + static::assertEquals( + [ + ['id' => 1234, 'deleted' => false, 'phase' => null], + ['id' => 4321, 'deleted' => true, 'phase' => 'launch'], + ], + $rows->toArray(), + ); } - public function test_merges_collection_together(): void + public function test_transforms_rows_to_array_without_keys(): void { - $rowsOne = rows(schema(int_schema('id')), row(['id' => 1]), row(['id' => 2])); - $rowsTwo = rows(schema(int_schema('id')), row(['id' => 3]), row(['id' => 4]), row(['id' => 5])); + $rows = array_to_rows( + [ + ['id' => 1234, 'deleted' => false, 'phase' => null], + ['id' => 4321, 'deleted' => true, 'phase' => 'launch'], + ], + schema(int_schema('id'), bool_schema('deleted'), string_schema('phase', nullable: true)), + ); - $rowsThree = rows(schema(int_schema('id')), row(['id' => 6]), row(['id' => 7])); + static::assertEquals( + [ + [1234, false, null], + [4321, true, 'launch'], + ], + $rows->toArray(withKeys: false), + ); + } - $merged = $rowsOne->merge($rowsTwo)->merge($rowsThree); + public function test_values_of_a_row_are_keyed_in_schema_order(): void + { + $createdAt = new DateTimeImmutable('2020-07-13 15:00'); static::assertEquals( - rows( - schema(int_schema('id')), - row(['id' => 1]), - row(['id' => 2]), - row(['id' => 3]), - row(['id' => 4]), - row(['id' => 5]), - row(['id' => 6]), - row(['id' => 7]), - ), - $merged, + [ + 'id' => 1234, + 'deleted' => false, + 'created-at' => $createdAt, + 'phase' => null, + 'items' => ['item-id' => 1, 'name' => 'one'], + 'statuses' => [1 => 'NEW'], + ], + array_to_rows( + [ + [ + 'id' => 1, + 'deleted' => true, + 'created-at' => $createdAt, + 'phase' => 'x', + 'items' => [ + 'item-id' => 0, + 'name' => 'zero', + ], + 'statuses' => [], + ], + [ + 'statuses' => [1 => 'NEW'], + 'phase' => null, + 'items' => [ + 'item-id' => 1, + 'name' => 'one', + ], + 'id' => 1234, + 'created-at' => $createdAt, + 'deleted' => false, + ], + ], + schema( + int_schema('id'), + bool_schema('deleted'), + datetime_schema('created-at'), + str_schema('phase', nullable: true), + structure_schema('items', type_structure(['item-id' => type_integer(), 'name' => type_string()])), + map_schema('statuses', type_map(type_integer(), type_string())), + ), + )->values(1), ); } - public function test_offset_exists_with_non_int_offset(): void + #[TestWith([-1])] + #[TestWith([1])] + public function test_values_outside_the_batch_are_refused(int $index): void { $this->expectException(InvalidArgumentException::class); + $this->expectExceptionMessage("Row {$index} does not exist in a batch of 1 rows"); - // @mago-ignore analysis:invalid-argument - rows(schema())->offsetExists('a'); + array_to_rows([['id' => 1]], schema(int_schema('id')))->values($index); } - public function test_offset_get_on_empty_rows(): void + public function test_an_absent_column_is_refused(): void { $this->expectException(InvalidArgumentException::class); + $this->expectExceptionMessage('Column "missing" does not exist'); - rows(schema())[5]; + array_to_rows([['id' => 1]], schema(int_schema('id')))->column('missing'); } - public function test_remove(): void + public function test_structure_field_order_is_a_schema_difference_not_a_value_difference(): void { - $rows = rows(schema(int_schema('id')), row(['id' => 1]), row(['id' => 2]), row(['id' => 3])); - - $rows = $rows->remove(1); + $left = array_to_rows([['s' => [ + 'a' => 1, + 'b' => 'x', + ]]], schema(structure_schema('s', type_structure([ + 'a' => type_integer(), + 'b' => type_string(), + ])))); + $right = array_to_rows([['s' => [ + 'b' => 'x', + 'a' => 1, + ]]], schema(structure_schema('s', type_structure([ + 'b' => type_string(), + 'a' => type_integer(), + ])))); + + // the field-order difference lives in the Schema, which 04c made positional, so R6 + // refuses the merge rather than emitting a batch whose declared order no row agrees with. + $this->expectException(InvalidArgumentException::class); + $this->expectExceptionMessage('Cannot concat Rows with different schemas'); - static::assertCount(2, $rows); - static::assertSame(1, $rows[0]->get('id')); - static::assertSame(3, $rows[1]->get('id')); + $left->concat(new PhpBackend(), $right); } - public function test_remove_on_empty_rows(): void + public function test_rows_hop_a_foreign_zone_at_ingest(): void { - $this->expectException(InvalidArgumentException::class); - - rows(schema())->remove(1); + static::assertSame( + '2026-01-01 22:04:05 UTC', + type_datetime() + ->assert( + array_to_rows([[ + 'at' => new DateTimeImmutable('2026-01-02 03:04:05+05:00'), + ]], schema(datetime_schema('at'))) + ->column('at') + ->value(0), + ) + ->format('Y-m-d H:i:s e'), + ); } - public function test_returns_first_row(): void + public function test_rows_hop_nested_datetimes(): void { - $rows = rows( - schema(int_schema('number'), str_schema('name')), - $first = row(['number' => 3, 'name' => 'three']), - row(['number' => 1, 'name' => 'one']), - row(['number' => 2, 'name' => 'two']), + static::assertSame( + '2026-01-02 04:04:05 Europe/Warsaw', + type_list(type_datetime()) + ->assert( + array_to_rows([['l' => [new DateTimeImmutable( + '2026-01-02 03:04:05', + new DateTimeZone('UTC'), + )]]], schema(list_schema('l', type_list(type_datetime('Europe/Warsaw'))))) + ->column('l') + ->value(0), + )[0]->format('Y-m-d H:i:s e'), ); + } - static::assertEquals($first, $rows->first()); + public function test_rows_hop_nested_optional_datetimes(): void + { + static::assertSame( + '2026-01-02 04:04:05 Europe/Warsaw', + type_list(type_optional(type_datetime())) + ->assert( + array_to_rows([['l' => [ + new DateTimeImmutable('2026-01-02 03:04:05', new DateTimeZone('UTC')), + null, + ]]], schema(list_schema('l', type_list(type_optional(type_datetime('Europe/Warsaw')))))) + ->column('l') + ->value(0), + )[0]?->format('Y-m-d H:i:s e'), + ); } - public function test_reverse(): void + /** + * @return Generator + */ + public static function match_to_refusals(): Generator { - $rows = rows(schema(int_schema('id')), row(['id' => 1]), row(['id' => 2]), row(['id' => 3])); + yield 'other column, same count' => [ + schema(int_schema('a'), int_schema('b')), + array_to_rows([['a' => 1, 'z' => 2]], schema(int_schema('a'), int_schema('z'))), + 'Rows do not match their schema: column "z" (row 0) is not declared by the schema', + ]; + yield 'null under not-null' => [ + schema(str_schema('a')), + array_to_rows([['a' => null]], schema(int_schema('a', nullable: true))), + 'column "a" (row 0): could not convert null to string, column is not nullable', + ]; + yield 'missing not-null column' => [ + schema(int_schema('a'), int_schema('b')), + array_to_rows([['a' => 1]], schema(int_schema('a'))), + 'Rows do not match their schema: column "b" (row 0) declared by the schema is missing from the row', + ]; + yield 'unknown column' => [ + schema(int_schema('a')), + array_to_rows([['a' => 1, 'b' => 2]], schema(int_schema('a'), int_schema('b'))), + 'Rows do not match their schema: column "b" (row 0) is not declared by the schema', + ]; + yield 'any column against schema()' => [ + schema(), + array_to_rows([['a' => 1]], schema(int_schema('a'))), + 'Rows do not match their schema: column "a" (row 0) is not declared by the schema', + ]; + yield 'same names, third value wrong' => [ + schema(int_schema('a'), str_schema('b'), str_schema('c')), + array_to_rows( + [['a' => 1, 'b' => 'x', 'c' => 3]], + schema(int_schema('a'), str_schema('b'), int_schema('c')), + ), + 'Rows do not match their schema: column "c" (row 0): could not convert 3 (integer) to string', + ]; + yield 'reordered, wrong type' => [ + schema(int_schema('a'), int_schema('b')), + array_to_rows([['b' => 'x', 'a' => 1]], schema(str_schema('b'), int_schema('a'))), + 'Rows do not match their schema: column "b" (row 0): could not convert \'x\' (string) to integer', + ]; + yield 'null under not-null datetime' => [ + schema(datetime_schema('at')), + array_to_rows([['at' => null]], schema(str_schema('at', nullable: true))), + 'column "at" (row 0): could not convert null to datetime, column is not nullable', + ]; + yield 'string in a datetime column' => [ + schema(datetime_schema('at')), + array_to_rows([['at' => '2026-01-02 03:04:05']], schema(str_schema('at'))), + 'column "at" (row 0): could not convert \'2026-01-02 03:04:05\' (string) to datetime', + ]; + } - $rows = $rows->reverse(); + #[DataProvider('match_to_refusals')] + public function test_match_to_refuses(Schema $schema, Rows $rows, string $message): void + { + $this->expectException(SchemaMismatchException::class); + $this->expectExceptionMessage($message); - static::assertCount(3, $rows); - static::assertSame(3, $rows[0]->get('id')); - static::assertSame(2, $rows[1]->get('id')); - static::assertSame(1, $rows[2]->get('id')); + $rows->matchTo($schema, new AdaptiveBackend()); } - #[DataProvider('rows_diff_left_provider')] - public function test_rows_diff_left(Rows $expected, Rows $left, Rows $right): void + public function test_from_rows_refuses_null_under_not_null_in_row_1(): void { - static::assertEquals($expected->toArray(), $left->diffLeft($right)->toArray()); + $this->expectException(SchemaMismatchException::class); + $this->expectExceptionMessage('column "id" (row 1)'); + + array_to_rows([['id' => 1], ['id' => null]], schema(int_schema('id'))); } - #[DataProvider('rows_diff_right_provider')] - public function test_rows_diff_right(Rows $expected, Rows $left, Rows $right): void + public function test_from_rows_reorders_and_hops_into_the_column_zone(): void { - static::assertEquals($expected->toArray(), $left->diffRight($right)->toArray()); + static::assertSame( + '2026-01-02 04:04:05 Europe/Warsaw', + type_datetime() + ->assert( + array_to_rows( + [['at' => new DateTimeImmutable('2026-01-02 03:04:05', new DateTimeZone('UTC')), 'id' => 1]], + schema(int_schema('id'), datetime_schema('at', zone: 'Europe/Warsaw')), + ) + ->column('at') + ->value(0), + ) + ->format('Y-m-d H:i:s e'), + ); } - public function test_rows_diff_right_carries_the_right_sides_schema(): void + public function test_of_takes_columns_verbatim(): void { - // the surviving rows come from the right side, so the left side's schema cannot describe them - $left = rows(schema(int_schema('number')), row(['number' => 1])); - $right = rows(schema(int_schema('number'), str_schema('name')), row(['number' => 2, 'name' => 'two'])); + $schema = schema(int_schema('number')); + $column = (new PhpBackend())->constant(int_schema('number'), 1, 2); + $rows = Rows::fromColumns($schema, ['number' => $column], 2); - static::assertTrue($left->diffRight($right)->schema()->isSame($right->schema())); + static::assertSame($column, $rows->column('number')); + static::assertSame([['number' => 1], ['number' => 1]], $rows->toArray()); } - public function test_rows_schema(): void + public function test_of_checks_names_and_counts_only(): void { - $rows = rows( - schema( - int_schema('id'), - str_schema('name', nullable: true), - list_schema('list', type_list(type_integer()), nullable: true), - list_schema('tags', type_list(type_string()), nullable: true), - ), - row(['id' => 1, 'name' => 'foo']), - row(['id' => 1, 'name' => null, 'list' => [1, 2]]), - row(['id' => 1, 'name' => 'bar', 'tags' => ['a', 'b']]), - row(['id' => 1, 'name' => 'baz']), + // a column of another type passes the structural tier - values are checked where they enter a column + $rows = Rows::fromColumns( + schema(str_schema('id')), + ['id' => (new PhpBackend())->constant(int_schema('id'), 1, 1)], + 1, ); - static::assertEquals( - schema( - int_schema('id'), - str_schema('name', nullable: true), - list_schema('list', type_list(type_integer()), nullable: true), - list_schema('tags', type_list(type_string()), nullable: true), - ), - $rows->schema(), + static::assertSame(1, $rows->count()); + } + + public function test_of_refuses_other_column_names(): void + { + $this->expectException(InvalidArgumentException::class); + $this->expectExceptionMessage('Rows::fromColumns() expects columns [id], got [other]'); + + Rows::fromColumns( + schema(int_schema('id')), + ['other' => (new PhpBackend())->constant(int_schema('other'), 1, 1)], + 1, ); } - public function test_rows_reject_a_row_whose_list_contradicts_the_declared_element_type(): void + public function test_of_refuses_a_column_of_another_count(): void + { + $this->expectException(InvalidArgumentException::class); + $this->expectExceptionMessage('Column "id" holds 2 rows, the batch 1'); + + Rows::fromColumns(schema(int_schema('id')), ['id' => (new PhpBackend())->constant(int_schema('id'), 1, 2)], 1); + } + + public function test_of_refuses_a_null_in_a_not_null_column(): void { $this->expectException(SchemaMismatchException::class); $this->expectExceptionMessage( - 'Rows do not match their schema: column "list" (row 1): could not convert array (list) to list', + 'Rows do not match their schema: column "id" (row 1): could not convert null to integer, column is not nullable', ); - rows( - schema(list_schema('list', type_list(type_string()))), - row(['list' => ['one', 'two']]), - row(['list' => [1, 2]]), + Rows::fromColumns( + schema(int_schema('id')), + [ + 'id' => array_to_rows([['id' => 1], ['id' => null]], schema(int_schema('id', nullable: true)))->column( + 'id', + ), + ], + 2, ); } - public function test_rows_serialization(): void + public function test_project_shares_kept_columns(): void { - $rows = rows(schema(int_schema('id')), row(['id' => 1]), row(['id' => 2]), row(['id' => 3])); + $rows = array_to_rows([['id' => 1, 'name' => 'a']], schema(int_schema('id'), str_schema('name'))); - $serialized = serialize($rows); + static::assertSame( + $rows->column('id'), + $rows->project(schema(int_schema('id')), new AdaptiveBackend())->column('id'), + ); + } - /** @var Rows $unserialized */ - $unserialized = unserialize($serialized); + public function test_concat_of_mixed_column_classes_rebuilds_in_the_given_backend(): void + { + $schema = schema(int_schema('id')); + $constant = Rows::fromColumns($schema, ['id' => (new PhpBackend())->constant(int_schema('id'), 7, 2)], 2); + $concatenated = $constant->concat(new PhpBackend(), array_to_rows([['id' => 1]], $schema, new PhpBackend())); - static::assertSame($rows->toArray(), $unserialized->toArray()); - static::assertTrue($rows->schema()->isSame($unserialized->schema())); + static::assertSame([['id' => 7], ['id' => 7], ['id' => 1]], $concatenated->toArray()); + static::assertInstanceOf(ScalarColumn::class, $concatenated->column('id')); } - #[DataProvider('unique_rows_provider')] - public function test_rows_unique( - Rows $expected, - Rows $notUnique, - Comparator $comparator = new NativeComparator(), - ): void { - static::assertEquals($expected, $notUnique->unique($comparator)); + public function test_adopted_by_a_backend_that_owns_every_column_is_the_same_batch(): void + { + $rows = array_to_rows([['id' => 1]], schema(int_schema('id')), new PhpBackend()); + + static::assertSame($rows, $rows->adoptedBy(new PhpBackend())); } - public function test_sort_rows_by_not_existing_column(): void + #[RequiresPhpExtension('flow_php')] + public function test_adopted_by_another_backend_copies_every_column_into_it(): void { - $this->expectException(InvalidArgumentException::class); - $this->expectExceptionMessage('Column "c" does not exist'); - - $rows = rows( - schema(int_schema('a'), int_schema('b')), - row(['a' => 3, 'b' => 2]), - row(['a' => 1, 'b' => 5]), - row(['a' => 1, 'b' => 4]), - row(['a' => 2, 'b' => 7]), - row(['a' => 3, 'b' => 10]), - row(['a' => 2, 'b' => 4]), + $rows = array_to_rows( + [['id' => 1, 'name' => 'a']], + schema(int_schema('id'), str_schema('name')), + new PhpBackend(), ); - $rows->sortBy(ref('c'), ref('b')->desc()); + $adopted = $rows->adoptedBy(new RustBackend()); + + static::assertSame($rows->toArray(), $adopted->toArray()); + static::assertSame('Flow\ETL\Column\RustColumn', $adopted->column('id')::class); + static::assertSame('Flow\ETL\Column\RustColumn', $adopted->column('name')::class); } - public function test_sort_rows_by_two_columns(): void + #[RequiresPhpExtension('flow_php')] + public function test_concat_of_two_php_parts_builds_in_the_given_backend(): void { - $rows = rows( - schema(int_schema('a'), int_schema('b')), - row(['a' => 3, 'b' => 2]), - row(['a' => 1, 'b' => 5]), - row(['a' => 1, 'b' => 4]), - row(['a' => 2, 'b' => 7]), - row(['a' => 3, 'b' => 10]), - row(['a' => 2, 'b' => 4]), + $schema = schema(int_schema('id')); + $concatenated = array_to_rows([['id' => 1]], $schema, new PhpBackend())->concat( + new RustBackend(), + array_to_rows([['id' => 2]], $schema, new PhpBackend()), ); - $ascending = $rows->sortBy(ref('a'), ref('b')->desc()); - $descending = $rows->sortBy(ref('a')->desc(), ref('b')); + static::assertSame([1, 2], $concatenated->column('id')->values()); + static::assertSame('Flow\ETL\Column\RustColumn', $concatenated->column('id')::class); + } - static::assertSame( - [ - ['a' => 1, 'b' => 5], - ['a' => 1, 'b' => 4], - ['a' => 2, 'b' => 7], - ['a' => 2, 'b' => 4], - ['a' => 3, 'b' => 10], - ['a' => 3, 'b' => 2], - ], - $ascending->toArray(), - ); - static::assertSame( - [ - ['a' => 3, 'b' => 2], - ['a' => 3, 'b' => 10], - ['a' => 2, 'b' => 4], - ['a' => 2, 'b' => 7], - ['a' => 1, 'b' => 4], - ['a' => 1, 'b' => 5], - ], - $descending->toArray(), - ); + #[RequiresPhpExtension('flow_php')] + public function test_concat_with_no_rows_still_answers_in_the_given_backend(): void + { + $empty = Rows::empty(schema(int_schema('id')), new PhpBackend()); + + static::assertSame('Flow\ETL\Column\RustColumn', $empty->concat(new RustBackend())->column('id')::class); } - public function test_sort_rows_without_changing_original_collection(): void + #[RequiresPhpExtension('flow_php')] + public function test_concat_of_a_php_part_and_a_rust_part_builds_in_the_given_backend(): void { - $rows = rows( - schema(int_schema('number'), str_schema('name')), - $three = row(['number' => 3, 'name' => 'three']), - $one = row(['number' => 1, 'name' => 'one']), - $five = row(['number' => 5, 'name' => 'five']), - $two = row(['number' => 2, 'name' => 'two']), - $four = row(['number' => 4, 'name' => 'four']), + $schema = schema(int_schema('id')); + $concatenated = array_to_rows([['id' => 1]], $schema, new PhpBackend())->concat( + new RustBackend(), + array_to_rows([['id' => 2]], $schema, new RustBackend()), ); - $ascending = $rows->sortAscending(ref('number')); - $descending = $rows->sortDescending(ref('number')); + static::assertSame([1, 2], $concatenated->column('id')->values()); + static::assertSame('Flow\ETL\Column\RustColumn', $concatenated->column('id')::class); + } - static::assertEquals( - rows(schema(int_schema('number'), str_schema('name')), $one, $two, $three, $four, $five), - $ascending, - ); - static::assertEquals( - rows(schema(int_schema('number'), str_schema('name')), $five, $four, $three, $two, $one), - $descending, + public function test_empty_builds_its_columns_in_the_given_backend(): void + { + static::assertInstanceOf( + ScalarColumn::class, + Rows::empty(schema(int_schema('id')), new PhpBackend())->column('id'), ); - static::assertNotEquals($ascending, $rows); - static::assertNotEquals($descending, $rows); } - public function test_tail(): void + public function test_match_to_builds_a_new_column_in_the_given_backend(): void { - $rows = rows( - schema(int_schema('id')), - row(['id' => 1]), - row(['id' => 2]), - row(['id' => 3]), - row(['id' => 4]), - row(['id' => 5]), + $matched = array_to_rows([['id' => 1]], schema(int_schema('id')), new PhpBackend())->matchTo( + schema(int_schema('id'), str_schema('name', nullable: true)), + new PhpBackend(), ); - $tail = $rows->tail(3); - - static::assertCount(3, $tail); - static::assertSame(3, $tail[0]->get('id')); - static::assertSame(4, $tail[1]->get('id')); - static::assertSame(5, $tail[2]->get('id')); + static::assertInstanceOf(ConstantColumn::class, $matched->column('name')); } - public function test_tail_maintains_correct_order(): void + public function test_match_to_reports_the_first_null_under_not_null(): void { - $rows = rows( + $this->expectException(SchemaMismatchException::class); + $this->expectExceptionMessage('(row 2)'); + + array_to_rows([['id' => 1], ['id' => 2], ['id' => null]], schema(int_schema('id', nullable: true)))->matchTo( schema(int_schema('id')), - row(['id' => 1]), - row(['id' => 2]), - row(['id' => 3]), - row(['id' => 4]), - row(['id' => 5]), + new PhpBackend(), ); + } - $tail = $rows->tail(2); + public function test_match_to_restamps_a_provable_change_without_reading_values(): void + { + $spy = new SpyColumn((new PhpBackend())->constant(int_schema('id'), 1, 2)); + $matched = Rows::fromColumns(schema(int_schema('id')), ['id' => $spy], 2)->matchTo( + schema(int_schema('id', nullable: true)), + new PhpBackend(), + ); - static::assertCount(2, $tail); - static::assertSame(4, $tail[0]->get('id')); - static::assertSame(5, $tail[1]->get('id')); + static::assertTrue($matched->schema()->get('id')->isNullable()); + static::assertNotContains('value', $spy->calls->getArrayCopy()); + static::assertNotContains('values', $spy->calls->getArrayCopy()); } - public function test_tail_on_empty_rows(): void + public function test_match_to_refuses_an_unprovable_change_at_the_bad_row(): void { - $tail = rows(schema())->tail(5); + $this->expectException(SchemaMismatchException::class); + $this->expectExceptionMessage('column "id" (row 1)'); - static::assertCount(0, $tail); + array_to_rows([['id' => null], ['id' => 5]], schema(int_schema('id', nullable: true)))->matchTo( + schema(str_schema('id', nullable: true)), + new PhpBackend(), + ); } - public function test_tail_with_count_larger_than_available(): void + public function test_concat_refuses_a_zero_row_input_with_another_schema(): void { - $rows = rows(schema(int_schema('id')), row(['id' => 1]), row(['id' => 2]), row(['id' => 3])); - - $tail = $rows->tail(10); + $this->expectException(InvalidArgumentException::class); + $this->expectExceptionMessage('Cannot concat Rows with different schemas'); - static::assertCount(3, $tail); - static::assertSame(1, $tail[0]->get('id')); - static::assertSame(2, $tail[1]->get('id')); - static::assertSame(3, $tail[2]->get('id')); + array_to_rows([['id' => 1]], schema(int_schema('id')))->concat( + new PhpBackend(), + rows(schema(str_schema('id'))), + ); } - public function test_tail_with_negative_count(): void + public function test_concat_of_empty_inputs_is_an_empty_batch(): void { - $this->expectException(InvalidArgumentException::class); - $this->expectExceptionMessage('Count must be greater than or equal to 0'); + $schema = schema(int_schema('id')); - $rows = rows(schema(int_schema('id')), row(['id' => 1]), row(['id' => 2]), row(['id' => 3])); + static::assertSame(0, rows($schema)->concat(new PhpBackend(), rows($schema))->count()); + } - $rows->tail(-1); + public function test_with_columns_beside_a_constant_column(): void + { + $schema = schema(int_schema('id'), str_schema('tag'), int_schema('n')); + $rows = Rows::fromColumns( + schema(int_schema('id'), str_schema('tag')), + [ + 'id' => (new PhpBackend())->constant(int_schema('id'), 1, 2), + 'tag' => (new PhpBackend())->constant(str_schema('tag'), 'a', 2), + ], + 2, + ); + $builder = (new PhpBackend())->builder(str_schema('tag')); + $builder->appendMany(['x', 'y']); + $n = (new PhpBackend())->builder(int_schema('n')); + $n->appendMany([5, 6]); + + static::assertSame( + [['id' => 1, 'tag' => 'x', 'n' => 5], ['id' => 1, 'tag' => 'y', 'n' => 6]], + $rows->withColumns($schema, ['n' => $n->finish(), 'tag' => $builder->finish()])->toArray(), + ); } - public function test_tail_with_zero_count(): void + public function test_empty_has_one_empty_column_per_definition(): void { - $rows = rows(schema(int_schema('id')), row(['id' => 1]), row(['id' => 2]), row(['id' => 3])); + $rows = Rows::empty(schema(int_schema('id'), str_schema('name', nullable: true)), new AdaptiveBackend()); - $tail = $rows->tail(0); + static::assertSame(0, $rows->count()); + static::assertSame(['id', 'name'], array_keys($rows->columns())); + static::assertSame(0, $rows->column('name')->count()); + static::assertEquals(schema(int_schema('id'), str_schema('name', nullable: true)), $rows->schema()); + } - static::assertCount(0, $tail); + public function test_empty_of_an_empty_schema(): void + { + static::assertSame([], Rows::empty(schema(), new AdaptiveBackend())->columns()); } - public function test_take(): void + public function test_from_columns_refuses_an_untyped_column(): void { - $rows = rows(schema(int_schema('id')), row(['id' => 1]), row(['id' => 2]), row(['id' => 3])); + $this->expectException(ColumnMismatchException::class); + $this->expectExceptionMessage('column "id": mixed cannot be a batch column'); - $rows = $rows->take(1); + Rows::fromColumns(schema(int_schema('id')), ['id' => new ValueColumn([1])], 1); + } - static::assertCount(1, $rows); - static::assertSame(1, $rows[0]->get('id')); + public function test_with_columns_refuses_an_untyped_column(): void + { + $this->expectException(ColumnMismatchException::class); + $this->expectExceptionMessage('column "n": mixed cannot be a batch column'); + + array_to_rows([['id' => 1]], schema(int_schema('id')))->withColumns( + schema(int_schema('id'), int_schema('n')), + ['n' => new ValueColumn([1])], + ); } - public function test_take_all(): void + public function test_with_columns_refuses_a_column_of_another_count(): void { - $rows = rows(schema(int_schema('id')), row(['id' => 1]), row(['id' => 2]), row(['id' => 3])); + $this->expectException(InvalidArgumentException::class); + $this->expectExceptionMessage('Column "n" holds 1 rows, the batch 2'); - $rows = $rows->take(3); + array_to_rows([['id' => 1], ['id' => 2]], schema(int_schema('id')))->withColumns( + schema(int_schema('id'), int_schema('n')), + ['n' => (new PhpBackend())->constant(int_schema('n'), 1, 1)], + ); + } - static::assertCount(3, $rows); - static::assertSame(1, $rows[0]->get('id')); - static::assertSame(2, $rows[1]->get('id')); - static::assertSame(3, $rows[2]->get('id')); + public function test_with_schema_renames_and_retypes_positionally(): void + { + $renamed = array_to_rows([['id' => 1]], schema(int_schema('id', nullable: true)))->withSchema(schema(int_schema( + 'key', + ))); + + static::assertSame([['key' => 1]], $renamed->toArray()); } - public function test_take_more_than_exists(): void + public function test_with_schema_refuses_another_definition_count(): void { - $rows = rows(schema(int_schema('id')), row(['id' => 1]), row(['id' => 2]), row(['id' => 3])); + $this->expectException(InvalidArgumentException::class); + $this->expectExceptionMessage('Rows::withSchema() expects 1 definitions, got 2'); - $rows = $rows->take(4); + array_to_rows([['id' => 1]], schema(int_schema('id')))->withSchema(schema(int_schema('a'), int_schema('b'))); + } - static::assertCount(3, $rows); - static::assertSame(1, $rows[0]->get('id')); - static::assertSame(2, $rows[1]->get('id')); - static::assertSame(3, $rows[2]->get('id')); + public function test_with_schema_refuses_a_null_under_not_null_at_its_row(): void + { + try { + array_to_rows([ + ['id' => 1], + ['id' => null], + ], schema(int_schema('id', nullable: true)))->withSchema(schema(int_schema('key'))); + static::fail('a null under NOT NULL must be refused'); + } catch (SchemaMismatchException $e) { + static::assertSame(1, $e->rowIndex); + static::assertStringContainsString('column "key"', $e->getMessage()); + } } - public function test_take_right(): void + public function test_with_schema_refuses_another_column_kind(): void { - $rows = rows(schema(int_schema('id')), row(['id' => 1]), row(['id' => 2]), row(['id' => 3])); + $this->expectException(ColumnMismatchException::class); + $this->expectExceptionMessage( + 'column "id": cannot retype the column to string, integer and string are different column kinds', + ); - $rows = $rows->takeRight(1); + array_to_rows([['id' => 1]], schema(int_schema('id')))->withSchema(schema(str_schema('id'))); + } - static::assertCount(1, $rows); - static::assertSame(3, $rows[0]->get('id')); + /** + * @return Generator + */ + public static function slices_out_of_range(): Generator + { + $rows = array_to_rows([['id' => 1], ['id' => 2], ['id' => 3]], schema(int_schema('id'))); + + yield 'negative offset' => [$rows, -1, 1]; + yield 'negative length' => [$rows, 0, -1]; + yield 'past the end' => [$rows, 2, 2]; + yield 'past the end of a batch without columns' => [array_to_rows([[], [], []], schema()), 1, 3]; } - public function test_take_right_all(): void + #[DataProvider('slices_out_of_range')] + public function test_slice_refuses_out_of_range(Rows $rows, int $offset, int $length): void { - $rows = rows(schema(int_schema('id')), row(['id' => 1]), row(['id' => 2]), row(['id' => 3])); + $this->expectException(InvalidArgumentException::class); + $this->expectExceptionMessage(sprintf( + 'Rows::slice(%d, %d) is outside a batch of %d rows', + $offset, + $length, + $rows->count(), + )); - $rows = $rows->takeRight(3); + $rows->slice($offset, $length); + } - static::assertCount(3, $rows); - static::assertSame(3, $rows[0]->get('id')); - static::assertSame(2, $rows[1]->get('id')); - static::assertSame(1, $rows[2]->get('id')); + /** + * @return Generator + */ + public static function gathers_out_of_range(): Generator + { + $rows = array_to_rows([['id' => 1], ['id' => 2], ['id' => 3]], schema(int_schema('id'))); + + yield 'negative' => [$rows, -1]; + yield 'the count' => [$rows, 3]; + yield 'the count of a batch without columns' => [array_to_rows([[], [], []], schema()), 3]; } - public function test_take_right_more_than_exists(): void + #[DataProvider('gathers_out_of_range')] + public function test_gather_refuses_out_of_range(Rows $rows, int $index): void { - $rows = rows(schema(int_schema('id')), row(['id' => 1]), row(['id' => 2]), row(['id' => 3])); + $this->expectException(InvalidArgumentException::class); + $this->expectExceptionMessage(sprintf( + 'Rows::gather() index %d is outside a batch of %d rows', + $index, + $rows->count(), + )); - $rows = $rows->takeRight(4); + $rows->gather([$index]); + } - static::assertCount(3, $rows); - static::assertSame(3, $rows[0]->get('id')); - static::assertSame(2, $rows[1]->get('id')); - static::assertSame(1, $rows[2]->get('id')); + public function test_gather_nothing_is_an_empty_batch(): void + { + static::assertSame(0, array_to_rows([['id' => 1]], schema(int_schema('id')))->gather([])->count()); } - public function test_transforms_rows_to_array(): void + public function test_a_batch_without_columns_keeps_its_rows(): void { - $rows = rows( - schema(int_schema('id'), bool_schema('deleted'), string_schema('phase', nullable: true)), - row(['id' => 1234, 'deleted' => false, 'phase' => null]), - row(['id' => 4321, 'deleted' => true, 'phase' => 'launch']), - ); + $rows = array_to_rows([[], [], []], schema()); - static::assertEquals( - [ - ['id' => 1234, 'deleted' => false, 'phase' => null], - ['id' => 4321, 'deleted' => true, 'phase' => 'launch'], - ], - $rows->toArray(), - ); + static::assertSame(3, $rows->count()); + static::assertSame(2, $rows->slice(1, 2)->count()); + static::assertSame([[], []], $rows->gather([0, 2])->toArray()); } - public function test_transforms_rows_to_array_without_keys(): void + public function test_dropping_nothing_keeps_the_batch(): void { - $rows = rows( - schema(int_schema('id'), bool_schema('deleted'), string_schema('phase', nullable: true)), - row(['id' => 1234, 'deleted' => false, 'phase' => null]), - row(['id' => 4321, 'deleted' => true, 'phase' => 'launch']), - ); + $rows = array_to_rows([['id' => 1]], schema(int_schema('id'))); - static::assertEquals( - [ - [1234, false, null], - [4321, true, 'launch'], - ], - $rows->toArray(withKeys: false), - ); + static::assertSame($rows, $rows->drop(0)); + static::assertSame($rows, $rows->dropRight(0)); } - public function test_structure_field_order_is_a_schema_difference_not_a_value_difference(): void + public function test_an_empty_batch_sorts_to_itself_and_reduces_to_nothing(): void { - $left = rows( - schema(structure_schema('s', type_structure([ - 'a' => type_integer(), - 'b' => type_string(), - ]))), - row(['s' => ['a' => 1, 'b' => 'x']]), - ); - $right = rows( - schema(structure_schema('s', type_structure([ - 'b' => type_string(), - 'a' => type_integer(), - ]))), - row(['s' => ['b' => 'x', 'a' => 1]]), - ); + $rows = rows(schema(int_schema('id'))); - // Values are compared by content (ArrayComparison), so the two rows are equal ... - static::assertCount(0, $left->diffLeft($right)); - static::assertCount(0, $left->diffRight($right)); + static::assertSame($rows, $rows->sortBy(ref('id'))); + static::assertSame($rows, $rows->sortBy(ref('id')->desc())); + static::assertSame([], $rows->reduceToArray('id')); + } - // ... and the field-order difference lives in the Schema, which 04c made positional, so R6 - // refuses the merge rather than emitting a batch whose declared order no row agrees with. - $this->expectException(InvalidArgumentException::class); - $this->expectExceptionMessage('Cannot merge Rows with different schemas'); + public function test_match_to_rebuilds_a_column_its_type_cannot_restamp(): void + { + // the child of a nullable structure carries null under the null parent row, so narrowing ?int to int cannot be a + // restamp - the matching values are rebuilt under the new type instead + $matched = array_to_rows([ + ['s' => ['a' => 1]], + ['s' => null], + ], schema(structure_schema('s', type_structure(['a' => type_optional(type_integer())]), nullable: true)))->matchTo( + schema(structure_schema('s', type_structure(['a' => type_integer()]), nullable: true)), + new AdaptiveBackend(), + ); - $left->merge($right); + static::assertSame([['s' => ['a' => 1]], ['s' => null]], $matched->toArray()); + static::assertSame('structure{a: integer}', $matched->column('s')->type()->toString()); } - public function test_a_row_in_zone_is_kept(): void + public function test_match_to_adopts_a_not_null_column_an_empty_batch_lacks(): void { - $row = row(['at' => new DateTimeImmutable('2026-01-02 03:04:05', new DateTimeZone('Europe/Warsaw'))]); + $matched = rows(schema(int_schema('id'))) + ->matchTo(schema(int_schema('id'), str_schema('name')), new AdaptiveBackend()); - static::assertSame($row, $row->matchTo(schema(datetime_schema('at', zone: 'Europe/Warsaw')))); + static::assertSame(0, $matched->count()); + static::assertSame(['id', 'name'], $matched->schema()->references()->names()); } - public function test_rows_hop_a_foreign_zone_at_ingest(): void + public function test_match_to_refuses_a_midnight_outside_utc_in_a_date_column(): void { - static::assertSame( - '2026-01-01 22:04:05 UTC', - type_datetime() - ->assert(rows( - schema(datetime_schema('at')), - row(['at' => new DateTimeImmutable('2026-01-02 03:04:05+05:00')]), - )->first()->get('at')) - ->format('Y-m-d H:i:s e'), + $this->expectException(SchemaMismatchException::class); + $this->expectExceptionMessage( + 'column "at" (row 0): could not convert 2026-01-01T00:00:00.000000+01:00 (datetime) to date', + ); + + array_to_rows([['at' => new DateTimeImmutable( + '2026-01-01 00:00:00', + new DateTimeZone('Europe/Warsaw'), + )]], schema(datetime_schema('at', zone: 'Europe/Warsaw')))->matchTo( + schema(date_schema('at')), + new AdaptiveBackend(), ); } - public function test_rows_hop_nested_datetimes(): void - { - static::assertSame( - '2026-01-02 04:04:05 Europe/Warsaw', - type_list(type_datetime()) - ->assert(rows( - schema(list_schema('l', type_list(type_datetime('Europe/Warsaw')))), - row(['l' => [new DateTimeImmutable('2026-01-02 03:04:05', new DateTimeZone('UTC'))]]), - )->first()->get('l'))[0]->format('Y-m-d H:i:s e'), + public function test_match_to_reports_a_bad_value_before_a_later_null_of_the_same_column(): void + { + try { + array_to_rows([ + ['n' => [1]], + ['n' => [null]], + ['n' => null], + ], schema(list_schema('n', type_list(type_optional(type_integer())), nullable: true)))->matchTo( + schema(list_schema('n', type_list(type_integer()))), + new PhpBackend(), + ); + static::fail('the bad value must be refused'); + } catch (SchemaMismatchException $e) { + static::assertSame(1, $e->rowIndex); + } + } + + public function test_match_to_reports_the_lowest_bad_row_and_its_column(): void + { + try { + array_to_rows( + [['a' => [1], 'b' => [1]], ['a' => [2], 'b' => [null]], ['a' => [null], 'b' => [3]]], + schema( + list_schema('a', type_list(type_optional(type_integer()))), + list_schema('b', type_list(type_optional(type_integer()))), + ), + )->matchTo( + schema(list_schema('a', type_list(type_integer())), list_schema('b', type_list(type_integer()))), + new PhpBackend(), + ); + static::fail('the bad value must be refused'); + } catch (SchemaMismatchException $e) { + static::assertSame(1, $e->rowIndex); + static::assertStringContainsString('"b"', $e->getMessage()); + } + } + + public function test_select_keeps_the_named_columns_as_they_are_in_schema_order(): void + { + $rows = array_to_rows( + [['id' => 1, 'name' => 'a', 'tag' => 'x']], + schema(int_schema('id'), str_schema('name'), str_schema('tag')), ); + $selected = $rows->select('tag', 'id'); + + static::assertSame(['id', 'tag'], $selected->schema()->references()->names()); + static::assertSame($rows->column('tag'), $selected->column('tag')); } - public function test_rows_hop_nested_optional_datetimes(): void + public function test_select_refuses_a_name_the_batch_lacks(): void { - static::assertSame( - '2026-01-02 04:04:05 Europe/Warsaw', - type_list(type_optional(type_datetime())) - ->assert(rows( - schema(list_schema('l', type_list(type_optional(type_datetime('Europe/Warsaw'))))), - row(['l' => [new DateTimeImmutable('2026-01-02 03:04:05', new DateTimeZone('UTC')), null]]), - )->first()->get('l'))[0]?->format('Y-m-d H:i:s e'), - ); + $this->expectException(SchemaDefinitionNotFoundException::class); + + array_to_rows([['id' => 1]], schema(int_schema('id')))->select('id', 'missing'); } } diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Schema/Definition/ZoneAlignmentTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Schema/Definition/ZoneAlignmentTest.php deleted file mode 100644 index b3bda662fd..0000000000 --- a/src/core/etl/tests/Flow/ETL/Tests/Unit/Schema/Definition/ZoneAlignmentTest.php +++ /dev/null @@ -1,107 +0,0 @@ -assert((new ZoneAlignment())->align( - datetime_schema('at', zone: 'Europe/Warsaw'), - new DateTimeImmutable('2026-01-02 03:04:05+05:00'), - )) - ->format('Y-m-d H:i:s e'), - ); - } - - public function test_align_keeps_null(): void - { - static::assertNull((new ZoneAlignment())->align(datetime_schema('at', nullable: true), null)); - } - - public function test_align_recasts_a_nested_list(): void - { - static::assertSame( - '2026-01-02 04:04:05 Europe/Warsaw', - type_list(type_datetime()) - ->assert((new ZoneAlignment())->align( - list_schema('l', type_list(type_datetime('Europe/Warsaw'))), - [new DateTimeImmutable('2026-01-02 03:04:05', new DateTimeZone('UTC'))], - ))[0]->format('Y-m-d H:i:s e'), - ); - } - - public function test_align_returns_the_same_object_in_zone(): void - { - $value = new DateTimeImmutable('2026-01-02 03:04:05', new DateTimeZone('Europe/Warsaw')); - - static::assertSame($value, (new ZoneAlignment())->align(datetime_schema('at', zone: 'Europe/Warsaw'), $value)); - } - - public function test_columns_find_top_level_and_nested_datetimes(): void - { - static::assertSame( - ['at', 'optional', 'list', 'map', 'structure'], - array_keys((new ZoneAlignment())->columns([ - 'at' => datetime_schema('at'), - 'optional' => list_schema('optional', type_list(type_optional(type_datetime()))), - 'list' => list_schema('list', type_list(type_datetime())), - 'map' => map_schema('map', type_map(type_string(), type_datetime())), - 'structure' => structure_schema('structure', type_structure(['at' => type_datetime()])), - 'integers' => list_schema('integers', type_list(type_integer())), - 'on' => date_schema('on'), - ])), - ); - static::assertTrue((new ZoneAlignment())->carriesDateTime(type_optional(type_datetime()))); - } - - public function test_columns_of_a_scalar_schema_are_empty(): void - { - static::assertSame( - [], - (new ZoneAlignment())->columns([ - 'id' => int_schema('id'), - 'name' => string_schema('name'), - ]), - ); - } - - public function test_in_zone_needs_a_datetime_column_in_the_same_zone(): void - { - $value = new DateTimeImmutable('2026-01-02 03:04:05', new DateTimeZone('Europe/Warsaw')); - - static::assertTrue((new ZoneAlignment())->inZone(datetime_schema('at', zone: 'Europe/Warsaw'), $value)); - static::assertFalse((new ZoneAlignment())->inZone(datetime_schema('at'), $value)); - static::assertFalse((new ZoneAlignment())->inZone( - list_schema('at', type_list(type_datetime('Europe/Warsaw'))), - $value, - )); - static::assertFalse((new ZoneAlignment())->inZone(datetime_schema('at', zone: 'Europe/Warsaw'), '2026-01-02')); - } -} diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Schema/Formatter/InlineSchemaFormatterTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Schema/Formatter/InlineSchemaFormatterTest.php new file mode 100644 index 0000000000..1d4fb38e0a --- /dev/null +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Schema/Formatter/InlineSchemaFormatterTest.php @@ -0,0 +1,35 @@ +format(schema())); + } + + public function test_formatting_names_nullability_and_types_in_schema_order(): void + { + static::assertSame( + 'id: integer, name: ?string, tags: list', + (new InlineSchemaFormatter())->format(schema( + int_schema('id'), + str_schema('name', nullable: true), + list_schema('tags', type_list(type_integer())), + )), + ); + } +} diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Schema/Inference/ColumnTypesTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Schema/Inference/ColumnTypesTest.php index 1fe18e5146..485816cffd 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Unit/Schema/Inference/ColumnTypesTest.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Schema/Inference/ColumnTypesTest.php @@ -4,9 +4,7 @@ namespace Flow\ETL\Tests\Unit\Schema\Inference; -use Flow\ETL\Row\RawRowValues; use Flow\ETL\Schema; -use Flow\ETL\Schema\Metadata; use Flow\ETL\Tests\FlowTestCase; use Flow\ETL\Tests\Mother\ColumnTypesMother; use Flow\Types\Type; @@ -173,7 +171,7 @@ public function test_a_header_only_source_yields_every_name_as_nullable_string() public function test_a_name_met_only_in_a_row_is_appended_after_the_header_names(): void { $columns = ColumnTypesMother::fromStrings(['a', 'b']); - $columns->observe(new RawRowValues(['c' => '1'])); + $columns->observe(['c' => '1']); static::assertSame(['a', 'b', 'c'], array_keys($columns->schema(ColumnTypesMother::floor())->definitions())); } @@ -181,9 +179,9 @@ public function test_a_name_met_only_in_a_row_is_appended_after_the_header_names public function test_a_null_first_row_then_free_text_yields_optional_string(): void { $columns = ColumnTypesMother::fromStrings(); - $columns->observe(new RawRowValues(['c' => null])); - $columns->observe(new RawRowValues(['c' => 'free text'])); - $columns->observe(new RawRowValues(['c' => '42'])); + $columns->observe(['c' => null]); + $columns->observe(['c' => 'free text']); + $columns->observe(['c' => '42']); static::assertEquals( new Schema(definition_from_type('c', type_optional(type_string()), nullable: true)), @@ -194,9 +192,8 @@ public function test_a_null_first_row_then_free_text_yields_optional_string(): v public function test_a_numeric_key_becomes_a_string_column_name(): void { $columns = ColumnTypesMother::fromStrings(); - // PHP re-keys a numeric column name to int; RawRowValues declares array, which cannot - // express that, so the fold has to survive an int key arriving through a documented-string array - $columns->observe(new RawRowValues(['1' => 'x'])); + // PHP re-keys a numeric column name to int, so the fold has to survive an int key + $columns->observe(['1' => 'x']); static::assertSame('1', $columns->schema(ColumnTypesMother::floor())->get('1')->entry()->name()); } @@ -204,9 +201,9 @@ public function test_a_numeric_key_becomes_a_string_column_name(): void public function test_a_saturated_string_column_widens_to_optional_string_on_a_later_null(): void { $columns = ColumnTypesMother::fromStrings(); - $columns->observe(new RawRowValues(['c' => 'free text'])); - $columns->observe(new RawRowValues(['c' => null])); - $columns->observe(new RawRowValues(['c' => 'more text'])); + $columns->observe(['c' => 'free text']); + $columns->observe(['c' => null]); + $columns->observe(['c' => 'more text']); static::assertEquals( new Schema(definition_from_type('c', type_optional(type_string()), nullable: true)), @@ -222,7 +219,7 @@ public function test_a_source_with_no_names_and_no_rows_yields_an_empty_schema() public function test_all_strings_floors_every_scalar_column(): void { $columns = ColumnTypesMother::fromStrings(); - $columns->observe(new RawRowValues(['a' => '1', 'b' => 'true', 'c' => '2024-01-01'])); + $columns->observe(['a' => '1', 'b' => 'true', 'c' => '2024-01-01']); static::assertEquals( new Schema( @@ -237,8 +234,8 @@ public function test_all_strings_floors_every_scalar_column(): void public function test_an_absent_key_is_read_as_null(): void { $columns = ColumnTypesMother::fromStrings(); - $columns->observe(new RawRowValues(['a' => '1'])); - $columns->observe(new RawRowValues(['b' => 'x'])); + $columns->observe(['a' => '1']); + $columns->observe(['b' => 'x']); static::assertEquals( new Schema( @@ -252,8 +249,8 @@ public function test_an_absent_key_is_read_as_null(): void public function test_an_integer_column_is_declared_nullable(): void { $columns = ColumnTypesMother::fromStrings(); - $columns->observe(new RawRowValues(['c' => '1'])); - $columns->observe(new RawRowValues(['c' => '2'])); + $columns->observe(['c' => '1']); + $columns->observe(['c' => '2']); $definition = $columns->schema(ColumnTypesMother::floor())->get('c'); @@ -264,9 +261,9 @@ public function test_an_integer_column_is_declared_nullable(): void public function test_an_optional_non_string_column_is_not_saturated(): void { $columns = ColumnTypesMother::fromStrings(); - $columns->observe(new RawRowValues(['c' => '1'])); - $columns->observe(new RawRowValues(['c' => null])); - $columns->observe(new RawRowValues(['c' => '1.5'])); + $columns->observe(['c' => '1']); + $columns->observe(['c' => null]); + $columns->observe(['c' => '1.5']); static::assertEquals( new Schema(definition_from_type('c', type_optional(type_float()), nullable: true)), @@ -277,8 +274,8 @@ public function test_an_optional_non_string_column_is_not_saturated(): void public function test_from_column_types_reproduces_an_observed_fold(): void { $observed = ColumnTypesMother::fromStrings(['id', 'name']); - $observed->observe(new RawRowValues(['id' => '1', 'name' => 'a'])); - $observed->observe(new RawRowValues(['id' => '2', 'name' => null])); + $observed->observe(['id' => '1', 'name' => 'a']); + $observed->observe(['id' => '2', 'name' => null]); $computed = ColumnTypesMother::fromColumnTypes([ 'id' => type_optional(type_integer()), @@ -295,9 +292,9 @@ public function test_from_column_types_reproduces_an_observed_fold(): void public function test_from_column_types_merges_with_an_observed_fold(): void { $left = ColumnTypesMother::fromStrings(['a']); - $left->observe(new RawRowValues(['a' => '1'])); + $left->observe(['a' => '1']); $right = ColumnTypesMother::fromStrings(['a']); - $right->observe(new RawRowValues(['a' => '1.5', 'b' => 'x'])); + $right->observe(['a' => '1.5', 'b' => 'x']); static::assertEquals( $left->merge($right, true)->schema(ColumnTypesMother::floor()), @@ -311,7 +308,7 @@ public function test_from_column_types_merges_with_an_observed_fold(): void public function test_from_column_types_carries_the_row_count_into_merge(): void { $observed = ColumnTypesMother::fromStrings(); - $observed->observe(new RawRowValues(['a' => '1'])); + $observed->observe(['a' => '1']); static::assertSame( 8, @@ -322,11 +319,11 @@ public function test_from_column_types_carries_the_row_count_into_merge(): void public function test_merge_is_associative_over_three_partials(): void { $left = ColumnTypesMother::fromStrings(['a']); - $left->observe(new RawRowValues(['a' => '1'])); + $left->observe(['a' => '1']); $middle = ColumnTypesMother::fromStrings(['b']); - $middle->observe(new RawRowValues(['b' => 'x'])); + $middle->observe(['b' => 'x']); $right = ColumnTypesMother::fromStrings(['c']); - $right->observe(new RawRowValues(['a' => '1.5', 'c' => 'true'])); + $right->observe(['a' => '1.5', 'c' => 'true']); $leftAssociated = $left->merge($middle, true)->merge($right, true)->schema(ColumnTypesMother::floor()); $rightAssociated = $left->merge($middle->merge($right, true), true)->schema(ColumnTypesMother::floor()); @@ -362,10 +359,10 @@ public function test_merge_keeps_the_left_operands_name_order_then_appends_the_r public function test_merge_sums_the_row_counts_and_leaves_both_operands_alone(): void { $left = ColumnTypesMother::fromStrings(); - $left->observe(new RawRowValues(['a' => '1'])); - $left->observe(new RawRowValues(['a' => '2'])); + $left->observe(['a' => '1']); + $left->observe(['a' => '2']); $right = ColumnTypesMother::fromStrings(); - $right->observe(new RawRowValues(['a' => '3'])); + $right->observe(['a' => '3']); static::assertSame(3, $left->merge($right, true)->rows()); static::assertSame(2, $left->rows()); @@ -376,7 +373,7 @@ public function test_merge_takes_a_seeded_only_partial_as_the_identity_of_the_fo { $seeded = ColumnTypesMother::fromStrings(['a', 'b']); $observed = ColumnTypesMother::fromStrings(['a', 'b']); - $observed->observe(new RawRowValues(['a' => '1'])); + $observed->observe(['a' => '1']); $expected = new Schema( definition_from_type('a', type_integer(), nullable: true), @@ -390,9 +387,9 @@ public function test_merge_takes_a_seeded_only_partial_as_the_identity_of_the_fo public function test_merge_unions_disjoint_name_sets(): void { $left = ColumnTypesMother::fromStrings(); - $left->observe(new RawRowValues(['a' => '1'])); + $left->observe(['a' => '1']); $right = ColumnTypesMother::fromStrings(); - $right->observe(new RawRowValues(['b' => 'x'])); + $right->observe(['b' => 'x']); static::assertEquals( new Schema( @@ -413,9 +410,9 @@ public function test_merged_types_do_not_depend_on_the_order_of_the_two_partials Type $expected, ): void { $left = ColumnTypesMother::fromStrings(); - $left->observe(new RawRowValues(['c' => $leftValue])); + $left->observe(['c' => $leftValue]); $right = ColumnTypesMother::fromStrings(); - $right->observe(new RawRowValues(['c' => $rightValue])); + $right->observe(['c' => $rightValue]); $leftFirst = $left->merge($right, true)->schema(ColumnTypesMother::floor())->get('c')->type(); $rightFirst = $right->merge($left, true)->schema(ColumnTypesMother::floor())->get('c')->type(); @@ -433,9 +430,9 @@ public function test_merged_types_do_not_depend_on_the_order_of_the_two_partials public function test_merged_types_do_not_depend_on_the_order_for_container_partials(): void { $left = ColumnTypesMother::fromTypedValues(); - $left->observe(new RawRowValues(['c' => []])); + $left->observe(['c' => []]); $right = ColumnTypesMother::fromTypedValues(); - $right->observe(new RawRowValues(['c' => [1]])); + $right->observe(['c' => [1]]); static::assertTrue(type_equals( type_list(type_optional(type_integer())), @@ -447,23 +444,12 @@ public function test_merged_types_do_not_depend_on_the_order_for_container_parti )); } - public function test_metadata_never_reaches_the_schema(): void - { - $columns = ColumnTypesMother::fromStrings(); - $columns->observe(new RawRowValues(['a' => '1'], ['a' => Metadata::empty()->add('k', 'v')])); - - static::assertEquals( - new Schema(definition_from_type('a', type_integer(), nullable: true)), - $columns->schema(ColumnTypesMother::floor()), - ); - } - public function test_null_never_reaches_the_ladder(): void { $columns = ColumnTypesMother::fromStrings(); - $columns->observe(new RawRowValues(['c' => '1'])); - $columns->observe(new RawRowValues(['c' => null])); - $columns->observe(new RawRowValues(['c' => '2'])); + $columns->observe(['c' => '1']); + $columns->observe(['c' => null]); + $columns->observe(['c' => '2']); static::assertEquals( new Schema(definition_from_type('c', type_optional(type_integer()), nullable: true)), @@ -474,8 +460,8 @@ public function test_null_never_reaches_the_ladder(): void public function test_observing_a_column_name_absent_from_the_header_still_records_its_type(): void { $columns = ColumnTypesMother::fromStrings(['a']); - $columns->observe(new RawRowValues(['a' => 'free text', 'b' => '1'])); - $columns->observe(new RawRowValues(['a' => 'more text', 'b' => '2'])); + $columns->observe(['a' => 'free text', 'b' => '1']); + $columns->observe(['a' => 'more text', 'b' => '2']); static::assertEquals( new Schema( @@ -489,12 +475,12 @@ public function test_observing_a_column_name_absent_from_the_header_still_record public function test_padded_cells_are_read_as_text_because_cast_does_not_trim(): void { $padded = ColumnTypesMother::fromStrings(); - $padded->observe(new RawRowValues(['c' => ' 12 '])); - $padded->observe(new RawRowValues(['c' => '13'])); + $padded->observe(['c' => ' 12 ']); + $padded->observe(['c' => '13']); $clean = ColumnTypesMother::fromStrings(); - $clean->observe(new RawRowValues(['c' => '12'])); - $clean->observe(new RawRowValues(['c' => '13'])); + $clean->observe(['c' => '12']); + $clean->observe(['c' => '13']); static::assertEquals( new Schema(definition_from_type('c', type_string(), nullable: true)), @@ -512,9 +498,9 @@ public function test_rows_counts_observations_not_columns(): void static::assertSame(0, $columns->rows()); - $columns->observe(new RawRowValues(['a' => '1', 'b' => '2'])); - $columns->observe(new RawRowValues(['a' => '3'])); - $columns->observe(new RawRowValues([])); + $columns->observe(['a' => '1', 'b' => '2']); + $columns->observe(['a' => '3']); + $columns->observe([]); static::assertSame(3, $columns->rows()); } @@ -526,7 +512,7 @@ public function test_rows_counts_observations_not_columns(): void public function test_the_declared_type_fits_the_cell_that_produced_it(string $cell): void { $columns = ColumnTypesMother::fromStrings(); - $columns->observe(new RawRowValues(['c' => $cell])); + $columns->observe(['c' => $cell]); $definition = $columns->schema(ColumnTypesMother::floor())->get('c'); @@ -536,11 +522,11 @@ public function test_the_declared_type_fits_the_cell_that_produced_it(string $ce public function test_the_null_rung_is_read_as_text_because_the_cell_is_text(): void { $mixed = ColumnTypesMother::fromStrings(); - $mixed->observe(new RawRowValues(['c' => 'null'])); - $mixed->observe(new RawRowValues(['c' => '1'])); + $mixed->observe(['c' => 'null']); + $mixed->observe(['c' => '1']); $only = ColumnTypesMother::fromStrings(); - $only->observe(new RawRowValues(['c' => 'nil'])); + $only->observe(['c' => 'nil']); static::assertEquals( new Schema(definition_from_type('c', type_string(), nullable: true)), @@ -563,7 +549,7 @@ public function test_typed_values_fold_through_the_instance_of_narrower(array $v /** @var mixed $value */ foreach ($values as $value) { - $columns->observe(new RawRowValues(['c' => $value])); + $columns->observe(['c' => $value]); } static::assertEquals( @@ -582,7 +568,7 @@ public function test_widening_across_rows_of_one_column(array $values, Type $exp $columns = ColumnTypesMother::fromStrings(); foreach ($values as $value) { - $columns->observe(new RawRowValues(['c' => $value])); + $columns->observe(['c' => $value]); } static::assertEquals( diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Schema/Inference/InferredTypesTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Schema/Inference/InferredTypesTest.php index f5e4b142fd..9fb7341b95 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Unit/Schema/Inference/InferredTypesTest.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Schema/Inference/InferredTypesTest.php @@ -65,7 +65,9 @@ public function test_an_empty_list_is_rejected(): void public function test_a_parameterised_type_admits_its_whole_family(): void { - static::assertTrue((new InferredTypes(type_enum(BackedStringEnum::class)))->allows(type_enum(BackedStringEnum::class))); + static::assertTrue((new InferredTypes(type_enum(BackedStringEnum::class)))->allows(type_enum( + BackedStringEnum::class, + ))); } public function test_default_is_every_rung_but_markup(): void diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Schema/Inference/SchemaInferrerTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Schema/Inference/SchemaInferrerTest.php index b6b1a732b1..751059f34d 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Unit/Schema/Inference/SchemaInferrerTest.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Schema/Inference/SchemaInferrerTest.php @@ -4,7 +4,6 @@ namespace Flow\ETL\Tests\Unit\Schema\Inference; -use Flow\ETL\Row\RawRowValues; use Flow\ETL\Schema; use Flow\ETL\Schema\Inference\InferredTypes; use Flow\ETL\Schema\Inference\SchemaInference; @@ -51,7 +50,7 @@ public static function budgets(): array * Heterogeneous on purpose: "a" widens integer -> float inside source 1, and source 2 introduces "c", so a * budget that stops early and a merge that drops a name are both observable. * - * @return list> + * @return list>> */ public static function threeSourcesOfTenRows(): array { @@ -71,7 +70,7 @@ public static function threeSourcesOfTenRows(): array $values['c'] = 'true'; } - $rows[] = new RawRowValues($values); + $rows[] = $values; } $sources[] = $rows; @@ -120,8 +119,8 @@ public function test_a_sample_that_sniffs_itself_is_not_asked_once_the_budget_is public function test_a_column_that_is_free_text_in_the_first_source_and_numeric_in_the_second_widens_to_string(): void { $fixture = [ - [new RawRowValues(['id' => '1', 'v' => 'free text']), new RawRowValues(['id' => '2', 'v' => 'more text'])], - [new RawRowValues(['id' => '3', 'v' => '42']), new RawRowValues(['id' => '4', 'v' => '43'])], + [['id' => '1', 'v' => 'free text'], ['id' => '2', 'v' => 'more text']], + [['id' => '3', 'v' => '42'], ['id' => '4', 'v' => '43']], ]; $inferrer = new SchemaInferrer( new SchemaInference(), @@ -148,8 +147,8 @@ public function test_a_row_less_source_does_not_spend_a_files_to_sniff_slot(): v { $sources = new RecordingSources([ [], - [new RawRowValues(['a' => '1', 'b' => 'x'])], - [new RawRowValues(['c' => '1'])], + [['a' => '1', 'b' => 'x']], + [['c' => '1']], ]); $schema = (new SchemaInferrer( @@ -163,7 +162,7 @@ public function test_a_row_less_source_does_not_spend_a_files_to_sniff_slot(): v public function test_a_single_generator_source_is_accepted(): void { - $sources = new RecordingSources([[new RawRowValues(['a' => '1'])]]); + $sources = new RecordingSources([[['a' => '1']]]); static::assertEquals( new Schema(definition_from_type('a', type_integer(), nullable: true)), @@ -332,7 +331,7 @@ public function test_sniff_seeds_the_names_it_is_handed_even_when_no_row_carries $partial = (new SchemaInferrer( new SchemaInference(), new StringTypeNarrower(InferredTypes::default()->toArray()), - ))->sniff(['ghost'], [new RawRowValues(['a' => '1'])], -1); + ))->sniff(['ghost'], [['a' => '1']], -1); static::assertSame(['ghost', 'a'], array_keys($partial->schema(ColumnTypesMother::floor())->definitions())); static::assertSame('string', $partial->schema(ColumnTypesMother::floor())->get('ghost')->type()->toString()); @@ -340,7 +339,7 @@ public function test_sniff_seeds_the_names_it_is_handed_even_when_no_row_carries public function test_sniff_observes_nothing_when_the_budget_is_already_spent(): void { - $sources = new RecordingSources([[new RawRowValues(['a' => '1'])]]); + $sources = new RecordingSources([[['a' => '1']]]); $partial = (new SchemaInferrer( new SchemaInference(), @@ -374,7 +373,7 @@ public function test_the_candidate_set_reaches_the_floor_through_infer(): void $schema = (new SchemaInferrer( new SchemaInference(types: new InferredTypes(type_integer(), type_string())), new StringTypeNarrower([type_integer(), type_string()]), - ))->infer([], [[new RawRowValues(['i' => '42', 'f' => '1.5', 'd' => '2024-01-01'])]]); + ))->infer([], [[['i' => '42', 'f' => '1.5', 'd' => '2024-01-01']]]); static::assertEquals( new Schema( @@ -389,8 +388,8 @@ public function test_the_candidate_set_reaches_the_floor_through_infer(): void public function test_union_by_name_over_sources_where_one_column_saturates_to_string(): void { $sources = new RecordingSources([ - [new RawRowValues(['id' => '1', 'v' => 'free text'])], - [new RawRowValues(['id' => '2', 'v' => '42', 'w' => '2024-01-01'])], + [['id' => '1', 'v' => 'free text']], + [['id' => '2', 'v' => '42', 'w' => '2024-01-01']], ]); static::assertEquals( diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Schema/SameSchemasTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Schema/SameSchemasTest.php new file mode 100644 index 0000000000..f942687696 --- /dev/null +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Schema/SameSchemasTest.php @@ -0,0 +1,28 @@ +isSame(schema(int_schema('id'), str_schema('name'))); + + static::assertTrue(type_instance_of(Schema::class)->assert(unserialize(serialize($schema)))->isSame($schema)); + } + + public function test_a_schema_proved_the_same_stays_the_same_and_another_stays_different(): void + { + $schema = schema(int_schema('id'), str_schema('name')); + $same = schema(int_schema('id'), str_schema('name')); + $other = schema(int_schema('id'), str_schema('name', nullable: true)); + + static::assertTrue($schema->isSame($same)); + static::assertTrue($schema->isSame($same)); + static::assertFalse($schema->isSame($other)); + static::assertFalse($schema->isSame($other)); + } + public static function provide_add_after_reference_inputs(): Generator { yield 'string reference' => ['id']; @@ -930,13 +952,4 @@ public function test_match_order_to_adds_and_drops_nothing(): void ), ); } - - public function test_zoned_definitions_are_computed_in_the_constructor(): void - { - $schema = schema(int_schema('i'), datetime_schema('at')); - - static::assertSame(['at'], array_keys($schema->zonedDefinitions())); - static::assertSame(['b'], array_keys($schema->rename('at', 'b')->zonedDefinitions())); - static::assertSame([], $schema->remove('at')->zonedDefinitions()); - } } diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Serializer/DomValueCodecTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Serializer/DomValueCodecTest.php deleted file mode 100644 index 089de187de..0000000000 --- a/src/core/etl/tests/Flow/ETL/Tests/Unit/Serializer/DomValueCodecTest.php +++ /dev/null @@ -1,174 +0,0 @@ - [type_xml()]; - yield 'xml_element' => [type_xml_element()]; - yield 'html' => [type_html()]; - yield 'html_element' => [type_html_element()]; - } - - public static function nonDomTypes(): iterable - { - yield 'string' => [type_string()]; - yield 'integer' => [type_integer()]; - } - - #[DataProvider('domTypes')] - public function test_handles_dom_types(Type $type): void - { - static::assertTrue((new DomValueCodec())->handles($type)); - } - - #[DataProvider('nonDomTypes')] - public function test_handles_rejects_non_dom_types(Type $type): void - { - static::assertFalse((new DomValueCodec())->handles($type)); - } - - #[DataProvider('domTypes')] - public function test_null_passes_through_untouched(Type $type): void - { - static::assertNull((new DomValueCodec())->encode($type, null)); - static::assertNull((new DomValueCodec())->decode($type, null)); - } - - #[DataProvider('nonDomTypes')] - public function test_non_dom_type_passes_through_untouched(Type $type): void - { - static::assertSame('1', (new DomValueCodec())->encode($type, '1')); - static::assertSame('1', (new DomValueCodec())->decode($type, '1')); - } - - public function test_xml_document_survives_a_round_trip(): void - { - $document = new DOMDocument(); - $document->loadXML('1'); - - $codec = new DomValueCodec(); - - /** @var DOMDocument $restored */ - $restored = $codec->decode(type_xml(), $codec->encode(type_xml(), $document)); - - static::assertInstanceOf(DOMDocument::class, $restored); - static::assertSame($document->C14N(), $restored->C14N()); - } - - public function test_xml_element_survives_a_round_trip(): void - { - $document = new DOMDocument(); - $document->loadXML('1'); - $element = $document->documentElement; - static::assertInstanceOf(DOMElement::class, $element); - - $codec = new DomValueCodec(); - - /** @var DOMElement $restored */ - $restored = $codec->decode(type_xml_element(), $codec->encode(type_xml_element(), $element)); - - static::assertInstanceOf(DOMElement::class, $restored); - static::assertSame('1', $restored->C14N()); - } - - public function test_a_restored_xml_element_does_not_canonicalize_to_an_empty_string(): void - { - $codec = new DomValueCodec(); - - $left = new DOMDocument(); - $left->loadXML('1'); - $right = new DOMDocument(); - $right->loadXML('999'); - - /** @var DOMElement $restoredLeft */ - $restoredLeft = $codec->decode(type_xml_element(), $codec->encode(type_xml_element(), $left->documentElement)); - /** @var DOMElement $restoredRight */ - $restoredRight = $codec->decode(type_xml_element(), $codec->encode( - type_xml_element(), - $right->documentElement, - )); - - static::assertNotSame('', $restoredLeft->C14N()); - static::assertNotSame($restoredLeft->C14N(), $restoredRight->C14N()); - } - - public function test_html_document_survives_a_round_trip(): void - { - if (!class_exists('\Dom\HTMLDocument')) { - static::markTestSkipped('\Dom\HTMLDocument requires PHP 8.4+'); - } - - $document = ValueDecoder::htmlDocumentFromString('

x

'); - - $codec = new DomValueCodec(); - - static::assertSame( - type_string()->cast($document), - type_string()->cast($codec->decode(type_html(), $codec->encode(type_html(), $document))), - ); - } - - public function test_html_element_survives_a_round_trip(): void - { - if (!class_exists('\Dom\HTMLDocument')) { - static::markTestSkipped('\Dom\HTMLDocument requires PHP 8.4+'); - } - - $element = ValueDecoder::htmlElementFromString('

x

'); - - $codec = new DomValueCodec(); - - static::assertSame( - type_string()->cast($element), - type_string()->cast($codec->decode(type_html_element(), $codec->encode(type_html_element(), $element))), - ); - } - - public function test_decode_rejects_a_value_that_is_not_base64(): void - { - $this->expectException(InvalidArgumentException::class); - $this->expectExceptionMessage('Given value is not valid base64'); - - (new DomValueCodec())->decode(type_xml(), '!!! not base64 !!!'); - } - - public function test_decode_rejects_a_value_that_is_not_gzcompressed(): void - { - $this->expectException(InvalidArgumentException::class); - $this->expectExceptionMessage('Given value is not valid gzcompressed XML'); - - (new DomValueCodec())->decode(type_xml(), base64_encode('plain, uncompressed')); - } - - public function test_decode_rejects_a_payload_that_is_not_xml(): void - { - $this->expectException(InvalidArgumentException::class); - $this->expectExceptionMessage('is not valid XML'); - - (new DomValueCodec())->decode(type_xml(), base64_encode((string) gzcompress('not xml at all'))); - } -} diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Sort/ExternalSortTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Sort/ExternalSortTest.php new file mode 100644 index 0000000000..6866d3cff1 --- /dev/null +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Sort/ExternalSortTest.php @@ -0,0 +1,311 @@ + [$seed]; + } + } + + public function test_data_under_the_limit_sorts_with_zero_spills(): void + { + $storage = new RecordingBucketsStorage(new MemoryBuckets()); + + static::assertSame( + [1, 2, 3, 4, 5], + IdRows::ids(ExternalSortMother::inMemory(refs('id'), $storage)->sort( + IdRows::batches([4, 2], [5], [1, 3]), + flow_context(config()), + )), + ); + static::assertSame([], $storage->appended); + static::assertSame([], $storage->read); + } + + public function test_data_under_the_limit_is_emitted_in_batches_of_the_configured_size(): void + { + $batches = iterator_to_array( + ExternalSortMother::inMemory(refs('id'), batchSize: 2)->sort( + IdRows::batches([5, 4, 3], [2, 1]), + flow_context(config()), + ), + false, + ); + + static::assertSame([2, 2, 1], array_map(static fn(Rows $rows): int => $rows->count(), $batches)); + } + + public function test_data_over_the_limit_spills_and_merges(): void + { + $storage = new SpyBucketsStorage(new MemoryBuckets()); + + static::assertSame( + [0, 1, 2, 3, 10, 11, 12, 13], + IdRows::ids(ExternalSortMother::spilling(refs('id'), $storage)->sort( + IdRows::batches([10, 0], [11, 1], [2, 12], [13, 3]), + flow_context(config()), + )), + ); + static::assertCount(4, array_filter($storage->readBucketIds(), static fn(string $id): bool => str_starts_with( + $id, + 'sort-', + ))); + static::assertSame([], $storage->liveBucketIds()); + } + + public function test_descending_single_column(): void + { + static::assertSame( + [4, 3, 2, 1], + IdRows::ids(ExternalSortMother::spilling(refs(ref('id')->desc()))->sort( + IdRows::batches([3, 1], [4, 2]), + flow_context(config()), + )), + ); + } + + public function test_sorts_by_multiple_columns(): void + { + $input = (static function (): Generator { + yield array_to_rows([['a' => 1, 'b' => 2], ['a' => 1, 'b' => 1]], schema(int_schema('a'), int_schema('b'))); + yield array_to_rows([['a' => 0, 'b' => 5], ['a' => 0, 'b' => 3]], schema(int_schema('a'), int_schema('b'))); + })(); + + $sorted = rows(schema(int_schema('a'), int_schema('b'))); + + foreach (ExternalSortMother::spilling(refs('a', 'b'))->sort($input, flow_context(config())) as $batch) { + $sorted = $sorted->concat(new AdaptiveBackend(), $batch); + } + + static::assertSame( + [['a' => 0, 'b' => 3], ['a' => 0, 'b' => 5], ['a' => 1, 'b' => 1], ['a' => 1, 'b' => 2]], + $sorted->toArray(), + ); + } + + public function test_empty_input_yields_nothing_and_writes_nothing(): void + { + $storage = new RecordingBucketsStorage(new MemoryBuckets()); + + static::assertSame( + [], + IdRows::ids(ExternalSortMother::spilling(refs('id'), $storage)->sort( + IdRows::batches([], []), + flow_context(config()), + )), + ); + static::assertSame([], $storage->appended); + } + + public function test_runs_past_the_fan_in_are_reduced_to_merged_runs_first(): void + { + $storage = new RecordingBucketsStorage(new MemoryBuckets()); + + static::assertSame( + range(0, 19), + IdRows::ids(ExternalSortMother::spilling(refs('id'), $storage, mergeFanIn: 2)->sort( + IdRows::batches(...array_map(static fn(int $i): array => [($i * 2) + 1, $i * 2], range(9, 0))), + flow_context(config()), + )), + ); + static::assertNotSame([], array_filter($storage->appended, static fn(string $id): bool => str_starts_with( + $id, + 'sort-merge-', + ))); + } + + public function test_merged_runs_go_to_the_merge_storage(): void + { + $spill = new RecordingBucketsStorage(new MemoryBuckets()); + $merge = new RecordingBucketsStorage(new MemoryBuckets()); + + IdRows::ids(ExternalSortMother::with(refs('id'), $spill, Unit::fromBytes(1), 2, mergeStorage: $merge)->sort( + IdRows::batches([3], [2], [1]), + flow_context(config()), + )); + + static::assertSame([], array_filter($spill->appended, static fn(string $id): bool => str_starts_with( + $id, + 'sort-merge-', + ))); + static::assertNotSame([], $merge->appended); + } + + public function test_a_limit_under_the_memory_limit_keeps_the_first_rows(): void + { + static::assertSame( + [1, 2], + IdRows::ids(ExternalSortMother::inMemory(refs('id'))->sort( + IdRows::batches([4, 2], [5], [1, 3]), + flow_context(config()), + 2, + )), + ); + } + + public function test_a_limit_over_the_memory_limit_stops_every_merge_at_the_limit(): void + { + $storage = new SpyBucketsStorage(new MemoryBuckets()); + + static::assertSame( + [0, 1, 2], + IdRows::ids(ExternalSortMother::spilling(refs('id'), $storage, mergeFanIn: 2)->sort( + IdRows::batches(...array_map(static fn(int $i): array => [$i + 10, $i], range(9, 0))), + flow_context(config()), + 3, + )), + ); + + $merged = array_filter( + $storage->appendedRows(), + static fn(string $id): bool => str_starts_with($id, 'sort-merge-'), + ARRAY_FILTER_USE_KEY, + ); + + static::assertNotSame([], $merged); + + foreach ($merged as $batches) { + static::assertLessThanOrEqual( + 3, + array_sum(array_map(static fn(Rows $rows): int => $rows->count(), $batches)), + ); + } + } + + #[DataProvider('random_datasets')] + public function test_spilled_sort_equals_one_in_memory_sort(int $seed): void + { + $dataset = SortDatasetMother::random($seed); + $input = (static function () use ($dataset): Generator { + foreach ($dataset['runs'] as $rows) { + yield array_to_rows($rows, $dataset['schema']); + } + })(); + + $sorted = rows($dataset['schema']); + + foreach (ExternalSortMother::spilling( + refs(...$dataset['references']), + mergeFanIn: 2, + batchSize: $dataset['mergeBatchSize'], + )->sort($input, flow_context(config())) as $batch) { + $sorted = $sorted->concat( + new AdaptiveBackend(), + $batch->matchTo($dataset['schema'], new AdaptiveBackend()), + ); + } + + // serialized, so NaN and -0.0 compare by what they are + static::assertSame( + serialize( + array_to_rows(array_merge(...$dataset['runs']), $dataset['schema']) + ->sortBy(...$dataset['references']) + ->toArray(), + ), + serialize($sorted->toArray()), + ); + } + + public function test_both_storages_are_cleared_when_the_first_clear_throws(): void + { + $spillStorage = new SpyBucketsStorage(new MemoryBuckets()); + $mergeStorage = new SpyBucketsStorage(new MemoryBuckets()); + $spill = new Buckets(new ThrowingRemoveBucketsStorage($spillStorage)); + $merge = new Buckets($mergeStorage); + + // the upstream throws during the drain, so reduce() never writes a merged run - seed one, or the + // merge-side assertion is true whether or not the nested finally ran + $mergeStorage->append('pre-existing', array_to_rows([['id' => 1]], schema(int_schema('id')))); + $merge->add(new Bucket('pre-existing', 1, 0)); + + $sort = new ExternalSort( + refs('id'), + $spill, + $merge, + new NativePHPRandomValueGenerator(), + Unit::fromBytes(1), + 2, + ); + + $upstream = (static function (): Generator { + yield from IdRows::batches([9, 8], [7, 6]); + + throw new RuntimeException('upstream failed'); + })(); + + try { + iterator_to_array($sort->sort($upstream, flow_context(config())), false); + static::fail('The upstream failure must escape.'); + } catch (RuntimeException $escaped) { + static::assertSame('spill clear failed', $escaped->getMessage()); + + $previous = $escaped->getPrevious(); + + static::assertInstanceOf(RuntimeException::class, $previous); + static::assertSame('upstream failed', $previous->getMessage()); + } + + static::assertSame([], $mergeStorage->liveBucketIds()); + } + + public function test_throws_when_batch_size_below_one(): void + { + $this->expectException(InvalidArgumentException::class); + $this->expectExceptionMessage('Batch size must be greater than 0, given: 0'); + + // @mago-ignore analysis:invalid-argument + ExternalSortMother::with(refs('id'), new MemoryBuckets(), Unit::fromGb(1), 10, 0); + } + + public function test_throws_when_merge_fan_in_below_one(): void + { + $this->expectException(InvalidArgumentException::class); + $this->expectExceptionMessage('Merge fan-in must be greater than 0, given: 0'); + + // @mago-ignore analysis:invalid-argument + ExternalSortMother::with(refs('id'), new MemoryBuckets(), Unit::fromGb(1), 0); + } +} diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Sort/Merge/BucketCursorTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Sort/Merge/BucketCursorTest.php index 5ac8878263..6e8d51f712 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Unit/Sort/Merge/BucketCursorTest.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Sort/Merge/BucketCursorTest.php @@ -4,12 +4,20 @@ namespace Flow\ETL\Tests\Unit\Sort\Merge; +use DateTimeImmutable; +use DateTimeZone; +use Flow\ETL\Column\AdaptiveBackend; use Flow\ETL\Sort\Merge\BucketCursor; +use Flow\ETL\Sort\RowOrder; +use Flow\ETL\Sort\SortKey; use Flow\ETL\Tests\FlowTestCase; use Generator; +use function array_map; +use function Flow\ETL\DSL\array_to_rows; +use function Flow\ETL\DSL\datetime_schema; use function Flow\ETL\DSL\int_schema; -use function Flow\ETL\DSL\row; +use function Flow\ETL\DSL\ref; use function Flow\ETL\DSL\rows; use function Flow\ETL\DSL\schema; @@ -18,17 +26,17 @@ final class BucketCursorTest extends FlowTestCase public function test_iterates_rows_across_batch_boundaries(): void { $batches = static function (): Generator { - yield rows(schema(int_schema('id')), row(['id' => 1]), row(['id' => 2])); - yield rows(schema(int_schema('id')), row(['id' => 3])); + yield array_to_rows([['id' => 1], ['id' => 2]], schema(int_schema('id'))); + yield array_to_rows([['id' => 3]], schema(int_schema('id'))); }; - $cursor = new BucketCursor($batches()); + $cursor = new BucketCursor($batches(), new RowOrder([ref('id')]), new AdaptiveBackend()); $ids = []; while ($cursor->valid()) { - $ids[] = $cursor->current()->get('id'); - $cursor->next(); + $ids[] = $cursor->batch()->column('id')->value($cursor->index()); + $cursor->advance($cursor->index() + 1); } static::assertSame([1, 2, 3], $ids); @@ -40,25 +48,25 @@ public function test_empty_batch_stream_is_invalid_from_the_start(): void yield from []; }; - static::assertFalse((new BucketCursor($batches()))->valid()); + static::assertFalse((new BucketCursor($batches(), new RowOrder([ref('id')]), new AdaptiveBackend()))->valid()); } public function test_skips_empty_batches(): void { $batches = static function (): Generator { yield rows(schema()); - yield rows(schema(int_schema('id')), row(['id' => 1])); + yield array_to_rows([['id' => 1]], schema(int_schema('id'))); yield rows(schema()); - yield rows(schema(int_schema('id')), row(['id' => 2])); + yield array_to_rows([['id' => 2]], schema(int_schema('id'))); }; - $cursor = new BucketCursor($batches()); + $cursor = new BucketCursor($batches(), new RowOrder([ref('id')]), new AdaptiveBackend()); $ids = []; while ($cursor->valid()) { - $ids[] = $cursor->current()->get('id'); - $cursor->next(); + $ids[] = $cursor->batch()->column('id')->value($cursor->index()); + $cursor->advance($cursor->index() + 1); } static::assertSame([1, 2], $ids); @@ -67,23 +75,24 @@ public function test_skips_empty_batches(): void public function test_next_batch_is_not_decoded_until_the_current_one_is_exhausted(): void { $batches = static function (): Generator { - yield rows(schema(int_schema('id')), row(['id' => 1]), row(['id' => 2])); - yield rows(schema(int_schema('id')), row(['id' => 3])); + yield array_to_rows([['id' => 1], ['id' => 2]], schema(int_schema('id'))); + yield array_to_rows([['id' => 3]], schema(int_schema('id'))); }; $generator = $batches(); - $cursor = new BucketCursor($generator); + $cursor = new BucketCursor($generator, new RowOrder([ref('id')]), new AdaptiveBackend()); static::assertSame(0, $generator->key()); - $cursor->next(); + $cursor->advance(1); static::assertSame(0, $generator->key()); - $cursor->next(); + $cursor->advance(2); static::assertSame(1, $generator->key()); - static::assertSame(3, $cursor->current()->get('id')); + static::assertSame(0, $cursor->index()); + static::assertSame([['id' => 3]], $cursor->batch()->toArray()); } public function test_stream_of_only_empty_batches_is_invalid(): void @@ -93,6 +102,36 @@ public function test_stream_of_only_empty_batches_is_invalid(): void yield rows(schema()); }; - static::assertFalse((new BucketCursor($batches()))->valid()); + static::assertFalse((new BucketCursor($batches(), new RowOrder([ref('id')]), new AdaptiveBackend()))->valid()); + } + + public function test_keys_are_the_sort_keys_of_every_ref_in_order(): void + { + $batches = static function (): Generator { + yield array_to_rows( + [['id' => 2, 'at' => new DateTimeImmutable('1970-01-01 00:00:01', new DateTimeZone('UTC'))]], + schema(int_schema('id'), datetime_schema('at')), + ); + }; + + $cursor = new BucketCursor($batches(), new RowOrder([ref('at'), ref('id')]), new AdaptiveBackend()); + + static::assertSame( + [[1_000_000], [2]], + array_map(static fn(SortKey $key): array => $key->values, $cursor->keys()), + ); + static::assertEquals(schema(int_schema('id'), datetime_schema('at')), $cursor->schema()); + } + + public function test_schema_of_an_empty_stream_is_empty(): void + { + $batches = static function (): Generator { + yield from []; + }; + + static::assertEquals( + schema(), + (new BucketCursor($batches(), new RowOrder([ref('id')]), new AdaptiveBackend()))->schema(), + ); } } diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Sort/Merge/KWayMergeTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Sort/Merge/KWayMergeTest.php index 6d16554b74..ade2a81326 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Unit/Sort/Merge/KWayMergeTest.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Sort/Merge/KWayMergeTest.php @@ -7,52 +7,109 @@ use Flow\ETL\Bucketing\BucketRun; use Flow\ETL\Bucketing\Buckets; use Flow\ETL\Bucketing\Storage\MemoryBuckets; +use Flow\ETL\Column\AdaptiveBackend; +use Flow\ETL\Constraint\SortedByConstraint; use Flow\ETL\Exception\InvalidArgumentException; -use Flow\ETL\Row; use Flow\ETL\Sort\Merge\KWayMerge; use Flow\ETL\Tests\Context\BucketsStorageContext; use Flow\ETL\Tests\FlowTestCase; +use Flow\ETL\Tests\Mother\SortDatasetMother; +use Generator; +use PHPUnit\Framework\Attributes\DataProvider; -use function array_map; +use function array_chunk; +use function array_merge; +use function Flow\ETL\DSL\array_to_rows; use function Flow\ETL\DSL\int_schema; use function Flow\ETL\DSL\ref; use function Flow\ETL\DSL\refs; -use function Flow\ETL\DSL\row; use function Flow\ETL\DSL\rows; use function Flow\ETL\DSL\schema; use function Flow\ETL\DSL\str_schema; +use function serialize; final class KWayMergeTest extends FlowTestCase { - public function test_merges_multiple_sorted_runs_ascending(): void + public static function random_datasets(): Generator + { + for ($seed = 1; $seed <= 200; $seed++) { + yield "seed {$seed}" => [$seed]; + } + } + + #[DataProvider('random_datasets')] + public function test_merging_sorted_runs_equals_one_in_memory_sort(int $seed): void { + $dataset = SortDatasetMother::random($seed); $buckets = new Buckets(new MemoryBuckets()); - $buckets->storage()->append('a', rows( - schema(int_schema('id')), - row(['id' => 1]), - row(['id' => 4]), - row(['id' => 7]), - )); - $buckets->storage()->append('b', rows( - schema(int_schema('id')), - row(['id' => 2]), - row(['id' => 5]), - row(['id' => 8]), - )); - $buckets->storage()->append('c', rows( - schema(int_schema('id')), - row(['id' => 3]), - row(['id' => 6]), - row(['id' => 9]), - )); + $runs = []; + + foreach ($dataset['runs'] as $run => $rows) { + foreach (array_to_rows($rows, $dataset['schema']) + ->sortBy(...$dataset['references']) + ->chunks($dataset['storedBatchSize']) as $batch) { + $buckets->storage()->append("run-{$run}", $batch); + } - $merge = new KWayMerge(refs(ref('id')->asc())); + $runs[] = new BucketRun("run-{$run}", $buckets); + } + + $merged = rows($dataset['schema']); + + foreach ((new KWayMerge( + refs(...$dataset['references']), + new AdaptiveBackend(), + $dataset['mergeBatchSize'], + ))->merge($runs) as $batch) { + $merged = $merged->concat( + new AdaptiveBackend(), + $batch->matchTo($dataset['schema'], new AdaptiveBackend()), + ); + } + + // serialized, so NaN and -0.0 compare by what they are + static::assertSame( + serialize( + array_to_rows(array_merge(...$dataset['runs']), $dataset['schema']) + ->sortBy(...$dataset['references']) + ->toArray(), + ), + serialize($merged->toArray()), + ); + static::assertNull((new SortedByConstraint(...$dataset['references']))->firstViolation($merged)); + } + + public function test_merges_multiple_sorted_runs_ascending(): void + { + $buckets = new Buckets(new MemoryBuckets()); + $buckets->storage()->append('a', array_to_rows([ + ['id' => 1], + ['id' => 4], + ['id' => 7], + ], schema(int_schema('id')))); + $buckets->storage()->append('b', array_to_rows([ + ['id' => 2], + ['id' => 5], + ['id' => 8], + ], schema(int_schema('id')))); + $buckets->storage()->append('c', array_to_rows([ + ['id' => 3], + ['id' => 6], + ['id' => 9], + ], schema(int_schema('id')))); + + $merge = new KWayMerge(refs(ref('id')->asc()), new AdaptiveBackend()); static::assertSame( [1, 2, 3, 4, 5, 6, 7, 8, 9], - array_map(static fn(Row $r): mixed => $r->get( + array_column( + BucketsStorageContext::rows($merge->merge([ + new BucketRun('a', $buckets), + new BucketRun('b', $buckets), + new BucketRun('c', $buckets), + ])), 'id', - ), BucketsStorageContext::rows($merge->merge([new BucketRun('a', $buckets), new BucketRun('b', $buckets), new BucketRun('c', $buckets)]))), + ), ); } @@ -61,130 +118,135 @@ public function test_merge_reads_each_run_through_its_own_storage(): void $spill = new Buckets(new MemoryBuckets()); $merged = new Buckets(new MemoryBuckets()); - $spill->storage()->append('sort-run-0', rows(schema(int_schema('id')), row(['id' => 1]), row(['id' => 3]))); - $merged->storage()->append('sort-merge-0', rows(schema(int_schema('id')), row(['id' => 2]), row(['id' => 4]))); + $spill->storage()->append('sort-run-0', array_to_rows([['id' => 1], ['id' => 3]], schema(int_schema('id')))); + $merged->storage()->append('sort-merge-0', array_to_rows([['id' => 2], ['id' => 4]], schema(int_schema('id')))); // neither storage holds the other's id, so a routing miss reads 0 batches instead of throwing static::assertSame([], BucketsStorageContext::rows($spill->storage()->get('sort-merge-0'))); static::assertSame([], BucketsStorageContext::rows($merged->storage()->get('sort-run-0'))); - $merge = new KWayMerge(refs(ref('id')->asc())); + $merge = new KWayMerge(refs(ref('id')->asc()), new AdaptiveBackend()); static::assertSame( [1, 2, 3, 4], - array_map(static fn(Row $r): mixed => $r->get( + array_column( + BucketsStorageContext::rows($merge->merge([ + new BucketRun('sort-run-0', $spill), + new BucketRun('sort-merge-0', $merged), + ])), 'id', - ), BucketsStorageContext::rows($merge->merge([ - new BucketRun('sort-run-0', $spill), - new BucketRun('sort-merge-0', $merged), - ]))), + ), ); } public function test_merges_multiple_sorted_runs_descending(): void { $buckets = new Buckets(new MemoryBuckets()); - $buckets->storage()->append('a', rows( - schema(int_schema('id')), - row(['id' => 7]), - row(['id' => 4]), - row(['id' => 1]), - )); - $buckets->storage()->append('b', rows( - schema(int_schema('id')), - row(['id' => 8]), - row(['id' => 5]), - row(['id' => 2]), - )); - - $merge = new KWayMerge(refs(ref('id')->desc())); + $buckets->storage()->append('a', array_to_rows([ + ['id' => 7], + ['id' => 4], + ['id' => 1], + ], schema(int_schema('id')))); + $buckets->storage()->append('b', array_to_rows([ + ['id' => 8], + ['id' => 5], + ['id' => 2], + ], schema(int_schema('id')))); + + $merge = new KWayMerge(refs(ref('id')->desc()), new AdaptiveBackend()); static::assertSame( [8, 7, 5, 4, 2, 1], - array_map(static fn(Row $r): mixed => $r->get( + array_column( + BucketsStorageContext::rows($merge->merge([ + new BucketRun('a', $buckets), + new BucketRun('b', $buckets), + ])), 'id', - ), BucketsStorageContext::rows($merge->merge([new BucketRun('a', $buckets), new BucketRun('b', $buckets)]))), + ), ); } public function test_merges_runs_of_uneven_length(): void { $buckets = new Buckets(new MemoryBuckets()); - $buckets->storage()->append('a', rows( - schema(int_schema('id')), - row(['id' => 1]), - row(['id' => 2]), - row(['id' => 3]), - )); - $buckets->storage()->append('b', rows(schema(int_schema('id')), row(['id' => 4]))); + $buckets->storage()->append('a', array_to_rows([ + ['id' => 1], + ['id' => 2], + ['id' => 3], + ], schema(int_schema('id')))); + $buckets->storage()->append('b', array_to_rows([['id' => 4]], schema(int_schema('id')))); - $merge = new KWayMerge(refs(ref('id')->asc())); + $merge = new KWayMerge(refs(ref('id')->asc()), new AdaptiveBackend()); static::assertSame( [1, 2, 3, 4], - array_map(static fn(Row $r): mixed => $r->get( + array_column( + BucketsStorageContext::rows($merge->merge([ + new BucketRun('a', $buckets), + new BucketRun('b', $buckets), + ])), 'id', - ), BucketsStorageContext::rows($merge->merge([new BucketRun('a', $buckets), new BucketRun('b', $buckets)]))), + ), ); } public function test_merges_non_numeric_values(): void { $buckets = new Buckets(new MemoryBuckets()); - $buckets->storage()->append('a', rows(schema(str_schema('id')), row(['id' => 'a']), row(['id' => 'c']))); - $buckets->storage()->append('b', rows(schema(str_schema('id')), row(['id' => 'b']), row(['id' => 'd']))); + $buckets->storage()->append('a', array_to_rows([['id' => 'a'], ['id' => 'c']], schema(str_schema('id')))); + $buckets->storage()->append('b', array_to_rows([['id' => 'b'], ['id' => 'd']], schema(str_schema('id')))); - $merge = new KWayMerge(refs(ref('id')->asc())); + $merge = new KWayMerge(refs(ref('id')->asc()), new AdaptiveBackend()); static::assertSame( ['a', 'b', 'c', 'd'], - array_map(static fn(Row $r): mixed => $r->get( + array_column( + BucketsStorageContext::rows($merge->merge([ + new BucketRun('a', $buckets), + new BucketRun('b', $buckets), + ])), 'id', - ), BucketsStorageContext::rows($merge->merge([new BucketRun('a', $buckets), new BucketRun('b', $buckets)]))), + ), ); } public function test_runs_under_different_schemas_are_conformed_to_the_first(): void { $buckets = new Buckets(new MemoryBuckets()); - $buckets->storage()->append('a', rows( + $buckets->storage()->append('a', array_to_rows( + [['id' => 1, 'name' => 'a']], schema(int_schema('id'), str_schema('name', true)), - row(['id' => 1, 'name' => 'a']), )); - $buckets->storage()->append('b', rows(schema(int_schema('id')), row(['id' => 2]))); + $buckets->storage()->append('b', array_to_rows([['id' => 2]], schema(int_schema('id')))); - $merge = new KWayMerge(refs(ref('id')->asc())); + $merge = new KWayMerge(refs(ref('id')->asc()), new AdaptiveBackend()); static::assertSame( [['id' => 1, 'name' => 'a'], ['id' => 2, 'name' => null]], - array_map( - static fn(Row $r): array => $r->toArray(), - BucketsStorageContext::rows($merge->merge([ - new BucketRun('a', $buckets), - new BucketRun('b', $buckets), - ])), - ), + BucketsStorageContext::rows($merge->merge([ + new BucketRun('a', $buckets), + new BucketRun('b', $buckets), + ])), ); } public function test_single_run_passthrough(): void { $buckets = new Buckets(new MemoryBuckets()); - $buckets->storage()->append('a', rows(schema(int_schema('id')), row(['id' => 1]), row(['id' => 2]))); + $buckets->storage()->append('a', array_to_rows([['id' => 1], ['id' => 2]], schema(int_schema('id')))); - $merge = new KWayMerge(refs(ref('id')->asc())); + $merge = new KWayMerge(refs(ref('id')->asc()), new AdaptiveBackend()); static::assertSame( [1, 2], - array_map(static fn(Row $r): mixed => $r->get( - 'id', - ), BucketsStorageContext::rows($merge->merge([new BucketRun('a', $buckets)]))), + array_column(BucketsStorageContext::rows($merge->merge([new BucketRun('a', $buckets)])), 'id'), ); } public function test_empty_bucket_set_yields_nothing(): void { - $merge = new KWayMerge(refs(ref('id')->asc())); + $merge = new KWayMerge(refs(ref('id')->asc()), new AdaptiveBackend()); static::assertSame([], BucketsStorageContext::rows($merge->merge([]))); } @@ -192,15 +254,14 @@ public function test_empty_bucket_set_yields_nothing(): void public function test_merged_rows_are_yielded_in_batches_of_batch_size(): void { $buckets = new Buckets(new MemoryBuckets()); - $buckets->storage()->append('a', rows( - schema(int_schema('id')), - row(['id' => 1]), - row(['id' => 3]), - row(['id' => 5]), - )); - $buckets->storage()->append('b', rows(schema(int_schema('id')), row(['id' => 2]), row(['id' => 4]))); + $buckets->storage()->append('a', array_to_rows([ + ['id' => 1], + ['id' => 3], + ['id' => 5], + ], schema(int_schema('id')))); + $buckets->storage()->append('b', array_to_rows([['id' => 2], ['id' => 4]], schema(int_schema('id')))); - $merge = new KWayMerge(refs(ref('id')->asc()), batchSize: 2); + $merge = new KWayMerge(refs(ref('id')->asc()), new AdaptiveBackend(), batchSize: 2); $sizes = []; @@ -217,6 +278,42 @@ public function test_throws_when_batch_size_below_one(): void $this->expectExceptionMessage('Batch size must be greater than 0, given: 0'); // @mago-ignore analysis:invalid-argument - new KWayMerge(refs(ref('id')->asc()), batchSize: 0); + new KWayMerge(refs(ref('id')->asc()), new AdaptiveBackend(), batchSize: 0); + } + + public function test_interleaves_runs_across_batch_boundaries_like_one_sort(): void + { + $buckets = new Buckets(new MemoryBuckets()); + $all = []; + + foreach (['a' => [2, 0], 'b' => [3, 1], 'c' => [5, 2]] as $run => [$batchSize, $offset]) { + $rows = []; + + for ($i = 0; $i < 10; $i++) { + $rows[] = ['id' => ($i * 3) + $offset, 'run' => $run]; + } + + $all = [...$all, ...$rows]; + + foreach (array_chunk($rows, $batchSize) as $chunk) { + $buckets->storage()->append($run, array_to_rows($chunk, schema(int_schema('id'), str_schema('run')))); + } + } + + $merged = []; + + foreach ((new KWayMerge(refs(ref('id')->asc()), new AdaptiveBackend(), batchSize: 4))->merge([ + new BucketRun('a', $buckets), + new BucketRun('b', $buckets), + new BucketRun('c', $buckets), + ]) as $batch) { + static::assertLessThanOrEqual(4, $batch->count()); + $merged = [...$merged, ...$batch->toArray()]; + } + + static::assertSame( + array_to_rows($all, schema(int_schema('id'), str_schema('run')))->sortBy(ref('id')->asc())->toArray(), + $merged, + ); } } diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Sort/Merge/RowsMinHeapTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Sort/Merge/RowsMinHeapTest.php deleted file mode 100644 index f330d50a47..0000000000 --- a/src/core/etl/tests/Flow/ETL/Tests/Unit/Sort/Merge/RowsMinHeapTest.php +++ /dev/null @@ -1,113 +0,0 @@ -asc()); - - $minHeap->push(row(['id' => 1]), 'cache_id'); - $minHeap->push(row(['id' => 2]), 'cache_id'); - $minHeap->push(row(['id' => 3]), 'cache_id'); - $minHeap->push(row(['id' => 4]), 'cache_id'); - $minHeap->push(row(['id' => 5]), 'cache_id'); - $minHeap->push(row(['id' => 6]), 'cache_id'); - - static::assertEquals( - [ - ['id' => 1], - ['id' => 2], - ['id' => 3], - ['id' => 4], - ['id' => 5], - ['id' => 6], - ], - array_map(static fn() => $minHeap->extract()->row->toArray(), range(1, count($minHeap))), - ); - } - - public function test_min_heap_desc(): void - { - $minHeap = new RowsMinHeap(ref('id')->desc()); - - $minHeap->push(row(['id' => 1]), 'cache_id'); - $minHeap->push(row(['id' => 2]), 'cache_id'); - $minHeap->push(row(['id' => 3]), 'cache_id'); - $minHeap->push(row(['id' => 4]), 'cache_id'); - $minHeap->push(row(['id' => 5]), 'cache_id'); - $minHeap->push(row(['id' => 6]), 'cache_id'); - - static::assertEquals( - [ - ['id' => 6], - ['id' => 5], - ['id' => 4], - ['id' => 3], - ['id' => 2], - ['id' => 1], - ], - array_map(static fn() => $minHeap->extract()->row->toArray(), range(1, count($minHeap))), - ); - } - - public function test_min_heap_on_non_numeric_values(): void - { - $minHeap = new RowsMinHeap(ref('id')->asc()); - - $minHeap->push(row(['id' => 'a']), 'cache_id'); - $minHeap->push(row(['id' => 'b']), 'cache_id'); - $minHeap->push(row(['id' => 'c']), 'cache_id'); - $minHeap->push(row(['id' => 'd']), 'cache_id'); - $minHeap->push(row(['id' => 'e']), 'cache_id'); - $minHeap->push(row(['id' => 'f']), 'cache_id'); - - static::assertEquals( - [ - ['id' => 'a'], - ['id' => 'b'], - ['id' => 'c'], - ['id' => 'd'], - ['id' => 'e'], - ['id' => 'f'], - ], - array_map(static fn() => $minHeap->extract()->row->toArray(), range(1, count($minHeap))), - ); - } - - public function test_min_heap_on_non_numeric_values_desc(): void - { - $minHeap = new RowsMinHeap(ref('id')->desc()); - - $minHeap->push(row(['id' => 'a']), 'cache_id'); - $minHeap->push(row(['id' => 'b']), 'cache_id'); - $minHeap->push(row(['id' => 'c']), 'cache_id'); - $minHeap->push(row(['id' => 'd']), 'cache_id'); - $minHeap->push(row(['id' => 'e']), 'cache_id'); - $minHeap->push(row(['id' => 'f']), 'cache_id'); - - static::assertEquals( - [ - ['id' => 'f'], - ['id' => 'e'], - ['id' => 'd'], - ['id' => 'c'], - ['id' => 'b'], - ['id' => 'a'], - ], - array_map(static fn() => $minHeap->extract()->row->toArray(), range(1, count($minHeap))), - ); - } -} diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Sort/RowOrderTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Sort/RowOrderTest.php new file mode 100644 index 0000000000..87592fcaca --- /dev/null +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Sort/RowOrderTest.php @@ -0,0 +1,258 @@ + 3], ['v' => null], ['v' => -1], ['v' => null]], schema(int_schema('v', true))); + $ascending = new RowOrder([ref('v')->asc()]); + $descending = new RowOrder([ref('v')->desc()]); + + static::assertSame([1, 3, 2, 0], $ascending->permutation($ascending->keys($rows), 4)); + static::assertSame([0, 2, 1, 3], $descending->permutation($descending->keys($rows), 4)); + } + + public function test_nan_sorts_after_every_value_and_ties_with_nan(): void + { + $rows = array_to_rows([ + ['v' => 1.0], + ['v' => NAN], + ['v' => -1.0], + ['v' => NAN], + ['v' => null], + ], schema(float_schema('v', true))); + $ascending = new RowOrder([ref('v')->asc()]); + $descending = new RowOrder([ref('v')->desc()]); + + static::assertSame([4, 2, 0, 1, 3], $ascending->permutation($ascending->keys($rows), 5)); + static::assertSame([1, 3, 0, 2, 4], $descending->permutation($descending->keys($rows), 5)); + static::assertSame(0, $ascending->compare($ascending->keys($rows), 1, $ascending->keys($rows), 3)); + } + + public function test_negative_zero_ties_with_zero(): void + { + $rows = array_to_rows([['v' => 0.0], ['v' => -0.0], ['v' => 0.0]], schema(float_schema('v'))); + $order = new RowOrder([ref('v')->desc()]); + + static::assertSame([0, 1, 2], $order->permutation($order->keys($rows), 3)); + } + + public function test_strings_compare_by_bytes_never_as_numbers(): void + { + $rows = array_to_rows([ + ['v' => '9'], + ['v' => '10'], + ['v' => 'a'], + ['v' => 'B'], + ['v' => ''], + ['v' => '09'], + ['v' => '100'], + ], schema(str_schema('v'))); + $order = new RowOrder([ref('v')->asc()]); + + static::assertSame( + ['', '09', '10', '100', '9', 'B', 'a'], + $rows + ->gather($order->permutation($order->keys($rows), 7)) + ->column('v') + ->values(), + ); + } + + public function test_false_sorts_before_true(): void + { + $rows = array_to_rows([['v' => true], ['v' => false], ['v' => null]], schema(bool_schema('v', true))); + $order = new RowOrder([ref('v')->asc()]); + + static::assertSame([2, 1, 0], $order->permutation($order->keys($rows), 3)); + } + + public function test_datetimes_sort_by_instant_whatever_their_zone(): void + { + $rows = array_to_rows([ + ['v' => new DateTimeImmutable('2026-01-01 02:00:00', new DateTimeZone('Europe/Warsaw'))], + ['v' => new DateTimeImmutable('2026-01-01 00:30:00', new DateTimeZone('UTC'))], + ['v' => new DateTimeImmutable('2026-01-01 01:00:00', new DateTimeZone('UTC'))], + ], schema(datetime_schema('v'))); + $order = new RowOrder([ref('v')->asc()]); + + static::assertSame([1, 0, 2], $order->permutation($order->keys($rows), 3)); + } + + public function test_dates_sort_by_day_and_times_by_duration(): void + { + $rows = array_to_rows( + [ + ['d' => new DateTimeImmutable('2026-03-01'), 't' => new DateInterval('PT2H')], + ['d' => new DateTimeImmutable('1969-12-31'), 't' => new DateInterval('PT90M')], + ['d' => new DateTimeImmutable('2026-02-28'), 't' => new DateInterval('PT1H30M1S')], + ], + schema(date_schema('d'), time_schema('t')), + ); + $byDay = new RowOrder([ref('d')->asc()]); + $byDuration = new RowOrder([ref('t')->asc()]); + + static::assertSame([1, 2, 0], $byDay->permutation($byDay->keys($rows), 3)); + static::assertSame([1, 2, 0], $byDuration->permutation($byDuration->keys($rows), 3)); + } + + public function test_values_without_an_order_tie_and_keep_their_input_order(): void + { + $rows = array_to_rows([ + ['v' => Uuid::fromString('00000000-0000-4000-8000-000000000002')], + ['v' => Uuid::fromString('00000000-0000-4000-8000-000000000001')], + ], schema(uuid_schema('v'))); + $order = new RowOrder([ref('v')->asc()]); + + static::assertSame([0, 1], $order->permutation($order->keys($rows), 2)); + } + + public function test_later_keys_break_ties_in_their_own_direction(): void + { + $rows = array_to_rows( + [['a' => 1, 'b' => 'x'], ['a' => 2, 'b' => 'y'], ['a' => 1, 'b' => 'y'], ['a' => 2, 'b' => 'x']], + schema(int_schema('a'), str_schema('b')), + ); + $order = new RowOrder([ref('a')->desc(), ref('b')->asc()]); + + static::assertSame([3, 1, 0, 2], $order->permutation($order->keys($rows), 4)); + } + + public function test_rows_equal_on_every_key_keep_their_input_order(): void + { + $rows = array_to_rows([['v' => 1], ['v' => 1], ['v' => 0], ['v' => 1]], schema(int_schema('v'))); + $order = new RowOrder([ref('v')->desc()]); + + static::assertSame([0, 1, 3, 2], $order->permutation($order->keys($rows), 4)); + } + + public function test_a_batch_of_fewer_than_two_rows_keeps_its_order(): void + { + $order = new RowOrder([ref('v')->asc()]); + + static::assertSame([], $order->permutation($order->keys(array_to_rows([], schema(int_schema('v')))), 0)); + static::assertSame( + [0], + $order->permutation($order->keys(array_to_rows([['v' => 1]], schema(int_schema('v')))), 1), + ); + } + + public function test_first_after_finds_the_first_row_past_the_bound(): void + { + $rows = array_to_rows([['v' => 1], ['v' => 2], ['v' => 2], ['v' => 3]], schema(int_schema('v'))); + $bound = array_to_rows([['v' => 2]], schema(int_schema('v'))); + $order = new RowOrder([ref('v')->asc()]); + $keys = $order->keys($rows); + + static::assertSame(3, $order->firstAfter($keys, 0, 4, $order->keys($bound), 0, true)); + static::assertSame(1, $order->firstAfter($keys, 0, 4, $order->keys($bound), 0, false)); + static::assertSame(3, $order->firstAfter($keys, 3, 4, $order->keys($bound), 0, false)); + } + + public function test_compare_agrees_with_the_permutation(): void + { + for ($seed = 1; $seed <= 50; $seed++) { + $dataset = SortDatasetMother::random($seed); + $rows = array_to_rows(array_merge(...$dataset['runs']), $dataset['schema']); + $order = new RowOrder($dataset['references']); + $keys = $order->keys($rows); + $permutation = $order->permutation($keys, $rows->count()); + + for ($i = 1; $i < $rows->count(); $i++) { + $comparison = $order->compare($keys, $permutation[$i - 1], $keys, $permutation[$i]); + + static::assertTrue( + $comparison < 0 || $comparison === 0 && $permutation[$i - 1] < $permutation[$i], + "seed {$seed}, position {$i}", + ); + } + } + } + + public function test_the_explicit_default_placement_orders_like_the_implicit_one(): void + { + $rows = array_to_rows([ + ['v' => 1.0], + ['v' => null], + ['v' => NAN], + ['v' => -1.0], + ], schema(float_schema('v', true))); + + foreach ([ + [ref('v')->asc(NullsOrder::FIRST), ref('v')->asc()], + [ref('v')->desc(NullsOrder::LAST), ref('v')->desc()], + ] as [$explicit, $implicit]) { + $explicitOrder = new RowOrder([$explicit]); + $implicitOrder = new RowOrder([$implicit]); + + static::assertSame( + $implicitOrder->permutation($implicitOrder->keys($rows), 4), + $explicitOrder->permutation($explicitOrder->keys($rows), 4), + ); + } + + $ascending = new RowOrder([ref('v')->asc(NullsOrder::FIRST)]); + $descending = new RowOrder([ref('v')->desc(NullsOrder::LAST)]); + + static::assertSame([1, 3, 0, 2], $ascending->permutation($ascending->keys($rows), 4)); + static::assertSame([2, 0, 3, 1], $descending->permutation($descending->keys($rows), 4)); + } + + public function test_a_reference_moves_nulls_to_the_other_end(): void + { + $rows = array_to_rows([ + ['v' => 1.0], + ['v' => null], + ['v' => NAN], + ['v' => -1.0], + ], schema(float_schema('v', true))); + $ascendingLast = new RowOrder([ref('v')->asc(NullsOrder::LAST)]); + $descendingFirst = new RowOrder([ref('v')->desc(NullsOrder::FIRST)]); + + static::assertSame([3, 0, 2, 1], $ascendingLast->permutation($ascendingLast->keys($rows), 4)); + static::assertSame([1, 2, 0, 3], $descendingFirst->permutation($descendingFirst->keys($rows), 4)); + } +} diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Sort/SortStepsTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Sort/SortStepsTest.php index 6a06ff227b..965062cf70 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Unit/Sort/SortStepsTest.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Sort/SortStepsTest.php @@ -5,17 +5,17 @@ namespace Flow\ETL\Tests\Unit\Sort; use Flow\ETL\Bucketing\Storage\MemoryBuckets; -use Flow\ETL\Processor\BucketingProcessor; +use Flow\ETL\Dataset\Memory\Unit; use Flow\ETL\Processor\MemorySortProcessor; use Flow\ETL\Processor\MergeSortProcessor; -use Flow\ETL\Row; +use Flow\ETL\Sort\ExternalSort; use Flow\ETL\Sort\SortSteps; use Flow\ETL\Tests\Double\RecordingBucketsStorage; use Flow\ETL\Tests\FlowTestCase; use Generator; use function array_filter; -use function array_map; +use function Flow\ETL\DSL\array_to_rows; use function Flow\ETL\DSL\config; use function Flow\ETL\DSL\config_builder; use function Flow\ETL\DSL\external_sort; @@ -24,8 +24,6 @@ use function Flow\ETL\DSL\memory_sort; use function Flow\ETL\DSL\ref; use function Flow\ETL\DSL\refs; -use function Flow\ETL\DSL\row; -use function Flow\ETL\DSL\rows; use function Flow\ETL\DSL\schema; use function iterator_to_array; use function range; @@ -33,16 +31,36 @@ final class SortStepsTest extends FlowTestCase { - public function test_external_sort_config_builds_bucketing_and_merge_processors(): void + public function test_external_sort_config_builds_one_merge_sort_processor(): void { $steps = SortSteps::of( refs(ref('id')), config_builder()->sort(external_sort()->storage(new MemoryBuckets()))->build(), ); - static::assertCount(2, $steps); - static::assertInstanceOf(BucketingProcessor::class, $steps[0]); - static::assertInstanceOf(MergeSortProcessor::class, $steps[1]); + static::assertCount(1, $steps); + static::assertInstanceOf(MergeSortProcessor::class, $steps[0]); + } + + public function test_external_carries_the_configured_memory_limit(): void + { + $sort = SortSteps::external( + refs(ref('id')), + config_builder()->sort(external_sort()->memoryLimit(Unit::fromMb(3)))->build(), + ); + + static::assertInstanceOf(ExternalSort::class, $sort); + static::assertSame(Unit::fromMb(3)->inBytes(), $sort->memoryLimit->inBytes()); + } + + public function test_external_is_null_for_the_memory_sort(): void + { + static::assertNull(SortSteps::external(refs(ref('id')), config_builder()->sort(memory_sort())->build())); + } + + public function test_pinned_algorithm_wins_over_configuration(): void + { + static::assertNull(SortSteps::external(refs(ref('id')), config_builder()->build(), memory_sort())); } public function test_external_sort_resolves_both_phases_over_the_same_storage_by_default(): void @@ -52,21 +70,21 @@ public function test_external_sort_resolves_both_phases_over_the_same_storage_by $steps = SortSteps::of( refs(ref('id')), - config_builder()->sort(external_sort()->storage($spill)->runSize(1)->bucketsCount(2))->build(), + config_builder() + ->sort(external_sort()->storage($spill)->memoryLimit(Unit::fromBytes(1))->bucketsCount(2)) + ->build(), ); $context = flow_context(config()); $input = (static function (): Generator { - yield rows( - schema(int_schema('id')), - ...array_map(static fn(int $i): Row => row(['id' => $i]), range(9, 0)), - ); + foreach (range(9, 0) as $i) { + yield array_to_rows([['id' => $i]], schema(int_schema('id'))); + } })(); - static::assertInstanceOf(BucketingProcessor::class, $steps[0]); - static::assertInstanceOf(MergeSortProcessor::class, $steps[1]); + static::assertInstanceOf(MergeSortProcessor::class, $steps[0]); - iterator_to_array($steps[1]->process($steps[0]->process($input, $context), $context), false); + iterator_to_array($steps[0]->process($input, $context), false); static::assertNotSame([], array_filter($spill->appended, static fn(string $id): bool => str_starts_with( $id, @@ -82,21 +100,26 @@ public function test_external_sort_routes_merged_runs_to_the_merge_storage_when_ $steps = SortSteps::of( refs(ref('id')), config_builder() - ->sort(external_sort()->storage($spill)->mergeStorage($merge)->runSize(1)->bucketsCount(2)) + ->sort( + external_sort() + ->storage($spill) + ->mergeStorage($merge) + ->memoryLimit(Unit::fromBytes(1)) + ->bucketsCount(2), + ) ->build(), ); $context = flow_context(config()); $input = (static function (): Generator { - yield rows( - schema(int_schema('id')), - ...array_map(static fn(int $i): Row => row(['id' => $i]), range(9, 0)), - ); + foreach (range(9, 0) as $i) { + yield array_to_rows([['id' => $i]], schema(int_schema('id'))); + } })(); - static::assertInstanceOf(MergeSortProcessor::class, $steps[1]); + static::assertInstanceOf(MergeSortProcessor::class, $steps[0]); - iterator_to_array($steps[1]->process($steps[0]->process($input, $context), $context), false); + iterator_to_array($steps[0]->process($input, $context), false); static::assertSame([], array_filter($spill->appended, static fn(string $id): bool => str_starts_with( $id, @@ -112,9 +135,8 @@ public function test_external_sort_is_the_default(): void { $steps = SortSteps::of(refs(ref('id')), config_builder()->build()); - static::assertCount(2, $steps); - static::assertInstanceOf(BucketingProcessor::class, $steps[0]); - static::assertInstanceOf(MergeSortProcessor::class, $steps[1]); + static::assertCount(1, $steps); + static::assertInstanceOf(MergeSortProcessor::class, $steps[0]); } public function test_memory_sort_config_builds_a_memory_sort_processor(): void diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Sort/ValuesSorterTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Sort/ValuesSorterTest.php deleted file mode 100644 index 94fcc05d30..0000000000 --- a/src/core/etl/tests/Flow/ETL/Tests/Unit/Sort/ValuesSorterTest.php +++ /dev/null @@ -1,105 +0,0 @@ -expectException(SchemaDefinitionNotUniqueException::class); $this->expectExceptionMessage('Entry definitions must be unique, duplicated entries: [id]'); - (new AddRowIndexTransformer('id', StartFrom::ZERO))->transform( - rows(schema(int_schema('id')), row(['id' => 1])), - flow_context(), - ); + (new AddRowIndexTransformer('id', StartFrom::ZERO))->transform(array_to_rows([[ + 'id' => 1, + ]], schema(int_schema('id'))), flow_context()); } public function test_bind_adds_an_int_index_column(): void @@ -45,11 +43,11 @@ public function test_index_keeps_incrementing_across_batches(): void static::assertSame( [['id' => 1, 'idx' => 0], ['id' => 2, 'idx' => 1]], - $transformer->transform(rows($schema, row(['id' => 1]), row(['id' => 2])), $context)->toArray(), + $transformer->transform(array_to_rows([['id' => 1], ['id' => 2]], $schema), $context)->toArray(), ); static::assertSame( [['id' => 3, 'idx' => 2]], - $transformer->transform(rows($schema, row(['id' => 3])), $context)->toArray(), + $transformer->transform(array_to_rows([['id' => 3]], $schema), $context)->toArray(), ); } @@ -59,14 +57,14 @@ public function test_a_fresh_instance_starts_counting_again(StartFrom $startFrom { $transformer = new AddRowIndexTransformer('idx', $startFrom); $schema = schema(int_schema('id')); - $transformer->transform(rows($schema, row(['id' => 1]), row(['id' => 2])), flow_context()); + $transformer->transform(array_to_rows([['id' => 1], ['id' => 2]], $schema), flow_context()); $fresh = $transformer->fresh(); static::assertNotSame($transformer, $fresh); static::assertSame( [['id' => 1, 'idx' => $first]], - $fresh->transform(rows($schema, row(['id' => 1])), flow_context())->toArray(), + $fresh->transform(array_to_rows([['id' => 1]], $schema), flow_context())->toArray(), ); } @@ -75,7 +73,7 @@ public function test_index_starting_from_one(): void static::assertSame( [['id' => 1, 'idx' => 1]], (new AddRowIndexTransformer('idx', StartFrom::ONE)) - ->transform(rows(schema(int_schema('id')), row(['id' => 1])), flow_context()) + ->transform(array_to_rows([['id' => 1]], schema(int_schema('id'))), flow_context()) ->toArray(), ); } diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Transformer/CollectReferencesTransformerTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Transformer/CollectReferencesTransformerTest.php index 22735c8b82..597efd0498 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Unit/Transformer/CollectReferencesTransformerTest.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Transformer/CollectReferencesTransformerTest.php @@ -8,11 +8,10 @@ use Flow\ETL\Transformer\CollectReferencesTransformer; use function array_map; +use function Flow\ETL\DSL\array_to_rows; use function Flow\ETL\DSL\flow_context; use function Flow\ETL\DSL\int_schema; use function Flow\ETL\DSL\refs; -use function Flow\ETL\DSL\row; -use function Flow\ETL\DSL\rows; use function Flow\ETL\DSL\schema; use function Flow\ETL\DSL\str_schema; @@ -28,7 +27,7 @@ public function test_bind_returns_the_input_schema(): void public function test_collecting_references_from_the_batch_schema(): void { $references = refs(); - $batch = rows(schema(int_schema('id'), str_schema('name')), row(['id' => 1, 'name' => 'John'])); + $batch = array_to_rows([['id' => 1, 'name' => 'John']], schema(int_schema('id'), str_schema('name'))); static::assertSame($batch, (new CollectReferencesTransformer($references))->transform($batch, flow_context())); static::assertSame( diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Transformer/CrossJoinRowsTransformerTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Transformer/CrossJoinRowsTransformerTest.php index c80bc24eeb..ae117d0a34 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Unit/Transformer/CrossJoinRowsTransformerTest.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Transformer/CrossJoinRowsTransformerTest.php @@ -9,11 +9,10 @@ use Flow\ETL\Tests\Mother\PhysicalPlanMother; use Flow\ETL\Transformer\CrossJoinRowsTransformer; +use function Flow\ETL\DSL\array_to_rows; use function Flow\ETL\DSL\flow_context; use function Flow\ETL\DSL\from_rows; use function Flow\ETL\DSL\int_schema; -use function Flow\ETL\DSL\row; -use function Flow\ETL\DSL\rows; use function Flow\ETL\DSL\schema; use function Flow\ETL\DSL\str_schema; @@ -24,7 +23,9 @@ public function test_bind_concatenates_the_left_and_the_right_schema(): void static::assertEquals( schema(int_schema('id'), str_schema('name')), (new CrossJoinRowsTransformer( - PhysicalPlanMother::reading(from_rows(rows(schema(str_schema('name')), row(['name' => 'Alice'])))), + PhysicalPlanMother::reading(from_rows(array_to_rows([[ + 'name' => 'Alice', + ]], schema(str_schema('name'))))), new Executor(), ))->bind(schema(int_schema('id')))->output, ); @@ -35,7 +36,9 @@ public function test_bind_prefixes_every_right_side_column(): void static::assertEquals( schema(int_schema('id'), str_schema('right_name')), (new CrossJoinRowsTransformer( - PhysicalPlanMother::reading(from_rows(rows(schema(str_schema('name')), row(['name' => 'Alice'])))), + PhysicalPlanMother::reading(from_rows(array_to_rows([[ + 'name' => 'Alice', + ]], schema(str_schema('name'))))), new Executor(), 'right_', ))->bind(schema(int_schema('id')))->output, @@ -45,18 +48,17 @@ public function test_bind_prefixes_every_right_side_column(): void public function test_the_right_side_is_fetched_through_a_frame_output(): void { $transformer = new CrossJoinRowsTransformer( - PhysicalPlanMother::reading(from_rows(rows( - schema(str_schema('name')), - row(['name' => 'Alice']), - row(['name' => 'Bob']), - ))), + PhysicalPlanMother::reading(from_rows(array_to_rows([ + ['name' => 'Alice'], + ['name' => 'Bob'], + ], schema(str_schema('name'))))), new Executor(), 'r_', ); static::assertSame( [['id' => 1, 'r_name' => 'Alice'], ['id' => 1, 'r_name' => 'Bob']], - $transformer->transform(rows(schema(int_schema('id')), row(['id' => 1])), flow_context())->toArray(), + $transformer->transform(array_to_rows([['id' => 1]], schema(int_schema('id'))), flow_context())->toArray(), ); } } diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Transformer/DerivedColumnsTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Transformer/DerivedColumnsTest.php index f99b465acb..ed0d31e347 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Unit/Transformer/DerivedColumnsTest.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Transformer/DerivedColumnsTest.php @@ -4,12 +4,16 @@ namespace Flow\ETL\Tests\Unit\Transformer; +use Flow\ETL\Column\PhpBackend; +use Flow\ETL\Column\ValueColumn; use Flow\ETL\Exception\SchemaMismatchException; +use Flow\ETL\Tests\Double\SpyBackend; use Flow\ETL\Tests\FlowTestCase; +use Flow\ETL\Tests\Mother\ColumnMother; use Flow\ETL\Transformer\DerivedColumns; +use function Flow\ETL\DSL\array_to_rows; use function Flow\ETL\DSL\int_schema; -use function Flow\ETL\DSL\row; use function Flow\ETL\DSL\schema; use function Flow\ETL\DSL\str_schema; @@ -31,46 +35,101 @@ public function test_declare_replaces_an_existing_column(): void ); } - public function test_value_casts_into_the_declared_type(): void + public function test_stored_adopts_a_column_of_the_same_type(): void { - static::assertSame(5, (new DerivedColumns())->value(int_schema('a'), '5', 0)); + $column = ColumnMother::of(int_schema('a', nullable: true), [1, 2]); + $backend = new SpyBackend(); + + $stored = (new DerivedColumns())->stored(int_schema('a'), $column, $backend); + + static::assertSame([1, 2], $stored->values()); + static::assertSame(1, $backend->adopts()); } - public function test_value_keeps_null_under_a_nullable_definition(): void + public function test_stored_rebuilds_a_column_of_another_type(): void { - static::assertNull((new DerivedColumns())->value(int_schema('a', nullable: true), null, 0)); + $stored = (new DerivedColumns())->stored( + int_schema('a'), + ColumnMother::of(str_schema('a'), ['5', '6']), + new PhpBackend(), + ); + + static::assertSame([5, 6], $stored->values()); } - public function test_value_names_the_row_of_a_null_under_a_not_null_definition(): void + public function test_stored_rebuilds_an_untyped_column(): void + { + static::assertSame( + [5, null], + (new DerivedColumns()) + ->stored(int_schema('a', nullable: true), new ValueColumn(['5', null]), new PhpBackend()) + ->values(), + ); + } + + public function test_stored_keeps_null_under_a_nullable_definition(): void + { + static::assertSame( + [null], + (new DerivedColumns()) + ->stored( + int_schema('a', nullable: true), + ColumnMother::of(int_schema('a', nullable: true), [null]), + new PhpBackend(), + ) + ->values(), + ); + } + + public function test_stored_names_the_row_of_a_null_under_a_not_null_definition(): void { $this->expectException(SchemaMismatchException::class); $this->expectExceptionMessage( 'Rows do not match their schema: column "a" (row 3): could not convert null to integer, column is not nullable', ); - (new DerivedColumns())->value(int_schema('a'), null, 3); + (new DerivedColumns())->stored(int_schema('a'), ColumnMother::of(int_schema('a', nullable: true), [ + 1, + 2, + 3, + null, + ]), new PhpBackend()); } - public function test_rows_trusts_a_batch_declared_under_the_output_schema(): void + public function test_rows_adds_the_derived_column_under_the_output_schema(): void { $output = schema(int_schema('a'), str_schema('note', nullable: true)); static::assertSame( - [['a' => 1]], + [['a' => 1, 'note' => 'x']], (new DerivedColumns()) - ->rows($output, $output, [row(['a' => 1])]) + ->rows( + array_to_rows([['a' => 1]], schema(int_schema('a'))), + $output, + $output, + 'note', + (new PhpBackend())->constant(str_schema('note', nullable: true), 'x', 1), + new PhpBackend(), + ) ->toArray(), ); } - public function test_rows_conforms_a_batch_declared_under_another_schema(): void + public function test_rows_matches_a_batch_declared_under_another_schema(): void { + $declared = schema(int_schema('a'), str_schema('note', nullable: true)); + static::assertSame( - [['a' => 1, 'note' => null]], + [['a' => 1, 'note' => 'x', 'extra' => null]], (new DerivedColumns()) - ->rows(schema(int_schema('a')), schema(int_schema('a'), str_schema('note', nullable: true)), [row([ - 'a' => 1, - ])]) + ->rows( + array_to_rows([['a' => 1]], schema(int_schema('a'))), + $declared, + $declared->add(str_schema('extra', nullable: true)), + 'note', + (new PhpBackend())->constant(str_schema('note', nullable: true), 'x', 1), + new PhpBackend(), + ) ->toArray(), ); } diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Transformer/DropDuplicatesTransformerTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Transformer/DropDuplicatesTransformerTest.php index af47667367..779f3c10be 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Unit/Transformer/DropDuplicatesTransformerTest.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Transformer/DropDuplicatesTransformerTest.php @@ -8,11 +8,10 @@ use Flow\ETL\Tests\FlowTestCase; use Flow\ETL\Transformer\DropDuplicatesTransformer; +use function Flow\ETL\DSL\array_to_rows; use function Flow\ETL\DSL\config; use function Flow\ETL\DSL\flow_context; use function Flow\ETL\DSL\int_schema; -use function Flow\ETL\DSL\row; -use function Flow\ETL\DSL\rows; use function Flow\ETL\DSL\schema; use function Flow\ETL\DSL\str_schema; @@ -37,21 +36,21 @@ public function test_dropping_duplicated_entries_from_rows(): void { $transformer = new DropDuplicatesTransformer('id'); - $rows = rows( + $rows = array_to_rows( + [ + ['id' => 1, 'name' => 'name1'], + ['id' => 1, 'name' => 'name1'], + ['id' => 2, 'name' => 'name2'], + ['id' => 2, 'name' => 'name2'], + ['id' => 3, 'name' => 'name3'], + ], schema(int_schema('id'), str_schema('name')), - row(['id' => 1, 'name' => 'name1']), - row(['id' => 1, 'name' => 'name1']), - row(['id' => 2, 'name' => 'name2']), - row(['id' => 2, 'name' => 'name2']), - row(['id' => 3, 'name' => 'name3']), ); static::assertEquals( - rows( + array_to_rows( + [['id' => 1, 'name' => 'name1'], ['id' => 2, 'name' => 'name2'], ['id' => 3, 'name' => 'name3']], schema(int_schema('id'), str_schema('name')), - row(['id' => 1, 'name' => 'name1']), - row(['id' => 2, 'name' => 'name2']), - row(['id' => 3, 'name' => 'name3']), ), $transformer->transform($rows, flow_context(config())), ); @@ -61,23 +60,27 @@ public function test_dropping_duplicates_when_not_all_rows_has_expected_entry(): { $transformer = new DropDuplicatesTransformer('id'); - $rows = rows( + $rows = array_to_rows( + [ + ['id' => 1, 'name' => 'name1'], + ['id' => 1, 'name' => 'name1'], + ['id' => 2, 'name' => 'name2'], + ['id' => 2, 'name' => 'name2'], + ['name' => 'name3'], + ['id' => 4, 'name' => 'name4'], + ], schema(int_schema('id', nullable: true), str_schema('name')), - row(['id' => 1, 'name' => 'name1']), - row(['id' => 1, 'name' => 'name1']), - row(['id' => 2, 'name' => 'name2']), - row(['id' => 2, 'name' => 'name2']), - row(['name' => 'name3']), - row(['id' => 4, 'name' => 'name4']), ); static::assertEquals( - rows( + array_to_rows( + [ + ['id' => 1, 'name' => 'name1'], + ['id' => 2, 'name' => 'name2'], + ['name' => 'name3'], + ['id' => 4, 'name' => 'name4'], + ], schema(int_schema('id', nullable: true), str_schema('name')), - row(['id' => 1, 'name' => 'name1']), - row(['id' => 2, 'name' => 'name2']), - row(['name' => 'name3']), - row(['id' => 4, 'name' => 'name4']), ), $transformer->transform($rows, flow_context(config())), ); @@ -87,21 +90,21 @@ public function test_dropping_duplications_based_on_two_entries(): void { $transformer = new DropDuplicatesTransformer('id', 'name'); - $rows = rows( + $rows = array_to_rows( + [ + ['id' => 1, 'name' => 'name1'], + ['id' => 1, 'name' => 'name1'], + ['id' => 2, 'name' => 'name2'], + ['id' => 2, 'name' => 'name2'], + ['id' => 3, 'name' => 'name3'], + ], schema(int_schema('id'), str_schema('name')), - row(['id' => 1, 'name' => 'name1']), - row(['id' => 1, 'name' => 'name1']), - row(['id' => 2, 'name' => 'name2']), - row(['id' => 2, 'name' => 'name2']), - row(['id' => 3, 'name' => 'name3']), ); static::assertEquals( - rows( + array_to_rows( + [['id' => 1, 'name' => 'name1'], ['id' => 2, 'name' => 'name2'], ['id' => 3, 'name' => 'name3']], schema(int_schema('id'), str_schema('name')), - row(['id' => 1, 'name' => 'name1']), - row(['id' => 2, 'name' => 'name2']), - row(['id' => 3, 'name' => 'name3']), ), $transformer->transform($rows, flow_context(config())), ); diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Transformer/DropEntriesTransformerTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Transformer/DropEntriesTransformerTest.php index 97fb86d64f..b55dfc5fa3 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Unit/Transformer/DropEntriesTransformerTest.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Transformer/DropEntriesTransformerTest.php @@ -7,12 +7,11 @@ use Flow\ETL\Tests\FlowTestCase; use Flow\ETL\Transformer\DropEntriesTransformer; +use function Flow\ETL\DSL\array_to_rows; use function Flow\ETL\DSL\config; use function Flow\ETL\DSL\flow_context; use function Flow\ETL\DSL\int_schema; use function Flow\ETL\DSL\json_schema; -use function Flow\ETL\DSL\row; -use function Flow\ETL\DSL\rows; use function Flow\ETL\DSL\schema; use function Flow\ETL\DSL\string_schema; use function Flow\Types\DSL\type_json; @@ -40,9 +39,9 @@ public function test_bind_removes_the_dropped_columns(): void public function test_dropping_entries(): void { - $rows = rows( + $rows = array_to_rows( + [['id' => 1, 'name' => 'Row Name', 'array' => type_json()->cast(['test'])]], schema(int_schema('id'), string_schema('name'), json_schema('array')), - row(['id' => 1, 'name' => 'Row Name', 'array' => type_json()->cast(['test'])]), ); $transformer = new DropEntriesTransformer('id', 'array'); @@ -56,9 +55,9 @@ public function test_dropping_entries(): void public function test_removing_not_existing_entries(): void { - $rows = rows( + $rows = array_to_rows( + [['id' => 1, 'name' => 'Row Name', 'array' => type_json()->cast(['test'])]], schema(int_schema('id'), string_schema('name'), json_schema('array')), - row(['id' => 1, 'name' => 'Row Name', 'array' => type_json()->cast(['test'])]), ); $transformer = new DropEntriesTransformer('not_existing'); diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Transformer/DuplicateRowTransformerTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Transformer/DuplicateRowTransformerTest.php index f8e53e1db0..c9e80c39e7 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Unit/Transformer/DuplicateRowTransformerTest.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Transformer/DuplicateRowTransformerTest.php @@ -5,6 +5,7 @@ namespace Flow\ETL\Tests\Unit\Transformer; use DateTimeImmutable; +use Flow\ETL\Column\AdaptiveBackend; use Flow\ETL\Exception\InvalidArgumentException; use Flow\ETL\Exception\SchemaDefinitionNotFoundException; use Flow\ETL\Tests\FlowTestCase; @@ -13,6 +14,7 @@ use Flow\ETL\WithEntry; use Flow\Types\Exception\InvalidTypeException; +use function Flow\ETL\DSL\array_to_rows; use function Flow\ETL\DSL\bool_schema; use function Flow\ETL\DSL\config; use function Flow\ETL\DSL\date_schema; @@ -21,7 +23,6 @@ use function Flow\ETL\DSL\int_schema; use function Flow\ETL\DSL\lit; use function Flow\ETL\DSL\ref; -use function Flow\ETL\DSL\row; use function Flow\ETL\DSL\rows; use function Flow\ETL\DSL\schema; use function Flow\ETL\DSL\string_schema; @@ -60,7 +61,7 @@ public function test_a_literal_condition_duplicates_every_row(): void ['id' => 2, 'flag' => true], ], (new DuplicateRowTransformer(true, with_entry('flag', lit(true)))) - ->transform(rows(schema(int_schema('id')), row(['id' => 1]), row(['id' => 2])), flow_context(config())) + ->transform(array_to_rows([['id' => 1], ['id' => 2]], schema(int_schema('id'))), flow_context(config())) ->toArray(), ); } @@ -71,10 +72,9 @@ public function test_a_transformation_that_adds_a_column_conforms_the_untouched_ ref('status')->equals(lit('inactive')), with_entry('flag', lit(true)), ))->transform( - rows( + array_to_rows( + [['id' => 1, 'status' => 'active'], ['id' => 2, 'status' => 'inactive']], schema(int_schema('id'), string_schema('status')), - row(['id' => 1, 'status' => 'active']), - row(['id' => 2, 'status' => 'inactive']), ), flow_context(config()), ); @@ -96,10 +96,12 @@ public function test_a_transformation_that_narrows_a_column_widens_it_back(): vo ref('status')->equals(lit('inactive')), with_entry('amount', ref('amount')->multiply(lit(-1))), ))->transform( - rows( + array_to_rows( + [ + ['id' => 1, 'status' => 'active', 'amount' => 100], + ['id' => 2, 'status' => 'inactive', 'amount' => 100], + ], schema(int_schema('id'), string_schema('status'), int_schema('amount', nullable: true)), - row(['id' => 1, 'status' => 'active', 'amount' => 100]), - row(['id' => 2, 'status' => 'inactive', 'amount' => 100]), ), flow_context(config()), ); @@ -111,7 +113,30 @@ public function test_a_transformation_that_narrows_a_column_widens_it_back(): vo public function test_applying_two_transformations(): void { - $rows = rows( + $rows = array_to_rows( + [ + [ + 'id' => 1, + 'status' => 'active', + 'amount' => 100, + 'date_created' => type_date()->cast('2025-01-01'), + 'date_deactivated' => null, + ], + [ + 'id' => 2, + 'status' => 'inactive', + 'amount' => 100, + 'date_created' => type_date()->cast('2025-01-01'), + 'date_deactivated' => type_date()->cast('2025-01-03'), + ], + [ + 'id' => 3, + 'status' => 'active', + 'amount' => 100, + 'date_created' => type_date()->cast('2025-01-01'), + 'date_deactivated' => null, + ], + ], schema( int_schema('id'), string_schema('status'), @@ -120,27 +145,6 @@ public function test_applying_two_transformations(): void date_schema('date_deactivated', nullable: true), date_schema('date_updated', nullable: true), ), - row([ - 'id' => 1, - 'status' => 'active', - 'amount' => 100, - 'date_created' => type_date()->cast('2025-01-01'), - 'date_deactivated' => null, - ]), - row([ - 'id' => 2, - 'status' => 'inactive', - 'amount' => 100, - 'date_created' => type_date()->cast('2025-01-01'), - 'date_deactivated' => type_date()->cast('2025-01-03'), - ]), - row([ - 'id' => 3, - 'status' => 'active', - 'amount' => 100, - 'date_created' => type_date()->cast('2025-01-01'), - 'date_deactivated' => null, - ]), ); $transformedRows = (new DuplicateRowTransformer( @@ -213,11 +217,13 @@ public function test_bind_refuses_a_condition_referencing_a_column_missing_from_ public function test_doing_nothing_when_condition_is_not_satisfied(): void { - $rows = rows( + $rows = array_to_rows( + [ + ['id' => 1, 'status' => 'active', 'amount' => 100], + ['id' => 2, 'status' => 'active', 'amount' => 100], + ['id' => 3, 'status' => 'active', 'amount' => 100], + ], schema(int_schema('id'), string_schema('status'), int_schema('amount')), - row(['id' => 1, 'status' => 'active', 'amount' => 100]), - row(['id' => 2, 'status' => 'active', 'amount' => 100]), - row(['id' => 3, 'status' => 'active', 'amount' => 100]), ); $transformedRows = (new DuplicateRowTransformer( @@ -239,11 +245,13 @@ public function test_doing_nothing_when_condition_is_not_satisfied(): void public function test_duplicating_row(): void { - $rows = rows( + $rows = array_to_rows( + [ + ['id' => 1, 'status' => 'active', 'amount' => 100], + ['id' => 2, 'status' => 'inactive', 'amount' => 100], + ['id' => 3, 'status' => 'active', 'amount' => 100], + ], schema(int_schema('id'), string_schema('status'), int_schema('amount')), - row(['id' => 1, 'status' => 'active', 'amount' => 100]), - row(['id' => 2, 'status' => 'inactive', 'amount' => 100]), - row(['id' => 3, 'status' => 'active', 'amount' => 100]), ); $transformedRows = (new DuplicateRowTransformer( @@ -303,7 +311,7 @@ public function test_an_unbound_transform_refuses_an_expand_in_an_entry(): void ); (new DuplicateRowTransformer(lit(true), with_entry('t', ref('tags')->expand())))->transform( - rows(ListColumnsMother::schema(), ListColumnsMother::row()), + ListColumnsMother::rows()->matchTo(ListColumnsMother::schema(), new AdaptiveBackend()), flow_context(config()), ); } diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Transformer/JoinEachRowsTransformerTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Transformer/JoinEachRowsTransformerTest.php index 49821aa428..9f55aabf13 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Unit/Transformer/JoinEachRowsTransformerTest.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Transformer/JoinEachRowsTransformerTest.php @@ -12,12 +12,11 @@ use Flow\ETL\Tests\FlowTestCase; use Flow\ETL\Transformer\JoinEachRowsTransformer; +use function Flow\ETL\DSL\array_to_rows; use function Flow\ETL\DSL\config; use function Flow\ETL\DSL\data_frame; use function Flow\ETL\DSL\flow_context; use function Flow\ETL\DSL\int_schema; -use function Flow\ETL\DSL\row; -use function Flow\ETL\DSL\rows; use function Flow\ETL\DSL\schema; use function Flow\ETL\DSL\str_schema; @@ -42,21 +41,21 @@ public function from(Rows $rows): DataFrame public function test_inner_join_rows(): void { - $left = rows( + $left = array_to_rows( + [['id' => 1, 'country' => 'PL'], ['id' => 2, 'country' => 'US'], ['id' => 3, 'country' => 'FR']], schema(int_schema('id'), str_schema('country')), - row(['id' => 1, 'country' => 'PL']), - row(['id' => 2, 'country' => 'US']), - row(['id' => 3, 'country' => 'FR']), ); $right = new class implements DataFrameFactory { public function from(Rows $rows): DataFrame { - return data_frame()->process(rows( + return data_frame()->process(array_to_rows( + [ + ['code' => 'PL', 'name' => 'Poland'], + ['code' => 'US', 'name' => 'United States'], + ['code' => 'GB', 'name' => 'Great Britain'], + ], schema(str_schema('code'), str_schema('name')), - row(['code' => 'PL', 'name' => 'Poland']), - row(['code' => 'US', 'name' => 'United States']), - row(['code' => 'GB', 'name' => 'Great Britain']), )); } }; @@ -74,20 +73,20 @@ public function from(Rows $rows): DataFrame public function test_left_join_rows(): void { - $left = rows( + $left = array_to_rows( + [['id' => 1, 'country' => 'PL'], ['id' => 2, 'country' => 'US'], ['id' => 3, 'country' => 'FR']], schema(int_schema('id'), str_schema('country')), - row(['id' => 1, 'country' => 'PL']), - row(['id' => 2, 'country' => 'US']), - row(['id' => 3, 'country' => 'FR']), ); $right = new class implements DataFrameFactory { public function from(Rows $rows): DataFrame { - return data_frame()->process(rows( + return data_frame()->process(array_to_rows( + [ + ['code' => 'PL', 'name' => 'Poland'], + ['code' => 'US', 'name' => 'United States'], + ['code' => 'GB', 'name' => 'Great Britain'], + ], schema(str_schema('code'), str_schema('name')), - row(['code' => 'PL', 'name' => 'Poland']), - row(['code' => 'US', 'name' => 'United States']), - row(['code' => 'GB', 'name' => 'Great Britain']), )); } }; @@ -106,20 +105,20 @@ public function from(Rows $rows): DataFrame public function test_right_join_rows(): void { - $left = rows( + $left = array_to_rows( + [['id' => 1, 'country' => 'PL'], ['id' => 2, 'country' => 'US'], ['id' => 3, 'country' => 'FR']], schema(int_schema('id'), str_schema('country')), - row(['id' => 1, 'country' => 'PL']), - row(['id' => 2, 'country' => 'US']), - row(['id' => 3, 'country' => 'FR']), ); $right = new class implements DataFrameFactory { public function from(Rows $rows): DataFrame { - return data_frame()->process(rows( + return data_frame()->process(array_to_rows( + [ + ['code' => 'PL', 'name' => 'Poland'], + ['code' => 'US', 'name' => 'United States'], + ['code' => 'GB', 'name' => 'Great Britain'], + ], schema(str_schema('code'), str_schema('name')), - row(['code' => 'PL', 'name' => 'Poland']), - row(['code' => 'US', 'name' => 'United States']), - row(['code' => 'GB', 'name' => 'Great Britain']), )); } }; diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Transformer/NestedExpandTransformerTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Transformer/NestedExpandTransformerTest.php index e457088287..85eca30f2e 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Unit/Transformer/NestedExpandTransformerTest.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Transformer/NestedExpandTransformerTest.php @@ -10,13 +10,12 @@ use Flow\ETL\Transformer\NestedExpandTransformer; use Flow\ETL\Transformer\ScalarFunctionTransformer; +use function Flow\ETL\DSL\array_to_rows; use function Flow\ETL\DSL\concat; use function Flow\ETL\DSL\config; use function Flow\ETL\DSL\flow_context; use function Flow\ETL\DSL\list_schema; use function Flow\ETL\DSL\ref; -use function Flow\ETL\DSL\row; -use function Flow\ETL\DSL\rows; use function Flow\ETL\DSL\schema; use function Flow\ETL\DSL\str_schema; use function Flow\ETL\DSL\structure; @@ -41,14 +40,10 @@ public function test_emits_one_row_per_element_and_none_for_an_empty_list(): voi ['id' => 'a', 'tags' => ['x', 'y'], 's' => ['tag' => 'x']], ['id' => 'a', 'tags' => ['x', 'y'], 's' => ['tag' => 'y']], ], - $step->transform( - rows( - ListColumnsMother::tagsSchema(), - row(['id' => 'a', 'tags' => ['x', 'y']]), - row(['id' => 'b', 'tags' => []]), - ), - flow_context(config()), - )->toArray(), + $step->transform(array_to_rows([ + ['id' => 'a', 'tags' => ['x', 'y']], + ['id' => 'b', 'tags' => []], + ], ListColumnsMother::tagsSchema()), flow_context(config()))->toArray(), ); } @@ -65,10 +60,10 @@ public function test_unpack_emits_one_row_per_element(): void ['id' => 'a', 'tags' => ['x', 'y'], 'u.tag' => 'x'], ['id' => 'a', 'tags' => ['x', 'y'], 'u.tag' => 'y'], ], - $step->transform( - rows(ListColumnsMother::tagsSchema(), row(['id' => 'a', 'tags' => ['x', 'y']])), - flow_context(config()), - )->toArray(), + $step->transform(array_to_rows([[ + 'id' => 'a', + 'tags' => ['x', 'y'], + ]], ListColumnsMother::tagsSchema()), flow_context(config()))->toArray(), ); } @@ -110,10 +105,10 @@ public function test_a_definition_entry_is_declared_as_given(): void ['id' => 'a', 'tags' => ['x', 'y'], 's' => 'ax'], ['id' => 'a', 'tags' => ['x', 'y'], 's' => 'ay'], ], - $step->transform( - rows(ListColumnsMother::tagsSchema(), row(['id' => 'a', 'tags' => ['x', 'y']])), - flow_context(config()), - )->toArray(), + $step->transform(array_to_rows([[ + 'id' => 'a', + 'tags' => ['x', 'y'], + ]], ListColumnsMother::tagsSchema()), flow_context(config()))->toArray(), ); } @@ -127,10 +122,10 @@ public function test_a_batch_under_another_schema_than_the_bound_one_is_conforme static::assertInstanceOf(NestedExpandTransformer::class, $step); - $result = $step->transform( - rows(ListColumnsMother::tagsSchema(), row(['id' => 'a', 'tags' => ['x']])), - flow_context(config()), - ); + $result = $step->transform(array_to_rows([[ + 'id' => 'a', + 'tags' => ['x'], + ]], ListColumnsMother::tagsSchema()), flow_context(config())); static::assertEquals( schema( @@ -154,6 +149,6 @@ public function test_a_null_under_a_not_null_definition_names_its_output_row(): $this->expectException(SchemaMismatchException::class); $this->expectExceptionMessage('column "s" (row 1)'); - $step->transform(rows($input, row(['tags' => ['x']]), row(['tags' => [null]])), flow_context(config())); + $step->transform(array_to_rows([['tags' => ['x']], ['tags' => [null]]], $input), flow_context(config())); } } diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Transformer/NestedExpansionTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Transformer/NestedExpansionTest.php index 831c2372e6..5c8562525f 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Unit/Transformer/NestedExpansionTest.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Transformer/NestedExpansionTest.php @@ -14,6 +14,7 @@ use Generator; use PHPUnit\Framework\Attributes\DataProvider; +use function Flow\ETL\DSL\array_to_rows; use function Flow\ETL\DSL\concat; use function Flow\ETL\DSL\config; use function Flow\ETL\DSL\flow_context; @@ -21,7 +22,6 @@ use function Flow\ETL\DSL\lit; use function Flow\ETL\DSL\map_schema; use function Flow\ETL\DSL\ref; -use function Flow\ETL\DSL\row; use function Flow\ETL\DSL\schema; use function Flow\ETL\DSL\str_schema; use function Flow\ETL\DSL\structure; @@ -29,7 +29,6 @@ use function Flow\Types\DSL\type_integer; use function Flow\Types\DSL\type_list; use function Flow\Types\DSL\type_map; -use function Flow\Types\DSL\type_mixed; use function Flow\Types\DSL\type_string; final class NestedExpansionTest extends FlowTestCase @@ -48,7 +47,7 @@ public function test_declares_the_zipped_type_and_gives_one_value_per_zipped_pos $expansion = NestedExpansionContext::of($tree); static::assertSame($returns, $expansion->returns()->toString()); - static::assertSame($eval, $expansion->eval(ListColumnsMother::row($override), flow_context(config()))); + static::assertSame($eval, $expansion->eval(ListColumnsMother::rows($override), flow_context(config()))[1]); } /** @@ -140,7 +139,10 @@ public function test_a_padded_null_follows_the_null_rule_of_the_function_reading $this->expectException(InvalidArgumentException::class); $this->expectExceptionMessage('Plus function requires non-null values'); - $expansion->eval(ListColumnsMother::row(['nums' => [1, 2], 'tags' => ['x', 'y', 'z']]), flow_context(config())); + $expansion->eval(ListColumnsMother::rows([ + 'nums' => [1, 2], + 'tags' => ['x', 'y', 'z'], + ]), flow_context(config())); } public function test_of_is_null_for_a_root_expand(): void @@ -159,30 +161,27 @@ public function test_of_is_null_without_an_expand(): void )); } - public function test_zipped_expands_over_mixed_lists_stay_mixed(): void - { - $expansion = NestedExpansionContext::of( - structure(['a' => ref('a')->expand(), 'b' => ref('b')->expand()]), - schema(list_schema('a', type_list(type_mixed())), list_schema('b', type_list(type_mixed()))), - ); - - static::assertSame('structure{a: mixed, b: mixed}', $expansion->returns()->toString()); - static::assertSame( - [['a' => 1, 'b' => 'x'], ['a' => 2, 'b' => null]], - $expansion->eval(row(['a' => [1, 2], 'b' => ['x']]), flow_context(config())), - ); - } - public function test_an_expand_over_a_map_reads_its_values_by_position(): void { static::assertSame( [['v' => 1], ['v' => 2]], NestedExpansionContext::of(structure([ 'v' => ref('m')->expand(), - ]), schema(map_schema('m', type_map(type_string(), type_integer()))))->eval(row(['m' => [ + ]), schema(map_schema('m', type_map(type_string(), type_integer()))))->eval(array_to_rows([['m' => [ 'a' => 1, 'b' => 2, - ]]), flow_context(config())), + ]]], schema(map_schema('m', type_map(type_string(), type_integer())))), flow_context(config()))[1], + ); + } + + public function test_every_element_names_the_row_it_comes_from(): void + { + static::assertSame( + [[0, 0, 1], [['tag' => 'x'], ['tag' => 'y'], ['tag' => 'z']]], + NestedExpansionContext::of(structure(['tag' => ref('tags')->expand()]))->eval(array_to_rows([ + ['id' => 'a', 'tags' => ['x', 'y']], + ['id' => 'b', 'tags' => ['z']], + ], ListColumnsMother::tagsSchema()), flow_context(config())), ); } @@ -193,7 +192,13 @@ public function test_the_synthesized_column_does_not_shadow_an_input_column(): v NestedExpansionContext::of( structure(['c' => ref("\0expand:0"), 't' => ref('tags')->expand()]), schema(str_schema("\0expand:0"), list_schema('tags', type_list(type_string()))), - )->eval(row(["\0expand:0" => 'kept', 'tags' => ['x']]), flow_context(config())), + )->eval( + array_to_rows( + [["\0expand:0" => 'kept', 'tags' => ['x']]], + schema(str_schema("\0expand:0"), list_schema('tags', type_list(type_string()))), + ), + flow_context(config()), + )[1], ); } } diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Transformer/PruneEntriesTransformerTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Transformer/PruneEntriesTransformerTest.php index 84252f39fb..b18cc603cd 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Unit/Transformer/PruneEntriesTransformerTest.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Transformer/PruneEntriesTransformerTest.php @@ -7,11 +7,10 @@ use Flow\ETL\Tests\FlowTestCase; use Flow\ETL\Transformer\PruneEntriesTransformer; +use function Flow\ETL\DSL\array_to_rows; use function Flow\ETL\DSL\flow_context; use function Flow\ETL\DSL\int_schema; use function Flow\ETL\DSL\ref; -use function Flow\ETL\DSL\row; -use function Flow\ETL\DSL\rows; use function Flow\ETL\DSL\schema; use function Flow\ETL\DSL\str_schema; @@ -43,9 +42,9 @@ public function test_keeping_only_the_given_entries(): void [['id' => 1, 'name' => 'a']], (new PruneEntriesTransformer(ref('id'), ref('name'))) ->transform( - rows( + array_to_rows( + [['id' => 1, 'name' => 'a', 'dropped' => 9]], schema(int_schema('id'), str_schema('name'), int_schema('dropped')), - row(['id' => 1, 'name' => 'a', 'dropped' => 9]), ), flow_context(), ) @@ -59,10 +58,9 @@ public function test_entry_missing_from_a_row_is_padded_from_its_nullable_declar [['id' => 1, 'name' => 'a'], ['id' => 2, 'name' => null]], (new PruneEntriesTransformer(ref('id'), ref('name'))) ->transform( - rows( + array_to_rows( + [['id' => 1, 'name' => 'a'], ['id' => 2]], schema(int_schema('id'), str_schema('name', nullable: true)), - row(['id' => 1, 'name' => 'a']), - row(['id' => 2]), ), flow_context(), ) @@ -75,7 +73,7 @@ public function test_row_without_any_of_the_given_entries_becomes_empty(): void static::assertSame( [[]], (new PruneEntriesTransformer(ref('missing'))) - ->transform(rows(schema(int_schema('id')), row(['id' => 1])), flow_context()) + ->transform(array_to_rows([['id' => 1]], schema(int_schema('id'))), flow_context()) ->toArray(), ); } diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Transformer/RenameEntryTransformerTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Transformer/RenameEntryTransformerTest.php index 7a399b354f..e83f1c0f7a 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Unit/Transformer/RenameEntryTransformerTest.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Transformer/RenameEntryTransformerTest.php @@ -7,14 +7,13 @@ use Flow\ETL\Tests\FlowTestCase; use Flow\ETL\Transformer\RenameEntryTransformer; +use function Flow\ETL\DSL\array_to_rows; use function Flow\ETL\DSL\bool_schema; use function Flow\ETL\DSL\config; use function Flow\ETL\DSL\datetime_schema; use function Flow\ETL\DSL\flow_context; use function Flow\ETL\DSL\integer_schema; use function Flow\ETL\DSL\json_schema; -use function Flow\ETL\DSL\row; -use function Flow\ETL\DSL\rows; use function Flow\ETL\DSL\schema; use function Flow\ETL\DSL\schema_metadata; use function Flow\ETL\DSL\string_schema; @@ -23,6 +22,17 @@ final class RenameEntryTransformerTest extends FlowTestCase { + public function test_renaming_keeps_the_column_position_and_its_values(): void + { + $renamed = (new RenameEntryTransformer('a', 'x'))->transform( + array_to_rows([['a' => 1, 'b' => 2]], schema(integer_schema('a'), integer_schema('b'))), + flow_context(config()), + ); + + static::assertSame(['x', 'b'], array_keys($renamed->values(0))); + static::assertSame([['x' => 1, 'b' => 2]], $renamed->toArray()); + } + public function test_bind_renames_the_column_in_place(): void { static::assertEquals( @@ -46,7 +56,16 @@ public function test_renaming_entries(): void $context = flow_context(config()); $rows = (new RenameEntryTransformer('old_int', 'new_int'))->transform( - rows( + array_to_rows( + [[ + 'old_int' => 1000, + 'id' => 1, + 'status' => 'PENDING', + 'enabled' => true, + 'datetime' => type_datetime()->cast('2020-01-01 00:00:00 UTC'), + 'json' => type_json()->cast(['foo', 'bar']), + 'null' => null, + ]], schema( integer_schema('old_int'), integer_schema('id'), @@ -56,21 +75,21 @@ public function test_renaming_entries(): void json_schema('json'), string_schema('null', nullable: true), ), - row([ - 'old_int' => 1000, - 'id' => 1, - 'status' => 'PENDING', - 'enabled' => true, - 'datetime' => type_datetime()->cast('2020-01-01 00:00:00 UTC'), - 'json' => type_json()->cast(['foo', 'bar']), - 'null' => null, - ]), ), $context, ); static::assertEquals( - rows( + array_to_rows( + [[ + 'new_int' => 1000, + 'id' => 1, + 'status' => 'PENDING', + 'enabled' => true, + 'datetime' => type_datetime()->cast('2020-01-01 00:00:00 UTC'), + 'json' => type_json()->cast(['foo', 'bar']), + 'nothing' => null, + ]], schema( integer_schema('new_int'), integer_schema('id'), @@ -80,15 +99,6 @@ public function test_renaming_entries(): void json_schema('json'), string_schema('nothing', nullable: true), ), - row([ - 'new_int' => 1000, - 'id' => 1, - 'status' => 'PENDING', - 'enabled' => true, - 'datetime' => type_datetime()->cast('2020-01-01 00:00:00 UTC'), - 'json' => type_json()->cast(['foo', 'bar']), - 'nothing' => null, - ]), ), (new RenameEntryTransformer('null', 'nothing'))->transform($rows, $context), ); @@ -98,18 +108,17 @@ public function test_renaming_entry_preserves_metadata(): void { $metadata = schema_metadata(['description' => 'test metadata', 'priority' => 1]); - $outputRows = (new RenameEntryTransformer('old_name', 'new_name'))->transform( - rows(schema(string_schema('old_name', metadata: $metadata)), row(['old_name' => 'test value'])), - flow_context(config()), - ); + $outputRows = (new RenameEntryTransformer('old_name', 'new_name'))->transform(array_to_rows([[ + 'old_name' => 'test value', + ]], schema(string_schema('old_name', metadata: $metadata))), flow_context(config())); - static::assertSame('test value', $outputRows->first()->get('new_name')); + static::assertSame('test value', $outputRows->column('new_name')->value(0)); static::assertTrue($outputRows->schema()->get('new_name')->metadata()->isEqual($metadata)); } public function test_renaming_to_same_name_returns_same_rows_instance(): void { - $inputRows = rows(schema(string_schema('name')), row(['name' => 'value'])); + $inputRows = array_to_rows([['name' => 'value']], schema(string_schema('name'))); static::assertSame($inputRows, (new RenameEntryTransformer('name', 'name'))->transform( $inputRows, diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Transformer/ScalarFunctionFilterTransformerTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Transformer/ScalarFunctionFilterTransformerTest.php index b98eb38f12..313fda2876 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Unit/Transformer/ScalarFunctionFilterTransformerTest.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Transformer/ScalarFunctionFilterTransformerTest.php @@ -10,12 +10,12 @@ use Flow\ETL\Tests\Mother\ListColumnsMother; use Flow\ETL\Transformer\ScalarFunctionFilterTransformer; +use function Flow\ETL\DSL\array_to_rows; use function Flow\ETL\DSL\config; use function Flow\ETL\DSL\flow_context; use function Flow\ETL\DSL\int_schema; use function Flow\ETL\DSL\lit; use function Flow\ETL\DSL\ref; -use function Flow\ETL\DSL\row; use function Flow\ETL\DSL\rows; use function Flow\ETL\DSL\schema; @@ -47,7 +47,7 @@ public function test_bind_returns_the_input_schema(): void public function test_equal(): void { - $rows = rows(schema(int_schema('a'), int_schema('b')), row(['a' => 1, 'b' => 1]), row(['a' => 1, 'b' => 2])); + $rows = array_to_rows([['a' => 1, 'b' => 1], ['a' => 1, 'b' => 2]], schema(int_schema('a'), int_schema('b'))); static::assertSame( [ @@ -61,7 +61,7 @@ public function test_equal(): void public function test_equal_on_literal(): void { - $rows = rows(schema(int_schema('a'), int_schema('b')), row(['a' => 1, 'b' => 1]), row(['a' => 1, 'b' => 2])); + $rows = array_to_rows([['a' => 1, 'b' => 1], ['a' => 1, 'b' => 2]], schema(int_schema('a'), int_schema('b'))); static::assertSame( [ @@ -75,7 +75,7 @@ public function test_equal_on_literal(): void public function test_greater_than(): void { - $rows = rows(schema(int_schema('a'), int_schema('b')), row(['a' => 1, 'b' => 2])); + $rows = array_to_rows([['a' => 1, 'b' => 2]], schema(int_schema('a'), int_schema('b'))); static::assertSame( [ @@ -89,7 +89,7 @@ public function test_greater_than(): void public function test_greater_than_or_equal(): void { - $rows = rows(schema(int_schema('a'), int_schema('b')), row(['a' => 1, 'b' => 1]), row(['a' => 1, 'b' => 2])); + $rows = array_to_rows([['a' => 1, 'b' => 1], ['a' => 1, 'b' => 2]], schema(int_schema('a'), int_schema('b'))); static::assertSame( [ @@ -104,7 +104,7 @@ public function test_greater_than_or_equal(): void public function test_less_than(): void { - $rows = rows(schema(int_schema('a'), int_schema('b')), row(['a' => 1, 'b' => 1]), row(['a' => 1, 'b' => 2])); + $rows = array_to_rows([['a' => 1, 'b' => 1], ['a' => 1, 'b' => 2]], schema(int_schema('a'), int_schema('b'))); static::assertSame( [ @@ -118,7 +118,7 @@ public function test_less_than(): void public function test_less_than_equal(): void { - $rows = rows(schema(int_schema('a'), int_schema('b')), row(['a' => 1, 'b' => 1]), row(['a' => 1, 'b' => 2])); + $rows = array_to_rows([['a' => 1, 'b' => 1], ['a' => 1, 'b' => 2]], schema(int_schema('a'), int_schema('b'))); static::assertSame( [ @@ -133,7 +133,7 @@ public function test_less_than_equal(): void public function test_not_equal(): void { - $rows = rows(schema(int_schema('a'), int_schema('b')), row(['a' => 1, 'b' => 1]), row(['a' => 1, 'b' => 2])); + $rows = array_to_rows([['a' => 1, 'b' => 1], ['a' => 1, 'b' => 2]], schema(int_schema('a'), int_schema('b'))); static::assertSame( [ @@ -147,7 +147,7 @@ public function test_not_equal(): void public function test_not_same(): void { - $rows = rows(schema(int_schema('a'), int_schema('b')), row(['a' => 1, 'b' => 1]), row(['a' => 1, 'b' => 2])); + $rows = array_to_rows([['a' => 1, 'b' => 1], ['a' => 1, 'b' => 2]], schema(int_schema('a'), int_schema('b'))); static::assertSame( [ @@ -161,7 +161,7 @@ public function test_not_same(): void public function test_same(): void { - $rows = rows(schema(int_schema('a'), int_schema('b')), row(['a' => 1, 'b' => 1]), row(['a' => 1, 'b' => 2])); + $rows = array_to_rows([['a' => 1, 'b' => 1], ['a' => 1, 'b' => 2]], schema(int_schema('a'), int_schema('b'))); static::assertSame( [ @@ -178,10 +178,10 @@ public function test_a_non_boolean_predicate_is_rejected_at_bind(): void $this->expectException(InvalidArgumentException::class); $this->expectExceptionMessage('filter() requires a predicate returning boolean'); - (new ScalarFunctionFilterTransformer(ref('score')))->transform( - rows(schema(int_schema('score', nullable: true)), row(['score' => 1]), row(['score' => null])), - flow_context(config()), - ); + (new ScalarFunctionFilterTransformer(ref('score')))->transform(array_to_rows([ + ['score' => 1], + ['score' => null], + ], schema(int_schema('score', nullable: true))), flow_context(config())); } public function test_a_null_propagating_predicate_over_a_nullable_column_is_accepted(): void @@ -191,15 +191,11 @@ public function test_a_null_propagating_predicate_over_a_nullable_column_is_acce ['score' => 11], ], (new ScalarFunctionFilterTransformer(ref('score')->greaterThan(lit(10)))) - ->transform( - rows( - schema(int_schema('score', nullable: true)), - row(['score' => 11]), - row(['score' => 2]), - row(['score' => null]), - ), - flow_context(config()), - ) + ->transform(array_to_rows([ + ['score' => 11], + ['score' => 2], + ['score' => null], + ], schema(int_schema('score', nullable: true))), flow_context(config())) ->toArray(), ); } diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Transformer/ScalarFunctionTransformer/MathTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Transformer/ScalarFunctionTransformer/MathTest.php index 3fed78a1b6..f81e061c54 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Unit/Transformer/ScalarFunctionTransformer/MathTest.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Transformer/ScalarFunctionTransformer/MathTest.php @@ -11,13 +11,12 @@ use Generator; use PHPUnit\Framework\Attributes\DataProvider; +use function Flow\ETL\DSL\array_to_rows; use function Flow\ETL\DSL\float_schema; use function Flow\ETL\DSL\flow_context; use function Flow\ETL\DSL\int_schema; use function Flow\ETL\DSL\integer_schema; use function Flow\ETL\DSL\ref; -use function Flow\ETL\DSL\row; -use function Flow\ETL\DSL\rows; use function Flow\ETL\DSL\schema; final class MathTest extends FlowTestCase @@ -52,7 +51,7 @@ public static function minus_data_provider(): Generator 0.3, float_schema('b'), 0.1, - ['result' => 0.2, 'a' => 0.3, 'b' => 0.1], + ['result' => 0.19999999999999998, 'a' => 0.3, 'b' => 0.1], ]; yield [ @@ -68,7 +67,7 @@ public static function minus_data_provider(): Generator 0.3, float_schema('b'), 0.1, - ['result' => 0.2, 'a' => 0.3_0000_0000_0000_000, 'b' => 0.1_0000_0000_0000_000], + ['result' => 0.19999999999999998, 'a' => 0.3_0000_0000_0000_000, 'b' => 0.1_0000_0000_0000_000], ]; } @@ -98,7 +97,7 @@ public static function plus_data_provider(): Generator 0.3, float_schema('b'), -0.1, - ['result' => 0.2, 'a' => 0.3, 'b' => -0.1], + ['result' => 0.19999999999999998, 'a' => 0.3, 'b' => -0.1], ]; yield [ @@ -125,7 +124,7 @@ public static function power_data_provider(): Generator -0.3, integer_schema('b'), 10, - ['result' => 5.9049E-6, 'a' => -0.3, 'b' => 10], + ['result' => 5.9048999999999975E-6, 'a' => -0.3, 'b' => 10], ]; } @@ -146,7 +145,7 @@ public function test_divide( ): void { $rows = (new ScalarFunctionTransformer('result', ref($a->entry()->name()) ->divide(ref($b->entry()->name()), $scale, $rounding)))->transform( - rows(schema($a, $b), row([$a->entry()->name() => $aValue, $b->entry()->name() => $bValue])), + array_to_rows([[$a->entry()->name() => $aValue, $b->entry()->name() => $bValue]], schema($a, $b)), flow_context(), ); @@ -165,7 +164,7 @@ public function test_minus(Definition $a, mixed $aValue, Definition $b, mixed $b 'result', ref($a->entry()->name())->minus(ref($b->entry()->name())), ))->transform( - rows(schema($a, $b), row([$a->entry()->name() => $aValue, $b->entry()->name() => $bValue])), + array_to_rows([[$a->entry()->name() => $aValue, $b->entry()->name() => $bValue]], schema($a, $b)), flow_context(), ); @@ -184,7 +183,7 @@ public function test_multiply(Definition $a, mixed $aValue, Definition $b, mixed 'result', ref($a->entry()->name())->multiply(ref($b->entry()->name())), ))->transform( - rows(schema($a, $b), row([$a->entry()->name() => $aValue, $b->entry()->name() => $bValue])), + array_to_rows([[$a->entry()->name() => $aValue, $b->entry()->name() => $bValue]], schema($a, $b)), flow_context(), ); @@ -203,7 +202,7 @@ public function test_plus(Definition $a, mixed $aValue, Definition $b, mixed $bV 'result', ref($a->entry()->name())->plus(ref($b->entry()->name())), ))->transform( - rows(schema($a, $b), row([$a->entry()->name() => $aValue, $b->entry()->name() => $bValue])), + array_to_rows([[$a->entry()->name() => $aValue, $b->entry()->name() => $bValue]], schema($a, $b)), flow_context(), ); @@ -222,7 +221,7 @@ public function test_power(Definition $a, mixed $aValue, Definition $b, mixed $b 'result', ref($a->entry()->name())->power(ref($b->entry()->name())), ))->transform( - rows(schema($a, $b), row([$a->entry()->name() => $aValue, $b->entry()->name() => $bValue])), + array_to_rows([[$a->entry()->name() => $aValue, $b->entry()->name() => $bValue]], schema($a, $b)), flow_context(), ); diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Transformer/ScalarFunctionTransformerTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Transformer/ScalarFunctionTransformerTest.php index d4387a0bb4..ed2090f73f 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Unit/Transformer/ScalarFunctionTransformerTest.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Transformer/ScalarFunctionTransformerTest.php @@ -7,6 +7,8 @@ use DOMDocument; use DOMNodeList; use DOMXPath; +use Flow\ETL\Column\AdaptiveBackend; +use Flow\ETL\Column\PhpBackend; use Flow\ETL\Exception\InvalidArgumentException; use Flow\ETL\Exception\InvalidLogicException; use Flow\ETL\Exception\SchemaDefinitionNotFoundException; @@ -22,6 +24,7 @@ use Generator; use PHPUnit\Framework\Attributes\DataProvider; +use function Flow\ETL\DSL\array_to_rows; use function Flow\ETL\DSL\concat; use function Flow\ETL\DSL\config; use function Flow\ETL\DSL\flow_context; @@ -29,7 +32,6 @@ use function Flow\ETL\DSL\list_schema; use function Flow\ETL\DSL\lit; use function Flow\ETL\DSL\ref; -use function Flow\ETL\DSL\row; use function Flow\ETL\DSL\rows; use function Flow\ETL\DSL\schema; use function Flow\ETL\DSL\str_schema; @@ -83,7 +85,7 @@ public function test_expand_results(): void ['array' => 3], ], (new ScalarFunctionTransformer('array', new ArrayExpand(lit([1, 2, 3]), ArrayExpand\ArrayExpand::VALUES))) - ->transform(rows(schema(), row([])), flow_context(config())) + ->transform(array_to_rows([[]], schema()), flow_context(config())) ->toArray(), ); } @@ -105,7 +107,7 @@ public function test_lit_expression_on_non_empty_rows(): void ['name' => 'Norbert', 'number' => 1], ], (new ScalarFunctionTransformer('number', lit(1))) - ->transform(rows(schema(str_schema('name')), row(['name' => 'Norbert'])), flow_context(config())) + ->transform(array_to_rows([['name' => 'Norbert']], schema(str_schema('name'))), flow_context(config())) ->toArray(), ); } @@ -128,7 +130,7 @@ public function test_plus_expression_on_non_empty_rows(): void ], (new ScalarFunctionTransformer('c', ref('a')->plus(ref('b')))) ->transform( - rows(schema(int_schema('a'), int_schema('b')), row(['a' => 1, 'b' => 2])), + array_to_rows([['a' => 1, 'b' => 2]], schema(int_schema('a'), int_schema('b'))), flow_context(config()), ) ->toArray(), @@ -145,7 +147,7 @@ public function test_plus_expression_on_non_existing_rows(): void ['a' => 1, 'number' => 0], ], (new ScalarFunctionTransformer('number', ref('num')->plus(ref('num1')))) - ->transform(rows(schema(int_schema('a')), row(['a' => 1])), flow_context(config())) + ->transform(array_to_rows([['a' => 1]], schema(int_schema('a'))), flow_context(config())) ->toArray(), ); } @@ -160,7 +162,7 @@ public function test_an_undeclared_payload_key_is_dropped(): void 'array', new ArrayUnpack(lit(['id' => 1, 'secret' => 'dropped']), schema(int_schema('id'))), )) - ->transform(rows(schema(), row([])), flow_context(config())) + ->transform(array_to_rows([[]], schema()), flow_context(config())) ->toArray(), ); } @@ -172,7 +174,7 @@ public function test_unpack_results_ignores_a_definition_passed_to_with_entry(): $result = (new ScalarFunctionTransformer( str_schema('array'), new ArrayUnpack(lit(['id' => 1, 'name' => 'Norbert']), schema(int_schema('id'), str_schema('name'))), - ))->transform(rows(schema(), row([])), flow_context(config())); + ))->transform(array_to_rows([[]], schema()), flow_context(config())); static::assertEquals([['array.id' => 1, 'array.name' => 'Norbert']], $result->toArray()); static::assertEquals( @@ -191,22 +193,21 @@ public function test_xml_xpath_expression_when_there_is_more_than_one_node_under $nodes = type_instance_of(DOMNodeList::class)->assert($xpath->query('/root/foo')); $expected = [$nodes->item(0), $nodes->item(1)]; - $result = (new ScalarFunctionTransformer('xpath', ref('xml')->xpath('/root/foo')))->transform( - rows(schema(xml_schema('xml')), row(['xml' => type_xml()->cast($xml)])), - flow_context(config()), - ); + $result = (new ScalarFunctionTransformer('xpath', ref('xml')->xpath('/root/foo')))->transform(array_to_rows([[ + 'xml' => type_xml()->cast($xml), + ]], schema(xml_schema('xml'))), flow_context(config())); - static::assertEquals($expected, $result->first()->get('xpath')); + static::assertEquals($expected, $result->column('xpath')->value(0)); static::assertEquals(type_list(type_xml_element()), $result->schema()->get('xpath')->type()); static::assertTrue($result->schema()->get('xpath')->isNullable()); } public function test_a_declared_output_definition_coerces_the_produced_value(): void { - $result = (new ScalarFunctionTransformer(str_schema('out'), ref('v')))->transform( - rows(schema(int_schema('v')), row(['v' => 1]), row(['v' => 2])), - flow_context(config()), - ); + $result = (new ScalarFunctionTransformer(str_schema('out'), ref('v')))->transform(array_to_rows([ + ['v' => 1], + ['v' => 2], + ], schema(int_schema('v'))), flow_context(config())); static::assertSame( [ @@ -219,25 +220,30 @@ public function test_a_declared_output_definition_coerces_the_produced_value(): public function test_the_produced_column_takes_the_declared_definition(): void { - $result = (new ScalarFunctionTransformer(str_schema('out'), ref('v')))->transform( - rows(schema(int_schema('v')), row(['v' => 1]), row(['v' => 2])), - flow_context(config()), - ); + $result = (new ScalarFunctionTransformer(str_schema('out'), ref('v')))->transform(array_to_rows([ + ['v' => 1], + ['v' => 2], + ], schema(int_schema('v'))), flow_context(config())); static::assertEquals(type_string(), $result->schema()->get('out')->type()); } public function test_the_produced_definition_does_not_depend_on_batch_size(): void { - $wholeBatch = (new ScalarFunctionTransformer('out', ref('v')->plus(lit(1))))->transform( - rows(schema(int_schema('v')), row(['v' => 1]), row(['v' => 2])), - flow_context(config()), - ); + $wholeBatch = (new ScalarFunctionTransformer('out', ref('v')->plus(lit(1))))->transform(array_to_rows([ + ['v' => 1], + ['v' => 2], + ], schema(int_schema('v'))), flow_context(config())); $transformer = new ScalarFunctionTransformer('out', ref('v')->plus(lit(1))); - $single = $transformer - ->transform(rows(schema(int_schema('v')), row(['v' => 1])), flow_context(config())) - ->merge($transformer->transform(rows(schema(int_schema('v')), row(['v' => 2])), flow_context(config()))); + $single = $transformer->transform(array_to_rows([[ + 'v' => 1, + ]], schema(int_schema('v'))), flow_context(config()))->concat( + new PhpBackend(), + $transformer->transform(array_to_rows([[ + 'v' => 2, + ]], schema(int_schema('v'))), flow_context(config())), + ); static::assertEquals($wholeBatch->schema()->get('out'), $single->schema()->get('out')); static::assertEquals($wholeBatch->toArray(), $single->toArray()); @@ -248,10 +254,10 @@ public function test_a_null_produced_under_a_not_null_declaration_names_its_row( $this->expectException(SchemaMismatchException::class); $this->expectExceptionMessage('column "out" (row 1)'); - (new ScalarFunctionTransformer(int_schema('out'), ref('v')))->transform( - rows(schema(int_schema('v', nullable: true)), row(['v' => 1]), row(['v' => null])), - flow_context(config()), - ); + (new ScalarFunctionTransformer(int_schema('out'), ref('v')))->transform(array_to_rows([ + ['v' => 1], + ['v' => null], + ], schema(int_schema('v', nullable: true))), flow_context(config())); } public function test_a_batch_arriving_under_another_schema_than_the_bound_one_is_conformed_to_the_bound_output(): void @@ -263,7 +269,7 @@ public function test_a_batch_arriving_under_another_schema_than_the_bound_one_is static::assertSame( [['id' => 1, 'name' => null, 'out' => 'x']], - $step->transform(rows(schema(int_schema('id')), row(['id' => 1])), flow_context(config()))->toArray(), + $step->transform(array_to_rows([['id' => 1]], schema(int_schema('id'))), flow_context(config()))->toArray(), ); } @@ -275,7 +281,7 @@ public function test_an_expanded_null_under_a_not_null_declaration_names_its_row (new ScalarFunctionTransformer( int_schema('out'), new ArrayExpand(lit([1, null]), ArrayExpand\ArrayExpand::VALUES), - ))->transform(rows(schema(), row([])), flow_context(config())); + ))->transform(array_to_rows([[]], schema()), flow_context(config())); } public function test_an_empty_batch_is_bound_against_its_own_schema(): void @@ -365,10 +371,10 @@ public function test_an_unbound_nested_expand_is_expanded(): void ['id' => 'a', 'tags' => ['x', 'y'], 's' => ['tag' => 'y']], ], (new ScalarFunctionTransformer('s', structure(['tag' => ref('tags')->expand()]))) - ->transform( - rows(ListColumnsMother::tagsSchema(), row(['id' => 'a', 'tags' => ['x', 'y']])), - flow_context(config()), - ) + ->transform(array_to_rows([[ + 'id' => 'a', + 'tags' => ['x', 'y'], + ]], ListColumnsMother::tagsSchema()), flow_context(config())) ->toArray(), ); } @@ -384,10 +390,10 @@ public function test_an_unbound_unpack_over_a_nested_expand_is_expanded(): void 'u', structure(['tag' => ref('tags')->expand()])->unpack(schema(str_schema('tag'))), )) - ->transform( - rows(ListColumnsMother::tagsSchema(), row(['id' => 'a', 'tags' => ['x', 'y']])), - flow_context(config()), - ) + ->transform(array_to_rows([[ + 'id' => 'a', + 'tags' => ['x', 'y'], + ]], ListColumnsMother::tagsSchema()), flow_context(config())) ->toArray(), ); } @@ -420,7 +426,7 @@ public function test_an_unbound_expand_inside_an_expand_is_refused(): void ); (new ScalarFunctionTransformer('v', ref('lists')->expand()->expand()))->transform( - rows(ListColumnsMother::schema(), ListColumnsMother::row()), + ListColumnsMother::rows()->matchTo(ListColumnsMother::schema(), new AdaptiveBackend()), flow_context(config()), ); } @@ -433,4 +439,39 @@ public function test_bind_refuses_an_unpack_whose_type_is_not_a_structure(): voi (new ScalarFunctionTransformer('u', new NonStructureUnpackStub()))->bind(schema()); } + + public function test_an_expand_explodes_every_row_of_the_batch_in_order(): void + { + $rows = array_to_rows( + [['id' => 1, 'list' => [1, 2]], ['id' => 2, 'list' => []], ['id' => 3, 'list' => [3]]], + schema(int_schema('id'), list_schema('list', type_list(type_integer()))), + ); + + static::assertSame( + [['id' => 1, 'value' => 1], ['id' => 1, 'value' => 2], ['id' => 3, 'value' => 3]], + (new ScalarFunctionTransformer('value', ref('list')->expand())) + ->transform($rows, flow_context()) + ->project(schema(int_schema('id'), int_schema('value')), new AdaptiveBackend()) + ->toArray(), + ); + } + + public function test_an_unpack_fills_one_column_per_declared_element_for_every_row(): void + { + $rows = array_to_rows([ + ['s' => ['a' => 1, 'b' => 'x']], + ['s' => ['a' => 2, 'b' => null]], + ], schema(structure_schema('s', type_structure(['a' => type_integer(), 'b' => type_optional(type_string())])))); + + static::assertSame( + [['s.a' => 1, 's.b' => 'x'], ['s.a' => 2, 's.b' => null]], + (new ScalarFunctionTransformer('s', ref('s')->unpack(schema(int_schema('a'), str_schema('b'))))) + ->transform($rows, flow_context()) + ->project( + schema(int_schema('s.a', nullable: true), str_schema('s.b', nullable: true)), + new AdaptiveBackend(), + ) + ->toArray(), + ); + } } diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Transformer/SelectEntriesTransformerTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Transformer/SelectEntriesTransformerTest.php index ba07261d74..cb0c616818 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Unit/Transformer/SelectEntriesTransformerTest.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Transformer/SelectEntriesTransformerTest.php @@ -8,12 +8,11 @@ use Flow\ETL\Tests\FlowTestCase; use Flow\ETL\Transformer\SelectEntriesTransformer; +use function Flow\ETL\DSL\array_to_rows; use function Flow\ETL\DSL\config; use function Flow\ETL\DSL\flow_context; use function Flow\ETL\DSL\int_schema; use function Flow\ETL\DSL\json_schema; -use function Flow\ETL\DSL\row; -use function Flow\ETL\DSL\rows; use function Flow\ETL\DSL\schema; use function Flow\ETL\DSL\str_schema; use function Flow\ETL\DSL\string_schema; @@ -43,9 +42,9 @@ public function test_bind_refuses_a_column_missing_from_the_input(): void public function test_selecting_entries(): void { - $rows = rows( + $rows = array_to_rows( + [['id' => 1, 'name' => 'Row Name', 'array' => type_json()->cast(['test'])]], schema(int_schema('id'), str_schema('name'), json_schema('array')), - row(['id' => 1, 'name' => 'Row Name', 'array' => type_json()->cast(['test'])]), ); $transformer = new SelectEntriesTransformer('name'); @@ -59,9 +58,9 @@ public function test_selecting_entries(): void public function test_selecting_not_existing_entries(): void { - $rows = rows( + $rows = array_to_rows( + [['id' => 1, 'name' => 'Row Name', 'array' => type_json()->cast(['test'])]], schema(int_schema('id'), string_schema('name'), json_schema('array')), - row(['id' => 1, 'name' => 'Row Name', 'array' => type_json()->cast(['test'])]), ); $this->expectException(SchemaDefinitionNotFoundException::class); @@ -78,10 +77,9 @@ public function test_selecting_an_entry_missing_from_some_rows_widens_to_a_nulla schema(int_schema('id', nullable: true)), $transformer ->transform( - rows( + array_to_rows( + [['id' => 1], ['name' => 'no id here']], schema(int_schema('id', nullable: true), str_schema('name', nullable: true)), - row(['id' => 1]), - row(['name' => 'no id here']), ), flow_context(config()), ) @@ -91,9 +89,9 @@ public function test_selecting_an_entry_missing_from_some_rows_widens_to_a_nulla public function test_using_select_entries_in_order_to_change_entries_order(): void { - $rows = rows( + $rows = array_to_rows( + [['id' => 1, 'name' => 'Row Name', 'array' => type_json()->cast(['test'])]], schema(int_schema('id'), str_schema('name'), json_schema('array')), - row(['id' => 1, 'name' => 'Row Name', 'array' => type_json()->cast(['test'])]), ); $result = (new SelectEntriesTransformer('name', 'id', 'array'))->transform($rows, flow_context(config())); @@ -113,10 +111,9 @@ public function test_select_carries_a_not_null_definition_through_unchanged(): v schema(int_schema('id')), (new SelectEntriesTransformer('id')) ->transform( - rows( + array_to_rows( + [['id' => 1, 'name' => 'Alice'], ['id' => 2, 'name' => 'Bob']], schema(int_schema('id'), str_schema('name')), - row(['id' => 1, 'name' => 'Alice']), - row(['id' => 2, 'name' => 'Bob']), ), flow_context(config()), ) diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Transformer/SerializeTransformerTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Transformer/SerializeTransformerTest.php index 8886a35717..f368647f47 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Unit/Transformer/SerializeTransformerTest.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Transformer/SerializeTransformerTest.php @@ -4,17 +4,18 @@ namespace Flow\ETL\Tests\Unit\Transformer; +use Flow\ETL\Column\AdaptiveBackend; +use Flow\ETL\Rows; use Flow\ETL\Tests\FlowTestCase; use Flow\ETL\Transformer\SerializeTransformer; use Flow\Floe\FloeSerializer; use Flow\Serializer\Base64Serializer; +use function Flow\ETL\DSL\array_to_rows; use function Flow\ETL\DSL\bool_schema; use function Flow\ETL\DSL\flow_context; use function Flow\ETL\DSL\int_schema; use function Flow\ETL\DSL\list_schema; -use function Flow\ETL\DSL\row; -use function Flow\ETL\DSL\rows; use function Flow\ETL\DSL\schema; use function Flow\ETL\DSL\str_schema; use function Flow\Serializer\DSL\serialize_to_string; @@ -52,7 +53,7 @@ public function test_bind_replaces_the_target_column_with_a_string_column(): voi public function test_serializing_empty_row_under_one_entry(): void { - $rows = rows($rowSchema = schema(), $row1 = row([])); + $rows = Rows::fromColumns(schema(), [], 1); $transformer = new SerializeTransformer('serialized'); $transformedRows = $transformer->transform($rows, flow_context()); @@ -61,8 +62,8 @@ public function test_serializing_empty_row_under_one_entry(): void [ [ 'serialized' => serialize_to_string( - new Base64Serializer(new FloeSerializer()), - rows($rowSchema, $row1), + new Base64Serializer(new FloeSerializer(new AdaptiveBackend())), + $rows, ), ], ], @@ -78,11 +79,10 @@ public function test_serializing_row_under_one_entry(): void bool_schema('active'), list_schema('tags', type_list(type_string())), ); - $rows = rows( - $rowSchema, - $row1 = row(['id' => 1, 'name' => 'John', 'active' => true, 'tags' => ['tag1', 'tag2']]), - $row2 = row(['id' => 2, 'name' => 'Jane', 'active' => false, 'tags' => ['tag3', 'tag4']]), - ); + $rows = array_to_rows([ + ['id' => 1, 'name' => 'John', 'active' => true, 'tags' => ['tag1', 'tag2']], + ['id' => 2, 'name' => 'Jane', 'active' => false, 'tags' => ['tag3', 'tag4']], + ], $rowSchema); $transformer = new SerializeTransformer('serialized'); @@ -96,8 +96,8 @@ public function test_serializing_row_under_one_entry(): void 'active' => true, 'tags' => ['tag1', 'tag2'], 'serialized' => serialize_to_string( - new Base64Serializer(new FloeSerializer()), - rows($rowSchema, $row1), + new Base64Serializer(new FloeSerializer(new AdaptiveBackend())), + $rows->slice(0, 1), ), ], [ @@ -106,8 +106,8 @@ public function test_serializing_row_under_one_entry(): void 'active' => false, 'tags' => ['tag3', 'tag4'], 'serialized' => serialize_to_string( - new Base64Serializer(new FloeSerializer()), - rows($rowSchema, $row2), + new Base64Serializer(new FloeSerializer(new AdaptiveBackend())), + $rows->slice(1, 1), ), ], ], @@ -123,11 +123,10 @@ public function test_serializing_row_under_standalone_entry(): void bool_schema('active'), list_schema('tags', type_list(type_string())), ); - $rows = rows( - $rowSchema, - $row1 = row(['id' => 1, 'name' => 'John', 'active' => true, 'tags' => ['tag1', 'tag2']]), - $row2 = row(['id' => 2, 'name' => 'Jane', 'active' => false, 'tags' => ['tag3', 'tag4']]), - ); + $rows = array_to_rows([ + ['id' => 1, 'name' => 'John', 'active' => true, 'tags' => ['tag1', 'tag2']], + ['id' => 2, 'name' => 'Jane', 'active' => false, 'tags' => ['tag3', 'tag4']], + ], $rowSchema); $transformer = new SerializeTransformer('serialized', true); @@ -137,14 +136,14 @@ public function test_serializing_row_under_standalone_entry(): void [ [ 'serialized' => serialize_to_string( - new Base64Serializer(new FloeSerializer()), - rows($rowSchema, $row1), + new Base64Serializer(new FloeSerializer(new AdaptiveBackend())), + $rows->slice(0, 1), ), ], [ 'serialized' => serialize_to_string( - new Base64Serializer(new FloeSerializer()), - rows($rowSchema, $row2), + new Base64Serializer(new FloeSerializer(new AdaptiveBackend())), + $rows->slice(1, 1), ), ], ], diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Transformer/SerializedPayloadDecoderTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Transformer/SerializedPayloadDecoderTest.php index cf974571b6..b556832870 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Unit/Transformer/SerializedPayloadDecoderTest.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Transformer/SerializedPayloadDecoderTest.php @@ -4,68 +4,89 @@ namespace Flow\ETL\Tests\Unit\Transformer; +use Flow\ETL\Column\AdaptiveBackend; use Flow\ETL\Tests\FlowTestCase; use Flow\ETL\Transformer\SerializedPayloadDecoder; use Flow\Floe\FloeSerializer; use Flow\Serializer\Base64Serializer; +use function Flow\ETL\DSL\array_to_rows; use function Flow\ETL\DSL\int_schema; use function Flow\ETL\DSL\ref; -use function Flow\ETL\DSL\row; -use function Flow\ETL\DSL\rows; use function Flow\ETL\DSL\schema; +use function Flow\ETL\DSL\str_schema; use function Flow\Serializer\DSL\serialize_to_string; final class SerializedPayloadDecoderTest extends FlowTestCase { public function test_a_multi_row_payload_gives_up(): void { - $payload = serialize_to_string( - new Base64Serializer(new FloeSerializer()), - rows(schema(int_schema('id')), row(['id' => 1]), row(['id' => 2])), - ); + $payload = serialize_to_string(new Base64Serializer(new FloeSerializer(new AdaptiveBackend())), array_to_rows([ + ['id' => 1], + ['id' => 2], + ], schema(int_schema('id')))); static::assertSame( - ['id' => null], - SerializedPayloadDecoderTest::decoder()->decode(row(['serialized' => $payload])), + [[]], + SerializedPayloadDecoderTest::decoder() + ->decode(array_to_rows([['serialized' => $payload]], schema(str_schema('serialized'))), 0) + ->toArray(), ); } public function test_a_non_string_value_gives_up(): void { - static::assertSame(['id' => null], SerializedPayloadDecoderTest::decoder()->decode(row(['serialized' => 123]))); + static::assertSame( + [[]], + SerializedPayloadDecoderTest::decoder() + ->decode(array_to_rows([['serialized' => 123]], schema(int_schema('serialized'))), 0) + ->toArray(), + ); } public function test_a_payload_that_does_not_deserialize_gives_up(): void { static::assertSame( - ['id' => null], - SerializedPayloadDecoderTest::decoder()->decode(row(['serialized' => 'not-serialized'])), + [[]], + SerializedPayloadDecoderTest::decoder() + ->decode(array_to_rows([['serialized' => 'not-serialized']], schema(str_schema('serialized'))), 0) + ->toArray(), ); } public function test_a_row_without_the_source_column_gives_up(): void { - static::assertSame(['id' => null], SerializedPayloadDecoderTest::decoder()->decode(row(['other' => 1]))); + static::assertSame( + [[]], + SerializedPayloadDecoderTest::decoder() + ->decode(array_to_rows([['other' => 1]], schema(int_schema('other'))), 0) + ->toArray(), + ); } public function test_it_reads_the_declared_columns_out_of_the_payload(): void { - $payload = serialize_to_string( - new Base64Serializer(new FloeSerializer()), - rows(schema(int_schema('id')), row(['id' => 7])), - ); + $payload = serialize_to_string(new Base64Serializer(new FloeSerializer(new AdaptiveBackend())), array_to_rows([[ + 'id' => 7, + ]], schema(int_schema('id')))); static::assertSame( - ['id' => 7], - SerializedPayloadDecoderTest::decoder()->decode(row(['serialized' => $payload])), + [['id' => 7]], + SerializedPayloadDecoderTest::decoder() + ->decode(array_to_rows([ + ['serialized' => 'not-serialized'], + ['serialized' => $payload], + ], schema(str_schema('serialized'))), 1) + ->toArray(), ); } public static function decoder(): SerializedPayloadDecoder { - return SerializedPayloadDecoder::of(ref('serialized'), new Base64Serializer(new FloeSerializer()), [ - 'id' => 'id', - ]); + return SerializedPayloadDecoder::of( + ref('serialized'), + new Base64Serializer(new FloeSerializer(new AdaptiveBackend())), + ['id'], + ); } } diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Transformer/UnpackedColumnsTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Transformer/UnpackedColumnsTest.php index 0a21880813..86f12aa8dd 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Unit/Transformer/UnpackedColumnsTest.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Transformer/UnpackedColumnsTest.php @@ -41,8 +41,8 @@ public function test_every_declared_column_is_added_nullable_and_prefixed(): voi public function test_values_prefixes_declared_keys_drops_undeclared_and_nulls_absent(): void { static::assertSame( - ['id' => 1, 'u.a' => 1, 'u.b' => null], - (new UnpackedColumns())->values(['id' => 1], 'u.', schema(int_schema('a'), str_schema('b')), [ + ['u.a' => 1, 'u.b' => null], + (new UnpackedColumns())->values('u.', schema(int_schema('a'), str_schema('b')), [ 'a' => '1', 'c' => 'x', ]), diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Transformer/UnserializeTransformerTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Transformer/UnserializeTransformerTest.php index e89a0eea42..d85a996fb5 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Unit/Transformer/UnserializeTransformerTest.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Transformer/UnserializeTransformerTest.php @@ -4,17 +4,18 @@ namespace Flow\ETL\Tests\Unit\Transformer; +use Flow\ETL\Column\AdaptiveBackend; +use Flow\ETL\Exception\SchemaMismatchException; use Flow\ETL\Tests\FlowTestCase; use Flow\ETL\Transformer\UnserializeTransformer; use Flow\Floe\FloeSerializer; use Flow\Serializer\Base64Serializer; +use function Flow\ETL\DSL\array_to_rows; use function Flow\ETL\DSL\bool_schema; use function Flow\ETL\DSL\flow_context; use function Flow\ETL\DSL\int_schema; use function Flow\ETL\DSL\list_schema; -use function Flow\ETL\DSL\row; -use function Flow\ETL\DSL\rows; use function Flow\ETL\DSL\schema; use function Flow\ETL\DSL\str_schema; use function Flow\Serializer\DSL\serialize_to_string; @@ -23,6 +24,20 @@ final class UnserializeTransformerTest extends FlowTestCase { + public function test_a_payload_written_with_another_type_is_refused(): void + { + $payload = serialize_to_string(new Base64Serializer(new FloeSerializer(new AdaptiveBackend())), array_to_rows([[ + 'id' => '12', + ]], schema(str_schema('id')))); + + $this->expectException(SchemaMismatchException::class); + $this->expectExceptionMessage('column "id" (row 0): could not convert \'12\' (string) to integer'); + + (new UnserializeTransformer('serialized', schema(int_schema('id'))))->transform(array_to_rows([[ + 'serialized' => $payload, + ]], schema(str_schema('serialized'))), flow_context()); + } + public function test_bind_expands_the_declared_target_into_nullable_columns(): void { static::assertEquals( @@ -65,26 +80,27 @@ public function test_bind_without_merging_keeps_only_the_declared_target(): void public function test_unserializing_row_from_entry(): void { - $row1 = row(['id' => 1, 'name' => 'John', 'active' => true, 'tags' => ['tag1', 'tag2']]); $rowSchema = schema( int_schema('id'), str_schema('name'), bool_schema('active'), list_schema('tags', type_list(type_string())), ); - $row2 = row(['id' => 2, 'name' => 'Jane', 'active' => false, 'tags' => ['tag3', 'tag4']]); - - $rows = rows( - schema(str_schema('serialized')), - row(['serialized' => serialize_to_string( - new Base64Serializer(new FloeSerializer()), - rows($rowSchema, $row1), - )]), - row(['serialized' => serialize_to_string( - new Base64Serializer(new FloeSerializer()), - rows($rowSchema, $row2), - )]), - ); + $source = array_to_rows([ + ['id' => 1, 'name' => 'John', 'active' => true, 'tags' => ['tag1', 'tag2']], + ['id' => 2, 'name' => 'Jane', 'active' => false, 'tags' => ['tag3', 'tag4']], + ], $rowSchema); + + $rows = array_to_rows([ + ['serialized' => serialize_to_string( + new Base64Serializer(new FloeSerializer(new AdaptiveBackend())), + $source->slice(0, 1), + )], + ['serialized' => serialize_to_string( + new Base64Serializer(new FloeSerializer(new AdaptiveBackend())), + $source->slice(1, 1), + )], + ], schema(str_schema('serialized'))); $transformer = new UnserializeTransformer('serialized', $rowSchema); @@ -94,8 +110,8 @@ public function test_unserializing_row_from_entry(): void [ [ 'serialized' => serialize_to_string( - new Base64Serializer(new FloeSerializer()), - rows($rowSchema, $row1), + new Base64Serializer(new FloeSerializer(new AdaptiveBackend())), + $source->slice(0, 1), ), 'id' => 1, 'name' => 'John', @@ -104,8 +120,8 @@ public function test_unserializing_row_from_entry(): void ], [ 'serialized' => serialize_to_string( - new Base64Serializer(new FloeSerializer()), - rows($rowSchema, $row2), + new Base64Serializer(new FloeSerializer(new AdaptiveBackend())), + $source->slice(1, 1), ), 'id' => 2, 'name' => 'Jane', @@ -119,7 +135,7 @@ public function test_unserializing_row_from_entry(): void public function test_unserializing_something_that_is_not_serialized_row(): void { - $rows = rows(schema(str_schema('serialized')), row(['serialized' => 'not-serialized'])); + $rows = array_to_rows([['serialized' => 'not-serialized']], schema(str_schema('serialized'))); $transformer = new UnserializeTransformer('serialized', schema(int_schema('id'))); @@ -131,7 +147,7 @@ public function test_unserializing_something_that_is_not_serialized_row(): void public function test_unserializing_row_without_source_column_emits_the_declared_shape(): void { - $rows = rows(schema(int_schema('id')), row(['id' => 1])); + $rows = array_to_rows([['id' => 1]], schema(int_schema('id'))); $transformer = new UnserializeTransformer('serialized', schema(str_schema('name'))); @@ -140,7 +156,7 @@ public function test_unserializing_row_without_source_column_emits_the_declared_ public function test_unserializing_non_string_value_emits_the_declared_shape(): void { - $rows = rows(schema(int_schema('serialized')), row(['serialized' => 123])); + $rows = array_to_rows([['serialized' => 123]], schema(int_schema('serialized'))); $transformer = new UnserializeTransformer('serialized', schema(str_schema('name'))); @@ -152,11 +168,11 @@ public function test_unserializing_non_string_value_emits_the_declared_shape(): public function test_unserializing_multi_row_payload_emits_the_declared_shape(): void { - $payload = serialize_to_string( - new Base64Serializer(new FloeSerializer()), - rows(schema(int_schema('id')), row(['id' => 1]), row(['id' => 2])), - ); - $rows = rows(schema(str_schema('serialized')), row(['serialized' => $payload])); + $payload = serialize_to_string(new Base64Serializer(new FloeSerializer(new AdaptiveBackend())), array_to_rows([ + ['id' => 1], + ['id' => 2], + ], schema(int_schema('id')))); + $rows = array_to_rows([['serialized' => $payload]], schema(str_schema('serialized'))); $transformer = new UnserializeTransformer('serialized', schema(int_schema('id'))); @@ -168,43 +184,45 @@ public function test_unserializing_multi_row_payload_emits_the_declared_shape(): public function test_the_declared_columns_land_under_the_merge_prefix(): void { - $payload = serialize_to_string( - new Base64Serializer(new FloeSerializer()), - rows(schema(int_schema('id')), row(['id' => 7])), - ); + $payload = serialize_to_string(new Base64Serializer(new FloeSerializer(new AdaptiveBackend())), array_to_rows([[ + 'id' => 7, + ]], schema(int_schema('id')))); $transformer = new UnserializeTransformer('serialized', schema(int_schema('id')), mergePrefix: 'payload_'); static::assertSame( [['serialized' => $payload, 'payload_id' => 7]], $transformer - ->transform(rows(schema(str_schema('serialized')), row(['serialized' => $payload])), flow_context()) + ->transform(array_to_rows([[ + 'serialized' => $payload, + ]], schema(str_schema('serialized'))), flow_context()) ->toArray(), ); } public function test_unserializing_without_merge(): void { - $row1 = row(['id' => 1, 'name' => 'John', 'active' => true, 'tags' => ['tag1', 'tag2']]); $rowSchema = schema( int_schema('id'), str_schema('name'), bool_schema('active'), list_schema('tags', type_list(type_string())), ); - $row2 = row(['id' => 2, 'name' => 'Jane', 'active' => false, 'tags' => ['tag3', 'tag4']]); - - $rows = rows( - schema(str_schema('serialized')), - row(['serialized' => serialize_to_string( - new Base64Serializer(new FloeSerializer()), - rows($rowSchema, $row1), - )]), - row(['serialized' => serialize_to_string( - new Base64Serializer(new FloeSerializer()), - rows($rowSchema, $row2), - )]), - ); + $source = array_to_rows([ + ['id' => 1, 'name' => 'John', 'active' => true, 'tags' => ['tag1', 'tag2']], + ['id' => 2, 'name' => 'Jane', 'active' => false, 'tags' => ['tag3', 'tag4']], + ], $rowSchema); + + $rows = array_to_rows([ + ['serialized' => serialize_to_string( + new Base64Serializer(new FloeSerializer(new AdaptiveBackend())), + $source->slice(0, 1), + )], + ['serialized' => serialize_to_string( + new Base64Serializer(new FloeSerializer(new AdaptiveBackend())), + $source->slice(1, 1), + )], + ], schema(str_schema('serialized'))); $transformer = new UnserializeTransformer('serialized', $rowSchema, false); @@ -231,18 +249,17 @@ public function test_unserializing_without_merge(): void public function test_a_declared_column_replaces_an_input_column_of_the_same_name(): void { - $payload = serialize_to_string( - new Base64Serializer(new FloeSerializer()), - rows(schema(int_schema('id')), row(['id' => 7])), - ); + $payload = serialize_to_string(new Base64Serializer(new FloeSerializer(new AdaptiveBackend())), array_to_rows([[ + 'id' => 7, + ]], schema(int_schema('id')))); static::assertSame( [['serialized' => $payload, 'id' => 7]], (new UnserializeTransformer('serialized', schema(int_schema('id')))) ->transform( - rows( + array_to_rows( + [['serialized' => $payload, 'id' => 1]], schema(str_schema('serialized'), int_schema('id')), - row(['serialized' => $payload, 'id' => 1]), ), flow_context(), ) @@ -252,15 +269,16 @@ public function test_a_declared_column_replaces_an_input_column_of_the_same_name public function test_a_declared_column_the_payload_does_not_carry_is_null(): void { - $payload = serialize_to_string( - new Base64Serializer(new FloeSerializer()), - rows(schema(int_schema('id')), row(['id' => 7])), - ); + $payload = serialize_to_string(new Base64Serializer(new FloeSerializer(new AdaptiveBackend())), array_to_rows([[ + 'id' => 7, + ]], schema(int_schema('id')))); static::assertSame( [['serialized' => $payload, 'id' => 7, 'absent' => null]], (new UnserializeTransformer('serialized', schema(int_schema('id'), str_schema('absent')))) - ->transform(rows(schema(str_schema('serialized')), row(['serialized' => $payload])), flow_context()) + ->transform(array_to_rows([[ + 'serialized' => $payload, + ]], schema(str_schema('serialized'))), flow_context()) ->toArray(), ); } diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Transformer/UntilTransformerTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Transformer/UntilTransformerTest.php index 13a7d21969..1a7740e582 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Unit/Transformer/UntilTransformerTest.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Transformer/UntilTransformerTest.php @@ -11,12 +11,12 @@ use Flow\ETL\Tests\Mother\ListColumnsMother; use Flow\ETL\Transformer\UntilTransformer; +use function Flow\ETL\DSL\array_to_rows; use function Flow\ETL\DSL\config; use function Flow\ETL\DSL\flow_context; use function Flow\ETL\DSL\int_schema; use function Flow\ETL\DSL\lit; use function Flow\ETL\DSL\ref; -use function Flow\ETL\DSL\row; use function Flow\ETL\DSL\rows; use function Flow\ETL\DSL\schema; @@ -27,7 +27,7 @@ public function test_a_batch_that_holds_the_predicate_throughout_passes_through( static::assertSame( [['id' => 1], ['id' => 2]], (new UntilTransformer(ref('id')->lessThan(lit(3)))) - ->transform(rows(schema(int_schema('id')), row(['id' => 1]), row(['id' => 2])), flow_context(config())) + ->transform(array_to_rows([['id' => 1], ['id' => 2]], schema(int_schema('id'))), flow_context(config())) ->toArray(), ); } @@ -51,10 +51,11 @@ public function test_passes_rows_until_the_predicate_stops_holding(): void $thrown = null; try { - (new UntilTransformer(ref('id')->lessThan(lit(3))))->transform( - rows(schema(int_schema('id')), row(['id' => 1]), row(['id' => 2]), row(['id' => 3])), - flow_context(config()), - ); + (new UntilTransformer(ref('id')->lessThan(lit(3))))->transform(array_to_rows([ + ['id' => 1], + ['id' => 2], + ['id' => 3], + ], schema(int_schema('id'))), flow_context(config())); } catch (LimitReachedException $e) { $thrown = $e; } @@ -68,17 +69,13 @@ public function test_it_stops_at_the_first_row_failing_the_predicate(): void $thrown = null; try { - (new UntilTransformer(ref('id')->lessThan(lit(3))))->transform( - rows( - schema(int_schema('id')), - row(['id' => 1]), - row(['id' => 2]), - row(['id' => 5]), - row(['id' => 1]), - row(['id' => 2]), - ), - flow_context(config()), - ); + (new UntilTransformer(ref('id')->lessThan(lit(3))))->transform(array_to_rows([ + ['id' => 1], + ['id' => 2], + ['id' => 5], + ['id' => 1], + ['id' => 2], + ], schema(int_schema('id'))), flow_context(config())); } catch (LimitReachedException $e) { $thrown = $e; } @@ -92,14 +89,14 @@ public function test_a_reached_limit_stops_the_next_batch(): void $transformer = new UntilTransformer(ref('id')->lessThan(lit(1))); try { - $transformer->transform(rows(schema(int_schema('id')), row(['id' => 5])), flow_context(config())); + $transformer->transform(array_to_rows([['id' => 5]], schema(int_schema('id'))), flow_context(config())); } catch (LimitReachedException) { } $thrown = null; try { - $transformer->transform(rows(schema(int_schema('id')), row(['id' => 0])), flow_context(config())); + $transformer->transform(array_to_rows([['id' => 0]], schema(int_schema('id'))), flow_context(config())); } catch (LimitReachedException $e) { $thrown = $e; } @@ -113,10 +110,9 @@ public function test_a_non_boolean_predicate_is_rejected_at_bind(): void $this->expectException(InvalidArgumentException::class); $this->expectExceptionMessage('until() requires a predicate returning boolean'); - (new UntilTransformer(ref('id')))->transform( - rows(schema(int_schema('id')), row(['id' => 1])), - flow_context(config()), - ); + (new UntilTransformer(ref('id')))->transform(array_to_rows([[ + 'id' => 1, + ]], schema(int_schema('id'))), flow_context(config())); } public function test_an_empty_batch_is_bound_against_its_own_schema(): void @@ -131,7 +127,7 @@ public function test_a_reached_limit_stops_an_empty_next_batch(): void $transformer = new UntilTransformer(ref('id')->lessThan(lit(1))); try { - $transformer->transform(rows(schema(int_schema('id')), row(['id' => 5])), flow_context(config())); + $transformer->transform(array_to_rows([['id' => 5]], schema(int_schema('id'))), flow_context(config())); } catch (LimitReachedException) { } diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Window/Accumulator/AverageAccumulatorTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Window/Accumulator/AverageAccumulatorTest.php index d4d8637727..81d0cde6b1 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Unit/Window/Accumulator/AverageAccumulatorTest.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Window/Accumulator/AverageAccumulatorTest.php @@ -9,25 +9,30 @@ use Flow\ETL\Tests\FlowTestCase; use Flow\ETL\Window\Accumulator\AverageAccumulator; +use function Flow\ETL\DSL\array_to_rows; use function Flow\ETL\DSL\flow_context; +use function Flow\ETL\DSL\int_schema; use function Flow\ETL\DSL\ref; -use function Flow\ETL\DSL\row; +use function Flow\ETL\DSL\schema; +use function Flow\ETL\DSL\str_schema; final class AverageAccumulatorTest extends FlowTestCase { public function test_averages_accumulated_values(): void { - $accumulator = new AverageAccumulator(ref('value'), 2, Rounding::HALF_UP, flow_context()); + $accumulator = new AverageAccumulator(ref('value'), 2, Rounding::HALF_UP, false, flow_context()); - $accumulator->accumulate(row(['value' => 10])); - $accumulator->accumulate(row(['value' => 20])); + $accumulator->accumulate(array_to_rows([['value' => 10]], schema(int_schema('value'))), 0); + $accumulator->accumulate(array_to_rows([['value' => 20]], schema(int_schema('value'))), 0); static::assertSame(15.0, $accumulator->value()); } public function test_empty_frame_averages_to_null(): void { - static::assertNull((new AverageAccumulator(ref('value'), 2, Rounding::HALF_UP, flow_context()))->value()); + static::assertNull( + (new AverageAccumulator(ref('value'), 2, Rounding::HALF_UP, false, flow_context()))->value(), + ); } public function test_missing_entry_throws_in_strict_mode(): void @@ -36,50 +41,52 @@ public function test_missing_entry_throws_in_strict_mode(): void $this->expectException(InvalidArgumentException::class); $this->expectExceptionMessageMatches('/^Average window function error: /'); - (new AverageAccumulator(ref('value'), 2, Rounding::HALF_UP, $context))->accumulate(row(['other' => 'x'])); + (new AverageAccumulator(ref('value'), 2, Rounding::HALF_UP, false, $context))->accumulate(array_to_rows([[ + 'other' => 'x', + ]], schema(str_schema('other'))), 0); } public function test_non_numeric_values_are_skipped(): void { - $accumulator = new AverageAccumulator(ref('value'), 2, Rounding::HALF_UP, flow_context()); + $accumulator = new AverageAccumulator(ref('value'), 2, Rounding::HALF_UP, false, flow_context()); - $accumulator->accumulate(row(['value' => 10])); - $accumulator->accumulate(row(['value' => 'not a number'])); - $accumulator->accumulate(row(['value' => null])); - $accumulator->accumulate(row(['value' => 20])); + $accumulator->accumulate(array_to_rows([['value' => 10]], schema(int_schema('value'))), 0); + $accumulator->accumulate(array_to_rows([['value' => 'not a number']], schema(str_schema('value'))), 0); + $accumulator->accumulate(array_to_rows([['value' => null]], schema(str_schema('value', nullable: true))), 0); + $accumulator->accumulate(array_to_rows([['value' => 20]], schema(int_schema('value'))), 0); static::assertSame(15.0, $accumulator->value()); } public function test_only_null_values_still_average_to_null(): void { - $accumulator = new AverageAccumulator(ref('value'), 2, Rounding::HALF_UP, flow_context()); + $accumulator = new AverageAccumulator(ref('value'), 2, Rounding::HALF_UP, false, flow_context()); - $accumulator->accumulate(row(['value' => null])); + $accumulator->accumulate(array_to_rows([['value' => null]], schema(str_schema('value', nullable: true))), 0); static::assertNull($accumulator->value()); } public function test_scale_and_rounding_are_applied(): void { - $accumulator = new AverageAccumulator(ref('value'), 3, Rounding::HALF_UP, flow_context()); + $accumulator = new AverageAccumulator(ref('value'), 3, Rounding::HALF_UP, false, flow_context()); - $accumulator->accumulate(row(['value' => 10])); - $accumulator->accumulate(row(['value' => 20])); - $accumulator->accumulate(row(['value' => 25])); + $accumulator->accumulate(array_to_rows([['value' => 10]], schema(int_schema('value'))), 0); + $accumulator->accumulate(array_to_rows([['value' => 20]], schema(int_schema('value'))), 0); + $accumulator->accumulate(array_to_rows([['value' => 25]], schema(int_schema('value'))), 0); static::assertSame(18.333, $accumulator->value()); } public function test_value_is_idempotent(): void { - $accumulator = new AverageAccumulator(ref('value'), 2, Rounding::HALF_UP, flow_context()); - $accumulator->accumulate(row(['value' => 10])); + $accumulator = new AverageAccumulator(ref('value'), 2, Rounding::HALF_UP, false, flow_context()); + $accumulator->accumulate(array_to_rows([['value' => 10]], schema(int_schema('value'))), 0); static::assertSame(10.0, $accumulator->value()); static::assertSame(10.0, $accumulator->value()); - $accumulator->accumulate(row(['value' => 20])); + $accumulator->accumulate(array_to_rows([['value' => 20]], schema(int_schema('value'))), 0); static::assertSame(15.0, $accumulator->value()); } diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Window/Accumulator/CountAccumulatorTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Window/Accumulator/CountAccumulatorTest.php index a458b45118..1dd10b00d7 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Unit/Window/Accumulator/CountAccumulatorTest.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Window/Accumulator/CountAccumulatorTest.php @@ -8,9 +8,12 @@ use Flow\ETL\Tests\FlowTestCase; use Flow\ETL\Window\Accumulator\CountAccumulator; +use function Flow\ETL\DSL\array_to_rows; use function Flow\ETL\DSL\flow_context; +use function Flow\ETL\DSL\int_schema; use function Flow\ETL\DSL\ref; -use function Flow\ETL\DSL\row; +use function Flow\ETL\DSL\schema; +use function Flow\ETL\DSL\str_schema; final class CountAccumulatorTest extends FlowTestCase { @@ -18,8 +21,8 @@ public function test_counts_every_row_when_reference_is_null(): void { $accumulator = new CountAccumulator(null, flow_context()); - $accumulator->accumulate(row(['value' => 1])); - $accumulator->accumulate(row(['other' => 'x'])); + $accumulator->accumulate(array_to_rows([['value' => 1]], schema(int_schema('value'))), 0); + $accumulator->accumulate(array_to_rows([['other' => 'x']], schema(str_schema('other'))), 0); static::assertSame(2, $accumulator->value()); } @@ -28,9 +31,9 @@ public function test_counts_only_non_null_values(): void { $accumulator = new CountAccumulator(ref('value'), flow_context()); - $accumulator->accumulate(row(['value' => 1])); - $accumulator->accumulate(row(['value' => null])); - $accumulator->accumulate(row(['value' => 3])); + $accumulator->accumulate(array_to_rows([['value' => 1]], schema(int_schema('value'))), 0); + $accumulator->accumulate(array_to_rows([['value' => null]], schema(int_schema('value', nullable: true))), 0); + $accumulator->accumulate(array_to_rows([['value' => 3]], schema(int_schema('value'))), 0); static::assertSame(2, $accumulator->value()); } @@ -51,18 +54,20 @@ public function test_missing_entry_throws_in_strict_mode(): void $this->expectException(InvalidArgumentException::class); $this->expectExceptionMessageMatches('/^Count window function error: /'); - (new CountAccumulator(ref('value'), $context))->accumulate(row(['other' => 'x'])); + (new CountAccumulator(ref('value'), $context))->accumulate(array_to_rows([[ + 'other' => 'x', + ]], schema(str_schema('other'))), 0); } public function test_value_is_idempotent(): void { $accumulator = new CountAccumulator(ref('value'), flow_context()); - $accumulator->accumulate(row(['value' => 1])); + $accumulator->accumulate(array_to_rows([['value' => 1]], schema(int_schema('value'))), 0); static::assertSame(1, $accumulator->value()); static::assertSame(1, $accumulator->value()); - $accumulator->accumulate(row(['value' => 2])); + $accumulator->accumulate(array_to_rows([['value' => 2]], schema(int_schema('value'))), 0); static::assertSame(2, $accumulator->value()); } diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Window/Accumulator/SumAccumulatorTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Window/Accumulator/SumAccumulatorTest.php index 12c299def2..2b38dc1fd5 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Unit/Window/Accumulator/SumAccumulatorTest.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Window/Accumulator/SumAccumulatorTest.php @@ -8,10 +8,13 @@ use Flow\ETL\Tests\FlowTestCase; use Flow\ETL\Window\Accumulator\SumAccumulator; +use function Flow\ETL\DSL\array_to_rows; +use function Flow\ETL\DSL\float_schema; use function Flow\ETL\DSL\flow_context; -use function Flow\ETL\DSL\lit; +use function Flow\ETL\DSL\int_schema; use function Flow\ETL\DSL\ref; -use function Flow\ETL\DSL\row; +use function Flow\ETL\DSL\schema; +use function Flow\ETL\DSL\str_schema; final class SumAccumulatorTest extends FlowTestCase { @@ -20,22 +23,12 @@ public function test_empty_frame_sums_to_null(): void static::assertNull((new SumAccumulator(ref('value'), false, flow_context()))->value()); } - public function test_exact_as_scalar_function_is_resolved_per_row(): void - { - $accumulator = new SumAccumulator(ref('value'), lit(true), flow_context()); - - $accumulator->accumulate(row(['value' => 0.1])); - $accumulator->accumulate(row(['value' => 0.2])); - - static::assertSame(0.3, $accumulator->value()); - } - public function test_exact_sum_avoids_floating_point_drift(): void { $accumulator = new SumAccumulator(ref('value'), true, flow_context()); - $accumulator->accumulate(row(['value' => 0.1])); - $accumulator->accumulate(row(['value' => 0.2])); + $accumulator->accumulate(array_to_rows([['value' => 0.1]], schema(float_schema('value'))), 0); + $accumulator->accumulate(array_to_rows([['value' => 0.2]], schema(float_schema('value'))), 0); static::assertSame(0.3, $accumulator->value()); } @@ -44,8 +37,8 @@ public function test_inexact_sum_keeps_native_addition(): void { $accumulator = new SumAccumulator(ref('value'), false, flow_context()); - $accumulator->accumulate(row(['value' => 0.1])); - $accumulator->accumulate(row(['value' => 0.2])); + $accumulator->accumulate(array_to_rows([['value' => 0.1]], schema(float_schema('value'))), 0); + $accumulator->accumulate(array_to_rows([['value' => 0.2]], schema(float_schema('value'))), 0); static::assertSame(0.1 + 0.2, $accumulator->value()); } @@ -56,17 +49,19 @@ public function test_missing_entry_throws_in_strict_mode(): void $this->expectException(InvalidArgumentException::class); $this->expectExceptionMessageMatches('/^Sum window function error: /'); - (new SumAccumulator(ref('value'), false, $context))->accumulate(row(['other' => 'x'])); + (new SumAccumulator(ref('value'), false, $context))->accumulate(array_to_rows([[ + 'other' => 'x', + ]], schema(str_schema('other'))), 0); } public function test_non_numeric_values_are_skipped(): void { $accumulator = new SumAccumulator(ref('value'), false, flow_context()); - $accumulator->accumulate(row(['value' => 10])); - $accumulator->accumulate(row(['value' => 'not a number'])); - $accumulator->accumulate(row(['value' => null])); - $accumulator->accumulate(row(['value' => 5])); + $accumulator->accumulate(array_to_rows([['value' => 10]], schema(int_schema('value'))), 0); + $accumulator->accumulate(array_to_rows([['value' => 'not a number']], schema(str_schema('value'))), 0); + $accumulator->accumulate(array_to_rows([['value' => null]], schema(str_schema('value', nullable: true))), 0); + $accumulator->accumulate(array_to_rows([['value' => 5]], schema(int_schema('value'))), 0); static::assertSame(15, $accumulator->value()); } @@ -75,8 +70,8 @@ public function test_numeric_strings_are_summed(): void { $accumulator = new SumAccumulator(ref('value'), false, flow_context()); - $accumulator->accumulate(row(['value' => '10'])); - $accumulator->accumulate(row(['value' => '2.5'])); + $accumulator->accumulate(array_to_rows([['value' => '10']], schema(str_schema('value'))), 0); + $accumulator->accumulate(array_to_rows([['value' => '2.5']], schema(str_schema('value'))), 0); static::assertSame(12.5, $accumulator->value()); } @@ -85,7 +80,7 @@ public function test_only_non_numeric_values_still_sums_to_null(): void { $accumulator = new SumAccumulator(ref('value'), false, flow_context()); - $accumulator->accumulate(row(['value' => null])); + $accumulator->accumulate(array_to_rows([['value' => null]], schema(str_schema('value', nullable: true))), 0); static::assertNull($accumulator->value()); } @@ -93,12 +88,12 @@ public function test_only_non_numeric_values_still_sums_to_null(): void public function test_value_is_idempotent(): void { $accumulator = new SumAccumulator(ref('value'), false, flow_context()); - $accumulator->accumulate(row(['value' => 10])); + $accumulator->accumulate(array_to_rows([['value' => 10]], schema(int_schema('value'))), 0); static::assertSame(10, $accumulator->value()); static::assertSame(10, $accumulator->value()); - $accumulator->accumulate(row(['value' => 5])); + $accumulator->accumulate(array_to_rows([['value' => 5]], schema(int_schema('value'))), 0); static::assertSame(15, $accumulator->value()); } @@ -107,8 +102,8 @@ public function test_whole_float_result_stays_float(): void { $accumulator = new SumAccumulator(ref('value'), false, flow_context()); - $accumulator->accumulate(row(['value' => 1.5])); - $accumulator->accumulate(row(['value' => 2.5])); + $accumulator->accumulate(array_to_rows([['value' => 1.5]], schema(float_schema('value'))), 0); + $accumulator->accumulate(array_to_rows([['value' => 2.5]], schema(float_schema('value'))), 0); static::assertSame(4.0, $accumulator->value()); } diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Window/PeerComparatorTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Window/PeerComparatorTest.php new file mode 100644 index 0000000000..9e77621b73 --- /dev/null +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Window/PeerComparatorTest.php @@ -0,0 +1,106 @@ + new DateTimeImmutable('2024-01-01 01:00:00', new DateTimeZone('+01:00')), 'n' => 1], + ['at' => new DateTimeImmutable('2024-01-01 00:00:00', new DateTimeZone('UTC')), 'n' => 1], + ['at' => new DateTimeImmutable('2024-01-01 00:00:00', new DateTimeZone('UTC')), 'n' => 2], + ], + schema(datetime_schema('at'), int_schema('n')), + ); + static::assertSame( + [false, true, false], + (new PeerComparator([ref('at'), ref('n')]))->peersOfPrevious($partition), + ); + } + + public function test_nan_is_a_peer_of_nan(): void + { + $partition = array_to_rows([['v' => NAN], ['v' => NAN], ['v' => 1.0]], schema(float_schema('v'))); + static::assertSame([false, true, false], (new PeerComparator([ref('v')]))->peersOfPrevious($partition)); + } + + public function test_peers_of_previous_marks_each_row_against_the_row_before_it(): void + { + $partition = array_to_rows( + [ + ['at' => new DateTimeImmutable('2024-01-01 01:00:00', new DateTimeZone('+01:00')), 'n' => 1], + ['at' => new DateTimeImmutable('2024-01-01 00:00:00', new DateTimeZone('UTC')), 'n' => 1], + ['at' => new DateTimeImmutable('2024-01-01 00:00:00', new DateTimeZone('UTC')), 'n' => 2], + ], + schema(datetime_schema('at'), int_schema('n')), + ); + + static::assertSame( + [false, true, false], + (new PeerComparator([ref('at'), ref('n')]))->peersOfPrevious($partition), + ); + } + + public function test_peers_of_previous_treats_nan_as_a_peer_of_nan(): void + { + static::assertSame( + [false, true, false], + (new PeerComparator([ref('v')]))->peersOfPrevious(array_to_rows([ + ['v' => NAN], + ['v' => NAN], + ['v' => 1.0], + ], schema(float_schema('v')))), + ); + static::assertSame( + [], + (new PeerComparator([ref('v')]))->peersOfPrevious(array_to_rows([], schema(float_schema('v')))), + ); + } + + public function test_peers_of_previous_compares_values_without_a_physical_order(): void + { + static::assertSame( + [false, true, false], + (new PeerComparator([ref('l')]))->peersOfPrevious(array_to_rows([ + ['l' => [1, 2]], + ['l' => [1, 2]], + ['l' => [2]], + ], schema(list_schema('l', type_list(type_integer()))))), + ); + } + + public function test_xml_elements_of_different_documents_are_peers_by_their_markup(): void + { + static::assertSame( + [false, true, false], + (new PeerComparator([ref('e')]))->peersOfPrevious(array_to_rows([ + ['e' => type_xml_element()->cast('

a

')->firstElementChild], + ['e' => type_xml_element()->cast('

a

')->firstElementChild], + ['e' => type_xml_element()->cast('

b

')->firstElementChild], + ], schema(xml_element_schema('e')))), + ); + } +} diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Window/PeerFrameTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Window/PeerFrameTest.php index 72b0c5418d..c753640480 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Unit/Window/PeerFrameTest.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Window/PeerFrameTest.php @@ -4,23 +4,29 @@ namespace Flow\ETL\Tests\Unit\Window; +use ArrayObject; use DateTimeImmutable; +use Flow\ETL\Rows; +use Flow\ETL\Tests\Double\SpyColumn; use Flow\ETL\Tests\FlowTestCase; use Flow\ETL\Window\PeerFrame; +use function array_map; +use function Flow\ETL\DSL\array_to_rows; use function Flow\ETL\DSL\datetime_schema; use function Flow\ETL\DSL\int_schema; use function Flow\ETL\DSL\ref; -use function Flow\ETL\DSL\row; use function Flow\ETL\DSL\rows; use function Flow\ETL\DSL\schema; use function Flow\ETL\DSL\str_schema; +use function intdiv; +use function range; final class PeerFrameTest extends FlowTestCase { public function test_all_rows_tied_covers_the_whole_partition(): void { - $partition = rows(schema(int_schema('d')), row(['d' => 1]), row(['d' => 1]), row(['d' => 1])); + $partition = array_to_rows([['d' => 1], ['d' => 1], ['d' => 1]], schema(int_schema('d'))); $frame = new PeerFrame([ref('d')]); @@ -29,14 +35,33 @@ public function test_all_rows_tied_covers_the_whole_partition(): void static::assertSame([0, 2], $frame->bounds(2, $partition)); } + public function test_the_partition_is_read_once_for_every_row_of_it(): void + { + $calls = new ArrayObject(); + $schema = schema(int_schema('v')); + $column = array_to_rows( + array_map(static fn(int $i): array => ['v' => intdiv($i, 10)], range(0, 49)), + $schema, + )->column('v'); + $partition = Rows::fromColumns($schema, ['v' => new SpyColumn($column, $calls)], 50); + $frame = new PeerFrame([ref('v')]); + $ends = []; + + for ($index = 0; $index < 50; $index++) { + $ends[] = $frame->bounds($index, $partition)[1]; + } + + static::assertSame(array_map(static fn(int $i): int => (intdiv($i, 10) * 10) + 9, range(0, 49)), $ends); + static::assertCount(1, $calls); + } + public function test_datetime_peers_are_matched_by_value_not_identity(): void { - $partition = rows( - schema(datetime_schema('d')), - row(['d' => new DateTimeImmutable('2024-01-01 00:00:00')]), - row(['d' => new DateTimeImmutable('2024-01-01 00:00:00')]), - row(['d' => new DateTimeImmutable('2024-01-02 00:00:00')]), - ); + $partition = array_to_rows([ + ['d' => new DateTimeImmutable('2024-01-01 00:00:00')], + ['d' => new DateTimeImmutable('2024-01-01 00:00:00')], + ['d' => new DateTimeImmutable('2024-01-02 00:00:00')], + ], schema(datetime_schema('d'))); $frame = new PeerFrame([ref('d')]); @@ -52,11 +77,9 @@ public function test_index_past_the_partition_end_is_empty(): void public function test_multiple_order_by_references_must_all_match(): void { - $partition = rows( + $partition = array_to_rows( + [['a' => 'x', 'b' => 1], ['a' => 'x', 'b' => 1], ['a' => 'x', 'b' => 2]], schema(str_schema('a'), int_schema('b')), - row(['a' => 'x', 'b' => 1]), - row(['a' => 'x', 'b' => 1]), - row(['a' => 'x', 'b' => 2]), ); $frame = new PeerFrame([ref('a'), ref('b')]); @@ -67,7 +90,7 @@ public function test_multiple_order_by_references_must_all_match(): void public function test_ties_extend_the_frame_to_the_last_peer(): void { - $partition = rows(schema(int_schema('d')), row(['d' => 1]), row(['d' => 1]), row(['d' => 2]), row(['d' => 3])); + $partition = array_to_rows([['d' => 1], ['d' => 1], ['d' => 2], ['d' => 3]], schema(int_schema('d'))); $frame = new PeerFrame([ref('d')]); @@ -79,7 +102,7 @@ public function test_ties_extend_the_frame_to_the_last_peer(): void public function test_without_ties_the_frame_ends_at_the_current_row(): void { - $partition = rows(schema(int_schema('d')), row(['d' => 1]), row(['d' => 2]), row(['d' => 3])); + $partition = array_to_rows([['d' => 1], ['d' => 2], ['d' => 3]], schema(int_schema('d'))); $frame = new PeerFrame([ref('d')]); diff --git a/src/core/etl/tests/Flow/ETL/Tests/Unit/Window/WindowContextTest.php b/src/core/etl/tests/Flow/ETL/Tests/Unit/Window/WindowContextTest.php index 9513f61655..166f837171 100644 --- a/src/core/etl/tests/Flow/ETL/Tests/Unit/Window/WindowContextTest.php +++ b/src/core/etl/tests/Flow/ETL/Tests/Unit/Window/WindowContextTest.php @@ -22,7 +22,16 @@ public function test_accessors_return_what_was_passed_in(): void static::assertSame(2, $context->index()); static::assertSame($partition, $context->partition()); - static::assertSame($partition[2]->values(), $context->row()->values()); + } + + public function test_frame_is_the_slice_of_the_partition_between_the_bounds(): void + { + static::assertSame( + [['id' => 2], ['id' => 3], ['id' => 4]], + WindowContextMother::atIndex(RowsMother::sequentialIds(5), 3, new RowsFrame(preceding(2), current_row())) + ->frame() + ->toArray(), + ); } public function test_empty_bounds_produce_an_empty_frame(): void @@ -67,8 +76,8 @@ public function test_frame_is_sliced_according_to_the_window_frame(): void )->frame(); static::assertCount(3, $frame); - static::assertSame(2, $frame[0]->get('id')); - static::assertSame(3, $frame[1]->get('id')); - static::assertSame(4, $frame[2]->get('id')); + static::assertSame(2, $frame->column('id')->value(0)); + static::assertSame(3, $frame->column('id')->value(1)); + static::assertSame(4, $frame->column('id')->value(2)); } } diff --git a/src/core/etl/tests/Flow/Floe/Tests/Context/FloeEngineContext.php b/src/core/etl/tests/Flow/Floe/Tests/Context/FloeEngineContext.php deleted file mode 100644 index 8d5e1043d4..0000000000 --- a/src/core/etl/tests/Flow/Floe/Tests/Context/FloeEngineContext.php +++ /dev/null @@ -1,126 +0,0 @@ - - */ - public static function readRows(FloeReader $reader, Path $path): array - { - $file = $reader->read($path); - $rows = []; - - foreach ($file->rows() as $batch) { - foreach ($batch as $row) { - $rows[] = $row; - } - } - - $file->close(); - - return $rows; - } - - /** - * A two-file Hive tree whose bodies carry only `id`, so `country` can only come from the path. - */ - public static function writePartitionedFiles(Filesystem $filesystem, string $root = 'memory://parts'): void - { - $schema = schema(int_schema('id')); - - foreach (['PL' => 1, 'DE' => 2] as $country => $id) { - self::writeAll( - self::phpWriter($filesystem, $schema), - path($root . '/country=' . $country . '/data.floe'), - [rows($schema, row(['id' => $id]))], - ); - } - } - - /** - * The same tree with a numerically-valued partition, so a declared partition type has something - * to convert - `country=PL` is already a string and would prove nothing. - */ - public static function writeYearPartitionedFiles(Filesystem $filesystem, string $root = 'memory://years'): void - { - $schema = schema(int_schema('id')); - - foreach (['2024' => 1, '2025' => 2] as $year => $id) { - self::writeAll( - self::phpWriter($filesystem, $schema), - path($root . '/year=' . $year . '/data.floe'), - [rows($schema, row(['id' => $id]))], - ); - } - } - - /** - * @param array $files - uri => the rows written to it - */ - public static function writeFiles(Filesystem $filesystem, array $files): void - { - foreach ($files as $uri => $rows) { - self::writeAll(self::phpWriter($filesystem, $rows->schema()), path($uri), [$rows]); - } - } - - /** - * @param array $batches - */ - public static function writeAll(FloeWriter $writer, Path $path, array $batches): void - { - $writer->create($path); - - foreach ($batches as $batch) { - $writer->write($batch); - } - - $writer->close(); - } -} diff --git a/src/core/etl/tests/Flow/Floe/Tests/Context/FloeFilesContext.php b/src/core/etl/tests/Flow/Floe/Tests/Context/FloeFilesContext.php new file mode 100644 index 0000000000..15612414c2 --- /dev/null +++ b/src/core/etl/tests/Flow/Floe/Tests/Context/FloeFilesContext.php @@ -0,0 +1,91 @@ + 1, 'DE' => 2] as $country => $id) { + self::writeAll( + self::phpWriter($filesystem, $schema), + path($root . '/country=' . $country . '/data.floe'), + [array_to_rows([['id' => $id]], $schema)], + ); + } + } + + /** + * The same tree with a numerically-valued partition, so a declared partition type has something + * to convert - `country=PL` is already a string and would prove nothing. + */ + public static function writeYearPartitionedFiles(Filesystem $filesystem, string $root = 'memory://years'): void + { + $schema = schema(int_schema('id')); + + foreach (['2024' => 1, '2025' => 2] as $year => $id) { + self::writeAll( + self::phpWriter($filesystem, $schema), + path($root . '/year=' . $year . '/data.floe'), + [array_to_rows([['id' => $id]], $schema)], + ); + } + } + + /** + * @param array $files - uri => the rows written to it + */ + public static function writeFiles(Filesystem $filesystem, array $files): void + { + foreach ($files as $uri => $rows) { + self::writeAll(self::phpWriter($filesystem, $rows->schema()), path($uri), [$rows]); + } + } + + /** + * @param array $batches + */ + public static function writeAll(FloeWriter $writer, Path $path, array $batches): void + { + $writer->create($path); + + foreach ($batches as $batch) { + $writer->write($batch); + } + + $writer->close(); + } +} diff --git a/src/core/etl/tests/Flow/Floe/Tests/Context/FloeGoldenContext.php b/src/core/etl/tests/Flow/Floe/Tests/Context/FloeGoldenContext.php index 7057b91052..dc060dbe2d 100644 --- a/src/core/etl/tests/Flow/Floe/Tests/Context/FloeGoldenContext.php +++ b/src/core/etl/tests/Flow/Floe/Tests/Context/FloeGoldenContext.php @@ -40,8 +40,8 @@ public static function path(string $name, string $version = self::VERSION_DIRECT /** * Rewrites every golden file from the batches above. Committed on purpose: fixtures are only - * trustworthy when the thing that made them is reviewable, and phase 5 of the format bump has - * to be reproducible by someone who is not the author. + * trustworthy when the thing that made them is reviewable and reproducible by someone who is not + * the author. */ public static function regenerate(string $version = self::VERSION_DIRECTORY): void { @@ -54,8 +54,8 @@ public static function regenerate(string $version = self::VERSION_DIRECTORY): vo $filesystem->rm($path); } - FloeEngineContext::writeAll( - FloeEngineContext::phpWriter($filesystem, self::schema($batches)), + FloeFilesContext::writeAll( + FloeFilesContext::phpWriter($filesystem, self::schema($batches)), $path, $batches, ); diff --git a/src/core/etl/tests/Flow/Floe/Tests/Context/FloeStreamReaderContext.php b/src/core/etl/tests/Flow/Floe/Tests/Context/FloeStreamReaderContext.php index 188c81e6ac..bd73effb1d 100644 --- a/src/core/etl/tests/Flow/Floe/Tests/Context/FloeStreamReaderContext.php +++ b/src/core/etl/tests/Flow/Floe/Tests/Context/FloeStreamReaderContext.php @@ -4,7 +4,8 @@ namespace Flow\Floe\Tests\Context; -use Flow\ETL\Row\Hydrator; +use Flow\ETL\Column\AdaptiveBackend; +use Flow\ETL\Column\PhpBackend; use Flow\ETL\Rows; use Flow\ETL\Schema; use Flow\ETL\Schema\Metadata; @@ -12,7 +13,6 @@ use Flow\Filesystem\Path; use Flow\Floe\Codec; use Flow\Floe\Codec\NoopCodec; -use Flow\Floe\FloeEngine; use Flow\Floe\FloeReader; use Flow\Floe\FloeWriter; use Flow\Floe\Footer; @@ -23,7 +23,6 @@ use Flow\Floe\Section; use Flow\Floe\Statistics; -use function count; use function Flow\ETL\DSL\rows; use function Flow\ETL\DSL\schema; use function Flow\Filesystem\DSL\memory_filesystem; @@ -76,13 +75,13 @@ public static function roundTrip(Rows $rows): Rows $filesystem = memory_filesystem(); $path = path('memory://round-trip.floe'); - $writer = new FloeWriter($filesystem, $rows->schema()); + $writer = new FloeWriter($filesystem, $rows->schema(), new AdaptiveBackend()); $writer->create($path); $writer->write($rows); $writer->close(); $batches = iterator_to_array( - (new FloeReader($filesystem)) + (new FloeReader($filesystem, new AdaptiveBackend())) ->read($path) ->rows(), ); @@ -109,16 +108,17 @@ public static function frameTypes(Filesystem $filesystem, Path $path): array */ public static function reconstruct(Filesystem $filesystem, Path $path): Rows { - $file = (new FloeReader($filesystem))->read($path); - $rows = []; + $file = (new FloeReader($filesystem, new AdaptiveBackend()))->read($path); + $rows = rows($file->footer()->schema()); foreach ($file->rows() as $batch) { - foreach ($batch->all() as $row) { - $rows[] = $row; - } + $rows = $rows->concat( + new AdaptiveBackend(), + $batch->matchTo($file->footer()->schema(), new AdaptiveBackend()), + ); } - return new Rows($file->footer()->schema(), ...$rows); + return $rows; } /** @@ -128,10 +128,10 @@ public static function readAll(Filesystem $filesystem, Path $path): Rows { $merged = null; - foreach ((new FloeReader($filesystem)) + foreach ((new FloeReader($filesystem, new AdaptiveBackend())) ->read($path) ->rows() as $batch) { - $merged = $merged === null ? $batch : $merged->merge($batch); + $merged = $merged === null ? $batch : $merged->concat(new PhpBackend(), $batch); } return $merged ?? rows(schema()); @@ -147,33 +147,36 @@ public static function write( array $metadata = [], Codec $codec = new NoopCodec(), ): void { - $writer = new FloeWriter($filesystem, $rows->schema(), new Options(codec: $codec)); + $writer = new FloeWriter($filesystem, $rows->schema(), new AdaptiveBackend(), new Options(codec: $codec)); $writer->create($path, Metadata::fromArray($metadata)); $writer->write($rows); $writer->close(); } /** - * A file holding exactly these ROW frame bodies under `$schema` - frames no FloeWriter would produce. - * - * @param list $bodies + * A file holding exactly these complete frames under `$schema` - frames no FloeWriter would produce. */ - public static function writeFrames(Filesystem $filesystem, Path $path, Schema $schema, array $bodies): void - { + public static function writeFrames( + Filesystem $filesystem, + Path $path, + Schema $schema, + string $frames, + int $rowCount, + ): void { /** @var array> $normalized */ $normalized = $schema->normalize(); $footerJson = (new Footer( Format::VERSION, 'test', $normalized, - [new Section(Format::HEADER_LENGTH, count($bodies))], - new Statistics(count($bodies), strlen(Format::rowFrames($bodies))), + [new Section(Format::HEADER_LENGTH, $rowCount)], + new Statistics($rowCount, strlen($frames)), Metadata::empty(), ))->toJson(); $stream = $filesystem->writeTo($path); $stream->append( - Format::header(0x00) . Format::rowFrames($bodies) + Format::header(0x00) . $frames . Format::frame(Format::FRAME_FOOTER, $footerJson . Format::trailer(strlen($footerJson))), ); $stream->close(); @@ -188,10 +191,8 @@ public static function writeBatches( Schema $schema, array $batches, Options $options = new Options(), - ?Hydrator $hydrator = null, - FloeEngine $engine = FloeEngine::adaptive, ): string { - $writer = new FloeWriter($filesystem, $schema, $options, $hydrator, $engine); + $writer = new FloeWriter($filesystem, $schema, new AdaptiveBackend(), $options); $writer->create($path); foreach ($batches as $batch) { @@ -210,7 +211,7 @@ public static function writeBatches( */ public static function writeWithoutFooter(Filesystem $filesystem, Path $path, Rows $rows): void { - $writer = new FloeWriter($filesystem, $rows->schema()); + $writer = new FloeWriter($filesystem, $rows->schema(), new AdaptiveBackend()); $writer->create($path); $writer->write($rows); $writer->close(); diff --git a/src/core/etl/tests/Flow/Floe/Tests/Context/FrameContext.php b/src/core/etl/tests/Flow/Floe/Tests/Context/FrameContext.php new file mode 100644 index 0000000000..974b6356f3 --- /dev/null +++ b/src/core/etl/tests/Flow/Floe/Tests/Context/FrameContext.php @@ -0,0 +1,37 @@ + the row, node and buffer counts, then every node and extent word + */ + public static function directory(string $body): array + { + /** @var array{1: int, 2: int, 3: int} $counts */ + $counts = unpack('V3', $body); + + /** @var list */ + return array_values((array) unpack('V' . (3 + (2 * ($counts[2] + $counts[3]))), $body)); + } + + /** + * @param array $words directory word index (0 = row count) => new value + */ + public static function patch(string $body, array $words): string + { + foreach ($words as $index => $word) { + $body = substr_replace($body, pack('V', $word), 4 * $index, 4); + } + + return $body; + } +} diff --git a/src/core/etl/tests/Flow/Floe/Tests/Double/ClosingSpySourceStream.php b/src/core/etl/tests/Flow/Floe/Tests/Double/ClosingSpySourceStream.php index e2a1044f6b..db71014ec4 100644 --- a/src/core/etl/tests/Flow/Floe/Tests/Double/ClosingSpySourceStream.php +++ b/src/core/etl/tests/Flow/Floe/Tests/Double/ClosingSpySourceStream.php @@ -8,10 +8,14 @@ use Flow\Filesystem\SourceStream; use Generator; +use function strlen; + final class ClosingSpySourceStream implements SourceStream { public int $closeCount = 0; + public int $readBytes = 0; + public function __construct( private readonly SourceStream $stream, ) {} @@ -24,7 +28,10 @@ public function close(): void public function content(): string { - return $this->stream->content(); + $content = $this->stream->content(); + $this->readBytes += strlen($content); + + return $content; } public function isOpen(): bool @@ -34,7 +41,11 @@ public function isOpen(): bool public function iterate(int $length = 1): Generator { - return $this->stream->iterate($length); + foreach ($this->stream->iterate($length) as $chunk) { + $this->readBytes += strlen($chunk); + + yield $chunk; + } } public function path(): Path @@ -44,7 +55,10 @@ public function path(): Path public function read(int $length, int $offset): string { - return $this->stream->read($length, $offset); + $bytes = $this->stream->read($length, $offset); + $this->readBytes += strlen($bytes); + + return $bytes; } public function readLines(string $separator = "\n", ?int $length = null): Generator diff --git a/src/core/etl/tests/Flow/Floe/Tests/Double/OverflowingColumnStub.php b/src/core/etl/tests/Flow/Floe/Tests/Double/OverflowingColumnStub.php new file mode 100644 index 0000000000..0f2b300011 --- /dev/null +++ b/src/core/etl/tests/Flow/Floe/Tests/Double/OverflowingColumnStub.php @@ -0,0 +1,94 @@ +column->at($i); + } + + public function concat(Column ...$others): Column + { + return $this->column->concat(...$others); + } + + public function count(): int + { + return $this->column->count(); + } + + public function encode(): array + { + $this->encodeCalls++; + + if ($this->count() > $this->maxRows) { + throw $this->failure; + } + + return $this->column->encode(); + } + + public function isNull(int $i): bool + { + return $this->column->isNull($i); + } + + public function nullCount(): int + { + return $this->column->nullCount(); + } + + public function physicals(): array + { + return $this->column->physicals(); + } + + public function slice(int $offset, int $length): Column + { + return new self($this->column->slice($offset, $length), $this->failure, $this->maxRows); + } + + public function take(array $indices): Column + { + return $this->column->take($indices); + } + + public function type(): Type + { + return $this->column->type(); + } + + public function value(int $i): mixed + { + return $this->column->value($i); + } + + public function values(): array + { + return $this->column->values(); + } + + public function withType(Type $type): Column + { + return $this->column->withType($type); + } +} diff --git a/src/core/etl/tests/Flow/Floe/Tests/Double/SpyHydrator.php b/src/core/etl/tests/Flow/Floe/Tests/Double/SpyHydrator.php deleted file mode 100644 index 5bf885e8cb..0000000000 --- a/src/core/etl/tests/Flow/Floe/Tests/Double/SpyHydrator.php +++ /dev/null @@ -1,31 +0,0 @@ -dehydrateCalls++; - - return (new AdaptiveRowHydrator())->dehydrate($rows); - } - - public function hydrate(array $batch, Schema $schema): Rows - { - $this->hydrateCalls++; - - return (new AdaptiveRowHydrator())->hydrate($batch, $schema); - } -} diff --git a/src/core/etl/tests/Flow/Floe/Tests/Double/TrailingZerosCodecStub.php b/src/core/etl/tests/Flow/Floe/Tests/Double/TrailingZerosCodecStub.php new file mode 100644 index 0000000000..d1d77a8795 --- /dev/null +++ b/src/core/etl/tests/Flow/Floe/Tests/Double/TrailingZerosCodecStub.php @@ -0,0 +1,37 @@ +read(from_floe($dir . '/*.floe'))->count()); } - public function test_writing_and_reading_with_explicit_native_engine(): void - { - if (!NativeFloeEncoder::isSupported()) { - static::markTestSkipped('flow_php extension is not loaded'); - } - - $path = $this->cacheDir->suffix('native-engine.floe'); - - data_frame() - ->read(from_array([['id' => 1], ['id' => 2]])) - ->write(to_floe($path, engine: FloeEngine::native)->saveMode(overwrite())) - ->run(); - - static::assertSame(2, data_frame()->read(from_floe($path, engine: FloeEngine::native))->count()); - } - - public function test_writing_and_reading_with_explicit_php_engine(): void + public function test_writing_and_reading_with_the_default_backend(): void { $path = $this->cacheDir->suffix('php-engine.floe'); - data_frame(config_builder()->hydrator(new PhpRowHydrator())) + data_frame(config_builder()) ->read(from_array([['id' => 1], ['id' => 2]])) - ->write(to_floe($path, engine: FloeEngine::php)->saveMode(overwrite())) + ->write(to_floe($path)->saveMode(overwrite())) ->run(); - static::assertSame( - 2, - data_frame(config_builder()->hydrator(new PhpRowHydrator())) - ->read(from_floe($path, engine: FloeEngine::php)) - ->count(), - ); + static::assertSame(2, data_frame(config_builder())->read(from_floe($path))->count()); } public function test_a_timezone_column_round_trips_through_floe(): void @@ -98,11 +73,10 @@ public function test_a_timezone_column_round_trips_through_floe(): void $path = $this->cacheDir->suffix('timezones.floe'); data_frame() - ->read(from_rows(rows( - schema(time_zone_schema('tz')), - row(['tz' => type_time_zone()->cast('Europe/Warsaw')]), - row(['tz' => type_time_zone()->cast('UTC')]), - ))) + ->read(from_rows(array_to_rows([ + ['tz' => type_time_zone()->cast('Europe/Warsaw')], + ['tz' => type_time_zone()->cast('UTC')], + ], schema(time_zone_schema('tz'))))) ->write(to_floe($path)->saveMode(overwrite())) ->run(); @@ -112,8 +86,8 @@ public function test_a_timezone_column_round_trips_through_floe(): void $names = []; - foreach ($read as $row) { - $names[] = type_instance_of(DateTimeZone::class)->assert($row->get('tz'))->getName(); + foreach ($read->toArray() as $row) { + $names[] = type_instance_of(DateTimeZone::class)->assert($row['tz'])->getName(); } static::assertSame(['Europe/Warsaw', 'UTC'], $names); @@ -139,12 +113,14 @@ public function test_inferred_nested_arrays_are_projected_to_json_and_round_trip // column nullable - NOT NULL is a declaration, never an inference. static::assertTrue($rows->schema()->get('body')->isNullable()); - $first = $rows[0]->get('body'); + // @mago-ignore analysis:mixed-assignment + $first = $rows->column('body')->value(0); static::assertIsArray($first); static::assertInstanceOf(Json::class, $first['data']); static::assertSame([1, 'a'], $first['data']->toArray()); - $second = $rows[1]->get('body'); + // @mago-ignore analysis:mixed-assignment + $second = $rows->column('body')->value(1); static::assertIsArray($second); static::assertInstanceOf(Json::class, $second['data']); static::assertSame([], $second['data']->toArray()); @@ -155,10 +131,10 @@ public function test_list_of_json_round_trip(): void $path = $this->cacheDir->suffix('list-json.floe'); data_frame() - ->read(from_rows(rows( - schema(list_schema('json_list', type_list(type_json()))), - row(['json_list' => [Json::fromArray(['a' => 1]), Json::fromArray([1, 'b'])]]), - ))) + ->read(from_rows(array_to_rows([['json_list' => [ + Json::fromArray(['a' => 1]), + Json::fromArray([1, 'b']), + ]]], schema(list_schema('json_list', type_list(type_json())))))) ->write(to_floe($path)->saveMode(overwrite())) ->run(); @@ -166,7 +142,8 @@ public function test_list_of_json_round_trip(): void static::assertSame('list', $rows->schema()->get('json_list')->type()->toString()); - $list = $rows[0]->get('json_list'); + // @mago-ignore analysis:mixed-assignment + $list = $rows->column('json_list')->value(0); static::assertIsArray($list); static::assertInstanceOf(Json::class, $list[0]); static::assertSame(['a' => 1], $list[0]->toArray()); @@ -191,7 +168,7 @@ public function test_schema_and_extract_agree_in_both_metadata_states(): void static::assertSame( array_keys($extractor->schema()->definitions()), - array_keys(data_frame()->read($extractor)->fetch()->first()->toArray()), + array_keys(data_frame()->read($extractor)->fetch()->toArray()[0]), ); } } @@ -207,7 +184,7 @@ public function test_input_file_uri_is_added_when_configured(): void $rows = data_frame()->read(from_floe($path)->withMetadataColumns(true))->fetch(); - static::assertTrue($rows->first()->has('_input_file_uri')); + static::assertNotNull($rows->schema()->findDefinition('_input_file_uri')); } public function test_offset_and_limit_pushdown(): void @@ -289,7 +266,7 @@ public function test_round_trip(): void $result = data_frame()->read(from_floe($path))->fetch(); static::assertSame(2, $result->count()); - static::assertSame(['id', 'name'], $result->first()->names()); + static::assertSame(['id', 'name'], array_keys($result->values(0))); } public function test_a_close_that_fails_during_closure_leaves_no_file(): void diff --git a/src/core/etl/tests/Flow/Floe/Tests/Integration/FloeExtractorTest.php b/src/core/etl/tests/Flow/Floe/Tests/Integration/FloeExtractorTest.php index 0ec907453d..382123a4ac 100644 --- a/src/core/etl/tests/Flow/Floe/Tests/Integration/FloeExtractorTest.php +++ b/src/core/etl/tests/Flow/Floe/Tests/Integration/FloeExtractorTest.php @@ -7,18 +7,31 @@ use DateTimeImmutable; use DateTimeZone; use Flow\ETL\Cardinality; +use Flow\ETL\Rows; +use Flow\ETL\Tests\Context\ExtractedRows; use Flow\ETL\Tests\Double\CountingFilesystem; use Flow\ETL\Tests\FlowIntegrationTestCase; +use Flow\Floe\Tests\Context\FloeFilesContext; use Flow\Floe\Tests\Context\FloeStreamReaderContext; +use function array_map; +use function array_merge; +use function Flow\ETL\DSL\array_to_rows; use function Flow\ETL\DSL\datetime_schema; use function Flow\ETL\DSL\df; +use function Flow\ETL\DSL\flow_context; +use function Flow\ETL\DSL\from_array; use function Flow\ETL\DSL\int_schema; -use function Flow\ETL\DSL\row; -use function Flow\ETL\DSL\rows; +use function Flow\ETL\DSL\ref; use function Flow\ETL\DSL\schema; +use function Flow\ETL\DSL\to_timezone; +use function Flow\Filesystem\DSL\memory_filesystem; +use function Flow\Filesystem\DSL\path; use function Flow\Floe\DSL\from_floe; +use function Flow\Floe\DSL\to_floe; use function Flow\Types\DSL\type_datetime; +use function Flow\Types\DSL\type_string; +use function iterator_to_array; final class FloeExtractorTest extends FlowIntegrationTestCase { @@ -27,12 +40,12 @@ public function test_a_glob_declares_an_approximate_row_count(): void FloeStreamReaderContext::write( $this->fs(), $this->cacheDir->suffix('glob/a.floe'), - rows(schema(int_schema('id')), row(['id' => 1]), row(['id' => 2])), + array_to_rows([['id' => 1], ['id' => 2]], schema(int_schema('id'))), ); FloeStreamReaderContext::write( $this->fs(), $this->cacheDir->suffix('glob/b.floe'), - rows(schema(int_schema('id')), row(['id' => 3])), + array_to_rows([['id' => 3]], schema(int_schema('id'))), ); $firstBytes = FloeStreamReaderContext::dataBytes($this->fs(), $this->cacheDir->suffix('glob/a.floe')); @@ -48,11 +61,10 @@ public function test_a_glob_declares_an_approximate_row_count(): void public function test_a_single_file_declares_exact_rows_and_byte_size(): void { $path = $this->cacheDir->suffix('single.floe'); - FloeStreamReaderContext::write( - $this->fs(), - $path, - rows(schema(int_schema('id')), row(['id' => 1]), row(['id' => 2])), - ); + FloeStreamReaderContext::write($this->fs(), $path, array_to_rows([ + ['id' => 1], + ['id' => 2], + ], schema(int_schema('id')))); $statistics = from_floe($path, filesystem: $this->fs())->statistics(); @@ -66,11 +78,10 @@ public function test_a_single_file_declares_exact_rows_and_byte_size(): void public function test_an_offset_is_subtracted_from_the_row_count(): void { $path = $this->cacheDir->suffix('offset.floe'); - FloeStreamReaderContext::write( - $this->fs(), - $path, - rows(schema(int_schema('id')), row(['id' => 1]), row(['id' => 2])), - ); + FloeStreamReaderContext::write($this->fs(), $path, array_to_rows([ + ['id' => 1], + ['id' => 2], + ], schema(int_schema('id')))); $extractor = from_floe($path, filesystem: $this->fs()); @@ -82,7 +93,7 @@ public function test_an_offset_is_subtracted_from_the_row_count(): void public function test_statistics_are_computed_at_most_once(): void { $path = $this->cacheDir->suffix('memo.floe'); - FloeStreamReaderContext::write($this->fs(), $path, rows(schema(int_schema('id')), row(['id' => 1]))); + FloeStreamReaderContext::write($this->fs(), $path, array_to_rows([['id' => 1]], schema(int_schema('id')))); $filesystem = new CountingFilesystem($this->fs()); $extractor = from_floe($path, filesystem: $filesystem); @@ -94,7 +105,7 @@ public function test_statistics_are_computed_at_most_once(): void public function test_schema_then_statistics_reads_the_footer_once(): void { $path = $this->cacheDir->suffix('schema-first.floe'); - FloeStreamReaderContext::write($this->fs(), $path, rows(schema(int_schema('id')), row(['id' => 1]))); + FloeStreamReaderContext::write($this->fs(), $path, array_to_rows([['id' => 1]], schema(int_schema('id')))); $filesystem = new CountingFilesystem($this->fs()); $extractor = from_floe($path, filesystem: $filesystem); @@ -104,28 +115,17 @@ public function test_schema_then_statistics_reads_the_footer_once(): void static::assertSame(1, $filesystem->readFromCalls); } - public function test_a_declared_schema_still_reads_the_footer_for_statistics(): void - { - $path = $this->cacheDir->suffix('declared.floe'); - FloeStreamReaderContext::write($this->fs(), $path, rows(schema(int_schema('id')), row(['id' => 1]))); - - static::assertEquals( - Cardinality::exact(1), - from_floe($path, filesystem: $this->fs())->withSchema(schema(int_schema('id')))->statistics()->rows, - ); - } - public function test_union_by_name_sums_every_footer_exactly(): void { FloeStreamReaderContext::write( $this->fs(), $this->cacheDir->suffix('union/a.floe'), - rows(schema(int_schema('id')), row(['id' => 1]), row(['id' => 2])), + array_to_rows([['id' => 1], ['id' => 2]], schema(int_schema('id'))), ); FloeStreamReaderContext::write( $this->fs(), $this->cacheDir->suffix('union/b.floe'), - rows(schema(int_schema('id')), row(['id' => 3])), + array_to_rows([['id' => 3]], schema(int_schema('id'))), ); $statistics = from_floe($this->cacheDir->suffix('union/*.floe'), filesystem: $this->fs()) @@ -145,7 +145,7 @@ public function test_union_by_name_sums_every_footer_exactly(): void public function test_union_by_name_invalidates_the_memo(): void { $path = $this->cacheDir->suffix('union-memo.floe'); - FloeStreamReaderContext::write($this->fs(), $path, rows(schema(int_schema('id')), row(['id' => 1]))); + FloeStreamReaderContext::write($this->fs(), $path, array_to_rows([['id' => 1]], schema(int_schema('id')))); $filesystem = new CountingFilesystem($this->fs()); $extractor = from_floe($path, filesystem: $filesystem); @@ -155,30 +155,24 @@ public function test_union_by_name_invalidates_the_memo(): void static::assertSame(2, $filesystem->readFromCalls); } - public function test_declared_zone_rezones_a_utc_file(): void + public function test_a_utc_file_is_rezoned_after_reading(): void { $path = $this->cacheDir->suffix('utc.floe'); - FloeStreamReaderContext::write( - $this->fs(), - $path, - rows( - schema(datetime_schema('at')), - row(['at' => new DateTimeImmutable('2026-01-02 03:04:05', new DateTimeZone('UTC'))]), - ), - ); + FloeStreamReaderContext::write($this->fs(), $path, array_to_rows([['at' => new DateTimeImmutable( + '2026-01-02 03:04:05', + new DateTimeZone('UTC'), + )]], schema(datetime_schema('at')))); static::assertSame( '2026-01-02 04:04:05 Europe/Warsaw', type_datetime() ->assert( df() - ->read(from_floe($path, filesystem: $this->fs())->withSchema(schema(datetime_schema( - 'at', - zone: 'Europe/Warsaw', - )))) + ->read(from_floe($path, filesystem: $this->fs())) + ->withEntry('at', to_timezone(ref('at'), 'Europe/Warsaw')) ->fetch() - ->first() - ->get('at'), + ->column('at') + ->value(0), ) ->format('Y-m-d H:i:s e'), ); @@ -189,28 +183,90 @@ public function test_two_files_read_in_the_first_footer_zone(): void FloeStreamReaderContext::write( $this->fs(), $this->cacheDir->suffix('zones/a.floe'), - rows( - schema(datetime_schema('at', zone: 'Europe/Warsaw')), - row(['at' => new DateTimeImmutable('2026-01-02 03:04:05', new DateTimeZone('Europe/Warsaw'))]), - ), + array_to_rows([['at' => new DateTimeImmutable( + '2026-01-02 03:04:05', + new DateTimeZone('Europe/Warsaw'), + )]], schema(datetime_schema('at', zone: 'Europe/Warsaw'))), ); FloeStreamReaderContext::write( $this->fs(), $this->cacheDir->suffix('zones/b.floe'), - rows( - schema(datetime_schema('at')), - row(['at' => new DateTimeImmutable('2026-01-02 03:04:05', new DateTimeZone('UTC'))]), - ), + array_to_rows([['at' => new DateTimeImmutable( + '2026-01-02 03:04:05', + new DateTimeZone('UTC'), + )]], schema(datetime_schema('at'))), ); $zones = []; foreach (df() ->read(from_floe($this->cacheDir->suffix('zones/*.floe'), filesystem: $this->fs())) - ->fetch() as $row) { - $zones[] = type_datetime()->assert($row->get('at'))->getTimezone()->getName(); + ->fetch() + ->toArray() as $row) { + $zones[] = type_datetime()->assert($row['at'])->getTimezone()->getName(); } static::assertSame(['Europe/Warsaw', 'Europe/Warsaw'], $zones); } + + public function test_xml_nodes_written_before_utf8_node_documents_read_back_and_load_as_their_text(): void + { + $loaded = $this->cacheDir->suffix('xml-node-utf8.floe'); + + df() + ->read(from_floe(__DIR__ . '/../Fixtures/xml-node-before-utf8.floe')) + ->write(to_floe($loaded, filesystem: $this->fs())) + ->run(); + + static::assertSame( + [ + ['node' => 'zażółć ☃ 😀'], + ['node' => ''], + ['node' => '<p> & ©'], + ], + df() + ->read(from_floe($loaded, filesystem: $this->fs())) + ->withEntry('node', ref('node')->cast(type_string())) + ->fetch() + ->toArray(), + ); + } + + public function test_an_offset_spanning_two_files_and_a_limit_inside_the_third_read_the_same_twice(): void + { + $filesystem = memory_filesystem(); + + foreach (['a' => [1, 2, 3], 'b' => [4, 5, 6], 'c' => [7, 8, 9]] as $name => $ids) { + df() + ->read(from_array(array_map(static fn(int $id): array => ['id' => $id], $ids))) + ->write(to_floe('memory://window/' . $name . '.floe', filesystem: $filesystem)) + ->run(); + } + + $extractor = from_floe('memory://window/*.floe', filesystem: $filesystem)->withOffset(4); + $read = static fn(): array => array_merge(...array_map( + static fn(Rows $rows): array => $rows->column('id')->values(), + iterator_to_array($extractor->extract(flow_context(), limit: 4), false), + )); + + static::assertSame([5, 6, 7, 8], $read()); + static::assertSame([5, 6, 7, 8], $read()); + } + + public function test_one_extractor_read_twice_interleaved_gives_each_read_every_row_and_closes_every_stream(): void + { + $filesystem = new CountingFilesystem(memory_filesystem()); + FloeFilesContext::writeFiles($filesystem, [ + 'memory://in/a.floe' => array_to_rows([['id' => 1], ['id' => 2]], schema(int_schema('id'))), + 'memory://in/b.floe' => array_to_rows([['id' => 3]], schema(int_schema('id'))), + ]); + [$first, $second] = ExtractedRows::interleaved(from_floe( + path('memory://in/*.floe'), + filesystem: $filesystem, + )->withBatchSize(1)); + + static::assertSame([['id' => 1], ['id' => 2], ['id' => 3]], $first->toArray()); + static::assertSame($first->toArray(), $second->toArray()); + static::assertSame($filesystem->readFromCalls, $filesystem->closedStreams()); + } } diff --git a/src/core/etl/tests/Flow/Floe/Tests/Integration/FloeGoldenTest.php b/src/core/etl/tests/Flow/Floe/Tests/Integration/FloeGoldenTest.php index 6f573af36b..51666fa52b 100644 --- a/src/core/etl/tests/Flow/Floe/Tests/Integration/FloeGoldenTest.php +++ b/src/core/etl/tests/Flow/Floe/Tests/Integration/FloeGoldenTest.php @@ -6,11 +6,14 @@ use Flow\ETL\Rows; use Flow\ETL\Tests\FlowIntegrationTestCase; -use Flow\Floe\Tests\Context\FloeEngineContext; +use Flow\Floe\Tests\Context\FloeFilesContext; use Flow\Floe\Tests\Context\FloeGoldenContext; use Generator; use PHPUnit\Framework\Attributes\DataProvider; +use function array_map; +use function array_merge; +use function iterator_to_array; use function ord; /** @@ -43,8 +46,8 @@ public function test_golden_bytes_are_unchanged(string $name, array $batches): v { $written = $this->cacheDir->suffix('golden-' . $name . '.floe'); - FloeEngineContext::writeAll( - FloeEngineContext::phpWriter($this->fs(), FloeGoldenContext::schema($batches)), + FloeFilesContext::writeAll( + FloeFilesContext::phpWriter($this->fs(), FloeGoldenContext::schema($batches)), $written, $batches, ); @@ -61,18 +64,14 @@ public function test_golden_bytes_are_unchanged(string $name, array $batches): v #[DataProvider('golden_fixtures')] public function test_golden_files_read_back_to_the_rows_that_wrote_them(string $name, array $batches): void { - $expected = []; + $file = FloeFilesContext::phpReader($this->fs())->read(FloeGoldenContext::path($name)); + $read = iterator_to_array($file->rows(), false); + $file->close(); - foreach ($batches as $batch) { - foreach ($batch as $row) { - $expected[] = $row; - } - } - - static::assertEquals($expected, FloeEngineContext::readRows( - FloeEngineContext::phpReader($this->fs()), - FloeGoldenContext::path($name), - )); + static::assertEquals( + array_merge(...array_map(static fn(Rows $batch) => $batch->toArray(), $batches)), + array_merge(...array_map(static fn(Rows $batch) => $batch->toArray(), $read)), + ); } #[DataProvider('golden_fixture_names')] @@ -80,4 +79,13 @@ public function test_golden_header_is_the_current_format_version(string $name): { static::assertSame(0x02, ord($this->fs()->readFrom(FloeGoldenContext::path($name))->content()[4])); } + + #[DataProvider('golden_fixture_names')] + public function test_golden_fixtures_hold_batch_frames(string $name): void + { + static::assertSame( + $name === 'empty' ? 0x06 : 0x05, + ord($this->fs()->readFrom(FloeGoldenContext::path($name))->content()[6]), + ); + } } diff --git a/src/core/etl/tests/Flow/Floe/Tests/Integration/FloeHardBreakTest.php b/src/core/etl/tests/Flow/Floe/Tests/Integration/FloeHardBreakTest.php index 3bdef57169..8a7bc43036 100644 --- a/src/core/etl/tests/Flow/Floe/Tests/Integration/FloeHardBreakTest.php +++ b/src/core/etl/tests/Flow/Floe/Tests/Integration/FloeHardBreakTest.php @@ -6,7 +6,7 @@ use Flow\ETL\Tests\FlowIntegrationTestCase; use Flow\Floe\Exception\FloeException; -use Flow\Floe\Tests\Context\FloeEngineContext; +use Flow\Floe\Tests\Context\FloeFilesContext; use function Flow\Filesystem\DSL\path; @@ -21,7 +21,7 @@ public function test_reading_a_file_written_before_the_statistics_block_fails_an $this->expectException(FloeException::class); $this->expectExceptionMessageMatches('/Floe footer is malformed: .*statistics/'); - FloeEngineContext::phpReader($this->fs()) + FloeFilesContext::phpReader($this->fs()) ->read(path(__DIR__ . '/../Fixtures/pre-statistics/heterogeneous.floe')) ->schema(); } diff --git a/src/core/etl/tests/Flow/Floe/Tests/Integration/FloeMergeDSLTest.php b/src/core/etl/tests/Flow/Floe/Tests/Integration/FloeMergeDSLTest.php index 85710a10cf..5b2cd18a33 100644 --- a/src/core/etl/tests/Flow/Floe/Tests/Integration/FloeMergeDSLTest.php +++ b/src/core/etl/tests/Flow/Floe/Tests/Integration/FloeMergeDSLTest.php @@ -4,12 +4,12 @@ namespace Flow\Floe\Tests\Integration; +use Flow\ETL\Column\AdaptiveBackend; use Flow\ETL\Tests\FlowIntegrationTestCase; use Flow\Floe\Tests\Context\FloeStreamReaderContext; +use function Flow\ETL\DSL\array_to_rows; use function Flow\ETL\DSL\int_schema; -use function Flow\ETL\DSL\row; -use function Flow\ETL\DSL\rows; use function Flow\ETL\DSL\schema; use function Flow\Floe\DSL\merge_floe; @@ -21,13 +21,13 @@ public function test_merge_floe_compact_over_path_objects(): void $b = $this->cacheDir->suffix('mc-b.floe'); $out = $this->cacheDir->suffix('mc-out.floe'); - FloeStreamReaderContext::write($this->fs(), $a, rows(schema(int_schema('id')), row(['id' => 1]))); - FloeStreamReaderContext::write($this->fs(), $b, rows(schema(int_schema('id')), row(['id' => 2]))); + FloeStreamReaderContext::write($this->fs(), $a, array_to_rows([['id' => 1]], schema(int_schema('id')))); + FloeStreamReaderContext::write($this->fs(), $b, array_to_rows([['id' => 2]], schema(int_schema('id')))); - merge_floe([$a, $b], $out, compact: true); + merge_floe([$a, $b], $out, new AdaptiveBackend(), compact: true); static::assertEquals( - rows(schema(int_schema('id')), row(['id' => 1]), row(['id' => 2])), + array_to_rows([['id' => 1], ['id' => 2]], schema(int_schema('id'))), FloeStreamReaderContext::readAll($this->fs(), $out), ); } @@ -38,13 +38,13 @@ public function test_merge_floe_splices_local_files_from_string_paths(): void $b = $this->cacheDir->suffix('ms-b.floe'); $out = $this->cacheDir->suffix('ms-out.floe'); - FloeStreamReaderContext::write($this->fs(), $a, rows(schema(int_schema('id')), row(['id' => 1]))); - FloeStreamReaderContext::write($this->fs(), $b, rows(schema(int_schema('id')), row(['id' => 2]))); + FloeStreamReaderContext::write($this->fs(), $a, array_to_rows([['id' => 1]], schema(int_schema('id')))); + FloeStreamReaderContext::write($this->fs(), $b, array_to_rows([['id' => 2]], schema(int_schema('id')))); - merge_floe([$a->path(), $b->path()], $out->path()); + merge_floe([$a->path(), $b->path()], $out->path(), new AdaptiveBackend()); static::assertEquals( - rows(schema(int_schema('id')), row(['id' => 1]), row(['id' => 2])), + array_to_rows([['id' => 1], ['id' => 2]], schema(int_schema('id'))), FloeStreamReaderContext::readAll($this->fs(), $out), ); } diff --git a/src/core/etl/tests/Flow/Floe/Tests/Integration/FloePreAllowExtraRemovalTest.php b/src/core/etl/tests/Flow/Floe/Tests/Integration/FloePreAllowExtraRemovalTest.php deleted file mode 100644 index 1bc671a8e3..0000000000 --- a/src/core/etl/tests/Flow/Floe/Tests/Integration/FloePreAllowExtraRemovalTest.php +++ /dev/null @@ -1,55 +0,0 @@ -fs()->readFrom(path(__DIR__ - . '/../Fixtures/pre-allow-extra-removal/all-entry-types.floe'))->content(), - ); - } - - public function test_php_reader_reads_a_footer_that_carries_allow_extra(): void - { - static::assertEquals( - array_values(RowsMother::withAllEntryTypes()->all()), - FloeEngineContext::readRows( - FloeEngineContext::phpReader($this->fs()), - path(__DIR__ . '/../Fixtures/pre-allow-extra-removal/all-entry-types.floe'), - ), - ); - } - - public function test_native_reader_reads_a_footer_that_carries_allow_extra(): void - { - if (!NativeFloeEncoder::isSupported()) { - static::markTestSkipped('flow_php extension with the RawRowValues pipeline is not loaded.'); - } - - static::assertEquals( - array_values(RowsMother::withAllEntryTypes()->all()), - FloeEngineContext::readRows( - FloeEngineContext::nativeReader($this->fs()), - path(__DIR__ . '/../Fixtures/pre-allow-extra-removal/all-entry-types.floe'), - ), - ); - } -} diff --git a/src/core/etl/tests/Flow/Floe/Tests/Integration/FloeReaderExtensionParityTest.php b/src/core/etl/tests/Flow/Floe/Tests/Integration/FloeReaderExtensionParityTest.php deleted file mode 100644 index 9d95a8f07b..0000000000 --- a/src/core/etl/tests/Flow/Floe/Tests/Integration/FloeReaderExtensionParityTest.php +++ /dev/null @@ -1,207 +0,0 @@ - [RowsMother::withAllEntryTypes()], - 'heterogeneous' => [RowsMother::heterogeneous()], - 'empty' => [rows(schema())], - ]; - } - - #[Override] - protected function setUp(): void - { - parent::setUp(); - - if (!NativeFloeEncoder::isSupported()) { - self::markTestSkipped('flow_php extension with the RawRowValues pipeline is not loaded.'); - } - } - - #[DataProvider('rows_datasets')] - public function test_extension_and_pure_php_hydrate_identical_rows(Rows $rows): void - { - $path = $this->cacheDir->suffix('parity.floe'); - - $writer = new FloeWriter($this->fs(), $rows->schema()); - $writer->create($path); - $writer->write($rows); - $writer->close(); - - static::assertEquals( - iterator_to_array(FloeEngineContext::phpReader($this->fs())->read($path)->rows()), - iterator_to_array(FloeEngineContext::nativeReader($this->fs())->read($path)->rows()), - ); - } - - public function test_extension_and_pure_php_seek_offset_identically(): void - { - $path = $this->cacheDir->suffix('parity-offset.floe'); - - $data = rows( - schema(int_schema('id'), str_schema('email', nullable: true)), - row(['id' => 1]), - row(['id' => 2]), - row(['id' => 3]), - row(['id' => 4, 'email' => null]), - row(['id' => 5, 'email' => 'x']), - row(['id' => 6]), - ); - $writer = new FloeWriter($this->fs(), $data->schema()); - $writer->create($path); - $writer->write($data); - $writer->close(); - - static::assertEquals( - iterator_to_array(FloeEngineContext::phpReader($this->fs())->read($path)->rows(1000, 2, 3)), - iterator_to_array(FloeEngineContext::nativeReader($this->fs())->read($path)->rows(1000, 2, 3)), - ); - } - - /** - * @return array - */ - public static function codec_matrix(): array - { - return [ - 'noop codec' => [new NoopCodec()], - 'transforming codec' => [new PrefixingCodecStub()], - ]; - } - - #[DataProvider('rows_datasets')] - public function test_extension_and_pure_php_read_identical_rows_with_transforming_codec(Rows $rows): void - { - $codec = new PrefixingCodecStub(); - $path = $this->cacheDir->suffix('read-codec.floe'); - - $writer = new FloeWriter($this->fs(), $rows->schema(), new Options(codec: $codec)); - $writer->create($path); - $writer->write($rows); - $writer->close(); - - static::assertEquals( - iterator_to_array(FloeEngineContext::phpReader($this->fs(), $codec)->read($path)->rows()), - iterator_to_array(FloeEngineContext::nativeReader($this->fs(), $codec)->read($path)->rows()), - ); - } - - /** - * Full write->read parity across the engine switch for both codecs and for - * plain, partitioned and schema-evolving files: a pure-PHP round-trip and a - * native round-trip must yield identical rows in every cell. - */ - #[DataProvider('codec_matrix')] - public function test_round_trip_matrix_matches_across_engines(Codec $codec): void - { - $cases = [ - 'plain' => [RowsMother::heterogeneous()], - 'multi-write' => [ - rows(schema(int_schema('id'), str_schema('email')), row(['id' => 1, 'email' => 'a'])), - rows(schema(int_schema('id'), str_schema('email')), row(['id' => 2, 'email' => 'x'])), - ], - ]; - - foreach ($cases as $name => $batches) { - $schema = array_reduce( - $batches, - static fn(Schema $carry, Rows $batch): Schema => $carry->merge($batch->schema()), - schema(), - ); - - $purePath = $this->cacheDir->suffix("matrix-{$name}-pure.floe"); - $extPath = $this->cacheDir->suffix("matrix-{$name}-ext.floe"); - - foreach ([[$purePath, false], [$extPath, true]] as [$path, $native]) { - $writer = $native - ? FloeEngineContext::nativeWriter($this->fs(), $schema, $codec) - : FloeEngineContext::phpWriter($this->fs(), $schema, $codec); - $writer->create($path); - - foreach ($batches as $batch) { - $writer->write($batch); - } - - $writer->close(); - } - - static::assertSame( - $this->fs()->readFrom($purePath)->content(), - $this->fs()->readFrom($extPath)->content(), - "byte-identity failed for {$name}", - ); - - static::assertEquals( - iterator_to_array(FloeEngineContext::phpReader($this->fs(), $codec)->read($purePath)->rows()), - iterator_to_array(FloeEngineContext::nativeReader($this->fs(), $codec)->read($extPath)->rows()), - "row parity failed for {$name}", - ); - } - } - - public function test_extension_and_pure_php_pad_merged_sections_identically(): void - { - // a merged file re-encodes to one union schema; base rows lack email and must pad - // identically across engines on read - $path = $this->cacheDir->suffix('parity-evolved.floe'); - $base = $this->cacheDir->suffix('parity-base.floe'); - $evolved = $this->cacheDir->suffix('parity-new.floe'); - - $baseRows = rows(schema(int_schema('id')), row(['id' => 1])); - $writer = new FloeWriter($this->fs(), $baseRows->schema()); - $writer->create($base); - $writer->write($baseRows); - $writer->close(); - - $evolvedRows = rows( - schema(int_schema('id'), str_schema('email', nullable: true)), - row(['id' => 2, 'email' => null]), - ); - $writer = new FloeWriter($this->fs(), $evolvedRows->schema()); - $writer->create($evolved); - $writer->write($evolvedRows); - $writer->close(); - - (new FloeMerger($this->fs()))->merge([$base, $evolved], $path); - - static::assertEquals( - iterator_to_array(FloeEngineContext::phpReader($this->fs())->read($path)->rows()), - iterator_to_array(FloeEngineContext::nativeReader($this->fs())->read($path)->rows()), - ); - } -} diff --git a/src/core/etl/tests/Flow/Floe/Tests/Integration/FloeStatisticsTest.php b/src/core/etl/tests/Flow/Floe/Tests/Integration/FloeStatisticsTest.php index a7c0cbeedb..72b503dd5b 100644 --- a/src/core/etl/tests/Flow/Floe/Tests/Integration/FloeStatisticsTest.php +++ b/src/core/etl/tests/Flow/Floe/Tests/Integration/FloeStatisticsTest.php @@ -4,13 +4,13 @@ namespace Flow\Floe\Tests\Integration; +use Flow\ETL\Column\AdaptiveBackend; use Flow\ETL\Tests\FlowIntegrationTestCase; use Flow\Floe\FloeWriter; use Flow\Floe\Tests\Context\FloeStreamReaderContext; +use function Flow\ETL\DSL\array_to_rows; use function Flow\ETL\DSL\int_schema; -use function Flow\ETL\DSL\row; -use function Flow\ETL\DSL\rows; use function Flow\ETL\DSL\schema; use function Flow\ETL\DSL\str_schema; use function Flow\Floe\DSL\merge_floe; @@ -23,15 +23,14 @@ public function test_a_compacted_merge_reports_the_same_rows_as_a_spliced_one(): $b = $this->cacheDir->suffix('compact-b.floe'); $spliced = $this->cacheDir->suffix('compact-spliced.floe'); $compacted = $this->cacheDir->suffix('compact-compacted.floe'); - FloeStreamReaderContext::write( - $this->fs(), - $a, - rows(schema(int_schema('id')), row(['id' => 1]), row(['id' => 2])), - ); - FloeStreamReaderContext::write($this->fs(), $b, rows(schema(int_schema('id')), row(['id' => 3]))); + FloeStreamReaderContext::write($this->fs(), $a, array_to_rows([ + ['id' => 1], + ['id' => 2], + ], schema(int_schema('id')))); + FloeStreamReaderContext::write($this->fs(), $b, array_to_rows([['id' => 3]], schema(int_schema('id')))); - merge_floe([$a, $b], $spliced); - merge_floe([$a, $b], $compacted, compact: true); + merge_floe([$a, $b], $spliced, new AdaptiveBackend()); + merge_floe([$a, $b], $compacted, new AdaptiveBackend(), compact: true); static::assertSame(3, FloeStreamReaderContext::footer($this->fs(), $spliced)->statistics->rows); static::assertSame(3, FloeStreamReaderContext::footer($this->fs(), $compacted)->statistics->rows); @@ -46,10 +45,14 @@ public function test_a_merged_file_reports_the_output_byte_size_not_the_inputs() $a = $this->cacheDir->suffix('splice-a.floe'); $b = $this->cacheDir->suffix('splice-b.floe'); $out = $this->cacheDir->suffix('splice-out.floe'); - FloeStreamReaderContext::write($this->fs(), $a, rows(schema(str_schema('name')), row(['name' => 'first']))); - FloeStreamReaderContext::write($this->fs(), $b, rows(schema(str_schema('name')), row(['name' => 'second']))); + FloeStreamReaderContext::write($this->fs(), $a, array_to_rows([[ + 'name' => 'first', + ]], schema(str_schema('name')))); + FloeStreamReaderContext::write($this->fs(), $b, array_to_rows([[ + 'name' => 'second', + ]], schema(str_schema('name')))); - merge_floe([$a, $b], $out); + merge_floe([$a, $b], $out, new AdaptiveBackend()); $statistics = FloeStreamReaderContext::footer($this->fs(), $out)->statistics; @@ -68,14 +71,14 @@ public function test_a_merged_file_reports_the_output_byte_size_not_the_inputs() public function test_a_multi_section_file_sums_its_sections(): void { $path = $this->cacheDir->suffix('sections.floe'); - FloeStreamReaderContext::write($this->fs(), $path, rows(schema(int_schema('id')), row(['id' => 1]))); + FloeStreamReaderContext::write($this->fs(), $path, array_to_rows([['id' => 1]], schema(int_schema('id')))); foreach ([[2, 3], [4, 5, 6]] as $ids) { - $writer = new FloeWriter($this->fs(), schema(int_schema('id'))); + $writer = new FloeWriter($this->fs(), schema(int_schema('id')), new AdaptiveBackend()); $writer->append($path); foreach ($ids as $id) { - $writer->write(rows(schema(int_schema('id')), row(['id' => $id]))); + $writer->write(array_to_rows([['id' => $id]], schema(int_schema('id')))); } $writer->close(); @@ -97,11 +100,10 @@ public function test_a_multi_section_file_sums_its_sections(): void public function test_a_written_file_reports_its_rows_and_byte_size(): void { $path = $this->cacheDir->suffix('written.floe'); - FloeStreamReaderContext::write( - $this->fs(), - $path, - rows(schema(int_schema('id')), row(['id' => 1]), row(['id' => 2])), - ); + FloeStreamReaderContext::write($this->fs(), $path, array_to_rows([ + ['id' => 1], + ['id' => 2], + ], schema(int_schema('id')))); $statistics = FloeStreamReaderContext::footer($this->fs(), $path)->statistics; @@ -113,12 +115,12 @@ public function test_a_written_file_reports_its_rows_and_byte_size(): void public function test_an_appended_file_reports_the_combined_totals(): void { $path = $this->cacheDir->suffix('appended.floe'); - FloeStreamReaderContext::write($this->fs(), $path, rows(schema(int_schema('id')), row(['id' => 1]))); + FloeStreamReaderContext::write($this->fs(), $path, array_to_rows([['id' => 1]], schema(int_schema('id')))); $before = FloeStreamReaderContext::footer($this->fs(), $path)->statistics->byteSize; - $writer = new FloeWriter($this->fs(), schema(int_schema('id'))); + $writer = new FloeWriter($this->fs(), schema(int_schema('id')), new AdaptiveBackend()); $writer->append($path); - $writer->write(rows(schema(int_schema('id')), row(['id' => 2]), row(['id' => 3]))); + $writer->write(array_to_rows([['id' => 2], ['id' => 3]], schema(int_schema('id')))); $writer->close(); $statistics = FloeStreamReaderContext::footer($this->fs(), $path)->statistics; diff --git a/src/core/etl/tests/Flow/Floe/Tests/Integration/FloeStreamReaderTest.php b/src/core/etl/tests/Flow/Floe/Tests/Integration/FloeStreamReaderTest.php index 118c2846ed..3f3813c64d 100644 --- a/src/core/etl/tests/Flow/Floe/Tests/Integration/FloeStreamReaderTest.php +++ b/src/core/etl/tests/Flow/Floe/Tests/Integration/FloeStreamReaderTest.php @@ -4,15 +4,15 @@ namespace Flow\Floe\Tests\Integration; +use Flow\ETL\Column\AdaptiveBackend; use Flow\ETL\Tests\FlowIntegrationTestCase; use Flow\Floe\FloeMerger; use Flow\Floe\FloeReader; use Flow\Floe\FloeWriter; use Flow\Floe\Tests\Mother\RowsMother; +use function Flow\ETL\DSL\array_to_rows; use function Flow\ETL\DSL\int_schema; -use function Flow\ETL\DSL\row; -use function Flow\ETL\DSL\rows; use function Flow\ETL\DSL\schema; use function Flow\ETL\DSL\str_schema; use function iterator_to_array; @@ -28,39 +28,42 @@ public function test_merged_multi_schema_file_reads_seamlessly(): void $fileTwo = $this->cacheDir->suffix('evolving-2.floe'); $fileThree = $this->cacheDir->suffix('evolving-3.floe'); - $rowsOne = rows(schema(int_schema('id')), row(['id' => 1])); - $writer = new FloeWriter($this->fs(), $rowsOne->schema()); + $rowsOne = array_to_rows([['id' => 1]], schema(int_schema('id'))); + $writer = new FloeWriter($this->fs(), $rowsOne->schema(), new AdaptiveBackend()); $writer->create($fileOne); $writer->write($rowsOne); $writer->close(); - $rowsTwo = rows( + $rowsTwo = array_to_rows( + [['id' => 2, 'email' => null]], schema(int_schema('id'), str_schema('email', nullable: true)), - row(['id' => 2, 'email' => null]), ); - $writer = new FloeWriter($this->fs(), $rowsTwo->schema()); + $writer = new FloeWriter($this->fs(), $rowsTwo->schema(), new AdaptiveBackend()); $writer->create($fileTwo); $writer->write($rowsTwo); $writer->close(); - $rowsThree = rows(schema(int_schema('id'), str_schema('email')), row(['id' => 3, 'email' => 'third@flow.php'])); - $writer = new FloeWriter($this->fs(), $rowsThree->schema()); + $rowsThree = array_to_rows( + [['id' => 3, 'email' => 'third@flow.php']], + schema(int_schema('id'), str_schema('email')), + ); + $writer = new FloeWriter($this->fs(), $rowsThree->schema(), new AdaptiveBackend()); $writer->create($fileThree); $writer->write($rowsThree); $writer->close(); - (new FloeMerger($this->fs()))->merge([$fileOne, $fileTwo, $fileThree], $path); + (new FloeMerger($this->fs(), new AdaptiveBackend()))->merge([$fileOne, $fileTwo, $fileThree], $path); - $reader = (new FloeReader($this->fs()))->read($path); + $reader = (new FloeReader($this->fs(), new AdaptiveBackend()))->read($path); static::assertSame(3, $reader->totalRows()); $read = []; foreach ($reader->rows() as $batch) { - foreach ($batch->all() as $row) { - static::assertSame(['id', 'email'], $row->names()); - $read[] = [$row->get('id'), $row->get('email')]; + foreach ($batch->toArray() as $row) { + static::assertSame(['id', 'email'], array_keys($row)); + $read[] = [$row['id'], $row['email']]; } } @@ -71,18 +74,18 @@ public function test_file_larger_than_compaction_threshold_round_trips(): void { $path = $this->cacheDir->suffix('large.floe'); - $schema = rows( + $schema = array_to_rows( + [['id' => 0, 'payload' => str_pad('row_0', 300, 'x')]], schema(int_schema('id'), str_schema('payload')), - row(['id' => 0, 'payload' => str_pad('row_0', 300, 'x')]), )->schema(); - $writer = new FloeWriter($this->fs(), $schema); + $writer = new FloeWriter($this->fs(), $schema, new AdaptiveBackend()); $writer->create($path); $written = 0; for ($i = 0; $i < 4000; $i++) { - $writer->write(rows( + $writer->write(array_to_rows( + [['id' => $i, 'payload' => str_pad('row_' . $i, 300, 'x')]], schema(int_schema('id'), str_schema('payload')), - row(['id' => $i, 'payload' => str_pad('row_' . $i, 300, 'x')]), )); $written++; } @@ -95,11 +98,11 @@ public function test_file_larger_than_compaction_threshold_round_trips(): void $read = 0; - foreach ((new FloeReader($this->fs(), chunkSize: 4096)) + foreach ((new FloeReader($this->fs(), new AdaptiveBackend(), chunkSize: 4096)) ->read($path) ->rows(500) as $batch) { - foreach ($batch->all() as $row) { - static::assertSame($read, $row->get('id')); + foreach ($batch->toArray() as $row) { + static::assertSame($read, $row['id']); $read++; } } @@ -112,19 +115,19 @@ public function test_round_trip_of_all_entry_types_through_local_filesystem(): v $rows = RowsMother::withAllEntryTypes(); $path = $this->cacheDir->suffix('all-types.floe'); - $writer = new FloeWriter($this->fs(), $rows->schema()); + $writer = new FloeWriter($this->fs(), $rows->schema(), new AdaptiveBackend()); $writer->create($path); $writer->write($rows); $writer->close(); $batches = iterator_to_array( - (new FloeReader($this->fs())) + (new FloeReader($this->fs(), new AdaptiveBackend())) ->read($path) ->rows(), ); static::assertCount(1, $batches); - static::assertEquals($rows->all(), $batches[0]->all()); + static::assertEquals($rows->toArray(), $batches[0]->toArray()); } public function test_round_trip_of_heterogeneous_rows_through_local_filesystem(): void @@ -132,14 +135,14 @@ public function test_round_trip_of_heterogeneous_rows_through_local_filesystem() $rows = RowsMother::heterogeneous(); $path = $this->cacheDir->suffix('heterogeneous.floe'); - $writer = new FloeWriter($this->fs(), $rows->schema()); + $writer = new FloeWriter($this->fs(), $rows->schema(), new AdaptiveBackend()); $writer->create($path); $writer->write($rows); $writer->close(); $read = 0; - foreach ((new FloeReader($this->fs())) + foreach ((new FloeReader($this->fs(), new AdaptiveBackend())) ->read($path) ->rows() as $batch) { $read += $batch->count(); diff --git a/src/core/etl/tests/Flow/Floe/Tests/Integration/FloeWriteContractTest.php b/src/core/etl/tests/Flow/Floe/Tests/Integration/FloeWriteContractTest.php index a55bef734a..9763c82b8c 100644 --- a/src/core/etl/tests/Flow/Floe/Tests/Integration/FloeWriteContractTest.php +++ b/src/core/etl/tests/Flow/Floe/Tests/Integration/FloeWriteContractTest.php @@ -4,126 +4,90 @@ namespace Flow\Floe\Tests\Integration; +use Flow\ETL\Column\AdaptiveBackend; use Flow\ETL\Rows; use Flow\ETL\Schema; use Flow\ETL\Tests\FlowIntegrationTestCase; use Flow\Floe\Exception\IncompatibleSchemaException; -use Flow\Floe\FloeEngine; use Flow\Floe\FloeWriter; -use Flow\Floe\NativeFloeEncoder; use Flow\Floe\Options; use Flow\Floe\Tests\Context\FloeStreamReaderContext; use PHPUnit\Framework\Attributes\DataProvider; -use Throwable; -use function array_key_exists; +use function Flow\ETL\DSL\array_to_rows; use function Flow\ETL\DSL\float_schema; use function Flow\ETL\DSL\int_schema; -use function Flow\ETL\DSL\row; -use function Flow\ETL\DSL\rows; use function Flow\ETL\DSL\schema; use function Flow\ETL\DSL\str_schema; final class FloeWriteContractTest extends FlowIntegrationTestCase { - public static function engines(): array - { - return ['php' => [FloeEngine::php], 'native' => [FloeEngine::native]]; - } - public static function rejected_values(): array { return [ 'string into int' => [ schema(int_schema('id')), - rows(schema(str_schema('id')), row(['id' => 'AB-1'])), + array_to_rows([['id' => 'AB-1']], schema(str_schema('id'))), 'expected: id, given: id', ], 'float into int' => [ schema(int_schema('id')), - rows(schema(float_schema('id')), row(['id' => 1.5])), + array_to_rows([['id' => 1.5]], schema(float_schema('id'))), 'expected: id, given: id', ], 'int into string' => [ schema(str_schema('name')), - rows(schema(int_schema('name')), row(['name' => 1000])), + array_to_rows([['name' => 1000]], schema(int_schema('name'))), 'expected: name, given: name', ], 'null into non nullable' => [ schema(str_schema('name')), - rows(schema(str_schema('name', nullable: true)), row(['name' => null])), + array_to_rows([['name' => null]], schema(str_schema('name', nullable: true))), 'expected: name, given: name', ], 'undeclared column' => [ schema(int_schema('id')), - rows(schema(int_schema('id'), int_schema('extra')), row(['id' => 1, 'extra' => 2])), + array_to_rows([['id' => 1, 'extra' => 2]], schema(int_schema('id'), int_schema('extra'))), 'new column "extra"', ], ]; } #[DataProvider('rejected_values')] - public function test_both_engines_reject_the_same_value_with_the_same_message( + public function test_the_writer_rejects_the_value_with_the_message( Schema $schema, Rows $batch, string $expectedMessage, ): void { - $messages = []; - - foreach ([FloeEngine::php, FloeEngine::native] as $engine) { - if ($engine === FloeEngine::native && !NativeFloeEncoder::isSupported()) { - continue; - } - - $name = $engine->value; - $path = $this->cacheDir->suffix('contract-' . $name . '-' . md5($expectedMessage) . '.floe'); - $writer = new FloeWriter($this->fs(), $schema, new Options(), null, $engine); - $writer->create($path); - - try { - $writer->write($batch); - static::fail($name . ' engine accepted a value it must reject'); - } catch (Throwable $e) { - $messages[$name] = $e::class . ': ' . $e->getMessage(); - } - - $writer->close(); - - static::assertSame( - [], - FloeStreamReaderContext::readAll($this->fs(), $path)->toArray(), - $name . ' engine emitted bytes for a rejected batch', - ); - } + $path = $this->cacheDir->suffix('contract-' . md5($expectedMessage) . '.floe'); + $writer = new FloeWriter($this->fs(), $schema, new AdaptiveBackend(), new Options()); + $writer->create($path); - foreach ($messages as $name => $message) { - static::assertStringContainsString($expectedMessage, $message, $name . ' engine message'); - static::assertStringContainsString( - IncompatibleSchemaException::class, - $message, - $name . ' exception class', - ); + try { + $writer->write($batch); + static::fail('the writer accepted a value it must reject'); + } catch (IncompatibleSchemaException $e) { + static::assertStringContainsString($expectedMessage, $e->getMessage()); } - if (array_key_exists('php', $messages) && array_key_exists('native', $messages)) { - static::assertSame($messages['php'], $messages['native'], 'engines disagree on the rejection'); - } + $writer->close(); + + static::assertSame( + [], + FloeStreamReaderContext::readAll($this->fs(), $path)->toArray(), + 'the writer emitted bytes for a rejected batch', + ); } - #[DataProvider('engines')] - public function test_a_rejected_batch_leaves_previously_written_rows_readable(FloeEngine $engine): void + public function test_a_rejected_batch_leaves_previously_written_rows_readable(): void { - if ($engine === FloeEngine::native && !NativeFloeEncoder::isSupported()) { - static::markTestSkipped('flow_php extension is not loaded.'); - } - - $path = $this->cacheDir->suffix('contract-survivor-' . $engine->value . '.floe'); - $writer = new FloeWriter($this->fs(), schema(int_schema('id')), new Options(), null, $engine); + $path = $this->cacheDir->suffix('contract-survivor.floe'); + $writer = new FloeWriter($this->fs(), schema(int_schema('id')), new AdaptiveBackend(), new Options()); $writer->create($path); - $writer->write(rows(schema(int_schema('id')), row(['id' => 1]), row(['id' => 2]))); + $writer->write(array_to_rows([['id' => 1], ['id' => 2]], schema(int_schema('id')))); try { - $writer->write(rows(schema(str_schema('id')), row(['id' => 'AB-1']))); + $writer->write(array_to_rows([['id' => 'AB-1']], schema(str_schema('id')))); static::fail('rejected batch was accepted'); } catch (IncompatibleSchemaException) { } @@ -133,32 +97,26 @@ public function test_a_rejected_batch_leaves_previously_written_rows_readable(Fl static::assertSame([['id' => 1], ['id' => 2]], FloeStreamReaderContext::readAll($this->fs(), $path)->toArray()); } - #[DataProvider('engines')] /** * SCHEMA EVOLUTION: the reverse - a first batch WITHOUT the column and a later one WITH it - * is rejected as a new column. Once append supports adding an optional column, that case * belongs here too. */ - public function test_a_batch_omitting_a_nullable_column_still_writes(FloeEngine $engine): void + public function test_a_batch_omitting_a_nullable_column_still_writes(): void { - if ($engine === FloeEngine::native && !NativeFloeEncoder::isSupported()) { - static::markTestSkipped('flow_php extension is not loaded.'); - } - - $path = $this->cacheDir->suffix('contract-absent-' . $engine->value . '.floe'); + $path = $this->cacheDir->suffix('contract-absent.floe'); $writer = new FloeWriter( $this->fs(), schema(int_schema('id'), str_schema('name', nullable: true)), + new AdaptiveBackend(), new Options(), - null, - $engine, ); $writer->create($path); - $writer->write(rows( + $writer->write(array_to_rows( + [['id' => 1, 'name' => 'a']], schema(int_schema('id'), str_schema('name', nullable: true)), - row(['id' => 1, 'name' => 'a']), )); - $writer->write(rows(schema(int_schema('id')), row(['id' => 2]))); + $writer->write(array_to_rows([['id' => 2]], schema(int_schema('id')))); $writer->close(); static::assertSame( @@ -171,27 +129,21 @@ public function test_a_batch_omitting_a_nullable_column_still_writes(FloeEngine * b57's writer half: the old check walked the row's own values, so a NOT NULL column the batch * simply omitted was never looked at and was encoded as VALUE_ABSENT. */ - #[DataProvider('engines')] - public function test_a_batch_omitting_a_not_null_column_is_refused(FloeEngine $engine): void + public function test_a_batch_omitting_a_not_null_column_is_refused(): void { - if ($engine === FloeEngine::native && !NativeFloeEncoder::isSupported()) { - static::markTestSkipped('flow_php extension is not loaded.'); - } - - $path = $this->cacheDir->suffix('contract-absent-not-null-' . $engine->value . '.floe'); + $path = $this->cacheDir->suffix('contract-absent-not-null.floe'); $writer = new FloeWriter( $this->fs(), schema(int_schema('id'), str_schema('name')), + new AdaptiveBackend(), new Options(), - null, - $engine, ); $writer->create($path); - $writer->write(rows(schema(int_schema('id'), str_schema('name')), row(['id' => 1, 'name' => 'a']))); + $writer->write(array_to_rows([['id' => 1, 'name' => 'a']], schema(int_schema('id'), str_schema('name')))); $this->expectException(IncompatibleSchemaException::class); $this->expectExceptionMessage('Missing Definitions'); - $writer->write(rows(schema(int_schema('id')), row(['id' => 2]))); + $writer->write(array_to_rows([['id' => 2]], schema(int_schema('id')))); } } diff --git a/src/core/etl/tests/Flow/Floe/Tests/Integration/FloeWriterExtensionParityTest.php b/src/core/etl/tests/Flow/Floe/Tests/Integration/FloeWriterExtensionParityTest.php deleted file mode 100644 index 680ffe765b..0000000000 --- a/src/core/etl/tests/Flow/Floe/Tests/Integration/FloeWriterExtensionParityTest.php +++ /dev/null @@ -1,165 +0,0 @@ - [RowsMother::withAllEntryTypes()], - 'heterogeneous' => [RowsMother::heterogeneous()], - 'empty' => [rows(schema())], - ]; - } - - #[Override] - protected function setUp(): void - { - parent::setUp(); - - if (!NativeFloeEncoder::isSupported()) { - self::markTestSkipped('flow_php extension is not loaded.'); - } - } - - public function test_extension_and_pure_php_reject_datetime_subclass_identically(): void - { - $rows = rows(schema(datetime_schema('at')), row(['at' => new CustomDateTime('2025-01-01 00:00:00 UTC')])); - $schema = $rows->schema(); - - $writers = [ - 'php' => FloeEngineContext::phpWriter($this->fs(), $schema), - 'ext' => FloeEngineContext::nativeWriter($this->fs(), $schema), - ]; - - foreach ($writers as $engine => $writer) { - $writer->create($this->cacheDir->suffix('subclass-' . $engine . '.floe')); - - try { - $writer->write($rows); - static::fail('Expected a FloeException for a custom datetime subclass'); - } catch (FloeException $e) { - static::assertStringContainsString('DateTime and DateTimeImmutable', $e->getMessage()); - } - } - } - - #[DataProvider('rows_datasets')] - public function test_extension_and_pure_php_write_byte_identical_files(Rows $rows): void - { - $purePath = $this->cacheDir->suffix('write-pure.floe'); - $extPath = $this->cacheDir->suffix('write-ext.floe'); - - $schema = $rows->schema(); - - FloeEngineContext::writeAll(FloeEngineContext::phpWriter($this->fs(), $schema), $purePath, [$rows]); - FloeEngineContext::writeAll(FloeEngineContext::nativeWriter($this->fs(), $schema), $extPath, [$rows]); - - static::assertSame($this->fs()->readFrom($purePath)->content(), $this->fs()->readFrom($extPath)->content()); - } - - public function test_extension_and_pure_php_write_byte_identical_files_across_multiple_writes(): void - { - $batches = [ - rows( - schema(int_schema('id'), str_schema('name')), - row(['id' => 1, 'name' => 'a']), - row(['id' => 2, 'name' => 'b']), - ), - rows(schema(int_schema('id'), str_schema('name')), row(['id' => 3, 'name' => 'c'])), - rows( - schema(int_schema('id'), str_schema('name')), - row(['id' => 4, 'name' => 'd']), - row(['id' => 5, 'name' => 'e']), - ), - ]; - - $purePath = $this->cacheDir->suffix('multi-write-pure.floe'); - $extPath = $this->cacheDir->suffix('multi-write-ext.floe'); - - $schema = schema(int_schema('id'), str_schema('name')); - - FloeEngineContext::writeAll(FloeEngineContext::phpWriter($this->fs(), $schema), $purePath, $batches); - FloeEngineContext::writeAll(FloeEngineContext::nativeWriter($this->fs(), $schema), $extPath, $batches); - - static::assertSame($this->fs()->readFrom($purePath)->content(), $this->fs()->readFrom($extPath)->content()); - } - - /** - * A transforming (non-identity) codec must not fork the engine: the native - * body-encode produces the same ROW bodies as the PHP engine, the codec runs - * in PHP either way, so the files stay byte-identical (the A1 fix). - */ - #[DataProvider('rows_datasets')] - public function test_extension_and_pure_php_write_byte_identical_files_with_transforming_codec(Rows $rows): void - { - $codec = new PrefixingCodecStub(); - - $purePath = $this->cacheDir->suffix('write-codec-pure.floe'); - $extPath = $this->cacheDir->suffix('write-codec-ext.floe'); - - $schema = $rows->schema(); - - FloeEngineContext::writeAll(FloeEngineContext::phpWriter($this->fs(), $schema, $codec), $purePath, [$rows]); - FloeEngineContext::writeAll(FloeEngineContext::nativeWriter($this->fs(), $schema, $codec), $extPath, [$rows]); - - static::assertSame($this->fs()->readFrom($purePath)->content(), $this->fs()->readFrom($extPath)->content()); - } - - public function test_extension_and_pure_php_append_byte_identically(): void - { - $purePath = $this->cacheDir->suffix('append-pure.floe'); - $extPath = $this->cacheDir->suffix('append-ext.floe'); - - $createRows = rows( - schema(int_schema('id'), str_schema('email', nullable: true)), - row(['id' => 1, 'email' => null]), - ); - $appendRows = rows(schema(int_schema('id'), str_schema('email')), row(['id' => 2, 'email' => 'x'])); - $schema = $createRows->schema()->merge($appendRows->schema()); - - foreach ([[$purePath, false], [$extPath, true]] as [$path, $native]) { - $create = $native - ? FloeEngineContext::nativeWriter($this->fs(), $schema) - : FloeEngineContext::phpWriter($this->fs(), $schema); - $create->create($path); - $create->write($createRows); - $create->close(); - - $append = $native - ? FloeEngineContext::nativeWriter($this->fs(), $schema) - : FloeEngineContext::phpWriter($this->fs(), $schema); - $append->append($path); - $append->write($appendRows); - $append->close(); - } - - static::assertSame($this->fs()->readFrom($purePath)->content(), $this->fs()->readFrom($extPath)->content()); - } -} diff --git a/src/core/etl/tests/Flow/Floe/Tests/Mother/DateTimeDecoderMother.php b/src/core/etl/tests/Flow/Floe/Tests/Mother/DateTimeDecoderMother.php deleted file mode 100644 index 48864658b8..0000000000 --- a/src/core/etl/tests/Flow/Floe/Tests/Mother/DateTimeDecoderMother.php +++ /dev/null @@ -1,22 +0,0 @@ - 1, 'name' => 'ab'], ['id' => 2, 'name' => null]], + schema(int_schema('id'), str_schema('name', nullable: true)), + ); + } + public static function ids(int $from, int $to): Rows { - return rows( + return array_to_rows( + array_map(static fn(int $id) => ['id' => $id], range($from, $to)), schema(int_schema('id')), - ...array_map(static fn(int $id) => row(['id' => $id]), range($from, $to)), ); } @@ -67,27 +75,24 @@ public static function ids(int $from, int $to): Rows */ public static function numbered(int $count): Rows { - return rows( - schema(int_schema('id'), str_schema('name', nullable: true), datetime_schema('at')), - ...array_map( - static fn(int $id) => row([ + return array_to_rows( + array_map( + static fn(int $id) => [ 'id' => $id, 'name' => ($id % 4) === 0 ? null : 'user_' . $id, 'at' => (new DateTimeImmutable('2026-01-01 00:00:00.000001 +00:00'))->modify("+{$id} hours"), - ]), + ], range(1, $count), ), + schema(int_schema('id'), str_schema('name', nullable: true), datetime_schema('at')), ); } public static function heterogeneous(): Rows { - return rows( + return array_to_rows( + [['a' => 1, 'b' => 'x'], ['a' => 2, 'b' => 'y'], ['a' => 3, 'c' => 1.5], ['b' => 'z', 'a' => 4]], schema(int_schema('a'), str_schema('b', nullable: true), float_schema('c', nullable: true)), - row(['a' => 1, 'b' => 'x']), - row(['a' => 2, 'b' => 'y']), - row(['a' => 3, 'c' => 1.5]), - row(['b' => 'z', 'a' => 4]), ); } @@ -106,36 +111,8 @@ public static function withAllEntryTypes(): Rows 'b' => type_string(), ]); - return rows( - schema( - int_schema('int'), - int_schema('int_min'), - int_schema('int_max'), - float_schema('float'), - bool_schema('bool'), - str_schema('string'), - str_schema('string_binary'), - str_schema('string_unicode'), - str_schema('string_null', nullable: true), - null_schema('string_from_null'), - datetime_schema('datetime_immutable', zone: 'Europe/Warsaw'), - datetime_schema('datetime_mutable', zone: 'America/New_York'), - datetime_schema('datetime_before_epoch'), - date_schema('date'), - time_schema('time'), - uuid_schema('uuid'), - json_schema('json'), - json_schema('json_object'), - enum_schema('enum_backed', BackedStringEnum::class), - enum_schema('enum_basic', BasicEnum::class), - list_schema('list', type_list(type_integer())), - map_schema('map', type_map(type_string(), type_integer())), - structure_schema('structure', $structure), - structure_schema('structure_interleaved', $interleaved), - xml_schema('xml'), - xml_element_schema('xml_element'), - ), - row([ + return array_to_rows( + [[ 'int' => 42, 'int_min' => PHP_INT_MIN, 'int_max' => PHP_INT_MAX, @@ -152,7 +129,7 @@ enum_schema('enum_basic', BasicEnum::class), ), 'datetime_mutable' => new DateTime('2025-06-15 12:30:45.654321', new DateTimeZone('America/New_York')), 'datetime_before_epoch' => new DateTimeImmutable('1969-07-20 20:17:00.500000 UTC'), - 'date' => new DateTimeImmutable('2025-06-15 00:00:00', new DateTimeZone('Europe/Warsaw')), + 'date' => new DateTimeImmutable('2025-06-15 00:00:00', new DateTimeZone('UTC')), 'time' => new DateInterval('PT2H30M15S'), 'uuid' => type_uuid()->cast('0196aecb-b568-7e57-a381-8ec8d3e4a531'), 'json' => type_json()->cast('[1,2,3]'), @@ -165,7 +142,35 @@ enum_schema('enum_basic', BasicEnum::class), 'structure_interleaved' => $interleaved->cast(['z' => 1, 'b' => 'x']), 'xml' => type_xml()->cast('text & entity'), 'xml_element' => type_xml_element()->cast('value'), - ]), + ]], + schema( + int_schema('int'), + int_schema('int_min'), + int_schema('int_max'), + float_schema('float'), + bool_schema('bool'), + str_schema('string'), + str_schema('string_binary'), + str_schema('string_unicode'), + str_schema('string_null', nullable: true), + null_schema('string_from_null'), + datetime_schema('datetime_immutable', zone: 'Europe/Warsaw'), + datetime_schema('datetime_mutable', zone: 'America/New_York'), + datetime_schema('datetime_before_epoch'), + date_schema('date'), + time_schema('time'), + uuid_schema('uuid'), + json_schema('json'), + json_schema('json_object'), + enum_schema('enum_backed', BackedStringEnum::class), + enum_schema('enum_basic', BasicEnum::class), + list_schema('list', type_list(type_integer())), + map_schema('map', type_map(type_string(), type_integer())), + structure_schema('structure', $structure), + structure_schema('structure_interleaved', $interleaved), + xml_schema('xml'), + xml_element_schema('xml_element'), + ), ); } } diff --git a/src/core/etl/tests/Flow/Floe/Tests/Unit/Decoding/BooleanDecoderTest.php b/src/core/etl/tests/Flow/Floe/Tests/Unit/Decoding/BooleanDecoderTest.php deleted file mode 100644 index a324590d1f..0000000000 --- a/src/core/etl/tests/Flow/Floe/Tests/Unit/Decoding/BooleanDecoderTest.php +++ /dev/null @@ -1,21 +0,0 @@ -decode("\x01\x00", $position)); - static::assertFalse($decoder->decode("\x01\x00", $position)); - static::assertSame(2, $position); - } -} diff --git a/src/core/etl/tests/Flow/Floe/Tests/Unit/Decoding/DateTimeDecoderTest.php b/src/core/etl/tests/Flow/Floe/Tests/Unit/Decoding/DateTimeDecoderTest.php deleted file mode 100644 index 3355942a60..0000000000 --- a/src/core/etl/tests/Flow/Floe/Tests/Unit/Decoding/DateTimeDecoderTest.php +++ /dev/null @@ -1,104 +0,0 @@ -encode($value); - $position = 0; - - $decoded = DateTimeDecoderMother::create()->decode($encoded, $position); - - static::assertInstanceOf(DateTimeImmutable::class, $decoded); - static::assertEquals($value, $decoded); - static::assertSame('Australia/Eucla', $decoded->getTimezone()->getName()); - static::assertSame('987654', $decoded->format('u')); - static::assertSame(strlen($encoded), $position); - } - - public function test_a_mutable_datetime_decodes_as_immutable(): void - { - $value = new DateTime('2025-01-01 00:00:00 UTC'); - $position = 0; - - $decoded = DateTimeDecoderMother::create()->decode((new DateTimeEncoder())->encode($value), $position); - - static::assertInstanceOf(DateTimeImmutable::class, $decoded); - static::assertSame($value->format('Y-m-d H:i:s.u e'), $decoded->format('Y-m-d H:i:s.u e')); - } - - public function test_round_trip_before_epoch(): void - { - $value = new DateTimeImmutable('1969-07-20 20:17:00.500000 UTC'); - $position = 0; - - static::assertEquals($value, DateTimeDecoderMother::create()->decode( - (new DateTimeEncoder())->encode($value), - $position, - )); - } - - public function test_a_truncated_datetime_value_throws(): void - { - $position = 0; - - $this->expectException(FloeException::class); - $this->expectExceptionMessage('Floe found a truncated datetime value'); - - DateTimeDecoderMother::create()->decode("\xEE", $position); - } - - public function test_column_zone_decoder_advances_the_position(): void - { - $value = new DateTimeImmutable('2025-06-15 12:30:45', new DateTimeZone('Australia/Eucla')); - $encoded = (new DateTimeEncoder())->encode($value) . (new DateTimeEncoder())->encode($value); - $position = 0; - - DateTimeDecoderMother::inColumnZone('Europe/Warsaw')->decode($encoded, $position); - - static::assertSame(strlen($encoded) / 2, $position); - } - - public function test_column_zone_decoder_ignores_an_unknown_stored_zone(): void - { - $position = 0; - - static::assertSame( - '2025-06-15 12:30:45.000000 Europe/Warsaw', - DateTimeDecoderMother::inColumnZone('Europe/Warsaw') - ->decode(pack('P', 1749983445) . pack('V', 0) . pack('V', 13) . 'Europe/Xarsaw', $position) - ->format('Y-m-d H:i:s.u e'), - ); - } - - public function test_decodes_into_the_column_zone(): void - { - $value = new DateTimeImmutable('2025-06-15 12:30:45.987654', new DateTimeZone('Australia/Eucla')); - $position = 0; - - $decoded = DateTimeDecoderMother::inColumnZone('Europe/Warsaw')->decode( - (new DateTimeEncoder())->encode($value), - $position, - ); - - static::assertSame('Europe/Warsaw', $decoded->getTimezone()->getName()); - static::assertSame($value->getTimestamp(), $decoded->getTimestamp()); - static::assertSame('987654', $decoded->format('u')); - } -} diff --git a/src/core/etl/tests/Flow/Floe/Tests/Unit/Decoding/EnumDecoderTest.php b/src/core/etl/tests/Flow/Floe/Tests/Unit/Decoding/EnumDecoderTest.php deleted file mode 100644 index 29f645da80..0000000000 --- a/src/core/etl/tests/Flow/Floe/Tests/Unit/Decoding/EnumDecoderTest.php +++ /dev/null @@ -1,50 +0,0 @@ -decode( - (new EnumEncoder())->encode(BackedStringEnum::one), - $position, - )); - } - - public function test_unknown_case_throws(): void - { - $class = BackedStringEnum::class; - $encoded = pack('V', strlen($class)) . $class . pack('V', 4) . 'nope'; - $position = 0; - - $this->expectException(FloeException::class); - $this->expectExceptionMessage('cannot restore enum case'); - - (new EnumDecoder())->decode($encoded, $position); - } - - public function test_unknown_class_throws(): void - { - $encoded = pack('V', 10) . 'NoSuchEnum' . pack('V', 3) . 'one'; - $position = 0; - - $this->expectException(FloeException::class); - $this->expectExceptionMessage('enum not found'); - - (new EnumDecoder())->decode($encoded, $position); - } -} diff --git a/src/core/etl/tests/Flow/Floe/Tests/Unit/Decoding/Float64DecoderTest.php b/src/core/etl/tests/Flow/Floe/Tests/Unit/Decoding/Float64DecoderTest.php deleted file mode 100644 index 947ac6de25..0000000000 --- a/src/core/etl/tests/Flow/Floe/Tests/Unit/Decoding/Float64DecoderTest.php +++ /dev/null @@ -1,21 +0,0 @@ -decode(pack('e', 3.5), $position)); - static::assertSame(8, $position); - } -} diff --git a/src/core/etl/tests/Flow/Floe/Tests/Unit/Decoding/HtmlDocumentDecoderTest.php b/src/core/etl/tests/Flow/Floe/Tests/Unit/Decoding/HtmlDocumentDecoderTest.php deleted file mode 100644 index d65240d33a..0000000000 --- a/src/core/etl/tests/Flow/Floe/Tests/Unit/Decoding/HtmlDocumentDecoderTest.php +++ /dev/null @@ -1,37 +0,0 @@ -x

'); - static::assertInstanceOf(HTMLDocument::class, $document); - - $encoder = new HtmlDocumentEncoder(); - $encoded = $encoder->encode($document); - $position = 0; - - $decoded = (new HtmlDocumentDecoder())->decode($encoded, $position); - static::assertInstanceOf(HTMLDocument::class, $decoded); - - static::assertSame($encoded, $encoder->encode($decoded)); - static::assertSame(strlen($encoded), $position); - } -} diff --git a/src/core/etl/tests/Flow/Floe/Tests/Unit/Decoding/HtmlElementDecoderTest.php b/src/core/etl/tests/Flow/Floe/Tests/Unit/Decoding/HtmlElementDecoderTest.php deleted file mode 100644 index 2b8451545d..0000000000 --- a/src/core/etl/tests/Flow/Floe/Tests/Unit/Decoding/HtmlElementDecoderTest.php +++ /dev/null @@ -1,34 +0,0 @@ -x

'); - static::assertInstanceOf(HTMLElement::class, $element); - - $position = 0; - - $decoded = (new HtmlElementDecoder())->decode((new HtmlElementEncoder())->encode($element), $position); - static::assertInstanceOf(HTMLElement::class, $decoded); - - static::assertSame(ValueEncoder::htmlElementToString($element), ValueEncoder::htmlElementToString($decoded)); - } -} diff --git a/src/core/etl/tests/Flow/Floe/Tests/Unit/Decoding/Int64DecoderTest.php b/src/core/etl/tests/Flow/Floe/Tests/Unit/Decoding/Int64DecoderTest.php deleted file mode 100644 index d69cfc92ab..0000000000 --- a/src/core/etl/tests/Flow/Floe/Tests/Unit/Decoding/Int64DecoderTest.php +++ /dev/null @@ -1,28 +0,0 @@ -decode(pack('P', $value), $position)); - static::assertSame(8, $position); - } - } -} diff --git a/src/core/etl/tests/Flow/Floe/Tests/Unit/Decoding/IntervalDecoderTest.php b/src/core/etl/tests/Flow/Floe/Tests/Unit/Decoding/IntervalDecoderTest.php deleted file mode 100644 index 9daad8038b..0000000000 --- a/src/core/etl/tests/Flow/Floe/Tests/Unit/Decoding/IntervalDecoderTest.php +++ /dev/null @@ -1,34 +0,0 @@ -invert = 1; - // @mago-ignore analysis:invalid-property-write - $value->f = 0.5; - - $position = 0; - $decoded = (new IntervalDecoder())->decode((new IntervalEncoder())->encode($value), $position); - - static::assertSame([1, 2, 3, 4, 0.5, 1], [ - $decoded->d, - $decoded->h, - $decoded->i, - $decoded->s, - $decoded->f, - $decoded->invert, - ]); - } -} diff --git a/src/core/etl/tests/Flow/Floe/Tests/Unit/Decoding/JsonDecoderTest.php b/src/core/etl/tests/Flow/Floe/Tests/Unit/Decoding/JsonDecoderTest.php deleted file mode 100644 index ad9c3f63f1..0000000000 --- a/src/core/etl/tests/Flow/Floe/Tests/Unit/Decoding/JsonDecoderTest.php +++ /dev/null @@ -1,26 +0,0 @@ -decode((new JsonEncoder())->encode($value), $position); - - static::assertSame($value->toString(), $decoded->toString()); - static::assertSame($value->isObject(), $decoded->isObject()); - } - } -} diff --git a/src/core/etl/tests/Flow/Floe/Tests/Unit/Decoding/ListDecoderTest.php b/src/core/etl/tests/Flow/Floe/Tests/Unit/Decoding/ListDecoderTest.php deleted file mode 100644 index e06d8b8501..0000000000 --- a/src/core/etl/tests/Flow/Floe/Tests/Unit/Decoding/ListDecoderTest.php +++ /dev/null @@ -1,25 +0,0 @@ -decode((new ListEncoder(new StringEncoder()))->encode(['a', 'bc']), $position), - ); - } -} diff --git a/src/core/etl/tests/Flow/Floe/Tests/Unit/Decoding/MapDecoderTest.php b/src/core/etl/tests/Flow/Floe/Tests/Unit/Decoding/MapDecoderTest.php deleted file mode 100644 index d192171d5e..0000000000 --- a/src/core/etl/tests/Flow/Floe/Tests/Unit/Decoding/MapDecoderTest.php +++ /dev/null @@ -1,42 +0,0 @@ - 1, 'b' => 2], - $decoder->decode((new MapEncoder(new StringKeyEncoder(), new Int64Encoder()))->encode([ - 'a' => 1, - 'b' => 2, - ]), $position), - ); - } - - public function test_round_trip_of_integer_keyed_map(): void - { - $decoder = new MapDecoder(new Int64Decoder(), new StringDecoder()); - $position = 0; - - static::assertSame( - [7 => 'x'], - $decoder->decode((new MapEncoder(new Int64Encoder(), new StringEncoder()))->encode([7 => 'x']), $position), - ); - } -} diff --git a/src/core/etl/tests/Flow/Floe/Tests/Unit/Decoding/NullDecoderTest.php b/src/core/etl/tests/Flow/Floe/Tests/Unit/Decoding/NullDecoderTest.php deleted file mode 100644 index 2ce8d77948..0000000000 --- a/src/core/etl/tests/Flow/Floe/Tests/Unit/Decoding/NullDecoderTest.php +++ /dev/null @@ -1,19 +0,0 @@ -decode('', $position)); - static::assertSame(0, $position); - } -} diff --git a/src/core/etl/tests/Flow/Floe/Tests/Unit/Decoding/OptionalDecoderTest.php b/src/core/etl/tests/Flow/Floe/Tests/Unit/Decoding/OptionalDecoderTest.php deleted file mode 100644 index 66e7405167..0000000000 --- a/src/core/etl/tests/Flow/Floe/Tests/Unit/Decoding/OptionalDecoderTest.php +++ /dev/null @@ -1,26 +0,0 @@ -decode($encoder->encode($value), $position)); - } - } -} diff --git a/src/core/etl/tests/Flow/Floe/Tests/Unit/Decoding/PackedListDecoderTest.php b/src/core/etl/tests/Flow/Floe/Tests/Unit/Decoding/PackedListDecoderTest.php deleted file mode 100644 index bd1ffcaad2..0000000000 --- a/src/core/etl/tests/Flow/Floe/Tests/Unit/Decoding/PackedListDecoderTest.php +++ /dev/null @@ -1,43 +0,0 @@ -decode((new PackedListEncoder('P'))->encode([10, -20, 30]), $position), - ); - } - - public function test_round_trip_of_float64_list(): void - { - $position = 0; - - static::assertSame( - [1.5, -2.5], - (new PackedListDecoder('e'))->decode((new PackedListEncoder('e'))->encode([1.5, -2.5]), $position), - ); - } - - public function test_empty_list(): void - { - $position = 0; - - static::assertSame( - [], - (new PackedListDecoder('P'))->decode((new PackedListEncoder('P'))->encode([]), $position), - ); - static::assertSame(4, $position); - } -} diff --git a/src/core/etl/tests/Flow/Floe/Tests/Unit/Decoding/StringDecoderTest.php b/src/core/etl/tests/Flow/Floe/Tests/Unit/Decoding/StringDecoderTest.php deleted file mode 100644 index a8cf3c0b22..0000000000 --- a/src/core/etl/tests/Flow/Floe/Tests/Unit/Decoding/StringDecoderTest.php +++ /dev/null @@ -1,26 +0,0 @@ -decode(pack('V', 3) . 'abc', $position)); - static::assertSame(7, $position); - - $position = 0; - static::assertSame('', $decoder->decode(pack('V', 0), $position)); - static::assertSame(4, $position); - } -} diff --git a/src/core/etl/tests/Flow/Floe/Tests/Unit/Decoding/StructureDecoderTest.php b/src/core/etl/tests/Flow/Floe/Tests/Unit/Decoding/StructureDecoderTest.php deleted file mode 100644 index c0005d5ee7..0000000000 --- a/src/core/etl/tests/Flow/Floe/Tests/Unit/Decoding/StructureDecoderTest.php +++ /dev/null @@ -1,49 +0,0 @@ - new Int64Encoder(), - 'name' => new StringEncoder(), - 'missing' => new Int64Encoder(), - ]); - $decoder = new StructureDecoder([ - 'id' => new Int64Decoder(), - 'name' => new StringDecoder(), - 'missing' => new Int64Decoder(), - ]); - - $position = 0; - - static::assertSame( - ['id' => 1, 'name' => null], - $decoder->decode($encoder->encode(['id' => 1, 'name' => null]), $position), - ); - } - - public function test_unknown_element_flag_throws(): void - { - $decoder = new StructureDecoder(['name' => new StringDecoder()]); - $position = 0; - - $this->expectException(FloeException::class); - $this->expectExceptionMessage('unknown structure element flag'); - - $decoder->decode("\xEE", $position); - } -} diff --git a/src/core/etl/tests/Flow/Floe/Tests/Unit/Decoding/TimeZoneDecoderTest.php b/src/core/etl/tests/Flow/Floe/Tests/Unit/Decoding/TimeZoneDecoderTest.php deleted file mode 100644 index 96b264e4a1..0000000000 --- a/src/core/etl/tests/Flow/Floe/Tests/Unit/Decoding/TimeZoneDecoderTest.php +++ /dev/null @@ -1,26 +0,0 @@ -decode(pack('V', 15) . 'Australia/Eucla', $position)->getName(), - ); - static::assertSame(19, $position); - } -} diff --git a/src/core/etl/tests/Flow/Floe/Tests/Unit/Decoding/UuidDecoderTest.php b/src/core/etl/tests/Flow/Floe/Tests/Unit/Decoding/UuidDecoderTest.php deleted file mode 100644 index de4d30a91a..0000000000 --- a/src/core/etl/tests/Flow/Floe/Tests/Unit/Decoding/UuidDecoderTest.php +++ /dev/null @@ -1,28 +0,0 @@ -decode(pack('V', strlen($uuid)) . $uuid, $position); - - static::assertInstanceOf(Uuid::class, $decoded); - static::assertSame($uuid, $decoded->toString()); - static::assertSame(4 + strlen($uuid), $position); - } -} diff --git a/src/core/etl/tests/Flow/Floe/Tests/Unit/Decoding/XmlDocumentDecoderTest.php b/src/core/etl/tests/Flow/Floe/Tests/Unit/Decoding/XmlDocumentDecoderTest.php deleted file mode 100644 index 59fa09db07..0000000000 --- a/src/core/etl/tests/Flow/Floe/Tests/Unit/Decoding/XmlDocumentDecoderTest.php +++ /dev/null @@ -1,37 +0,0 @@ -loadXML('value'); - - $position = 0; - $decoded = (new XmlDocumentDecoder())->decode((new XmlDocumentEncoder())->encode($document), $position); - - static::assertSame('item', $decoded->documentElement?->tagName); - } - - public function test_invalid_xml_throws(): void - { - $position = 0; - - $this->expectException(FloeException::class); - $this->expectExceptionMessage('failed to restore DOMDocument'); - - (new XmlDocumentDecoder())->decode(pack('V', 9) . 'not < xml', $position); - } -} diff --git a/src/core/etl/tests/Flow/Floe/Tests/Unit/Decoding/XmlElementDecoderTest.php b/src/core/etl/tests/Flow/Floe/Tests/Unit/Decoding/XmlElementDecoderTest.php deleted file mode 100644 index 1b34b528da..0000000000 --- a/src/core/etl/tests/Flow/Floe/Tests/Unit/Decoding/XmlElementDecoderTest.php +++ /dev/null @@ -1,28 +0,0 @@ -loadXML('value'); - $element = $document->documentElement; - static::assertInstanceOf(DOMElement::class, $element); - - $position = 0; - $decoded = (new XmlElementDecoder())->decode((new XmlElementEncoder())->encode($element), $position); - - static::assertSame('item', $decoded->tagName); - static::assertSame('5', $decoded->getAttribute('id')); - } -} diff --git a/src/core/etl/tests/Flow/Floe/Tests/Unit/Encoding/BooleanEncoderTest.php b/src/core/etl/tests/Flow/Floe/Tests/Unit/Encoding/BooleanEncoderTest.php deleted file mode 100644 index 81adc6792a..0000000000 --- a/src/core/etl/tests/Flow/Floe/Tests/Unit/Encoding/BooleanEncoderTest.php +++ /dev/null @@ -1,19 +0,0 @@ -encode(true)); - static::assertSame("\x00", $encoder->encode(false)); - } -} diff --git a/src/core/etl/tests/Flow/Floe/Tests/Unit/Encoding/DateTimeEncoderTest.php b/src/core/etl/tests/Flow/Floe/Tests/Unit/Encoding/DateTimeEncoderTest.php deleted file mode 100644 index 86341512f4..0000000000 --- a/src/core/etl/tests/Flow/Floe/Tests/Unit/Encoding/DateTimeEncoderTest.php +++ /dev/null @@ -1,44 +0,0 @@ -encode($value), $encoder->encode(DateTime::createFromImmutable($value))); - } - - public function test_encodes_timestamp_microseconds_and_timezone(): void - { - $value = new DateTimeImmutable('2025-01-01 00:00:00.123456 UTC'); - - static::assertSame( - pack('P', $value->getTimestamp()) . pack('V', 123456) . pack('V', 3) . 'UTC', - (new DateTimeEncoder())->encode($value), - ); - } - - public function test_custom_datetime_class_throws(): void - { - $this->expectException(FloeException::class); - $this->expectExceptionMessage('Floe supports only DateTime and DateTimeImmutable, got ' - . CustomDateTime::class); - - (new DateTimeEncoder())->encode(new CustomDateTime('2025-01-01 00:00:00 UTC')); - } -} diff --git a/src/core/etl/tests/Flow/Floe/Tests/Unit/Encoding/EnumEncoderTest.php b/src/core/etl/tests/Flow/Floe/Tests/Unit/Encoding/EnumEncoderTest.php deleted file mode 100644 index c7c8b9d4e9..0000000000 --- a/src/core/etl/tests/Flow/Floe/Tests/Unit/Encoding/EnumEncoderTest.php +++ /dev/null @@ -1,25 +0,0 @@ -encode(BackedStringEnum::one), - ); - } -} diff --git a/src/core/etl/tests/Flow/Floe/Tests/Unit/Encoding/Float64EncoderTest.php b/src/core/etl/tests/Flow/Floe/Tests/Unit/Encoding/Float64EncoderTest.php deleted file mode 100644 index 2ce9f18af0..0000000000 --- a/src/core/etl/tests/Flow/Floe/Tests/Unit/Encoding/Float64EncoderTest.php +++ /dev/null @@ -1,21 +0,0 @@ -encode(3.5)); - static::assertSame(pack('e', -0.25), $encoder->encode(-0.25)); - } -} diff --git a/src/core/etl/tests/Flow/Floe/Tests/Unit/Encoding/HtmlDocumentEncoderTest.php b/src/core/etl/tests/Flow/Floe/Tests/Unit/Encoding/HtmlDocumentEncoderTest.php deleted file mode 100644 index 08b7ca0f3a..0000000000 --- a/src/core/etl/tests/Flow/Floe/Tests/Unit/Encoding/HtmlDocumentEncoderTest.php +++ /dev/null @@ -1,34 +0,0 @@ -x

'); - static::assertInstanceOf(HTMLDocument::class, $document); - - $encoded = (new HtmlDocumentEncoder())->encode($document); - - static::assertSame(pack('V', strlen($encoded) - 4), substr($encoded, 0, 4)); - static::assertTrue(str_contains($encoded, '

x

')); - } -} diff --git a/src/core/etl/tests/Flow/Floe/Tests/Unit/Encoding/HtmlElementEncoderTest.php b/src/core/etl/tests/Flow/Floe/Tests/Unit/Encoding/HtmlElementEncoderTest.php deleted file mode 100644 index 179275acac..0000000000 --- a/src/core/etl/tests/Flow/Floe/Tests/Unit/Encoding/HtmlElementEncoderTest.php +++ /dev/null @@ -1,31 +0,0 @@ -x

'); - static::assertInstanceOf(HTMLElement::class, $element); - $html = ValueEncoder::htmlElementToString($element); - - static::assertSame(pack('V', strlen($html)) . $html, (new HtmlElementEncoder())->encode($element)); - } -} diff --git a/src/core/etl/tests/Flow/Floe/Tests/Unit/Encoding/Int64EncoderTest.php b/src/core/etl/tests/Flow/Floe/Tests/Unit/Encoding/Int64EncoderTest.php deleted file mode 100644 index 432d9efa56..0000000000 --- a/src/core/etl/tests/Flow/Floe/Tests/Unit/Encoding/Int64EncoderTest.php +++ /dev/null @@ -1,26 +0,0 @@ -encode(PHP_INT_MIN)); - static::assertSame(pack('P', PHP_INT_MAX), $encoder->encode(PHP_INT_MAX)); - static::assertSame("\xFB\xFF\xFF\xFF\xFF\xFF\xFF\xFF", $encoder->encode(-5)); - static::assertSame("\x2A\x00\x00\x00\x00\x00\x00\x00", $encoder->encode(42)); - } -} diff --git a/src/core/etl/tests/Flow/Floe/Tests/Unit/Encoding/IntervalEncoderTest.php b/src/core/etl/tests/Flow/Floe/Tests/Unit/Encoding/IntervalEncoderTest.php deleted file mode 100644 index dccd16a929..0000000000 --- a/src/core/etl/tests/Flow/Floe/Tests/Unit/Encoding/IntervalEncoderTest.php +++ /dev/null @@ -1,28 +0,0 @@ -invert = 1; - // @mago-ignore analysis:invalid-property-write - $value->f = 0.5; - - static::assertSame( - "\x01" . pack('VVVVVV', 1, 2, 3, 4, 5, 6) . pack('e', 0.5), - (new IntervalEncoder())->encode($value), - ); - } -} diff --git a/src/core/etl/tests/Flow/Floe/Tests/Unit/Encoding/JsonEncoderTest.php b/src/core/etl/tests/Flow/Floe/Tests/Unit/Encoding/JsonEncoderTest.php deleted file mode 100644 index 785328de95..0000000000 --- a/src/core/etl/tests/Flow/Floe/Tests/Unit/Encoding/JsonEncoderTest.php +++ /dev/null @@ -1,22 +0,0 @@ -encode(new Json('[1,2,3]'))); - static::assertSame(pack('V', 10) . '{"a":true}' . "\x01", $encoder->encode(new Json('{"a":true}'))); - } -} diff --git a/src/core/etl/tests/Flow/Floe/Tests/Unit/Encoding/ListEncoderTest.php b/src/core/etl/tests/Flow/Floe/Tests/Unit/Encoding/ListEncoderTest.php deleted file mode 100644 index 8fc99e6262..0000000000 --- a/src/core/etl/tests/Flow/Floe/Tests/Unit/Encoding/ListEncoderTest.php +++ /dev/null @@ -1,22 +0,0 @@ -encode(['a', 'bc'])); - static::assertSame(pack('V', 0), $encoder->encode([])); - } -} diff --git a/src/core/etl/tests/Flow/Floe/Tests/Unit/Encoding/MapEncoderTest.php b/src/core/etl/tests/Flow/Floe/Tests/Unit/Encoding/MapEncoderTest.php deleted file mode 100644 index 1fae624359..0000000000 --- a/src/core/etl/tests/Flow/Floe/Tests/Unit/Encoding/MapEncoderTest.php +++ /dev/null @@ -1,30 +0,0 @@ -encode(['a' => 42])); - } - - public function test_encodes_integer_keys(): void - { - $encoder = new MapEncoder(new Int64Encoder(), new StringEncoder()); - - static::assertSame(pack('V', 1) . pack('P', 7) . pack('V', 1) . 'x', $encoder->encode([7 => 'x'])); - } -} diff --git a/src/core/etl/tests/Flow/Floe/Tests/Unit/Encoding/NullEncoderTest.php b/src/core/etl/tests/Flow/Floe/Tests/Unit/Encoding/NullEncoderTest.php deleted file mode 100644 index 6a143bab36..0000000000 --- a/src/core/etl/tests/Flow/Floe/Tests/Unit/Encoding/NullEncoderTest.php +++ /dev/null @@ -1,16 +0,0 @@ -encode(null)); - } -} diff --git a/src/core/etl/tests/Flow/Floe/Tests/Unit/Encoding/OptionalEncoderTest.php b/src/core/etl/tests/Flow/Floe/Tests/Unit/Encoding/OptionalEncoderTest.php deleted file mode 100644 index 25fe70185c..0000000000 --- a/src/core/etl/tests/Flow/Floe/Tests/Unit/Encoding/OptionalEncoderTest.php +++ /dev/null @@ -1,23 +0,0 @@ -encode(null)); - static::assertSame(Format::VALUE_PRESENT_BYTE . pack('P', 42), $encoder->encode(42)); - } -} diff --git a/src/core/etl/tests/Flow/Floe/Tests/Unit/Encoding/PackedListEncoderTest.php b/src/core/etl/tests/Flow/Floe/Tests/Unit/Encoding/PackedListEncoderTest.php deleted file mode 100644 index 81fafd3499..0000000000 --- a/src/core/etl/tests/Flow/Floe/Tests/Unit/Encoding/PackedListEncoderTest.php +++ /dev/null @@ -1,28 +0,0 @@ -encode([10, -20, 30])); - static::assertSame(pack('V', 0), $encoder->encode([])); - } - - public function test_encodes_float64_list_in_bulk(): void - { - $encoder = new PackedListEncoder('e'); - - static::assertSame(pack('V', 2) . pack('e*', 1.5, -2.5), $encoder->encode([1.5, -2.5])); - } -} diff --git a/src/core/etl/tests/Flow/Floe/Tests/Unit/Encoding/StringEncoderTest.php b/src/core/etl/tests/Flow/Floe/Tests/Unit/Encoding/StringEncoderTest.php deleted file mode 100644 index adfacbf642..0000000000 --- a/src/core/etl/tests/Flow/Floe/Tests/Unit/Encoding/StringEncoderTest.php +++ /dev/null @@ -1,21 +0,0 @@ -encode('abc')); - static::assertSame(pack('V', 0), $encoder->encode('')); - } -} diff --git a/src/core/etl/tests/Flow/Floe/Tests/Unit/Encoding/StringKeyEncoderTest.php b/src/core/etl/tests/Flow/Floe/Tests/Unit/Encoding/StringKeyEncoderTest.php deleted file mode 100644 index 6734fc8437..0000000000 --- a/src/core/etl/tests/Flow/Floe/Tests/Unit/Encoding/StringKeyEncoderTest.php +++ /dev/null @@ -1,24 +0,0 @@ -encode('abc')); - } - - public function test_encodes_numeric_string_key_arriving_as_integer(): void - { - // PHP turns numeric-string array keys into integers - static::assertSame(pack('V', 3) . '123', (new StringKeyEncoder())->encode(123)); - } -} diff --git a/src/core/etl/tests/Flow/Floe/Tests/Unit/Encoding/StructureEncoderTest.php b/src/core/etl/tests/Flow/Floe/Tests/Unit/Encoding/StructureEncoderTest.php deleted file mode 100644 index 151fc7cbf4..0000000000 --- a/src/core/etl/tests/Flow/Floe/Tests/Unit/Encoding/StructureEncoderTest.php +++ /dev/null @@ -1,37 +0,0 @@ - new Int64Encoder(), - 'name' => new StringEncoder(), - 'missing' => new Int64Encoder(), - ]); - - static::assertSame( - Format::VALUE_PRESENT_BYTE . pack('P', 1) . Format::VALUE_NULL_BYTE . Format::VALUE_ABSENT_BYTE, - $encoder->encode(['id' => 1, 'name' => null]), - ); - } - - public function test_undeclared_keys_are_not_encoded(): void - { - $encoder = new StructureEncoder(['id' => new Int64Encoder()]); - - static::assertSame(Format::VALUE_PRESENT_BYTE . pack('P', 1), $encoder->encode(['id' => 1, 'extra' => 2])); - } -} diff --git a/src/core/etl/tests/Flow/Floe/Tests/Unit/Encoding/TimeZoneEncoderTest.php b/src/core/etl/tests/Flow/Floe/Tests/Unit/Encoding/TimeZoneEncoderTest.php deleted file mode 100644 index c96516c443..0000000000 --- a/src/core/etl/tests/Flow/Floe/Tests/Unit/Encoding/TimeZoneEncoderTest.php +++ /dev/null @@ -1,22 +0,0 @@ -encode(new DateTimeZone('Australia/Eucla')), - ); - } -} diff --git a/src/core/etl/tests/Flow/Floe/Tests/Unit/Encoding/UuidEncoderTest.php b/src/core/etl/tests/Flow/Floe/Tests/Unit/Encoding/UuidEncoderTest.php deleted file mode 100644 index 5d8a0a8514..0000000000 --- a/src/core/etl/tests/Flow/Floe/Tests/Unit/Encoding/UuidEncoderTest.php +++ /dev/null @@ -1,22 +0,0 @@ -encode(new Uuid($uuid))); - } -} diff --git a/src/core/etl/tests/Flow/Floe/Tests/Unit/Encoding/XmlDocumentEncoderTest.php b/src/core/etl/tests/Flow/Floe/Tests/Unit/Encoding/XmlDocumentEncoderTest.php deleted file mode 100644 index 300eaccc22..0000000000 --- a/src/core/etl/tests/Flow/Floe/Tests/Unit/Encoding/XmlDocumentEncoderTest.php +++ /dev/null @@ -1,26 +0,0 @@ -loadXML('value'); - - $xml = ValueEncoder::xmlDocumentToString($document); - - static::assertSame(pack('V', strlen($xml)) . $xml, (new XmlDocumentEncoder())->encode($document)); - } -} diff --git a/src/core/etl/tests/Flow/Floe/Tests/Unit/Encoding/XmlElementEncoderTest.php b/src/core/etl/tests/Flow/Floe/Tests/Unit/Encoding/XmlElementEncoderTest.php deleted file mode 100644 index 8e8b763163..0000000000 --- a/src/core/etl/tests/Flow/Floe/Tests/Unit/Encoding/XmlElementEncoderTest.php +++ /dev/null @@ -1,29 +0,0 @@ -loadXML('value'); - $element = $document->documentElement; - static::assertInstanceOf(DOMElement::class, $element); - - static::assertSame( - pack('V', strlen('value')) . 'value', - (new XmlElementEncoder())->encode($element), - ); - } -} diff --git a/src/core/etl/tests/Flow/Floe/Tests/Unit/FloeEngineTest.php b/src/core/etl/tests/Flow/Floe/Tests/Unit/FloeEngineTest.php deleted file mode 100644 index fe6e249af9..0000000000 --- a/src/core/etl/tests/Flow/Floe/Tests/Unit/FloeEngineTest.php +++ /dev/null @@ -1,54 +0,0 @@ -encoder($this->schema()), - ); - } - - public function test_native_engine_builds_native_encoder_when_supported(): void - { - if (!NativeFloeEncoder::isSupported()) { - static::markTestSkipped('flow_php extension is not loaded'); - } - - static::assertInstanceOf(NativeFloeEncoder::class, FloeEngine::native->encoder($this->schema())); - } - - public function test_php_engine_builds_php_encoder(): void - { - static::assertInstanceOf(PhpFloeEncoder::class, FloeEngine::php->encoder($this->schema())); - } - - public function test_case_values_are_stable_identifiers(): void - { - static::assertSame('adaptive', FloeEngine::adaptive->value); - static::assertSame('native', FloeEngine::native->value); - static::assertSame('php', FloeEngine::php->value); - } - - private function schema(): Schema - { - return schema_from_json(FloeSchemaContext::schemaBody(schema(int_schema('id'), str_schema('name')))); - } -} diff --git a/src/core/etl/tests/Flow/Floe/Tests/Unit/FloeExtractorTest.php b/src/core/etl/tests/Flow/Floe/Tests/Unit/FloeExtractorTest.php index 2c598399fc..6dc5defddd 100644 --- a/src/core/etl/tests/Flow/Floe/Tests/Unit/FloeExtractorTest.php +++ b/src/core/etl/tests/Flow/Floe/Tests/Unit/FloeExtractorTest.php @@ -8,21 +8,21 @@ use Flow\ETL\Exception\InvalidArgumentException; use Flow\ETL\Extractor\Signal; use Flow\ETL\Tests\Double\CountingFilesystem; +use Flow\ETL\Tests\Double\SpyBackend; use Flow\ETL\Tests\FlowTestCase; use Flow\ETL\Tests\Mother\RowsMother; use Flow\Filesystem\Tests\Double\RejectingFilter; use Flow\Floe\FloeExtractor; -use Flow\Floe\Tests\Context\FloeEngineContext; +use Flow\Floe\Tests\Context\FloeFilesContext; use function array_sum; +use function Flow\ETL\DSL\array_to_rows; use function Flow\ETL\DSL\config; use function Flow\ETL\DSL\config_builder; use function Flow\ETL\DSL\df; use function Flow\ETL\DSL\flow_context; use function Flow\ETL\DSL\int_schema; use function Flow\ETL\DSL\partition_types; -use function Flow\ETL\DSL\row; -use function Flow\ETL\DSL\rows; use function Flow\ETL\DSL\schema; use function Flow\ETL\DSL\str_schema; use function Flow\Filesystem\DSL\memory_filesystem; @@ -38,14 +38,14 @@ final class FloeExtractorTest extends FlowTestCase public function test_a_later_file_with_other_columns_throws_naming_both_files(): void { $memory = memory_filesystem(); - FloeEngineContext::writeFiles($memory, [ - 'memory://glob/a.floe' => rows(schema(int_schema('id')), row(['id' => 1])), - 'memory://glob/b.floe' => rows( - schema(int_schema('id'), str_schema('extra')), - row([ + FloeFilesContext::writeFiles($memory, [ + 'memory://glob/a.floe' => array_to_rows([['id' => 1]], schema(int_schema('id'))), + 'memory://glob/b.floe' => array_to_rows( + [[ 'id' => 2, 'extra' => 'x', - ]), + ]], + schema(int_schema('id'), str_schema('extra')), ), ]); @@ -64,9 +64,9 @@ public function test_a_later_file_with_other_columns_throws_naming_both_files(): public function test_a_later_file_with_another_column_type_throws(): void { $memory = memory_filesystem(); - FloeEngineContext::writeFiles($memory, [ - 'memory://glob/a.floe' => rows(schema(int_schema('id')), row(['id' => 1])), - 'memory://glob/b.floe' => rows(schema(str_schema('id')), row(['id' => 'x'])), + FloeFilesContext::writeFiles($memory, [ + 'memory://glob/a.floe' => array_to_rows([['id' => 1]], schema(int_schema('id'))), + 'memory://glob/b.floe' => array_to_rows([['id' => 'x']], schema(str_schema('id'))), ]); $this->expectException(InferredSchemaException::class); @@ -80,20 +80,20 @@ public function test_a_later_file_with_another_column_type_throws(): void public function test_a_later_file_with_the_columns_in_another_order_follows_the_first(): void { $memory = memory_filesystem(); - FloeEngineContext::writeFiles($memory, [ - 'memory://glob/a.floe' => rows( - schema(int_schema('id'), str_schema('name')), - row([ + FloeFilesContext::writeFiles($memory, [ + 'memory://glob/a.floe' => array_to_rows( + [[ 'id' => 1, 'name' => 'a', - ]), + ]], + schema(int_schema('id'), str_schema('name')), ), - 'memory://glob/b.floe' => rows( - schema(str_schema('name'), int_schema('id')), - row([ + 'memory://glob/b.floe' => array_to_rows( + [[ 'name' => 'b', 'id' => 2, - ]), + ]], + schema(str_schema('name'), int_schema('id')), ), ]); @@ -109,14 +109,14 @@ public function test_a_later_file_with_the_columns_in_another_order_follows_the_ public function test_union_by_name_reads_every_file_under_one_schema(): void { $memory = memory_filesystem(); - FloeEngineContext::writeFiles($memory, [ - 'memory://glob/a.floe' => rows(schema(int_schema('id')), row(['id' => 1])), - 'memory://glob/b.floe' => rows( - schema(int_schema('id'), str_schema('extra')), - row([ + FloeFilesContext::writeFiles($memory, [ + 'memory://glob/a.floe' => array_to_rows([['id' => 1]], schema(int_schema('id'))), + 'memory://glob/b.floe' => array_to_rows( + [[ 'id' => 2, 'extra' => 'x', - ]), + ]], + schema(int_schema('id'), str_schema('extra')), ), ]); @@ -136,13 +136,13 @@ public function test_extract_adds_input_file_uri_when_configured(): void $path = path('memory://input-uri.floe'); $loader = to_floe($path, filesystem: $memory); - $loader->load(rows(schema(int_schema('id')), row(['id' => 1])), $context); + $loader->load(array_to_rows([['id' => 1]], schema(int_schema('id'))), $context); $loader->closure($context); $extractor = from_floe($path, filesystem: $memory)->withMetadataColumns(true); $batches = iterator_to_array($extractor->extract($context)); - static::assertSame($path->uri(), $batches[0]->first()->get('_input_file_uri')); + static::assertSame($path->uri(), $batches[0]->column('_input_file_uri')->value(0)); } public function test_extract_applies_offset_within_a_file(): void @@ -152,14 +152,14 @@ public function test_extract_applies_offset_within_a_file(): void $path = path('memory://offset-within.floe'); $loader = to_floe($path, filesystem: $memory); - $loader->load(rows(schema(int_schema('id')), row(['id' => 1]), row(['id' => 2]), row(['id' => 3])), $context); + $loader->load(array_to_rows([['id' => 1], ['id' => 2], ['id' => 3]], schema(int_schema('id'))), $context); $loader->closure($context); $ids = []; foreach (from_floe($path, filesystem: $memory)->withOffset(1)->extract($context) as $batch) { - foreach ($batch->all() as $extractedRow) { - $ids[] = $extractedRow->get('id'); + foreach ($batch->toArray() as $extractedRow) { + $ids[] = $extractedRow['id']; } } @@ -173,7 +173,7 @@ public function test_extract_honors_limit(): void $path = path('memory://limited.floe'); $loader = to_floe($path, filesystem: $memory); - $loader->load(rows(schema(int_schema('id')), row(['id' => 1]), row(['id' => 2]), row(['id' => 3])), $context); + $loader->load(array_to_rows([['id' => 1], ['id' => 2], ['id' => 3]], schema(int_schema('id'))), $context); $loader->closure($context); $extractor = from_floe($path, filesystem: $memory); @@ -181,8 +181,8 @@ public function test_extract_honors_limit(): void $ids = []; foreach ($extractor->extract($context, limit: 2) as $batch) { - foreach ($batch->all() as $extractedRow) { - $ids[] = $extractedRow->get('id'); + foreach ($batch->toArray() as $extractedRow) { + $ids[] = $extractedRow['id']; } } @@ -196,12 +196,12 @@ public function test_extract_omits_input_file_uri_by_default(): void $path = path('memory://no-input-uri.floe'); $loader = to_floe($path, filesystem: $memory); - $loader->load(rows(schema(int_schema('id')), row(['id' => 1])), $context); + $loader->load(array_to_rows([['id' => 1]], schema(int_schema('id'))), $context); $loader->closure($context); $batches = iterator_to_array(from_floe($path, filesystem: $memory)->extract($context)); - static::assertSame(['id'], $batches[0]->first()->names()); + static::assertSame(['id'], array_keys($batches[0]->values(0))); } public function test_extract_reads_rows(): void @@ -211,14 +211,14 @@ public function test_extract_reads_rows(): void $path = path('memory://read.floe'); $loader = to_floe($path, filesystem: $memory); - $loader->load(rows(schema(int_schema('id')), row(['id' => 1]), row(['id' => 2])), $context); + $loader->load(array_to_rows([['id' => 1], ['id' => 2]], schema(int_schema('id'))), $context); $loader->closure($context); $ids = []; foreach (from_floe($path, filesystem: $memory)->extract($context) as $batch) { - foreach ($batch->all() as $extractedRow) { - $ids[] = $extractedRow->get('id'); + foreach ($batch->toArray() as $extractedRow) { + $ids[] = $extractedRow['id']; } } @@ -232,10 +232,10 @@ public function test_extract_skips_whole_files_with_offset(): void foreach (['a' => [1, 2], 'b' => [10, 11]] as $name => $values) { $loader = to_floe(path('memory://skip-files/' . $name . '.floe'), filesystem: $memory); - $loader->load( - rows(schema(int_schema('id')), row(['id' => $values[0]]), row(['id' => $values[1]])), - $context, - ); + $loader->load(array_to_rows([ + ['id' => $values[0]], + ['id' => $values[1]], + ], schema(int_schema('id'))), $context); $loader->closure($context); } @@ -244,8 +244,8 @@ public function test_extract_skips_whole_files_with_offset(): void foreach (from_floe(path('memory://skip-files/*.floe'), filesystem: $memory) ->withOffset(2) ->extract($context) as $batch) { - foreach ($batch->all() as $extractedRow) { - $ids[] = $extractedRow->get('id'); + foreach ($batch->toArray() as $extractedRow) { + $ids[] = $extractedRow['id']; } } @@ -259,7 +259,7 @@ public function test_extract_stops_on_stop_signal(): void $path = path('memory://stop.floe'); $loader = to_floe($path, filesystem: $memory); - $loader->load(rows(schema(int_schema('id')), row(['id' => 1]), row(['id' => 2])), $context); + $loader->load(array_to_rows([['id' => 1], ['id' => 2]], schema(int_schema('id'))), $context); $loader->closure($context); $generator = from_floe($path, filesystem: $memory)->extract($context); @@ -280,17 +280,44 @@ public function test_extract_yields_the_metadata_column_schema_promises(): void $path = path('memory://declared-with-metadata.floe'); $loader = to_floe($path, filesystem: $memory); - $loader->load(rows(schema(int_schema('id')), row(['id' => 1])), $context); + $loader->load(array_to_rows([['id' => 1]], schema(int_schema('id'))), $context); $loader->closure($context); - $extractor = from_floe($path, filesystem: $memory) - ->withMetadataColumns(true) - ->withSchema(schema(int_schema('id'))); + $extractor = from_floe($path, filesystem: $memory)->withMetadataColumns(true); $batches = iterator_to_array($extractor->extract($context)); - static::assertSame($extractor->schema()->references()->names(), $batches[0]->first()->names()); - static::assertSame($path->uri(), $batches[0]->first()->get('_input_file_uri')); + static::assertSame($extractor->schema()->references()->names(), array_keys($batches[0]->values(0))); + static::assertSame($path->uri(), $batches[0]->column('_input_file_uri')->value(0)); + } + + public function test_with_schema_is_refused(): void + { + $this->expectException(InvalidArgumentException::class); + $this->expectExceptionMessage( + 'Floe is a self-describing format and does not accept a schema; declaring one is not supported yet.', + ); + + from_floe(path('memory://x.floe'), filesystem: memory_filesystem())->withSchema(schema(int_schema('id'))); + } + + public function test_extract_builds_through_the_config_backend(): void + { + $memory = memory_filesystem(); + $path = path('memory://backend.floe'); + $context = flow_context(config()); + + $loader = to_floe($path, filesystem: $memory); + $loader->load(array_to_rows([['id' => 1]], schema(int_schema('id'))), $context); + $loader->closure($context); + + $backend = new SpyBackend(); + + iterator_to_array(from_floe($path, filesystem: $memory)->extract( + flow_context(config_builder()->backend($backend)->build()), + )); + + static::assertGreaterThanOrEqual(1, $backend->decodes()); } public function test_negative_offset_throws(): void @@ -307,7 +334,7 @@ public function test_schema_reads_footer_only(): void $path = path('memory://schema.floe'); $loader = to_floe($path, filesystem: $memory); - $loader->load(rows(schema(int_schema('id')), row(['id' => 1])), $context); + $loader->load(array_to_rows([['id' => 1]], schema(int_schema('id'))), $context); $loader->closure($context); $schema = from_floe($path, filesystem: $memory)->schema(); @@ -318,7 +345,7 @@ public function test_schema_reads_footer_only(): void public function test_schema_declares_partition_columns_from_the_path(): void { $memory = memory_filesystem(); - FloeEngineContext::writePartitionedFiles($memory); + FloeFilesContext::writePartitionedFiles($memory); static::assertSame( ['id', 'country'], @@ -329,14 +356,14 @@ public function test_schema_declares_partition_columns_from_the_path(): void public function test_extract_fills_partition_columns_from_the_path(): void { $memory = memory_filesystem(); - FloeEngineContext::writePartitionedFiles($memory); + FloeFilesContext::writePartitionedFiles($memory); $extractor = from_floe(path('memory://parts/*/*.floe'), filesystem: $memory); $values = []; foreach ($extractor->extract(flow_context(config())) as $rows) { - foreach ($rows as $row) { - $values[] = $row->toArray(); + foreach ($rows->toArray() as $row) { + $values[] = $row; } } @@ -346,7 +373,7 @@ public function test_extract_fills_partition_columns_from_the_path(): void public function test_schema_opens_only_the_first_file(): void { $counting = new CountingFilesystem($memory = memory_filesystem()); - FloeEngineContext::writePartitionedFiles($memory); + FloeFilesContext::writePartitionedFiles($memory); from_floe(path('memory://parts/*/*.floe'), filesystem: $counting)->schema(); @@ -356,7 +383,7 @@ public function test_schema_opens_only_the_first_file(): void public function test_schema_is_memoised(): void { $counting = new CountingFilesystem($memory = memory_filesystem()); - FloeEngineContext::writePartitionedFiles($memory); + FloeFilesContext::writePartitionedFiles($memory); $extractor = from_floe(path('memory://parts/*/*.floe'), filesystem: $counting); @@ -367,7 +394,7 @@ public function test_schema_is_memoised(): void public function test_union_by_name_opens_every_file(): void { $counting = new CountingFilesystem($memory = memory_filesystem()); - FloeEngineContext::writePartitionedFiles($memory); + FloeFilesContext::writePartitionedFiles($memory); from_floe(path('memory://parts/*/*.floe'), filesystem: $counting)->unionByName()->schema(); @@ -377,7 +404,7 @@ public function test_union_by_name_opens_every_file(): void public function test_schema_closes_every_reader_it_opens(): void { $counting = new CountingFilesystem($memory = memory_filesystem()); - FloeEngineContext::writePartitionedFiles($memory); + FloeFilesContext::writePartitionedFiles($memory); from_floe(path('memory://parts/*/*.floe'), filesystem: $counting)->unionByName()->schema(); @@ -387,7 +414,7 @@ public function test_schema_closes_every_reader_it_opens(): void public function test_declared_partition_types_reach_the_rows(): void { $memory = memory_filesystem(); - FloeEngineContext::writeYearPartitionedFiles($memory); + FloeFilesContext::writeYearPartitionedFiles($memory); $extractor = from_floe(path('memory://years/*/*.floe'), filesystem: $memory)->partitionTypes( partition_types(year: type_integer()), @@ -395,26 +422,7 @@ public function test_declared_partition_types_reach_the_rows(): void foreach ($extractor->extract(flow_context(config())) as $rows) { static::assertEquals($extractor->schema(), $rows->schema()); - static::assertSame(2024, $rows->first()->get('year')); - - return; - } - - static::fail('extractor yielded nothing'); - } - - public function test_a_declared_schema_may_name_the_partition_column(): void - { - $memory = memory_filesystem(); - FloeEngineContext::writeYearPartitionedFiles($memory); - - $extractor = from_floe(path('memory://years/*/*.floe'), filesystem: $memory)->withSchema(schema( - int_schema('id'), - int_schema('year'), - )); - - foreach ($extractor->extract(flow_context(config())) as $rows) { - static::assertSame(2024, $rows->first()->get('year')); + static::assertSame(2024, $rows->column('year')->value(0)); return; } @@ -425,7 +433,7 @@ public function test_a_declared_schema_may_name_the_partition_column(): void public function test_extract_closes_the_reader_when_the_generator_is_abandoned(): void { $counting = new CountingFilesystem($memory = memory_filesystem()); - FloeEngineContext::writePartitionedFiles($memory); + FloeFilesContext::writePartitionedFiles($memory); $generator = from_floe(path('memory://parts/*/*.floe'), filesystem: $counting)->extract(flow_context(config())); $generator->current(); @@ -437,7 +445,7 @@ public function test_extract_closes_the_reader_when_the_generator_is_abandoned() public function test_extract_closes_every_reader_it_opens(): void { $counting = new CountingFilesystem($memory = memory_filesystem()); - FloeEngineContext::writePartitionedFiles($memory); + FloeFilesContext::writePartitionedFiles($memory); foreach (from_floe(path('memory://parts/*/*.floe'), filesystem: $counting)->extract( flow_context(config()), @@ -450,7 +458,7 @@ public function test_extract_closes_every_reader_it_opens(): void public function test_extract_closes_the_reader_it_skips_for_the_offset(): void { $counting = new CountingFilesystem($memory = memory_filesystem()); - FloeEngineContext::writePartitionedFiles($memory); + FloeFilesContext::writePartitionedFiles($memory); foreach (from_floe(path('memory://parts/*/*.floe'), filesystem: $counting) ->withOffset(1) @@ -463,7 +471,7 @@ public function test_extract_closes_the_reader_it_skips_for_the_offset(): void public function test_extract_closes_the_reader_when_the_pipeline_stops(): void { $counting = new CountingFilesystem($memory = memory_filesystem()); - FloeEngineContext::writePartitionedFiles($memory); + FloeFilesContext::writePartitionedFiles($memory); $generator = from_floe(path('memory://parts/*/*.floe'), filesystem: $counting)->extract(flow_context(config())); @@ -476,7 +484,7 @@ public function test_extract_closes_the_reader_when_the_pipeline_stops(): void public function test_a_path_filter_narrows_the_read_but_not_the_schema(): void { $counting = new CountingFilesystem($memory = memory_filesystem()); - FloeEngineContext::writePartitionedFiles($memory); + FloeFilesContext::writePartitionedFiles($memory); $extractor = from_floe(path('memory://parts/*/*.floe'), filesystem: $counting); $schema = $extractor->schema(); diff --git a/src/core/etl/tests/Flow/Floe/Tests/Unit/FloeFileBatchesTest.php b/src/core/etl/tests/Flow/Floe/Tests/Unit/FloeFileBatchesTest.php new file mode 100644 index 0000000000..4fa80011ab --- /dev/null +++ b/src/core/etl/tests/Flow/Floe/Tests/Unit/FloeFileBatchesTest.php @@ -0,0 +1,138 @@ + array_to_rows([ + ['id' => 1], + ['id' => 2], + ['id' => 3], + ], schema(int_schema('id')))]); + $file = (new FloeFileBatches( + $filesystem, + new NoopCodec(), + 65536, + FileColumnsContext::discovering(names: []), + null, + ))->batches( + new SourceFile(path('memory://a.floe')), + schema(int_schema('id')), + 10, + new PhpBackend(), + new ReadWindow(3), + ); + + static::assertSame([], iterator_to_array($file, false)); + static::assertSame(3, $file->getReturn()); + } + + public function test_an_offset_inside_the_file_starts_the_read_there_and_is_consumed(): void + { + $filesystem = memory_filesystem(); + FloeFilesContext::writeFiles($filesystem, ['memory://a.floe' => array_to_rows([ + ['id' => 1], + ['id' => 2], + ['id' => 3], + ], schema(int_schema('id')))]); + $file = (new FloeFileBatches( + $filesystem, + new NoopCodec(), + 65536, + FileColumnsContext::discovering(names: []), + null, + ))->batches( + new SourceFile(path('memory://a.floe')), + schema(int_schema('id')), + 10, + new PhpBackend(), + new ReadWindow(1), + ); + + static::assertSame( + [[['id' => 2], ['id' => 3]]], + array_map(static fn(Rows $rows): array => $rows->toArray(), iterator_to_array($file, false)), + ); + static::assertSame(1, $file->getReturn()); + } + + public function test_a_file_narrower_than_the_body_is_padded_with_nulls(): void + { + $filesystem = memory_filesystem(); + FloeFilesContext::writeFiles($filesystem, ['memory://a.floe' => array_to_rows([[ + 'id' => 1, + ]], schema(int_schema('id')))]); + $file = (new FloeFileBatches( + $filesystem, + new NoopCodec(), + 65536, + FileColumnsContext::discovering(names: []), + null, + ))->batches( + new SourceFile(path('memory://a.floe')), + schema(int_schema('id'), str_schema('name', nullable: true)), + 10, + new PhpBackend(), + new ReadWindow(), + ); + + static::assertSame( + [[['id' => 1, 'name' => null]]], + array_map(static fn(Rows $rows): array => $rows->toArray(), iterator_to_array($file, false)), + ); + } + + public function test_a_file_that_diverges_from_the_derived_schema_is_refused_and_closed(): void + { + $filesystem = new CountingFilesystem(memory_filesystem()); + FloeFilesContext::writeFiles($filesystem, ['memory://b.floe' => array_to_rows([[ + 'name' => 'x', + ]], schema(str_schema('name')))]); + + try { + iterator_to_array((new FloeFileBatches( + $filesystem, + new NoopCodec(), + 65536, + FileColumnsContext::discovering(names: []), + new DerivedSchema(schema(int_schema('id')), 'memory://a.floe'), + ))->batches( + new SourceFile(path('memory://b.floe')), + schema(int_schema('id')), + 10, + new PhpBackend(), + new ReadWindow(), + )); + static::fail('a diverging file must be refused'); + } catch (InferredSchemaException) { + static::assertSame($filesystem->readFromCalls, $filesystem->closedStreams()); + } + } +} diff --git a/src/core/etl/tests/Flow/Floe/Tests/Unit/FloeFileSinkTest.php b/src/core/etl/tests/Flow/Floe/Tests/Unit/FloeFileSinkTest.php new file mode 100644 index 0000000000..e845923a3c --- /dev/null +++ b/src/core/etl/tests/Flow/Floe/Tests/Unit/FloeFileSinkTest.php @@ -0,0 +1,53 @@ +writeTo(path('memory://a.floe')); + + (new FloeFileSinks($filesystem, null, null, new Options())) + ->open($stream, new PhpBackend()) + ->close(); + $stream->close(); + + static::assertSame('', $filesystem->readFrom(path('memory://a.floe'))->content()); + } + + public function test_the_first_write_opens_the_writer_and_every_write_lands_in_the_file(): void + { + $filesystem = memory_filesystem(); + $stream = $filesystem->writeTo(path('memory://a.floe')); + $sink = (new FloeFileSinks($filesystem, null, null, new Options()))->open($stream, new PhpBackend()); + + $sink->write(array_to_rows([['id' => 1]], schema(int_schema('id')))); + $sink->write(array_to_rows([['id' => 2]], schema(int_schema('id')))); + $sink->close(); + $stream->close(); + + $rows = []; + + foreach (FloeFilesContext::phpReader($filesystem)->read(path('memory://a.floe'))->rows() as $batch) { + $rows = [...$rows, ...$batch->toArray()]; + } + + static::assertSame([['id' => 1], ['id' => 2]], $rows); + } +} diff --git a/src/core/etl/tests/Flow/Floe/Tests/Unit/FloeFileSinksTest.php b/src/core/etl/tests/Flow/Floe/Tests/Unit/FloeFileSinksTest.php new file mode 100644 index 0000000000..2bdcdb0d20 --- /dev/null +++ b/src/core/etl/tests/Flow/Floe/Tests/Unit/FloeFileSinksTest.php @@ -0,0 +1,58 @@ +open($filesystem->writeTo(path('memory://a.floe')), new PhpBackend()); + $second = $sinks->open($filesystem->writeTo(path('memory://b.floe')), new PhpBackend()); + + $first->write(array_to_rows([['id' => 1, 'name' => 'a']], $schema)); + $second->write(array_to_rows([['id' => 2, 'name' => null]], $schema)); + $first->close(); + $second->close(); + + static::assertEquals( + $schema, + FloeFilesContext::phpReader($filesystem)->read(path('memory://b.floe'))->schema(), + ); + } + + public function test_a_declared_schema_is_the_written_one(): void + { + $filesystem = memory_filesystem(); + $declared = schema(int_schema('id', nullable: true)); + $sink = (new FloeFileSinks($filesystem, $declared, null, new Options()))->open( + $filesystem->writeTo(path('memory://a.floe')), + new PhpBackend(), + ); + + $sink->write(array_to_rows([['id' => 1]], schema(int_schema('id')))); + $sink->close(); + + static::assertEquals( + $declared, + FloeFilesContext::phpReader($filesystem)->read(path('memory://a.floe'))->schema(), + ); + } +} diff --git a/src/core/etl/tests/Flow/Floe/Tests/Unit/FloeLoaderTest.php b/src/core/etl/tests/Flow/Floe/Tests/Unit/FloeLoaderTest.php index 9f1f96b68b..422d42db74 100644 --- a/src/core/etl/tests/Flow/Floe/Tests/Unit/FloeLoaderTest.php +++ b/src/core/etl/tests/Flow/Floe/Tests/Unit/FloeLoaderTest.php @@ -5,25 +5,25 @@ namespace Flow\Floe\Tests\Unit; use Flow\ETL\Exception\RuntimeException; +use Flow\ETL\Filesystem\SaveMode; use Flow\Floe\Exception\IncompatibleSchemaException; -use Flow\Floe\Tests\Double\SpyHydrator; +use Flow\Floe\Tests\Context\FloeFilesContext; use PHPUnit\Framework\TestCase; +use function Flow\ETL\DSL\array_to_rows; use function Flow\ETL\DSL\config; -use function Flow\ETL\DSL\config_builder; use function Flow\ETL\DSL\data_frame; use function Flow\ETL\DSL\flow_context; use function Flow\ETL\DSL\from_array; use function Flow\ETL\DSL\int_schema; use function Flow\ETL\DSL\partition_by; -use function Flow\ETL\DSL\row; -use function Flow\ETL\DSL\rows; use function Flow\ETL\DSL\schema; use function Flow\ETL\DSL\str_schema; use function Flow\Filesystem\DSL\memory_filesystem; use function Flow\Filesystem\DSL\path; use function Flow\Floe\DSL\from_floe; use function Flow\Floe\DSL\to_floe; +use function implode; use function sort; final class FloeLoaderTest extends TestCase @@ -35,32 +35,43 @@ public function test_closure_writes_a_readable_non_torn_file(): void $path = path('memory://closed.floe'); $loader = to_floe($path, filesystem: $memory); - $loader->load(rows(schema(int_schema('id')), row(['id' => 1]), row(['id' => 2])), $context); + $loader->load(array_to_rows([['id' => 1], ['id' => 2]], schema(int_schema('id'))), $context); $loader->closure($context); $ids = []; foreach (from_floe($path, filesystem: $memory)->extract($context) as $batch) { - foreach ($batch->all() as $extractedRow) { - $ids[] = $extractedRow->get('id'); + foreach ($batch->toArray() as $extractedRow) { + $ids[] = $extractedRow['id']; } } static::assertSame([1, 2], $ids); } - public function test_load_honors_the_context_hydrator(): void + public function test_a_second_run_writes_under_its_own_schema(): void { - $hydrator = new SpyHydrator(); - $context = flow_context(config_builder()->hydrator($hydrator)->build()); + $context = flow_context(config()); $memory = memory_filesystem(); - $path = path('memory://hydrator.floe'); + $loader = to_floe(path('memory://runs/data.floe'), filesystem: $memory)->saveMode(SaveMode::Append); - $loader = to_floe($path, filesystem: $memory); - $loader->load(rows(schema(int_schema('id')), row(['id' => 1]), row(['id' => 2])), $context); + $loader->load(array_to_rows([['id' => 1]], schema(int_schema('id'))), $context); $loader->closure($context); + $loader->load(array_to_rows([['name' => 'a']], schema(str_schema('name'))), $context); + $loader->closure($context); + + $schemas = []; - static::assertGreaterThan(0, $hydrator->dehydrateCalls); + foreach ($memory->list(path('memory://runs/*.floe')) as $file) { + $schemas[] = implode( + ',', + FloeFilesContext::phpReader($memory)->read($file->path)->schema()->references()->names(), + ); + } + + sort($schemas); + + static::assertSame(['id', 'name'], $schemas); } public function test_destination_returns_path(): void @@ -80,10 +91,9 @@ public function test_inferred_schema_preserves_nullability(): void $loader = to_floe($path, filesystem: $memory); $loader->load( - rows( + array_to_rows( + [['id' => 1, 'note' => 'a'], ['id' => 2, 'note' => null]], schema(int_schema('id'), str_schema('note', nullable: true)), - row(['id' => 1, 'note' => 'a']), - row(['id' => 2, 'note' => null]), ), $context, ); @@ -99,10 +109,9 @@ public function test_load_to_path_without_extension_reports_failure(): void { $this->expectException(RuntimeException::class); - to_floe(path('memory://no-extension'), filesystem: memory_filesystem())->load( - rows(schema(int_schema('id')), row(['id' => 1])), - flow_context(config()), - ); + to_floe(path('memory://no-extension'), filesystem: memory_filesystem())->load(array_to_rows([[ + 'id' => 1, + ]], schema(int_schema('id'))), flow_context(config())); } public function test_partitioned_batches_write_one_file_per_partition(): void @@ -114,10 +123,9 @@ public function test_partitioned_batches_write_one_file_per_partition(): void // one batch carrying both combinations: the loader routes it, B39 $loader = to_floe($base, filesystem: $memory)->partitionBy(partition_by('country')); $loader->load( - rows( + array_to_rows( + [['id' => 1, 'country' => 'PL'], ['id' => 2, 'country' => 'US']], schema(int_schema('id'), str_schema('country')), - row(['id' => 1, 'country' => 'PL']), - row(['id' => 2, 'country' => 'US']), ), $context, ); @@ -129,8 +137,8 @@ public function test_partitioned_batches_write_one_file_per_partition(): void $ids = []; foreach (from_floe(path('memory://parts/**/*.floe'), filesystem: $memory)->extract($context) as $batch) { - foreach ($batch->all() as $extractedRow) { - $ids[] = $extractedRow->get('id'); + foreach ($batch->toArray() as $extractedRow) { + $ids[] = $extractedRow['id']; } } @@ -146,16 +154,16 @@ public function test_repeated_loads_write_a_single_file(): void $path = path('memory://repeated.floe'); $loader = to_floe($path, filesystem: $memory); - $loader->load(rows(schema(int_schema('id')), row(['id' => 1]), row(['id' => 2])), $context); - $loader->load(rows(schema(int_schema('id')), row(['id' => 3])), $context); + $loader->load(array_to_rows([['id' => 1], ['id' => 2]], schema(int_schema('id'))), $context); + $loader->load(array_to_rows([['id' => 3]], schema(int_schema('id'))), $context); $loader->closure($context); $ids = []; // reading the concrete (non-pattern) path proves all rows landed in one file foreach (from_floe($path, filesystem: $memory)->extract($context) as $batch) { - foreach ($batch->all() as $extractedRow) { - $ids[] = $extractedRow->get('id'); + foreach ($batch->toArray() as $extractedRow) { + $ids[] = $extractedRow['id']; } } diff --git a/src/core/etl/tests/Flow/Floe/Tests/Unit/FloeMergerTest.php b/src/core/etl/tests/Flow/Floe/Tests/Unit/FloeMergerTest.php index 95c7818a84..818f4d439b 100644 --- a/src/core/etl/tests/Flow/Floe/Tests/Unit/FloeMergerTest.php +++ b/src/core/etl/tests/Flow/Floe/Tests/Unit/FloeMergerTest.php @@ -6,6 +6,7 @@ use DateTimeImmutable; use DateTimeZone; +use Flow\ETL\Column\AdaptiveBackend; use Flow\ETL\Schema\Metadata; use Flow\Floe\Exception\FloeException; use Flow\Floe\Exception\IncompatibleSchemaException; @@ -17,9 +18,9 @@ use PHPUnit\Framework\TestCase; use function chr; +use function Flow\ETL\DSL\array_to_rows; use function Flow\ETL\DSL\datetime_schema; use function Flow\ETL\DSL\int_schema; -use function Flow\ETL\DSL\row; use function Flow\ETL\DSL\rows; use function Flow\ETL\DSL\schema; use function Flow\ETL\DSL\str_schema; @@ -37,12 +38,19 @@ public function test_compact_reads_identically_to_splice(): void FloeStreamReaderContext::write( $fs, path('memory://a.floe'), - rows(schema(int_schema('id')), row(['id' => 1]), row(['id' => 2])), + array_to_rows([['id' => 1], ['id' => 2]], schema(int_schema('id'))), + ); + FloeStreamReaderContext::write( + $fs, + path('memory://b.floe'), + array_to_rows([['id' => 3]], schema(int_schema('id'))), ); - FloeStreamReaderContext::write($fs, path('memory://b.floe'), rows(schema(int_schema('id')), row(['id' => 3]))); - (new FloeMerger($fs))->merge([path('memory://a.floe'), path('memory://b.floe')], path('memory://splice.floe')); - (new FloeMerger($fs))->merge( + (new FloeMerger($fs, new AdaptiveBackend()))->merge([ + path('memory://a.floe'), + path('memory://b.floe'), + ], path('memory://splice.floe')); + (new FloeMerger($fs, new AdaptiveBackend()))->merge( [path('memory://a.floe'), path('memory://b.floe')], path('memory://compact.floe'), compact: true, @@ -57,16 +65,27 @@ public function test_compact_reads_identically_to_splice(): void public function test_compact_coalesces_same_schema_sources_into_one_section(): void { $fs = memory_filesystem(); - FloeStreamReaderContext::write($fs, path('memory://a.floe'), rows(schema(int_schema('id')), row(['id' => 1]))); - FloeStreamReaderContext::write($fs, path('memory://b.floe'), rows(schema(int_schema('id')), row(['id' => 2]))); + FloeStreamReaderContext::write( + $fs, + path('memory://a.floe'), + array_to_rows([['id' => 1]], schema(int_schema('id'))), + ); + FloeStreamReaderContext::write( + $fs, + path('memory://b.floe'), + array_to_rows([['id' => 2]], schema(int_schema('id'))), + ); - (new FloeMerger($fs))->merge( + (new FloeMerger($fs, new AdaptiveBackend()))->merge( [path('memory://a.floe'), path('memory://b.floe')], path('memory://compact.floe'), compact: true, ); - static::assertCount(1, (new FloeReader($fs))->read(path('memory://compact.floe'))->footer()->sections); + static::assertCount( + 1, + (new FloeReader($fs, new AdaptiveBackend()))->read(path('memory://compact.floe'))->footer()->sections, + ); } public function test_merge_of_same_columns_in_different_order_re_encodes_instead_of_splicing(): void @@ -75,15 +94,18 @@ public function test_merge_of_same_columns_in_different_order_re_encodes_instead FloeStreamReaderContext::write( $fs, path('memory://a.floe'), - rows(schema(int_schema('a'), int_schema('b')), row(['a' => 1, 'b' => 100])), + array_to_rows([['a' => 1, 'b' => 100]], schema(int_schema('a'), int_schema('b'))), ); FloeStreamReaderContext::write( $fs, path('memory://b.floe'), - rows(schema(int_schema('b'), int_schema('a')), row(['b' => 200, 'a' => 2])), + array_to_rows([['b' => 200, 'a' => 2]], schema(int_schema('b'), int_schema('a'))), ); - (new FloeMerger($fs))->merge([path('memory://a.floe'), path('memory://b.floe')], path('memory://out.floe')); + (new FloeMerger($fs, new AdaptiveBackend()))->merge([ + path('memory://a.floe'), + path('memory://b.floe'), + ], path('memory://out.floe')); static::assertSame( [ @@ -93,7 +115,10 @@ public function test_merge_of_same_columns_in_different_order_re_encodes_instead FloeStreamReaderContext::readAll($fs, path('memory://out.floe'))->toArray(), ); // the compact path coalesces sources into one section - splicing would keep one per source - static::assertCount(1, (new FloeReader($fs))->read(path('memory://out.floe'))->footer()->sections); + static::assertCount( + 1, + (new FloeReader($fs, new AdaptiveBackend()))->read(path('memory://out.floe'))->footer()->sections, + ); } public function test_merge_of_same_columns_in_different_order_with_mixed_types_round_trips(): void @@ -102,15 +127,18 @@ public function test_merge_of_same_columns_in_different_order_with_mixed_types_r FloeStreamReaderContext::write( $fs, path('memory://a.floe'), - rows(schema(int_schema('a'), str_schema('b')), row(['a' => 1, 'b' => 'one'])), + array_to_rows([['a' => 1, 'b' => 'one']], schema(int_schema('a'), str_schema('b'))), ); FloeStreamReaderContext::write( $fs, path('memory://b.floe'), - rows(schema(str_schema('b'), int_schema('a')), row(['b' => 'two', 'a' => 2])), + array_to_rows([['b' => 'two', 'a' => 2]], schema(str_schema('b'), int_schema('a'))), ); - (new FloeMerger($fs))->merge([path('memory://a.floe'), path('memory://b.floe')], path('memory://out.floe')); + (new FloeMerger($fs, new AdaptiveBackend()))->merge([ + path('memory://a.floe'), + path('memory://b.floe'), + ], path('memory://out.floe')); static::assertSame( [ @@ -124,18 +152,26 @@ public function test_merge_of_same_columns_in_different_order_with_mixed_types_r public function test_empty_source_contributes_no_rows(): void { $fs = memory_filesystem(); - FloeStreamReaderContext::write($fs, path('memory://a.floe'), rows(schema(int_schema('id')), row(['id' => 1]))); + FloeStreamReaderContext::write( + $fs, + path('memory://a.floe'), + array_to_rows([['id' => 1]], schema(int_schema('id'))), + ); FloeStreamReaderContext::write($fs, path('memory://empty.floe'), rows(schema())); - FloeStreamReaderContext::write($fs, path('memory://b.floe'), rows(schema(int_schema('id')), row(['id' => 2]))); + FloeStreamReaderContext::write( + $fs, + path('memory://b.floe'), + array_to_rows([['id' => 2]], schema(int_schema('id'))), + ); - (new FloeMerger($fs))->merge([ + (new FloeMerger($fs, new AdaptiveBackend()))->merge([ path('memory://a.floe'), path('memory://empty.floe'), path('memory://b.floe'), ], path('memory://out.floe')); static::assertEquals( - rows(schema(int_schema('id')), row(['id' => 1]), row(['id' => 2])), + array_to_rows([['id' => 1], ['id' => 2]], schema(int_schema('id'))), FloeStreamReaderContext::readAll($fs, path('memory://out.floe')), ); } @@ -145,17 +181,21 @@ public function test_empty_sources_list_throws(): void $this->expectException(FloeException::class); $this->expectExceptionMessage('at least one source'); - (new FloeMerger(memory_filesystem()))->merge([], path('memory://out.floe')); + (new FloeMerger(memory_filesystem(), new AdaptiveBackend()))->merge([], path('memory://out.floe')); } public function test_merging_a_source_that_adds_a_non_nullable_column_throws(): void { $fs = memory_filesystem(); - FloeStreamReaderContext::write($fs, path('memory://a.floe'), rows(schema(int_schema('id')), row(['id' => 1]))); + FloeStreamReaderContext::write( + $fs, + path('memory://a.floe'), + array_to_rows([['id' => 1]], schema(int_schema('id'))), + ); FloeStreamReaderContext::write( $fs, path('memory://b.floe'), - rows(schema(int_schema('id'), str_schema('email')), row(['id' => 2, 'email' => 'x@y'])), + array_to_rows([['id' => 2, 'email' => 'x@y']], schema(int_schema('id'), str_schema('email'))), ); // "email" is non-nullable in B but absent from A - padding A's rows with null would @@ -163,7 +203,10 @@ public function test_merging_a_source_that_adds_a_non_nullable_column_throws(): $this->expectException(IncompatibleSchemaException::class); $this->expectExceptionMessage('Floe merge cannot reconcile the schema of'); - (new FloeMerger($fs))->merge([path('memory://a.floe'), path('memory://b.floe')], path('memory://out.floe')); + (new FloeMerger($fs, new AdaptiveBackend()))->merge([ + path('memory://a.floe'), + path('memory://b.floe'), + ], path('memory://out.floe')); } public function test_merging_a_source_that_drops_a_non_nullable_column_throws(): void @@ -172,14 +215,21 @@ public function test_merging_a_source_that_drops_a_non_nullable_column_throws(): FloeStreamReaderContext::write( $fs, path('memory://a.floe'), - rows(schema(int_schema('id'), str_schema('email')), row(['id' => 1, 'email' => 'x@y'])), + array_to_rows([['id' => 1, 'email' => 'x@y']], schema(int_schema('id'), str_schema('email'))), + ); + FloeStreamReaderContext::write( + $fs, + path('memory://b.floe'), + array_to_rows([['id' => 2]], schema(int_schema('id'))), ); - FloeStreamReaderContext::write($fs, path('memory://b.floe'), rows(schema(int_schema('id')), row(['id' => 2]))); $this->expectException(IncompatibleSchemaException::class); $this->expectExceptionMessage('Floe merge cannot reconcile the schema of'); - (new FloeMerger($fs))->merge([path('memory://a.floe'), path('memory://b.floe')], path('memory://out.floe')); + (new FloeMerger($fs, new AdaptiveBackend()))->merge([ + path('memory://a.floe'), + path('memory://b.floe'), + ], path('memory://out.floe')); } public function test_merging_sources_with_a_nullable_column_pads_missing_rows(): void @@ -188,20 +238,29 @@ public function test_merging_sources_with_a_nullable_column_pads_missing_rows(): FloeStreamReaderContext::write( $fs, path('memory://a.floe'), - rows(schema(int_schema('id'), str_schema('email', nullable: true)), row(['id' => 1, 'email' => null])), + array_to_rows( + [['id' => 1, 'email' => null]], + schema(int_schema('id'), str_schema('email', nullable: true)), + ), + ); + FloeStreamReaderContext::write( + $fs, + path('memory://b.floe'), + array_to_rows([['id' => 2]], schema(int_schema('id'))), ); - FloeStreamReaderContext::write($fs, path('memory://b.floe'), rows(schema(int_schema('id')), row(['id' => 2]))); - (new FloeMerger($fs))->merge([path('memory://a.floe'), path('memory://b.floe')], path('memory://out.floe')); + (new FloeMerger($fs, new AdaptiveBackend()))->merge([ + path('memory://a.floe'), + path('memory://b.floe'), + ], path('memory://out.floe')); // "email" is nullable in A, so B's rows are legitimately padded with null on read-back. FloeStreamReaderContext::write( $fs, path('memory://reference.floe'), - rows( + array_to_rows( + [['id' => 1, 'email' => null], ['id' => 2]], schema(int_schema('id'), str_schema('email', nullable: true)), - row(['id' => 1, 'email' => null]), - row(['id' => 2]), ), ); @@ -214,16 +273,26 @@ public function test_merging_sources_with_a_nullable_column_pads_missing_rows(): public function test_metadata_is_merged_with_new_keys_winning(): void { $fs = memory_filesystem(); - FloeStreamReaderContext::write($fs, path('memory://a.floe'), rows(schema(int_schema('id')), row(['id' => 1])), [ - 'a' => '1', - 'shared' => 'from-a', - ]); - FloeStreamReaderContext::write($fs, path('memory://b.floe'), rows(schema(int_schema('id')), row(['id' => 2])), [ - 'b' => '2', - 'shared' => 'from-b', - ]); + FloeStreamReaderContext::write( + $fs, + path('memory://a.floe'), + array_to_rows([['id' => 1]], schema(int_schema('id'))), + [ + 'a' => '1', + 'shared' => 'from-a', + ], + ); + FloeStreamReaderContext::write( + $fs, + path('memory://b.floe'), + array_to_rows([['id' => 2]], schema(int_schema('id'))), + [ + 'b' => '2', + 'shared' => 'from-b', + ], + ); - (new FloeMerger($fs))->merge( + (new FloeMerger($fs, new AdaptiveBackend()))->merge( [path('memory://a.floe'), path('memory://b.floe')], path('memory://out.floe'), metadata: Metadata::fromArray(['shared' => 'override']), @@ -231,7 +300,7 @@ public function test_metadata_is_merged_with_new_keys_winning(): void static::assertSame( ['a' => '1', 'shared' => 'override', 'b' => '2'], - (new FloeReader($fs)) + (new FloeReader($fs, new AdaptiveBackend())) ->read(path('memory://out.floe')) ->metadata() ->normalize(), @@ -243,7 +312,9 @@ public function test_missing_source_throws(): void $this->expectException(FloeException::class); $this->expectExceptionMessage('does not exist'); - (new FloeMerger(memory_filesystem()))->merge([path('memory://missing.floe')], path('memory://out.floe')); + (new FloeMerger(memory_filesystem(), new AdaptiveBackend()))->merge([path( + 'memory://missing.floe', + )], path('memory://out.floe')); } public function test_offset_read_after_merge_seeks_across_sources(): void @@ -252,24 +323,25 @@ public function test_offset_read_after_merge_seeks_across_sources(): void FloeStreamReaderContext::write( $fs, path('memory://a.floe'), - rows(schema(int_schema('id')), row(['id' => 1]), row(['id' => 2])), + array_to_rows([['id' => 1], ['id' => 2]], schema(int_schema('id'))), ); FloeStreamReaderContext::write( $fs, path('memory://b.floe'), - rows(schema(int_schema('id')), row(['id' => 3]), row(['id' => 4])), + array_to_rows([['id' => 3], ['id' => 4]], schema(int_schema('id'))), ); - (new FloeMerger($fs))->merge([path('memory://a.floe'), path('memory://b.floe')], path('memory://out.floe')); + (new FloeMerger($fs, new AdaptiveBackend()))->merge([ + path('memory://a.floe'), + path('memory://b.floe'), + ], path('memory://out.floe')); $read = []; - foreach ((new FloeReader($fs)) + foreach ((new FloeReader($fs, new AdaptiveBackend())) ->read(path('memory://out.floe')) ->rows(batchSize: 100, offset: 3) as $batch) { - foreach ($batch as $r) { - $read[] = $r->get('id'); - } + $read = [...$read, ...$batch->column('id')->values()]; } static::assertSame([4], $read); @@ -278,10 +350,10 @@ public function test_offset_read_after_merge_seeks_across_sources(): void public function test_single_source_is_a_copy(): void { $fs = memory_filesystem(); - $value = rows(schema(int_schema('id')), row(['id' => 1]), row(['id' => 2])); + $value = array_to_rows([['id' => 1], ['id' => 2]], schema(int_schema('id'))); FloeStreamReaderContext::write($fs, path('memory://a.floe'), $value); - (new FloeMerger($fs))->merge([path('memory://a.floe')], path('memory://out.floe')); + (new FloeMerger($fs, new AdaptiveBackend()))->merge([path('memory://a.floe')], path('memory://out.floe')); static::assertEquals($value, FloeStreamReaderContext::readAll($fs, path('memory://out.floe'))); } @@ -289,18 +361,30 @@ public function test_single_source_is_a_copy(): void public function test_splices_three_files(): void { $fs = memory_filesystem(); - FloeStreamReaderContext::write($fs, path('memory://a.floe'), rows(schema(int_schema('id')), row(['id' => 1]))); - FloeStreamReaderContext::write($fs, path('memory://b.floe'), rows(schema(int_schema('id')), row(['id' => 2]))); - FloeStreamReaderContext::write($fs, path('memory://c.floe'), rows(schema(int_schema('id')), row(['id' => 3]))); + FloeStreamReaderContext::write( + $fs, + path('memory://a.floe'), + array_to_rows([['id' => 1]], schema(int_schema('id'))), + ); + FloeStreamReaderContext::write( + $fs, + path('memory://b.floe'), + array_to_rows([['id' => 2]], schema(int_schema('id'))), + ); + FloeStreamReaderContext::write( + $fs, + path('memory://c.floe'), + array_to_rows([['id' => 3]], schema(int_schema('id'))), + ); - (new FloeMerger($fs))->merge([ + (new FloeMerger($fs, new AdaptiveBackend()))->merge([ path('memory://a.floe'), path('memory://b.floe'), path('memory://c.floe'), ], path('memory://out.floe')); static::assertEquals( - rows(schema(int_schema('id')), row(['id' => 1]), row(['id' => 2]), row(['id' => 3])), + array_to_rows([['id' => 1], ['id' => 2], ['id' => 3]], schema(int_schema('id'))), FloeStreamReaderContext::readAll($fs, path('memory://out.floe')), ); } @@ -311,18 +395,21 @@ public function test_splices_two_files_with_the_same_schema(): void FloeStreamReaderContext::write( $fs, path('memory://a.floe'), - rows(schema(int_schema('id')), row(['id' => 1]), row(['id' => 2])), + array_to_rows([['id' => 1], ['id' => 2]], schema(int_schema('id'))), ); FloeStreamReaderContext::write( $fs, path('memory://b.floe'), - rows(schema(int_schema('id')), row(['id' => 3]), row(['id' => 4])), + array_to_rows([['id' => 3], ['id' => 4]], schema(int_schema('id'))), ); - (new FloeMerger($fs))->merge([path('memory://a.floe'), path('memory://b.floe')], path('memory://out.floe')); + (new FloeMerger($fs, new AdaptiveBackend()))->merge([ + path('memory://a.floe'), + path('memory://b.floe'), + ], path('memory://out.floe')); static::assertEquals( - rows(schema(int_schema('id')), row(['id' => 1]), row(['id' => 2]), row(['id' => 3]), row(['id' => 4])), + array_to_rows([['id' => 1], ['id' => 2], ['id' => 3], ['id' => 4]], schema(int_schema('id'))), FloeStreamReaderContext::readAll($fs, path('memory://out.floe')), ); } @@ -330,28 +417,41 @@ public function test_splices_two_files_with_the_same_schema(): void public function test_type_conflict_throws(): void { $fs = memory_filesystem(); - FloeStreamReaderContext::write($fs, path('memory://a.floe'), rows(schema(int_schema('id')), row(['id' => 1]))); + FloeStreamReaderContext::write( + $fs, + path('memory://a.floe'), + array_to_rows([['id' => 1]], schema(int_schema('id'))), + ); FloeStreamReaderContext::write( $fs, path('memory://b.floe'), - rows(schema(str_schema('id')), row(['id' => 'x'])), + array_to_rows([['id' => 'x']], schema(str_schema('id'))), ); $this->expectException(IncompatibleSchemaException::class); $this->expectExceptionMessage('Floe merge cannot reconcile the schema of'); - (new FloeMerger($fs))->merge([path('memory://a.floe'), path('memory://b.floe')], path('memory://out.floe')); + (new FloeMerger($fs, new AdaptiveBackend()))->merge([ + path('memory://a.floe'), + path('memory://b.floe'), + ], path('memory://out.floe')); } public function test_unsized_source_throws(): void { $fs = memory_filesystem(); - FloeStreamReaderContext::write($fs, path('memory://a.floe'), rows(schema(int_schema('id')), row(['id' => 1]))); + FloeStreamReaderContext::write( + $fs, + path('memory://a.floe'), + array_to_rows([['id' => 1]], schema(int_schema('id'))), + ); $this->expectException(FloeException::class); $this->expectExceptionMessage('does not report its size'); - (new FloeMerger(new UnsizedFilesystem($fs)))->merge([path('memory://a.floe')], path('memory://out.floe')); + (new FloeMerger(new UnsizedFilesystem($fs), new AdaptiveBackend()))->merge([path( + 'memory://a.floe', + )], path('memory://out.floe')); } public function test_source_smaller_than_header_and_trailer_throws(): void @@ -365,7 +465,7 @@ public function test_source_smaller_than_header_and_trailer_throws(): void $this->expectException(FloeException::class); $this->expectExceptionMessage('too small'); - (new FloeMerger($fs))->merge([path('memory://tiny.floe')], path('memory://out.floe')); + (new FloeMerger($fs, new AdaptiveBackend()))->merge([path('memory://tiny.floe')], path('memory://out.floe')); } public function test_non_noop_codec_source_throws(): void @@ -379,7 +479,7 @@ public function test_non_noop_codec_source_throws(): void $this->expectException(FloeException::class); $this->expectExceptionMessage('written with codec 0x05'); - (new FloeMerger($fs))->merge([path('memory://codec.floe')], path('memory://out.floe')); + (new FloeMerger($fs, new AdaptiveBackend()))->merge([path('memory://codec.floe')], path('memory://out.floe')); } public function test_source_with_footer_larger_than_file_throws(): void @@ -393,7 +493,7 @@ public function test_source_with_footer_larger_than_file_throws(): void $this->expectException(FloeException::class); $this->expectExceptionMessage('footer does not fit'); - (new FloeMerger($fs))->merge([path('memory://torn.floe')], path('memory://out.floe')); + (new FloeMerger($fs, new AdaptiveBackend()))->merge([path('memory://torn.floe')], path('memory://out.floe')); } public function test_merge_of_different_datetime_zones_widens_to_utc(): void @@ -404,15 +504,15 @@ public function test_merge_of_different_datetime_zones_widens_to_utc(): void FloeStreamReaderContext::write( $fs, path('memory://warsaw.floe'), - rows(schema(datetime_schema('at', zone: 'Europe/Warsaw')), row(['at' => $warsaw])), + array_to_rows([['at' => $warsaw]], schema(datetime_schema('at', zone: 'Europe/Warsaw'))), ); FloeStreamReaderContext::write( $fs, path('memory://utc.floe'), - rows(schema(datetime_schema('at')), row(['at' => $utc])), + array_to_rows([['at' => $utc]], schema(datetime_schema('at'))), ); - (new FloeMerger($fs))->merge([ + (new FloeMerger($fs, new AdaptiveBackend()))->merge([ path('memory://warsaw.floe'), path('memory://utc.floe'), ], path('memory://merged.floe')); @@ -424,8 +524,8 @@ public function test_merge_of_different_datetime_zones_widens_to_utc(): void FloeStreamReaderContext::footer($fs, path('memory://merged.floe'))->schema()->get('at')->type()->toString(), ); static::assertSame([$warsaw->getTimestamp(), $utc->getTimestamp()], [ - type_datetime()->assert($merged->all()[0]->get('at'))->getTimestamp(), - type_datetime()->assert($merged->all()[1]->get('at'))->getTimestamp(), + type_datetime()->assert($merged->column('at')->value(0))->getTimestamp(), + type_datetime()->assert($merged->column('at')->value(1))->getTimestamp(), ]); } } diff --git a/src/core/etl/tests/Flow/Floe/Tests/Unit/FloeReaderTest.php b/src/core/etl/tests/Flow/Floe/Tests/Unit/FloeReaderTest.php index e054630a2b..abe2a24bfa 100644 --- a/src/core/etl/tests/Flow/Floe/Tests/Unit/FloeReaderTest.php +++ b/src/core/etl/tests/Flow/Floe/Tests/Unit/FloeReaderTest.php @@ -4,7 +4,7 @@ namespace Flow\Floe\Tests\Unit; -use Flow\ETL\Row\AdaptiveRowHydrator; +use Flow\ETL\Column\AdaptiveBackend; use Flow\ETL\Rows; use Flow\ETL\Schema\Metadata; use Flow\Floe\Exception\FloeException; @@ -13,8 +13,6 @@ use Flow\Floe\FloeWriter; use Flow\Floe\Format; use Flow\Floe\Options; -use Flow\Floe\PhpFloeEncoder; -use Flow\Floe\Tests\Context\FloeSchemaContext; use Flow\Floe\Tests\Context\FloeStreamReaderContext; use Flow\Floe\Tests\Double\CodecStub; use Flow\Floe\Tests\Double\UnsizedFilesystem; @@ -23,13 +21,11 @@ use PHPUnit\Framework\TestCase; use function array_map; +use function array_merge; use function array_slice; -use function count; +use function Flow\ETL\DSL\array_to_rows; use function Flow\ETL\DSL\int_schema; -use function Flow\ETL\DSL\row; -use function Flow\ETL\DSL\rows; use function Flow\ETL\DSL\schema; -use function Flow\ETL\DSL\schema_from_json; use function Flow\ETL\DSL\str_schema; use function Flow\Filesystem\DSL\memory_filesystem; use function Flow\Filesystem\DSL\path; @@ -45,19 +41,19 @@ public function test_all_entry_types_round_trip(): void $path = path('memory://all-types.floe'); $rows = RowsMother::withAllEntryTypes(); - $writer = new FloeWriter($filesystem, $rows->schema()); + $writer = new FloeWriter($filesystem, $rows->schema(), new AdaptiveBackend()); $writer->create($path); $writer->write($rows); $writer->close(); $batches = iterator_to_array( - (new FloeReader($filesystem)) + (new FloeReader($filesystem, new AdaptiveBackend())) ->read($path) ->rows(), ); static::assertCount(1, $batches); - static::assertEquals($rows->all(), $batches[0]->all()); + static::assertEquals($rows->toArray(), $batches[0]->toArray()); } public function test_batches_follow_requested_size(): void @@ -65,20 +61,19 @@ public function test_batches_follow_requested_size(): void $filesystem = memory_filesystem(); $path = path('memory://batches.floe'); - $data = rows( - schema(int_schema('id')), - row(['id' => 1]), - row(['id' => 2]), - row(['id' => 3]), - row(['id' => 4]), - row(['id' => 5]), - ); - $writer = new FloeWriter($filesystem, $data->schema()); + $data = array_to_rows([ + ['id' => 1], + ['id' => 2], + ['id' => 3], + ['id' => 4], + ['id' => 5], + ], schema(int_schema('id'))); + $writer = new FloeWriter($filesystem, $data->schema(), new AdaptiveBackend()); $writer->create($path); $writer->write($data); $writer->close(); - $reader = (new FloeReader($filesystem))->read($path); + $reader = (new FloeReader($filesystem, new AdaptiveBackend()))->read($path); $sizes = /** @@ -88,7 +83,7 @@ public function test_batches_follow_requested_size(): void */ static fn(int $batchSize): array => array_map( static fn(Rows $batch): int => $batch->count(), - iterator_to_array($reader->rows($batchSize)), + iterator_to_array($reader->rows($batchSize), false), ); static::assertSame([2, 2, 1], $sizes(2)); @@ -102,19 +97,19 @@ public function test_limit_caps_returned_rows(): void $filesystem = memory_filesystem(); $path = path('memory://limit.floe'); - $data = rows(schema(int_schema('id')), row(['id' => 1]), row(['id' => 2]), row(['id' => 3]), row(['id' => 4])); - $writer = new FloeWriter($filesystem, $data->schema()); + $data = array_to_rows([['id' => 1], ['id' => 2], ['id' => 3], ['id' => 4]], schema(int_schema('id'))); + $writer = new FloeWriter($filesystem, $data->schema(), new AdaptiveBackend()); $writer->create($path); $writer->write($data); $writer->close(); $ids = []; - foreach ((new FloeReader($filesystem)) + foreach ((new FloeReader($filesystem, new AdaptiveBackend())) ->read($path) ->rows(batchSize: 100, offset: 0, limit: 2) as $batch) { - foreach ($batch->all() as $row) { - $ids[] = $row->get('id'); + foreach ($batch->toArray() as $row) { + $ids[] = $row['id']; } } @@ -126,27 +121,27 @@ public function test_limit_stops_within_a_batch(): void $filesystem = memory_filesystem(); $path = path('memory://limit-batch.floe'); - $data = rows(schema(int_schema('id')), row(['id' => 1]), row(['id' => 2]), row(['id' => 3])); - $writer = new FloeWriter($filesystem, $data->schema()); + $data = array_to_rows([['id' => 1], ['id' => 2], ['id' => 3]], schema(int_schema('id'))); + $writer = new FloeWriter($filesystem, $data->schema(), new AdaptiveBackend()); $writer->create($path); $writer->write($data); $writer->close(); $batches = iterator_to_array( - (new FloeReader($filesystem)) + (new FloeReader($filesystem, new AdaptiveBackend())) ->read($path) ->rows(batchSize: 100, limit: 2), ); static::assertCount(1, $batches); - static::assertCount(2, $batches[0]->all()); + static::assertCount(2, $batches[0]); } public function test_negative_offset_throws(): void { $filesystem = memory_filesystem(); $path = path('memory://neg-offset.floe'); - $writer = new FloeWriter($filesystem, schema()); + $writer = new FloeWriter($filesystem, schema(), new AdaptiveBackend()); $writer->create($path); $writer->close(); @@ -154,7 +149,7 @@ public function test_negative_offset_throws(): void $this->expectExceptionMessage('offset must be greater or equal to 0'); iterator_to_array( - (new FloeReader($filesystem)) + (new FloeReader($filesystem, new AdaptiveBackend())) ->read($path) ->rows(offset: -1), ); @@ -164,7 +159,7 @@ public function test_non_positive_limit_throws(): void { $filesystem = memory_filesystem(); $path = path('memory://zero-limit.floe'); - $writer = new FloeWriter($filesystem, schema()); + $writer = new FloeWriter($filesystem, schema(), new AdaptiveBackend()); $writer->create($path); $writer->close(); @@ -172,7 +167,7 @@ public function test_non_positive_limit_throws(): void $this->expectExceptionMessage('limit must be greater than 0'); iterator_to_array( - (new FloeReader($filesystem)) + (new FloeReader($filesystem, new AdaptiveBackend())) ->read($path) ->rows(limit: 0), ); @@ -183,19 +178,19 @@ public function test_offset_skips_leading_rows_within_a_single_section(): void $filesystem = memory_filesystem(); $path = path('memory://offset-single.floe'); - $data = rows(schema(int_schema('id')), row(['id' => 1]), row(['id' => 2]), row(['id' => 3]), row(['id' => 4])); - $writer = new FloeWriter($filesystem, $data->schema()); + $data = array_to_rows([['id' => 1], ['id' => 2], ['id' => 3], ['id' => 4]], schema(int_schema('id'))); + $writer = new FloeWriter($filesystem, $data->schema(), new AdaptiveBackend()); $writer->create($path); $writer->write($data); $writer->close(); $ids = []; - foreach ((new FloeReader($filesystem)) + foreach ((new FloeReader($filesystem, new AdaptiveBackend())) ->read($path) ->rows(offset: 2) as $batch) { - foreach ($batch->all() as $row) { - $ids[] = $row->get('id'); + foreach ($batch->toArray() as $row) { + $ids[] = $row['id']; } } @@ -207,17 +202,19 @@ public function test_offset_skips_whole_sections(): void $filesystem = memory_filesystem(); $path = path('memory://offset-sections.floe'); - $data = rows( + $data = array_to_rows( + [ + ['id' => 1], + ['id' => 2], + ['id' => 3], + ['id' => 4, 'email' => 'a'], + ['id' => 5, 'email' => 'b'], + ['id' => 6], + ['id' => 7], + ], schema(int_schema('id'), str_schema('email', nullable: true)), - row(['id' => 1]), - row(['id' => 2]), - row(['id' => 3]), - row(['id' => 4, 'email' => 'a']), - row(['id' => 5, 'email' => 'b']), - row(['id' => 6]), - row(['id' => 7]), ); - $writer = new FloeWriter($filesystem, $data->schema()); + $writer = new FloeWriter($filesystem, $data->schema(), new AdaptiveBackend()); $writer->create($path); $writer->write($data); $writer->close(); @@ -226,13 +223,13 @@ public function test_offset_skips_whole_sections(): void $names = []; $emails = []; - foreach ((new FloeReader($filesystem)) + foreach ((new FloeReader($filesystem, new AdaptiveBackend())) ->read($path) ->rows(offset: 4) as $batch) { - foreach ($batch->all() as $row) { - $ids[] = $row->get('id'); - $names[] = $row->names(); - $emails[] = $row->get('email'); + foreach ($batch->toArray() as $row) { + $ids[] = $row['id']; + $names[] = array_keys($row); + $emails[] = $row['email']; } } @@ -247,8 +244,8 @@ public function test_offset_beyond_total_rows_yields_nothing(): void $filesystem = memory_filesystem(); $path = path('memory://offset-past-end.floe'); - $data = rows(schema(int_schema('id')), row(['id' => 1]), row(['id' => 2])); - $writer = new FloeWriter($filesystem, $data->schema()); + $data = array_to_rows([['id' => 1], ['id' => 2]], schema(int_schema('id'))); + $writer = new FloeWriter($filesystem, $data->schema(), new AdaptiveBackend()); $writer->create($path); $writer->write($data); $writer->close(); @@ -256,7 +253,7 @@ public function test_offset_beyond_total_rows_yields_nothing(): void static::assertSame( [], iterator_to_array( - (new FloeReader($filesystem)) + (new FloeReader($filesystem, new AdaptiveBackend())) ->read($path) ->rows(offset: 2), ), @@ -264,7 +261,7 @@ public function test_offset_beyond_total_rows_yields_nothing(): void static::assertSame( [], iterator_to_array( - (new FloeReader($filesystem)) + (new FloeReader($filesystem, new AdaptiveBackend())) ->read($path) ->rows(offset: 5), ), @@ -276,8 +273,8 @@ public function test_offset_respects_batch_size(): void $filesystem = memory_filesystem(); $path = path('memory://offset-batches.floe'); - $data = rows(schema(int_schema('id')), row(['id' => 1]), row(['id' => 2]), row(['id' => 3]), row(['id' => 4])); - $writer = new FloeWriter($filesystem, $data->schema()); + $data = array_to_rows([['id' => 1], ['id' => 2], ['id' => 3], ['id' => 4]], schema(int_schema('id'))); + $writer = new FloeWriter($filesystem, $data->schema(), new AdaptiveBackend()); $writer->create($path); $writer->write($data); $writer->close(); @@ -285,7 +282,7 @@ public function test_offset_respects_batch_size(): void $sizes = array_map( static fn(Rows $batch): int => $batch->count(), iterator_to_array( - (new FloeReader($filesystem)) + (new FloeReader($filesystem, new AdaptiveBackend())) ->read($path) ->rows(batchSize: 1, offset: 1), ), @@ -299,26 +296,25 @@ public function test_offset_and_limit_combined(): void $filesystem = memory_filesystem(); $path = path('memory://offset-limit.floe'); - $data = rows( - schema(int_schema('id')), - row(['id' => 1]), - row(['id' => 2]), - row(['id' => 3]), - row(['id' => 4]), - row(['id' => 5]), - ); - $writer = new FloeWriter($filesystem, $data->schema()); + $data = array_to_rows([ + ['id' => 1], + ['id' => 2], + ['id' => 3], + ['id' => 4], + ['id' => 5], + ], schema(int_schema('id'))); + $writer = new FloeWriter($filesystem, $data->schema(), new AdaptiveBackend()); $writer->create($path); $writer->write($data); $writer->close(); $ids = []; - foreach ((new FloeReader($filesystem)) + foreach ((new FloeReader($filesystem, new AdaptiveBackend())) ->read($path) ->rows(batchSize: 100, offset: 1, limit: 2) as $batch) { - foreach ($batch->all() as $row) { - $ids[] = $row->get('id'); + foreach ($batch->toArray() as $row) { + $ids[] = $row['id']; } } @@ -330,27 +326,29 @@ public function test_offset_matches_tail_of_full_read(): void $filesystem = memory_filesystem(); $path = path('memory://offset-tail.floe'); - $data = rows( + $data = array_to_rows( + [ + ['id' => 1], + ['id' => 2], + ['id' => 3], + ['id' => 4, 'email' => 'a'], + ['id' => 5, 'email' => 'b'], + ['id' => 6], + ['id' => 7], + ], schema(int_schema('id'), str_schema('email', nullable: true)), - row(['id' => 1]), - row(['id' => 2]), - row(['id' => 3]), - row(['id' => 4, 'email' => 'a']), - row(['id' => 5, 'email' => 'b']), - row(['id' => 6]), - row(['id' => 7]), ); - $writer = new FloeWriter($filesystem, $data->schema()); + $writer = new FloeWriter($filesystem, $data->schema(), new AdaptiveBackend()); $writer->create($path); $writer->write($data); $writer->close(); - $reader = (new FloeReader($filesystem))->read($path); + $reader = (new FloeReader($filesystem, new AdaptiveBackend()))->read($path); $full = []; foreach ($reader->rows() as $batch) { - foreach ($batch->all() as $row) { - $full[] = $row->get('id'); + foreach ($batch->toArray() as $row) { + $full[] = $row['id']; } } @@ -358,8 +356,8 @@ public function test_offset_matches_tail_of_full_read(): void $tail = []; foreach ($reader->rows(offset: $offset) as $batch) { - foreach ($batch->all() as $row) { - $tail[] = $row->get('id'); + foreach ($batch->toArray() as $row) { + $tail[] = $row['id']; } } @@ -372,19 +370,24 @@ public function test_offset_reads_through_a_custom_identity_codec(): void $filesystem = memory_filesystem(); $path = path('memory://offset-codec.floe'); - $data = rows(schema(int_schema('id')), row(['id' => 1]), row(['id' => 2]), row(['id' => 3])); - $writer = new FloeWriter($filesystem, $data->schema(), new Options(codec: new CodecStub(0x00))); + $data = array_to_rows([['id' => 1], ['id' => 2], ['id' => 3]], schema(int_schema('id'))); + $writer = new FloeWriter( + $filesystem, + $data->schema(), + new AdaptiveBackend(), + new Options(codec: new CodecStub(0x00)), + ); $writer->create($path); $writer->write($data); $writer->close(); $ids = []; - foreach ((new FloeReader($filesystem, new CodecStub(0x00))) + foreach ((new FloeReader($filesystem, new AdaptiveBackend(), new CodecStub(0x00))) ->read($path) ->rows(offset: 1) as $batch) { - foreach ($batch->all() as $extractedRow) { - $ids[] = $extractedRow->get('id'); + foreach ($batch->toArray() as $extractedRow) { + $ids[] = $extractedRow['id']; } } @@ -398,16 +401,16 @@ public function test_corrupt_row_body_throws_wrapped_exception(): void $footerJson = FooterMother::footer(schema: schema(int_schema('id'))->normalize())->toJson(); $stream = $filesystem->writeTo($path); $stream->append( - Format::header(0x00) . Format::frame(Format::FRAME_ROW, "\xEE") + Format::header(0x00) . Format::frame(Format::FRAME_BATCH, "\xEE") . Format::frame(Format::FRAME_FOOTER, $footerJson . Format::trailer(strlen($footerJson))), ); $stream->close(); $this->expectException(FloeException::class); - $this->expectExceptionMessage('unknown value flag'); + $this->expectExceptionMessage('Floe BATCH frame is truncated, its directory is incomplete'); iterator_to_array( - (new FloeReader($filesystem)) + (new FloeReader($filesystem, new AdaptiveBackend())) ->read($path) ->rows(), ); @@ -429,7 +432,7 @@ public function test_corrupted_frame_chain_throws(): void $this->expectExceptionMessage('frame body is incomplete'); iterator_to_array( - (new FloeReader($filesystem)) + (new FloeReader($filesystem, new AdaptiveBackend())) ->read($path) ->rows(), ); @@ -440,26 +443,29 @@ public function test_dead_footers_of_appended_files_are_skipped(): void $filesystem = memory_filesystem(); $path = path('memory://appended.floe'); - $created = rows(schema(int_schema('id')), row(['id' => 1])); - $writer = new FloeWriter($filesystem, $created->schema()); + $created = array_to_rows([['id' => 1]], schema(int_schema('id'))); + $writer = new FloeWriter($filesystem, $created->schema(), new AdaptiveBackend()); $writer->create($path); $writer->write($created); $writer->close(); - $appended = rows(schema(int_schema('id')), row(['id' => 2])); - $writer = new FloeWriter($filesystem, $appended->schema()); + $appended = array_to_rows([['id' => 2]], schema(int_schema('id'))); + $writer = new FloeWriter($filesystem, $appended->schema(), new AdaptiveBackend()); $writer->append($path); $writer->write($appended); $writer->close(); $batches = iterator_to_array( - (new FloeReader($filesystem)) + (new FloeReader($filesystem, new AdaptiveBackend())) ->read($path) ->rows(), + false, ); - static::assertCount(1, $batches); - static::assertSame([1, 2], array_map(static fn($row) => $row->get('id'), $batches[0]->all())); + static::assertSame( + [[['id' => 1]], [['id' => 2]]], + array_map(static fn(Rows $batch) => $batch->toArray(), $batches), + ); } public function test_empty_file_yields_no_batches(): void @@ -467,13 +473,13 @@ public function test_empty_file_yields_no_batches(): void $filesystem = memory_filesystem(); $path = path('memory://empty.floe'); - $writer = new FloeWriter($filesystem, schema()); + $writer = new FloeWriter($filesystem, schema(), new AdaptiveBackend()); $writer->create($path); $writer->close(); - $reader = (new FloeReader($filesystem))->read($path); + $reader = (new FloeReader($filesystem, new AdaptiveBackend()))->read($path); - static::assertSame([], iterator_to_array($reader->rows())); + static::assertSame([], iterator_to_array($reader->rows(), false)); static::assertSame(0, $reader->totalRows()); static::assertCount(0, $reader->schema()->definitions()); } @@ -487,39 +493,39 @@ public function test_evolved_file_rows_match_merged_schema(): void $baseFile = path('memory://evolved-base.floe'); $evolvedFile = path('memory://evolved-new.floe'); - $baseRows = rows(schema(int_schema('id')), row(['id' => 1]), row(['id' => 2])); - $writer = new FloeWriter($filesystem, $baseRows->schema()); + $baseRows = array_to_rows([['id' => 1], ['id' => 2]], schema(int_schema('id'))); + $writer = new FloeWriter($filesystem, $baseRows->schema(), new AdaptiveBackend()); $writer->create($baseFile); $writer->write($baseRows); $writer->close(); - $evolvedRows = rows( + $evolvedRows = array_to_rows( + [['id' => 3, 'email' => null], ['id' => 4, 'email' => 'x@flow.php']], schema(int_schema('id'), str_schema('email', nullable: true)), - row(['id' => 3, 'email' => null]), - row(['id' => 4, 'email' => 'x@flow.php']), ); - $writer = new FloeWriter($filesystem, $evolvedRows->schema()); + $writer = new FloeWriter($filesystem, $evolvedRows->schema(), new AdaptiveBackend()); $writer->create($evolvedFile); $writer->write($evolvedRows); $writer->close(); - (new FloeMerger($filesystem))->merge([$baseFile, $evolvedFile], $path); + (new FloeMerger($filesystem, new AdaptiveBackend()))->merge([$baseFile, $evolvedFile], $path); $batches = iterator_to_array( - (new FloeReader($filesystem)) + (new FloeReader($filesystem, new AdaptiveBackend())) ->read($path) ->rows(), + false, ); - $rows = $batches[0]->all(); - - static::assertCount(4, $rows); - - foreach ($rows as $row) { - static::assertSame(['id', 'email'], $row->names()); - } - static::assertNull($rows[0]->get('email')); - static::assertSame('x@flow.php', $rows[3]->get('email')); + static::assertSame( + [ + ['id' => 1, 'email' => null], + ['id' => 2, 'email' => null], + ['id' => 3, 'email' => null], + ['id' => 4, 'email' => 'x@flow.php'], + ], + array_merge(...array_map(static fn(Rows $batch) => $batch->toArray(), $batches)), + ); // the merged schema is what carries nullability now, not a per-cell definition $merged = $batches[0]->schema(); @@ -538,7 +544,7 @@ public function test_footer_length_exceeding_file_throws(): void $this->expectException(FloeException::class); $this->expectExceptionMessage('footer does not fit inside the file'); - (new FloeReader($filesystem)) + (new FloeReader($filesystem, new AdaptiveBackend())) ->read($path) ->footer(); } @@ -554,7 +560,7 @@ public function test_footer_of_file_smaller_than_header_and_trailer_throws(): vo $this->expectException(FloeException::class); $this->expectExceptionMessage('too small'); - (new FloeReader($filesystem)) + (new FloeReader($filesystem, new AdaptiveBackend())) ->read($path) ->footer(); } @@ -564,14 +570,14 @@ public function test_footer_over_unsized_stream_throws(): void $filesystem = memory_filesystem(); $path = path('memory://unsized.floe'); - $writer = new FloeWriter($filesystem, schema()); + $writer = new FloeWriter($filesystem, schema(), new AdaptiveBackend()); $writer->create($path); $writer->close(); $this->expectException(FloeException::class); $this->expectExceptionMessage('requires a sized stream'); - (new FloeReader(new UnsizedFilesystem($filesystem))) + (new FloeReader(new UnsizedFilesystem($filesystem), new AdaptiveBackend())) ->read($path) ->footer(); } @@ -581,13 +587,13 @@ public function test_footer_accessors(): void $filesystem = memory_filesystem(); $path = path('memory://accessors.floe'); - $data = rows(schema(int_schema('id'), str_schema('name')), row(['id' => 1, 'name' => 'x'])); - $writer = new FloeWriter($filesystem, $data->schema()); + $data = array_to_rows([['id' => 1, 'name' => 'x']], schema(int_schema('id'), str_schema('name'))); + $writer = new FloeWriter($filesystem, $data->schema(), new AdaptiveBackend()); $writer->create($path, Metadata::fromArray(['source' => 'unit'])); $writer->write($data); $writer->close(); - $reader = (new FloeReader($filesystem))->read($path); + $reader = (new FloeReader($filesystem, new AdaptiveBackend()))->read($path); static::assertSame(Format::VERSION, $reader->footer()->version); static::assertSame(['source' => 'unit'], $reader->metadata()->normalize()); @@ -600,14 +606,14 @@ public function test_open_with_non_noop_codec_throws(): void { $filesystem = memory_filesystem(); $path = path('memory://x.floe'); - $writer = new FloeWriter($filesystem, schema()); + $writer = new FloeWriter($filesystem, schema(), new AdaptiveBackend()); $writer->create($path); $writer->close(); $this->expectException(FloeException::class); $this->expectExceptionMessage('supports only the no-op codec, got codec 0x09'); - (new FloeReader($filesystem, new CodecStub(0x09)))->read($path); + (new FloeReader($filesystem, new AdaptiveBackend(), new CodecStub(0x09)))->read($path); } public function test_reader_with_mismatched_codec_flags_throws(): void @@ -621,7 +627,7 @@ public function test_reader_with_mismatched_codec_flags_throws(): void $this->expectException(FloeException::class); $this->expectExceptionMessage('written with codec 0x07'); - (new FloeReader($filesystem)) + (new FloeReader($filesystem, new AdaptiveBackend())) ->read($path) ->footer(); } @@ -631,94 +637,25 @@ public function test_rows_through_a_custom_identity_codec(): void $filesystem = memory_filesystem(); $path = path('memory://custom-codec.floe'); - $data = rows(schema(int_schema('id')), row(['id' => 1]), row(['id' => 2])); - $writer = new FloeWriter($filesystem, $data->schema(), new Options(codec: new CodecStub(0x00))); + $data = array_to_rows([['id' => 1], ['id' => 2]], schema(int_schema('id'))); + $writer = new FloeWriter( + $filesystem, + $data->schema(), + new AdaptiveBackend(), + new Options(codec: new CodecStub(0x00)), + ); $writer->create($path); $writer->write($data); $writer->close(); $batches = iterator_to_array( - (new FloeReader($filesystem, new CodecStub(0x00))) + (new FloeReader($filesystem, new AdaptiveBackend(), new CodecStub(0x00))) ->read($path) ->rows(), ); static::assertCount(1, $batches); - static::assertCount(2, $batches[0]->all()); - } - - public function test_rows_through_a_custom_identity_codec_with_long_row_body_throws(): void - { - $filesystem = memory_filesystem(); - $path = path('memory://custom-codec-long-row.floe'); - $schemaBody = FloeSchemaContext::schemaBody(schema(int_schema('id'))); - $rowBody = (new PhpFloeEncoder(schema_from_json( - $schemaBody, - )))->encode((new AdaptiveRowHydrator())->dehydrate(rows(schema(int_schema('id')), row(['id' => 1]))))[0]; - $footerJson = FooterMother::footer(schema: schema(int_schema('id'))->normalize())->toJson(); - $stream = $filesystem->writeTo($path); - $stream->append( - Format::header(0x00) . Format::frame(Format::FRAME_ROW, $rowBody . 'extra-bytes') - . Format::frame(Format::FRAME_FOOTER, $footerJson . Format::trailer(strlen($footerJson))), - ); - $stream->close(); - - $this->expectException(FloeException::class); - $this->expectExceptionMessage('row frame length does not match its content'); - - iterator_to_array( - (new FloeReader($filesystem, new CodecStub(0x00))) - ->read($path) - ->rows(), - ); - } - - public function test_row_frame_not_described_by_the_footer_schema_throws(): void - { - $filesystem = memory_filesystem(); - $path = path('memory://row-first.floe'); - $footerJson = FooterMother::footer()->toJson(); - $stream = $filesystem->writeTo($path); - $stream->append( - Format::header(0x00) . Format::frame(Format::FRAME_ROW, 'row-bytes') - . Format::frame(Format::FRAME_FOOTER, $footerJson . Format::trailer(strlen($footerJson))), - ); - $stream->close(); - - $this->expectException(FloeException::class); - $this->expectExceptionMessage('row frame length does not match its content'); - - iterator_to_array( - (new FloeReader($filesystem)) - ->read($path) - ->rows(), - ); - } - - public function test_row_body_longer_than_hydrated_content_throws(): void - { - $filesystem = memory_filesystem(); - $path = path('memory://long-row.floe'); - $schemaBody = FloeSchemaContext::schemaBody(schema(int_schema('id'))); - $rowBody = (new PhpFloeEncoder(schema_from_json( - $schemaBody, - )))->encode((new AdaptiveRowHydrator())->dehydrate(rows(schema(int_schema('id')), row(['id' => 1]))))[0]; - $footerJson = FooterMother::footer(schema: schema(int_schema('id'))->normalize())->toJson(); - $stream = $filesystem->writeTo($path); - $stream->append( - Format::header(0x00) . Format::frame(Format::FRAME_ROW, $rowBody . 'extra-bytes') - . Format::frame(Format::FRAME_FOOTER, $footerJson . Format::trailer(strlen($footerJson))), - ); - $stream->close(); - - $this->expectException(FloeException::class); - $this->expectExceptionMessage('row frame length does not match its content'); - - iterator_to_array( - (new FloeReader($filesystem)) - ->read($path) - ->rows(), - ); + static::assertCount(2, $batches[0]); } public function test_torn_file_without_footer_throws_in_strict_mode(): void @@ -726,17 +663,15 @@ public function test_torn_file_without_footer_throws_in_strict_mode(): void $filesystem = memory_filesystem(); $path = path('memory://strict-torn.floe'); - FloeStreamReaderContext::writeWithoutFooter( - $filesystem, - $path, - rows(schema(int_schema('id')), row(['id' => 1])), - ); + FloeStreamReaderContext::writeWithoutFooter($filesystem, $path, array_to_rows([[ + 'id' => 1, + ]], schema(int_schema('id')))); $this->expectException(FloeException::class); $this->expectExceptionMessage('magic'); iterator_to_array( - (new FloeReader($filesystem)) + (new FloeReader($filesystem, new AdaptiveBackend())) ->read($path) ->rows(), ); @@ -758,7 +693,7 @@ public function test_unknown_frame_type_throws(): void $this->expectExceptionMessage('unknown frame type 0x55'); iterator_to_array( - (new FloeReader($filesystem)) + (new FloeReader($filesystem, new AdaptiveBackend())) ->read($path) ->rows(), ); @@ -769,26 +704,25 @@ public function test_head_returns_first_rows(): void $filesystem = memory_filesystem(); $path = path('memory://head.floe'); - $data = rows( - schema(int_schema('id')), - row(['id' => 1]), - row(['id' => 2]), - row(['id' => 3]), - row(['id' => 4]), - row(['id' => 5]), - ); - $writer = new FloeWriter($filesystem, $data->schema()); + $data = array_to_rows([ + ['id' => 1], + ['id' => 2], + ['id' => 3], + ['id' => 4], + ['id' => 5], + ], schema(int_schema('id'))); + $writer = new FloeWriter($filesystem, $data->schema(), new AdaptiveBackend()); $writer->create($path); $writer->write($data); $writer->close(); $ids = []; - foreach ((new FloeReader($filesystem)) + foreach ((new FloeReader($filesystem, new AdaptiveBackend())) ->read($path) ->head(3) as $batch) { - foreach ($batch->all() as $row) { - $ids[] = $row->get('id'); + foreach ($batch->toArray() as $row) { + $ids[] = $row['id']; } } @@ -800,19 +734,19 @@ public function test_head_count_beyond_total_returns_all_rows(): void $filesystem = memory_filesystem(); $path = path('memory://head-all.floe'); - $data = rows(schema(int_schema('id')), row(['id' => 1]), row(['id' => 2])); - $writer = new FloeWriter($filesystem, $data->schema()); + $data = array_to_rows([['id' => 1], ['id' => 2]], schema(int_schema('id'))); + $writer = new FloeWriter($filesystem, $data->schema(), new AdaptiveBackend()); $writer->create($path); $writer->write($data); $writer->close(); $ids = []; - foreach ((new FloeReader($filesystem)) + foreach ((new FloeReader($filesystem, new AdaptiveBackend())) ->read($path) ->head(5) as $batch) { - foreach ($batch->all() as $row) { - $ids[] = $row->get('id'); + foreach ($batch->toArray() as $row) { + $ids[] = $row['id']; } } @@ -824,16 +758,16 @@ public function test_head_respects_batch_size(): void $filesystem = memory_filesystem(); $path = path('memory://head-batches.floe'); - $data = rows(schema(int_schema('id')), row(['id' => 1]), row(['id' => 2]), row(['id' => 3]), row(['id' => 4])); - $writer = new FloeWriter($filesystem, $data->schema()); + $data = array_to_rows([['id' => 1], ['id' => 2], ['id' => 3], ['id' => 4]], schema(int_schema('id'))); + $writer = new FloeWriter($filesystem, $data->schema(), new AdaptiveBackend()); $writer->create($path); $writer->write($data); $writer->close(); $sizes = array_map( - static fn(Rows $batch): int => count($batch->all()), + static fn(Rows $batch): int => $batch->count(), iterator_to_array( - (new FloeReader($filesystem)) + (new FloeReader($filesystem, new AdaptiveBackend())) ->read($path) ->head(3, batchSize: 1), ), @@ -846,7 +780,7 @@ public function test_head_non_positive_count_throws(): void { $filesystem = memory_filesystem(); $path = path('memory://head-zero.floe'); - $writer = new FloeWriter($filesystem, schema()); + $writer = new FloeWriter($filesystem, schema(), new AdaptiveBackend()); $writer->create($path); $writer->close(); @@ -854,7 +788,7 @@ public function test_head_non_positive_count_throws(): void $this->expectExceptionMessage('head count must be greater than 0'); iterator_to_array( - (new FloeReader($filesystem)) + (new FloeReader($filesystem, new AdaptiveBackend())) ->read($path) ->head(0), ); @@ -865,17 +799,19 @@ public function test_tail_returns_last_rows_across_sections(): void $filesystem = memory_filesystem(); $path = path('memory://tail-sections.floe'); - $data = rows( + $data = array_to_rows( + [ + ['id' => 1], + ['id' => 2], + ['id' => 3], + ['id' => 4, 'email' => 'a'], + ['id' => 5, 'email' => 'b'], + ['id' => 6], + ['id' => 7], + ], schema(int_schema('id'), str_schema('email', nullable: true)), - row(['id' => 1]), - row(['id' => 2]), - row(['id' => 3]), - row(['id' => 4, 'email' => 'a']), - row(['id' => 5, 'email' => 'b']), - row(['id' => 6]), - row(['id' => 7]), ); - $writer = new FloeWriter($filesystem, $data->schema()); + $writer = new FloeWriter($filesystem, $data->schema(), new AdaptiveBackend()); $writer->create($path); $writer->write($data); $writer->close(); @@ -883,12 +819,12 @@ public function test_tail_returns_last_rows_across_sections(): void $ids = []; $names = []; - foreach ((new FloeReader($filesystem)) + foreach ((new FloeReader($filesystem, new AdaptiveBackend())) ->read($path) ->tail(3) as $batch) { - foreach ($batch->all() as $row) { - $ids[] = $row->get('id'); - $names[] = $row->names(); + foreach ($batch->toArray() as $row) { + $ids[] = $row['id']; + $names[] = array_keys($row); } } @@ -901,19 +837,19 @@ public function test_tail_count_beyond_total_returns_all_rows(): void $filesystem = memory_filesystem(); $path = path('memory://tail-all.floe'); - $data = rows(schema(int_schema('id')), row(['id' => 1]), row(['id' => 2])); - $writer = new FloeWriter($filesystem, $data->schema()); + $data = array_to_rows([['id' => 1], ['id' => 2]], schema(int_schema('id'))); + $writer = new FloeWriter($filesystem, $data->schema(), new AdaptiveBackend()); $writer->create($path); $writer->write($data); $writer->close(); $ids = []; - foreach ((new FloeReader($filesystem)) + foreach ((new FloeReader($filesystem, new AdaptiveBackend())) ->read($path) ->tail(5) as $batch) { - foreach ($batch->all() as $row) { - $ids[] = $row->get('id'); + foreach ($batch->toArray() as $row) { + $ids[] = $row['id']; } } @@ -925,16 +861,16 @@ public function test_tail_respects_batch_size(): void $filesystem = memory_filesystem(); $path = path('memory://tail-batches.floe'); - $data = rows(schema(int_schema('id')), row(['id' => 1]), row(['id' => 2]), row(['id' => 3]), row(['id' => 4])); - $writer = new FloeWriter($filesystem, $data->schema()); + $data = array_to_rows([['id' => 1], ['id' => 2], ['id' => 3], ['id' => 4]], schema(int_schema('id'))); + $writer = new FloeWriter($filesystem, $data->schema(), new AdaptiveBackend()); $writer->create($path); $writer->write($data); $writer->close(); $sizes = array_map( - static fn(Rows $batch): int => count($batch->all()), + static fn(Rows $batch): int => $batch->count(), iterator_to_array( - (new FloeReader($filesystem)) + (new FloeReader($filesystem, new AdaptiveBackend())) ->read($path) ->tail(3, batchSize: 1), ), @@ -948,28 +884,27 @@ public function test_tail_matches_slice_of_full_read(): void $filesystem = memory_filesystem(); $path = path('memory://tail-slice.floe'); - $data = rows( - schema(int_schema('id')), - row(['id' => 1]), - row(['id' => 2]), - row(['id' => 3]), - row(['id' => 4]), - row(['id' => 5]), - row(['id' => 6]), - row(['id' => 7]), - ); - $writer = new FloeWriter($filesystem, $data->schema()); + $data = array_to_rows([ + ['id' => 1], + ['id' => 2], + ['id' => 3], + ['id' => 4], + ['id' => 5], + ['id' => 6], + ['id' => 7], + ], schema(int_schema('id'))); + $writer = new FloeWriter($filesystem, $data->schema(), new AdaptiveBackend()); $writer->create($path); $writer->write($data); $writer->close(); - $reader = (new FloeReader($filesystem))->read($path); + $reader = (new FloeReader($filesystem, new AdaptiveBackend()))->read($path); $full = []; foreach ($reader->rows() as $batch) { - foreach ($batch->all() as $row) { - $full[] = $row->get('id'); + foreach ($batch->toArray() as $row) { + $full[] = $row['id']; } } @@ -977,8 +912,8 @@ public function test_tail_matches_slice_of_full_read(): void $tail = []; foreach ($reader->tail($count) as $batch) { - foreach ($batch->all() as $row) { - $tail[] = $row->get('id'); + foreach ($batch->toArray() as $row) { + $tail[] = $row['id']; } } @@ -990,7 +925,7 @@ public function test_tail_non_positive_count_throws(): void { $filesystem = memory_filesystem(); $path = path('memory://tail-zero.floe'); - $writer = new FloeWriter($filesystem, schema()); + $writer = new FloeWriter($filesystem, schema(), new AdaptiveBackend()); $writer->create($path); $writer->close(); @@ -998,7 +933,7 @@ public function test_tail_non_positive_count_throws(): void $this->expectExceptionMessage('tail count must be greater than 0'); iterator_to_array( - (new FloeReader($filesystem)) + (new FloeReader($filesystem, new AdaptiveBackend())) ->read($path) ->tail(0), ); diff --git a/src/core/etl/tests/Flow/Floe/Tests/Unit/FloeSerializerTest.php b/src/core/etl/tests/Flow/Floe/Tests/Unit/FloeSerializerTest.php index 04da98eedc..97b5a6e1f1 100644 --- a/src/core/etl/tests/Flow/Floe/Tests/Unit/FloeSerializerTest.php +++ b/src/core/etl/tests/Flow/Floe/Tests/Unit/FloeSerializerTest.php @@ -4,8 +4,9 @@ namespace Flow\Floe\Tests\Unit; -use Flow\ETL\Row; +use Flow\ETL\Column\AdaptiveBackend; use Flow\ETL\Rows; +use Flow\ETL\Tests\Double\SpyBackend; use Flow\Floe\Exception\FloeException; use Flow\Floe\FloeSerializer; use Flow\Floe\Format; @@ -15,8 +16,8 @@ use PHPUnit\Framework\TestCase; use function array_map; +use function Flow\ETL\DSL\array_to_rows; use function Flow\ETL\DSL\int_schema; -use function Flow\ETL\DSL\row; use function Flow\ETL\DSL\rows; use function Flow\ETL\DSL\schema; use function Flow\ETL\DSL\str_schema; @@ -33,8 +34,11 @@ final class FloeSerializerTest extends TestCase public static function values(): array { return [ - 'single row' => [rows(schema(int_schema('id'), str_schema('name')), row(['id' => 1, 'name' => 'John']))], - 'rows' => [rows(schema(int_schema('id')), row(['id' => 1]), row(['id' => 2]))], + 'single row' => [array_to_rows( + [['id' => 1, 'name' => 'John']], + schema(int_schema('id'), str_schema('name')), + )], + 'rows' => [array_to_rows([['id' => 1], ['id' => 2]], schema(int_schema('id')))], 'heterogeneous rows' => [RowsMother::heterogeneous()], 'all entry types' => [RowsMother::withAllEntryTypes()], 'empty rows' => [rows(schema())], @@ -60,14 +64,14 @@ public static function exactRoundTripValues(): array #[DataProvider('exactRoundTripValues')] public function test_serialize_unserialize_round_trip(Rows $value): void { - $serializer = new FloeSerializer(); + $serializer = new FloeSerializer(new AdaptiveBackend()); static::assertEquals($value, unserialize_from_string($serializer, serialize_to_string($serializer, $value))); } public function test_streaming_mode_round_trip(): void { - $serializer = new FloeSerializer(2); + $serializer = new FloeSerializer(new AdaptiveBackend(), 2); $value = RowsMother::withAllEntryTypes(); static::assertEquals($value, unserialize_from_string($serializer, serialize_to_string($serializer, $value))); @@ -75,29 +79,26 @@ public function test_streaming_mode_round_trip(): void public function test_cache_round_trip_preserves_rows_without_a_partition_table(): void { - $value = rows( + $value = array_to_rows( + [['id' => 1, 'country' => 'PL'], ['id' => 2, 'country' => 'PL']], schema(int_schema('id'), str_schema('country')), - row(['id' => 1, 'country' => 'PL']), - row(['id' => 2, 'country' => 'PL']), ); static::assertEquals($value, unserialize_from_string( - new FloeSerializer(1), - serialize_to_string(new FloeSerializer(), $value), + new FloeSerializer(new AdaptiveBackend(), 1), + serialize_to_string(new FloeSerializer(new AdaptiveBackend()), $value), )); } public function test_heterogeneous_rows_round_trip_unpadded_but_union_widened(): void { - $serializer = new FloeSerializer(); + $serializer = new FloeSerializer(new AdaptiveBackend()); // one write session = one schema: rows keep their own columns (unpadded) but // entry types widen to the batch union - id and name become nullable because // each is absent from some row. Values are unchanged; only nullability widens. - $value = rows( + $value = array_to_rows( + [['id' => 1], ['id' => 2, 'name' => 'John'], ['name' => 'Jane']], schema(int_schema('id', nullable: true), str_schema('name', nullable: true)), - row(['id' => 1]), - row(['id' => 2, 'name' => 'John']), - row(['name' => 'Jane']), ); $result = unserialize_from_string($serializer, serialize_to_string($serializer, $value)); @@ -110,19 +111,19 @@ public function test_heterogeneous_rows_round_trip_unpadded_but_union_widened(): #[DataProvider('values')] public function test_read_batch_size_does_not_change_decoded_rows(Rows $value): void { - $bytes = serialize_to_string(new FloeSerializer(), $value); + $bytes = serialize_to_string(new FloeSerializer(new AdaptiveBackend()), $value); static::assertEquals( - unserialize_from_string(new FloeSerializer(), $bytes), - unserialize_from_string(new FloeSerializer(1), $bytes), + unserialize_from_string(new FloeSerializer(new AdaptiveBackend()), $bytes), + unserialize_from_string(new FloeSerializer(new AdaptiveBackend(), 1), $bytes), ); } #[DataProvider('exactRoundTripValues')] public function test_batch_sizes_decode_each_others_bytes(Rows $value): void { - $small = new FloeSerializer(1); - $default = new FloeSerializer(); + $small = new FloeSerializer(new AdaptiveBackend(), 1); + $default = new FloeSerializer(new AdaptiveBackend()); static::assertEquals($value, unserialize_from_string($default, serialize_to_string($small, $value))); static::assertEquals($value, unserialize_from_string($small, serialize_to_string($default, $value))); @@ -130,22 +131,22 @@ public function test_batch_sizes_decode_each_others_bytes(Rows $value): void public function test_batch_size_smaller_than_row_count(): void { - $serializer = new FloeSerializer(3); - $value = rows( + $serializer = new FloeSerializer(new AdaptiveBackend(), 3); + $value = array_to_rows( + array_map(static fn(int $id): array => ['id' => $id], range(1, 10)), schema(int_schema('id')), - ...array_map(static fn(int $id): Row => row(['id' => $id]), range(1, 10)), ); static::assertEquals($value, unserialize_from_string($serializer, serialize_to_string( - new FloeSerializer(), + new FloeSerializer(new AdaptiveBackend()), $value, ))); } public function test_batch_size_larger_than_row_count(): void { - $serializer = new FloeSerializer(100); - $value = rows(schema(int_schema('id')), row(['id' => 1]), row(['id' => 2])); + $serializer = new FloeSerializer(new AdaptiveBackend(), 100); + $value = array_to_rows([['id' => 1], ['id' => 2]], schema(int_schema('id'))); static::assertEquals($value, unserialize_from_string($serializer, serialize_to_string($serializer, $value))); } @@ -156,14 +157,14 @@ public function test_batch_size_below_one_throws(): void $this->expectExceptionMessage('Serializer batch size must be at least 1'); // @mago-ignore analysis:invalid-argument - new FloeSerializer(0); + new FloeSerializer(new AdaptiveBackend(), 0); } public function test_unserialize_rejects_torn_bytes(): void { $this->expectException(SerializationException::class); - unserialize_from_string(new FloeSerializer(), 'not a valid floe payload'); + unserialize_from_string(new FloeSerializer(new AdaptiveBackend()), 'not a valid floe payload'); } public function test_unserialize_rejects_empty_payload(): void @@ -171,12 +172,14 @@ public function test_unserialize_rejects_empty_payload(): void $this->expectException(SerializationException::class); $this->expectExceptionMessage('too small'); - unserialize_from_string(new FloeSerializer(), ''); + unserialize_from_string(new FloeSerializer(new AdaptiveBackend()), ''); } public function test_unserialize_rejects_row_count_mismatch(): void { - $bytes = serialize_to_string(new FloeSerializer(), rows(schema(int_schema('id')), row(['id' => 1]))); + $bytes = serialize_to_string(new FloeSerializer(new AdaptiveBackend()), array_to_rows([[ + 'id' => 1, + ]], schema(int_schema('id')))); // inflate the footer's row count while the body still holds a single row; the JSON // byte-length is unchanged (1 -> 2) so the trailer stays valid and the read reaches // the whole-value row-count guard @@ -185,7 +188,7 @@ public function test_unserialize_rejects_row_count_mismatch(): void $this->expectException(SerializationException::class); $this->expectExceptionMessage('decoded 1 of 2 rows'); - unserialize_from_string(new FloeSerializer(), $corrupted); + unserialize_from_string(new FloeSerializer(new AdaptiveBackend()), $corrupted); } public function test_unserialize_rejects_payload_smaller_than_header_and_trailer(): void @@ -193,7 +196,7 @@ public function test_unserialize_rejects_payload_smaller_than_header_and_trailer $this->expectException(SerializationException::class); $this->expectExceptionMessage('too small'); - unserialize_from_string(new FloeSerializer(), 'tiny'); + unserialize_from_string(new FloeSerializer(new AdaptiveBackend()), 'tiny'); } public function test_unserialize_rejects_footer_that_does_not_fit(): void @@ -201,6 +204,28 @@ public function test_unserialize_rejects_footer_that_does_not_fit(): void $this->expectException(SerializationException::class); $this->expectExceptionMessage('footer does not fit'); - unserialize_from_string(new FloeSerializer(), Format::header(0x00) . Format::trailer(1000)); + unserialize_from_string( + new FloeSerializer(new AdaptiveBackend()), + Format::header(0x00) . Format::trailer(1000), + ); + } + + public function test_unserialize_builds_through_the_given_backend(): void + { + $backend = new SpyBackend(); + $serializer = new FloeSerializer(backend: $backend); + + unserialize_from_string($serializer, serialize_to_string($serializer, RowsMother::numbered(3))); + + static::assertGreaterThanOrEqual(1, $backend->decodes()); + } + + public function test_an_empty_payload_unserializes_to_an_empty_batch_of_its_schema(): void + { + $serializer = new FloeSerializer(new AdaptiveBackend()); + $rows = unserialize_from_string($serializer, serialize_to_string($serializer, rows(schema(int_schema('id'))))); + + static::assertSame(0, $rows->count()); + static::assertSame(schema(int_schema('id'))->normalize(), $rows->schema()->normalize()); } } diff --git a/src/core/etl/tests/Flow/Floe/Tests/Unit/FloeSourceFileTest.php b/src/core/etl/tests/Flow/Floe/Tests/Unit/FloeSourceFileTest.php index 19a4b5c34e..9e7fc59d04 100644 --- a/src/core/etl/tests/Flow/Floe/Tests/Unit/FloeSourceFileTest.php +++ b/src/core/etl/tests/Flow/Floe/Tests/Unit/FloeSourceFileTest.php @@ -4,11 +4,11 @@ namespace Flow\Floe\Tests\Unit; -use Flow\ETL\Extractor\SourceFile; +use Flow\ETL\Extractor\File\SourceFile; use Flow\ETL\Tests\Double\CountingFilesystem; use Flow\ETL\Tests\FlowTestCase; use Flow\Floe\FloeSourceFile; -use Flow\Floe\Tests\Context\FloeEngineContext; +use Flow\Floe\Tests\Context\FloeFilesContext; use function Flow\ETL\DSL\int_schema; use function Flow\ETL\DSL\schema; @@ -20,10 +20,10 @@ final class FloeSourceFileTest extends FlowTestCase public function test_close_releases_the_reader(): void { $counting = new CountingFilesystem($memory = memory_filesystem()); - FloeEngineContext::writePartitionedFiles($memory); + FloeFilesContext::writePartitionedFiles($memory); (new FloeSourceFile( - FloeEngineContext::phpReader($counting)->read(path('memory://parts/country=PL/data.floe')), + FloeFilesContext::phpReader($counting)->read(path('memory://parts/country=PL/data.floe')), new SourceFile(path('memory://parts/country=PL/data.floe')), ))->close(); @@ -33,12 +33,12 @@ public function test_close_releases_the_reader(): void public function test_schema_comes_from_the_footer(): void { $memory = memory_filesystem(); - FloeEngineContext::writePartitionedFiles($memory); + FloeFilesContext::writePartitionedFiles($memory); static::assertEquals( schema(int_schema('id')), (new FloeSourceFile( - FloeEngineContext::phpReader($memory)->read(path('memory://parts/country=PL/data.floe')), + FloeFilesContext::phpReader($memory)->read(path('memory://parts/country=PL/data.floe')), new SourceFile(path('memory://parts/country=PL/data.floe')), ))->schema(), ); @@ -47,12 +47,12 @@ public function test_schema_comes_from_the_footer(): void public function test_source_returns_the_listed_file(): void { $memory = memory_filesystem(); - FloeEngineContext::writePartitionedFiles($memory); + FloeFilesContext::writePartitionedFiles($memory); static::assertEquals( new SourceFile(path('memory://parts/country=PL/data.floe')), (new FloeSourceFile( - FloeEngineContext::phpReader($memory)->read(path('memory://parts/country=PL/data.floe')), + FloeFilesContext::phpReader($memory)->read(path('memory://parts/country=PL/data.floe')), new SourceFile(path('memory://parts/country=PL/data.floe')), ))->source(), ); diff --git a/src/core/etl/tests/Flow/Floe/Tests/Unit/FloeStreamReaderTest.php b/src/core/etl/tests/Flow/Floe/Tests/Unit/FloeStreamReaderTest.php index 66a178f70e..4d51b2c161 100644 --- a/src/core/etl/tests/Flow/Floe/Tests/Unit/FloeStreamReaderTest.php +++ b/src/core/etl/tests/Flow/Floe/Tests/Unit/FloeStreamReaderTest.php @@ -4,200 +4,191 @@ namespace Flow\Floe\Tests\Unit; -use Closure; -use Flow\ETL\Row\Hydrator; -use Flow\ETL\Row\PhpRowHydrator; -use Flow\ETL\Row\TypedRowValues; +use Flow\ETL\Column\AdaptiveBackend; use Flow\ETL\Rows; -use Flow\ETL\Schema\Metadata; use Flow\Floe\Codec\NoopCodec; -use Flow\Floe\FloeEngine; +use Flow\Floe\Exception\FloeException; use Flow\Floe\FloeReader; use Flow\Floe\FloeStreamReader; use Flow\Floe\FloeWriter; -use Flow\Floe\NativeFloeEncoder; -use Flow\Floe\PhpFloeEncoder; +use Flow\Floe\Format; use Flow\Floe\Tests\Context\FloeStreamReaderContext; use Flow\Floe\Tests\Double\ClosingSpySourceStream; -use Flow\Floe\Tests\Double\PrefixingCodecStub; -use Flow\Floe\Tests\Double\SpyHydrator; use Flow\Floe\Tests\Mother\RowsMother; -use Generator; -use PHPUnit\Framework\Attributes\DataProvider; use PHPUnit\Framework\TestCase; +use function array_keys; +use function array_map; +use function Flow\ETL\DSL\array_to_rows; use function Flow\ETL\DSL\int_schema; -use function Flow\ETL\DSL\row; -use function Flow\ETL\DSL\rows; use function Flow\ETL\DSL\schema; use function Flow\Filesystem\DSL\memory_filesystem; use function Flow\Filesystem\DSL\path; -use function Flow\Types\DSL\type_integer; use function iterator_to_array; -use function range; -use function serialize; final class FloeStreamReaderTest extends TestCase { - /** - * @return array>}> - */ - public static function reads(): array - { - return [ - 'every row' => [ - static fn(FloeStreamReader $reader): Generator => $reader->rows(10), - [range(1, 10), range(11, 20), range(21, 25)], - ], - 'batch larger than the file' => [ - static fn(FloeStreamReader $reader): Generator => $reader->rows(100), - [range(1, 25)], - ], - 'limit below the batch size' => [ - static fn(FloeStreamReader $reader): Generator => $reader->rows(10, 0, 3), - [range(1, 3)], - ], - 'limit on a batch boundary' => [ - static fn(FloeStreamReader $reader): Generator => $reader->rows(10, 0, 20), - [range(1, 10), range(11, 20)], - ], - 'limit inside a batch' => [ - static fn(FloeStreamReader $reader): Generator => $reader->rows(10, 0, 15), - [range(1, 10), range(11, 15)], - ], - 'offset' => [ - static fn(FloeStreamReader $reader): Generator => $reader->rows(10, 4), - [range(5, 14), range(15, 24), [25]], - ], - 'offset and limit' => [ - static fn(FloeStreamReader $reader): Generator => $reader->rows(10, 4, 12), - [range(5, 14), [15, 16]], - ], - 'head' => [static fn(FloeStreamReader $reader): Generator => $reader->head(7, 5), [range(1, 5), [6, 7]]], - 'tail' => [ - static fn(FloeStreamReader $reader): Generator => $reader->tail(7, 5), - [range(19, 23), [24, 25]], - ], - ]; - } - public function test_close_closes_the_source_stream(): void { $filesystem = memory_filesystem(); $path = path('memory://close.floe'); - $data = rows(schema(int_schema('id')), row(['id' => 1])); - $writer = new FloeWriter($filesystem, $data->schema()); + $data = array_to_rows([['id' => 1]], schema(int_schema('id'))); + $writer = new FloeWriter($filesystem, $data->schema(), new AdaptiveBackend()); $writer->create($path); $writer->write($data); $writer->close(); $source = new ClosingSpySourceStream($filesystem->readFrom($path)); - (new FloeStreamReader($source, new NoopCodec(), 65_536))->close(); + (new FloeStreamReader($source, new NoopCodec(), 65_536, new AdaptiveBackend()))->close(); static::assertSame(1, $source->closeCount); } - /** - * @param Closure(FloeStreamReader): Generator $read - * @param list> $batches - */ - #[DataProvider('reads')] - public function test_fused_read_yields_the_batches_of_the_two_step_read(Closure $read, array $batches): void + public function test_a_row_frame_is_an_unknown_frame_type(): void { - if (!NativeFloeEncoder::isSupported()) { - static::markTestSkipped('flow_php extension with RustFloeEncoderNative is not loaded'); - } - $filesystem = memory_filesystem(); - $path = path('memory://fused.floe'); - FloeStreamReaderContext::write($filesystem, $path, RowsMother::numbered(25)); - $spy = new SpyHydrator(); + $path = path('memory://row-frame.floe'); + FloeStreamReaderContext::writeFrames( + $filesystem, + $path, + schema(int_schema('id')), + Format::frame(0x02, "\x01"), + 1, + ); - $twoStep = iterator_to_array($read((new FloeReader($filesystem, hydrator: $spy))->read($path))); - $fused = iterator_to_array($read((new FloeReader($filesystem))->read($path))); + $this->expectException(FloeException::class); + $this->expectExceptionMessage('Floe found unknown frame type 0x02'); - static::assertSame($batches, array_map(static fn(Rows $rows): array => $rows->reduceToArray('id'), $fused)); - static::assertEquals($twoStep, $fused); - static::assertGreaterThan(0, $spy->hydrateCalls); + iterator_to_array( + (new FloeReader($filesystem, new AdaptiveBackend())) + ->read($path) + ->rows(), + false, + ); } - /** - * @param Closure(FloeStreamReader): Generator $read - * @param list> $batches - */ - #[DataProvider('reads')] - public function test_native_engine_reads_a_transforming_codec_like_the_php_engine( - Closure $read, - array $batches, - ): void { - if (!NativeFloeEncoder::isSupported()) { - static::markTestSkipped('flow_php extension with RustFloeEncoderNative is not loaded'); - } - + public function test_offset_skips_a_whole_batch_frame_and_slices_the_next(): void + { $filesystem = memory_filesystem(); - $codec = new PrefixingCodecStub(); - $path = path('memory://codec.floe'); - FloeStreamReaderContext::write($filesystem, $path, RowsMother::numbered(25), codec: $codec); - - $php = iterator_to_array($read((new FloeReader($filesystem, $codec, engine: FloeEngine::php))->read($path))); - $native = iterator_to_array($read((new FloeReader($filesystem, $codec, engine: FloeEngine::native))->read( + $path = path('memory://offset.floe'); + FloeStreamReaderContext::writeFrames( + $filesystem, $path, - ))); + schema(int_schema('id')), + Format::frame(Format::FRAME_BATCH, RowsMother::ids(1, 3)->encodeFrame()) + . Format::frame(Format::FRAME_BATCH, RowsMother::ids(4, 6)->encodeFrame()), + 6, + ); - static::assertEquals($php, $native); - static::assertSame($batches, array_map(static fn(Rows $rows): array => $rows->reduceToArray('id'), $native)); + static::assertSame( + [[['id' => 5], ['id' => 6]]], + array_map( + static fn(Rows $batch) => $batch->toArray(), + iterator_to_array( + (new FloeReader($filesystem, new AdaptiveBackend())) + ->read($path) + ->rows(offset: 4), + false, + ), + ), + ); } - /** - * @param Closure(FloeStreamReader): Generator $read - * @param list> $batches - */ - #[DataProvider('reads')] - public function test_native_engine_reads_like_the_php_engine(Closure $read, array $batches): void + public function test_limit_slices_a_batch_frame(): void { - if (!NativeFloeEncoder::isSupported()) { - static::markTestSkipped('flow_php extension with RustFloeEncoderNative is not loaded'); - } - $filesystem = memory_filesystem(); - $path = path('memory://engines.floe'); - FloeStreamReaderContext::write($filesystem, $path, RowsMother::numbered(25)); - - $php = iterator_to_array($read((new FloeReader($filesystem, engine: FloeEngine::php))->read($path))); - $native = iterator_to_array($read((new FloeReader($filesystem, engine: FloeEngine::native))->read($path))); + $path = path('memory://limit.floe'); + FloeStreamReaderContext::writeFrames( + $filesystem, + $path, + schema(int_schema('id')), + Format::frame(Format::FRAME_BATCH, RowsMother::ids(1, 5)->encodeFrame()), + 5, + ); - static::assertSame(serialize($php), serialize($native)); - static::assertSame($batches, array_map(static fn(Rows $rows): array => $rows->reduceToArray('id'), $native)); + static::assertSame( + [[['id' => 1], ['id' => 2]]], + array_map( + static fn(Rows $batch) => $batch->toArray(), + iterator_to_array( + (new FloeReader($filesystem, new AdaptiveBackend())) + ->read($path) + ->rows(limit: 2), + false, + ), + ), + ); } - /** - * @return array - */ - public static function hydrators(): array + public function test_batch_size_is_an_upper_bound(): void { - return ['default' => [null], 'php' => [new PhpRowHydrator()]]; + $filesystem = memory_filesystem(); + $one = path('memory://one-frame.floe'); + $two = path('memory://two-frames.floe'); + FloeStreamReaderContext::writeFrames( + $filesystem, + $one, + schema(int_schema('id')), + Format::frame(Format::FRAME_BATCH, RowsMother::ids(1, 5)->encodeFrame()), + 5, + ); + FloeStreamReaderContext::writeFrames( + $filesystem, + $two, + schema(int_schema('id')), + Format::frame(Format::FRAME_BATCH, RowsMother::ids(1, 2)->encodeFrame()) + . Format::frame(Format::FRAME_BATCH, RowsMother::ids(3, 4)->encodeFrame()), + 4, + ); + + static::assertSame( + [2, 2, 1], + array_map( + static fn(Rows $batch) => $batch->count(), + iterator_to_array( + (new FloeReader($filesystem, new AdaptiveBackend())) + ->read($one) + ->rows(2), + false, + ), + ), + ); + static::assertSame( + [2, 2], + array_map( + static fn(Rows $batch) => $batch->count(), + iterator_to_array( + (new FloeReader($filesystem, new AdaptiveBackend())) + ->read($two) + ->rows(10), + false, + ), + ), + ); } - #[DataProvider('hydrators')] - public function test_batches_carry_the_file_schema_when_frames_carry_per_value_metadata(?Hydrator $hydrator): void + public function test_chunked_frames_yield_unique_keys(): void { $filesystem = memory_filesystem(); - $path = path('memory://per-value-metadata.floe'); - $schema = schema(int_schema('id')); - FloeStreamReaderContext::writeFrames($filesystem, $path, $schema, (new PhpFloeEncoder($schema))->encode([ - new TypedRowValues(['id' => 1], ['id' => type_integer()], ['id' => Metadata::fromArray(['k' => 'v'])]), - ])); - - $batches = iterator_to_array( - (new FloeReader($filesystem, hydrator: $hydrator)) - ->read($path) - ->rows(), + $path = path('memory://keys.floe'); + FloeStreamReaderContext::writeFrames( + $filesystem, + $path, + schema(int_schema('id')), + Format::frame(Format::FRAME_BATCH, RowsMother::ids(1, 3)->encodeFrame()) + . Format::frame(Format::FRAME_BATCH, RowsMother::ids(4, 6)->encodeFrame()), + 6, ); - static::assertCount(1, $batches); - static::assertSame($schema->normalize(), $batches[0]->schema()->normalize()); - static::assertSame([1], $batches[0]->reduceToArray('id')); + static::assertSame( + [0, 1, 2, 3], + array_keys(iterator_to_array( + (new FloeReader($filesystem, new AdaptiveBackend())) + ->read($path) + ->rows(2), + )), + ); } } diff --git a/src/core/etl/tests/Flow/Floe/Tests/Unit/FloeStreamWriterTest.php b/src/core/etl/tests/Flow/Floe/Tests/Unit/FloeStreamWriterTest.php index d3f7986bd4..6a06069aa9 100644 --- a/src/core/etl/tests/Flow/Floe/Tests/Unit/FloeStreamWriterTest.php +++ b/src/core/etl/tests/Flow/Floe/Tests/Unit/FloeStreamWriterTest.php @@ -4,31 +4,33 @@ namespace Flow\Floe\Tests\Unit; -use DateTimeImmutable; +use Flow\ETL\Column\AdaptiveBackend; +use Flow\ETL\Exception\InvalidArgumentException; +use Flow\ETL\Exception\OffsetOverflow; +use Flow\ETL\Rows; use Flow\ETL\Schema\Metadata; use Flow\Floe\Exception\FloeException; use Flow\Floe\Exception\IncompatibleSchemaException; -use Flow\Floe\FloeEngine; +use Flow\Floe\FloeReader; use Flow\Floe\FloeStreamWriter; use Flow\Floe\FloeWriter; use Flow\Floe\Format; -use Flow\Floe\NativeFloeEncoder; use Flow\Floe\Options; use Flow\Floe\Tests\Context\FloeStreamReaderContext; use Flow\Floe\Tests\Double\CodecStub; -use Flow\Floe\Tests\Double\PrefixingCodecStub; -use Flow\Floe\Tests\Double\SpyHydrator; -use Flow\Floe\Tests\Mother\RowsMother; +use Flow\Floe\Tests\Double\OverflowingColumnStub; use PHPUnit\Framework\TestCase; -use function Flow\ETL\DSL\datetime_schema; +use function array_map; +use function array_merge; +use function Flow\ETL\DSL\array_to_rows; use function Flow\ETL\DSL\int_schema; -use function Flow\ETL\DSL\row; use function Flow\ETL\DSL\rows; use function Flow\ETL\DSL\schema; use function Flow\ETL\DSL\str_schema; use function Flow\Filesystem\DSL\memory_filesystem; use function Flow\Filesystem\DSL\path; +use function iterator_to_array; final class FloeStreamWriterTest extends TestCase { @@ -37,19 +39,18 @@ public function test_create_on_stream_is_byte_identical_to_path_create(): void $filesystem = memory_filesystem(); $viaCreate = path('memory://via-create.floe'); $viaStream = path('memory://via-stream.floe'); - $data = rows( + $data = array_to_rows( + [['id' => 1, 'name' => 'a'], ['id' => 2, 'name' => 'b']], schema(int_schema('id'), str_schema('name')), - row(['id' => 1, 'name' => 'a']), - row(['id' => 2, 'name' => 'b']), ); $schema = $data->schema(); - $create = new FloeWriter($filesystem, $schema); + $create = new FloeWriter($filesystem, $schema, new AdaptiveBackend()); $create->create($viaCreate); $create->write($data); $create->close(); - $onStream = new FloeStreamWriter($schema); + $onStream = new FloeStreamWriter($schema, new AdaptiveBackend()); $onStream->create($filesystem->writeTo($viaStream)); $onStream->write($data); $onStream->close(); @@ -62,21 +63,19 @@ public function test_small_buffer_size_produces_byte_identical_output(): void $filesystem = memory_filesystem(); $default = path('memory://default-buffer.floe'); $tiny = path('memory://tiny-buffer.floe'); - $data = rows( + $data = array_to_rows( + [['id' => 1, 'name' => 'alpha'], ['id' => 2, 'name' => 'beta'], ['id' => 3, 'name' => 'gamma']], schema(int_schema('id'), str_schema('name')), - row(['id' => 1, 'name' => 'alpha']), - row(['id' => 2, 'name' => 'beta']), - row(['id' => 3, 'name' => 'gamma']), ); $schema = $data->schema(); - $defaultWriter = new FloeStreamWriter($schema); + $defaultWriter = new FloeStreamWriter($schema, new AdaptiveBackend()); $defaultWriter->create($filesystem->writeTo($default)); $defaultWriter->write($data); $defaultWriter->close(); - $tinyWriter = new FloeStreamWriter($schema, new Options(bufferSize: 4)); + $tinyWriter = new FloeStreamWriter($schema, new AdaptiveBackend(), new Options(bufferSize: 4)); $tinyWriter->create($filesystem->writeTo($tiny)); $tinyWriter->write($data); $tinyWriter->close(); @@ -89,9 +88,9 @@ public function test_create_on_stream_round_trips(): void $filesystem = memory_filesystem(); $path = path('memory://on-stream.floe'); - $data = rows(schema(int_schema('id')), row(['id' => 1]), row(['id' => 2]), row(['id' => 3])); + $data = array_to_rows([['id' => 1], ['id' => 2], ['id' => 3]], schema(int_schema('id'))); - $writer = new FloeStreamWriter($data->schema()); + $writer = new FloeStreamWriter($data->schema(), new AdaptiveBackend()); $writer->create($filesystem->writeTo($path), Metadata::fromArray(['source' => 'stream'])); $writer->write($data); $writer->close(); @@ -101,7 +100,7 @@ public function test_create_on_stream_round_trips(): void static::assertSame(3, $footer->statistics->rows); static::assertSame(['source' => 'stream'], $footer->metadata->normalize()); static::assertSame( - [Format::FRAME_ROW, Format::FRAME_ROW, Format::FRAME_ROW, Format::FRAME_FOOTER], + [Format::FRAME_BATCH, Format::FRAME_FOOTER], FloeStreamReaderContext::frameTypes($filesystem, $path), ); } @@ -111,12 +110,12 @@ public function test_construct_with_non_noop_codec_throws(): void $this->expectException(FloeException::class); $this->expectExceptionMessage('supports only the no-op codec, got codec 0x05'); - new FloeStreamWriter(schema(), new Options(codec: new CodecStub(0x05))); + new FloeStreamWriter(schema(), new AdaptiveBackend(), new Options(codec: new CodecStub(0x05))); } public function test_creating_a_second_session_throws(): void { - $writer = new FloeStreamWriter(schema()); + $writer = new FloeStreamWriter(schema(), new AdaptiveBackend()); $writer->create(memory_filesystem()->writeTo(path('memory://first.floe'))); $this->expectException(FloeException::class); @@ -127,39 +126,57 @@ public function test_creating_a_second_session_throws(): void public function test_write_before_create_throws(): void { - $writer = new FloeStreamWriter(schema()); + $writer = new FloeStreamWriter(schema(), new AdaptiveBackend()); $this->expectException(FloeException::class); $this->expectExceptionMessage('Floe writer session is not open'); - $writer->write(rows(schema(int_schema('id')), row(['id' => 1]))); + $writer->write(array_to_rows([['id' => 1]], schema(int_schema('id')))); } - public function test_aligned_multi_batch_session_is_byte_identical_to_a_single_batch(): void + public function test_a_multi_batch_session_reads_back_the_rows_of_a_single_batch(): void { $filesystem = memory_filesystem(); $multi = path('memory://multi-batch.floe'); $single = path('memory://single-batch.floe'); - $data = rows( + $data = array_to_rows( + [['id' => 1, 'name' => 'a'], ['id' => 2, 'name' => 'b']], schema(int_schema('id'), str_schema('name')), - row(['id' => 1, 'name' => 'a']), - row(['id' => 2, 'name' => 'b']), ); $schema = $data->schema(); - $multiWriter = new FloeStreamWriter($schema); + $multiWriter = new FloeStreamWriter($schema, new AdaptiveBackend()); $multiWriter->create($filesystem->writeTo($multi)); - $multiWriter->write(rows(schema(int_schema('id'), str_schema('name')), row(['id' => 1, 'name' => 'a']))); - $multiWriter->write(rows(schema(int_schema('id'), str_schema('name')), row(['id' => 2, 'name' => 'b']))); + $multiWriter->write(array_to_rows([['id' => 1, 'name' => 'a']], schema(int_schema('id'), str_schema('name')))); + $multiWriter->write(array_to_rows([['id' => 2, 'name' => 'b']], schema(int_schema('id'), str_schema('name')))); $multiWriter->close(); - $singleWriter = new FloeStreamWriter($schema); + $singleWriter = new FloeStreamWriter($schema, new AdaptiveBackend()); $singleWriter->create($filesystem->writeTo($single)); $singleWriter->write($data); $singleWriter->close(); - static::assertSame($filesystem->readFrom($single)->content(), $filesystem->readFrom($multi)->content()); + static::assertSame( + [Format::FRAME_BATCH, Format::FRAME_BATCH, Format::FRAME_FOOTER], + FloeStreamReaderContext::frameTypes($filesystem, $multi), + ); + static::assertSame( + (new FloeReader($filesystem, new AdaptiveBackend())) + ->read($single) + ->rows() + ->current() + ->toArray(), + array_merge(...array_map( + static fn(Rows $batch) => $batch->toArray(), + iterator_to_array( + (new FloeReader($filesystem, new AdaptiveBackend())) + ->read($multi) + ->rows(), + false, + ), + )), + ); $footer = FloeStreamReaderContext::footer($filesystem, $multi); static::assertNotSame([], $footer->schema); @@ -171,18 +188,21 @@ public function test_batch_omitting_a_nullable_session_column_is_accepted_and_ro $filesystem = memory_filesystem(); $path = path('memory://subset.floe'); - $first = rows(schema(int_schema('id'), str_schema('name', nullable: true)), row(['id' => 1, 'name' => 'a'])); + $first = array_to_rows( + [['id' => 1, 'name' => 'a']], + schema(int_schema('id'), str_schema('name', nullable: true)), + ); - $writer = new FloeStreamWriter($first->schema()); + $writer = new FloeStreamWriter($first->schema(), new AdaptiveBackend()); $writer->create($filesystem->writeTo($path)); $writer->write($first); - $writer->write(rows(schema(int_schema('id')), row(['id' => 2]))); + $writer->write(array_to_rows([['id' => 2]], schema(int_schema('id')))); $writer->close(); $footer = FloeStreamReaderContext::footer($filesystem, $path); static::assertSame(2, $footer->statistics->rows); static::assertNotSame([], $footer->schema); - static::assertCount(2, FloeStreamReaderContext::readAll($filesystem, $path)->all()); + static::assertCount(2, FloeStreamReaderContext::readAll($filesystem, $path)); } /** @@ -191,54 +211,54 @@ public function test_batch_omitting_a_nullable_session_column_is_accepted_and_ro */ public function test_batch_omitting_a_not_null_session_column_is_refused(): void { - $first = rows(schema(int_schema('id'), str_schema('name')), row(['id' => 1, 'name' => 'a'])); + $first = array_to_rows([['id' => 1, 'name' => 'a']], schema(int_schema('id'), str_schema('name'))); - $writer = new FloeStreamWriter($first->schema()); + $writer = new FloeStreamWriter($first->schema(), new AdaptiveBackend()); $writer->create(memory_filesystem()->writeTo(path('memory://subset-not-null.floe'))); $writer->write($first); $this->expectException(IncompatibleSchemaException::class); $this->expectExceptionMessage('Missing Definitions'); - $writer->write(rows(schema(int_schema('id')), row(['id' => 2]))); + $writer->write(array_to_rows([['id' => 2]], schema(int_schema('id')))); } public function test_batch_introducing_a_new_column_throws_naming_the_column(): void { - $writer = new FloeStreamWriter(schema(int_schema('id'))); + $writer = new FloeStreamWriter(schema(int_schema('id')), new AdaptiveBackend()); $writer->create(memory_filesystem()->writeTo(path('memory://new-column.floe'))); - $writer->write(rows(schema(int_schema('id')), row(['id' => 1]))); + $writer->write(array_to_rows([['id' => 1]], schema(int_schema('id')))); $this->expectException(IncompatibleSchemaException::class); $this->expectExceptionMessage('new column "email"'); - $writer->write(rows(schema(int_schema('id'), str_schema('email')), row(['id' => 2, 'email' => 'x']))); + $writer->write(array_to_rows([['id' => 2, 'email' => 'x']], schema(int_schema('id'), str_schema('email')))); } public function test_batch_with_an_incompatible_type_throws_naming_the_column(): void { - $writer = new FloeStreamWriter(schema(int_schema('id'))); + $writer = new FloeStreamWriter(schema(int_schema('id')), new AdaptiveBackend()); $writer->create(memory_filesystem()->writeTo(path('memory://type-drift.floe'))); - $writer->write(rows(schema(int_schema('id')), row(['id' => 1]))); + $writer->write(array_to_rows([['id' => 1]], schema(int_schema('id')))); $this->expectException(IncompatibleSchemaException::class); $this->expectExceptionMessage('expected: id, given: id'); - $writer->write(rows(schema(str_schema('id')), row(['id' => 'x']))); + $writer->write(array_to_rows([['id' => 'x']], schema(str_schema('id')))); } public function test_new_column_message_names_the_session_schema_and_the_column(): void { - $writer = new FloeStreamWriter(schema(int_schema('id'))); + $writer = new FloeStreamWriter(schema(int_schema('id')), new AdaptiveBackend()); $writer->create(memory_filesystem()->writeTo(path('memory://hint.floe'))); - $writer->write(rows(schema(int_schema('id')), row(['id' => 1]))); + $writer->write(array_to_rows([['id' => 1]], schema(int_schema('id')))); $this->expectException(IncompatibleSchemaException::class); $this->expectExceptionMessage( 'Floe write session schema is fixed and this batch does not fit it: new column "email".', ); - $writer->write(rows(schema(int_schema('id'), str_schema('email')), row(['id' => 2, 'email' => 'x']))); + $writer->write(array_to_rows([['id' => 2, 'email' => 'x']], schema(int_schema('id'), str_schema('email')))); } public function test_explicit_session_schema_is_used_for_the_footer(): void @@ -246,9 +266,9 @@ public function test_explicit_session_schema_is_used_for_the_footer(): void $filesystem = memory_filesystem(); $path = path('memory://explicit-schema.floe'); - $writer = new FloeStreamWriter(schema(int_schema('id'), str_schema('name'))); + $writer = new FloeStreamWriter(schema(int_schema('id'), str_schema('name')), new AdaptiveBackend()); $writer->create($filesystem->writeTo($path)); - $writer->write(rows(schema(int_schema('id'), str_schema('name')), row(['id' => 1, 'name' => 'a']))); + $writer->write(array_to_rows([['id' => 1, 'name' => 'a']], schema(int_schema('id'), str_schema('name')))); $writer->close(); $fileSchema = FloeStreamReaderContext::footer($filesystem, $path)->schema(); @@ -261,7 +281,7 @@ public function test_empty_batch_writes_no_schema_or_section(): void $filesystem = memory_filesystem(); $path = path('memory://empty.floe'); - $writer = new FloeStreamWriter(schema()); + $writer = new FloeStreamWriter(schema(), new AdaptiveBackend()); $writer->create($filesystem->writeTo($path)); $writer->write(rows(schema())); $writer->close(); @@ -276,9 +296,9 @@ public function test_empty_batch_writes_no_schema_or_section(): void public function test_writing_a_column_name_with_invalid_utf8_throws(): void { - $badRows = rows(schema(str_schema('badÿname')), row(['badÿname' => 'x'])); + $badRows = array_to_rows([['badÿname' => 'x']], schema(str_schema('badÿname'))); - $writer = new FloeStreamWriter($badRows->schema()); + $writer = new FloeStreamWriter($badRows->schema(), new AdaptiveBackend()); $writer->create(memory_filesystem()->writeTo(path('memory://bad-name.floe'))); $this->expectException(FloeException::class); @@ -290,62 +310,36 @@ public function test_writing_a_column_name_with_invalid_utf8_throws(): void $writer->close(); } - public function test_multi_batch_is_byte_identical_to_single_batch(): void - { - $filesystem = memory_filesystem(); - $multi = path('memory://off-multi.floe'); - $single = path('memory://off-single.floe'); - $data = rows( - schema(int_schema('id'), str_schema('name')), - row(['id' => 1, 'name' => 'a']), - row(['id' => 2, 'name' => 'b']), - ); - $schema = $data->schema(); - - $multiWriter = new FloeStreamWriter($schema); - $multiWriter->create($filesystem->writeTo($multi)); - $multiWriter->write(rows(schema(int_schema('id'), str_schema('name')), row(['id' => 1, 'name' => 'a']))); - $multiWriter->write(rows(schema(int_schema('id'), str_schema('name')), row(['id' => 2, 'name' => 'b']))); - $multiWriter->close(); - - $singleWriter = new FloeStreamWriter($schema); - $singleWriter->create($filesystem->writeTo($single)); - $singleWriter->write($data); - $singleWriter->close(); - - static::assertSame($filesystem->readFrom($single)->content(), $filesystem->readFrom($multi)->content()); - } - public function test_a_later_null_into_a_non_nullable_session_column_is_rejected(): void { - $first = rows(schema(int_schema('id'), str_schema('opt')), row(['id' => 1, 'opt' => 'present'])); + $first = array_to_rows([['id' => 1, 'opt' => 'present']], schema(int_schema('id'), str_schema('opt'))); - $writer = new FloeStreamWriter($first->schema()); + $writer = new FloeStreamWriter($first->schema(), new AdaptiveBackend()); $writer->create(memory_filesystem()->writeTo(path('memory://off-present-then-null.floe'))); $writer->write($first); $this->expectException(IncompatibleSchemaException::class); $this->expectExceptionMessage('expected: opt, given: opt'); - $writer->write(rows( + $writer->write(array_to_rows( + [['id' => 2, 'opt' => null]], schema(int_schema('id'), str_schema('opt', nullable: true)), - row(['id' => 2, 'opt' => null]), )); } public function test_validation_on_rejects_a_later_null_into_a_non_nullable_session_column(): void { - $first = rows(schema(int_schema('id'), str_schema('opt')), row(['id' => 1, 'opt' => 'present'])); + $first = array_to_rows([['id' => 1, 'opt' => 'present']], schema(int_schema('id'), str_schema('opt'))); - $writer = new FloeStreamWriter($first->schema()); + $writer = new FloeStreamWriter($first->schema(), new AdaptiveBackend()); $writer->create(memory_filesystem()->writeTo(path('memory://on-present-then-null.floe'))); $writer->write($first); $this->expectException(IncompatibleSchemaException::class); - $writer->write(rows( + $writer->write(array_to_rows( + [['id' => 2, 'opt' => null]], schema(int_schema('id'), str_schema('opt', nullable: true)), - row(['id' => 2, 'opt' => null]), )); } @@ -356,148 +350,96 @@ public function test_validation_on_rejects_a_later_null_into_a_non_nullable_sess */ public function test_column_absent_from_the_session_schema_throws(): void { - $writer = new FloeStreamWriter(schema(int_schema('a'))); + $writer = new FloeStreamWriter(schema(int_schema('a')), new AdaptiveBackend()); $writer->create(memory_filesystem()->writeTo(path('memory://off-new-column.floe'))); $this->expectException(IncompatibleSchemaException::class); $this->expectExceptionMessage('new column "b"'); - $writer->write(rows(schema(int_schema('a'), int_schema('b')), row(['a' => 1, 'b' => 2]))); + $writer->write(array_to_rows([['a' => 1, 'b' => 2]], schema(int_schema('a'), int_schema('b')))); } - public function test_fused_write_is_byte_identical_to_the_two_step_write_across_sections(): void + public function test_an_overflowing_batch_is_split_in_halves(): void { - if (!NativeFloeEncoder::isSupported()) { - static::markTestSkipped('flow_php extension with RustFloeEncoderNative is not loaded'); - } - $filesystem = memory_filesystem(); - $schema = schema(int_schema('id')); - $batches = [RowsMother::ids(1, 100_000), RowsMother::ids(100_001, 100_002)]; - $spy = new SpyHydrator(); - - $twoStep = FloeStreamReaderContext::writeBatches( - $filesystem, - path('memory://two-step.floe'), + $path = path('memory://split.floe'); + $schema = schema(int_schema('id'), str_schema('name')); + $data = array_to_rows([ + ['id' => 1, 'name' => 'a'], + ['id' => 2, 'name' => 'b'], + ['id' => 3, 'name' => 'c'], + ['id' => 4, 'name' => 'd'], + ], $schema); + $rows = Rows::fromColumns( $schema, - $batches, - hydrator: $spy, + [ + 'id' => $data->column('id'), + 'name' => new OverflowingColumnStub($data->column('name'), new OffsetOverflow('overflow'), maxRows: 2), + ], + 4, ); - $fused = FloeStreamReaderContext::writeBatches($filesystem, path('memory://fused.floe'), $schema, $batches); - - static::assertSame($twoStep, $fused); - static::assertSame(2, $spy->dehydrateCalls); - static::assertCount(2, FloeStreamReaderContext::footer($filesystem, path('memory://fused.floe'))->sections); - } - public function test_fused_write_is_byte_identical_to_the_two_step_write_around_an_empty_batch(): void - { - if (!NativeFloeEncoder::isSupported()) { - static::markTestSkipped('flow_php extension with RustFloeEncoderNative is not loaded'); - } - - $filesystem = memory_filesystem(); - $numbered = RowsMother::numbered(5); - $batches = [rows($numbered->schema()), $numbered, rows($numbered->schema())]; + $writer = new FloeStreamWriter($schema, new AdaptiveBackend()); + $writer->create($filesystem->writeTo($path)); + $writer->write($rows); + $writer->close(); static::assertSame( - FloeStreamReaderContext::writeBatches( - $filesystem, - path('memory://two-step.floe'), - $numbered->schema(), - $batches, - hydrator: new SpyHydrator(), - ), - FloeStreamReaderContext::writeBatches( - $filesystem, - path('memory://fused.floe'), - $numbered->schema(), - $batches, + [Format::FRAME_BATCH, Format::FRAME_BATCH, Format::FRAME_FOOTER], + FloeStreamReaderContext::frameTypes($filesystem, $path), + ); + static::assertSame( + [[1, 2], [3, 4]], + array_map( + static fn(Rows $batch) => $batch->reduceToArray('id'), + iterator_to_array( + (new FloeReader($filesystem, new AdaptiveBackend())) + ->read($path) + ->rows(), + false, + ), ), ); } - public function test_fused_write_is_byte_identical_to_the_two_step_write_for_a_batch_that_needs_match_to(): void + public function test_a_corrupt_column_fails_on_the_first_encode(): void { - if (!NativeFloeEncoder::isSupported()) { - static::markTestSkipped('flow_php extension with RustFloeEncoderNative is not loaded'); - } - $filesystem = memory_filesystem(); - $session = RowsMother::numbered(1)->schema(); - $batches = [rows( - schema(datetime_schema('at'), int_schema('id')), - row(['at' => new DateTimeImmutable('2026-01-01 00:00:00'), 'id' => 1]), - )]; - - static::assertSame( - FloeStreamReaderContext::writeBatches( - $filesystem, - path('memory://two-step.floe'), - $session, - $batches, - hydrator: new SpyHydrator(), - ), - FloeStreamReaderContext::writeBatches($filesystem, path('memory://fused.floe'), $session, $batches), + $schema = schema(str_schema('name')); + $name = new OverflowingColumnStub( + array_to_rows([['name' => 'a'], ['name' => 'b']], $schema)->column('name'), + new InvalidArgumentException('corrupt offsets'), + maxRows: 0, ); - } - public function test_native_engine_writes_a_transforming_codec_like_the_php_engine(): void - { - if (!NativeFloeEncoder::isSupported()) { - static::markTestSkipped('flow_php extension with RustFloeEncoderNative is not loaded'); + $writer = new FloeStreamWriter($schema, new AdaptiveBackend()); + $writer->create($filesystem->writeTo(path('memory://corrupt.floe'))); + + try { + $writer->write(Rows::fromColumns($schema, ['name' => $name], 2)); + static::fail('A corrupt column must fail the write'); + } catch (InvalidArgumentException $e) { + static::assertSame('corrupt offsets', $e->getMessage()); + static::assertSame(1, $name->encodeCalls); } + } + public function test_a_single_overflowing_row_rethrows_offset_overflow(): void + { $filesystem = memory_filesystem(); - $options = new Options(codec: new PrefixingCodecStub()); - $numbered = RowsMother::numbered(10); - $batches = [$numbered, RowsMother::numbered(3)]; - - static::assertSame( - FloeStreamReaderContext::writeBatches( - $filesystem, - path('memory://php.floe'), - $numbered->schema(), - $batches, - $options, - engine: FloeEngine::php, - ), - FloeStreamReaderContext::writeBatches( - $filesystem, - path('memory://native.floe'), - $numbered->schema(), - $batches, - $options, - engine: FloeEngine::native, - ), + $schema = schema(str_schema('name')); + $name = new OverflowingColumnStub( + array_to_rows([['name' => 'a']], $schema)->column('name'), + new OffsetOverflow('overflow'), + maxRows: 0, ); - } - public function test_native_engine_writes_like_the_php_engine(): void - { - if (!NativeFloeEncoder::isSupported()) { - static::markTestSkipped('flow_php extension with RustFloeEncoderNative is not loaded'); - } + $writer = new FloeStreamWriter($schema, new AdaptiveBackend()); + $writer->create($filesystem->writeTo(path('memory://single.floe'))); - $filesystem = memory_filesystem(); - $numbered = RowsMother::numbered(10); - $batches = [$numbered, RowsMother::numbered(3)]; + $this->expectException(OffsetOverflow::class); + $this->expectExceptionMessage('overflow (row 0)'); - static::assertSame( - FloeStreamReaderContext::writeBatches( - $filesystem, - path('memory://php.floe'), - $numbered->schema(), - $batches, - engine: FloeEngine::php, - ), - FloeStreamReaderContext::writeBatches( - $filesystem, - path('memory://native.floe'), - $numbered->schema(), - $batches, - engine: FloeEngine::native, - ), - ); + $writer->write(Rows::fromColumns($schema, ['name' => $name], 1)); } } diff --git a/src/core/etl/tests/Flow/Floe/Tests/Unit/FloeValueRoundTripTest.php b/src/core/etl/tests/Flow/Floe/Tests/Unit/FloeValueRoundTripTest.php index 4393d24d29..f2f9d6fc66 100644 --- a/src/core/etl/tests/Flow/Floe/Tests/Unit/FloeValueRoundTripTest.php +++ b/src/core/etl/tests/Flow/Floe/Tests/Unit/FloeValueRoundTripTest.php @@ -14,6 +14,7 @@ use PHPUnit\Framework\Attributes\RequiresPhp; use PHPUnit\Framework\TestCase; +use function Flow\ETL\DSL\array_to_rows; use function Flow\ETL\DSL\bool_schema; use function Flow\ETL\DSL\date_schema; use function Flow\ETL\DSL\datetime_schema; @@ -25,8 +26,6 @@ use function Flow\ETL\DSL\list_schema; use function Flow\ETL\DSL\map_schema; use function Flow\ETL\DSL\null_schema; -use function Flow\ETL\DSL\row; -use function Flow\ETL\DSL\rows; use function Flow\ETL\DSL\schema; use function Flow\ETL\DSL\str_schema; use function Flow\ETL\DSL\structure_schema; @@ -62,15 +61,8 @@ final class FloeValueRoundTripTest extends TestCase public function test_datetime_entries(): void { static::assertEquals( - $rows = rows( - schema( - datetime_schema('immutable', zone: 'Europe/Warsaw'), - datetime_schema('mutable', zone: 'America/New_York'), - datetime_schema('offset_timezone', zone: '+02:30'), - datetime_schema('before_epoch'), - date_schema('date'), - ), - row([ + $rows = array_to_rows( + [[ 'immutable' => new DateTimeImmutable( '2025-06-15 12:30:45.123456', new DateTimeZone('Europe/Warsaw'), @@ -78,8 +70,15 @@ public function test_datetime_entries(): void 'mutable' => new DateTime('2025-06-15 12:30:45.654321', new DateTimeZone('America/New_York')), 'offset_timezone' => new DateTimeImmutable('2025-06-15 12:30:45', new DateTimeZone('+02:30')), 'before_epoch' => new DateTimeImmutable('1969-07-20 20:17:00 UTC'), - 'date' => new DateTimeImmutable('2025-06-15 00:00:00', new DateTimeZone('Europe/Warsaw')), - ]), + 'date' => new DateTimeImmutable('2025-06-15 00:00:00', new DateTimeZone('UTC')), + ]], + schema( + datetime_schema('immutable', zone: 'Europe/Warsaw'), + datetime_schema('mutable', zone: 'America/New_York'), + datetime_schema('offset_timezone', zone: '+02:30'), + datetime_schema('before_epoch'), + date_schema('date'), + ), ), FloeStreamReaderContext::roundTrip($rows), ); @@ -87,7 +86,22 @@ public function test_datetime_entries(): void public function test_entries_with_null_values(): void { - $rows = rows( + $rows = array_to_rows( + [[ + 'int' => null, + 'float' => null, + 'bool' => null, + 'str' => null, + 'from_null' => null, + 'list' => null, + 'map' => null, + 'json' => null, + 'uuid' => null, + 'datetime' => null, + 'time' => null, + 'enum' => null, + 'xml' => null, + ]], schema( int_schema('int', nullable: true), float_schema('float', nullable: true), @@ -103,21 +117,6 @@ public function test_entries_with_null_values(): void enum_schema('enum', BackedStringEnum::class, nullable: true), xml_schema('xml', nullable: true), ), - row([ - 'int' => null, - 'float' => null, - 'bool' => null, - 'str' => null, - 'from_null' => null, - 'list' => null, - 'map' => null, - 'json' => null, - 'uuid' => null, - 'datetime' => null, - 'time' => null, - 'enum' => null, - 'xml' => null, - ]), ); static::assertEquals($rows, FloeStreamReaderContext::roundTrip($rows)); @@ -126,9 +125,9 @@ enum_schema('enum', BackedStringEnum::class, nullable: true), public function test_enum_entries(): void { static::assertEquals( - $rows = rows( + $rows = array_to_rows( + [['backed' => BackedStringEnum::two, 'basic' => BasicEnum::three]], schema(enum_schema('backed', BackedStringEnum::class), enum_schema('basic', BasicEnum::class)), - row(['backed' => BackedStringEnum::two, 'basic' => BasicEnum::three]), ), FloeStreamReaderContext::roundTrip($rows), ); @@ -137,14 +136,13 @@ public function test_enum_entries(): void #[RequiresPhp('>= 8.4.0')] public function test_html_entries(): void { - $rows = rows( - schema(html_schema('html')), - row(['html' => type_html()->cast('

hello

')]), - ); + $rows = array_to_rows([[ + 'html' => type_html()->cast('

hello

'), + ]], schema(html_schema('html'))); static::assertSame( - type_string()->cast($rows->first()->get('html')), - type_string()->cast(FloeStreamReaderContext::roundTrip($rows)->first()->get('html')), + type_string()->cast($rows->column('html')->value(0)), + type_string()->cast(FloeStreamReaderContext::roundTrip($rows)->column('html')->value(0)), ); } @@ -159,14 +157,14 @@ public function test_interval_entries(): void $fractional->f = 0.123456; static::assertEquals( - $rows = rows( - schema(time_schema('time'), time_schema('negative'), time_schema('fractional'), time_schema('days')), - row([ + $rows = array_to_rows( + [[ 'time' => new DateInterval('PT2H30M15S'), 'negative' => $negative, 'fractional' => $fractional, 'days' => new DateInterval('P3D'), - ]), + ]], + schema(time_schema('time'), time_schema('negative'), time_schema('fractional'), time_schema('days')), ), FloeStreamReaderContext::roundTrip($rows), ); @@ -175,19 +173,19 @@ public function test_interval_entries(): void public function test_json_entries(): void { static::assertEquals( - $rows = rows( + $rows = array_to_rows( + [[ + 'list' => type_json()->cast('[1,2,3]'), + 'object' => type_json()->cast('{"a":1,"b":[true,null]}'), + 'empty_object' => type_json()->cast('{}'), + 'empty_list' => type_json()->cast('[]'), + ]], schema( json_schema('list'), json_schema('object'), json_schema('empty_object'), json_schema('empty_list'), ), - row([ - 'list' => type_json()->cast('[1,2,3]'), - 'object' => type_json()->cast('{"a":1,"b":[true,null]}'), - 'empty_object' => type_json()->cast('{}'), - 'empty_list' => type_json()->cast('[]'), - ]), ), FloeStreamReaderContext::roundTrip($rows), ); @@ -196,7 +194,17 @@ public function test_json_entries(): void public function test_list_entries(): void { static::assertEquals( - $rows = rows( + $rows = array_to_rows( + [[ + 'int' => [1, -2, PHP_INT_MAX], + 'int_empty' => [], + 'float' => [1.5, -2.25], + 'float_empty' => [], + 'string' => ['a', 'b', ''], + 'bool' => [true, false], + 'nullable_int' => [1, null, 3], + 'list_of_lists' => [[1, 2], [3]], + ]], schema( list_schema('int', type_list(type_integer())), list_schema('int_empty', type_list(type_integer())), @@ -207,16 +215,6 @@ public function test_list_entries(): void list_schema('nullable_int', type_list(type_optional(type_integer()))), list_schema('list_of_lists', type_list(type_list(type_integer()))), ), - row([ - 'int' => [1, -2, PHP_INT_MAX], - 'int_empty' => [], - 'float' => [1.5, -2.25], - 'float_empty' => [], - 'string' => ['a', 'b', ''], - 'bool' => [true, false], - 'nullable_int' => [1, null, 3], - 'list_of_lists' => [[1, 2], [3]], - ]), ), FloeStreamReaderContext::roundTrip($rows), ); @@ -225,13 +223,13 @@ public function test_list_entries(): void public function test_map_entries(): void { static::assertEquals( - $rows = rows( + $rows = array_to_rows( + [['string_keys' => ['a' => 1, 'b' => 2], 'int_keys' => [10 => 'x', 20 => 'y'], 'empty' => []]], schema( map_schema('string_keys', type_map(type_string(), type_integer())), map_schema('int_keys', type_map(type_integer(), type_string())), map_schema('empty', type_map(type_string(), type_integer())), ), - row(['string_keys' => ['a' => 1, 'b' => 2], 'int_keys' => [10 => 'x', 20 => 'y'], 'empty' => []]), ), FloeStreamReaderContext::roundTrip($rows), ); @@ -240,13 +238,13 @@ public function test_map_entries(): void public function test_uuid_entries_round_trip_under_format_v2(): void { static::assertEquals( - $rows = rows( - schema(uuid_schema('uuid'), uuid_schema('uuid_max'), str_schema('after')), - row([ + $rows = array_to_rows( + [[ 'uuid' => type_uuid()->cast('0196aecb-b568-7e57-a381-8ec8d3e4a531'), 'uuid_max' => type_uuid()->cast('ffffffff-ffff-ffff-ffff-ffffffffffff'), 'after' => 'not desynchronised', - ]), + ]], + schema(uuid_schema('uuid'), uuid_schema('uuid_max'), str_schema('after')), ), FloeStreamReaderContext::roundTrip($rows), ); @@ -255,7 +253,20 @@ public function test_uuid_entries_round_trip_under_format_v2(): void public function test_scalar_entries(): void { static::assertEquals( - $rows = rows( + $rows = array_to_rows( + [[ + 'int' => 42, + 'int_min' => PHP_INT_MIN, + 'int_max' => PHP_INT_MAX, + 'float' => 3.14159, + 'float_negative' => -1.0E-10, + 'bool_true' => true, + 'bool_false' => false, + 'string' => 'hello', + 'string_empty' => '', + 'string_binary' => "line\nbreak\x00null\xFFbyte", + 'string_unicode' => 'zażółć gęślą jaźń 🚀', + ]], schema( int_schema('int'), int_schema('int_min'), @@ -269,19 +280,6 @@ public function test_scalar_entries(): void str_schema('string_binary'), str_schema('string_unicode'), ), - row([ - 'int' => 42, - 'int_min' => PHP_INT_MIN, - 'int_max' => PHP_INT_MAX, - 'float' => 3.14159, - 'float_negative' => -1.0E-10, - 'bool_true' => true, - 'bool_false' => false, - 'string' => 'hello', - 'string_empty' => '', - 'string_binary' => "line\nbreak\x00null\xFFbyte", - 'string_unicode' => 'zażółć gęślą jaźń 🚀', - ]), ), FloeStreamReaderContext::roundTrip($rows), ); @@ -296,7 +294,18 @@ public function test_structure_entries(): void ]); static::assertEquals( - $rows = rows( + $rows = array_to_rows( + [[ + 'full' => [ + 'street' => 'Main', + 'nested' => ['count' => 5, 'tags' => ['a']], + 'optional_zip' => '00-001', + ], + 'without_optional' => ['street' => 'Side', 'nested' => ['count' => 0, 'tags' => []]], + 'nullable_element' => ['street' => null, 'nested' => ['count' => 1, 'tags' => []]], + 'with_timezone' => ['tz' => new DateTimeZone('Europe/Warsaw')], + 'with_null_type' => ['nothing' => null], + ]], schema( structure_schema('full', $type), structure_schema('without_optional', $type), @@ -309,17 +318,6 @@ public function test_structure_entries(): void ])), structure_schema('with_null_type', type_structure(['nothing' => type_null()])), ), - row([ - 'full' => [ - 'street' => 'Main', - 'nested' => ['count' => 5, 'tags' => ['a']], - 'optional_zip' => '00-001', - ], - 'without_optional' => ['street' => 'Side', 'nested' => ['count' => 0, 'tags' => []]], - 'nullable_element' => ['street' => null, 'nested' => ['count' => 1, 'tags' => []]], - 'with_timezone' => ['tz' => new DateTimeZone('Europe/Warsaw')], - 'with_null_type' => ['nothing' => null], - ]), ), FloeStreamReaderContext::roundTrip($rows), ); @@ -334,9 +332,9 @@ public function test_interleaved_structure_entries(): void ]); static::assertEquals( - $rows = rows( + $rows = array_to_rows( + [['full' => ['z' => 1, 'a' => 'present', 'b' => 'x'], 'without_optional' => ['z' => 2, 'b' => 'y']]], schema(structure_schema('full', $type), structure_schema('without_optional', $type)), - row(['full' => ['z' => 1, 'a' => 'present', 'b' => 'x'], 'without_optional' => ['z' => 2, 'b' => 'y']]), ), FloeStreamReaderContext::roundTrip($rows), ); @@ -346,26 +344,22 @@ public function test_round_trip_rewrites_value_key_order_into_schema_order(): vo { $type = type_structure(['a' => type_integer(), 'b' => type_string()]); - $inOrder = rows(schema(structure_schema('s', $type)), row(['s' => ['a' => 1, 'b' => 'x']])); + $inOrder = array_to_rows([['s' => ['a' => 1, 'b' => 'x']]], schema(structure_schema('s', $type))); - static::assertSame( - $inOrder->first()->values(), - FloeStreamReaderContext::roundTrip($inOrder)->first()->values(), - ); + static::assertSame($inOrder->values(0), FloeStreamReaderContext::roundTrip($inOrder)->values(0)); - $outOfOrder = rows(schema(structure_schema('s', $type)), row(['s' => ['b' => 'x', 'a' => 1]])); - $decoded = FloeStreamReaderContext::roundTrip($outOfOrder)->first(); + $outOfOrder = array_to_rows([['s' => ['b' => 'x', 'a' => 1]]], schema(structure_schema('s', $type))); + $decoded = FloeStreamReaderContext::roundTrip($outOfOrder); - static::assertSame(['a' => 1, 'b' => 'x'], $decoded->get('s')); + static::assertSame(['a' => 1, 'b' => 'x'], $decoded->column('s')->value(0)); } public function test_uuid_entries(): void { static::assertEquals( - $rows = rows( - schema(uuid_schema('uuid')), - row(['uuid' => type_uuid()->cast('0196aecb-b568-7e57-a381-8ec8d3e4a531')]), - ), + $rows = array_to_rows([[ + 'uuid' => type_uuid()->cast('0196aecb-b568-7e57-a381-8ec8d3e4a531'), + ]], schema(uuid_schema('uuid'))), FloeStreamReaderContext::roundTrip($rows), ); } @@ -373,12 +367,12 @@ public function test_uuid_entries(): void public function test_xml_entries(): void { static::assertEquals( - $rows = rows( - schema(xml_schema('xml'), xml_element_schema('element')), - row([ + $rows = array_to_rows( + [[ 'xml' => type_xml()->cast('text & entity'), 'element' => type_xml_element()->cast('value'), - ]), + ]], + schema(xml_schema('xml'), xml_element_schema('element')), ), FloeStreamReaderContext::roundTrip($rows), ); diff --git a/src/core/etl/tests/Flow/Floe/Tests/Unit/FloeWriterTest.php b/src/core/etl/tests/Flow/Floe/Tests/Unit/FloeWriterTest.php index 849ae08380..7e0edd396d 100644 --- a/src/core/etl/tests/Flow/Floe/Tests/Unit/FloeWriterTest.php +++ b/src/core/etl/tests/Flow/Floe/Tests/Unit/FloeWriterTest.php @@ -5,6 +5,7 @@ namespace Flow\Floe\Tests\Unit; use DateTimeImmutable; +use Flow\ETL\Column\AdaptiveBackend; use Flow\ETL\Schema\Metadata; use Flow\Floe\Exception\FloeException; use Flow\Floe\Exception\IncompatibleSchemaException; @@ -16,10 +17,9 @@ use Flow\Floe\Tests\Double\UnsizedFilesystem; use PHPUnit\Framework\TestCase; +use function Flow\ETL\DSL\array_to_rows; use function Flow\ETL\DSL\datetime_schema; use function Flow\ETL\DSL\int_schema; -use function Flow\ETL\DSL\row; -use function Flow\ETL\DSL\rows; use function Flow\ETL\DSL\schema; use function Flow\ETL\DSL\str_schema; use function Flow\Filesystem\DSL\memory_filesystem; @@ -34,19 +34,18 @@ public function test_create_for_stream_is_byte_identical_to_create_by_path(): vo $filesystem = memory_filesystem(); $viaPath = path('memory://via-path.floe'); $viaStream = path('memory://via-stream.floe'); - $data = rows( + $data = array_to_rows( + [['id' => 1, 'name' => 'a'], ['id' => 2, 'name' => 'b']], schema(int_schema('id'), str_schema('name')), - row(['id' => 1, 'name' => 'a']), - row(['id' => 2, 'name' => 'b']), ); $schema = $data->schema(); - $byPath = new FloeWriter($filesystem, $schema); + $byPath = new FloeWriter($filesystem, $schema, new AdaptiveBackend()); $byPath->create($viaPath); $byPath->write($data); $byPath->close(); - $byStream = new FloeWriter($filesystem, $schema); + $byStream = new FloeWriter($filesystem, $schema, new AdaptiveBackend()); $byStream->createForStream($filesystem->writeTo($viaStream)); $byStream->write($data); $byStream->close(); @@ -59,20 +58,19 @@ public function test_buffer_size_only_changes_flush_cadence_not_bytes(): void $filesystem = memory_filesystem(); $default = path('memory://default-buffer.floe'); $tiny = path('memory://tiny-buffer.floe'); - $data = rows( + $data = array_to_rows( + [['id' => 1, 'name' => 'alpha'], ['id' => 2, 'name' => 'beta']], schema(int_schema('id'), str_schema('name')), - row(['id' => 1, 'name' => 'alpha']), - row(['id' => 2, 'name' => 'beta']), ); $schema = $data->schema(); - $defaultWriter = new FloeWriter($filesystem, $schema); + $defaultWriter = new FloeWriter($filesystem, $schema, new AdaptiveBackend()); $defaultWriter->create($default); $defaultWriter->write($data); $defaultWriter->close(); - $tinyWriter = new FloeWriter($filesystem, $schema, new Options(bufferSize: 4)); + $tinyWriter = new FloeWriter($filesystem, $schema, new AdaptiveBackend(), new Options(bufferSize: 4)); $tinyWriter->create($tiny); $tinyWriter->write($data); $tinyWriter->close(); @@ -85,9 +83,9 @@ public function test_append_on_missing_file_behaves_like_create(): void $filesystem = memory_filesystem(); $path = path('memory://missing.floe'); - $writer = new FloeWriter($filesystem, schema(int_schema('id'))); + $writer = new FloeWriter($filesystem, schema(int_schema('id')), new AdaptiveBackend()); $writer->append($path); - $writer->write(rows(schema(int_schema('id')), row(['id' => 1]))); + $writer->write(array_to_rows([['id' => 1]], schema(int_schema('id')))); $writer->close(); static::assertSame(1, FloeStreamReaderContext::footer($filesystem, $path)->statistics->rows); @@ -99,9 +97,9 @@ public function test_append_on_empty_file_behaves_like_create(): void $path = path('memory://zero-bytes.floe'); $filesystem->writeTo($path)->close(); - $writer = new FloeWriter($filesystem, schema(int_schema('id'))); + $writer = new FloeWriter($filesystem, schema(int_schema('id')), new AdaptiveBackend()); $writer->append($path); - $writer->write(rows(schema(int_schema('id')), row(['id' => 1]))); + $writer->write(array_to_rows([['id' => 1]], schema(int_schema('id')))); $writer->close(); static::assertSame(1, FloeStreamReaderContext::footer($filesystem, $path)->statistics->rows); @@ -112,13 +110,13 @@ public function test_append_to_a_closed_zero_row_file_derives_the_schema_from_th $filesystem = memory_filesystem(); $path = path('memory://zero-rows.floe'); - $writer = new FloeWriter($filesystem, schema()); + $writer = new FloeWriter($filesystem, schema(), new AdaptiveBackend()); $writer->create($path); $writer->close(); - $writer = new FloeWriter($filesystem, schema(int_schema('id'))); + $writer = new FloeWriter($filesystem, schema(int_schema('id')), new AdaptiveBackend()); $writer->append($path); - $writer->write(rows(schema(int_schema('id')), row(['id' => 1]))); + $writer->write(array_to_rows([['id' => 1]], schema(int_schema('id')))); $writer->close(); static::assertSame([['id' => 1]], FloeStreamReaderContext::readAll($filesystem, $path)->toArray()); @@ -129,17 +127,17 @@ public function test_append_to_a_zero_row_file_with_an_explicit_schema_validates $filesystem = memory_filesystem(); $path = path('memory://zero-rows-schema.floe'); - $writer = new FloeWriter($filesystem, schema(int_schema('id'))); + $writer = new FloeWriter($filesystem, schema(int_schema('id')), new AdaptiveBackend()); $writer->create($path); $writer->close(); - $writer = new FloeWriter($filesystem, schema(int_schema('id'))); + $writer = new FloeWriter($filesystem, schema(int_schema('id')), new AdaptiveBackend()); $writer->append($path); $this->expectException(IncompatibleSchemaException::class); $this->expectExceptionMessage('new column "name"'); - $writer->write(rows(schema(str_schema('name')), row(['name' => 'flow']))); + $writer->write(array_to_rows([['name' => 'flow']], schema(str_schema('name')))); } public function test_create_with_an_explicit_schema_and_no_writes_records_the_schema_in_the_footer(): void @@ -147,7 +145,7 @@ public function test_create_with_an_explicit_schema_and_no_writes_records_the_sc $filesystem = memory_filesystem(); $path = path('memory://schema-only.floe'); - $writer = new FloeWriter($filesystem, schema(int_schema('id'))); + $writer = new FloeWriter($filesystem, schema(int_schema('id')), new AdaptiveBackend()); $writer->create($path); $writer->close(); @@ -162,14 +160,14 @@ public function test_append_over_unsized_stream_throws(): void $filesystem = memory_filesystem(); $path = path('memory://unsized.floe'); - $writer = new FloeWriter($filesystem, schema()); + $writer = new FloeWriter($filesystem, schema(), new AdaptiveBackend()); $writer->create($path); $writer->close(); $this->expectException(FloeException::class); $this->expectExceptionMessage('requires a sized stream'); - (new FloeWriter(new UnsizedFilesystem($filesystem), schema()))->append($path); + (new FloeWriter(new UnsizedFilesystem($filesystem), schema(), new AdaptiveBackend()))->append($path); } public function test_append_with_footer_length_exceeding_file_throws(): void @@ -183,7 +181,7 @@ public function test_append_with_footer_length_exceeding_file_throws(): void $this->expectException(FloeException::class); $this->expectExceptionMessage('footer does not fit inside the file'); - (new FloeWriter($filesystem, schema()))->append($path); + (new FloeWriter($filesystem, schema(), new AdaptiveBackend()))->append($path); } public function test_append_metadata_merges_over_existing_footer_metadata(): void @@ -191,10 +189,10 @@ public function test_append_metadata_merges_over_existing_footer_metadata(): voi $filesystem = memory_filesystem(); $path = path('memory://meta.floe'); - $writer = new FloeWriter($filesystem, schema()); + $writer = new FloeWriter($filesystem, schema(), new AdaptiveBackend()); $writer->create($path, Metadata::fromArray(['source' => 'create', 'kept' => 'yes'])); $writer->close(); - $writer = new FloeWriter($filesystem, schema()); + $writer = new FloeWriter($filesystem, schema(), new AdaptiveBackend()); $writer->append($path, Metadata::fromArray(['source' => 'append'])); $writer->close(); @@ -215,7 +213,7 @@ public function test_append_to_file_with_different_codec_flags_throws(): void $this->expectException(FloeException::class); $this->expectExceptionMessage('written with codec 0x05'); - (new FloeWriter($filesystem, schema()))->append($path); + (new FloeWriter($filesystem, schema(), new AdaptiveBackend()))->append($path); } public function test_append_to_torn_file_throws(): void @@ -223,13 +221,13 @@ public function test_append_to_torn_file_throws(): void $filesystem = memory_filesystem(); $path = path('memory://torn.floe'); $stream = $filesystem->writeTo($path); - $stream->append(Format::header(0x00) . Format::frame(Format::FRAME_ROW, 'not closed')); + $stream->append(Format::header(0x00) . Format::frame(Format::FRAME_BATCH, 'not closed')); $stream->close(); $this->expectException(FloeException::class); $this->expectExceptionMessage('torn'); - (new FloeWriter($filesystem, schema()))->append($path); + (new FloeWriter($filesystem, schema(), new AdaptiveBackend()))->append($path); } public function test_append_to_truncated_stub_throws(): void @@ -243,7 +241,7 @@ public function test_append_to_truncated_stub_throws(): void $this->expectException(FloeException::class); $this->expectExceptionMessage('too small'); - (new FloeWriter($filesystem, schema()))->append($path); + (new FloeWriter($filesystem, schema(), new AdaptiveBackend()))->append($path); } public function test_append_with_incompatible_schema_keeps_previous_rows_readable(): void @@ -251,16 +249,16 @@ public function test_append_with_incompatible_schema_keeps_previous_rows_readabl $filesystem = memory_filesystem(); $path = path('memory://safe.floe'); - $writer = new FloeWriter($filesystem, schema(int_schema('id'))); + $writer = new FloeWriter($filesystem, schema(int_schema('id')), new AdaptiveBackend()); $writer->create($path); - $writer->write(rows(schema(int_schema('id')), row(['id' => 1]))); + $writer->write(array_to_rows([['id' => 1]], schema(int_schema('id')))); $writer->close(); - $writer = new FloeWriter($filesystem, schema(int_schema('id'))); + $writer = new FloeWriter($filesystem, schema(int_schema('id')), new AdaptiveBackend()); $writer->append($path); try { - $writer->write(rows(schema(str_schema('id')), row(['id' => 'no longer an int']))); + $writer->write(array_to_rows([['id' => 'no longer an int']], schema(str_schema('id')))); static::fail('expected ' . IncompatibleSchemaException::class); } catch (IncompatibleSchemaException) { } @@ -278,17 +276,17 @@ public function test_append_with_new_non_nullable_column_throws(): void $filesystem = memory_filesystem(); $path = path('memory://evolve.floe'); - $writer = new FloeWriter($filesystem, schema(int_schema('id'))); + $writer = new FloeWriter($filesystem, schema(int_schema('id')), new AdaptiveBackend()); $writer->create($path); - $writer->write(rows(schema(int_schema('id')), row(['id' => 1]))); + $writer->write(array_to_rows([['id' => 1]], schema(int_schema('id')))); $writer->close(); $this->expectException(IncompatibleSchemaException::class); $this->expectExceptionMessage('new column "email"'); - $writer = new FloeWriter($filesystem, schema(int_schema('id'))); + $writer = new FloeWriter($filesystem, schema(int_schema('id')), new AdaptiveBackend()); $writer->append($path); - $writer->write(rows(schema(int_schema('id'), str_schema('email')), row(['id' => 2, 'email' => 'x']))); + $writer->write(array_to_rows([['id' => 2, 'email' => 'x']], schema(int_schema('id'), str_schema('email')))); } public function test_append_with_new_nullable_column_throws(): void @@ -296,20 +294,20 @@ public function test_append_with_new_nullable_column_throws(): void $filesystem = memory_filesystem(); $path = path('memory://merge.floe'); - $writer = new FloeWriter($filesystem, schema(int_schema('id'))); + $writer = new FloeWriter($filesystem, schema(int_schema('id')), new AdaptiveBackend()); $writer->create($path); - $writer->write(rows(schema(int_schema('id')), row(['id' => 1]))); + $writer->write(array_to_rows([['id' => 1]], schema(int_schema('id')))); $writer->close(); - $writer = new FloeWriter($filesystem, schema(int_schema('id'))); + $writer = new FloeWriter($filesystem, schema(int_schema('id')), new AdaptiveBackend()); $writer->append($path); $this->expectException(IncompatibleSchemaException::class); $this->expectExceptionMessage('new column "email"'); - $writer->write(rows( + $writer->write(array_to_rows( + [['id' => 2, 'email' => null]], schema(int_schema('id'), str_schema('email', nullable: true)), - row(['id' => 2, 'email' => null]), )); } @@ -318,18 +316,18 @@ public function test_append_with_same_schema_adds_a_section_reusing_the_footer_s $filesystem = memory_filesystem(); $path = path('memory://same-schema.floe'); - $writer = new FloeWriter($filesystem, schema(int_schema('id'))); + $writer = new FloeWriter($filesystem, schema(int_schema('id')), new AdaptiveBackend()); $writer->create($path); - $writer->write(rows(schema(int_schema('id')), row(['id' => 1]))); + $writer->write(array_to_rows([['id' => 1]], schema(int_schema('id')))); $writer->close(); - $writer = new FloeWriter($filesystem, schema(int_schema('id'))); + $writer = new FloeWriter($filesystem, schema(int_schema('id')), new AdaptiveBackend()); $writer->append($path); - $writer->write(rows(schema(int_schema('id')), row(['id' => 2]))); + $writer->write(array_to_rows([['id' => 2]], schema(int_schema('id')))); $writer->close(); static::assertSame( - [Format::FRAME_ROW, Format::FRAME_FOOTER, Format::FRAME_ROW, Format::FRAME_FOOTER], + [Format::FRAME_BATCH, Format::FRAME_FOOTER, Format::FRAME_BATCH, Format::FRAME_FOOTER], FloeStreamReaderContext::frameTypes($filesystem, $path), ); @@ -342,7 +340,7 @@ public function test_append_with_same_schema_adds_a_section_reusing_the_footer_s public function test_close_twice_throws(): void { - $writer = new FloeWriter(memory_filesystem(), schema()); + $writer = new FloeWriter(memory_filesystem(), schema(), new AdaptiveBackend()); $writer->create(path('memory://closed.floe')); $writer->close(); @@ -354,7 +352,7 @@ public function test_close_twice_throws(): void public function test_opening_a_second_session_throws(): void { - $writer = new FloeWriter(memory_filesystem(), schema()); + $writer = new FloeWriter(memory_filesystem(), schema(), new AdaptiveBackend()); $writer->create(path('memory://first.floe')); $this->expectException(FloeException::class); @@ -368,7 +366,7 @@ public function test_closing_empty_file_writes_header_and_footer_only(): void $filesystem = memory_filesystem(); $path = path('memory://empty.floe'); - $writer = new FloeWriter($filesystem, schema()); + $writer = new FloeWriter($filesystem, schema(), new AdaptiveBackend()); $writer->create($path); $writer->close(); @@ -386,9 +384,12 @@ public function test_create_with_non_noop_codec_throws(): void $this->expectException(FloeException::class); $this->expectExceptionMessage('supports only the no-op codec, got codec 0x05'); - (new FloeWriter(memory_filesystem(), schema(), new Options(codec: new CodecStub(0x05))))->create(path( - 'memory://codec.floe', - )); + (new FloeWriter( + memory_filesystem(), + schema(), + new AdaptiveBackend(), + new Options(codec: new CodecStub(0x05)), + ))->create(path('memory://codec.floe')); } public function test_two_writes_with_the_same_schema_in_one_session_share_one_section(): void @@ -396,10 +397,10 @@ public function test_two_writes_with_the_same_schema_in_one_session_share_one_se $filesystem = memory_filesystem(); $path = path('memory://continuation.floe'); - $writer = new FloeWriter($filesystem, schema(int_schema('id'))); + $writer = new FloeWriter($filesystem, schema(int_schema('id')), new AdaptiveBackend()); $writer->create($path); - $writer->write(rows(schema(int_schema('id')), row(['id' => 1]))); - $writer->write(rows(schema(int_schema('id')), row(['id' => 2]))); + $writer->write(array_to_rows([['id' => 1]], schema(int_schema('id')))); + $writer->write(array_to_rows([['id' => 2]], schema(int_schema('id')))); $writer->close(); $footer = FloeStreamReaderContext::footer($filesystem, $path); @@ -407,7 +408,7 @@ public function test_two_writes_with_the_same_schema_in_one_session_share_one_se static::assertSame(2, $footer->statistics->rows); static::assertCount(1, $footer->sections); static::assertSame( - [Format::FRAME_ROW, Format::FRAME_ROW, Format::FRAME_FOOTER], + [Format::FRAME_BATCH, Format::FRAME_BATCH, Format::FRAME_FOOTER], FloeStreamReaderContext::frameTypes($filesystem, $path), ); } @@ -417,13 +418,13 @@ public function test_nothing_reaches_the_stream_before_buffer_fills_or_close(): $filesystem = memory_filesystem(); $path = path('memory://buffered.floe'); - $writer = new FloeWriter($filesystem, schema(str_schema('data'))); + $writer = new FloeWriter($filesystem, schema(str_schema('data')), new AdaptiveBackend()); $writer->create($path); - $writer->write(rows(schema(str_schema('data')), row(['data' => 'x']))); + $writer->write(array_to_rows([['data' => 'x']], schema(str_schema('data')))); static::assertSame(0, $filesystem->readFrom($path)->size()); - $writer->write(rows(schema(str_schema('data')), row(['data' => str_repeat('x', 70_000)]))); + $writer->write(array_to_rows([['data' => str_repeat('x', 70_000)]], schema(str_schema('data')))); static::assertGreaterThan(0, $filesystem->readFrom($path)->size()); } @@ -433,14 +434,12 @@ public function test_within_batch_heterogeneous_rows_encode_under_one_union_sche $filesystem = memory_filesystem(); $path = path('memory://sections.floe'); - $data = rows( + $data = array_to_rows( + [['a' => 1, 'b' => 'x'], ['a' => 2, 'c' => 'y'], ['a' => 3, 'b' => 'z']], schema(int_schema('a'), str_schema('b', nullable: true), str_schema('c', nullable: true)), - row(['a' => 1, 'b' => 'x']), - row(['a' => 2, 'c' => 'y']), - row(['a' => 3, 'b' => 'z']), ); - $writer = new FloeWriter($filesystem, $data->schema()); + $writer = new FloeWriter($filesystem, $data->schema(), new AdaptiveBackend()); $writer->create($path); $writer->write($data); $writer->close(); @@ -456,7 +455,7 @@ public function test_within_batch_heterogeneous_rows_encode_under_one_union_sche static::assertNotNull($fileSchema->findDefinition('a')); static::assertTrue($fileSchema->findDefinition('b')?->isNullable()); static::assertTrue($fileSchema->findDefinition('c')?->isNullable()); - static::assertCount(3, FloeStreamReaderContext::readAll($filesystem, $path)->all()); + static::assertCount(3, FloeStreamReaderContext::readAll($filesystem, $path)); } public function test_section_offsets_point_at_frames(): void @@ -464,13 +463,12 @@ public function test_section_offsets_point_at_frames(): void $filesystem = memory_filesystem(); $path = path('memory://offsets.floe'); - $data = rows( + $data = array_to_rows( + [['a' => 1], ['b' => 'x']], schema(int_schema('a', nullable: true), str_schema('b', nullable: true)), - row(['a' => 1]), - row(['b' => 'x']), ); - $writer = new FloeWriter($filesystem, $data->schema()); + $writer = new FloeWriter($filesystem, $data->schema(), new AdaptiveBackend()); $writer->create($path); $writer->write($data); $writer->close(); @@ -481,20 +479,20 @@ public function test_section_offsets_point_at_frames(): void static::assertSame(Format::HEADER_LENGTH, $footer->sections[0]->offset); foreach ($footer->sections as $section) { - static::assertSame(Format::FRAME_ROW, ord($source->read(1, $section->offset))); + static::assertSame(Format::FRAME_BATCH, ord($source->read(1, $section->offset))); } } public function test_write_after_close_throws(): void { - $writer = new FloeWriter(memory_filesystem(), schema()); + $writer = new FloeWriter(memory_filesystem(), schema(), new AdaptiveBackend()); $writer->create(path('memory://done.floe')); $writer->close(); $this->expectException(FloeException::class); $this->expectExceptionMessage('Floe writer session is not open'); - $writer->write(rows(schema(int_schema('id')), row(['id' => 1]))); + $writer->write(array_to_rows([['id' => 1]], schema(int_schema('id')))); } public function test_append_refuses_a_different_datetime_zone(): void @@ -502,17 +500,20 @@ public function test_append_refuses_a_different_datetime_zone(): void $filesystem = memory_filesystem(); $path = path('memory://zoned.floe'); - $writer = new FloeWriter($filesystem, schema(datetime_schema('at'))); + $writer = new FloeWriter($filesystem, schema(datetime_schema('at')), new AdaptiveBackend()); $writer->create($path); - $writer->write(rows( - schema(datetime_schema('at')), - row(['at' => new DateTimeImmutable('2026-01-02 03:04:05')]), - )); + $writer->write(array_to_rows([[ + 'at' => new DateTimeImmutable('2026-01-02 03:04:05'), + ]], schema(datetime_schema('at')))); $writer->close(); $this->expectException(IncompatibleSchemaException::class); $this->expectExceptionMessage('Floe append schema does not match the existing file schema.'); - (new FloeWriter($filesystem, schema(datetime_schema('at', zone: 'Europe/Warsaw'))))->append($path); + (new FloeWriter( + $filesystem, + schema(datetime_schema('at', zone: 'Europe/Warsaw')), + new AdaptiveBackend(), + ))->append($path); } } diff --git a/src/core/etl/tests/Flow/Floe/Tests/Unit/FooterReaderTest.php b/src/core/etl/tests/Flow/Floe/Tests/Unit/FooterReaderTest.php index 28e5f7e642..86dcd665f0 100644 --- a/src/core/etl/tests/Flow/Floe/Tests/Unit/FooterReaderTest.php +++ b/src/core/etl/tests/Flow/Floe/Tests/Unit/FooterReaderTest.php @@ -4,6 +4,7 @@ namespace Flow\Floe\Tests\Unit; +use Flow\ETL\Column\AdaptiveBackend; use Flow\Filesystem\Stream\MemorySourceStream; use Flow\Floe\Codec\NoopCodec; use Flow\Floe\Exception\FloeException; @@ -15,9 +16,8 @@ use PHPUnit\Framework\TestCase; use function chr; +use function Flow\ETL\DSL\array_to_rows; use function Flow\ETL\DSL\int_schema; -use function Flow\ETL\DSL\row; -use function Flow\ETL\DSL\rows; use function Flow\ETL\DSL\schema; use function Flow\Filesystem\DSL\memory_filesystem; use function Flow\Filesystem\DSL\path; @@ -30,8 +30,8 @@ public function test_reads_footer_location_of_a_written_file(): void { $fs = memory_filesystem(); $path = path('memory://footer.floe'); - $data = rows(schema(int_schema('id')), row(['id' => 1]), row(['id' => 2])); - $writer = new FloeWriter($fs, $data->schema()); + $data = array_to_rows([['id' => 1], ['id' => 2]], schema(int_schema('id'))); + $writer = new FloeWriter($fs, $data->schema(), new AdaptiveBackend()); $writer->create($path); $writer->write($data); $writer->close(); @@ -47,7 +47,7 @@ public function test_unsized_stream_throws(): void { $fs = memory_filesystem(); $path = path('memory://unsized.floe'); - $writer = new FloeWriter($fs, schema()); + $writer = new FloeWriter($fs, schema(), new AdaptiveBackend()); $writer->create($path); $writer->close(); @@ -95,8 +95,8 @@ public function test_leaves_the_source_open_on_success(): void { $fs = memory_filesystem(); $path = path('memory://close-ok.floe'); - $data = rows(schema(int_schema('id')), row(['id' => 1])); - $writer = new FloeWriter($fs, $data->schema()); + $data = array_to_rows([['id' => 1]], schema(int_schema('id'))); + $writer = new FloeWriter($fs, $data->schema(), new AdaptiveBackend()); $writer->create($path); $writer->write($data); $writer->close(); diff --git a/src/core/etl/tests/Flow/Floe/Tests/Unit/FooterTest.php b/src/core/etl/tests/Flow/Floe/Tests/Unit/FooterTest.php index ecd34de58e..bbfbc3067a 100644 --- a/src/core/etl/tests/Flow/Floe/Tests/Unit/FooterTest.php +++ b/src/core/etl/tests/Flow/Floe/Tests/Unit/FooterTest.php @@ -4,6 +4,7 @@ namespace Flow\Floe\Tests\Unit; +use Flow\ETL\Column\AdaptiveBackend; use Flow\Floe\Exception\FloeException; use Flow\Floe\FloeWriter; use Flow\Floe\Footer; @@ -12,8 +13,8 @@ use Flow\Floe\Tests\Mother\FooterMother; use PHPUnit\Framework\TestCase; +use function Flow\ETL\DSL\array_to_rows; use function Flow\ETL\DSL\int_schema; -use function Flow\ETL\DSL\row; use function Flow\ETL\DSL\rows; use function Flow\ETL\DSL\schema; use function Flow\ETL\DSL\str_schema; @@ -193,9 +194,9 @@ public function test_reconstruct_rows_round_trips_unpartitioned_rows(): void { $filesystem = memory_filesystem(); $path = path('memory://footer-rows.floe'); - $value = rows(schema(int_schema('id')), row(['id' => 1]), row(['id' => 2])); + $value = array_to_rows([['id' => 1], ['id' => 2]], schema(int_schema('id'))); - $writer = new FloeWriter($filesystem, $value->schema()); + $writer = new FloeWriter($filesystem, $value->schema(), new AdaptiveBackend()); $writer->create($path); $writer->write($value); $writer->close(); @@ -209,7 +210,7 @@ public function test_reconstruct_rows_round_trips_empty_rows(): void $path = path('memory://footer-empty.floe'); $value = rows(schema()); - $writer = new FloeWriter($filesystem, $value->schema()); + $writer = new FloeWriter($filesystem, $value->schema(), new AdaptiveBackend()); $writer->create($path); $writer->write($value); $writer->close(); diff --git a/src/core/etl/tests/Flow/Floe/Tests/Unit/FormatTest.php b/src/core/etl/tests/Flow/Floe/Tests/Unit/FormatTest.php index 63390952cf..3cbc74c9f5 100644 --- a/src/core/etl/tests/Flow/Floe/Tests/Unit/FormatTest.php +++ b/src/core/etl/tests/Flow/Floe/Tests/Unit/FormatTest.php @@ -16,16 +16,7 @@ final class FormatTest extends TestCase { public function test_frame_prefixes_type_and_length(): void { - static::assertSame(chr(Format::FRAME_ROW) . pack('V', 3) . 'abc', Format::frame(Format::FRAME_ROW, 'abc')); - } - - public function test_row_frames_frames_every_body_in_order(): void - { - static::assertSame( - chr(Format::FRAME_ROW) . pack('V', 1) . 'a' . chr(Format::FRAME_ROW) . pack('V', 0), - Format::rowFrames(['a', '']), - ); - static::assertSame('', Format::rowFrames([])); + static::assertSame(chr(Format::FRAME_BATCH) . pack('V', 3) . 'abc', Format::frame(Format::FRAME_BATCH, 'abc')); } public function test_header_starts_with_magic_version_and_flags(): void diff --git a/src/core/etl/tests/Flow/Floe/Tests/Unit/FrameDecoderTest.php b/src/core/etl/tests/Flow/Floe/Tests/Unit/FrameDecoderTest.php new file mode 100644 index 0000000000..e749253715 --- /dev/null +++ b/src/core/etl/tests/Flow/Floe/Tests/Unit/FrameDecoderTest.php @@ -0,0 +1,280 @@ + + */ + public static function round_trips(): Generator + { + $cases = [ + 'all entry types' => RowsMother::withAllEntryTypes(), + 'worked example' => RowsMother::workedExample(), + 'list with a null slot' => array_to_rows([ + ['l' => [1, 2]], + ['l' => null], + ['l' => []], + ], schema(list_schema('l', type_list(type_integer()), nullable: true))), + 'map' => array_to_rows([ + ['m' => ['a' => 1]], + ['m' => ['b' => 2, 'c' => 3]], + ], schema(map_schema('m', type_map(type_string(), type_integer())))), + 'structure with a null parent' => array_to_rows([ + ['s' => ['a' => 1]], + ['s' => null], + ], schema(structure_schema('s', type_structure(['a' => type_integer()]), nullable: true))), + 'null column' => array_to_rows([['n' => null], ['n' => null]], schema(null_schema('n'))), + 'list of nulls' => array_to_rows([ + ['l' => [null, null]], + ['l' => [null]], + ], schema(list_schema('l', type_list(type_null())))), + 'map of null values' => array_to_rows([['m' => [ + 'a' => null, + ]]], schema(map_schema('m', type_map(type_string(), type_null())))), + 'no columns' => array_to_rows([[], [], []], schema()), + ]; + + foreach ($cases as $name => $rows) { + yield $name . ' / noop' => [$rows, new NoopCodec()]; + yield $name . ' / trailing zeros' => [$rows, new TrailingZerosCodecStub()]; + yield $name . ' / prefixing' => [$rows, new PrefixingCodecStub()]; + } + } + + #[DataProvider('round_trips')] + public function test_round_trips(Rows $rows, Codec $codec): void + { + $decoded = (new FrameDecoder($codec))->decode( + (new FrameEncoder($codec))->encode($rows), + $rows->schema(), + new PhpBackend(), + ); + + static::assertSame($rows->count(), $decoded->count()); + static::assertEquals($rows->toArray(), $decoded->toArray()); + } + + public function test_a_numeric_column_name_round_trips(): void + { + $rows = array_to_rows([['0' => 1], ['0' => null]], schema(int_schema('0', nullable: true))); + + static::assertSame( + $rows->toArray(), + (new FrameDecoder()) + ->decode($rows->encodeFrame(), $rows->schema(), new PhpBackend()) + ->toArray(), + ); + } + + public function test_a_null_struct_child_under_a_null_parent_is_accepted(): void + { + $rows = array_to_rows([ + ['s' => ['a' => 1, 'b' => 'x']], + ['s' => null], + ], schema(structure_schema('s', type_structure(['a' => type_integer(), 'b' => type_string()]), nullable: true))); + + static::assertSame( + $rows->toArray(), + (new FrameDecoder()) + ->decode($rows->encodeFrame(), $rows->schema(), new PhpBackend()) + ->toArray(), + ); + } + + /** + * @return Generator + */ + public static function refusals(): Generator + { + $body = RowsMother::workedExample()->encodeFrame(); + + yield 'a body shorter than the directory counts' => [ + substr($body, 0, 11), + 'Floe BATCH frame is truncated, its directory is incomplete', + ]; + yield 'a directory cut short' => [ + substr($body, 0, 40), + 'Floe BATCH frame is truncated, its directory is incomplete', + ]; + yield 'a node count other than the schema' => [ + FrameContext::patch($body, [1 => 3]), + 'Floe BATCH frame holds 3 nodes, the schema describes 2', + ]; + yield 'a buffer count other than the schema' => [ + FrameContext::patch($body, [2 => 4]), + 'Floe BATCH frame holds 4 buffers, the schema describes 5', + ]; + yield 'a column of another row count' => [ + FrameContext::patch($body, [3 => 3]), + 'Floe BATCH frame column "id" holds 3 rows, the frame 2', + ]; + yield 'more nulls than rows' => [ + FrameContext::patch($body, [4 => 3]), + 'Floe BATCH frame column "id" has 3 nulls in 2 rows', + ]; + yield 'a buffer past the area' => [ + FrameContext::patch($body, [10 => 200]), + 'Floe BATCH frame buffer 1 lies outside the 80-byte buffer area', + ]; + yield 'a buffer shorter than its prefix' => [ + FrameContext::patch($body, [10 => 4]), + 'Floe BATCH frame buffer 1 is shorter than its 8-byte length prefix', + ]; + yield 'a null count other than the bitmap' => [ + FrameContext::patch($body, [6 => 0]), + 'Floe BATCH frame column "name" is malformed: Column null count 0 disagrees with its validity bitmap (1 nulls in 2 rows)', + ]; + } + + #[DataProvider('refusals')] + public function test_refuses(string $body, string $message): void + { + $this->expectException(FloeException::class); + $this->expectExceptionMessage($message); + + (new FrameDecoder())->decode($body, RowsMother::workedExample()->schema(), new PhpBackend()); + } + + public function test_refuses_a_buffer_that_decodes_to_another_length_than_its_prefix(): void + { + $body = (new FrameEncoder(new TrailingZerosCodecStub()))->encode(RowsMother::workedExample()); + + $this->expectException(FloeException::class); + $this->expectExceptionMessage('Floe BATCH frame buffer 1 decodes to 16 bytes, its prefix declares 15'); + + (new FrameDecoder(new TrailingZerosCodecStub()))->decode( + substr_replace($body, pack('P', 15), 72, 8), + RowsMother::workedExample()->schema(), + new PhpBackend(), + ); + } + + public function test_refuses_child_nodes_that_disagree_with_the_buffers(): void + { + $rows = array_to_rows([ + ['l' => [1, 2]], + ['l' => null], + ], schema(list_schema('l', type_list(type_integer()), nullable: true))); + + $this->expectException(FloeException::class); + $this->expectExceptionMessage('Floe BATCH frame column "l" declares nodes that disagree with its buffers'); + + (new FrameDecoder())->decode( + FrameContext::patch($rows->encodeFrame(), [5 => 3]), + $rows->schema(), + new PhpBackend(), + ); + } + + public function test_refuses_a_backend_refusal_as_malformed(): void + { + $body = FrameContext::patch(RowsMother::workedExample()->encodeFrame(), [10 => 10]); + + try { + (new FrameDecoder())->decode($body, RowsMother::workedExample()->schema(), new PhpBackend()); + static::fail('A cut values buffer must be refused'); + } catch (FloeException $e) { + static::assertSame( + 'Floe BATCH frame column "id" is malformed: Int64 values buffer of 2 bytes, expected 16 for 2 rows', + $e->getMessage(), + ); + static::assertInstanceOf(InvalidArgumentException::class, $e->getPrevious()); + } + } + + public function test_a_not_null_column_with_a_null_node_is_refused(): void + { + $body = array_to_rows([['id' => 1], ['id' => null]], schema(int_schema('id', nullable: true)))->encodeFrame(); + + $this->expectException(FloeException::class); + $this->expectExceptionMessage( + 'Floe BATCH frame is malformed: Rows do not match their schema: column "id" (row 1): could not convert null to integer, column is not nullable', + ); + + (new FrameDecoder())->decode($body, schema(int_schema('id')), new PhpBackend()); + } + + public function test_a_null_in_a_non_optional_list_element_is_refused(): void + { + $body = array_to_rows([['l' => [ + 1, + null, + ]]], schema(list_schema('l', type_list(type_optional(type_integer())))))->encodeFrame(); + + try { + (new FrameDecoder())->decode($body, schema(list_schema('l', type_list(type_integer()))), new PhpBackend()); + static::fail('A null list element must be refused'); + } catch (FloeException $e) { + static::assertSame( + 'Floe BATCH frame column "l" is malformed: List child holds 1 nulls in a non-nullable integer', + $e->getMessage(), + ); + static::assertInstanceOf(InvalidArgumentException::class, $e->getPrevious()); + } + } + + public function test_a_null_map_key_is_refused(): void + { + $rows = array_to_rows([['m' => [ + 1 => null, + ]]], schema(map_schema('m', type_map(type_integer(), type_optional(type_integer()))))); + $words = FrameContext::directory($rows->encodeFrame()); + $body = FrameContext::patch($rows->encodeFrame(), [ + 8 => 1, + 10 => 0, + 17 => $words[21], + 18 => $words[22], + 21 => $words[17], + 22 => $words[18], + ]); + + try { + (new FrameDecoder())->decode($body, $rows->schema(), new PhpBackend()); + static::fail('A null map key must be refused'); + } catch (FloeException $e) { + static::assertSame( + 'Floe BATCH frame column "m" is malformed: Map key child holds 1 nulls in a non-nullable integer', + $e->getMessage(), + ); + static::assertInstanceOf(InvalidArgumentException::class, $e->getPrevious()); + } + } +} diff --git a/src/core/etl/tests/Flow/Floe/Tests/Unit/FrameEncoderTest.php b/src/core/etl/tests/Flow/Floe/Tests/Unit/FrameEncoderTest.php new file mode 100644 index 0000000000..0ea366bad1 --- /dev/null +++ b/src/core/etl/tests/Flow/Floe/Tests/Unit/FrameEncoderTest.php @@ -0,0 +1,104 @@ +encode(RowsMother::workedExample())), + ); + } + + public function test_a_codec_wraps_each_buffer_behind_its_length(): void + { + static::assertSame( + '020000000200000005000000020000000000000002000000010000000000000000000000000000001200000018000000090000002800000012000000400000000a00000000000000100000000000000001000000000000000207000000000000ffffffffffffffff01000000000000000c0000000000000000000000020000000203000000000000ffffffffffffffff6162000000000000', + bin2hex((new FrameEncoder(new TrailingZerosCodecStub()))->encode(RowsMother::workedExample())), + ); + } + + public function test_a_codec_whose_output_is_not_smaller_stores_raw(): void + { + static::assertSame( + (new FrameEncoder())->encode(RowsMother::workedExample()), + (new FrameEncoder(new PrefixingCodecStub()))->encode(RowsMother::workedExample()), + ); + } + + public function test_a_list_with_a_null_slot(): void + { + $body = (new FrameEncoder())->encode(array_to_rows([ + ['l' => [1, 2]], + ['l' => null], + ], schema(list_schema('l', type_list(type_integer()), nullable: true)))); + + static::assertSame([2, 2, 4, 2, 1, 2, 0, 0, 9, 16, 20, 0, 0, 40, 24], FrameContext::directory($body)); + } + + public function test_a_map_entries_node_and_extent(): void + { + $body = (new FrameEncoder())->encode(array_to_rows([ + ['m' => ['a' => 1]], + ['m' => ['b' => 2, 'c' => 3]], + ], schema(map_schema('m', type_map(type_string(), type_integer()))))); + + static::assertSame( + [2, 4, 8, 2, 0, 3, 0, 3, 0, 3, 0, 0, 0, 0, 20, 0, 0, 0, 0, 24, 24, 48, 11, 0, 0, 64, 32], + FrameContext::directory($body), + ); + } + + public function test_a_structure_with_a_null_parent(): void + { + $body = (new FrameEncoder())->encode(array_to_rows([ + ['s' => ['a' => 1]], + ['s' => null], + ], schema(structure_schema('s', type_structure(['a' => type_integer()]), nullable: true)))); + + static::assertSame([2, 2, 3, 2, 1, 2, 1, 0, 9, 16, 9, 32, 24], FrameContext::directory($body)); + } + + public function test_a_null_column_has_a_full_null_node_and_no_extent(): void + { + static::assertSame( + '020000000100000000000000020000000200000000000000', + bin2hex((new FrameEncoder())->encode(array_to_rows([ + ['n' => null], + ['n' => null], + ], schema(null_schema('n'))))), + ); + } + + public function test_a_batch_without_columns_keeps_its_row_count(): void + { + static::assertSame( + '03000000000000000000000000000000', + bin2hex((new FrameEncoder())->encode(array_to_rows([[], [], []], schema()))), + ); + } +} diff --git a/src/core/etl/tests/Flow/Floe/Tests/Unit/FrameReaderTest.php b/src/core/etl/tests/Flow/Floe/Tests/Unit/FrameReaderTest.php index d1ae478866..b6788bd735 100644 --- a/src/core/etl/tests/Flow/Floe/Tests/Unit/FrameReaderTest.php +++ b/src/core/etl/tests/Flow/Floe/Tests/Unit/FrameReaderTest.php @@ -38,13 +38,13 @@ public function test_frames_crossing_compaction_threshold_are_streamed(): void $bytes = Format::header(0x00); for ($i = 0; $i < 200; $i++) { - $bytes .= Format::frame(Format::FRAME_ROW, $body); + $bytes .= Format::frame(Format::FRAME_BATCH, $body); } $read = 0; foreach (FrameReaderMother::overBytes($bytes, chunkSize: 4096)->frames() as [$type, $frameBody]) { - static::assertSame(Format::FRAME_ROW, $type); + static::assertSame(Format::FRAME_BATCH, $type); static::assertSame($body, $frameBody); $read++; } @@ -56,12 +56,12 @@ public function test_frames_are_yielded_in_order_with_bodies(): void { $bytes = Format::header(0x00) - . Format::frame(Format::FRAME_ROW, 'row-1') + . Format::frame(Format::FRAME_BATCH, 'row-1') . Format::frame(Format::FRAME_FOOTER, 'footer'); static::assertSame( [ - [Format::FRAME_ROW, 'row-1'], + [Format::FRAME_BATCH, 'row-1'], [Format::FRAME_FOOTER, 'footer'], ], iterator_to_array(FrameReaderMother::overBytes($bytes)->frames()), @@ -70,14 +70,14 @@ public function test_frames_are_yielded_in_order_with_bodies(): void public function test_stream_ending_at_frame_boundary_ends_cleanly(): void { - $bytes = Format::header(0x00) . Format::frame(Format::FRAME_ROW, 'row'); + $bytes = Format::header(0x00) . Format::frame(Format::FRAME_BATCH, 'row'); static::assertCount(1, iterator_to_array(FrameReaderMother::overBytes($bytes)->frames())); } public function test_truncated_frame_body_throws(): void { - $bytes = Format::header(0x00) . chr(Format::FRAME_ROW) . pack('V', 100) . 'short'; + $bytes = Format::header(0x00) . chr(Format::FRAME_BATCH) . pack('V', 100) . 'short'; $this->expectException(FloeException::class); $this->expectExceptionMessage('frame body is incomplete'); @@ -87,7 +87,7 @@ public function test_truncated_frame_body_throws(): void public function test_truncated_frame_header_throws(): void { - $bytes = Format::header(0x00) . chr(Format::FRAME_ROW) . 'xy'; + $bytes = Format::header(0x00) . chr(Format::FRAME_BATCH) . 'xy'; $this->expectException(FloeException::class); $this->expectExceptionMessage('frame header is incomplete'); diff --git a/src/core/etl/tests/Flow/Floe/Tests/Unit/FrameWriterTest.php b/src/core/etl/tests/Flow/Floe/Tests/Unit/FrameWriterTest.php index 5f11627997..4b90e47e0a 100644 --- a/src/core/etl/tests/Flow/Floe/Tests/Unit/FrameWriterTest.php +++ b/src/core/etl/tests/Flow/Floe/Tests/Unit/FrameWriterTest.php @@ -28,15 +28,15 @@ public function test_header_emits_magic_version_and_codec(): void static::assertSame(Format::HEADER_LENGTH, $writer->position()); } - public function test_row_frame_is_type_length_prefixed_body(): void + public function test_frame_is_type_length_prefixed_body(): void { $sink = new StringDestinationStream(path('memory://frame.floe')); $writer = new FrameWriter($sink, 0x00); - $writer->row('abc'); + $writer->frame(Format::FRAME_BATCH, 'abc'); $writer->flush(); - static::assertSame(chr(Format::FRAME_ROW) . pack('V', 3) . 'abc', $sink->content()); + static::assertSame(chr(Format::FRAME_BATCH) . pack('V', 3) . 'abc', $sink->content()); static::assertSame(Format::FRAME_HEADER_LENGTH + 3, $writer->position()); } @@ -71,7 +71,7 @@ public function test_start_position_seeds_the_logical_position(): void static::assertSame(128, $writer->position()); - $writer->row('x'); + $writer->frame(Format::FRAME_BATCH, 'x'); static::assertSame(128 + Format::FRAME_HEADER_LENGTH + 1, $writer->position()); } @@ -93,9 +93,9 @@ public function test_close_flushes_pending_bytes(): void $sink = new StringDestinationStream(path('memory://frame.floe')); $writer = new FrameWriter($sink, 0x00); - $writer->row('pending'); + $writer->frame(Format::FRAME_BATCH, 'pending'); $writer->close(); - static::assertSame(chr(Format::FRAME_ROW) . pack('V', 7) . 'pending', $sink->content()); + static::assertSame(chr(Format::FRAME_BATCH) . pack('V', 7) . 'pending', $sink->content()); } } diff --git a/src/core/etl/tests/Flow/Floe/Tests/Unit/NativeFloeEncoderTest.php b/src/core/etl/tests/Flow/Floe/Tests/Unit/NativeFloeEncoderTest.php deleted file mode 100644 index 83db5c9b0b..0000000000 --- a/src/core/etl/tests/Flow/Floe/Tests/Unit/NativeFloeEncoderTest.php +++ /dev/null @@ -1,215 +0,0 @@ - 1, 'name' => 'flow'], [ - 'id' => type_integer(), - 'name' => type_string(), - ])]; - - static::assertSame( - (new PhpFloeEncoder($schema))->encode($encoded), - (new NativeFloeEncoder($schema))->encode($encoded), - ); - } - - public function test_native_decode_matches_the_php_engine(): void - { - $data = rows( - schema(int_schema('id'), str_schema('name', nullable: true)), - row(['id' => 1, 'name' => 'flow']), - row(['id' => 2, 'name' => null]), - ); - - $schema = schema_from_json(FloeSchemaContext::schemaBody($data->schema())); - $bodies = (new PhpFloeEncoder($schema))->encode((new PhpRowHydrator())->dehydrate($data)); - - static::assertEquals( - (new PhpFloeEncoder($schema))->decode($bodies), - (new NativeFloeEncoder($schema))->decode($bodies), - ); - } - - public function test_native_metadata_bearing_frames_match_the_php_engine(): void - { - $schema = schema_from_json(FloeSchemaContext::schemaBody(schema( - int_schema('id'), - str_schema('name', nullable: true), - ))); - - $encoded = [new TypedRowValues( - ['id' => 2, 'name' => null], - ['id' => type_integer(), 'name' => type_string()], - ['id' => Metadata::fromArray(['source' => 'trusted', 'weight' => 3])], - )]; - - $bodies = (new PhpFloeEncoder($schema))->encode($encoded); - - static::assertEquals( - (new PhpFloeEncoder($schema))->decode($bodies), - (new NativeFloeEncoder($schema))->decode($bodies), - ); - static::assertSame( - (new PhpFloeEncoder($schema))->encode($encoded), - (new NativeFloeEncoder($schema))->encode($encoded), - ); - } - - /** - * Flags, not instances: PHPUnit builds provider data before setUp() skips, and a native hydrator cannot be - * constructed without the extension. - * - * @return array - */ - public static function native_hydrators(): array - { - return [ - 'native' => [false], - 'adaptive' => [true], - ]; - } - - #[DataProvider('native_hydrators')] - public function test_decode_rows_matches_hydrate_of_decode(bool $adaptive): void - { - $data = rows( - schema(int_schema('id'), str_schema('name', nullable: true), datetime_schema('at')), - row([ - 'id' => 1, - 'name' => 'flow', - 'at' => new DateTimeImmutable('2026-01-01 10:00:00.5', new DateTimeZone('Europe/Warsaw')), - ]), - row(['id' => 2, 'name' => null, 'at' => new DateTimeImmutable('2026-01-02T00:00:00Z')]), - ); - $schema = schema_from_json(FloeSchemaContext::schemaBody($data->schema())); - $encoder = new NativeFloeEncoder($schema); - $bodies = $encoder->encode((new PhpRowHydrator())->dehydrate($data)); - - static::assertEquals( - (new NativeRowHydrator())->hydrate($encoder->decode($bodies), $schema), - $encoder->decodeRows($bodies, $schema, $adaptive ? new AdaptiveRowHydrator() : new NativeRowHydrator()), - ); - } - - public function test_decode_rows_hands_a_non_native_hydrator_the_decoded_values(): void - { - $data = rows(schema(int_schema('id')), row(['id' => 1]), row(['id' => 2])); - $schema = schema_from_json(FloeSchemaContext::schemaBody($data->schema())); - $encoder = new NativeFloeEncoder($schema); - $bodies = $encoder->encode((new PhpRowHydrator())->dehydrate($data)); - $spy = new SpyHydrator(); - - static::assertEquals( - (new PhpRowHydrator())->hydrate($encoder->decode($bodies), $schema), - $encoder->decodeRows($bodies, $schema, $spy), - ); - static::assertSame(1, $spy->hydrateCalls); - } - - public function test_decode_rows_throws_the_schema_mismatch_the_hydrator_throws(): void - { - $data = rows(schema(int_schema('id')), row(['id' => 1])); - $schema = schema_from_json(FloeSchemaContext::schemaBody($data->schema())); - $encoder = new NativeFloeEncoder($schema); - $bodies = $encoder->encode((new PhpRowHydrator())->dehydrate($data)); - - $this->expectException(SchemaMismatchException::class); - $this->expectExceptionMessage('column "name" (row 0) declared by the schema is missing from the row'); - - $encoder->decodeRows($bodies, schema(int_schema('id'), str_schema('name')), new NativeRowHydrator()); - } - - public function test_decode_rows_turns_a_corrupt_frame_into_a_floe_exception(): void - { - $data = rows(schema(int_schema('id')), row(['id' => 1])); - $schema = schema_from_json(FloeSchemaContext::schemaBody($data->schema())); - $encoder = new NativeFloeEncoder($schema); - $bodies = $encoder->encode((new PhpRowHydrator())->dehydrate($data)); - - $this->expectException(FloeException::class); - $this->expectExceptionMessage('flow_php row frame length does not match its content'); - - $encoder->decodeRows([$bodies[0] . "\xEF"], $schema, new NativeRowHydrator()); - } - - #[DataProvider('native_hydrators')] - public function test_encode_frames_matches_framing_encode_of_dehydrate(bool $adaptive): void - { - $data = RowsMother::numbered(4); - $encoder = new NativeFloeEncoder($data->schema()); - - static::assertSame( - Format::rowFrames($encoder->encode((new PhpRowHydrator())->dehydrate($data))), - $encoder->encodeFrames($data, $adaptive ? new AdaptiveRowHydrator() : new NativeRowHydrator()), - ); - } - - public function test_encode_frames_hands_a_non_native_hydrator_the_rows(): void - { - $data = RowsMother::numbered(2); - $encoder = new NativeFloeEncoder($data->schema()); - $spy = new SpyHydrator(); - - static::assertSame( - Format::rowFrames($encoder->encode((new PhpRowHydrator())->dehydrate($data))), - $encoder->encodeFrames($data, $spy), - ); - static::assertSame(1, $spy->dehydrateCalls); - } - - public function test_encode_frames_turns_a_row_without_a_declared_column_into_a_floe_exception(): void - { - $schema = schema(int_schema('id'), str_schema('name')); - - $this->expectException(FloeException::class); - $this->expectExceptionMessage('flow_php found a row that does not carry the declared column "name"'); - - (new NativeFloeEncoder($schema))->encodeFrames(Rows::trusted($schema, [row([ - 'id' => 1, - ])]), new NativeRowHydrator()); - } -} diff --git a/src/core/etl/tests/Flow/Floe/Tests/Unit/PhpFloeEncoderTest.php b/src/core/etl/tests/Flow/Floe/Tests/Unit/PhpFloeEncoderTest.php deleted file mode 100644 index 77d8d97210..0000000000 --- a/src/core/etl/tests/Flow/Floe/Tests/Unit/PhpFloeEncoderTest.php +++ /dev/null @@ -1,209 +0,0 @@ - 1, 'name' => 'flow', 'price' => 1.5]), - row(['id' => 2, 'name' => null, 'price' => 0.5]), - ); - - $encoder = new PhpFloeEncoder(schema_from_json(FloeSchemaContext::schemaBody($original->schema()))); - - $decoded = $encoder->decode($encoder->encode((new PhpRowHydrator())->dehydrate($original))); - - static::assertSame(['id' => 1, 'name' => 'flow', 'price' => 1.5], $decoded[0]->values); - static::assertSame([], $decoded[0]->metadata); - static::assertSame(['id' => 2, 'name' => null, 'price' => 0.5], $decoded[1]->values); - } - - public function test_encode_from_dehydrated_rows_round_trips(): void - { - $original = rows( - schema(int_schema('id'), str_schema('name', nullable: true)), - row(['id' => 1, 'name' => 'flow']), - row(['id' => 2, 'name' => null]), - ); - - $encoder = new PhpFloeEncoder(schema_from_json(FloeSchemaContext::schemaBody($original->schema()))); - - $decoded = $encoder->decode($encoder->encode((new PhpRowHydrator())->dehydrate($original))); - - static::assertEquals( - [ - new RawRowValues(['id' => 1, 'name' => 'flow']), - new RawRowValues(['id' => 2, 'name' => null]), - ], - $decoded, - ); - } - - public function test_encode_refuses_a_row_that_does_not_carry_a_declared_column(): void - { - // a row always carries every column its schema declares, so absence is a broken contract, - // not a value the format can express - VALUE_ABSENT is a structure-element flag only - $encoder = new PhpFloeEncoder(schema_from_json(FloeSchemaContext::schemaBody(schema( - int_schema('id'), - str_schema('name'), - )))); - - $this->expectException(FloeException::class); - $this->expectExceptionMessage('Floe found a row that does not carry the declared column "name"'); - - $encoder->encode([new TypedRowValues(['id' => 4], ['id' => type_integer()])]); - } - - public function test_encode_decode_round_trip_preserves_flags(): void - { - // the column carries nulls, so the declaration admits them - a null on a NOT NULL column is - // refused by the encoder - $schema = schema_from_json(FloeSchemaContext::schemaBody(schema( - int_schema('id'), - str_schema('name', nullable: true), - ))); - - $encoder = new PhpFloeEncoder($schema); - - $types = ['id' => type_integer(), 'name' => type_string()]; - $encoded = [ - new TypedRowValues(['id' => 1, 'name' => 'flow'], $types), - new TypedRowValues(['id' => 2, 'name' => null], $types), - new TypedRowValues(['id' => 3, 'name' => null], $types, ['name' => Metadata::fromArray(['tag' => 'x'])]), - ]; - - $decoded = $encoder->decode($encoder->encode($encoded)); - - static::assertEquals( - [ - new RawRowValues(['id' => 1, 'name' => 'flow']), - new RawRowValues(['id' => 2, 'name' => null]), - new RawRowValues(['id' => 3, 'name' => null], ['name' => Metadata::fromArray(['tag' => 'x'])]), - ], - $decoded, - ); - static::assertSame('x', $decoded[2]->metadata['name']->get('tag')); - } - - public function test_encode_decode_round_trip_preserves_arbitrary_metadata(): void - { - $schema = schema_from_json(FloeSchemaContext::schemaBody(schema(int_schema('id'), str_schema('name')))); - - $encoder = new PhpFloeEncoder($schema); - - $metadata = ['name' => Metadata::fromArray(['source' => 'trusted', 'weight' => 3])]; - $encoded = [ - new TypedRowValues( - ['id' => 1, 'name' => 'flow'], - ['id' => type_integer(), 'name' => type_string()], - $metadata, - ), - ]; - - $decoded = $encoder->decode($encoder->encode($encoded)); - - static::assertEquals([new RawRowValues(['id' => 1, 'name' => 'flow'], $metadata)], $decoded); - static::assertSame('trusted', $decoded[0]->metadata['name']->get('source')); - static::assertSame(3, $decoded[0]->metadata['name']->get('weight')); - } - - public function test_null_entry_in_a_typed_column_encodes_as_a_plain_null(): void - { - $schema = schema_from_json(FloeSchemaContext::schemaBody(schema( - int_schema('id'), - str_schema('name', nullable: true), - ))); - - $encoder = new PhpFloeEncoder($schema); - $body = $encoder->encode((new PhpRowHydrator())->dehydrate(rows( - schema(int_schema('id'), null_schema('name')), - row(['id' => 2, 'name' => null]), - )))[0]; - - $decoded = $encoder->decode([$body]); - - static::assertNull($decoded[0]->values['name']); - static::assertArrayNotHasKey('name', $decoded[0]->metadata); - } - - public function test_row_body_with_trailing_garbage_throws(): void - { - $schema = schema_from_json(FloeSchemaContext::schemaBody(schema(int_schema('id'), str_schema('name')))); - - $encoder = new PhpFloeEncoder($schema); - $body = $encoder->encode((new PhpRowHydrator())->dehydrate(rows( - schema(int_schema('id'), str_schema('name')), - row(['id' => 1, 'name' => 'flow']), - )))[0]; - - $this->expectException(FloeException::class); - $this->expectExceptionMessage('Floe row frame length does not match its content'); - - $encoder->decode([$body . "\x00"]); - } - - public function test_unknown_value_flag_throws(): void - { - $schema = schema_from_json(FloeSchemaContext::schemaBody(schema(int_schema('id'), str_schema('name')))); - - $this->expectException(FloeException::class); - $this->expectExceptionMessage('unknown value flag'); - - (new PhpFloeEncoder($schema))->decode(["\xEF"]); - } - - public function test_decode_rows_hydrates_the_decoded_values(): void - { - $data = rows( - schema(int_schema('id'), str_schema('name', nullable: true)), - row(['id' => 1, 'name' => 'flow']), - row(['id' => 2, 'name' => null]), - ); - $schema = schema_from_json(FloeSchemaContext::schemaBody($data->schema())); - $encoder = new PhpFloeEncoder($schema); - $bodies = $encoder->encode((new PhpRowHydrator())->dehydrate($data)); - - static::assertEquals( - (new PhpRowHydrator())->hydrate($encoder->decode($bodies), $schema), - $encoder->decodeRows($bodies, $schema, new PhpRowHydrator()), - ); - } - - public function test_encode_frames_frames_the_encoded_dehydrated_rows(): void - { - $data = RowsMother::numbered(3); - $encoder = new PhpFloeEncoder($data->schema()); - - static::assertSame( - Format::rowFrames($encoder->encode((new PhpRowHydrator())->dehydrate($data))), - $encoder->encodeFrames($data, new PhpRowHydrator()), - ); - } -} diff --git a/src/core/etl/tests/Flow/Floe/Tests/Unit/SchemaDecoderTest.php b/src/core/etl/tests/Flow/Floe/Tests/Unit/SchemaDecoderTest.php deleted file mode 100644 index 185e5ab4f5..0000000000 --- a/src/core/etl/tests/Flow/Floe/Tests/Unit/SchemaDecoderTest.php +++ /dev/null @@ -1,69 +0,0 @@ -decode($schemaJson); - - static::assertCount(1, $plan); - static::assertSame('name', $plan[0]->name); - static::assertFalse($plan[0]->definition->isNullable()); - } - - public function test_decoded_plan_maps_a_null_column_to_a_null_definition(): void - { - $schemaJson = FloeSchemaContext::schemaBody(schema(null_schema('name'))); - - $plan = (new SchemaDecoder(new ValueDecoder()))->decode($schemaJson); - - static::assertInstanceOf(NullDefinition::class, $plan[0]->definition); - static::assertTrue($plan[0]->definition->isNullable()); - } - - public function test_decoded_plan_preserves_custom_metadata(): void - { - $schemaJson = FloeSchemaContext::schemaBody(schema(int_schema('id', metadata: Metadata::fromArray([ - 'custom' => 'meta', - ])))); - - $plan = (new SchemaDecoder(new ValueDecoder()))->decode($schemaJson); - - static::assertSame('meta', $plan[0]->definition->metadata()->get('custom')); - } - - public function test_decoding_invalid_json_throws(): void - { - $this->expectException(FloeException::class); - $this->expectExceptionMessage('failed to decode schema JSON'); - - (new SchemaDecoder(new ValueDecoder()))->decode('{invalid'); - } - - public function test_decoding_non_list_json_throws(): void - { - $this->expectException(FloeException::class); - $this->expectExceptionMessage('expected schema JSON to be a list'); - - (new SchemaDecoder(new ValueDecoder()))->decode('"scalar"'); - } -} diff --git a/src/core/etl/tests/Flow/Floe/Tests/Unit/ValueDecoderTest.php b/src/core/etl/tests/Flow/Floe/Tests/Unit/ValueDecoderTest.php deleted file mode 100644 index 7156cb49e7..0000000000 --- a/src/core/etl/tests/Flow/Floe/Tests/Unit/ValueDecoderTest.php +++ /dev/null @@ -1,253 +0,0 @@ -decoderFor(enum_schema('c', BackedStringEnum::class)); - $position = 0; - - $this->expectException(FloeException::class); - $this->expectExceptionMessage('cannot restore enum case'); - - $decoder->decode($encoded, $position); - } - - public function test_decoding_enum_of_unknown_class_throws(): void - { - $encoded = pack('V', 10) . 'NoSuchEnum' . pack('V', 3) . 'one'; - $decoder = (new ValueDecoder())->decoderFor(enum_schema('c', BackedStringEnum::class)); - $position = 0; - - $this->expectException(FloeException::class); - $this->expectExceptionMessage('enum not found'); - - $decoder->decode($encoded, $position); - } - - public function test_decoding_html_document_below_php84_throws(): void - { - if (class_exists('\Dom\HTMLDocument')) { - static::markTestSkipped('requires PHP < 8.4'); - } - - $this->expectException(FloeException::class); - $this->expectExceptionMessage('requires PHP 8.4+'); - - ValueDecoder::htmlDocumentFromString('

x

'); - } - - public function test_decoding_html_element_below_php84_throws(): void - { - if (class_exists('\Dom\HTMLDocument')) { - static::markTestSkipped('requires PHP < 8.4'); - } - - $this->expectException(FloeException::class); - $this->expectExceptionMessage('requires PHP 8.4+'); - - ValueDecoder::htmlElementFromString('

x

'); - } - - public function test_decoding_structure_with_unknown_element_flag_throws(): void - { - $decoder = (new ValueDecoder())->decoderFor(structure_schema('c', type_structure(['name' => type_string()]))); - $position = 0; - - $this->expectException(FloeException::class); - $this->expectExceptionMessage('unknown structure element flag'); - - $decoder->decode("\xEE", $position); - } - - public function test_decoding_invalid_xml_throws(): void - { - $encoded = pack('V', 9) . 'not < xml'; - $decoder = (new ValueDecoder())->decoderFor(xml_schema('c')); - $position = 0; - - $this->expectException(FloeException::class); - $this->expectExceptionMessage('failed to restore DOMDocument'); - - $decoder->decode($encoded, $position); - } - - public function test_decoding_value_of_unsupported_type_throws(): void - { - $this->expectException(FloeException::class); - $this->expectExceptionMessage('Floe does not support columns of type "mixed"'); - - (new ValueDecoder())->decoderFor(new ForeignTypeDefinition('c', type_mixed())); - } - - public function test_round_trip_of_datetime_reads_the_column_zone(): void - { - $value = new DateTimeImmutable('2025-06-15 12:30:45.987654', new DateTimeZone('Australia/Eucla')); - $encoded = (new ValueEncoder()) - ->encoderFor(datetime_schema('c')) - ->encode($value); - $position = 0; - - // @mago-ignore analysis:mixed-assignment - $decoded = (new ValueDecoder()) - ->decoderFor(datetime_schema('c')) - ->decode($encoded, $position); - - static::assertInstanceOf(DateTimeImmutable::class, $decoded); - static::assertEquals($value, $decoded); - static::assertSame('UTC', $decoded->getTimezone()->getName()); - static::assertSame('987654', $decoded->format('u')); - static::assertSame(strlen($encoded), $position); - } - - public function test_round_trip_of_date_keeps_the_stored_zone(): void - { - $value = new DateTimeImmutable('2025-06-15 00:00:00', new DateTimeZone('Australia/Eucla')); - $encoded = (new ValueEncoder()) - ->encoderFor(date_schema('c')) - ->encode($value); - $position = 0; - - // @mago-ignore analysis:mixed-assignment - $decoded = (new ValueDecoder()) - ->decoderFor(date_schema('c')) - ->decode($encoded, $position); - - static::assertInstanceOf(DateTimeImmutable::class, $decoded); - static::assertEquals($value, $decoded); - static::assertSame('Australia/Eucla', $decoded->getTimezone()->getName()); - static::assertSame(strlen($encoded), $position); - } - - public function test_round_trip_of_datetime_before_epoch(): void - { - $value = new DateTimeImmutable('1969-07-20 20:17:00.500000 UTC'); - $encoded = (new ValueEncoder()) - ->encoderFor(datetime_schema('c')) - ->encode($value); - $position = 0; - - static::assertEquals($value, (new ValueDecoder()) - ->decoderFor(datetime_schema('c')) - ->decode($encoded, $position)); - } - - public function test_round_trip_of_integer_signed_edges(): void - { - $encoder = (new ValueEncoder())->encoderFor(int_schema('c')); - $decoder = (new ValueDecoder())->decoderFor(int_schema('c')); - - foreach ([PHP_INT_MIN, PHP_INT_MAX, -1, 0, 42] as $value) { - $position = 0; - - static::assertSame($value, $decoder->decode($encoder->encode($value), $position)); - } - } - - public function test_round_trip_of_interval_preserves_all_fields(): void - { - $value = new DateInterval('P1DT2H3M4S'); - // @mago-ignore analysis:invalid-property-write - $value->invert = 1; - // @mago-ignore analysis:invalid-property-write - $value->f = 0.5; - - $encoded = (new ValueEncoder()) - ->encoderFor(time_schema('c')) - ->encode($value); - $position = 0; - - // @mago-ignore analysis:mixed-assignment - $decoded = (new ValueDecoder()) - ->decoderFor(time_schema('c')) - ->decode($encoded, $position); - - static::assertInstanceOf(DateInterval::class, $decoded); - static::assertSame([1, 2, 3, 4, 0.5, 1], [ - $decoded->d, - $decoded->h, - $decoded->i, - $decoded->s, - $decoded->f, - $decoded->invert, - ]); - } - - public function test_restoring_xml_element_from_string(): void - { - static::assertSame('item', ValueDecoder::xmlElementFromString('value')->tagName); - } - - public function test_round_trip_of_uuid_skips_validation(): void - { - $uuid = new Uuid('0196aecb-b568-7e57-a381-8ec8d3e4a531'); - $encoded = (new ValueEncoder()) - ->encoderFor(uuid_schema('c')) - ->encode($uuid); - $position = 0; - - // @mago-ignore analysis:mixed-assignment - $decoded = (new ValueDecoder()) - ->decoderFor(uuid_schema('c')) - ->decode($encoded, $position); - - static::assertInstanceOf(Uuid::class, $decoded); - static::assertSame($uuid->toString(), $decoded->toString()); - } - - public function test_round_trip_of_a_list_with_null_or_t_elements(): void - { - $definition = list_schema('c', type_list(type_union(type_integer(), type_null()))); - $encoded = (new ValueEncoder()) - ->encoderFor($definition) - ->encode([1, null]); - $position = 0; - - static::assertSame( - [1, null], - (new ValueDecoder()) - ->decoderFor($definition) - ->decode($encoded, $position), - ); - } -} diff --git a/src/core/etl/tests/Flow/Floe/Tests/Unit/ValueEncoderTest.php b/src/core/etl/tests/Flow/Floe/Tests/Unit/ValueEncoderTest.php deleted file mode 100644 index 355ba4216e..0000000000 --- a/src/core/etl/tests/Flow/Floe/Tests/Unit/ValueEncoderTest.php +++ /dev/null @@ -1,103 +0,0 @@ -expectException(FloeException::class); - $this->expectExceptionMessage('Floe supports only DateTime and DateTimeImmutable, got ' - . CustomDateTime::class); - - (new DateTimeEncoder())->encode(new CustomDateTime('2025-01-01 00:00:00 UTC')); - } - - public function test_encoding_a_column_of_an_unsupported_type_throws(): void - { - $this->expectException(FloeException::class); - $this->expectExceptionMessage('Floe does not support columns of type "mixed"'); - - (new ValueEncoder())->encoderFor(new ForeignTypeDefinition('c', type_mixed())); - } - - public function test_encoding_list_of_mixed_elements_throws(): void - { - $this->expectException(FloeException::class); - $this->expectExceptionMessage('Floe does not support values of type "mixed"'); - - (new ValueEncoder())->encoderFor(list_schema('c', type_list(type_mixed()))); - } - - public function test_encoding_map_with_unsupported_key_type_throws(): void - { - $this->expectException(FloeException::class); - $this->expectExceptionMessage('Floe does not support map keys of type "non_empty_string"'); - - (new ValueEncoder())->encoderFor(map_schema('c', type_map(type_non_empty_string(), type_string()))); - } - - public function test_encoding_xml_element_without_owner_document_throws(): void - { - $this->expectException(FloeException::class); - $this->expectExceptionMessage('failed to convert DOMElement to XML string'); - - ValueEncoder::xmlElementToString(new DOMElement('detached')); - } - - public function test_integer_encoding_is_little_endian_for_signed_edges(): void - { - $encoder = (new ValueEncoder())->encoderFor(int_schema('c')); - - static::assertSame(pack('P', PHP_INT_MIN), $encoder->encode(PHP_INT_MIN)); - static::assertSame(pack('P', PHP_INT_MAX), $encoder->encode(PHP_INT_MAX)); - static::assertSame("\xFB\xFF\xFF\xFF\xFF\xFF\xFF\xFF", $encoder->encode(-5)); - static::assertSame("\x2A\x00\x00\x00\x00\x00\x00\x00", $encoder->encode(42)); - } - - public function test_integer_list_fast_path_is_bulk_packed(): void - { - $encoder = (new ValueEncoder())->encoderFor(list_schema('c', type_list(type_integer()))); - - static::assertSame(pack('V', 3) . pack('P*', 10, -20, 30), $encoder->encode([10, -20, 30])); - static::assertSame(pack('V', 0), $encoder->encode([])); - } - - public function test_stateful_encoders_are_shared_across_calls(): void - { - $factory = new ValueEncoder(); - - static::assertSame($factory->encoderFor(datetime_schema('c')), $factory->encoderFor(datetime_schema('c'))); - static::assertSame($factory->encoderFor(datetime_schema('c')), $factory->encoderFor(date_schema('c'))); - static::assertSame($factory->encoderFor(uuid_schema('c')), $factory->encoderFor(uuid_schema('c'))); - static::assertSame($factory->encoderFor(json_schema('c')), $factory->encoderFor(json_schema('c'))); - } -} diff --git a/src/core/etl/tests/Flow/Serializer/Tests/Unit/Base64SerializerTest.php b/src/core/etl/tests/Flow/Serializer/Tests/Unit/Base64SerializerTest.php index 4ae3bec8e1..ddbdb8ec6e 100644 --- a/src/core/etl/tests/Flow/Serializer/Tests/Unit/Base64SerializerTest.php +++ b/src/core/etl/tests/Flow/Serializer/Tests/Unit/Base64SerializerTest.php @@ -5,19 +5,17 @@ namespace Flow\Serializer\Tests\Unit; use DateTimeImmutable; -use Flow\ETL\Row; use Flow\Serializer\Base64Serializer; use Flow\Serializer\Exception\SerializationException; use Flow\Serializer\NativePHPSerializer; use PHPUnit\Framework\TestCase; use function array_map; +use function Flow\ETL\DSL\array_to_rows; use function Flow\ETL\DSL\bool_schema; use function Flow\ETL\DSL\datetime_schema; use function Flow\ETL\DSL\float_schema; use function Flow\ETL\DSL\int_schema; -use function Flow\ETL\DSL\row; -use function Flow\ETL\DSL\rows; use function Flow\ETL\DSL\schema; use function Flow\ETL\DSL\str_schema; use function Flow\ETL\DSL\structure_schema; @@ -32,18 +30,9 @@ final class Base64SerializerTest extends TestCase { public function test_serializing_rows(): void { - $rows = rows( - schema( - int_schema('integer'), - str_schema('string'), - bool_schema('boolean'), - datetime_schema('datetime'), - str_schema('null', nullable: true), - float_schema('float'), - structure_schema('struct', type_structure(['integer' => type_integer(), 'string' => type_string()])), - ), - ...array_map( - static fn(): Row => row([ + $rows = array_to_rows( + array_map( + static fn(): array => [ 'integer' => 1, 'string' => 'string', 'boolean' => true, @@ -51,9 +40,18 @@ public function test_serializing_rows(): void 'null' => null, 'float' => 0.12, 'struct' => ['integer' => 1, 'string' => 'string'], - ]), + ], range(0, 100), ), + schema( + int_schema('integer'), + str_schema('string'), + bool_schema('boolean'), + datetime_schema('datetime'), + str_schema('null', nullable: true), + float_schema('float'), + structure_schema('struct', type_structure(['integer' => type_integer(), 'string' => type_string()])), + ), ); $serializer = new Base64Serializer(new NativePHPSerializer()); diff --git a/src/core/etl/tests/Flow/Serializer/Tests/Unit/CompressingSerializerTest.php b/src/core/etl/tests/Flow/Serializer/Tests/Unit/CompressingSerializerTest.php index 66aa108c29..7e7833093b 100644 --- a/src/core/etl/tests/Flow/Serializer/Tests/Unit/CompressingSerializerTest.php +++ b/src/core/etl/tests/Flow/Serializer/Tests/Unit/CompressingSerializerTest.php @@ -5,18 +5,16 @@ namespace Flow\Serializer\Tests\Unit; use DateTimeImmutable; -use Flow\ETL\Row; use Flow\Serializer\CompressingSerializer; use Flow\Serializer\NativePHPSerializer; use PHPUnit\Framework\TestCase; use function array_map; +use function Flow\ETL\DSL\array_to_rows; use function Flow\ETL\DSL\bool_schema; use function Flow\ETL\DSL\datetime_schema; use function Flow\ETL\DSL\float_schema; use function Flow\ETL\DSL\int_schema; -use function Flow\ETL\DSL\row; -use function Flow\ETL\DSL\rows; use function Flow\ETL\DSL\schema; use function Flow\ETL\DSL\str_schema; use function Flow\ETL\DSL\structure_schema; @@ -39,18 +37,9 @@ protected function setUp(): void public function test_serializing_rows(): void { - $rows = rows( - schema( - int_schema('integer'), - str_schema('string'), - bool_schema('boolean'), - datetime_schema('datetime'), - str_schema('null', nullable: true), - float_schema('float'), - structure_schema('struct', type_structure(['integer' => type_integer(), 'string' => type_string()])), - ), - ...array_map( - static fn(): Row => row([ + $rows = array_to_rows( + array_map( + static fn(): array => [ 'integer' => 1, 'string' => 'string', 'boolean' => true, @@ -58,9 +47,18 @@ public function test_serializing_rows(): void 'null' => null, 'float' => 0.12, 'struct' => ['integer' => 1, 'string' => 'string'], - ]), + ], range(0, 100), ), + schema( + int_schema('integer'), + str_schema('string'), + bool_schema('boolean'), + datetime_schema('datetime'), + str_schema('null', nullable: true), + float_schema('float'), + structure_schema('struct', type_structure(['integer' => type_integer(), 'string' => type_string()])), + ), ); $serializer = new CompressingSerializer(new NativePHPSerializer()); diff --git a/src/core/etl/tests/Flow/Serializer/Tests/Unit/NativePHPSerializerTest.php b/src/core/etl/tests/Flow/Serializer/Tests/Unit/NativePHPSerializerTest.php index 67ca850ebd..89179c306a 100644 --- a/src/core/etl/tests/Flow/Serializer/Tests/Unit/NativePHPSerializerTest.php +++ b/src/core/etl/tests/Flow/Serializer/Tests/Unit/NativePHPSerializerTest.php @@ -5,18 +5,16 @@ namespace Flow\Serializer\Tests\Unit; use DateTimeImmutable; -use Flow\ETL\Row; use Flow\Serializer\Exception\SerializationException; use Flow\Serializer\NativePHPSerializer; use PHPUnit\Framework\TestCase; use function array_map; +use function Flow\ETL\DSL\array_to_rows; use function Flow\ETL\DSL\bool_schema; use function Flow\ETL\DSL\datetime_schema; use function Flow\ETL\DSL\float_schema; use function Flow\ETL\DSL\int_schema; -use function Flow\ETL\DSL\row; -use function Flow\ETL\DSL\rows; use function Flow\ETL\DSL\schema; use function Flow\ETL\DSL\str_schema; use function Flow\ETL\DSL\structure_schema; @@ -32,18 +30,9 @@ final class NativePHPSerializerTest extends TestCase { public function test_serializing_rows(): void { - $rows = rows( - schema( - int_schema('integer'), - str_schema('string'), - bool_schema('boolean'), - datetime_schema('datetime'), - str_schema('null', nullable: true), - float_schema('float'), - structure_schema('struct', type_structure(['integer' => type_integer(), 'string' => type_string()])), - ), - ...array_map( - static fn(): Row => row([ + $rows = array_to_rows( + array_map( + static fn(): array => [ 'integer' => 1, 'string' => 'string', 'boolean' => true, @@ -51,9 +40,18 @@ public function test_serializing_rows(): void 'null' => null, 'float' => 0.12, 'struct' => ['integer' => 1, 'string' => 'string'], - ]), + ], range(0, 100), ), + schema( + int_schema('integer'), + str_schema('string'), + bool_schema('boolean'), + datetime_schema('datetime'), + str_schema('null', nullable: true), + float_schema('float'), + structure_schema('struct', type_structure(['integer' => type_integer(), 'string' => type_string()])), + ), ); $serializer = new NativePHPSerializer(); diff --git a/src/extension/arrow-ext/Cargo.lock b/src/extension/arrow-ext/Cargo.lock index af8a2539ff..d69bb02fa8 100644 --- a/src/extension/arrow-ext/Cargo.lock +++ b/src/extension/arrow-ext/Cargo.lock @@ -81,6 +81,7 @@ dependencies = [ "arrow-schema", "bytes", "ext-php-rs", + "ext-php-rs-build", "parquet", ] @@ -97,6 +98,7 @@ dependencies = [ "chrono", "half", "hashbrown 0.17.1", + "libc", "num-complex", "num-integer", "num-traits", @@ -147,6 +149,7 @@ version = "59.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "10fab8d4563491417ba801fab29d205104d20d4bdf37bda6cd1cf425cff598cd" dependencies = [ + "bitflags", "serde_core", "serde_json", ] @@ -189,6 +192,24 @@ version = "1.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2af50177e190e07a26ab74f8b1efbfe2ef87da2116221318cb1c2e82baf7de06" +[[package]] +name = "bindgen" +version = "0.73.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "787ef8ef523575546b106a58213d6e6b06198a05c2f757258c68a74273670cfa" +dependencies = [ + "bitflags", + "cexpr", + "clang-sys", + "log", + "proc-macro2", + "quote", + "regex", + "rustc-hash", + "shlex 2.0.1", + "syn 2.0.117", +] + [[package]] name = "bitflags" version = "2.11.0" @@ -292,7 +313,7 @@ dependencies = [ "find-msvc-tools", "jobserver", "libc", - "shlex", + "shlex 1.3.0", ] [[package]] @@ -370,9 +391,9 @@ checksum = "3d52eff69cd5e647efe296129160853a42795992097e8af39800e1060caeea9b" [[package]] name = "convert_case" -version = "0.11.0" +version = "0.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "affbf0190ed2caf063e3def54ff444b449371d55c58e513a95ab98eca50adb49" +checksum = "1af709f1f33454bf52eadfc8c78b3b9ef9cb26fb54d16dc9cd9a7299f899fd1b" dependencies = [ "unicode-segmentation", ] @@ -429,9 +450,9 @@ dependencies = [ [[package]] name = "darling" -version = "0.23.0" +version = "0.24.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "25ae13da2f202d56bd7f91c25fba009e7717a1e4a1cc98a76d844b65ae912e9d" +checksum = "ed17f5901b6630b993ca003def43f2f8ef4014fc13b047b57aad617ff32bc2ec" dependencies = [ "darling_core", "darling_macro", @@ -439,26 +460,26 @@ dependencies = [ [[package]] name = "darling_core" -version = "0.23.0" +version = "0.24.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9865a50f7c335f53564bb694ef660825eb8610e0a53d3e11bf1b0d3df31e03b0" +checksum = "6837e2cf7485aaae18f86181d2f0e9a7ed297a025e220aeabf63fdebd3a2ddff" dependencies = [ "ident_case", "proc-macro2", "quote", "strsim", - "syn", + "syn 3.0.6", ] [[package]] name = "darling_macro" -version = "0.23.0" +version = "0.24.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ac3984ec7bd6cfa798e62b4a642426a5be0e68f9401cfc2a01e3fa9ea2fcdb8d" +checksum = "2ac7135c3ef02b2f7833bbeb1be5ba7f966dcde8a87c6b87f65a778d71a02785" dependencies = [ "darling_core", "quote", - "syn", + "syn 3.0.6", ] [[package]] @@ -530,46 +551,28 @@ dependencies = [ [[package]] name = "ext-php-rs" -version = "0.15.15" +version = "0.16.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "abe62f25cd053f5f95dc7bbf94e5b41818ea3cf39b36cc25de8ca6b4de68a0eb" +checksum = "530651e90f34e23c956cbd1a33ec4878c05e19ea6b50dd13926b94dd6a2f2134" dependencies = [ "anyhow", + "bindgen", "bitflags", "cc", "cfg-if", - "ext-php-rs-bindgen", "ext-php-rs-build", "ext-php-rs-derive", + "ext-php-rs-introspection", "inventory", "native-tls", "once_cell", "parking_lot", "skeptic", + "thiserror", "ureq", "zip", ] -[[package]] -name = "ext-php-rs-bindgen" -version = "0.72.1-extphprs.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "32ceb73da3b8b18a3424765dd6926dbc29464a0a7bc31ab491600bfa02adc4bc" -dependencies = [ - "bitflags", - "cexpr", - "clang-sys", - "itertools", - "log", - "prettyplease", - "proc-macro2", - "quote", - "regex", - "rustc-hash", - "shlex", - "syn", -] - [[package]] name = "ext-php-rs-build" version = "0.1.1" @@ -581,18 +584,24 @@ dependencies = [ [[package]] name = "ext-php-rs-derive" -version = "0.11.14" +version = "0.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f57a075f878ad51bcc56745fb21efe601429a509204f8a385bbe526566597320" +checksum = "cb3ec2dc5b2228bffa41f5a569b5656e48f964c39cd71633032eebae51e15ef1" dependencies = [ "convert_case", "darling", "itertools", "proc-macro2", "quote", - "syn", + "syn 3.0.6", ] +[[package]] +name = "ext-php-rs-introspection" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3a31bb462ea3b026fc419c2c4a38283b270980ac34fed3888dd5afd635c3ef94" + [[package]] name = "fastrand" version = "2.3.0" @@ -833,9 +842,9 @@ dependencies = [ [[package]] name = "itertools" -version = "0.14.0" +version = "0.15.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2b192c782037fadd9cfa75548310488aabdbf3d2da73885b31bd0abd03351285" +checksum = "8b4baf93f58d4425749ca49a51c50ebab072c5df6994d08fed93541c331481dc" dependencies = [ "either", ] @@ -1060,7 +1069,7 @@ checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.117", ] [[package]] @@ -1185,7 +1194,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "479ca8adacdd7ce8f1fb39ce9ecccbfe93a3f1344b3d0d97f20bc0196208f62b" dependencies = [ "proc-macro2", - "syn", + "syn 2.0.117", ] [[package]] @@ -1400,7 +1409,7 @@ checksum = "d540f220d3187173da220f885ab66608367b6574e925011a9353e4badda91d79" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.117", ] [[package]] @@ -1444,6 +1453,12 @@ version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64" +[[package]] +name = "shlex" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8fadd59c855ef2080decdef8ff161eb6661b86933c9d82e5ba29dc602a55aba" + [[package]] name = "simd-adler32" version = "0.3.8" @@ -1506,6 +1521,17 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "syn" +version = "3.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8593e8e72159ed2257d083c7a454a85cbf854f37a0966d8d483aff8c8a3ebcee" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + [[package]] name = "tempfile" version = "3.27.0" @@ -1519,6 +1545,26 @@ dependencies = [ "windows-sys", ] +[[package]] +name = "thiserror" +version = "2.0.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09e52cb86a36cede5cb101bf8908837b3e4c6e5e59fe7fd85c23fb56200d189e" +dependencies = [ + "thiserror-impl", +] + +[[package]] +name = "thiserror-impl" +version = "2.0.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fe5197923287db20a58125f0bc85c062f7f2c892de97b18c356f9efb14b28524" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.6", +] + [[package]] name = "time" version = "0.3.47" @@ -1704,7 +1750,7 @@ dependencies = [ "bumpalo", "proc-macro2", "quote", - "syn", + "syn 2.0.117", "wasm-bindgen-shared", ] @@ -1790,7 +1836,7 @@ checksum = "053e2e040ab57b9dc951b72c264860db7eb3b0200ba345b4e4c3b14f67855ddf" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.117", ] [[package]] @@ -1801,7 +1847,7 @@ checksum = "3f316c4a2570ba26bbec722032c4099d8c8bc095efccdc15688708623367e358" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.117", ] [[package]] @@ -1867,7 +1913,7 @@ dependencies = [ "heck", "indexmap", "prettyplease", - "syn", + "syn 2.0.117", "wasm-metadata", "wit-bindgen-core", "wit-component", @@ -1883,7 +1929,7 @@ dependencies = [ "prettyplease", "proc-macro2", "quote", - "syn", + "syn 2.0.117", "wit-bindgen-core", "wit-bindgen-rust", ] @@ -1942,7 +1988,7 @@ checksum = "7e6cc098ea4d3bd6246687de65af3f920c430e236bee1e3bf2e441463f08a02f" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.117", ] [[package]] diff --git a/src/extension/arrow-ext/Cargo.toml b/src/extension/arrow-ext/Cargo.toml index 9de8e105d4..e528e0f770 100644 --- a/src/extension/arrow-ext/Cargo.toml +++ b/src/extension/arrow-ext/Cargo.toml @@ -7,13 +7,12 @@ publish = false crate-type = ["cdylib"] [dependencies] -ext-php-rs = "0.15" +ext-php-rs = "0.16" parquet = { version = "59", default-features = false, features = ["arrow", "arrow_canonical_extension_types", "snap", "flate2", "flate2-zlib-rs", "zstd", "lz4", "brotli", "simdutf8"] } -arrow-array = "59" +arrow-array = { version = "59", features = ["ffi"] } arrow-buffer = "59" arrow-schema = { version = "59", features = ["canonical_extension_types"] } bytes = "1" -[[example]] -name = "generate_fixtures" -path = "examples/generate_fixtures.rs" +[build-dependencies] +ext-php-rs-build = "0.1" diff --git a/src/extension/arrow-ext/Makefile b/src/extension/arrow-ext/Makefile index b8bdfd6eac..fae03b517e 100644 --- a/src/extension/arrow-ext/Makefile +++ b/src/extension/arrow-ext/Makefile @@ -31,14 +31,26 @@ build: # Run PHPT tests test: build @echo "Running PHPT tests..." - @failed=0; total=0; passed=0; \ + @failed=0; total=0; passed=0; skipped=0; \ for f in tests/phpt/*.phpt; do \ total=$$((total + 1)); \ test_name=$$(basename "$$f"); \ tmp="$$(dirname "$$f")/_run_$${test_name%.phpt}.php"; \ + skipif="$$(dirname "$$f")/_skipif_$${test_name%.phpt}.php"; \ + sed -n '/^--SKIPIF--$$/,/^--FILE--$$/p' "$$f" | sed '1d;$$d' > "$$skipif"; \ + skip_out=""; \ + if [ -s "$$skipif" ]; then \ + skip_out=$$($(PHP) -d extension=$$(realpath $(EXTENSION_SO)) "$$skipif" 2>&1) || true; \ + fi; \ + rm -f "$$skipif"; \ + case "$$skip_out" in skip*) \ + echo "SKIP: $$test_name ($$skip_out)"; \ + skipped=$$((skipped + 1)); \ + continue;; \ + esac; \ sed -n '/^--FILE--$$/,/^--EXPECT/p' "$$f" | sed '1d;$$d' > "$$tmp"; \ expected=$$(sed -n '/^--EXPECT\(F\)\{0,1\}--$$/,$$p' "$$f" | sed '1d' | tr -d '\r'); \ - actual=$$($(PHP) -n -d extension=$$(realpath $(EXTENSION_SO)) "$$tmp" 2>&1) || true; \ + actual=$$($(PHP) -d extension=$$(realpath $(EXTENSION_SO)) "$$tmp" 2>&1) || true; \ actual=$$(printf '%s' "$$actual" | tr -d '\r'); \ rm -f "$$tmp"; \ if [ "$$actual" = "$$expected" ]; then \ @@ -51,7 +63,7 @@ test: build failed=1; \ fi; \ done; \ - echo "$$passed/$$total tests passed"; \ + echo "$$passed/$$total tests passed ($$skipped skipped)"; \ if [ $$failed -eq 1 ]; then exit 1; fi # Install to current PHP extension directory diff --git a/src/extension/arrow-ext/build.rs b/src/extension/arrow-ext/build.rs index 0f9cd4c10d..fcb72c499f 100644 --- a/src/extension/arrow-ext/build.rs +++ b/src/extension/arrow-ext/build.rs @@ -7,12 +7,34 @@ fn main() { println!("cargo:rustc-env=ARROW_VERSION={version}"); println!("cargo:rerun-if-changed=Cargo.lock"); - let arrow_version = - resolve_dep_version("arrow-schema").unwrap_or_else(|| "unknown".to_string()); + let arrow_version = resolve_dep_version("arrow-schema").unwrap_or_else(|| "unknown".to_string()); let parquet_version = resolve_dep_version("parquet").unwrap_or_else(|| "unknown".to_string()); println!("cargo:rustc-env=ARROW_LIB_VERSION={arrow_version}"); println!("cargo:rustc-env=PARQUET_LIB_VERSION={parquet_version}"); + + emit_php_version_cfg(); +} + +/// The `php84`/`php85` cfgs ext-php-rs sets for itself, for the same PHP (`PHP` first, then `PATH`): the extension +/// binary is bound to the PHP minor it is built against, so version gates are compile-time. +fn emit_php_version_cfg() { + use ext_php_rs_build::{ + emit_check_cfg, emit_php_cfg_flags, emit_rerun_if_env_changed, find_php, ApiVersion, PHPInfo, + }; + + emit_rerun_if_env_changed(); + emit_check_cfg(); + + let php = find_php().expect("cannot find the php executable"); + let info = PHPInfo::get(&php).expect("cannot read php -i"); + let version: ApiVersion = info + .zend_version() + .expect("cannot read the Zend API version") + .try_into() + .expect("unsupported Zend API version"); + + emit_php_cfg_flags(version); } fn extension_version(env_name: &str) -> String { @@ -67,7 +89,8 @@ fn git(args: &[&str]) -> Option { .filter(|stdout| !stdout.is_empty()) } -/// `0.43.0-107-gf998a45d0` (git describe) becomes `0.43.0+107.gf998a45d0`, semver build metadata Composer reads as 0.43.0. +/// `0.44.1-65-g4525c4adc` (git describe) becomes `0.45.0-dev+65.g4525c4adc`: commits past a tag are a pre-release of the +/// next minor, outside Composer's `<0.45` conflict of the libraries. A tag, or any other string, passes through. fn as_semver(describe: &str) -> String { let mut parts = describe.rsplitn(3, '-'); @@ -75,9 +98,17 @@ fn as_semver(describe: &str) -> String { (Some(hash), Some(distance), Some(tag)) if hash.starts_with('g') && !distance.is_empty() - && distance.bytes().all(|b| b.is_ascii_digit()) => + && distance.bytes().all(|b| b.is_ascii_digit()) + && distance != "0" => { - format!("{tag}+{distance}.{hash}") + let mut numbers = tag.split('.').map(|number| number.parse::().ok()); + + match (numbers.next().flatten(), numbers.next().flatten()) { + (Some(major), Some(minor)) => { + format!("{major}.{}.0-dev+{distance}.{hash}", minor + 1) + } + _ => describe.to_string(), + } } _ => describe.to_string(), } @@ -92,11 +123,7 @@ fn resolve_dep_version(crate_name: &str) -> Option { for line in chunk.lines() { let line = line.trim(); if line.starts_with("version = ") { - return Some( - line.trim_start_matches("version = ") - .trim_matches('"') - .to_string(), - ); + return Some(line.trim_start_matches("version = ").trim_matches('"').to_string()); } } } diff --git a/src/extension/arrow-ext/composer.json b/src/extension/arrow-ext/composer.json index 324b979249..d99b1fbed7 100644 --- a/src/extension/arrow-ext/composer.json +++ b/src/extension/arrow-ext/composer.json @@ -1,7 +1,7 @@ { "name": "flow-php/arrow-ext", "type": "php-ext", - "description": "Apache Arrow PHP extension powered by Rust", + "description": "Apache Arrow and Parquet bindings for flow-php/parquet (Rust)", "keywords": ["arrow", "parquet", "extension", "rust", "native"], "license": "MIT", "require": { @@ -9,7 +9,8 @@ }, "autoload": { "psr-4": { - "Flow\\Arrow\\": "php/Flow/Arrow/" + "Flow\\Arrow\\": "php/Flow/Arrow/", + "Flow\\Parquet\\Engine\\": "php/Flow/Parquet/Engine/" } }, "scripts": { diff --git a/src/extension/arrow-ext/examples/generate_fixtures.rs b/src/extension/arrow-ext/examples/generate_fixtures.rs deleted file mode 100644 index b4492eab15..0000000000 --- a/src/extension/arrow-ext/examples/generate_fixtures.rs +++ /dev/null @@ -1,388 +0,0 @@ -//! Standalone binary to generate test fixtures. -//! Run with: cargo run --example generate_fixtures - -use std::fs::File; -use std::sync::Arc; - -fn generate_simple() { - use parquet::basic::Type as PhysicalType; - use parquet::basic::{ConvertedType, Repetition}; - use parquet::column::writer::ColumnWriter; - use parquet::file::properties::WriterProperties; - use parquet::file::writer::SerializedFileWriter; - use parquet::schema::types::Type; - - let id_field = Arc::new( - Type::primitive_type_builder("id", PhysicalType::INT64) - .with_repetition(Repetition::REQUIRED) - .build() - .unwrap(), - ); - - let name_field = Arc::new( - Type::primitive_type_builder("name", PhysicalType::BYTE_ARRAY) - .with_repetition(Repetition::REQUIRED) - .with_converted_type(ConvertedType::UTF8) - .build() - .unwrap(), - ); - - let schema = Arc::new( - Type::group_type_builder("schema") - .with_fields(vec![id_field, name_field]) - .build() - .unwrap(), - ); - - let props = Arc::new( - WriterProperties::builder() - .set_compression(parquet::basic::Compression::UNCOMPRESSED) - .build(), - ); - - let file = File::create("tests/fixtures/simple.parquet").unwrap(); - let mut writer = SerializedFileWriter::new(file, schema, props).unwrap(); - - let mut row_group_writer = writer.next_row_group().unwrap(); - - // Write id column - if let Some(mut col_writer) = row_group_writer.next_column().unwrap() { - if let ColumnWriter::Int64ColumnWriter(ref mut w) = col_writer.untyped() { - w.write_batch(&[1, 2, 3], None, None).unwrap(); - } - col_writer.close().unwrap(); - } - - // Write name column - if let Some(mut col_writer) = row_group_writer.next_column().unwrap() { - if let ColumnWriter::ByteArrayColumnWriter(ref mut w) = col_writer.untyped() { - let values: Vec = - vec!["Alice".into(), "Bob".into(), "Charlie".into()]; - w.write_batch(&values, None, None).unwrap(); - } - col_writer.close().unwrap(); - } - - row_group_writer.close().unwrap(); - writer.close().unwrap(); - - println!("Generated tests/fixtures/simple.parquet"); -} - -fn generate_all_flat_types() { - use arrow_array::*; - use arrow_schema::{DataType, Field, Schema}; - use parquet::arrow::ArrowWriter; - - let schema = Arc::new(Schema::new(vec![ - Field::new("col_bool", DataType::Boolean, true), - Field::new("col_int8", DataType::Int8, true), - Field::new("col_int16", DataType::Int16, true), - Field::new("col_int32", DataType::Int32, true), - Field::new("col_int64", DataType::Int64, true), - Field::new("col_uint8", DataType::UInt8, true), - Field::new("col_uint16", DataType::UInt16, true), - Field::new("col_uint32", DataType::UInt32, true), - Field::new("col_uint64", DataType::UInt64, true), - Field::new("col_float", DataType::Float32, true), - Field::new("col_double", DataType::Float64, true), - Field::new("col_string", DataType::Utf8, true), - Field::new("col_binary", DataType::Binary, true), - Field::new("col_date", DataType::Date32, true), - Field::new( - "col_timestamp", - DataType::Timestamp(arrow_schema::TimeUnit::Microsecond, None), - true, - ), - Field::new("col_decimal", DataType::Decimal128(10, 2), true), - ])); - - let col_bool = BooleanArray::from(vec![Some(true), Some(false), None]); - let col_int8 = Int8Array::from(vec![Some(1i8), Some(-128i8), None]); - let col_int16 = Int16Array::from(vec![Some(256i16), Some(-32768i16), None]); - let col_int32 = Int32Array::from(vec![Some(100000i32), Some(-100000i32), None]); - let col_int64 = Int64Array::from(vec![Some(1000000000i64), Some(-1000000000i64), None]); - let col_uint8 = UInt8Array::from(vec![Some(0u8), Some(255u8), None]); - let col_uint16 = UInt16Array::from(vec![Some(0u16), Some(65535u16), None]); - let col_uint32 = UInt32Array::from(vec![Some(0u32), Some(4294967295u32), None]); - let col_uint64 = UInt64Array::from(vec![Some(0u64), Some(u64::MAX), None]); - let col_float = Float32Array::from(vec![Some(1.5f32), Some(-2.5f32), None]); - let col_double = Float64Array::from(vec![Some(1.23456f64), Some(-7.89012f64), None]); - let col_string = StringArray::from(vec![Some("hello"), Some("world"), None]); - - let col_binary: BinaryArray = vec![ - Some(b"\x00\x01\x02" as &[u8]), - Some(b"\xff\xfe" as &[u8]), - None, - ] - .into_iter() - .collect(); - - // 19000 days since epoch - let col_date = Date32Array::from(vec![Some(19000i32), Some(0i32), None]); - let col_timestamp = - TimestampMicrosecondArray::from(vec![Some(1700000000000000i64), Some(0i64), None]); - - // Decimal128: value is unscaled integer. 123.45 with scale=2 → 12345, -999.99 → -99999 - let col_decimal = Decimal128Array::from(vec![Some(12345i128), Some(-99999i128), None]) - .with_precision_and_scale(10, 2) - .unwrap(); - - let batch = RecordBatch::try_new( - schema.clone(), - vec![ - Arc::new(col_bool), - Arc::new(col_int8), - Arc::new(col_int16), - Arc::new(col_int32), - Arc::new(col_int64), - Arc::new(col_uint8), - Arc::new(col_uint16), - Arc::new(col_uint32), - Arc::new(col_uint64), - Arc::new(col_float), - Arc::new(col_double), - Arc::new(col_string), - Arc::new(col_binary), - Arc::new(col_date), - Arc::new(col_timestamp), - Arc::new(col_decimal), - ], - ) - .unwrap(); - - let file = File::create("tests/fixtures/all_flat_types.parquet").unwrap(); - let mut writer = ArrowWriter::try_new(file, schema, None).unwrap(); - writer.write(&batch).unwrap(); - writer.close().unwrap(); - - println!("Generated tests/fixtures/all_flat_types.parquet"); -} - -fn generate_nested() { - use arrow_array::builder::*; - use arrow_array::*; - use arrow_schema::{DataType, Field, Fields, Schema}; - use parquet::arrow::ArrowWriter; - - let schema = Arc::new(Schema::new(vec![ - Field::new("id", DataType::Int64, false), - Field::new( - "tags", - DataType::List(Arc::new(Field::new("item", DataType::Utf8, true))), - true, - ), - Field::new( - "metadata", - DataType::Struct(Fields::from(vec![ - Field::new("key", DataType::Utf8, false), - Field::new("value", DataType::Int64, false), - ])), - true, - ), - Field::new( - "props", - DataType::Map( - Arc::new(Field::new( - "entries", - DataType::Struct(Fields::from(vec![ - Field::new("keys", DataType::Utf8, false), - Field::new("values", DataType::Int64, true), - ])), - false, - )), - false, - ), - true, - ), - ])); - - // id column - let id_array = Int64Array::from(vec![1, 2, 3]); - - // tags column: Row 0: ["a","b","c"], Row 1: [] (empty), Row 2: null - let mut tags_builder = ListBuilder::new(StringBuilder::new()); - tags_builder.values().append_value("a"); - tags_builder.values().append_value("b"); - tags_builder.values().append_value("c"); - tags_builder.append(true); - tags_builder.append(true); // empty list - tags_builder.append(false); // null - let tags_array = tags_builder.finish(); - - // metadata column: Row 0: {key:"x", value:10}, Row 1: null, Row 2: {key:"y", value:20} - let meta_fields = Fields::from(vec![ - Field::new("key", DataType::Utf8, false), - Field::new("value", DataType::Int64, false), - ]); - let mut meta_builder = StructBuilder::new( - meta_fields, - vec![ - Box::new(StringBuilder::new()), - Box::new(Int64Builder::new()), - ], - ); - // Row 0: {key:"x", value:10} - meta_builder - .field_builder::(0) - .unwrap() - .append_value("x"); - meta_builder - .field_builder::(1) - .unwrap() - .append_value(10); - meta_builder.append(true); - // Row 1: null - meta_builder - .field_builder::(0) - .unwrap() - .append_null(); - meta_builder - .field_builder::(1) - .unwrap() - .append_null(); - meta_builder.append(false); - // Row 2: {key:"y", value:20} - meta_builder - .field_builder::(0) - .unwrap() - .append_value("y"); - meta_builder - .field_builder::(1) - .unwrap() - .append_value(20); - meta_builder.append(true); - let metadata_array = meta_builder.finish(); - - // props column: Row 0: {"p1":100, "p2":200}, Row 1: {"p3":300}, Row 2: null - let mut props_builder = MapBuilder::new(None, StringBuilder::new(), Int64Builder::new()); - props_builder.keys().append_value("p1"); - props_builder.values().append_value(100); - props_builder.keys().append_value("p2"); - props_builder.values().append_value(200); - props_builder.append(true).unwrap(); - props_builder.keys().append_value("p3"); - props_builder.values().append_value(300); - props_builder.append(true).unwrap(); - props_builder.append(false).unwrap(); // null - let props_array = props_builder.finish(); - - let batch = RecordBatch::try_new( - schema.clone(), - vec![ - Arc::new(id_array), - Arc::new(tags_array), - Arc::new(metadata_array), - Arc::new(props_array), - ], - ) - .unwrap(); - - let file = File::create("tests/fixtures/nested.parquet").unwrap(); - let mut writer = ArrowWriter::try_new(file, schema, None).unwrap(); - writer.write(&batch).unwrap(); - writer.close().unwrap(); - - println!("Generated tests/fixtures/nested.parquet"); -} - -fn generate_deeply_nested() { - use arrow_array::builder::*; - use arrow_array::*; - use arrow_schema::{DataType, Field, Fields, Schema}; - use parquet::arrow::ArrowWriter; - - let scores_field = Field::new( - "scores", - DataType::List(Arc::new(Field::new("item", DataType::Int64, true))), - true, - ); - let struct_fields = Fields::from(vec![ - Field::new("name", DataType::Utf8, false), - scores_field, - ]); - let groups_field = Field::new( - "groups", - DataType::List(Arc::new(Field::new( - "item", - DataType::Struct(struct_fields.clone()), - true, - ))), - true, - ); - - let schema = Arc::new(Schema::new(vec![groups_field])); - - // Build using nested builders - let scores_builder = ListBuilder::new(Int64Builder::new()); - let struct_builder = StructBuilder::new( - struct_fields, - vec![Box::new(StringBuilder::new()), Box::new(scores_builder)], - ); - let mut groups_builder = ListBuilder::new(struct_builder); - - // Row 0: [{name:"alpha", scores:[10,20]}, {name:"beta", scores:[30]}] - groups_builder - .values() - .field_builder::(0) - .unwrap() - .append_value("alpha"); - let scores_b = groups_builder - .values() - .field_builder::>(1) - .unwrap(); - scores_b.values().append_value(10); - scores_b.values().append_value(20); - scores_b.append(true); - groups_builder.values().append(true); - - groups_builder - .values() - .field_builder::(0) - .unwrap() - .append_value("beta"); - let scores_b = groups_builder - .values() - .field_builder::>(1) - .unwrap(); - scores_b.values().append_value(30); - scores_b.append(true); - groups_builder.values().append(true); - groups_builder.append(true); - - // Row 1: [{name:"gamma", scores:[]}] - groups_builder - .values() - .field_builder::(0) - .unwrap() - .append_value("gamma"); - let scores_b = groups_builder - .values() - .field_builder::>(1) - .unwrap(); - scores_b.append(true); // empty list - groups_builder.values().append(true); - groups_builder.append(true); - - // Row 2: null - groups_builder.append(false); - - let groups_array = groups_builder.finish(); - - let batch = RecordBatch::try_new(schema.clone(), vec![Arc::new(groups_array)]).unwrap(); - - let file = File::create("tests/fixtures/deeply_nested.parquet").unwrap(); - let mut writer = ArrowWriter::try_new(file, schema, None).unwrap(); - writer.write(&batch).unwrap(); - writer.close().unwrap(); - - println!("Generated tests/fixtures/deeply_nested.parquet"); -} - -fn main() { - std::fs::create_dir_all("tests/fixtures").unwrap(); - generate_simple(); - generate_all_flat_types(); - generate_nested(); - generate_deeply_nested(); -} diff --git a/src/extension/arrow-ext/php/Flow/Arrow/OutputStream.php b/src/extension/arrow-ext/php/Flow/Arrow/OutputStream.php deleted file mode 100644 index 781267cee9..0000000000 --- a/src/extension/arrow-ext/php/Flow/Arrow/OutputStream.php +++ /dev/null @@ -1,16 +0,0 @@ - $options - */ - public function __construct(RandomAccessFile $source, array $options = []) - { - throw new RuntimeException( - 'The arrow PHP extension is not loaded. Install ext-arrow to use Flow\Arrow\Parquet\Reader.', - ); - } - - public function close(): void {} - - /** - * @return array - */ - public function metadata(): array - { - return []; - } - - /** - * @param null|array $columns - * - * @return null|array> - */ - public function readRowGroup(?array $columns = null): ?array - { - return null; - } - - /** - * @return array> - */ - public function schema(): array - { - return []; - } -} diff --git a/src/extension/arrow-ext/php/Flow/Arrow/Parquet/RustBatchReader.php b/src/extension/arrow-ext/php/Flow/Arrow/Parquet/RustBatchReader.php new file mode 100644 index 0000000000..6ea532f5c8 --- /dev/null +++ b/src/extension/arrow-ext/php/Flow/Arrow/Parquet/RustBatchReader.php @@ -0,0 +1,49 @@ + $columns root column names + * @param int<1, max> $batchSize + */ + public function __construct(RustParquetFileReader $file, array $columns, int $batchSize, ?int $offset, ?int $limit) + { + throw new RuntimeException('arrow extension is not loaded'); + } + + public function close(): void + { + throw new RuntimeException('arrow extension is not loaded'); + } + + /** + * At most $batchSize rows as one Arrow C Data struct array, null after the last. + */ + public function next(): ?RustParquetBatch + { + throw new RuntimeException('arrow extension is not loaded'); + } + + /** + * The struct schema every batch has, before any batch is read. + */ + public function schema(): RustArrowSchema + { + throw new RuntimeException('arrow extension is not loaded'); + } +} diff --git a/src/extension/arrow-ext/php/Flow/Arrow/Parquet/RustColumnsReader.php b/src/extension/arrow-ext/php/Flow/Arrow/Parquet/RustColumnsReader.php new file mode 100644 index 0000000000..bf7ee0a131 --- /dev/null +++ b/src/extension/arrow-ext/php/Flow/Arrow/Parquet/RustColumnsReader.php @@ -0,0 +1,62 @@ +>> + */ +final class RustColumnsReader implements Iterator +{ + /** + * @param list $columns root names or struct paths + * @param int<1, max> $batchSize + */ + public function __construct(RustParquetFileReader $file, array $columns, int $batchSize, ?int $offset, ?int $limit) + { + throw new RuntimeException('arrow extension is not loaded'); + } + + /** + * At most $batchSize rows, keyed by $columns. + * + * @return null|array> + */ + public function current(): ?array + { + throw new RuntimeException('arrow extension is not loaded'); + } + + public function key(): ?int + { + throw new RuntimeException('arrow extension is not loaded'); + } + + public function next(): void + { + throw new RuntimeException('arrow extension is not loaded'); + } + + public function rewind(): void + { + throw new RuntimeException('arrow extension is not loaded'); + } + + public function valid(): bool + { + throw new RuntimeException('arrow extension is not loaded'); + } +} diff --git a/src/extension/arrow-ext/php/Flow/Arrow/Parquet/Writer.php b/src/extension/arrow-ext/php/Flow/Arrow/Parquet/Writer.php deleted file mode 100644 index 701f675123..0000000000 --- a/src/extension/arrow-ext/php/Flow/Arrow/Parquet/Writer.php +++ /dev/null @@ -1,45 +0,0 @@ -> $schema - * @param array $options - */ - public function __construct( - OutputStream $stream, - array $schema, - string $compression = 'SNAPPY', - array $options = [], - ) { - throw new RuntimeException( - 'The arrow PHP extension is not loaded. Install ext-arrow to use Flow\Arrow\Parquet\Writer.', - ); - } - - public function close(): void {} - - /** - * @param array> $batch - */ - public function writeBatch(array $batch): void {} -} diff --git a/src/extension/arrow-ext/php/Flow/Arrow/RandomAccessFile.php b/src/extension/arrow-ext/php/Flow/Arrow/RandomAccessFile.php deleted file mode 100644 index 3729d8ff55..0000000000 --- a/src/extension/arrow-ext/php/Flow/Arrow/RandomAccessFile.php +++ /dev/null @@ -1,18 +0,0 @@ -> $rows + */ + public function writeRows( + DestinationStream $stream, + Schema $schema, + Compressions $compression, + Options $options, + iterable $rows, + ): void { + throw new RuntimeException('arrow extension is not loaded'); + } +} diff --git a/src/extension/arrow-ext/php/Flow/Parquet/Engine/RustParquetFileReader.php b/src/extension/arrow-ext/php/Flow/Parquet/Engine/RustParquetFileReader.php new file mode 100644 index 0000000000..7e041d8abc --- /dev/null +++ b/src/extension/arrow-ext/php/Flow/Parquet/Engine/RustParquetFileReader.php @@ -0,0 +1,79 @@ + $columns resolved names, never empty + * @param int<1, max> $batchSize upper bound of rows per chunk + */ + public function readColumns(array $columns, int $batchSize, ?int $limit, ?int $offset): RustColumnsReader + { + throw new RuntimeException('arrow extension is not loaded'); + } + + public function rowsNumber(): int + { + throw new RuntimeException('arrow extension is not loaded'); + } + + public function schema(): Schema + { + throw new RuntimeException('arrow extension is not loaded'); + } + + /** + * The whole footer, decoded on the first call. + */ + public function thrift(): FileMetaData + { + throw new RuntimeException('arrow extension is not loaded'); + } + + /** + * Σ row group total_byte_size (uncompressed). + */ + public function totalByteSize(): int + { + throw new RuntimeException('arrow extension is not loaded'); + } +} diff --git a/src/extension/arrow-ext/php/Flow/Parquet/Engine/RustParquetFileWriter.php b/src/extension/arrow-ext/php/Flow/Parquet/Engine/RustParquetFileWriter.php new file mode 100644 index 0000000000..6c81f53bc3 --- /dev/null +++ b/src/extension/arrow-ext/php/Flow/Parquet/Engine/RustParquetFileWriter.php @@ -0,0 +1,88 @@ + $schema Flow\Parquet\Engine\Arrow\SchemaConverter::toExtension() + * @param array $options Flow\Parquet\Engine\Arrow\OptionsConverter::toExtension() + * @param int $batchSize rows per written batch; below 1 is refused + * + * @throws \Flow\Parquet\Exception\RuntimeException for Compressions::LZO + */ + public function __construct( + DestinationStream $stream, + array $schema, + Compressions $compression, + array $options, + int $batchSize, + ) { + throw new RuntimeException('arrow extension is not loaded'); + } + + /** + * Buffered rows, the footer, then the stream closed. + */ + public function close(): void + { + throw new RuntimeException('arrow extension is not loaded'); + } + + /** + * An Arrow C Data struct batch, exported by an extension: its children by writer column name, a writer column + * the batch lacks is nulls, a child the writer lacks is ignored. + */ + public function writeArrowBatch(object $batch): void + { + throw new RuntimeException('arrow extension is not loaded'); + } + + /** + * An array as writeRows() (keys ignored), a Traversable row by row. + * + * @param iterable> $rows + */ + public function writeBatch(iterable $rows): void + { + throw new RuntimeException('arrow extension is not loaded'); + } + + /** + * @param array> $columns by top-level column name, every list of one length; a writer column + * the array lacks is nulls, a key the writer lacks is ignored + */ + public function writeColumns(array $columns): void + { + throw new RuntimeException('arrow extension is not loaded'); + } + + /** + * @param array $row a writer column the row lacks is null + */ + public function writeRow(array $row): void + { + throw new RuntimeException('arrow extension is not loaded'); + } + + /** + * @param list> $rows + */ + public function writeRows(array $rows): void + { + throw new RuntimeException('arrow extension is not loaded'); + } +} diff --git a/src/extension/arrow-ext/rustfmt.toml b/src/extension/arrow-ext/rustfmt.toml new file mode 100644 index 0000000000..7530651796 --- /dev/null +++ b/src/extension/arrow-ext/rustfmt.toml @@ -0,0 +1 @@ +max_width = 120 diff --git a/src/extension/arrow-ext/src/alloc.rs b/src/extension/arrow-ext/src/alloc.rs new file mode 100644 index 0000000000..11e181d473 --- /dev/null +++ b/src/extension/arrow-ext/src/alloc.rs @@ -0,0 +1,61 @@ +//! The test global allocator: `System`, counting the bytes each thread holds, so a cargo test pins what arrow's +//! native heap keeps while the other tests allocate on their own threads. + +use std::alloc::{GlobalAlloc, Layout, System}; +use std::cell::Cell; + +thread_local! { + static ALLOCATED: Cell = const { Cell::new(0) }; +} + +fn counted(delta: i64) { + // a thread being torn down has no counter left: nothing reads it again + let _ = ALLOCATED.try_with(|allocated| allocated.set(allocated.get() + delta)); +} + +struct Counting; + +unsafe impl GlobalAlloc for Counting { + unsafe fn alloc(&self, layout: Layout) -> *mut u8 { + let ptr = System.alloc(layout); + + if !ptr.is_null() { + counted(layout.size() as i64); + } + + ptr + } + + unsafe fn alloc_zeroed(&self, layout: Layout) -> *mut u8 { + let ptr = System.alloc_zeroed(layout); + + if !ptr.is_null() { + counted(layout.size() as i64); + } + + ptr + } + + unsafe fn dealloc(&self, ptr: *mut u8, layout: Layout) { + System.dealloc(ptr, layout); + counted(-(layout.size() as i64)); + } + + unsafe fn realloc(&self, ptr: *mut u8, layout: Layout, new_size: usize) -> *mut u8 { + let new = System.realloc(ptr, layout, new_size); + + if !new.is_null() { + counted(new_size as i64 - layout.size() as i64); + } + + new + } +} + +#[global_allocator] +static GLOBAL: Counting = Counting; + +/// What the current thread holds. +pub fn allocated_bytes() -> i64 { + ALLOCATED.with(Cell::get) +} diff --git a/src/extension/arrow-ext/src/exception.rs b/src/extension/arrow-ext/src/exception.rs new file mode 100644 index 0000000000..6479c2bda6 --- /dev/null +++ b/src/extension/arrow-ext/src/exception.rs @@ -0,0 +1,14 @@ +use ext_php_rs::exception::PhpException; +use ext_php_rs::zend::{ce, ClassEntry}; + +/// What arrow throws for its own failures: the lib's runtime exception, plain `Exception` when the lib is not +/// autoloadable. +const EXCEPTION_CLASS: &str = "Flow\\Parquet\\Exception\\RuntimeException"; + +fn exception_ce() -> &'static ClassEntry { + ClassEntry::try_find(EXCEPTION_CLASS).unwrap_or_else(|| ce::exception()) +} + +pub fn ext_exception(message: impl Into) -> PhpException { + PhpException::new(message.into(), 0, exception_ce()) +} diff --git a/src/extension/arrow-ext/src/interfaces.rs b/src/extension/arrow-ext/src/interfaces.rs new file mode 100644 index 0000000000..cc8589a753 --- /dev/null +++ b/src/extension/arrow-ext/src/interfaces.rs @@ -0,0 +1,152 @@ +//! `Flow\Parquet\{ParquetEngine, ParquetFileReader, ParquetFileWriter}`: registered at MINIT with the package files' +//! exact signatures, so the extension's classes implement them and the package files (autoloaded only without arrow) +//! stay unused. + +use std::sync::atomic::{AtomicPtr, Ordering}; + +use ext_php_rs::args::{Arg, ArgInfoTables}; +use ext_php_rs::builders::{ClassBuilder, FunctionBuilder}; +use ext_php_rs::error::Result; +use ext_php_rs::flags::{ClassFlags, DataType, MethodFlags}; +use ext_php_rs::zend::ClassEntry; + +const SOURCE_STREAM: DataType = DataType::object("Flow\\Filesystem\\SourceStream"); +const DESTINATION_STREAM: DataType = DataType::object("Flow\\Filesystem\\DestinationStream"); +const SCHEMA: DataType = DataType::object("Flow\\Parquet\\ParquetFile\\Schema"); +const METADATA: DataType = DataType::object("Flow\\Parquet\\ParquetFile\\Metadata"); +const COMPRESSIONS: DataType = DataType::object("Flow\\Parquet\\ParquetFile\\Compressions"); +const OPTIONS: DataType = DataType::object("Flow\\Parquet\\Options"); +const PARQUET_FILE_READER: DataType = DataType::object("Flow\\Parquet\\ParquetFileReader"); +const PARQUET_FILE_WRITER: DataType = DataType::object("Flow\\Parquet\\ParquetFileWriter"); +const ITERATOR: DataType = DataType::object("Iterator"); + +// ClassEntry::try_find reads EG(class_table), which MINIT has not set, so each CE is kept from its registration +static PARQUET_ENGINE_CE: AtomicPtr = AtomicPtr::new(std::ptr::null_mut()); +static PARQUET_FILE_READER_CE: AtomicPtr = AtomicPtr::new(std::ptr::null_mut()); +static PARQUET_FILE_WRITER_CE: AtomicPtr = AtomicPtr::new(std::ptr::null_mut()); + +fn store_parquet_engine(ce: &'static mut ClassEntry, arg_info: ArgInfoTables) { + keep(arg_info); + PARQUET_ENGINE_CE.store(ce, Ordering::Release); +} + +fn store_parquet_file_reader(ce: &'static mut ClassEntry, arg_info: ArgInfoTables) { + keep(arg_info); + PARQUET_FILE_READER_CE.store(ce, Ordering::Release); +} + +fn store_parquet_file_writer(ce: &'static mut ClassEntry, arg_info: ArgInfoTables) { + keep(arg_info); + PARQUET_FILE_WRITER_CE.store(ce, Ordering::Release); +} + +/// The engine borrows an interface's argument info tables for the life of the process. +pub fn keep(arg_info: ArgInfoTables) { + std::mem::forget(arg_info); +} + +fn load(ce: &AtomicPtr, name: &str) -> &'static ClassEntry { + unsafe { ce.load(Ordering::Acquire).as_ref() }.unwrap_or_else(|| panic!("{name} not registered")) +} + +pub fn parquet_engine_ce() -> &'static ClassEntry { + load(&PARQUET_ENGINE_CE, "Flow\\Parquet\\ParquetEngine") +} + +pub fn parquet_file_reader_ce() -> &'static ClassEntry { + load(&PARQUET_FILE_READER_CE, "Flow\\Parquet\\ParquetFileReader") +} + +pub fn parquet_file_writer_ce() -> &'static ClassEntry { + load(&PARQUET_FILE_WRITER_CE, "Flow\\Parquet\\ParquetFileWriter") +} + +fn method(name: &str, args: Vec>, returns: DataType) -> FunctionBuilder<'static> { + args.into_iter() + .fold(FunctionBuilder::new_abstract(name), FunctionBuilder::arg) + .returns(returns, false, false) +} + +fn interface( + name: &str, + store: fn(&'static mut ClassEntry, ArgInfoTables), + methods: Vec>, +) -> Result<()> { + methods + .into_iter() + .fold( + ClassBuilder::new(name).flags(ClassFlags::Interface).registration(store), + |class, method| class.method(method, MethodFlags::Public | MethodFlags::Abstract), + ) + .register() +} + +pub fn register() -> Result<()> { + interface( + "Flow\\Parquet\\ParquetFileReader", + store_parquet_file_reader, + vec![ + method("close", vec![], DataType::Void), + method("metadata", vec![], METADATA), + method( + "readColumns", + vec![ + Arg::new("columns", DataType::Array), + Arg::new("batchSize", DataType::Long), + Arg::new("limit", DataType::Long).allow_null(), + Arg::new("offset", DataType::Long).allow_null(), + ], + ITERATOR, + ), + method("rowsNumber", vec![], DataType::Long), + method("schema", vec![], SCHEMA), + method("totalByteSize", vec![], DataType::Long), + ], + )?; + interface( + "Flow\\Parquet\\ParquetFileWriter", + store_parquet_file_writer, + vec![ + method("close", vec![], DataType::Void), + method("writeBatch", vec![Arg::new("rows", DataType::Iterable)], DataType::Void), + method( + "writeColumns", + vec![Arg::new("columns", DataType::Array)], + DataType::Void, + ), + method("writeRow", vec![Arg::new("row", DataType::Array)], DataType::Void), + ], + )?; + interface( + "Flow\\Parquet\\ParquetEngine", + store_parquet_engine, + vec![ + method( + "openForRead", + vec![Arg::new("stream", SOURCE_STREAM)], + PARQUET_FILE_READER, + ), + method( + "openForWrite", + vec![ + Arg::new("stream", DESTINATION_STREAM), + Arg::new("schema", SCHEMA), + Arg::new("compression", COMPRESSIONS), + Arg::new("options", OPTIONS), + ], + PARQUET_FILE_WRITER, + ), + method( + "writeRows", + vec![ + Arg::new("stream", DESTINATION_STREAM), + Arg::new("schema", SCHEMA), + Arg::new("compression", COMPRESSIONS), + Arg::new("options", OPTIONS), + Arg::new("rows", DataType::Iterable), + ], + DataType::Void, + ), + ], + ) +} diff --git a/src/extension/arrow-ext/src/lib.rs b/src/extension/arrow-ext/src/lib.rs index 3f477a690b..8804b3bff3 100644 --- a/src/extension/arrow-ext/src/lib.rs +++ b/src/extension/arrow-ext/src/lib.rs @@ -1,14 +1,21 @@ +#[cfg(test)] +mod alloc; +mod exception; +mod interfaces; mod parquet; -mod stream; +mod php; +mod render; +mod thrift; +mod values; +#[cfg(not(test))] use std::alloc::System; use ext_php_rs::prelude::*; use ext_php_rs::zend::ModuleEntry; use ext_php_rs::{info_table_end, info_table_row, info_table_start}; -use parquet::reader::Reader; -use parquet::writer::Writer; +#[cfg(not(test))] #[global_allocator] static GLOBAL: System = System; @@ -21,21 +28,17 @@ pub extern "C" fn php_module_info(_module: *mut ModuleEntry) { info_table_end!(); } +/// The contract version flow-php/parquet expects (`Flow\Parquet\Engine\ArrowExtension::ABI`); both change together +/// whenever a registered class or interface changes. +const FLOW_ARROW_ABI: i64 = 1; + /// # Safety /// /// Invoked by the PHP/Zend engine during module startup. Must only be called by /// the engine through the registered startup hook, never directly. pub unsafe extern "C" fn module_startup(_type: i32, _module_number: i32) -> i32 { - if let Err(e) = stream::output_stream::register() { - eprintln!("arrow: failed to register Flow\\Arrow\\OutputStream: {e}"); - return -1; - } - if let Err(e) = stream::random_access_file::register() { - eprintln!("arrow: failed to register Flow\\Arrow\\RandomAccessFile: {e}"); - return -1; - } - if let Err(e) = parquet::exception::register() { - eprintln!("arrow: failed to register Flow\\Arrow\\Parquet\\Exception: {e}"); + if let Err(e) = interfaces::register() { + eprintln!("arrow: failed to register the Flow\\Parquet interfaces: {e}"); return -1; } 0 @@ -47,6 +50,15 @@ pub fn get_module(module: ModuleBuilder) -> ModuleBuilder { module .version(env!("ARROW_VERSION")) .info_function(php_module_info) - .class::() - .class::() + .constant(("FLOW_ARROW_ABI", FLOW_ARROW_ABI, &[])) + .globals(&php::GLOBALS) + .request_startup_function(php::request_startup) + .request_shutdown_function(php::request_shutdown) + .class::() + .class::() + .class::() + .class::() + .class::() + .class::() + .class::() } diff --git a/src/extension/arrow-ext/src/parquet/batch.rs b/src/extension/arrow-ext/src/parquet/batch.rs new file mode 100644 index 0000000000..60d5fde10f --- /dev/null +++ b/src/extension/arrow-ext/src/parquet/batch.rs @@ -0,0 +1,274 @@ +//! `Flow\Arrow\Parquet\RustBatchReader` and the Arrow C Data carriers: one batch = one struct-typed `FFI_ArrowSchema` (one +//! child per column) plus one struct-typed `FFI_ArrowArray`. The carrier owns both until a consumer moves the array +//! out; any extension's internal class exposing the two addresses is accepted. + +use std::collections::HashMap; +use std::ffi::{c_char, CStr}; +use std::sync::Arc; + +use arrow_array::ffi::{from_ffi, to_ffi, FFI_ArrowArray, FFI_ArrowSchema}; +use arrow_array::{Array, StructArray}; +use arrow_schema::{DataType, Field, Fields}; +use ext_php_rs::flags::ClassFlags; +use ext_php_rs::prelude::*; +use ext_php_rs::types::{ZendObject, Zval}; +use ext_php_rs::zend::ClassEntry; + +use crate::parquet::error::Error; +use crate::parquet::library::{batch_size, rows, RustParquetFileReader, INVALID_ARGUMENT}; +use crate::parquet::read::{ParquetReader, ReadPlan}; +use crate::parquet::source::PhpStream; +use crate::php::{call_handle_transparent, ce_method_ref}; +use crate::render::{exception, parquet_exception, Side}; + +/// Child field metadata: the arrow type of the file column the canonical child was read from. +pub const SOURCE_TYPE: &str = "parquet.source_type"; + +/// `ZEND_INTERNAL_CLASS` (zend_compile.h): a class an extension registered, which userland can neither declare nor +/// extend once final. +const ZEND_INTERNAL_CLASS: c_char = 1; + +#[php_class] +#[php(name = "Flow\\Arrow\\RustParquetBatch", flags = ClassFlags::Final)] +pub struct RustParquetBatch { + schema: FFI_ArrowSchema, + array: FFI_ArrowArray, + rows: usize, +} + +#[php_impl] +impl RustParquetBatch { + pub fn count(&self) -> i64 { + self.rows as i64 + } + + #[php(name = "arrowSchemaAddress")] + pub fn arrow_schema_address(&self) -> i64 { + std::ptr::from_ref(&self.schema) as i64 + } + + #[php(name = "arrowArrayAddress")] + pub fn arrow_array_address(&self) -> i64 { + std::ptr::from_ref(&self.array) as i64 + } +} + +#[php_class] +#[php(name = "Flow\\Arrow\\RustArrowSchema", flags = ClassFlags::Final)] +pub struct RustArrowSchema { + schema: FFI_ArrowSchema, +} + +#[php_impl] +impl RustArrowSchema { + #[php(name = "arrowSchemaAddress")] + pub fn arrow_schema_address(&self) -> i64 { + std::ptr::from_ref(&self.schema) as i64 + } +} + +#[php_class] +#[php(name = "Flow\\Arrow\\Parquet\\RustBatchReader", flags = ClassFlags::Final)] +pub struct RustBatchReader { + reader: Option, + stream: Arc, + /// The projection: canonical children named by column, each with its `SOURCE_TYPE`. + fields: Fields, +} + +#[php_impl] +impl RustBatchReader { + /// `$columns`: root names; `$batchSize` rows at most per `next()`. + pub fn __construct( + file: &RustParquetFileReader, + columns: Vec, + batch_size: i64, + offset: Option, + limit: Option, + ) -> PhpResult { + let batch_size = self::batch_size(batch_size)?; + let (offset, limit) = (rows(offset, "offset")?.unwrap_or(0), rows(limit, "limit")?); + let (stream, size, meta) = file.read()?; + let stream = Arc::clone(stream); + let reader = ParquetReader::open( + Arc::clone(&stream), + size, + meta, + &columns, + ReadPlan::new(meta, offset, limit), + batch_size, + ) + .map_err(|error| parquet_exception(error, &stream, Side::Read))?; + let fields = columns + .iter() + .zip(reader.fields()) + .zip(reader.types()) + .map(|((name, field), canonical)| { + Field::new(name, canonical.clone(), true).with_metadata(HashMap::from([( + SOURCE_TYPE.to_string(), + field.data_type().to_string(), + )])) + }) + .collect(); + + Ok(Self { + reader: Some(reader), + stream, + fields, + }) + } + + /// The struct schema of every batch, known before the first. + pub fn schema(&self) -> PhpResult { + Ok(RustArrowSchema { + schema: FFI_ArrowSchema::try_from(&DataType::Struct(self.fields.clone())) + .map_err(|error| parquet_exception(Error::Arrow(error), &self.stream, Side::Read))?, + }) + } + + /// At most `$batchSize` rows as one struct batch, null after the last. + pub fn next(&mut self) -> PhpResult> { + let Some(reader) = self.reader.as_mut() else { + return Ok(None); + }; + + let (rows, arrays) = match reader.next() { + None => { + self.reader = None; + + return Ok(None); + } + Some(batch) => batch.map_err(|error| parquet_exception(error, &self.stream, Side::Read))?, + }; + + let exported = StructArray::try_new_with_length(self.fields.clone(), arrays, None, rows) + .and_then(|array| to_ffi(&array.to_data())) + .map_err(|error| parquet_exception(Error::Arrow(error), &self.stream, Side::Read))?; + + Ok(Some(RustParquetBatch { + array: exported.0, + schema: exported.1, + rows, + })) + } + + pub fn close(&mut self) { + self.reader = None; + } +} + +fn refused(message: String) -> PhpException { + exception(INVALID_ARGUMENT, message) +} + +/// `$object->$method()` of an address method: a pointer the exporter owns. +fn address(object: &ZendObject, ce: &ClassEntry, method: &str) -> PhpResult { + let address = call_handle_transparent(ce_method_ref(ce, method)?, Some(object), &mut [])?; + + address + .long() + .and_then(|address| usize::try_from(address).ok()) + .filter(|address| *address != 0) + .ok_or_else(|| refused(format!("Arrow C Data {method}() must return a non-null address"))) +} + +/// The struct batch `$batch` carries, moved out of it: an internal class's `arrowSchemaAddress()` (read by reference) +/// and `arrowArrayAddress()` (moved, its release left NULL), refused before the move unless the schema is a struct +/// of as many children as the array and the array was not imported before. +pub fn import(batch: &Zval) -> PhpResult { + let object = batch + .object() + .ok_or_else(|| refused("Arrow C Data batch must be an object".to_string()))?; + let ce = unsafe { object.ce.as_ref() }.ok_or_else(|| refused("Arrow C Data batch has no class".to_string()))?; + + if ce.type_ != ZEND_INTERNAL_CLASS { + return Err(refused(format!( + "Arrow C Data batch must be an extension's class, got {}", + ce.name().unwrap_or_default() + ))); + } + + let schema = unsafe { &*(address(object, ce, "arrowSchemaAddress")? as *const FFI_ArrowSchema) }; + let source = unsafe { &mut *(address(object, ce, "arrowArrayAddress")? as *mut FFI_ArrowArray) }; + + if schema.format.is_null() || unsafe { CStr::from_ptr(schema.format) }.to_bytes() != b"+s" { + return Err(refused("Arrow C Data batch must be a struct array".to_string())); + } + + if source.release.is_none() { + return Err(refused("Arrow C Data batch is already imported".to_string())); + } + + if source.n_children != schema.n_children || source.length < 0 { + return Err(refused(format!( + "Arrow C Data batch of {} children does not match its schema of {}", + source.n_children, schema.n_children + ))); + } + + let moved = unsafe { std::ptr::read(source) }; + source.release = None; + + let data = unsafe { from_ffi(moved, schema) } + .and_then(|data| data.validate().map(|()| data)) + .map_err(|error| refused(format!("Arrow C Data batch is not a valid struct array: {error}")))?; + + Ok(StructArray::from(data)) +} + +#[cfg(test)] +mod tests { + use std::sync::Arc; + + use arrow_array::ffi::{from_ffi, to_ffi}; + use arrow_array::types::Int64Type; + use arrow_array::{Array, ArrayRef, Int64Array, ListArray, StructArray}; + use arrow_schema::{DataType, Field, Fields}; + + use crate::alloc::allocated_bytes; + + fn batch() -> StructArray { + let ids: ArrayRef = Arc::new(Int64Array::from_iter((0..1000).map(|i| (i % 7 != 0).then_some(i)))); + let lists: ArrayRef = Arc::new(ListArray::from_iter_primitive::( + (0..1000).map(|i| Some(vec![Some(i), None, Some(i * 2)])), + )); + + StructArray::new( + Fields::from(vec![ + Field::new("id", DataType::Int64, true), + Field::new("list", lists.data_type().clone(), true), + ]), + vec![ids, lists], + None, + ) + } + + #[test] + fn an_exported_batch_dropped_unimported_frees_everything() { + let baseline = allocated_bytes(); + + { + let exported = to_ffi(&batch().to_data()).unwrap(); + assert!(allocated_bytes() > baseline); + drop(exported); + } + + assert_eq!(allocated_bytes(), baseline); + } + + #[test] + fn an_exported_batch_imported_then_dropped_frees_everything() { + let baseline = allocated_bytes(); + + { + let (array, schema) = to_ffi(&batch().to_data()).unwrap(); + let imported = StructArray::from(unsafe { from_ffi(array, &schema) }.unwrap()); + assert_eq!(imported.len(), 1000); + assert_eq!(imported, batch()); + drop(schema); + drop(imported); + } + + assert_eq!(allocated_bytes(), baseline); + } +} diff --git a/src/extension/arrow-ext/src/parquet/canonical.rs b/src/extension/arrow-ext/src/parquet/canonical.rs new file mode 100644 index 0000000000..45ddb218c9 --- /dev/null +++ b/src/extension/arrow-ext/src/parquet/canonical.rs @@ -0,0 +1,1353 @@ +//! The cast table between what arrow-rs reads from / writes to Parquet and the canonical arrays: one arrow type per +//! storage (the layout flow_php's columns store), keyed on the arrow `DataType` alone. Temporal and decimal +//! casts are this module's own - `arrow_cast` rounds toward zero and nulls out overflows. + +use std::sync::Arc; + +use arrow_array::cast::AsArray; +use arrow_array::types::{ + Date32Type, Date64Type, Decimal128Type, Decimal256Type, DecimalType, DurationMicrosecondType, Float32Type, + Float64Type, GenericBinaryType, Int16Type, Int32Type, Int64Type, Int8Type, Time32MillisecondType, Time32SecondType, + Time64NanosecondType, TimestampMicrosecondType, TimestampMillisecondType, TimestampNanosecondType, + TimestampSecondType, UInt16Type, UInt32Type, UInt64Type, UInt8Type, +}; +use arrow_array::{ + make_array, Array, ArrayRef, ArrowPrimitiveType, BinaryArray, FixedSizeBinaryArray, Float64Array, GenericByteArray, + ListArray, MapArray, PrimitiveArray, StructArray, +}; +use arrow_buffer::{OffsetBuffer, ScalarBuffer}; +use arrow_schema::{DataType, Field, FieldRef, Fields, TimeUnit}; + +use crate::parquet::error::{parent_row, Error}; + +const UTC: &str = "UTC"; + +fn unsupported(data_type: &DataType) -> Error { + Error::Unsupported { + column: String::new(), + parquet: data_type.to_string(), + } +} + +fn timestamp() -> DataType { + DataType::Timestamp(TimeUnit::Microsecond, Some(UTC.into())) +} + +fn item(data_type: DataType) -> FieldRef { + Arc::new(Field::new("item", data_type, true)) +} + +fn entries(key: DataType, value: DataType) -> FieldRef { + Arc::new(Field::new( + "entries", + DataType::Struct(Fields::from(vec![ + Field::new("key", key, false), + Field::new("value", value, true), + ])), + false, + )) +} + +/// The canonical type `canonical()` turns an array of `field`'s type into. +// Must equal flow-batch-frame's kind::data_type() (flow-php-ext), nested names and nullability included - pinned by flow-php-ext phpt 091. +pub fn canonical_type(field: &Field) -> Result { + canonical_data_type(field.data_type()).map_err(|e| e.in_column(field.name())) +} + +fn canonical_data_type(data_type: &DataType) -> Result { + Ok(match data_type { + DataType::Null + | DataType::Boolean + | DataType::Int64 + | DataType::Float64 + | DataType::Date32 + | DataType::FixedSizeBinary(16) => data_type.clone(), + DataType::Int8 + | DataType::Int16 + | DataType::Int32 + | DataType::UInt8 + | DataType::UInt16 + | DataType::UInt32 + | DataType::UInt64 => DataType::Int64, + DataType::Float32 | DataType::Decimal128(_, _) | DataType::Decimal256(_, _) => DataType::Float64, + DataType::Utf8 + | DataType::Binary + | DataType::LargeUtf8 + | DataType::LargeBinary + | DataType::FixedSizeBinary(_) => DataType::Binary, + DataType::Timestamp(_, _) => timestamp(), + DataType::Time64(TimeUnit::Microsecond | TimeUnit::Nanosecond) + | DataType::Time32(TimeUnit::Millisecond | TimeUnit::Second) => DataType::Duration(TimeUnit::Microsecond), + DataType::Date64 => DataType::Date32, + DataType::List(element) | DataType::LargeList(element) => { + DataType::List(item(canonical_data_type(element.data_type())?)) + } + DataType::Map(field, _) => { + let (key, value) = map_fields(field)?; + + DataType::Map( + entries( + canonical_data_type(key.data_type())?, + canonical_data_type(value.data_type())?, + ), + false, + ) + } + DataType::Struct(fields) => DataType::Struct(struct_fields(fields)?), + other => return Err(unsupported(other)), + }) +} + +fn map_fields(entries: &FieldRef) -> Result<(&FieldRef, &FieldRef), Error> { + match entries.data_type() { + DataType::Struct(fields) if fields.len() == 2 => Ok((&fields[0], &fields[1])), + other => Err(unsupported(other)), + } +} + +fn struct_fields(fields: &Fields) -> Result { + fields + .iter() + .map(|field| Ok(Field::new(field.name(), canonical_data_type(field.data_type())?, true))) + .collect() +} + +/// `array` (of `field`'s type, as arrow-rs read it) as its canonical array; refusals name `field`. +pub fn canonical(field: &Field, array: &ArrayRef) -> Result { + cast(array).map_err(|e| e.in_column(field.name())) +} + +fn widened(array: &ArrayRef) -> ArrayRef +where + T::Native: Into, +{ + Arc::new(array.as_primitive::().unary::<_, Int64Type>(Into::into)) +} + +/// Every valid value through `f`; the first `None` refuses its row. +fn checked( + array: &ArrayRef, + f: impl Fn(T::Native) -> Option, +) -> Result, Error> { + let input = array.as_primitive::(); + let mut values = Vec::with_capacity(input.len()); + + for (row, value) in input.values().iter().enumerate() { + values.push(if input.is_null(row) { + O::Native::default() + } else { + f(*value).ok_or(Error::Overflow { + column: String::new(), + row, + })? + }); + } + + Ok(PrimitiveArray::new(ScalarBuffer::from(values), input.nulls().cloned())) +} + +fn retagged(array: &ArrayRef, to: DataType) -> ArrayRef { + make_array(unsafe { array.to_data().into_builder().data_type(to).build_unchecked() }) +} + +fn micros>( + array: &ArrayRef, + f: impl Fn(i64) -> Option, +) -> Result { + Ok(Arc::new( + checked::(array, f)?.with_timezone(UTC), + )) +} + +fn decimals(array: &ArrayRef, precision: u8, scale: i8) -> Result { + let input = array.as_primitive::(); + let mut values = Vec::with_capacity(input.len()); + + for (row, value) in input.values().iter().enumerate() { + values.push(if input.is_null(row) { + 0.0 + } else { + T::format_decimal(*value, precision, scale) + .parse::() + .map_err(|e| Error::Arrow(arrow_schema::ArrowError::ParseError(e.to_string())))? + }); + } + + Ok(Arc::new(Float64Array::new( + ScalarBuffer::from(values), + input.nulls().cloned(), + ))) +} + +/// Large offsets narrowed to i32 and rebased to 0, with the values they span. +fn narrowed(offsets: &[i64]) -> Result, Error> { + let first = offsets[0]; + let narrowed = offsets + .iter() + .enumerate() + .map(|(i, offset)| { + i32::try_from(offset - first).map_err(|_| Error::Overflow { + column: String::new(), + row: i.saturating_sub(1), + }) + }) + .collect::, _>>()?; + + Ok(OffsetBuffer::new(ScalarBuffer::from(narrowed))) +} + +fn large_binary(array: &ArrayRef) -> Result { + let data = array.to_data(); + let offsets = data.buffers()[0].typed_data::(); + let offsets = &offsets[data.offset()..data.offset() + data.len() + 1]; + let values = data.buffers()[1].slice_with_length(offsets[0] as usize, (offsets[data.len()] - offsets[0]) as usize); + + Ok(Arc::new(GenericByteArray::>::new( + narrowed(offsets)?, + values, + data.nulls().cloned(), + ))) +} + +/// Fixed-size values as Binary over the same value buffer: offsets `n·i`. +fn fixed_binary(array: &ArrayRef) -> Result { + let fixed = array.as_fixed_size_binary(); + let size = i32::try_from(fixed.value_size()).map_err(|_| Error::Overflow { + column: String::new(), + row: 0, + })?; + let offsets = (0..=fixed.len()) + .map(|row| { + i32::try_from(row) + .ok() + .and_then(|row| row.checked_mul(size)) + .ok_or(Error::Overflow { + column: String::new(), + row: row.saturating_sub(1), + }) + }) + .collect::, _>>()?; + + Ok(Arc::new(BinaryArray::new( + OffsetBuffer::new(ScalarBuffer::from(offsets)), + fixed.values().clone(), + fixed.nulls().cloned(), + ))) +} + +fn cast(array: &ArrayRef) -> Result { + Ok(match array.data_type() { + DataType::Null + | DataType::Boolean + | DataType::Int64 + | DataType::Float64 + | DataType::Date32 + | DataType::FixedSizeBinary(16) => Arc::clone(array), + DataType::Timestamp(TimeUnit::Microsecond, Some(zone)) if zone.as_ref() == UTC => Arc::clone(array), + DataType::Int8 => widened::(array), + DataType::Int16 => widened::(array), + DataType::Int32 => widened::(array), + DataType::UInt8 => widened::(array), + DataType::UInt16 => widened::(array), + DataType::UInt32 => widened::(array), + DataType::UInt64 => Arc::new(checked::(array, |value| { + i64::try_from(value).ok() + })?), + DataType::Float32 => Arc::new(array.as_primitive::().unary::<_, Float64Type>(f64::from)), + DataType::Utf8 | DataType::Binary => retagged(array, DataType::Binary), + DataType::LargeUtf8 | DataType::LargeBinary => large_binary(array)?, + DataType::FixedSizeBinary(_) => fixed_binary(array)?, + DataType::Timestamp(TimeUnit::Microsecond, _) => retagged(array, timestamp()), + DataType::Timestamp(TimeUnit::Millisecond, _) => { + micros::(array, |ms| ms.checked_mul(1_000))? + } + DataType::Timestamp(TimeUnit::Second, _) => micros::(array, |s| s.checked_mul(1_000_000))?, + DataType::Timestamp(TimeUnit::Nanosecond, _) => { + micros::(array, |ns| Some(ns.div_euclid(1_000)))? + } + DataType::Time64(TimeUnit::Microsecond) => retagged(array, DataType::Duration(TimeUnit::Microsecond)), + DataType::Time64(TimeUnit::Nanosecond) => Arc::new( + array + .as_primitive::() + .unary::<_, DurationMicrosecondType>(|ns| ns.div_euclid(1_000)), + ), + DataType::Time32(TimeUnit::Millisecond) => Arc::new( + array + .as_primitive::() + .unary::<_, DurationMicrosecondType>(|ms| i64::from(ms) * 1_000), + ), + DataType::Time32(TimeUnit::Second) => Arc::new( + array + .as_primitive::() + .unary::<_, DurationMicrosecondType>(|s| i64::from(s) * 1_000_000), + ), + DataType::Date64 => Arc::new(checked::(array, |ms| { + i32::try_from(ms.div_euclid(86_400_000)).ok() + })?), + DataType::Decimal128(precision, scale) => decimals::(array, *precision, *scale)?, + DataType::Decimal256(precision, scale) => decimals::(array, *precision, *scale)?, + DataType::List(_) => { + let list = array.as_list::(); + let values = cast(list.values()).map_err(|e| e.at_row(|child| parent_row(list.value_offsets(), child)))?; + + Arc::new(ListArray::try_new( + item(values.data_type().clone()), + list.offsets().clone(), + values, + list.nulls().cloned(), + )?) + } + DataType::LargeList(_) => { + let list = array.as_list::(); + let offsets = list.value_offsets(); + let (first, last) = (offsets[0] as usize, offsets[offsets.len() - 1] as usize); + let values = cast(&list.values().slice(first, last - first)) + .map_err(|e| e.at_row(|child| parent_row(offsets, child + first)))?; + + Arc::new(ListArray::try_new( + item(values.data_type().clone()), + narrowed(offsets)?, + values, + list.nulls().cloned(), + )?) + } + DataType::Map(field, _) => { + map_fields(field)?; + let map = array.as_map(); + let at_parent = |e: Error| e.at_row(|child| parent_row(map.value_offsets(), child)); + let keys = cast(map.keys()).map_err(at_parent)?; + let values = cast(map.values()).map_err(at_parent)?; + let field = entries(keys.data_type().clone(), values.data_type().clone()); + let DataType::Struct(fields) = field.data_type() else { + unreachable!("entries() builds a struct") + }; + + Arc::new(MapArray::try_new( + Arc::clone(&field), + map.offsets().clone(), + StructArray::try_new_with_length(fields.clone(), vec![keys, values], None, map.entries().len())?, + map.nulls().cloned(), + false, + )?) + } + DataType::Struct(fields) => { + let structure = array.as_struct(); + let children = structure.columns().iter().map(cast).collect::, _>>()?; + let fields = fields + .iter() + .zip(&children) + .map(|(field, child)| Field::new(field.name(), child.data_type().clone(), true)) + .collect::(); + + Arc::new(StructArray::try_new_with_length( + fields, + children, + structure.nulls().cloned(), + structure.len(), + )?) + } + other => return Err(unsupported(other)), + }) +} + +/// A canonical array as the writer schema's `target` type: the inverse retags, UTF-8 validated where Parquet requires +/// it, nested children renamed to the target's fields. Refusals carry no column - the writer names it. +pub fn to_parquet(array: &ArrayRef, target: &DataType) -> Result { + if array.data_type() == target { + return Ok(Arc::clone(array)); + } + + Ok(match (array.data_type(), target) { + (DataType::Binary, DataType::Utf8) => utf8(array)?, + (DataType::Int64, DataType::Int8) => Arc::new(narrowed_int::(array)?), + (DataType::Int64, DataType::Int16) => Arc::new(narrowed_int::(array)?), + (DataType::Int64, DataType::Int32) => Arc::new(narrowed_int::(array)?), + (DataType::Int64, DataType::UInt8) => Arc::new(narrowed_int::(array)?), + (DataType::Int64, DataType::UInt16) => Arc::new(narrowed_int::(array)?), + (DataType::Int64, DataType::UInt32) => Arc::new(narrowed_int::(array)?), + (DataType::Int64, DataType::UInt64) => Arc::new(narrowed_int::(array)?), + (DataType::Float64, DataType::Float32) => Arc::new( + array + .as_primitive::() + .unary::<_, Float32Type>(|value| value as f32), + ), + (DataType::Float64, DataType::Decimal128(precision, scale)) => Arc::new( + checked_values::(array, |value| decimal_unscaled(value, *precision, *scale))? + .with_precision_and_scale(*precision, *scale)?, + ), + (DataType::Binary, DataType::FixedSizeBinary(size)) => fixed_size(array, *size)?, + (DataType::Duration(TimeUnit::Microsecond), DataType::Time64(TimeUnit::Microsecond)) => { + retagged(array, target.clone()) + } + (DataType::Duration(TimeUnit::Microsecond), DataType::Time32(TimeUnit::Millisecond)) => { + Arc::new(checked::( + array, + |us| i32::try_from(us.div_euclid(1_000)).ok(), + )?) + } + (DataType::Duration(TimeUnit::Microsecond), DataType::Time64(TimeUnit::Nanosecond)) => { + Arc::new(checked::(array, |us| { + us.checked_mul(1_000) + })?) + } + (DataType::Timestamp(TimeUnit::Microsecond, Some(zone)), DataType::Timestamp(TimeUnit::Millisecond, to)) + if zone.as_ref() == UTC => + { + Arc::new( + checked::(array, |us| Some(us.div_euclid(1_000)))? + .with_timezone_opt(to.clone()), + ) + } + (DataType::Timestamp(TimeUnit::Microsecond, Some(zone)), DataType::Timestamp(TimeUnit::Nanosecond, to)) + if zone.as_ref() == UTC => + { + Arc::new( + checked::(array, |us| us.checked_mul(1_000))? + .with_timezone_opt(to.clone()), + ) + } + (DataType::Timestamp(TimeUnit::Microsecond, Some(zone)), DataType::Timestamp(TimeUnit::Microsecond, _)) + if zone.as_ref() == UTC => + { + retagged(array, target.clone()) + } + (DataType::List(_), DataType::List(element)) => { + let list = array.as_list::(); + + Arc::new(ListArray::try_new( + Arc::clone(element), + list.offsets().clone(), + to_parquet(list.values(), element.data_type()) + .map_err(|e| e.at_row(|child| parent_row(list.value_offsets(), child)))?, + list.nulls().cloned(), + )?) + } + (DataType::Map(_, _), DataType::Map(field, sorted)) => { + let (key, value) = map_fields(field)?; + let DataType::Struct(fields) = field.data_type() else { + unreachable!("map_fields() matched a struct") + }; + let map = array.as_map(); + let at_parent = |e: Error| e.at_row(|child| parent_row(map.value_offsets(), child)); + + Arc::new(MapArray::try_new( + Arc::clone(field), + map.offsets().clone(), + StructArray::try_new_with_length( + fields.clone(), + vec![ + to_parquet(map.keys(), key.data_type()).map_err(at_parent)?, + to_parquet(map.values(), value.data_type()).map_err(at_parent)?, + ], + None, + map.entries().len(), + )?, + map.nulls().cloned(), + *sorted, + )?) + } + (DataType::Struct(from), DataType::Struct(fields)) if from.len() == fields.len() => { + let structure = array.as_struct(); + + Arc::new(StructArray::try_new_with_length( + fields.clone(), + structure + .columns() + .iter() + .zip(fields.iter()) + .map(|(child, field)| to_parquet(child, field.data_type())) + .collect::, _>>()?, + structure.nulls().cloned(), + structure.len(), + )?) + } + _ => return Err(unsupported(target)), + }) +} + +/// Int64 values in `T`'s range; the first one outside refuses its row. +fn narrowed_int(array: &ArrayRef) -> Result, Error> +where + T::Native: TryFrom, +{ + checked::(array, |value| T::Native::try_from(value).ok()) +} + +/// Every valid value through `f`, which refuses a row with the error it returns. +fn checked_values( + array: &ArrayRef, + f: impl Fn(T::Native) -> Result, +) -> Result, Error> { + let input = array.as_primitive::(); + let mut values = Vec::with_capacity(input.len()); + + for (row, value) in input.values().iter().enumerate() { + values.push(if input.is_null(row) { + O::Native::default() + } else { + f(*value).map_err(|error| error.at_row(|_| row))? + }); + } + + Ok(PrimitiveArray::new(ScalarBuffer::from(values), input.nulls().cloned())) +} + +/// `Flow\Parquet\Binary\decimal_unscaled()`: the float's shortest round-trip digits (Rust `{}` prints the digits PHP's +/// `var_export()` does, never an exponent), rounded half away from zero at `scale`; more than `precision` digits +/// overflow. +fn decimal_unscaled(value: f64, precision: u8, scale: i8) -> Result { + if !value.is_finite() { + return Err(Error::Value { + column: String::new(), + row: 0, + expected: "a finite number", + got: if value.is_nan() { + "NAN".to_string() + } else if value > 0.0 { + "INF".to_string() + } else { + "-INF".to_string() + }, + }); + } + + let overflow = || Error::Overflow { + column: String::new(), + row: 0, + }; + let text = format!("{}", value.abs()); + let (integer, fraction) = text.split_once('.').unwrap_or((&text, "")); + let scale = scale.max(0) as usize; + let kept = fraction.get(..scale.min(fraction.len())).unwrap_or(""); + let mut digits = format!("{integer}{kept}{}", "0".repeat(scale - kept.len())); + let round_up = fraction.as_bytes().get(scale).is_some_and(|digit| *digit >= b'5'); + digits = digits.trim_start_matches('0').to_string(); + + if digits.len() > usize::from(precision) || digits.len() > 38 { + return Err(overflow()); + } + + let mut unscaled = if digits.is_empty() { + 0 + } else { + digits.parse::().map_err(|_| overflow())? + }; + + if round_up { + unscaled += 1; + + if unscaled.to_string().len() > usize::from(precision) { + return Err(overflow()); + } + } + + Ok(if value.is_sign_negative() { -unscaled } else { unscaled }) +} + +/// Binary values of exactly `size` bytes as fixed-size binary; the first of another length refuses its row. +fn fixed_size(array: &ArrayRef, size: i32) -> Result { + let binary = array.as_binary::(); + let width = size as usize; + let mut values = vec![0u8; binary.len() * width]; + + for row in 0..binary.len() { + if binary.is_null(row) { + continue; + } + + let value = binary.value(row); + + if value.len() != width { + return Err(Error::Length { + column: String::new(), + row, + expected: width, + }); + } + + values[row * width..(row + 1) * width].copy_from_slice(value); + } + + Ok(Arc::new(FixedSizeBinaryArray::new( + size, + values.into(), + binary.nulls().cloned(), + ))) +} + +/// Binary as Utf8 over the same buffers once every value is valid UTF-8; else the first row that is not. +fn utf8(array: &ArrayRef) -> Result { + match array.to_data().into_builder().data_type(DataType::Utf8).build() { + Ok(data) => Ok(make_array(data)), + Err(error) => { + let binary: &BinaryArray = array.as_binary::(); + + Err((0..binary.len()) + .find(|row| binary.is_valid(*row) && std::str::from_utf8(binary.value(*row)).is_err()) + .map_or(Error::Arrow(error), |row| Error::InvalidUtf8 { + column: String::new(), + row, + })) + } + } +} + +#[cfg(test)] +mod tests { + use std::sync::Arc; + + use arrow_array::builder::{Int32Builder, ListBuilder, MapBuilder, StringBuilder}; + use arrow_array::cast::AsArray; + use arrow_array::types::{ + Date32Type, Decimal128Type, DurationMicrosecondType, Float32Type, Float64Type, Int64Type, Int8Type, + Time32MillisecondType, Time64MicrosecondType, Time64NanosecondType, TimestampMicrosecondType, + TimestampMillisecondType, TimestampNanosecondType, UInt64Type, + }; + use arrow_array::{ + Array, ArrayRef, BinaryArray, BooleanArray, Date32Array, Date64Array, Decimal128Array, Decimal256Array, + DurationMicrosecondArray, FixedSizeBinaryArray, Float32Array, Float64Array, Int16Array, Int32Array, Int64Array, + Int8Array, LargeBinaryArray, LargeListArray, LargeStringArray, ListArray, StringArray, StructArray, + Time32MillisecondArray, Time32SecondArray, Time64MicrosecondArray, Time64NanosecondArray, + TimestampMicrosecondArray, TimestampMillisecondArray, TimestampNanosecondArray, TimestampSecondArray, + UInt16Array, UInt32Array, UInt64Array, UInt8Array, + }; + use arrow_buffer::i256; + use arrow_schema::{DataType, Field, Fields, TimeUnit}; + + use super::{canonical, canonical_type, to_parquet}; + use crate::parquet::error::Error; + + fn read(array: ArrayRef) -> ArrayRef { + let field = Field::new("column", array.data_type().clone(), true); + let canonical = canonical(&field, &array).unwrap(); + + assert_eq!(canonical.data_type(), &canonical_type(&field).unwrap()); + + canonical + } + + fn refusal(array: ArrayRef) -> Error { + canonical(&Field::new("column", array.data_type().clone(), true), &array).unwrap_err() + } + + fn int64(array: &ArrayRef) -> Vec> { + array.as_primitive::().iter().collect() + } + + /// The canonical list layout: a nullable `item` child. + fn list_of(element: DataType) -> DataType { + DataType::List(Arc::new(Field::new("item", element, true))) + } + + /// The canonical map layout: `entries` of a non-null `key` and a nullable `value`. + fn map_of(key: DataType, value: DataType) -> DataType { + DataType::Map( + Arc::new(Field::new( + "entries", + DataType::Struct(Fields::from(vec![ + Field::new("key", key, false), + Field::new("value", value, true), + ])), + false, + )), + false, + ) + } + + #[test] + fn canonical_types_are_kept() { + let arrays: Vec = vec![ + Arc::new(BooleanArray::from(vec![Some(true), None])), + Arc::new(Int64Array::from(vec![Some(1), None])), + Arc::new(Float64Array::from(vec![Some(1.5), None])), + Arc::new(Date32Array::from(vec![Some(-1), None])), + Arc::new(FixedSizeBinaryArray::try_from_iter(vec![[7u8; 16]].into_iter()).unwrap()), + Arc::new(TimestampMicrosecondArray::from(vec![Some(-1), None]).with_timezone("UTC")), + ]; + + for array in arrays { + assert!(Arc::ptr_eq(&read(Arc::clone(&array)), &array)); + } + } + + #[test] + fn integers_widen_to_int64() { + let expected = vec![Some(-3), None, Some(7)]; + + assert_eq!( + int64(&read(Arc::new(Int8Array::from(vec![Some(-3), None, Some(7)])))), + expected + ); + assert_eq!( + int64(&read(Arc::new(Int16Array::from(vec![Some(-3), None, Some(7)])))), + expected + ); + assert_eq!( + int64(&read(Arc::new(Int32Array::from(vec![Some(-3), None, Some(7)])))), + expected + ); + assert_eq!( + int64(&read(Arc::new(UInt8Array::from(vec![Some(255), None])))), + vec![Some(255), None] + ); + assert_eq!( + int64(&read(Arc::new(UInt16Array::from(vec![Some(65_535), None])))), + vec![Some(65_535), None] + ); + assert_eq!( + int64(&read(Arc::new(UInt32Array::from(vec![Some(u32::MAX), None])))), + vec![Some(i64::from(u32::MAX)), None] + ); + assert_eq!( + int64(&read(Arc::new(UInt64Array::from(vec![Some(i64::MAX as u64), None])))), + vec![Some(i64::MAX), None] + ); + } + + #[test] + fn uint64_above_int64_is_refused_at_its_row() { + let error = refusal(Arc::new(UInt64Array::from(vec![ + Some(1), + None, + Some(i64::MAX as u64 + 1), + ]))); + + assert!(matches!(error, Error::Overflow { column, row: 2 } if column == "column")); + } + + #[test] + fn a_null_slot_never_overflows() { + let values = UInt64Array::new(vec![1, u64::MAX].into(), Some(vec![true, false].into())); + + assert_eq!(int64(&read(Arc::new(values))), vec![Some(1), None]); + } + + #[test] + fn float32_widens_to_float64() { + let canonical = read(Arc::new(Float32Array::from(vec![Some(0.5), None]))); + + assert_eq!( + canonical.as_primitive::().iter().collect::>(), + vec![Some(0.5), None] + ); + } + + #[test] + fn strings_and_binaries_are_binary() { + let expected: Vec> = vec![Some(b"a"), None, Some(b"")]; + + for array in [ + Arc::new(StringArray::from(vec![Some("a"), None, Some("")])) as ArrayRef, + Arc::new(BinaryArray::from(expected.clone())), + Arc::new(LargeStringArray::from(vec![Some("a"), None, Some("")])), + Arc::new(LargeBinaryArray::from(expected.clone())), + ] { + let canonical = read(array); + + assert_eq!(canonical.as_binary::().iter().collect::>(), expected); + } + } + + #[test] + fn a_sliced_large_string_is_rebased() { + let array = LargeStringArray::from(vec!["skipped", "kept", "too"]).slice(1, 2); + + assert_eq!( + read(Arc::new(array)).as_binary::().iter().collect::>(), + vec![Some(&b"kept"[..]), Some(&b"too"[..])] + ); + } + + #[test] + fn timestamps_are_utc_microseconds() { + let micros = |array: ArrayRef| -> Vec> { + read(array).as_primitive::().iter().collect() + }; + + assert_eq!( + micros(Arc::new(TimestampMicrosecondArray::from(vec![Some(-7), None]))), + vec![Some(-7), None] + ); + assert_eq!( + micros(Arc::new( + TimestampMicrosecondArray::from(vec![Some(5)]).with_timezone("Europe/Warsaw") + )), + vec![Some(5)] + ); + assert_eq!( + micros(Arc::new(TimestampMillisecondArray::from(vec![Some(-2), None]))), + vec![Some(-2_000), None] + ); + assert_eq!( + micros(Arc::new(TimestampSecondArray::from(vec![Some(3)]))), + vec![Some(3_000_000)] + ); + assert_eq!( + micros(Arc::new(TimestampNanosecondArray::from(vec![ + Some(-1_500), + Some(1_999), + None + ]))), + vec![Some(-2), Some(1), None] + ); + } + + #[test] + fn a_timestamp_beyond_microseconds_is_refused() { + let error = refusal(Arc::new(TimestampMillisecondArray::from(vec![Some(0), Some(i64::MAX)]))); + + assert!(matches!(error, Error::Overflow { row: 1, .. })); + assert!(matches!( + refusal(Arc::new(TimestampSecondArray::from(vec![Some(i64::MIN)]))), + Error::Overflow { row: 0, .. } + )); + } + + #[test] + fn times_are_microsecond_durations() { + let micros = |array: ArrayRef| -> Vec> { + read(array).as_primitive::().iter().collect() + }; + + assert_eq!( + micros(Arc::new(Time64MicrosecondArray::from(vec![Some(5), None]))), + vec![Some(5), None] + ); + assert_eq!( + micros(Arc::new(Time64NanosecondArray::from(vec![Some(1_999), None]))), + vec![Some(1), None] + ); + assert_eq!( + micros(Arc::new(Time32MillisecondArray::from(vec![Some(2), None]))), + vec![Some(2_000), None] + ); + assert_eq!( + micros(Arc::new(Time32SecondArray::from(vec![Some(3)]))), + vec![Some(3_000_000)] + ); + } + + #[test] + fn date64_floors_to_days() { + let days = read(Arc::new(Date64Array::from(vec![Some(-1), Some(86_400_000), None]))); + + assert_eq!( + days.as_primitive::().iter().collect::>(), + vec![Some(-1), Some(1), None] + ); + assert!(matches!( + refusal(Arc::new(Date64Array::from(vec![Some(i64::MAX)]))), + Error::Overflow { row: 0, .. } + )); + } + + #[test] + fn decimals_are_floats_through_their_decimal_text() { + let decimal128 = Decimal128Array::from(vec![Some(12_345), Some(-5), None]) + .with_precision_and_scale(9, 2) + .unwrap(); + let decimal256 = Decimal256Array::from(vec![Some(i256::from_i128(10_000_000_001)), None]) + .with_precision_and_scale(50, 10) + .unwrap(); + + assert_eq!( + read(Arc::new(decimal128)) + .as_primitive::() + .iter() + .collect::>(), + vec![Some(123.45), Some(-0.05), None] + ); + assert_eq!( + read(Arc::new(decimal256)) + .as_primitive::() + .iter() + .collect::>(), + vec![Some(1.0000000001), None] + ); + } + + #[test] + fn lists_are_renamed_to_the_kind_layout() { + let mut builder = + ListBuilder::new(Int32Builder::new()).with_field(Field::new("element", DataType::Int32, false)); + builder.append_value([Some(1), Some(2)]); + builder.append_null(); + builder.append_value([Some(3)]); + let canonical = read(Arc::new(builder.finish())); + + assert_eq!(canonical.data_type(), &list_of(DataType::Int64)); + assert_eq!( + int64(canonical.as_list::().values()), + vec![Some(1), Some(2), Some(3)] + ); + assert!(canonical.is_null(1)); + } + + #[test] + fn large_lists_narrow_their_offsets() { + let array = LargeListArray::from_iter_primitive::(vec![ + Some(vec![Some(9)]), + Some(vec![Some(1), None]), + None, + ]) + .slice(1, 2); + let canonical = read(Arc::new(array)); + let list = canonical.as_list::(); + + assert_eq!(canonical.data_type(), &list_of(DataType::Int64)); + assert_eq!(list.value_offsets(), &[0, 2, 2]); + assert_eq!(int64(list.values()), vec![Some(1), None]); + } + + #[test] + fn maps_are_renamed_to_the_kind_layout() { + let mut builder = MapBuilder::new(None, StringBuilder::new(), Int32Builder::new()); + builder.keys().append_value("a"); + builder.values().append_value(1); + builder.append(true).unwrap(); + builder.append(false).unwrap(); + let canonical = read(Arc::new(builder.finish())); + + assert_eq!(canonical.data_type(), &map_of(DataType::Binary, DataType::Int64)); + assert_eq!(int64(canonical.as_map().values()), vec![Some(1)]); + assert!(canonical.is_null(1)); + } + + #[test] + fn structs_are_relabelled_nullable() { + let array = StructArray::from(vec![ + ( + Arc::new(Field::new("id", DataType::Int32, false)), + Arc::new(Int32Array::from(vec![1, 2])) as ArrayRef, + ), + ( + Arc::new(Field::new("name", DataType::Utf8, false)), + Arc::new(StringArray::from(vec!["a", "b"])) as ArrayRef, + ), + ]); + let canonical = read(Arc::new(array)); + + assert_eq!( + canonical.data_type(), + &DataType::Struct(Fields::from(vec![ + Field::new("id", DataType::Int64, true), + Field::new("name", DataType::Binary, true), + ])) + ); + assert_eq!(int64(canonical.as_struct().column(0)), vec![Some(1), Some(2)]); + } + + #[test] + fn a_nested_overflow_is_refused_at_its_parent_row() { + let values = Arc::new(UInt64Array::from(vec![1, 2, u64::MAX])); + let list = ListArray::new( + Arc::new(Field::new("element", DataType::UInt64, false)), + arrow_buffer::OffsetBuffer::new(vec![0, 2, 3].into()), + values, + None, + ); + + assert!(matches!(refusal(Arc::new(list)), Error::Overflow { row: 1, .. })); + } + + #[test] + fn a_list_of_nulls_is_kept() { + let list: ArrayRef = Arc::new(ListArray::new( + Arc::new(Field::new("element", DataType::Null, true)), + arrow_buffer::OffsetBuffer::new(vec![0, 0, 2].into()), + Arc::new(arrow_array::NullArray::new(2)), + None, + )); + let canonical = read(list); + + assert_eq!(canonical.as_list::().value_offsets(), &[0, 0, 2]); + assert_eq!(canonical.as_list::().values().data_type(), &DataType::Null); + } + + #[test] + fn unsupported_types_are_refused_naming_the_type() { + let interval = DataType::Interval(arrow_schema::IntervalUnit::DayTime); + let field = Field::new( + "intervals", + DataType::List(Arc::new(Field::new("element", interval, true))), + true, + ); + + assert!(matches!( + canonical_type(&field), + Err(Error::Unsupported { column, parquet }) if column == "intervals" && parquet == "Interval(DayTime)" + )); + assert!(matches!( + refusal(Arc::new(arrow_array::DurationSecondArray::from(vec![1]))), + Error::Unsupported { .. } + )); + } + + #[test] + fn fixed_size_binaries_of_another_size_than_16_are_binary() { + let array = FixedSizeBinaryArray::try_from_sparse_iter_with_size( + vec![Some(*b"abcd"), None, Some(*b"wxyz"), Some(*b"skip")].into_iter(), + 4, + ) + .unwrap() + .slice(0, 3); + let canonical = read(Arc::new(array)); + + assert_eq!( + canonical.as_binary::().iter().collect::>(), + vec![Some(&b"abcd"[..]), None, Some(&b"wxyz"[..])] + ); + } + + #[test] + fn to_parquet_narrows_integers_in_range() { + let array: ArrayRef = Arc::new(Int64Array::from(vec![Some(-128), None, Some(127)])); + + assert_eq!( + to_parquet(&array, &DataType::Int8) + .unwrap() + .as_primitive::() + .iter() + .collect::>(), + vec![Some(-128), None, Some(127)] + ); + + let unsigned: ArrayRef = Arc::new(Int64Array::from(vec![Some(4_000_000_000), Some(i64::MAX)])); + + assert_eq!( + to_parquet(&unsigned, &DataType::UInt64) + .unwrap() + .as_primitive::() + .values() + .to_vec(), + vec![4_000_000_000, i64::MAX as u64] + ); + + for target in [ + DataType::Int16, + DataType::Int32, + DataType::UInt8, + DataType::UInt16, + DataType::UInt32, + ] { + assert_eq!(to_parquet(&array.slice(1, 1), &target).unwrap().data_type(), &target); + } + } + + #[test] + fn to_parquet_refuses_integers_out_of_the_target_range_at_their_row() { + let cases = [ + (DataType::Int8, vec![1, 128]), + (DataType::Int16, vec![1, -32_769]), + (DataType::Int32, vec![1, 2_147_483_648]), + (DataType::UInt8, vec![1, 256]), + (DataType::UInt16, vec![1, -1]), + (DataType::UInt32, vec![1, 4_294_967_296]), + (DataType::UInt64, vec![1, -1]), + ]; + + for (target, values) in cases { + let array: ArrayRef = Arc::new(Int64Array::from(values)); + + assert!( + matches!(to_parquet(&array, &target), Err(Error::Overflow { row: 1, .. })), + "{target}" + ); + } + } + + #[test] + fn to_parquet_narrows_floats() { + let array: ArrayRef = Arc::new(Float64Array::from(vec![Some(0.5), None])); + + assert_eq!( + to_parquet(&array, &DataType::Float32) + .unwrap() + .as_primitive::() + .iter() + .collect::>(), + vec![Some(0.5), None] + ); + } + + fn unscaled(values: Vec>, precision: u8, scale: i8) -> Result>, Error> { + let array: ArrayRef = Arc::new(Float64Array::from(values)); + + to_parquet(&array, &DataType::Decimal128(precision, scale)) + .map(|decimal| decimal.as_primitive::().iter().collect()) + } + + #[test] + fn to_parquet_writes_decimals_from_the_shortest_digits_half_away_from_zero() { + assert_eq!( + unscaled( + vec![ + Some(123.45), + Some(-0.125), + Some(0.125), + Some(0.1249), + None, + Some(-0.001) + ], + 9, + 2 + ) + .unwrap(), + vec![Some(12_345), Some(-13), Some(13), Some(12), None, Some(0)] + ); + assert_eq!(unscaled(vec![Some(1e20)], 38, 2).unwrap(), vec![Some(10i128.pow(22))]); + assert_eq!( + unscaled(vec![Some(0.1 + 0.2)], 20, 17).unwrap(), + vec![Some(30_000_000_000_000_004)] + ); + assert_eq!(unscaled(vec![Some(-5.0)], 1, -1).unwrap(), vec![Some(-5)]); + } + + #[test] + fn to_parquet_rounds_decimals_of_binary_fractions_half_away_from_zero() { + assert_eq!( + unscaled( + vec![ + Some(0.125), + Some(2.675), + Some(-0.125), + Some(1.005), + Some(9.995), + Some(-99.995) + ], + 9, + 2 + ) + .unwrap(), + vec![Some(13), Some(268), Some(-13), Some(101), Some(1_000), Some(-10_000)] + ); + } + + #[test] + fn to_parquet_refuses_decimals_beyond_their_precision() { + assert!(matches!( + unscaled(vec![Some(1.0), Some(100.0)], 4, 2), + Err(Error::Overflow { row: 1, .. }) + )); + assert!(matches!( + unscaled(vec![Some(99.995)], 4, 2), + Err(Error::Overflow { row: 0, .. }) + )); + assert!(matches!( + unscaled(vec![Some(1e300)], 38, 0), + Err(Error::Overflow { row: 0, .. }) + )); + } + + #[test] + fn to_parquet_refuses_decimals_that_are_not_finite() { + for (value, got) in [(f64::NAN, "NAN"), (f64::INFINITY, "INF"), (f64::NEG_INFINITY, "-INF")] { + assert!(matches!( + unscaled(vec![Some(1.0), Some(value)], 9, 2), + Err(Error::Value { row: 1, expected, got: text, .. }) if expected == "a finite number" && text == got + )); + } + } + + #[test] + fn to_parquet_writes_fixed_size_binaries_of_their_length() { + let array: ArrayRef = Arc::new(BinaryArray::from(vec![Some(&b"abcd"[..]), None])); + let fixed = to_parquet(&array, &DataType::FixedSizeBinary(4)).unwrap(); + + assert_eq!( + fixed.as_fixed_size_binary().iter().collect::>(), + vec![Some(&b"abcd"[..]), None] + ); + assert!(matches!( + to_parquet(&array, &DataType::FixedSizeBinary(16)), + Err(Error::Length { + row: 0, + expected: 16, + .. + }) + )); + } + + #[test] + fn to_parquet_writes_times_in_milliseconds_and_nanoseconds() { + let time: ArrayRef = Arc::new(DurationMicrosecondArray::from(vec![Some(1_999), Some(-1), None])); + + assert_eq!( + to_parquet(&time, &DataType::Time32(TimeUnit::Millisecond)) + .unwrap() + .as_primitive::() + .iter() + .collect::>(), + vec![Some(1), Some(-1), None] + ); + assert_eq!( + to_parquet(&time, &DataType::Time64(TimeUnit::Nanosecond)) + .unwrap() + .as_primitive::() + .iter() + .collect::>(), + vec![Some(1_999_000), Some(-1_000), None] + ); + + let long: ArrayRef = Arc::new(DurationMicrosecondArray::from(vec![0, i64::MAX])); + + assert!(matches!( + to_parquet(&long, &DataType::Time32(TimeUnit::Millisecond)), + Err(Error::Overflow { row: 1, .. }) + )); + assert!(matches!( + to_parquet(&long, &DataType::Time64(TimeUnit::Nanosecond)), + Err(Error::Overflow { row: 1, .. }) + )); + } + + #[test] + fn to_parquet_writes_timestamps_in_milliseconds_and_nanoseconds() { + let timestamp: ArrayRef = + Arc::new(TimestampMicrosecondArray::from(vec![Some(1_999), Some(-1), None]).with_timezone("UTC")); + let millis = to_parquet(×tamp, &DataType::Timestamp(TimeUnit::Millisecond, None)).unwrap(); + let nanos = to_parquet( + ×tamp, + &DataType::Timestamp(TimeUnit::Nanosecond, Some("UTC".into())), + ) + .unwrap(); + + assert_eq!(millis.data_type(), &DataType::Timestamp(TimeUnit::Millisecond, None)); + assert_eq!( + millis + .as_primitive::() + .iter() + .collect::>(), + vec![Some(1), Some(-1), None] + ); + assert_eq!( + nanos.data_type(), + &DataType::Timestamp(TimeUnit::Nanosecond, Some("UTC".into())) + ); + assert_eq!( + nanos + .as_primitive::() + .iter() + .collect::>(), + vec![Some(1_999_000), Some(-1_000), None] + ); + + let far: ArrayRef = Arc::new(TimestampMicrosecondArray::from(vec![0, i64::MAX]).with_timezone("UTC")); + + assert!(matches!( + to_parquet(&far, &DataType::Timestamp(TimeUnit::Nanosecond, None)), + Err(Error::Overflow { row: 1, .. }) + )); + } + + #[test] + fn to_parquet_keeps_equal_types() { + let array: ArrayRef = Arc::new(Int64Array::from(vec![1])); + + assert!(Arc::ptr_eq(&to_parquet(&array, &DataType::Int64).unwrap(), &array)); + } + + #[test] + fn to_parquet_retags_temporal_types() { + let time: ArrayRef = Arc::new(DurationMicrosecondArray::from(vec![Some(5), None])); + let timestamp: ArrayRef = Arc::new(TimestampMicrosecondArray::from(vec![Some(-1)]).with_timezone("UTC")); + + assert_eq!( + to_parquet(&time, &DataType::Time64(TimeUnit::Microsecond)) + .unwrap() + .as_primitive::() + .iter() + .collect::>(), + vec![Some(5), None] + ); + assert_eq!( + to_parquet(×tamp, &DataType::Timestamp(TimeUnit::Microsecond, None)) + .unwrap() + .data_type(), + &DataType::Timestamp(TimeUnit::Microsecond, None) + ); + } + + #[test] + fn to_parquet_writes_valid_utf8_as_strings() { + let binary: ArrayRef = Arc::new(BinaryArray::from(vec![Some(&b"zazolc"[..]), None])); + + assert_eq!( + to_parquet(&binary, &DataType::Utf8) + .unwrap() + .as_string::() + .iter() + .collect::>(), + vec![Some("zazolc"), None] + ); + } + + #[test] + fn to_parquet_refuses_invalid_utf8_at_its_row() { + let binary: ArrayRef = Arc::new(BinaryArray::from(vec![Some(&b"ok"[..]), None, Some(&b"\xff\xfe"[..])])); + + assert!(matches!( + to_parquet(&binary, &DataType::Utf8), + Err(Error::InvalidUtf8 { row: 2, .. }) + )); + } + + #[test] + fn to_parquet_refuses_invalid_utf8_at_its_parent_row() { + let list: ArrayRef = Arc::new(ListArray::new( + Arc::new(Field::new("item", DataType::Binary, true)), + arrow_buffer::OffsetBuffer::new(vec![0, 1, 3].into()), + Arc::new(BinaryArray::from(vec![&b"a"[..], b"b", b"\xff"])), + None, + )); + let target = DataType::List(Arc::new(Field::new("element", DataType::Utf8, true))); + + assert!(matches!( + to_parquet(&list, &target), + Err(Error::InvalidUtf8 { row: 1, .. }) + )); + } + + #[test] + fn to_parquet_renames_nested_children_to_the_target() { + let map = read({ + let mut builder = MapBuilder::new(None, StringBuilder::new(), Int32Builder::new()); + builder.keys().append_value("a"); + builder.values().append_value(1); + builder.append(true).unwrap(); + Arc::new(builder.finish()) + }); + let target = DataType::Map( + Arc::new(Field::new( + "key_value", + DataType::Struct(Fields::from(vec![ + Field::new("key", DataType::Utf8, false), + Field::new("value", DataType::Int64, true), + ])), + false, + )), + false, + ); + + assert_eq!(map.data_type(), &map_of(DataType::Binary, DataType::Int64)); + assert_eq!(to_parquet(&map, &target).unwrap().data_type(), &target); + + let structure = read(Arc::new(StructArray::from(vec![( + Arc::new(Field::new("id", DataType::Int64, true)), + Arc::new(Int64Array::from(vec![1])) as ArrayRef, + )]))); + let target = DataType::Struct(Fields::from(vec![Field::new("id", DataType::Int64, false)])); + + assert_eq!(to_parquet(&structure, &target).unwrap().data_type(), &target); + } + + #[test] + fn to_parquet_refuses_what_has_no_cast() { + let array: ArrayRef = Arc::new(Int64Array::from(vec![1])); + + assert!(matches!( + to_parquet(&array, &DataType::Float32), + Err(Error::Unsupported { parquet, .. }) if parquet == "Float32" + )); + } +} diff --git a/src/extension/arrow-ext/src/parquet/cells.rs b/src/extension/arrow-ext/src/parquet/cells.rs new file mode 100644 index 0000000000..34426e151c --- /dev/null +++ b/src/extension/arrow-ext/src/parquet/cells.rs @@ -0,0 +1,1147 @@ +//! The `Flow\Parquet` lib's per-cell values over the core's canonical arrays: what `PhpParquetEngine` returns for a +//! Parquet value on read, and what it accepts per target type on write. + +use std::any::Any; +use std::sync::Arc; + +use arrow_array::builder::{ + ArrayBuilder, BinaryBuilder, BooleanBuilder, Date32Builder, DurationMicrosecondBuilder, FixedSizeBinaryBuilder, + Float64Builder, Int64Builder, ListBuilder, MapBuilder, MapFieldNames, TimestampMicrosecondBuilder, +}; +use arrow_array::cast::AsArray; +use arrow_array::types::{Date32Type, DurationMicrosecondType, Float64Type, Int64Type, TimestampMicrosecondType}; +use arrow_array::{Array, ArrayRef, StructArray}; +use arrow_buffer::NullBufferBuilder; +use arrow_schema::{DataType, Field, Fields, TimeUnit}; +use ext_php_rs::exception::PhpException; +use ext_php_rs::types::{ZendHashTable, ZendObject, Zval}; +use ext_php_rs::zend::{ClassEntry, Function}; + +use crate::exception::ext_exception; +use crate::parquet::error::Error; +use crate::php::{ + call_handle_transparent, ce_method_ref, find_class, ht_for_each, ht_get, ht_insert, ht_insert_long, read_property, + zval_long, zval_str, +}; +use crate::values::{date_from_days, datetime_days, datetime_from_micros, datetime_micros, uuid_bytes, uuid_text}; + +const UUID: &str = "arrow.uuid"; + +fn is_uuid(field: &Field) -> bool { + field.extension_type_name() == Some(UUID) +} + +fn unsupported(data_type: &DataType) -> Error { + Error::Unsupported { + column: String::new(), + parquet: data_type.to_string(), + } +} + +fn collected(error: impl std::fmt::Debug) -> PhpException { + ext_exception(format!("arrow failed to collect a Parquet value: {error:?}")) +} + +/// How a canonical array's values become PHP values, per `PhpParquetEngine`. +pub enum ReadCell { + Null, + Bool, + Int, + Float, + Bytes, + /// `FixedSizeBinary(16)` under `arrow.uuid`: the lowercase 36-char text. + Uuid, + Date, + Timestamp, + Time, + List(Box), + Map(Box, Box), + Struct(Vec<(Vec, ReadCell)>), +} + +impl ReadCell { + /// `canonical`: the type the core reads the column as; `parquet`: the field it was read from (its extension type + /// tells a uuid from 16 bytes), walked in parallel with the canonical type's children. + pub fn new(canonical: &DataType, parquet: &Field) -> Result { + Ok(match canonical { + DataType::Null => ReadCell::Null, + DataType::Boolean => ReadCell::Bool, + DataType::Int64 => ReadCell::Int, + DataType::Float64 => ReadCell::Float, + DataType::Binary => ReadCell::Bytes, + DataType::FixedSizeBinary(16) if is_uuid(parquet) => ReadCell::Uuid, + DataType::FixedSizeBinary(16) => ReadCell::Bytes, + DataType::Date32 => ReadCell::Date, + DataType::Timestamp(_, _) => ReadCell::Timestamp, + DataType::Duration(_) => ReadCell::Time, + DataType::List(element) => ReadCell::List(Box::new(Self::new(element.data_type(), child(parquet, 0)?)?)), + DataType::Map(entries, _) => { + let DataType::Struct(fields) = entries.data_type() else { + return Err(unsupported(canonical)); + }; + let parquet_entries = child(parquet, 0)?; + let key = Self::new(fields[0].data_type(), child(parquet_entries, 0)?)?; + + if !matches!(key, ReadCell::Int | ReadCell::Bytes | ReadCell::Uuid) { + return Err(Error::MapKey { + column: String::new(), + key: key.php_type(), + }); + } + + ReadCell::Map( + Box::new(key), + Box::new(Self::new(fields[1].data_type(), child(parquet_entries, 1)?)?), + ) + } + DataType::Struct(fields) => ReadCell::Struct( + fields + .iter() + .enumerate() + .map(|(index, field)| { + Ok(( + field.name().as_bytes().to_vec(), + Self::new(field.data_type(), child(parquet, index)?)?, + )) + }) + .collect::>()?, + ), + other => return Err(unsupported(other)), + }) + } + + /// The PHP type this cell's values have: what a MAP key refusal names. + fn php_type(&self) -> &'static str { + match self { + ReadCell::Null => "null", + ReadCell::Bool => "bool", + ReadCell::Int => "int", + ReadCell::Float => "float", + ReadCell::Bytes | ReadCell::Uuid => "string", + ReadCell::Date | ReadCell::Timestamp => "DateTimeImmutable", + ReadCell::Time => "DateInterval", + ReadCell::List(_) | ReadCell::Map(_, _) | ReadCell::Struct(_) => "array", + } + } + + pub fn needs_times(&self) -> bool { + match self { + ReadCell::Time => true, + ReadCell::List(element) => element.needs_times(), + ReadCell::Map(key, value) => key.needs_times() || value.needs_times(), + ReadCell::Struct(children) => children.iter().any(|(_, child)| child.needs_times()), + _ => false, + } + } + + /// Every value of `array` as a PHP list. + pub fn values(&self, array: &ArrayRef, times: Option<&Times>) -> Result { + let mut list = ZendHashTable::with_capacity(u32::try_from(array.len()).unwrap_or(u32::MAX)); + + for row in 0..array.len() { + list.push(self.value(array.as_ref(), row, times)?).map_err(collected)?; + } + + let mut zv = Zval::new(); + zv.set_hashtable(list); + + Ok(zv) + } + + fn value(&self, array: &dyn Array, row: usize, times: Option<&Times>) -> Result { + let mut zv = Zval::new(); + + if array.is_null(row) { + return Ok(zv); + } + + match self { + ReadCell::Null => {} + ReadCell::Bool => zv.set_bool(array.as_boolean().value(row)), + ReadCell::Int => zv = zval_long(array.as_primitive::().value(row)), + ReadCell::Float => zv.set_double(array.as_primitive::().value(row)), + ReadCell::Bytes => { + zv = zval_str(match array.data_type() { + DataType::FixedSizeBinary(_) => array.as_fixed_size_binary().value(row), + _ => array.as_binary::().value(row), + }); + } + ReadCell::Uuid => zv = zval_str(&uuid_text(array.as_fixed_size_binary().value(row))), + ReadCell::Date => zv = date_from_days(i64::from(array.as_primitive::().value(row)))?, + ReadCell::Timestamp => { + zv = datetime_from_micros(array.as_primitive::().value(row), b"+00:00")?; + } + ReadCell::Time => { + zv = times + .ok_or_else(|| ext_exception("arrow Parquet reader has no TIME base"))? + .interval(array.as_primitive::().value(row))?; + } + ReadCell::List(element) => { + let list = array.as_list::(); + let offsets = list.value_offsets(); + let mut values = ZendHashTable::with_capacity((offsets[row + 1] - offsets[row]) as u32); + + for index in offsets[row] as usize..offsets[row + 1] as usize { + values + .push(element.value(list.values().as_ref(), index, times)?) + .map_err(collected)?; + } + + zv.set_hashtable(values); + } + ReadCell::Map(key, value) => { + let map = array.as_map(); + let offsets = map.value_offsets(); + let mut entries = ZendHashTable::with_capacity((offsets[row + 1] - offsets[row]) as u32); + + for index in offsets[row] as usize..offsets[row + 1] as usize { + let entry = value.value(map.values().as_ref(), index, times)?; + + match key.value(map.keys().as_ref(), index, times)? { + k if k.is_long() => ht_insert_long(&mut entries, k.long().unwrap_or_default(), entry), + k => ht_insert(&mut entries, k.zend_str().map_or(&b""[..], |key| key.as_bytes()), entry), + } + } + + zv.set_hashtable(entries); + } + ReadCell::Struct(children) => { + let structure = array.as_struct(); + let mut values = ZendHashTable::with_capacity(children.len() as u32); + + for ((name, child), column) in children.iter().zip(structure.columns()) { + ht_insert(&mut values, name, child.value(column.as_ref(), row, times)?); + } + + zv.set_hashtable(values); + } + } + + Ok(zv) + } +} + +/// Child `index` of a nested Parquet-side field. +fn child(parquet: &Field, index: usize) -> Result<&Field, Error> { + match parquet.data_type() { + DataType::List(element) | DataType::LargeList(element) | DataType::FixedSizeList(element, _) if index == 0 => { + Ok(element) + } + DataType::Map(entries, _) if index == 0 => Ok(entries), + DataType::Struct(fields) if index < fields.len() => Ok(&fields[index]), + other => Err(unsupported(other)), + } +} + +/// `TimeConverter::toDateInterval()`: `$base->diff($base->modify("+{µs} microseconds"))` - `diff()` normalises the +/// duration into d/h and, past a month, m/d exactly as the PHP engine's value. A time of day (< 24 h) normalises into +/// nothing, so it is built directly with `DateInterval::__set_state()`, one call instead of two. +pub struct Times { + base: Zval, + modify: &'static Function, + diff: &'static Function, + set_state: &'static Function, +} + +const DAY: i64 = 86_400_000_000; + +impl Times { + /// `$base = new DateTimeImmutable('1970-01-01 00:00:00.000000', new DateTimeZone('UTC'))`, built once per reader. + pub fn new() -> Result { + let ce = find_class("DateTimeImmutable")?; + + Ok(Self { + base: datetime_from_micros(0, b"UTC")?, + modify: ce_method_ref(ce, "modify")?, + diff: ce_method_ref(ce, "diff")?, + set_state: ce_method_ref(find_class("DateInterval")?, "__set_state")?, + }) + } + + fn interval(&self, micros: i64) -> Result { + if (0..DAY).contains(µs) { + return self.time_of_day(micros); + } + + let base = self + .base + .object() + .ok_or_else(|| ext_exception("arrow expected a TIME base datetime"))?; + let target = call_handle_transparent( + self.modify, + Some(base), + &mut [zval_str(format!("+{micros} microseconds").as_bytes())], + )?; + + call_handle_transparent(self.diff, Some(base), &mut [target]) + } + + /// `diff()`'s interval of a time of day: `days` 0, `f` from the whole microseconds. + fn time_of_day(&self, micros: i64) -> Result { + let mut state = ZendHashTable::with_capacity(10); + let double = |value: f64| { + let mut zv = Zval::new(); + zv.set_double(value); + zv + }; + + for (name, value) in [ + ("y", 0), + ("m", 0), + ("d", 0), + ("h", micros / 3_600_000_000), + ("i", micros / 60_000_000 % 60), + ("s", micros / 1_000_000 % 60), + ] { + ht_insert(&mut state, name.as_bytes(), zval_long(value)); + } + + // __set_state() stores `f` as (int) (f * 1e6): the half microsecond keeps the truncation on the exact value + // (0.519488 * 1e6 = 519487.99…), and `f` reads back as that value / 1e6, as diff()'s does + ht_insert( + &mut state, + b"f", + double(((micros % 1_000_000) as f64 + 0.5) / 1_000_000.0), + ); + ht_insert(&mut state, b"invert", zval_long(0)); + ht_insert(&mut state, b"days", zval_long(0)); + let mut from_string = Zval::new(); + from_string.set_bool(false); + ht_insert(&mut state, b"from_string", from_string); + + let mut array = Zval::new(); + array.set_hashtable(state); + + call_handle_transparent(self.set_state, None, &mut [array]) + } +} + +/// A value `WriteCell::check` refuses. +pub enum Refusal { + /// What the target type accepts and what it got. + Value { expected: &'static str, got: String }, + /// A byte buffer past i32 offsets, or a day past i32. + Overflow, + /// A PHP exception a call into the value threw. + Php(PhpException), +} + +impl From for Refusal { + fn from(exception: PhpException) -> Self { + Refusal::Php(exception) + } +} + +/// `get_debug_type()`. +fn debug_type(value: &Zval) -> String { + if value.is_null() { + "null".to_string() + } else if value.is_bool() { + "bool".to_string() + } else if value.is_long() { + "int".to_string() + } else if value.is_double() { + "float".to_string() + } else if value.is_string() { + "string".to_string() + } else if value.is_array() { + "array".to_string() + } else if let Some(object) = value.object() { + object.get_class_name().unwrap_or_else(|_| "object".to_string()) + } else { + "unknown".to_string() + } +} + +fn refused(expected: &'static str, value: &Zval) -> Refusal { + Refusal::Value { + expected, + got: debug_type(value), + } +} + +/// The classes the write cells test instances of, resolved once per writer. +pub struct Classes { + datetime: &'static ClassEntry, + interval: &'static ClassEntry, +} + +impl Classes { + pub fn new() -> Result { + Ok(Self { + datetime: find_class("DateTimeInterface")?, + interval: find_class("DateInterval")?, + }) + } +} + +/// A value a scalar cell accepted. `Copy`: a flat writer's row buffer clears without dropping anything. +#[derive(Clone, Copy)] +pub enum Scalar<'a> { + Null, + Bool(bool), + Int(i64), + Float(f64), + Bytes(&'a [u8]), + Uuid([u8; 16]), + Date(i32), + Micros(i64), +} + +impl Scalar<'_> { + /// The same value with its bytes, if any, read from `bytes`: a map key checked from a key zval built for the + /// check, whose bytes are the array key's. + fn rebound(self, bytes: &[u8]) -> Scalar<'_> { + match self { + Scalar::Null => Scalar::Null, + Scalar::Bool(value) => Scalar::Bool(value), + Scalar::Int(value) => Scalar::Int(value), + Scalar::Float(value) => Scalar::Float(value), + Scalar::Bytes(_) => Scalar::Bytes(bytes), + Scalar::Uuid(value) => Scalar::Uuid(value), + Scalar::Date(days) => Scalar::Date(days), + Scalar::Micros(micros) => Scalar::Micros(micros), + } + } +} + +fn missing<'a>(nullable: bool) -> Result, Refusal> { + match nullable { + true => Ok(Scalar::Null), + false => Err(Refusal::Value { + expected: "a value", + got: "null".to_string(), + }), + } +} + +/// A value a cell accepted, as its canonical builder appends it. +pub enum Validated<'a> { + Scalar(Scalar<'a>), + List(Vec>), + Map(Vec<(Validated<'a>, Validated<'a>)>), + Struct(Vec>), +} + +/// What a writer buffers per cell of a row until the whole row is checked: `Scalar` when every writer column is +/// scalar, `Validated` otherwise. +pub trait Checked<'a>: Sized { + fn check(cell: &WriteCell, value: &'a Zval, nullable: bool, classes: &Classes) -> Result; + + /// A writer column the row lacks: null, refused where not `nullable`. + fn missing(nullable: bool) -> Result; + + /// Whether `builder` takes the value with its offsets in i32. + fn fits(&self, cell: &WriteCell, builder: &mut ColumnBuilder) -> bool; + + fn append(&self, cell: &WriteCell, builder: &mut ColumnBuilder); +} + +impl<'a> Checked<'a> for Scalar<'a> { + fn check(cell: &WriteCell, value: &'a Zval, nullable: bool, classes: &Classes) -> Result { + cell.check_scalar(value, nullable, classes) + } + + fn missing(nullable: bool) -> Result { + missing(nullable) + } + + fn fits(&self, _: &WriteCell, builder: &mut ColumnBuilder) -> bool { + match (self, builder) { + (Scalar::Bytes(bytes), ColumnBuilder::Bytes(builder)) => { + offsets_fit(builder.values_slice().len(), bytes.len()) + } + _ => true, + } + } + + fn append(&self, cell: &WriteCell, builder: &mut ColumnBuilder) { + cell.append_scalar(builder, self); + } +} + +impl<'a> Checked<'a> for Validated<'a> { + fn check(cell: &WriteCell, value: &'a Zval, nullable: bool, classes: &Classes) -> Result { + cell.check(value, nullable, classes) + } + + fn missing(nullable: bool) -> Result { + missing(nullable).map(Validated::Scalar) + } + + fn fits(&self, cell: &WriteCell, builder: &mut ColumnBuilder) -> bool { + cell.fits(builder, &[self]) + } + + fn append(&self, cell: &WriteCell, builder: &mut ColumnBuilder) { + cell.append(builder, self); + } +} + +/// The builder of a writer cell's canonical type, resolved once per column: appends match it with the cell, nothing +/// is downcast per value. +pub enum ColumnBuilder { + Bool(BooleanBuilder), + Int(Int64Builder), + Float(Float64Builder), + Bytes(BinaryBuilder), + Uuid(FixedSizeBinaryBuilder), + Date(Date32Builder), + Timestamp(TimestampMicrosecondBuilder), + Time(DurationMicrosecondBuilder), + List(Box>), + Map(Box>), + Struct(Box), +} + +pub struct StructColumnBuilder { + fields: Fields, + children: Vec, + nulls: NullBufferBuilder, +} + +impl StructColumnBuilder { + fn finish(&mut self) -> ArrayRef { + let length = self.nulls.len(); + let children = self.children.iter_mut().map(ArrayBuilder::finish).collect(); + + Arc::new( + StructArray::try_new_with_length(self.fields.clone(), children, self.nulls.finish(), length) + .expect("every child appends a slot per struct slot"), + ) + } + + fn finish_cloned(&self) -> ArrayRef { + let children = self.children.iter().map(ArrayBuilder::finish_cloned).collect(); + + Arc::new( + StructArray::try_new_with_length( + self.fields.clone(), + children, + self.nulls.finish_cloned(), + self.nulls.len(), + ) + .expect("every child appends a slot per struct slot"), + ) + } +} + +impl ColumnBuilder { + /// `canonical`: what `WriteCell::new()` returned for the cell. + pub fn new(canonical: &DataType, capacity: usize) -> Self { + match canonical { + DataType::Boolean => ColumnBuilder::Bool(BooleanBuilder::with_capacity(capacity)), + DataType::Int64 => ColumnBuilder::Int(Int64Builder::with_capacity(capacity)), + DataType::Float64 => ColumnBuilder::Float(Float64Builder::with_capacity(capacity)), + DataType::Binary => ColumnBuilder::Bytes(BinaryBuilder::with_capacity(capacity, 1024)), + DataType::FixedSizeBinary(size) => { + ColumnBuilder::Uuid(FixedSizeBinaryBuilder::with_capacity(capacity, *size)) + } + DataType::Date32 => ColumnBuilder::Date(Date32Builder::with_capacity(capacity)), + DataType::Timestamp(_, _) => ColumnBuilder::Timestamp( + TimestampMicrosecondBuilder::with_capacity(capacity).with_data_type(canonical.clone()), + ), + DataType::Duration(_) => ColumnBuilder::Time(DurationMicrosecondBuilder::with_capacity(capacity)), + DataType::List(element) => ColumnBuilder::List(Box::new( + ListBuilder::with_capacity(Self::new(element.data_type(), capacity), capacity) + .with_field(Arc::clone(element)), + )), + DataType::Map(entries, _) => { + let DataType::Struct(fields) = entries.data_type() else { + unreachable!("WriteCell::new() builds map entries as a struct") + }; + + ColumnBuilder::Map(Box::new( + MapBuilder::with_capacity( + Some(MapFieldNames { + entry: entries.name().clone(), + key: fields[0].name().clone(), + value: fields[1].name().clone(), + }), + Self::new(fields[0].data_type(), capacity), + Self::new(fields[1].data_type(), capacity), + capacity, + ) + .with_keys_field(Arc::clone(&fields[0])) + .with_values_field(Arc::clone(&fields[1])), + )) + } + DataType::Struct(fields) => ColumnBuilder::Struct(Box::new(StructColumnBuilder { + fields: fields.clone(), + children: fields + .iter() + .map(|field| Self::new(field.data_type(), capacity)) + .collect(), + nulls: NullBufferBuilder::new(capacity), + })), + other => unreachable!("WriteCell::new() builds no {other} column"), + } + } +} + +impl ArrayBuilder for ColumnBuilder { + fn len(&self) -> usize { + match self { + ColumnBuilder::Bool(builder) => builder.len(), + ColumnBuilder::Int(builder) => builder.len(), + ColumnBuilder::Float(builder) => builder.len(), + ColumnBuilder::Bytes(builder) => builder.len(), + ColumnBuilder::Uuid(builder) => builder.len(), + ColumnBuilder::Date(builder) => builder.len(), + ColumnBuilder::Timestamp(builder) => builder.len(), + ColumnBuilder::Time(builder) => builder.len(), + ColumnBuilder::List(builder) => builder.len(), + ColumnBuilder::Map(builder) => builder.len(), + ColumnBuilder::Struct(builder) => builder.nulls.len(), + } + } + + fn finish(&mut self) -> ArrayRef { + match self { + ColumnBuilder::Bool(builder) => Arc::new(builder.finish()), + ColumnBuilder::Int(builder) => Arc::new(builder.finish()), + ColumnBuilder::Float(builder) => Arc::new(builder.finish()), + ColumnBuilder::Bytes(builder) => Arc::new(builder.finish()), + ColumnBuilder::Uuid(builder) => Arc::new(builder.finish()), + ColumnBuilder::Date(builder) => Arc::new(builder.finish()), + ColumnBuilder::Timestamp(builder) => Arc::new(builder.finish()), + ColumnBuilder::Time(builder) => Arc::new(builder.finish()), + ColumnBuilder::List(builder) => Arc::new(builder.finish()), + ColumnBuilder::Map(builder) => Arc::new(builder.finish()), + ColumnBuilder::Struct(builder) => builder.finish(), + } + } + + fn finish_cloned(&self) -> ArrayRef { + match self { + ColumnBuilder::Bool(builder) => Arc::new(builder.finish_cloned()), + ColumnBuilder::Int(builder) => Arc::new(builder.finish_cloned()), + ColumnBuilder::Float(builder) => Arc::new(builder.finish_cloned()), + ColumnBuilder::Bytes(builder) => Arc::new(builder.finish_cloned()), + ColumnBuilder::Uuid(builder) => Arc::new(builder.finish_cloned()), + ColumnBuilder::Date(builder) => Arc::new(builder.finish_cloned()), + ColumnBuilder::Timestamp(builder) => Arc::new(builder.finish_cloned()), + ColumnBuilder::Time(builder) => Arc::new(builder.finish_cloned()), + ColumnBuilder::List(builder) => Arc::new(builder.finish_cloned()), + ColumnBuilder::Map(builder) => Arc::new(builder.finish_cloned()), + ColumnBuilder::Struct(builder) => builder.finish_cloned(), + } + } + + fn as_any(&self) -> &dyn Any { + self + } + + fn as_any_mut(&mut self) -> &mut dyn Any { + self + } + + fn into_box_any(self: Box) -> Box { + self + } +} + +/// Offsets stay i32: `current` entries plus `added` must not pass `i32::MAX`. +fn offsets_fit(current: usize, added: usize) -> bool { + current + .checked_add(added) + .is_some_and(|total| total <= i32::MAX as usize) +} + +/// What a writer column accepts per `PhpParquetEngine`'s target type, appended to its canonical type's builder; the +/// cast to the Parquet type (`to_parquet`) refuses what is out of the target's range. +pub enum WriteCell { + Bool, + Int, + Float, + /// DECIMAL: a float or an int. + Decimal, + Bytes, + /// UUID: a string, or an object's `toString()` / `__toString()`, whose dash-less text is 32 hex digits. + Uuid, + Date, + Timestamp, + Time, + List(Box, bool), + Map(Box, Box, bool), + Struct(Vec<(Vec, WriteCell, bool)>), +} + +impl WriteCell { + /// The cell for a writer schema field (`Flow\Parquet\Engine\Arrow\SchemaConverter::toExtension()`), and the + /// canonical type its values are built in. + pub fn new(target: &Field) -> Result<(Self, DataType), Error> { + Ok(match target.data_type() { + DataType::Boolean => (WriteCell::Bool, DataType::Boolean), + DataType::Int8 + | DataType::Int16 + | DataType::Int32 + | DataType::Int64 + | DataType::UInt8 + | DataType::UInt16 + | DataType::UInt32 + | DataType::UInt64 => (WriteCell::Int, DataType::Int64), + DataType::Float32 | DataType::Float64 => (WriteCell::Float, DataType::Float64), + DataType::Decimal128(_, _) => (WriteCell::Decimal, DataType::Float64), + DataType::Utf8 | DataType::Binary => (WriteCell::Bytes, DataType::Binary), + DataType::FixedSizeBinary(16) if is_uuid(target) => (WriteCell::Uuid, DataType::FixedSizeBinary(16)), + DataType::FixedSizeBinary(_) => (WriteCell::Bytes, DataType::Binary), + DataType::Date32 => (WriteCell::Date, DataType::Date32), + DataType::Timestamp(_, _) => ( + WriteCell::Timestamp, + DataType::Timestamp(TimeUnit::Microsecond, Some("UTC".into())), + ), + DataType::Time32(_) | DataType::Time64(_) => (WriteCell::Time, DataType::Duration(TimeUnit::Microsecond)), + DataType::List(element) => { + let (cell, canonical) = Self::new(element)?; + + ( + WriteCell::List(Box::new(cell), element.is_nullable()), + DataType::List(Arc::new(Field::new("item", canonical, true))), + ) + } + DataType::Map(entries, _) => { + let DataType::Struct(fields) = entries.data_type() else { + return Err(unsupported(target.data_type())); + }; + let (key, key_type) = Self::new(&fields[0])?; + let (value, value_type) = Self::new(&fields[1])?; + + ( + WriteCell::Map(Box::new(key), Box::new(value), fields[1].is_nullable()), + DataType::Map( + Arc::new(Field::new( + "entries", + DataType::Struct(Fields::from(vec![ + Field::new("key", key_type, false), + Field::new("value", value_type, true), + ])), + false, + )), + false, + ), + ) + } + DataType::Struct(fields) => { + let mut cells = Vec::with_capacity(fields.len()); + let mut children = Vec::with_capacity(fields.len()); + + for field in fields { + let (cell, canonical) = Self::new(field)?; + cells.push((field.name().as_bytes().to_vec(), cell, field.is_nullable())); + children.push(Field::new(field.name(), canonical, true)); + } + + (WriteCell::Struct(cells), DataType::Struct(Fields::from(children))) + } + other => return Err(unsupported(other)), + }) + } + + /// `value` as this scalar cell accepts it, a null only where `nullable`; a nested cell refuses it as not an + /// array. + pub fn check_scalar<'a>(&self, value: &'a Zval, nullable: bool, classes: &Classes) -> Result, Refusal> { + let value = value.dereference(); + + if value.is_null() { + return if nullable { + Ok(Scalar::Null) + } else { + Err(refused("a value", value)) + }; + } + + Ok(match self { + WriteCell::Bool => Scalar::Bool(value.bool().ok_or_else(|| refused("bool", value))?), + WriteCell::Int => Scalar::Int(value.long().ok_or_else(|| refused("int", value))?), + WriteCell::Float => Scalar::Float(value.double().ok_or_else(|| refused("float", value))?), + WriteCell::Decimal => Scalar::Float(match (value.double(), value.long()) { + (Some(float), _) => float, + (_, Some(int)) => int as f64, + _ => return Err(refused("float or int", value)), + }), + WriteCell::Bytes => Scalar::Bytes(value.zend_str().ok_or_else(|| refused("string", value))?.as_bytes()), + WriteCell::Uuid => { + let text = match value.object() { + Some(object) => uuid_text_of(object)?.ok_or_else(|| refused("uuid string or object", value))?, + None => value.shallow_clone(), + }; + + Scalar::Uuid( + text.zend_str() + .and_then(|text| uuid_bytes(text.as_bytes())) + .ok_or_else(|| refused("uuid string or object", value))?, + ) + } + WriteCell::Date => { + let datetime = datetime(value, classes).ok_or_else(|| refused("DateTimeInterface", value))?; + let days = match datetime_days(datetime) { + Some(days) => days, + None => fallback_days(datetime)?, + }; + + Scalar::Date(i32::try_from(days).map_err(|_| Refusal::Overflow)?) + } + WriteCell::Timestamp => { + let datetime = datetime(value, classes).ok_or_else(|| refused("DateTimeInterface", value))?; + + Scalar::Micros(match datetime_micros(datetime) { + Some(micros) => micros, + None => fallback_micros(datetime)?, + }) + } + WriteCell::Time => { + let interval = value + .object() + .filter(|object| object.instance_of(classes.interval)) + .ok_or_else(|| refused("DateInterval", value))?; + + Scalar::Micros(interval_micros(interval, value)?) + } + WriteCell::List(_, _) | WriteCell::Map(_, _, _) | WriteCell::Struct(_) => { + return Err(refused("array", value)) + } + }) + } + + /// `value` as this cell accepts it; a null only where `nullable`. + pub fn check<'a>(&self, value: &'a Zval, nullable: bool, classes: &Classes) -> Result, Refusal> { + let value = value.dereference(); + + if value.is_null() { + return if nullable { + Ok(Validated::Scalar(Scalar::Null)) + } else { + Err(refused("a value", value)) + }; + } + + Ok(match self { + WriteCell::List(element, element_nullable) => { + let items = value.array().ok_or_else(|| refused("array", value))?; + let mut values = Vec::with_capacity(items.len()); + let mut failure = None; + + ht_for_each(items, |_, _, item| { + if failure.is_none() { + match element.check(item, *element_nullable, classes) { + Ok(validated) => values.push(validated), + Err(refusal) => failure = Some(refusal), + } + } + + Ok(()) + })?; + + if let Some(refusal) = failure { + return Err(refusal); + } + + Validated::List(values) + } + WriteCell::Map(key, map_value, value_nullable) => { + let items = value.array().ok_or_else(|| refused("array", value))?; + let mut entries = Vec::with_capacity(items.len()); + let mut failure = None; + + ht_for_each(items, |name, index, item| { + if failure.is_some() { + return Ok(()); + } + + let entry_key = match name { + Some(name) => zval_str(name.as_bytes()), + None => zval_long(index as i64), + }; + let checked = key + .check_scalar(&entry_key, false, classes) + .map(|key| Validated::Scalar(key.rebound(name.map_or(&[][..], |name| name.as_bytes())))) + .and_then(|key| Ok((key, map_value.check(item, *value_nullable, classes)?))); + + match checked { + Ok(entry) => entries.push(entry), + Err(refusal) => failure = Some(refusal), + } + + Ok(()) + })?; + + if let Some(refusal) = failure { + return Err(refusal); + } + + Validated::Map(entries) + } + WriteCell::Struct(children) => { + let values = value.array().ok_or_else(|| refused("array", value))?; + let mut checked = Vec::with_capacity(children.len()); + + for (name, cell, child_nullable) in children { + checked.push(match ht_get(values, name) { + Some(child) => cell.check(child, *child_nullable, classes)?, + None => Validated::Scalar(missing(*child_nullable)?), + }); + } + + Validated::Struct(checked) + } + scalar => Validated::Scalar(scalar.check_scalar(value, nullable, classes)?), + }) + } + + /// Whether `builder` takes `values` (one row's, every value bound for it) with its offsets in i32; only cells + /// holding offsets can pass it. + pub fn fits(&self, builder: &mut ColumnBuilder, values: &[&Validated]) -> bool { + match (self, builder) { + (WriteCell::Bytes, ColumnBuilder::Bytes(bytes)) => offsets_fit( + bytes.values_slice().len(), + values + .iter() + .map(|value| match value { + Validated::Scalar(Scalar::Bytes(bytes)) => bytes.len(), + _ => 0, + }) + .sum(), + ), + (WriteCell::List(element, _), ColumnBuilder::List(list)) => { + let items = values + .iter() + .flat_map(|value| match value { + Validated::List(items) => items.as_slice(), + _ => &[], + }) + .collect::>(); + + offsets_fit(list.values().len(), items.len()) && element.fits(list.values(), &items) + } + (WriteCell::Map(key, value, _), ColumnBuilder::Map(map)) => { + let entries = values + .iter() + .flat_map(|value| match value { + Validated::Map(entries) => entries.as_slice(), + _ => &[], + }) + .collect::>(); + let keys = entries.iter().map(|(key, _)| key).collect::>(); + let items = entries.iter().map(|(_, value)| value).collect::>(); + let (key_builder, value_builder) = map.entries(); + + offsets_fit(key_builder.len(), entries.len()) + && key.fits(key_builder, &keys) + && value.fits(value_builder, &items) + } + (WriteCell::Struct(children), ColumnBuilder::Struct(structure)) => children + .iter() + .zip(&mut structure.children) + .enumerate() + .all(|(index, ((_, cell, _), child))| { + let values = values + .iter() + .filter_map(|value| match value { + Validated::Struct(children) => Some(&children[index]), + _ => None, + }) + .collect::>(); + + cell.fits(child, &values) + }), + _ => true, + } + } + + /// `value`, checked by this cell and fit by [`WriteCell::fits`], appended to `builder`. + pub fn append(&self, builder: &mut ColumnBuilder, value: &Validated) { + match (self, builder, value) { + (_, builder, Validated::Scalar(scalar)) => self.append_scalar(builder, scalar), + (WriteCell::List(element, _), ColumnBuilder::List(list), Validated::List(items)) => { + for item in items { + element.append(list.values(), item); + } + + list.append(true); + } + (WriteCell::Map(key, map_value, _), ColumnBuilder::Map(map), Validated::Map(entries)) => { + for (entry_key, entry_value) in entries { + let (keys, values) = map.entries(); + key.append(keys, entry_key); + map_value.append(values, entry_value); + } + + map.append(true).expect("every entry appends a key and a value"); + } + (WriteCell::Struct(children), ColumnBuilder::Struct(structure), Validated::Struct(values)) => { + for (((_, cell, _), child), value) in children.iter().zip(&mut structure.children).zip(values) { + cell.append(child, value); + } + + structure.nulls.append_non_null(); + } + _ => unreachable!("a cell appends only what it checked, to the builder of its canonical type"), + } + } + + /// `scalar`, checked by this cell, appended to `builder`. + pub fn append_scalar(&self, builder: &mut ColumnBuilder, scalar: &Scalar) { + match (self, builder, scalar) { + (_, builder, Scalar::Null) => self.append_null(builder), + (WriteCell::Bool, ColumnBuilder::Bool(builder), Scalar::Bool(value)) => builder.append_value(*value), + (WriteCell::Int, ColumnBuilder::Int(builder), Scalar::Int(value)) => builder.append_value(*value), + (WriteCell::Float | WriteCell::Decimal, ColumnBuilder::Float(builder), Scalar::Float(value)) => { + builder.append_value(*value); + } + (WriteCell::Bytes, ColumnBuilder::Bytes(builder), Scalar::Bytes(bytes)) => builder.append_value(bytes), + (WriteCell::Uuid, ColumnBuilder::Uuid(builder), Scalar::Uuid(bytes)) => { + builder.append_value(bytes).expect("a uuid is 16 bytes"); + } + (WriteCell::Date, ColumnBuilder::Date(builder), Scalar::Date(days)) => builder.append_value(*days), + (WriteCell::Timestamp, ColumnBuilder::Timestamp(builder), Scalar::Micros(micros)) => { + builder.append_value(*micros); + } + (WriteCell::Time, ColumnBuilder::Time(builder), Scalar::Micros(micros)) => builder.append_value(*micros), + _ => unreachable!("a cell appends only what it checked, to the builder of its canonical type"), + } + } + + /// Whether this cell holds one value per row: no list, map or struct. + pub fn is_scalar(&self) -> bool { + !matches!( + self, + WriteCell::List(_, _) | WriteCell::Map(_, _, _) | WriteCell::Struct(_) + ) + } + + /// A null slot; a struct's children get one too. + fn append_null(&self, builder: &mut ColumnBuilder) { + match (self, builder) { + (WriteCell::Struct(children), ColumnBuilder::Struct(structure)) => { + for ((_, cell, _), child) in children.iter().zip(&mut structure.children) { + cell.append_null(child); + } + + structure.nulls.append_null(); + } + (_, ColumnBuilder::Bool(builder)) => builder.append_null(), + (_, ColumnBuilder::Int(builder)) => builder.append_null(), + (_, ColumnBuilder::Float(builder)) => builder.append_null(), + (_, ColumnBuilder::Bytes(builder)) => builder.append_null(), + (_, ColumnBuilder::Uuid(builder)) => builder.append_null(), + (_, ColumnBuilder::Date(builder)) => builder.append_null(), + (_, ColumnBuilder::Timestamp(builder)) => builder.append_null(), + (_, ColumnBuilder::Time(builder)) => builder.append_null(), + (_, ColumnBuilder::List(builder)) => builder.append_null(), + (_, ColumnBuilder::Map(builder)) => builder.append(false).expect("a null map appends no entries"), + (_, ColumnBuilder::Struct(_)) => { + unreachable!("only a struct cell builds a struct column") + } + } + } +} + +/// `UuidConverter::toParquetType()`'s text of an object: `toString()`, else `__toString()`; `None` for neither. +fn uuid_text_of(object: &ZendObject) -> Result, PhpException> { + let ce = unsafe { object.ce.as_ref() }.ok_or_else(|| ext_exception("arrow expected a class entry"))?; + + for method in ["toString", "__toString"] { + if let Ok(function) = ce_method_ref(ce, method) { + return call_handle_transparent(function, Some(object), &mut []).map(Some); + } + } + + Ok(None) +} + +fn datetime<'a>(value: &'a Zval, classes: &Classes) -> Option<&'a ZendObject> { + value.object().filter(|object| object.instance_of(classes.datetime)) +} + +fn method_long(object: &ZendObject, method: &str, args: &mut [Zval]) -> Result { + let ce = unsafe { object.ce.as_ref() }.ok_or_else(|| ext_exception("arrow expected a class entry"))?; + let result = call_handle_transparent(ce_method_ref(ce, method)?, Some(object), args)?; + + result + .long() + .or_else(|| { + result + .zend_str() + .and_then(|text| std::str::from_utf8(text.as_bytes()).ok()?.parse().ok()) + }) + .ok_or_else(|| ext_exception(format!("arrow expected {method}() to return an int"))) +} + +/// `getTimestamp() * 1_000_000 + format('u')`, for a datetime timelib has not brought up to date. +fn fallback_micros(datetime: &ZendObject) -> Result { + let seconds = method_long(datetime, "getTimestamp", &mut [])?; + let micros = method_long(datetime, "format", &mut [zval_str(b"u")])?; + + seconds + .checked_mul(1_000_000) + .and_then(|us| us.checked_add(micros)) + .ok_or(Refusal::Overflow) +} + +/// `(getTimestamp() + getOffset())` floored to days, for a datetime timelib has not brought up to date. +fn fallback_days(datetime: &ZendObject) -> Result { + let seconds = method_long(datetime, "getTimestamp", &mut [])?; + let offset = method_long(datetime, "getOffset", &mut [])?; + + Ok((seconds + offset).div_euclid(86_400)) +} + +fn interval_part(interval: &ZendObject, name: &str) -> Result { + Ok(read_property(interval, name)?.long().unwrap_or_default()) +} + +/// `TimeConverter::toParquetType()`: `((d·24+h)·3600+i·60+s)·1e6 + (int)(f·1e6)`, `invert` ignored; y or m ≠ 0 is +/// refused. +fn interval_micros(interval: &ZendObject, value: &Zval) -> Result { + if interval_part(interval, "y")? != 0 || interval_part(interval, "m")? != 0 { + return Err(refused("DateInterval without years or months", value)); + } + + let seconds = ((interval_part(interval, "d")? * 24 + interval_part(interval, "h")?) * 3_600 + + interval_part(interval, "i")? * 60 + + interval_part(interval, "s")?) + .checked_mul(1_000_000) + .ok_or(Refusal::Overflow)?; + let fraction = (read_property(interval, "f")?.double().unwrap_or_default() * 1_000_000.0) as i64; + + seconds.checked_add(fraction).ok_or(Refusal::Overflow) +} + +/// Rows appended to one writer column, finished into canonical arrays in batches. +pub struct WriteColumn { + pub name: String, + pub cell: WriteCell, + pub nullable: bool, + pub builder: ColumnBuilder, +} + +impl WriteColumn { + pub fn new(target: &Field, capacity: usize) -> Result { + let (cell, canonical) = WriteCell::new(target).map_err(|e| e.in_column(target.name()))?; + + Ok(Self { + name: target.name().clone(), + builder: ColumnBuilder::new(&canonical, capacity), + cell, + nullable: target.is_nullable(), + }) + } + + /// The rows appended since the last take, as a canonical array; the builder starts over. + pub fn take(&mut self) -> ArrayRef { + self.builder.finish() + } +} + +#[cfg(test)] +mod tests { + use super::offsets_fit; + + #[test] + fn offsets_fit_up_to_i32_max() { + assert!(offsets_fit(i32::MAX as usize - 10, 10)); + assert!(!offsets_fit(i32::MAX as usize - 10, 11)); + assert!(!offsets_fit(usize::MAX, 1)); + } +} diff --git a/src/extension/arrow-ext/src/parquet/engine.rs b/src/extension/arrow-ext/src/parquet/engine.rs new file mode 100644 index 0000000000..4bb49a1518 --- /dev/null +++ b/src/extension/arrow-ext/src/parquet/engine.rs @@ -0,0 +1,98 @@ +//! `Flow\Parquet\Engine\RustParquetEngine`: `flow-php/parquet`'s arrow engine, its files read and written by +//! `RustParquetFileReader` / `RustParquetFileWriter`. + +use ext_php_rs::flags::ClassFlags; +use ext_php_rs::prelude::*; +use ext_php_rs::types::Zval; + +use crate::exception::ext_exception; +use crate::interfaces::parquet_engine_ce; +use crate::parquet::lib_calls::{call_static, OPTIONS_TO_ENGINE, OPTIONS_TO_EXTENSION, SCHEMA_TO_EXTENSION}; +use crate::parquet::library::{RustParquetFileReader, RustParquetFileWriter}; +use crate::php::{ht_get, IterableArg}; + +#[php_class] +#[php( + name = "Flow\\Parquet\\Engine\\RustParquetEngine", + flags = ClassFlags::Final, + implements(ce = parquet_engine_ce, stub = "Flow\\Parquet\\ParquetEngine") +)] +pub struct RustParquetEngine { + int96_as_datetime: bool, + write_batch_size: i64, +} + +#[php_impl] +impl RustParquetEngine { + pub fn __construct(options: Option<&Zval>) -> PhpResult { + let mut options = [options.map_or_else(Zval::new, Zval::shallow_clone)]; + let engine = call_static(OPTIONS_TO_ENGINE, &mut options)?; + let engine = engine + .array() + .ok_or_else(|| ext_exception("arrow expected OptionsConverter::toEngine() to return an array"))?; + + Ok(Self { + int96_as_datetime: ht_get(engine, b"INT_96_AS_DATETIME") + .and_then(Zval::bool) + .ok_or_else(|| { + ext_exception("arrow expected OptionsConverter::toEngine() to hold INT_96_AS_DATETIME") + })?, + write_batch_size: ht_get(engine, b"ARROW_WRITE_BATCH_SIZE") + .and_then(Zval::long) + .ok_or_else(|| { + ext_exception("arrow expected OptionsConverter::toEngine() to hold ARROW_WRITE_BATCH_SIZE") + })?, + }) + } + + #[php(name = "openForRead")] + pub fn open_for_read(&self, stream: &Zval) -> PhpResult { + RustParquetFileReader::__construct(stream, self.int96_as_datetime) + } + + #[php(name = "openForWrite")] + pub fn open_for_write( + &self, + stream: &Zval, + schema: &Zval, + compression: &Zval, + options: &Zval, + ) -> PhpResult { + let schema = call_static(SCHEMA_TO_EXTENSION, &mut [schema.shallow_clone()])?; + let options = call_static(OPTIONS_TO_EXTENSION, &mut [options.shallow_clone()])?; + + RustParquetFileWriter::__construct( + stream, + schema + .array() + .ok_or_else(|| ext_exception("arrow expected SchemaConverter::toExtension() to return an array"))?, + compression, + options + .array() + .ok_or_else(|| ext_exception("arrow expected OptionsConverter::toExtension() to return an array"))?, + self.write_batch_size, + ) + } + + /// The writer is closed even when a row is refused; the refusal is what throws. + #[php(name = "writeRows")] + pub fn write_rows( + &self, + stream: &Zval, + schema: &Zval, + compression: &Zval, + options: &Zval, + rows: IterableArg, + ) -> PhpResult<()> { + let mut writer = self.open_for_write(stream, schema, compression, options)?; + + match writer.write_batch(rows) { + Ok(()) => writer.close(), + Err(e) => { + let _ = writer.close(); + + Err(e) + } + } + } +} diff --git a/src/extension/arrow-ext/src/parquet/error.rs b/src/extension/arrow-ext/src/parquet/error.rs new file mode 100644 index 0000000000..4817f37754 --- /dev/null +++ b/src/extension/arrow-ext/src/parquet/error.rs @@ -0,0 +1,132 @@ +//! What the Parquet core refuses, as data: `render.rs` turns each variant into the PHP exception and its message. + +use arrow_schema::ArrowError; +use parquet::errors::ParquetError; + +#[derive(Debug)] +pub enum Error { + Parquet(ParquetError), + Arrow(ArrowError), + MissingColumn(String), + /// `parquet`: the arrow type (or codec) the core has no cast for. + Unsupported { + column: String, + parquet: String, + }, + Overflow { + column: String, + row: usize, + }, + InvalidUtf8 { + column: String, + row: usize, + }, + /// A fixed-length target the value's byte length does not match. + Length { + column: String, + row: usize, + expected: usize, + }, + /// MAP keys of a type a PHP array does not key by (int or string), named by the PHP type they would read as. + MapKey { + column: String, + key: &'static str, + }, + /// A value the target type does not accept: what it expects and what it got. + Value { + column: String, + row: usize, + expected: &'static str, + got: String, + }, + /// Bytes that are not a Parquet file: no footer where it must be. + NotParquet(String), + Options(String), + Stream(String), +} + +impl Error { + /// The same refusal named after the root column it happened in. + pub fn in_column(self, name: &str) -> Self { + match self { + Error::Unsupported { parquet, .. } => Error::Unsupported { + column: name.to_string(), + parquet, + }, + Error::Overflow { row, .. } => Error::Overflow { + column: name.to_string(), + row, + }, + Error::InvalidUtf8 { row, .. } => Error::InvalidUtf8 { + column: name.to_string(), + row, + }, + Error::Length { row, expected, .. } => Error::Length { + column: name.to_string(), + row, + expected, + }, + Error::MapKey { key, .. } => Error::MapKey { + column: name.to_string(), + key, + }, + Error::Value { row, expected, got, .. } => Error::Value { + column: name.to_string(), + row, + expected, + got, + }, + error => error, + } + } + + /// The same refusal at the parent row `parent` maps a child row to. + pub fn at_row(self, parent: impl Fn(usize) -> usize) -> Self { + match self { + Error::Overflow { column, row } => Error::Overflow { + column, + row: parent(row), + }, + Error::InvalidUtf8 { column, row } => Error::InvalidUtf8 { + column, + row: parent(row), + }, + Error::Length { column, row, expected } => Error::Length { + column, + row: parent(row), + expected, + }, + Error::Value { + column, + row, + expected, + got, + } => Error::Value { + column, + row: parent(row), + expected, + got, + }, + error => error, + } + } +} + +impl From for Error { + fn from(error: ParquetError) -> Self { + Error::Parquet(error) + } +} + +impl From for Error { + fn from(error: ArrowError) -> Self { + Error::Arrow(error) + } +} + +/// The parent row of child row `child` under list/map `offsets`. +pub fn parent_row>(offsets: &[O], child: usize) -> usize { + offsets + .partition_point(|offset| (*offset).into() <= child as i64) + .saturating_sub(1) +} diff --git a/src/extension/arrow-ext/src/parquet/exception.rs b/src/extension/arrow-ext/src/parquet/exception.rs deleted file mode 100644 index 2bf15073c3..0000000000 --- a/src/extension/arrow-ext/src/parquet/exception.rs +++ /dev/null @@ -1,25 +0,0 @@ -use ext_php_rs::builders::ClassBuilder; -use ext_php_rs::error::Result; -use ext_php_rs::exception::PhpException; -use ext_php_rs::flags::ClassFlags; -use ext_php_rs::zend::{ce, ClassEntry}; - -fn noop(_ce: &'static mut ClassEntry) {} - -pub fn register() -> Result<()> { - ClassBuilder::new("Flow\\Arrow\\Parquet\\Exception") - .extends((ce::exception, "Exception")) - .flags(ClassFlags::Final) - .registration(noop) - .register()?; - - Ok(()) -} - -fn parquet_exception_ce() -> &'static ClassEntry { - ClassEntry::try_find("Flow\\Arrow\\Parquet\\Exception").unwrap_or_else(|| ce::exception()) -} - -pub fn parquet_exception(message: impl Into) -> PhpException { - PhpException::new(message.into(), 0, parquet_exception_ce()) -} diff --git a/src/extension/arrow-ext/src/parquet/fetch.rs b/src/extension/arrow-ext/src/parquet/fetch.rs new file mode 100644 index 0000000000..ce141cc755 --- /dev/null +++ b/src/extension/arrow-ext/src/parquet/fetch.rs @@ -0,0 +1,328 @@ +//! A planned read's column chunks, fetched a row group at a time: the first page request inside a planned row group +//! reads every projected column chunk of that group - adjacent chunks in one read - and every later page header and +//! page body of the group is a slice of those buffers. arrow-rs's sync page reader asks for each page header +//! separately (`get_read`), so answering from the stream per request would read the file many times over. +//! +//! arrow-rs advances each column's pages on its own, so one batch may take column `a` into the next row group while +//! column `b` still reads the previous one: a group's buffers are dropped once every projected column has moved past +//! it. Memory: the projected compressed chunks of the row groups one batch spans - the bound arrow-rs's async reader +//! and DuckDB accept, row groups being the writer's unit. + +use std::collections::btree_map::Entry; +use std::collections::BTreeMap; +use std::ops::Range; +use std::sync::{Arc, Mutex}; + +use bytes::buf::Reader; +use bytes::{Buf, Bytes}; +use parquet::errors::ParquetError; +use parquet::file::metadata::ParquetMetaData; +use parquet::file::reader::{ChunkReader, Length}; + +use crate::parquet::source::{exact, ReadAt}; + +/// A column chunk of the file and, when the read plans it, where: its planned group, its projected column, and the +/// coalesced read of that group holding it. +struct Chunk { + range: Range, + planned: Option<(usize, usize, usize)>, +} + +#[derive(Default)] +struct Fetched { + /// Planned group → its reads' bytes. + buffers: BTreeMap>, + /// Per projected column, the planned group it reads now. + progress: Vec, +} + +pub struct ChunkFetch { + source: Arc, + size: u64, + /// Per planned group, its projected chunks with every run of adjacent ones merged, in file order. + reads: Vec>>, + /// Every column chunk of the file, by start: page requests find theirs by binary search. + chunks: Vec, + fetched: Mutex, +} + +impl ChunkFetch { + /// `row_groups`: the planned groups; `leaves`: the projected leaf columns. + pub fn new(source: Arc, size: u64, meta: &ParquetMetaData, row_groups: &[usize], leaves: &[usize]) -> Self { + let range = |(start, length): (u64, u64)| start..start + length; + let planned = row_groups + .iter() + .map(|group| { + let columns = meta.row_group(*group).columns(); + + leaves.iter().map(|leaf| range(columns[*leaf].byte_range())).collect() + }) + .collect(); + let mut is_planned = vec![false; meta.num_row_groups()]; + let mut is_projected = vec![false; meta.file_metadata().schema_descr().num_columns()]; + row_groups.iter().for_each(|group| is_planned[*group] = true); + leaves.iter().for_each(|leaf| is_projected[*leaf] = true); + let (is_planned, is_projected) = (&is_planned, &is_projected); + let unplanned = meta + .row_groups() + .iter() + .enumerate() + .flat_map(|(group, columns)| { + columns + .columns() + .iter() + .enumerate() + .filter(move |(leaf, _)| !is_planned[group] || !is_projected[*leaf]) + .map(|(_, chunk)| range(chunk.byte_range())) + }) + .collect(); + + Self::from_ranges(source, size, planned, unplanned) + } + + /// `planned`: per planned group, its projected chunks, by projected column; `unplanned`: every other chunk. + fn from_ranges(source: Arc, size: u64, planned: Vec>>, unplanned: Vec>) -> Self { + let columns = planned.first().map_or(0, Vec::len); + let reads = planned + .iter() + .map(|chunks| coalesced(chunks.clone())) + .collect::>(); + let mut chunks = unplanned + .into_iter() + .map(|range| Chunk { range, planned: None }) + .chain(planned.into_iter().enumerate().flat_map(|(group, chunks)| { + let reads = &reads[group]; + + chunks + .into_iter() + .enumerate() + .map(move |(column, range)| { + let read = reads + .iter() + .position(|read| read.start <= range.start && range.end <= read.end); + + Chunk { + planned: read.map(|read| (group, column, read)), + range, + } + }) + .collect::>() + })) + .collect::>(); + chunks.sort_by_key(|chunk| chunk.range.start); + + Self { + source, + size, + reads, + chunks, + fetched: Mutex::new(Fetched { + buffers: BTreeMap::new(), + progress: vec![0; columns], + }), + } + } + + /// The column chunk holding `offset`. + fn chunk_at(&self, offset: u64) -> Option<&Chunk> { + let after = self.chunks.partition_point(|chunk| chunk.range.start <= offset); + + after + .checked_sub(1) + .map(|index| &self.chunks[index]) + .filter(|chunk| offset < chunk.range.end) + } + + /// The fetched bytes of [start, end) when a planned column chunk holds it, fetching its row group first. + fn planned(&self, start: u64, end: u64) -> Result, ParquetError> { + let Some((group, column, read)) = self.chunk_at(start).and_then(|chunk| chunk.planned) else { + return Ok(None); + }; + let range = &self.reads[group][read]; + + if end > range.end { + return Ok(None); + } + + let mut fetched = self + .fetched + .lock() + .map_err(|_| ParquetError::General("chunk fetch poisoned".into()))?; + + if let Entry::Vacant(entry) = fetched.buffers.entry(group) { + entry.insert( + self.reads[group] + .iter() + .map(|range| exact(self.source.as_ref(), range.start, (range.end - range.start) as usize)) + .collect::, _>>()?, + ); + } + + let bytes = fetched.buffers[&group][read].slice((start - range.start) as usize..(end - range.start) as usize); + fetched.progress[column] = group; + let slowest = fetched.progress.iter().copied().min().unwrap_or(group); + fetched.buffers.retain(|fetched_group, _| *fetched_group >= slowest); + + Ok(Some(bytes)) + } +} + +/// Sorted ranges with every run of adjacent ones merged. +fn coalesced(mut ranges: Vec>) -> Vec> { + ranges.sort_by_key(|range| range.start); + let mut merged: Vec> = Vec::with_capacity(ranges.len()); + + for range in ranges { + match merged.last_mut() { + Some(last) if last.end == range.start => last.end = range.end, + _ => merged.push(range), + } + } + + merged +} + +impl Length for ChunkFetch { + fn len(&self) -> u64 { + self.size + } +} + +impl ChunkReader for ChunkFetch { + type T = Reader; + + /// A page header: the rest of its planned read; outside the plan, one exact read to the end of the column chunk + /// holding `start` (of the file when none does). + fn get_read(&self, start: u64) -> Result { + let end = self.chunk_at(start).map_or(self.size, |chunk| match chunk.planned { + Some((group, _, read)) => self.reads[group][read].end, + None => chunk.range.end, + }); + + Ok(match self.planned(start, end)? { + Some(bytes) => bytes, + None => exact(self.source.as_ref(), start, (end - start) as usize)?, + } + .reader()) + } + + fn get_bytes(&self, start: u64, length: usize) -> Result { + match self.planned(start, start + length as u64)? { + Some(bytes) => Ok(bytes), + None => exact(self.source.as_ref(), start, length), + } + } +} + +#[cfg(test)] +mod tests { + use std::io::Read; + use std::ops::Range; + use std::sync::Arc; + + use parquet::file::reader::ChunkReader; + + use super::{coalesced, ChunkFetch}; + use crate::parquet::source::fake::Counting; + + #[test] + fn adjacent_ranges_are_coalesced() { + assert_eq!(coalesced(vec![10..20, 0..10, 30..40, 20..25]), vec![0..25, 30..40]); + assert!(coalesced(vec![]).is_empty()); + } + + fn fetch( + source: &Arc, + planned: Vec>>, + unplanned: Vec>, + ) -> ChunkFetch { + ChunkFetch::from_ranges(Arc::clone(source), source.data.len() as u64, planned, unplanned) + } + + #[test] + fn a_read_outside_the_plan_is_one_exact_read_to_the_end_of_its_chunk() { + let source = Arc::new(Counting::new((0..100).collect())); + let fetch = fetch(&source, vec![vec![10..20]], vec![30..35, 40..60]); + let mut rest = Vec::new(); + + fetch.get_read(45).unwrap().read_to_end(&mut rest).unwrap(); + + assert_eq!(rest, (45..60).collect::>()); + assert_eq!(source.counted(), (1, 15)); + + rest.clear(); + fetch.get_read(90).unwrap().read_to_end(&mut rest).unwrap(); + + assert_eq!(rest, (90..100).collect::>()); + assert_eq!(fetch.get_bytes(70, 2).unwrap().to_vec(), vec![70, 71]); + assert_eq!(source.counted(), (3, 27)); + } + + #[test] + fn a_planned_group_is_fetched_once_in_coalesced_reads() { + let source = Arc::new(Counting::new((0..100).collect())); + let fetch = fetch(&source, vec![vec![0..10, 10..15, 20..30]], vec![]); + let mut header = Vec::new(); + + assert_eq!(fetch.get_bytes(22, 3).unwrap().to_vec(), vec![22, 23, 24]); + fetch.get_read(12).unwrap().read_to_end(&mut header).unwrap(); + assert_eq!(fetch.get_bytes(1, 2).unwrap().to_vec(), vec![1, 2]); + + assert_eq!(header, (12..15).collect::>()); + assert_eq!(source.counted(), (2, 25)); + } + + #[test] + fn a_group_is_dropped_once_every_column_moved_past_it() { + let source = Arc::new(Counting::new((0..100).collect())); + let fetch = fetch(&source, vec![vec![0..10, 20..30], vec![50..60, 70..80]], vec![]); + let groups = |fetch: &ChunkFetch| { + fetch + .fetched + .lock() + .unwrap() + .buffers + .keys() + .copied() + .collect::>() + }; + + fetch.get_bytes(0, 1).unwrap(); + fetch.get_bytes(50, 1).unwrap(); + assert_eq!(groups(&fetch), vec![0, 1]); + + fetch.get_bytes(21, 1).unwrap(); + assert_eq!(groups(&fetch), vec![0, 1]); + + fetch.get_bytes(71, 1).unwrap(); + assert_eq!(groups(&fetch), vec![1]); + assert_eq!(source.counted(), (4, 40)); + } + + #[test] + fn page_requests_find_their_chunk_among_many() { + let source = Arc::new(Counting::new((0..=255).cycle().take(40_000).collect())); + let planned = (0..100).map(|group| vec![group * 400..group * 400 + 100, group * 400 + 200..group * 400 + 300]); + let fetch = fetch( + &source, + planned.collect(), + (0..100).map(|group| group * 400 + 100..group * 400 + 200).collect(), + ); + + assert_eq!( + fetch.get_bytes(99 * 400 + 250, 2).unwrap().to_vec(), + source.data[39_850..39_852].to_vec() + ); + assert_eq!(source.counted(), (2, 200)); + + let mut header = Vec::new(); + fetch + .get_read(50 * 400 + 150) + .unwrap() + .read_to_end(&mut header) + .unwrap(); + + assert_eq!(header, source.data[20_150..20_200].to_vec()); + assert_eq!(source.counted(), (3, 250)); + } +} diff --git a/src/extension/arrow-ext/src/parquet/footer.rs b/src/extension/arrow-ext/src/parquet/footer.rs new file mode 100644 index 0000000000..eab46a8ae6 --- /dev/null +++ b/src/extension/arrow-ext/src/parquet/footer.rs @@ -0,0 +1,179 @@ +//! A Parquet file's footer, read and decoded once: the readers plan and read from its `ParquetMetaData`, the lib +//! surface decodes its thrift bytes into PHP objects. + +use std::sync::Arc; + +use bytes::{Bytes, BytesMut}; +use parquet::file::metadata::{ParquetMetaData, ParquetMetaDataReader}; +use parquet::file::reader::ChunkReader; + +use crate::parquet::error::Error; + +/// What the first read takes from the end of the file: the whole footer of every file it is shorter than. +const TAIL: u64 = 64 * 1024; + +const MAGIC: &[u8; 4] = b"PAR1"; + +pub struct Footer { + /// The thrift `FileMetaData`. + bytes: Bytes, + meta: Arc, +} + +impl Footer { + /// One read of the last min(size, 64 KiB) bytes, a second only when the footer is longer. + pub fn read(source: &R) -> Result { + let size = source.len(); + + if size < 8 { + return Err(corrupt(format!( + "Invalid Parquet file. Size is {size} bytes, smaller than the footer" + ))); + } + + let read = size.min(TAIL); + let tail = source.get_bytes(size - read, read as usize)?; + let trailer = &tail[tail.len() - 8..]; + + if &trailer[4..] != MAGIC { + return Err(corrupt("Invalid Parquet file. Corrupt footer".to_string())); + } + + let length = u64::from(u32::from_le_bytes([trailer[0], trailer[1], trailer[2], trailer[3]])); + + if length + 8 > size { + return Err(corrupt(format!( + "Invalid Parquet file. Reported metadata length of {length} + 8 byte footer, but file is only {size} \ + bytes" + ))); + } + + let end = tail.len() - 8; + let bytes = if length + 8 <= read { + tail.slice(end - length as usize..end) + } else { + let mut bytes = BytesMut::with_capacity(length as usize); + bytes.extend_from_slice(&source.get_bytes(size - 8 - length, (length + 8 - read) as usize)?); + bytes.extend_from_slice(&tail[..end]); + bytes.freeze() + }; + let meta = ParquetMetaDataReader::decode_metadata(&bytes)?; + + Ok(Self { + bytes, + meta: Arc::new(meta), + }) + } + + pub fn meta(&self) -> &Arc { + &self.meta + } + + pub fn bytes(&self) -> &Bytes { + &self.bytes + } +} + +fn corrupt(message: String) -> Error { + Error::NotParquet(message) +} + +#[cfg(test)] +mod tests { + use std::sync::Arc; + + use arrow_array::{Int64Array, RecordBatch}; + use arrow_schema::{DataType, Field, Schema}; + use bytes::Bytes; + use parquet::arrow::ArrowWriter; + use parquet::errors::ParquetError; + use parquet::file::metadata::KeyValue; + use parquet::file::properties::WriterProperties; + use parquet::file::reader::{ChunkReader, Length}; + + use super::Footer; + use crate::parquet::error::Error; + use crate::parquet::source::fake::Counting; + use crate::parquet::source::{exact, WindowedRead}; + + struct Source(Arc); + + impl Length for Source { + fn len(&self) -> u64 { + self.0.data.len() as u64 + } + } + + impl ChunkReader for Source { + type T = WindowedRead; + + fn get_read(&self, start: u64) -> Result { + Ok(WindowedRead::new(Arc::clone(&self.0), start, self.len())) + } + + fn get_bytes(&self, start: u64, length: usize) -> Result { + exact(self.0.as_ref(), start, length) + } + } + + /// A file whose footer carries `padding` bytes of key/value metadata. + fn file(padding: usize) -> Vec { + let schema = Arc::new(Schema::new(vec![Field::new("id", DataType::Int64, false)])); + let props = WriterProperties::builder() + .set_key_value_metadata(Some(vec![KeyValue::new("padding".to_string(), "x".repeat(padding))])) + .build(); + let mut file = Vec::new(); + let mut writer = ArrowWriter::try_new(&mut file, Arc::clone(&schema), Some(props)).unwrap(); + writer + .write(&RecordBatch::try_new(schema, vec![Arc::new(Int64Array::from(vec![1, 2, 3]))]).unwrap()) + .unwrap(); + writer.close().unwrap(); + + file + } + + #[test] + fn a_footer_inside_the_tail_is_read_once() { + let counting = Arc::new(Counting::new(file(10))); + let footer = Footer::read(&Source(Arc::clone(&counting))).unwrap(); + + assert_eq!(counting.counted().0, 1); + assert_eq!(footer.meta().file_metadata().num_rows(), 3); + + let data = &counting.data; + let length = u32::from_le_bytes(data[data.len() - 8..data.len() - 4].try_into().unwrap()) as usize; + assert_eq!(footer.bytes().as_ref(), &data[data.len() - 8 - length..data.len() - 8]); + } + + #[test] + fn a_footer_longer_than_the_tail_takes_a_second_read() { + let counting = Arc::new(Counting::new(file(100 * 1024))); + let footer = Footer::read(&Source(Arc::clone(&counting))).unwrap(); + + assert_eq!(counting.counted().0, 2); + assert_eq!(footer.meta().file_metadata().num_rows(), 3); + assert_eq!( + footer.meta().file_metadata().key_value_metadata().unwrap()[0] + .value + .as_ref() + .map(String::len), + Some(100 * 1024) + ); + } + + #[test] + fn a_file_without_the_magic_is_refused() { + let mut data = file(10); + let end = data.len(); + data[end - 1] = b'X'; + + assert!(matches!( + Footer::read(&Source(Arc::new(Counting::new(data)))), + Err(Error::NotParquet(message)) if message.contains("Corrupt footer") + )); + assert!(matches!( + Footer::read(&Source(Arc::new(Counting::new(vec![1, 2, 3])))), + Err(Error::NotParquet(_)) + )); + } +} diff --git a/src/extension/arrow-ext/src/parquet/lib_calls.rs b/src/extension/arrow-ext/src/parquet/lib_calls.rs new file mode 100644 index 0000000000..1cd3227ecd --- /dev/null +++ b/src/extension/arrow-ext/src/parquet/lib_calls.rs @@ -0,0 +1,22 @@ +//! The `flow-php/parquet` converters arrow calls by name: mago cannot see these calls, the phpts pin each one. + +use ext_php_rs::prelude::*; +use ext_php_rs::types::Zval; + +use crate::exception::ext_exception; +use crate::php::{call_handle_transparent, ce_method_ref, find_class}; + +pub const METADATA_FROM_THRIFT: &str = "Flow\\Parquet\\ParquetFile\\Metadata::fromThrift"; +pub const SCHEMA_FROM_THRIFT: &str = "Flow\\Parquet\\ParquetFile\\Schema::fromThrift"; +pub const SCHEMA_TO_EXTENSION: &str = "Flow\\Parquet\\Engine\\Arrow\\SchemaConverter::toExtension"; +pub const OPTIONS_TO_EXTENSION: &str = "Flow\\Parquet\\Engine\\Arrow\\OptionsConverter::toExtension"; +pub const OPTIONS_TO_ENGINE: &str = "Flow\\Parquet\\Engine\\Arrow\\OptionsConverter::toEngine"; + +/// `Class::method(...$args)`; an exception the method throws surfaces as itself. +pub fn call_static(name: &str, args: &mut [Zval]) -> PhpResult { + let (class, method) = name + .rsplit_once("::") + .ok_or_else(|| ext_exception(format!("arrow expected a static method name, got \"{name}\"")))?; + + call_handle_transparent(ce_method_ref(find_class(class)?, method)?, None, args) +} diff --git a/src/extension/arrow-ext/src/parquet/library.rs b/src/extension/arrow-ext/src/parquet/library.rs new file mode 100644 index 0000000000..b89418385f --- /dev/null +++ b/src/extension/arrow-ext/src/parquet/library.rs @@ -0,0 +1,875 @@ +//! `Flow\Parquet\Engine\{RustParquetFileReader, RustParquetFileWriter}` and `Flow\Arrow\Parquet\RustColumnsReader`: the +//! `flow-php/parquet` lib's values over the core - +//! one footer read per file, columns as lists of the values `PhpParquetEngine` returns, rows written as it accepts +//! them, and struct batches written as another extension exports them. + +use std::sync::Arc; + +use arrow_array::{new_null_array, Array, ArrayRef}; +use ext_php_rs::convert::IntoZval; +use ext_php_rs::error::Result as ZvalResult; +use ext_php_rs::flags::{ClassFlags, DataType}; +use ext_php_rs::prelude::*; +use ext_php_rs::types::{ZendHashTable, Zval}; +use ext_php_rs::zend::ce; +use parquet::basic::Type as PhysicalType; +use parquet::file::metadata::ParquetMetaData; +use parquet::file::reader::Length; + +use crate::interfaces::{parquet_file_reader_ce, parquet_file_writer_ce}; +use crate::parquet::batch; +use crate::parquet::cells::{Checked, Classes, ReadCell, Refusal, Scalar, Times, Validated, WriteColumn}; +use crate::parquet::error::Error; +use crate::parquet::footer::Footer; +use crate::parquet::lib_calls::{call_static, METADATA_FROM_THRIFT, SCHEMA_FROM_THRIFT}; +use crate::parquet::options; +use crate::parquet::read::{ParquetReader, ReadPlan}; +use crate::parquet::sink::PhpSink; +use crate::parquet::source::{PhpSource, PhpStream}; +use crate::parquet::write::ParquetWriter; +use crate::php::{ + call_method, expect_object, for_each_value, ht_for_each, ht_get, ht_insert, read_property, IterableArg, +}; +use crate::render::{exception, parquet_exception, Side}; +use crate::thrift; + +const FILE_META_DATA: &str = "Flow\\Parquet\\ThriftModel\\FileMetaData"; +/// `FileMetaData::$schema`'s field id. +const SCHEMA: i16 = 2; +const RUNTIME: &str = "Flow\\Parquet\\Exception\\RuntimeException"; +pub const INVALID_ARGUMENT: &str = "Flow\\Parquet\\Exception\\InvalidArgumentException"; + +fn closed() -> PhpException { + exception(RUNTIME, "Reader is not open".to_string()) +} + +/// `ParquetFileReader::metadata()`'s return, typed `Flow\Parquet\ParquetFile\Metadata`. +pub struct MetadataObject(Zval); + +impl IntoZval for MetadataObject { + const TYPE: DataType = DataType::object("Flow\\Parquet\\ParquetFile\\Metadata"); + const NULLABLE: bool = false; + + fn set_zval(self, zv: &mut Zval, _persistent: bool) -> ZvalResult<()> { + *zv = self.0; + + Ok(()) + } +} + +/// `ParquetFileReader::schema()`'s return, typed `Flow\Parquet\ParquetFile\Schema`. +pub struct SchemaObject(Zval); + +impl IntoZval for SchemaObject { + const TYPE: DataType = DataType::object("Flow\\Parquet\\ParquetFile\\Schema"); + const NULLABLE: bool = false; + + fn set_zval(self, zv: &mut Zval, _persistent: bool) -> ZvalResult<()> { + *zv = self.0; + + Ok(()) + } +} + +pub fn rows(value: Option, name: &str) -> PhpResult> { + value + .map(|value| { + u64::try_from(value).map_err(|_| { + exception( + INVALID_ARGUMENT, + format!("arrow Parquet {name} must be greater or equal to 0"), + ) + }) + }) + .transpose() +} + +pub fn batch_size(value: i64) -> PhpResult { + usize::try_from(value).ok().filter(|size| *size > 0).ok_or_else(|| { + exception( + INVALID_ARGUMENT, + "arrow Parquet batch size must be greater than 0".to_string(), + ) + }) +} + +#[php_class] +#[php( + name = "Flow\\Parquet\\Engine\\RustParquetFileReader", + flags = ClassFlags::Final, + implements(ce = parquet_file_reader_ce, stub = "Flow\\Parquet\\ParquetFileReader") +)] +pub struct RustParquetFileReader { + stream: Arc, + source: PhpSource, + footer: Footer, + int96_as_datetime: bool, + metadata: Option, + schema: Option, + thrift: Option, + closed: bool, +} + +impl RustParquetFileReader { + fn open(&self) -> PhpResult<()> { + if self.closed { + return Err(closed()); + } + + Ok(()) + } + + /// The file's stream, size and footer, for a reader of it: the lib's `RustColumnsReader` or a `RustBatchReader`. + pub fn read(&self) -> PhpResult<(&Arc, u64, &Arc)> { + self.open()?; + + Ok((&self.stream, self.source.len(), self.footer.meta())) + } + + /// `FileMetaData::$schema`: the list of `SchemaElement`s. + fn elements(&self) -> PhpResult { + thrift::decode(self.footer.bytes(), FILE_META_DATA, Some(SCHEMA)) + } + + /// Without `int96AsDatetime`, a requested column (a leaf path, or the path of its parent) holding INT96. + fn refuse_int96(&self, columns: &ZendHashTable) -> PhpResult<()> { + if self.int96_as_datetime { + return Ok(()); + } + + let leaves = self + .footer + .meta() + .file_metadata() + .schema_descr() + .columns() + .iter() + .filter(|leaf| leaf.physical_type() == PhysicalType::INT96) + .map(|leaf| leaf.path().string()) + .collect::>(); + + for column in columns.values().filter_map(Zval::str) { + if leaves + .iter() + .any(|leaf| leaf == column || leaf.starts_with(&format!("{column}."))) + { + return Err(exception( + INVALID_ARGUMENT, + format!( + "Parquet column \"{column}\" holds INT96, which arrow reads only as a datetime \ + (Option::INT_96_AS_DATETIME = true); read the file with \\Flow\\Parquet\\Reader::php()" + ), + )); + } + } + + Ok(()) + } +} + +#[php_impl] +impl RustParquetFileReader { + /// Reads and decodes the footer; owns the stream from here on. `$int96AsDatetime`: false refuses to read an + /// INT96 column, which arrow reads only as a datetime. + #[php(defaults(int96AsDatetime = true))] + pub fn __construct(stream: &Zval, int96AsDatetime: bool) -> PhpResult { + let stream = Arc::new(PhpStream::new(stream)?); + let refused = |error| parquet_exception(error, &stream, Side::Read); + let source = PhpSource::new(Arc::clone(&stream)).map_err(refused)?; + let footer = Footer::read(&source).map_err(refused)?; + + Ok(Self { + stream, + source, + footer, + int96_as_datetime: int96AsDatetime, + metadata: None, + schema: None, + thrift: None, + closed: false, + }) + } + + /// Closes the stream; any later call throws. + pub fn close(&mut self) -> PhpResult<()> { + self.open()?; + self.closed = true; + self.metadata = None; + self.schema = None; + self.thrift = None; + + match self.stream.call("close", &mut []) { + Some(_) => Ok(()), + None => Err(parquet_exception( + Error::Stream("close() threw".to_string()), + &self.stream, + Side::Read, + )), + } + } + + /// The whole footer as `Metadata`, built on the first call. + pub fn metadata(&mut self) -> PhpResult { + self.open()?; + + if let Some(metadata) = &self.metadata { + return Ok(MetadataObject(metadata.shallow_clone())); + } + + let metadata = call_static(METADATA_FROM_THRIFT, &mut [self.thrift()?])?; + self.metadata = Some(metadata.shallow_clone()); + + Ok(MetadataObject(metadata)) + } + + /// `$columns` (resolved names) in chunks of at most `$batchSize` rows, keyed by column. + #[php(name = "readColumns")] + pub fn read_columns( + &self, + columns: &ZendHashTable, + batchSize: i64, + limit: Option, + offset: Option, + ) -> PhpResult { + self.open()?; + self.refuse_int96(columns)?; + + RustColumnsReader::__construct(self, columns, batchSize, offset, limit) + } + + #[php(name = "rowsNumber")] + pub fn rows_number(&self) -> PhpResult { + self.open()?; + + Ok(self.footer.meta().file_metadata().num_rows()) + } + + /// The file schema, without building the row-group metadata. + pub fn schema(&mut self) -> PhpResult { + self.open()?; + + if let Some(metadata) = &self.metadata { + return Ok(SchemaObject(call_method(metadata, "schema", &mut [])?)); + } + + if let Some(schema) = &self.schema { + return Ok(SchemaObject(schema.shallow_clone())); + } + + let schema = call_static(SCHEMA_FROM_THRIFT, &mut [self.elements()?])?; + self.schema = Some(schema.shallow_clone()); + + Ok(SchemaObject(schema)) + } + + /// Σ row group `total_byte_size` (uncompressed). + #[php(name = "totalByteSize")] + pub fn total_byte_size(&self) -> PhpResult { + self.open()?; + + Ok(self + .footer + .meta() + .row_groups() + .iter() + .map(|group| group.total_byte_size()) + .sum()) + } + + /// The whole footer as `FileMetaData`, decoded on the first call. + pub fn thrift(&mut self) -> PhpResult { + self.open()?; + + if let Some(thrift) = &self.thrift { + return Ok(thrift.shallow_clone()); + } + + let thrift = thrift::decode(self.footer.bytes(), FILE_META_DATA, None)?; + self.thrift = Some(thrift.shallow_clone()); + + Ok(thrift) + } +} + +/// A chunk of `RustColumnsReader::current()`, typed `array`. +pub struct Chunk(Zval); + +impl IntoZval for Chunk { + const TYPE: DataType = DataType::Array; + const NULLABLE: bool = false; + + fn set_zval(self, zv: &mut Zval, _persistent: bool) -> ZvalResult<()> { + *zv = self.0; + + Ok(()) + } +} + +#[php_class] +#[php( + name = "Flow\\Arrow\\Parquet\\RustColumnsReader", + flags = ClassFlags::Final, + implements(ce = ce::iterator, stub = "\\Iterator") +)] +pub struct RustColumnsReader { + reader: Option, + stream: Arc, + columns: Vec<(Vec, ReadCell)>, + times: Option, + current: Option, + /// The index of `current`. + key: i64, + started: bool, + /// `next()` moved past the first chunk: `rewind()` throws from here on. + advanced: bool, +} + +impl RustColumnsReader { + /// As a Generator, the reader starts on its first `rewind()`, `valid()`, `current()`, `key()` or `next()`. + fn start(&mut self) -> PhpResult<()> { + if self.started { + return Ok(()); + } + + self.started = true; + self.pull() + } + + /// A refused chunk ends the reader: it is never read again. + fn pull(&mut self) -> PhpResult<()> { + self.current = None; + let fetched = self.fetch(); + + if fetched.is_err() { + self.reader = None; + } + + self.current = fetched?; + + Ok(()) + } + + /// At most `$batchSize` rows as lists keyed by column, `None` after the last. + fn fetch(&mut self) -> PhpResult> { + let Some(reader) = self.reader.as_mut() else { + return Ok(None); + }; + + let arrays: Vec = match reader.next() { + None => { + self.reader = None; + + return Ok(None); + } + Some(batch) => { + batch + .map_err(|error| parquet_exception(error, &self.stream, Side::Read))? + .1 + } + }; + + let mut chunk = ZendHashTable::with_capacity(self.columns.len() as u32); + + for ((name, cell), array) in self.columns.iter().zip(&arrays) { + ht_insert(&mut chunk, name, cell.values(array, self.times.as_ref())?); + } + + let mut zv = Zval::new(); + zv.set_hashtable(chunk); + + Ok(Some(zv)) + } +} + +#[php_impl] +impl RustColumnsReader { + /// `$columns`: root names or struct paths; `$batchSize` rows at most per chunk. + pub fn __construct( + file: &RustParquetFileReader, + columns: &ZendHashTable, + batch_size: i64, + offset: Option, + limit: Option, + ) -> PhpResult { + let batch_size = self::batch_size(batch_size)?; + let (offset, limit) = (rows(offset, "offset")?.unwrap_or(0), rows(limit, "limit")?); + let (stream, size, meta) = file.read()?; + let stream = Arc::clone(stream); + let refused = |error| parquet_exception(error, &stream, Side::Read); + let mut names = Vec::with_capacity(columns.len()); + + ht_for_each(columns, |_, _, name| { + names.push( + name.str() + .ok_or_else(|| { + exception( + INVALID_ARGUMENT, + "arrow Parquet column names must be strings".to_string(), + ) + })? + .to_string(), + ); + + Ok(()) + })?; + + let reader = ParquetReader::open( + Arc::clone(&stream), + size, + meta, + &names, + ReadPlan::new(meta, offset, limit), + batch_size, + ) + .map_err(refused)?; + let cells = reader + .fields() + .iter() + .zip(reader.types()) + .map(|(field, canonical)| ReadCell::new(canonical, field).map_err(|e| e.in_column(field.name()))) + .collect::, Error>>() + .map_err(refused)?; + let times = if cells.iter().any(ReadCell::needs_times) { + Some(Times::new()?) + } else { + None + }; + + Ok(Self { + reader: Some(reader), + stream, + columns: names.into_iter().map(String::into_bytes).zip(cells).collect(), + times, + current: None, + key: 0, + started: false, + advanced: false, + }) + } + + /// Reads the first chunk; the reader streams once, so it refuses to rewind after `next()`. + pub fn rewind(&mut self) -> PhpResult<()> { + if self.advanced { + return Err(exception(RUNTIME, "RustColumnsReader cannot rewind".to_string())); + } + + self.start() + } + + pub fn valid(&mut self) -> PhpResult { + self.start()?; + + Ok(self.current.is_some()) + } + + pub fn current(&mut self) -> PhpResult> { + self.start()?; + + Ok(self.current.as_ref().map(|chunk| Chunk(chunk.shallow_clone()))) + } + + /// The chunk index from 0, null once the reader ended (as a Generator's). + pub fn key(&mut self) -> PhpResult> { + self.start()?; + + Ok(self.current.is_some().then_some(self.key)) + } + + pub fn next(&mut self) -> PhpResult<()> { + self.start()?; + + if self.current.is_none() { + return Ok(()); + } + + self.advanced = true; + self.key += 1; + self.pull() + } +} + +/// `Compressions::$name` as the Arrow codec name. +fn codec(compression: &Zval) -> PhpResult<&'static str> { + let name = read_property(expect_object(compression, "the compression")?, "name")?; + + match name.str() { + Some("UNCOMPRESSED") => Ok("UNCOMPRESSED"), + Some("SNAPPY") => Ok("SNAPPY"), + Some("GZIP") => Ok("GZIP"), + Some("BROTLI") => Ok("BROTLI"), + Some("LZ4" | "LZ4_RAW") => Ok("LZ4_RAW"), + Some("ZSTD") => Ok("ZSTD"), + Some("LZO") => Err(exception( + RUNTIME, + "LZO compression is not supported by the Arrow engine".to_string(), + )), + _ => Err(exception( + INVALID_ARGUMENT, + "arrow Parquet compression must be a Flow\\Parquet\\ParquetFile\\Compressions case".to_string(), + )), + } +} + +#[php_class] +#[php( + name = "Flow\\Parquet\\Engine\\RustParquetFileWriter", + flags = ClassFlags::Final, + implements(ce = parquet_file_writer_ce, stub = "Flow\\Parquet\\ParquetFileWriter") +)] +pub struct RustParquetFileWriter { + writer: Option, + stream: Arc, + columns: Vec, + classes: Classes, + buffered: usize, + /// Every column holds one value per row: rows are checked into `Scalar`s. + scalar: bool, + batch_size: usize, +} + +impl Drop for RustParquetFileWriter { + fn drop(&mut self) { + self.stream.detach(); + } +} + +impl RustParquetFileWriter { + fn refused(&self, error: Error) -> PhpException { + parquet_exception(error, &self.stream, Side::Write) + } + + fn open(&self) -> PhpResult<()> { + match self.writer { + Some(_) => Ok(()), + None => Err(exception(RUNTIME, "Writer is not open".to_string())), + } + } + + /// The refusal of `column` at buffered row `row`. + fn refusal(&self, refusal: Refusal, column: String, row: usize) -> PhpException { + match refusal { + Refusal::Value { expected, got } => self.refused(Error::Value { + column, + row, + expected, + got, + }), + Refusal::Overflow => self.refused(Error::Overflow { column, row }), + Refusal::Php(exception) => exception, + } + } + + /// The buffered rows through `ParquetWriter::write`. + fn flush(&mut self) -> PhpResult<()> { + if self.buffered == 0 { + return Ok(()); + } + + let arrays = self.columns.iter_mut().map(WriteColumn::take).collect::>(); + let rows = std::mem::take(&mut self.buffered); + let writer = self + .writer + .as_mut() + .ok_or_else(|| exception(RUNTIME, "Writer is not open".to_string()))?; + + writer + .write(rows, &arrays) + .map_err(|error| parquet_exception(error, &self.stream, Side::Write)) + } + + /// One row whose every column was checked (`value` of column index): appended, unless a column's offsets would + /// pass i32, which refuses the row whole before anything is appended. + fn append_checked<'a, 'b, V: Checked<'a> + 'b>(&mut self, value: impl Fn(usize) -> &'b V) -> PhpResult<()> { + let overflow = self + .columns + .iter_mut() + .enumerate() + .position(|(index, column)| !value(index).fits(&column.cell, &mut column.builder)); + + if let Some(index) = overflow { + return Err(self.refused(Error::Overflow { + column: self.columns[index].name.clone(), + row: self.buffered, + })); + } + + for (index, column) in self.columns.iter_mut().enumerate() { + value(index).append(&column.cell, &mut column.builder); + } + + self.buffered += 1; + + Ok(()) + } + + /// A row is checked whole into `checked` first; a refused row appends nothing. + fn append<'a, V: Checked<'a>>(&mut self, row: &'a Zval, checked: &mut Vec) -> PhpResult<()> { + self.open()?; + + let row = row + .array() + .ok_or_else(|| exception(INVALID_ARGUMENT, "arrow Parquet writer rows must be arrays".to_string()))?; + checked.clear(); + + for column in &self.columns { + let value = match ht_get(row, column.name.as_bytes()) { + Some(value) => V::check(&column.cell, value, column.nullable, &self.classes), + None => V::missing(column.nullable), + }; + + match value { + Ok(value) => checked.push(value), + Err(refusal) => return Err(self.refusal(refusal, column.name.clone(), self.buffered)), + } + } + + self.append_checked(|index| &checked[index])?; + + if self.buffered >= self.batch_size { + self.flush()?; + } + + Ok(()) + } + + fn append_rows<'a, V: Checked<'a>>(&mut self, rows: &'a ZendHashTable) -> PhpResult<()> { + let mut checked = Vec::with_capacity(self.columns.len()); + + for row in rows.values() { + self.append::(row, &mut checked)?; + } + + Ok(()) + } + + /// `rows` values of every column from `first` on, column by column; a refused cell lowers the row limit of the + /// columns after it, so the refusal raised and the rows kept are those of `append()` row by row. + fn append_columns<'a, V: Checked<'a>>( + &mut self, + lists: &[Option>], + first: usize, + rows: usize, + ) -> PhpResult<()> { + let mut limit = rows; + let mut refused = None; + let mut checked: Vec> = Vec::with_capacity(self.columns.len()); + + for (index, column) in self.columns.iter().enumerate() { + let mut values = Vec::with_capacity(limit); + let mut stop = None; + + for row in 0..limit { + let value = match &lists[index] { + Some(values) => V::check(&column.cell, values[first + row], column.nullable, &self.classes), + None => V::missing(column.nullable), + }; + + match value { + Ok(value) => values.push(value), + Err(refusal) => { + stop = Some((row, refusal)); + + break; + } + } + } + + if let Some((row, refusal)) = stop { + limit = row; + refused = Some((column.name.clone(), refusal)); + } + + checked.push(values); + } + + (0..limit).try_for_each(|row| self.append_checked(|index| &checked[index][row]))?; + + if let Some((name, refusal)) = refused { + return Err(self.refusal(refusal, name, self.buffered)); + } + + if self.buffered >= self.batch_size { + self.flush()?; + } + + Ok(()) + } +} + +#[php_impl] +impl RustParquetFileWriter { + /// `$schema` / `$options`: `Flow\Parquet\Engine\Arrow\SchemaConverter` / `OptionsConverter::toExtension()`; + /// `$batchSize` rows per `ParquetWriter::write`. + pub fn __construct( + stream: &Zval, + schema: &ZendHashTable, + compression: &Zval, + options: &ZendHashTable, + batch_size: i64, + ) -> PhpResult { + let batch_size = self::batch_size(batch_size)?; + let compression = codec(compression)?; + let stream = Arc::new(PhpStream::new(stream)?); + let refused = |error| parquet_exception(error, &stream, Side::Write); + let schema = options::schema(schema).map_err(refused)?; + let columns = schema + .fields() + .iter() + .map(|field| WriteColumn::new(field, batch_size)) + .collect::, Error>>() + .map_err(refused)?; + let scalar = columns.iter().all(|column| column.cell.is_scalar()); + let writer = ParquetWriter::open( + PhpSink::new(Arc::clone(&stream)), + schema, + options::properties(compression, options).map_err(refused)?, + ) + .map_err(refused)?; + + Ok(Self { + writer: Some(writer), + stream, + columns, + classes: Classes::new()?, + buffered: 0, + scalar, + batch_size, + }) + } + + /// A writer column the row lacks is null. + #[php(name = "writeRow")] + pub fn write_row(&mut self, row: &Zval) -> PhpResult<()> { + if self.scalar { + self.append::(row, &mut Vec::with_capacity(self.columns.len())) + } else { + self.append::(row, &mut Vec::with_capacity(self.columns.len())) + } + } + + #[php(name = "writeRows")] + pub fn write_rows(&mut self, rows: &ZendHashTable) -> PhpResult<()> { + if self.scalar { + self.append_rows::(rows) + } else { + self.append_rows::(rows) + } + } + + /// Every list of one length; a writer column `$columns` lacks is nulls, a key the writer lacks is ignored. + #[php(name = "writeColumns")] + pub fn write_columns(&mut self, columns: &ZendHashTable) -> PhpResult<()> { + if self.writer.is_none() { + return Err(exception(RUNTIME, "Writer is not open".to_string())); + } + + let mut lengths = Vec::with_capacity(columns.len()); + + for (key, values) in columns.iter() { + let values = values.array().ok_or_else(|| { + exception( + INVALID_ARGUMENT, + "arrow Parquet writer columns must be arrays".to_string(), + ) + })?; + + lengths.push((key.to_string(), values.len())); + } + + let count = lengths.first().map_or(0, |(_, length)| *length); + + if lengths.iter().any(|(_, length)| *length != count) { + let described = lengths + .iter() + .map(|(name, length)| format!("\"{name}\": {length}")) + .collect::>(); + + return Err(exception( + INVALID_ARGUMENT, + format!("writeColumns() takes lists of one length, got {}", described.join(", ")), + )); + } + + let lists = self + .columns + .iter() + .map(|column| { + ht_get(columns, column.name.as_bytes()) + .and_then(Zval::array) + .map(|values| values.values().collect::>()) + }) + .collect::>(); + let mut first = 0; + + // in the row door's flush steps, so a refusal names the row index append() names + while first < count { + let rows = (count - first).min(self.batch_size - self.buffered); + + if self.scalar { + self.append_columns::(&lists, first, rows)?; + } else { + self.append_columns::(&lists, first, rows)?; + } + first += rows; + } + + Ok(()) + } + + /// An array as `writeRows()` (keys ignored), a `Traversable` row by row. + #[php(name = "writeBatch")] + pub fn write_batch(&mut self, rows: IterableArg) -> PhpResult<()> { + self.open()?; + + if let Some(rows) = rows.0.array() { + return self.write_rows(rows); + } + + for_each_value(rows.0, |row| self.write_row(row)) + } + + /// An Arrow C Data struct batch another extension exported, moved out of `$batch`: its children by writer column + /// name, a writer column the batch lacks as nulls, a child the writer lacks ignored. Buffered rows go first. + #[php(name = "writeArrowBatch")] + pub fn write_arrow_batch(&mut self, batch: &Zval) -> PhpResult<()> { + self.open()?; + self.flush()?; + + let batch = batch::import(batch)?; + let rows = batch.len(); + let writer = self + .writer + .as_mut() + .ok_or_else(|| exception(RUNTIME, "Writer is not open".to_string()))?; + let arrays = writer + .schema() + .fields() + .iter() + .map(|field| match batch.column_by_name(field.name()) { + Some(column) => Arc::clone(column), + None => new_null_array(field.data_type(), rows), + }) + .collect::>(); + + writer + .write(rows, &arrays) + .map_err(|error| parquet_exception(error, &self.stream, Side::Write)) + } + + /// Buffered rows, the footer, then the stream closed - closed even when a flush or the footer is refused; the + /// first refusal is the one thrown. + pub fn close(&mut self) -> PhpResult<()> { + let flushed = self.flush(); + let writer = self + .writer + .take() + .ok_or_else(|| exception(RUNTIME, "Writer is not open".to_string()))?; + let written = flushed.and_then(|()| writer.close().map(|_| ()).map_err(|error| self.refused(error))); + let closed = match self.stream.call("close", &mut []) { + Some(_) => Ok(()), + None => Err(self.refused(Error::Stream("close() threw".to_string()))), + }; + + written.and(closed) + } +} diff --git a/src/extension/arrow-ext/src/parquet/mod.rs b/src/extension/arrow-ext/src/parquet/mod.rs index d4036fa50a..3a3b825f67 100644 --- a/src/extension/arrow-ext/src/parquet/mod.rs +++ b/src/extension/arrow-ext/src/parquet/mod.rs @@ -1,4 +1,21 @@ -pub mod exception; -pub mod reader; -pub mod type_converter; -pub mod writer; +//! Parquet in arrow: one reader, one cast table and one writer over Flow filesystem streams, surfaced as the +//! `flow-php/parquet` lib's values (`library`) and as Arrow C Data struct batches (`batch`). + +pub mod batch; +pub mod canonical; +pub mod cells; +pub mod engine; +pub mod error; +pub mod fetch; +pub mod footer; +pub mod lib_calls; +#[allow( + non_snake_case, + reason = "PHP takes a parameter name from its Rust identifier: the contracts' are camelCase" +)] +pub mod library; +pub mod options; +pub mod read; +pub mod sink; +pub mod source; +pub mod write; diff --git a/src/extension/arrow-ext/src/parquet/options.rs b/src/extension/arrow-ext/src/parquet/options.rs new file mode 100644 index 0000000000..99dbd48b40 --- /dev/null +++ b/src/extension/arrow-ext/src/parquet/options.rs @@ -0,0 +1,325 @@ +//! Ported verbatim from arrow-ext (`src/parquet/type_converter.rs` `parse_time_unit`, `php_schema_entry_to_field`, +//! `php_schema_to_arrow`; `src/parquet/writer.rs` `parse_compression`, `parse_encoding`, `apply_writer_options` and +//! the writer constructor's properties): the arrays of `Flow\Parquet\Engine\Arrow\SchemaConverter::toExtension()` and +//! `OptionsConverter::toExtension()`. + +use std::str::FromStr; +use std::sync::Arc; + +use arrow_schema::{ + extension::{Json, Uuid}, + DataType, Field, Fields, Schema, TimeUnit, +}; +use ext_php_rs::types::{ArrayKey, ZendHashTable}; +use parquet::basic::{BrotliLevel, Compression, Encoding, GzipLevel, ZstdLevel}; +use parquet::file::properties::{WriterProperties, WriterVersion}; +use parquet::schema::types::ColumnPath; + +use crate::parquet::error::Error; + +pub fn schema(ht: &ZendHashTable) -> Result { + php_schema_to_arrow(ht).map_err(Error::Options) +} + +pub fn properties(compression: &str, options: &ZendHashTable) -> Result { + if options.is_empty() { + let codec = parse_compression(compression, None, None, None).map_err(Error::Options)?; + + return Ok(WriterProperties::builder().set_compression(codec).build()); + } + + Ok(apply_writer_options(WriterProperties::builder(), options, compression) + .map_err(Error::Options)? + .build()) +} + +fn parse_time_unit(entry_ht: &ZendHashTable, name: &str) -> Result { + match entry_ht.get("unit").and_then(|z| z.str()) { + None | Some("MICROS") => Ok(TimeUnit::Microsecond), + Some("MILLIS") => Ok(TimeUnit::Millisecond), + Some("NANOS") => Ok(TimeUnit::Nanosecond), + Some(s) => Err(format!( + "Column '{}': unsupported unit '{}', expected MILLIS, MICROS or NANOS", + name, s + )), + } +} + +fn php_schema_entry_to_field(entry_ht: &ZendHashTable) -> Result { + let name = entry_ht + .get("name") + .and_then(|z| z.str()) + .ok_or_else(|| "Schema entry must have a 'name' string key".to_string())? + .to_string(); + + let type_str = entry_ht + .get("type") + .and_then(|z| z.str()) + .ok_or_else(|| "Schema entry must have a 'type' string key".to_string())? + .to_string(); + + let nullable = entry_ht.get("optional").and_then(|z| z.bool()).unwrap_or(true); + + let data_type = match type_str.as_str() { + "BOOLEAN" => DataType::Boolean, + "INT8" => DataType::Int8, + "INT16" => DataType::Int16, + "INT32" => DataType::Int32, + "INT64" => DataType::Int64, + "UINT8" => DataType::UInt8, + "UINT16" => DataType::UInt16, + "UINT32" => DataType::UInt32, + "UINT64" => DataType::UInt64, + "FLOAT" => DataType::Float32, + "DOUBLE" => DataType::Float64, + "STRING" => DataType::Utf8, + "BINARY" => DataType::Binary, + "DATE" => DataType::Date32, + "TIMESTAMP" => { + let unit = parse_time_unit(entry_ht, &name)?; + let utc = entry_ht.get("utc").and_then(|z| z.bool()).unwrap_or(true); + DataType::Timestamp(unit, if utc { Some("UTC".into()) } else { None }) + } + "TIME" => match parse_time_unit(entry_ht, &name)? { + TimeUnit::Millisecond => DataType::Time32(TimeUnit::Millisecond), + unit => DataType::Time64(unit), + }, + "DECIMAL" => { + let precision = entry_ht + .get("precision") + .and_then(|z| z.long()) + .ok_or_else(|| format!("Column '{}': DECIMAL type requires 'precision' integer key", name))? + as u8; + let scale = entry_ht + .get("scale") + .and_then(|z| z.long()) + .ok_or_else(|| format!("Column '{}': DECIMAL type requires 'scale' integer key", name))? + as i8; + DataType::Decimal128(precision, scale) + } + "UUID" => { + let mut field = Field::new(&name, DataType::FixedSizeBinary(16), nullable); + field + .try_with_extension_type(Uuid) + .map_err(|e| format!("Column '{}': failed to set UUID extension type: {}", name, e))?; + return Ok(field); + } + "JSON" => { + let mut field = Field::new(&name, DataType::Utf8, nullable); + field + .try_with_extension_type(Json::default()) + .map_err(|e| format!("Column '{}': failed to set JSON extension type: {}", name, e))?; + return Ok(field); + } + "FIXED_SIZE_BINARY" => { + let length = entry_ht.get("length").and_then(|z| z.long()).ok_or_else(|| { + format!( + "Column '{}': FIXED_SIZE_BINARY type requires 'length' integer key", + name + ) + })? as i32; + DataType::FixedSizeBinary(length) + } + "LIST" => { + let children = entry_ht + .get("children") + .and_then(|z| z.array()) + .ok_or_else(|| format!("Column '{}': LIST type requires 'children' array", name))?; + let child_zv = children + .values() + .next() + .ok_or_else(|| format!("Column '{}': LIST type requires exactly one child", name))?; + let child_ht = child_zv + .array() + .ok_or_else(|| format!("Column '{}': LIST child must be an array", name))?; + let child_field = php_schema_entry_to_field(child_ht)?; + DataType::List(Arc::new(child_field)) + } + "STRUCT" => { + let children = entry_ht + .get("children") + .and_then(|z| z.array()) + .ok_or_else(|| format!("Column '{}': STRUCT type requires 'children' array", name))?; + let fields: Result, String> = children + .values() + .map(|child_zv| { + let child_ht = child_zv + .array() + .ok_or_else(|| format!("Column '{}': STRUCT child must be an array", name))?; + php_schema_entry_to_field(child_ht) + }) + .collect(); + DataType::Struct(Fields::from(fields?)) + } + "MAP" => { + let children = entry_ht + .get("children") + .and_then(|z| z.array()) + .ok_or_else(|| format!("Column '{}': MAP type requires 'children' array", name))?; + let mut child_iter = children.values(); + let key_zv = child_iter + .next() + .ok_or_else(|| format!("Column '{}': MAP type requires key and value children", name))?; + let val_zv = child_iter + .next() + .ok_or_else(|| format!("Column '{}': MAP type requires key and value children", name))?; + let key_field = php_schema_entry_to_field( + key_zv + .array() + .ok_or_else(|| format!("Column '{}': MAP key child must be an array", name))?, + )?; + let val_field = php_schema_entry_to_field( + val_zv + .array() + .ok_or_else(|| format!("Column '{}': MAP value child must be an array", name))?, + )?; + let entries_struct = DataType::Struct(Fields::from(vec![key_field, val_field])); + DataType::Map(Arc::new(Field::new("entries", entries_struct, false)), false) + } + other => { + return Err(format!("Column '{}': Unsupported type '{}'", name, other)); + } + }; + + Ok(Field::new(&name, data_type, nullable)) +} + +fn php_schema_to_arrow(schema: &ZendHashTable) -> Result { + if schema.is_empty() { + return Err("Schema must have at least one column".into()); + } + + let mut fields = Vec::with_capacity(schema.len()); + + for (_key, zv) in schema.iter() { + let entry_ht = zv.array().ok_or_else(|| "Schema entry must be an array".to_string())?; + fields.push(php_schema_entry_to_field(entry_ht)?); + } + + Ok(Schema::new(fields)) +} + +fn parse_compression( + s: &str, + gzip_level: Option, + brotli_level: Option, + zstd_level: Option, +) -> Result { + match s.to_uppercase().as_str() { + "UNCOMPRESSED" => Ok(Compression::UNCOMPRESSED), + "SNAPPY" => Ok(Compression::SNAPPY), + "GZIP" => { + let level = match gzip_level { + Some(l) => GzipLevel::try_new(l).map_err(|e| format!("Invalid GZIP compression level: {}", e))?, + None => Default::default(), + }; + Ok(Compression::GZIP(level)) + } + "ZSTD" => { + let level = match zstd_level { + Some(l) => ZstdLevel::try_new(l).map_err(|e| format!("Invalid ZSTD compression level: {}", e))?, + None => Default::default(), + }; + Ok(Compression::ZSTD(level)) + } + "LZ4" | "LZ4_RAW" => Ok(Compression::LZ4_RAW), + "BROTLI" => { + let level = match brotli_level { + Some(l) => BrotliLevel::try_new(l).map_err(|e| format!("Invalid BROTLI compression level: {}", e))?, + None => Default::default(), + }; + Ok(Compression::BROTLI(level)) + } + other => Err(format!( + "Unsupported compression codec '{}'. Supported: UNCOMPRESSED, SNAPPY, GZIP, ZSTD, LZ4, LZ4_RAW, BROTLI", + other + )), + } +} + +fn parse_encoding(s: &str) -> Result { + Encoding::from_str(&s.to_uppercase()).map_err(|e| format!("Unsupported encoding '{}': {}", s, e)) +} + +fn apply_writer_options( + mut builder: parquet::file::properties::WriterPropertiesBuilder, + options: &ZendHashTable, + compression_str: &str, +) -> Result { + let gzip_level = options + .get("GZIP_COMPRESSION_LEVEL") + .and_then(|z| z.long()) + .map(|v| v as u32); + + let brotli_level = options + .get("BROTLI_COMPRESSION_LEVEL") + .and_then(|z| z.long()) + .map(|v| v as u32); + + let zstd_level = options + .get("ZSTD_COMPRESSION_LEVEL") + .and_then(|z| z.long()) + .map(|v| v as i32); + + let codec = parse_compression(compression_str, gzip_level, brotli_level, zstd_level)?; + builder = builder.set_compression(codec); + + if let Some(val) = options.get("ROW_GROUP_SIZE_BYTES").and_then(|z| z.long()) { + builder = builder.set_max_row_group_bytes(Some(val as usize)); + } + + if let Some(val) = options.get("PAGE_SIZE_BYTES").and_then(|z| z.long()) { + builder = builder.set_data_page_size_limit(val as usize); + } + + if let Some(val) = options.get("DICTIONARY_PAGE_SIZE").and_then(|z| z.long()) { + builder = builder.set_dictionary_page_size_limit(val as usize); + } + + if let Some(val) = options.get("WRITER_VERSION").and_then(|z| z.long()) { + let version = match val { + 1 => WriterVersion::PARQUET_1_0, + 2 => WriterVersion::PARQUET_2_0, + other => return Err(format!("Invalid WRITER_VERSION '{}'. Must be 1 or 2", other)), + }; + builder = builder.set_writer_version(version); + } + + if let Some(col_comp_zval) = options.get("COLUMNS_COMPRESSIONS") { + if let Some(col_comp_ht) = col_comp_zval.array() { + for (key, val) in col_comp_ht.iter() { + let col_path_str = match key { + ArrayKey::String(ref s) => s.clone(), + ArrayKey::Str(s) => s.to_string(), + _ => continue, + }; + if let Some(codec_str) = val.str() { + let parts: Vec = col_path_str.split('.').map(|s| s.to_string()).collect(); + let col_path = ColumnPath::new(parts); + let col_codec = parse_compression(codec_str, gzip_level, brotli_level, zstd_level)?; + builder = builder.set_column_compression(col_path, col_codec); + } + } + } + } + + if let Some(col_enc_zval) = options.get("COLUMNS_ENCODINGS") { + if let Some(col_enc_ht) = col_enc_zval.array() { + for (key, val) in col_enc_ht.iter() { + let col_path_str = match key { + ArrayKey::String(ref s) => s.clone(), + ArrayKey::Str(s) => s.to_string(), + _ => continue, + }; + if let Some(enc_str) = val.str() { + let parts: Vec = col_path_str.split('.').map(|s| s.to_string()).collect(); + let col_path = ColumnPath::new(parts); + let encoding = parse_encoding(enc_str)?; + builder = builder.set_column_encoding(col_path, encoding); + } + } + } + } + + Ok(builder) +} diff --git a/src/extension/arrow-ext/src/parquet/read.rs b/src/extension/arrow-ext/src/parquet/read.rs new file mode 100644 index 0000000000..bb661f2cf8 --- /dev/null +++ b/src/extension/arrow-ext/src/parquet/read.rs @@ -0,0 +1,742 @@ +//! Row groups of a Parquet file as canonical arrays: only the groups an offset/limit window overlaps are decoded, and +//! every projected column is checked against the cast table before the first batch. + +use std::sync::Arc; + +use arrow_array::cast::AsArray; +use arrow_array::{make_array, Array, ArrayRef, RecordBatchReader}; +use arrow_buffer::NullBuffer; +use arrow_schema::{DataType, Field, FieldRef, Fields, Schema, SchemaRef, TimeUnit}; +use parquet::arrow::arrow_reader::{ + ArrowReaderMetadata, ArrowReaderOptions, ParquetRecordBatchReader, ParquetRecordBatchReaderBuilder, +}; +use parquet::arrow::ProjectionMask; +use parquet::basic::{Compression, Type}; +use parquet::file::metadata::{PageIndexPolicy, ParquetMetaData, ParquetMetaDataReader}; +use parquet::schema::types::SchemaDescriptor; + +use crate::parquet::canonical::{canonical, canonical_type}; +use crate::parquet::error::Error; +use crate::parquet::fetch::ChunkFetch; +use crate::parquet::source::ReadAt; + +/// The row groups overlapping [offset, offset + limit) and the rows to skip in the first of them. +#[derive(Debug, PartialEq, Eq)] +pub struct ReadPlan { + pub row_groups: Vec, + pub skip: usize, + pub limit: Option, +} + +impl ReadPlan { + pub fn new(meta: &ParquetMetaData, offset: u64, limit: Option) -> Self { + let (mut row_groups, mut skip, mut start) = (Vec::new(), 0, 0); + + for (index, group) in meta.row_groups().iter().enumerate() { + let end = start + group.num_rows() as u64; + + if end > offset && limit.is_none_or(|limit| start < offset + limit) { + if row_groups.is_empty() { + skip = (offset - start) as usize; + } + + row_groups.push(index); + } + + start = end; + } + + Self { + row_groups, + skip, + limit: limit.map(|limit| limit as usize), + } + } +} + +/// A projected column: a root name, or a path through structs ("structure.int64") - the root first, then the child +/// names down to the column. +struct Projected { + path: Vec, + field: FieldRef, +} + +impl Projected { + /// A root named `name` as it is; else `name` as a path whose every segment but the last is a struct. + fn resolve(schema: &Schema, name: &str) -> Result { + if let Ok(root) = schema.field_with_name(name) { + return Ok(Self { + path: vec![name.to_string()], + field: Arc::new(root.clone()), + }); + } + + let missing = || Error::MissingColumn(name.to_string()); + let mut segments = name.split('.'); + let root = segments.next().ok_or_else(missing)?; + let mut field = schema.field_with_name(root).map_err(|_| missing())?; + let mut nullable = field.is_nullable(); + let mut path = vec![root.to_string()]; + + for segment in segments { + field = match field.data_type() { + DataType::Struct(children) => children.find(segment).ok_or_else(missing)?.1, + DataType::List(_) | DataType::LargeList(_) | DataType::FixedSizeList(_, _) | DataType::Map(_, _) => { + return Err(Error::Unsupported { + column: name.to_string(), + parquet: "a path into LIST/MAP".to_string(), + }) + } + _ => return Err(missing()), + }; + nullable |= field.is_nullable(); + path.push(segment.to_string()); + } + + Ok(Self { + path, + field: Arc::new(field.clone().with_name(name).with_nullable(nullable)), + }) + } + + fn reads_leaf(&self, parts: &[String]) -> bool { + parts.starts_with(&self.path) + } +} + +pub struct ParquetReader { + batches: ParquetRecordBatchReader, + /// Each column's field as the file stores it, named as it was asked for. + fields: Vec, + /// `canonical_type()` of each of `fields`. + types: Vec, + /// Where each of `fields` sits in the projected batches: its root, then the struct children down to it. + positions: Vec<(usize, Vec)>, +} + +impl ParquetReader { + /// `source`: the file's bytes (`PhpStream`), `size` long; `meta`: its footer, which `plan` was built from. Only + /// the projected column chunks of the planned row groups are fetched (`ChunkFetch`); the page index is read on a + /// copy of `meta`, only when the plan skips rows inside its first row group. + pub fn open( + source: Arc, + size: u64, + meta: &Arc, + columns: &[String], + plan: ReadPlan, + batch_size: usize, + ) -> Result { + let inferred = ArrowReaderMetadata::try_new(Arc::clone(meta), ArrowReaderOptions::new())?; + let options = match int96_hint(inferred.schema(), meta.file_metadata().schema_descr()) { + Some(hint) => ArrowReaderOptions::new().with_schema(hint), + None => ArrowReaderOptions::new(), + }; + let schema = ArrowReaderMetadata::try_new(Arc::clone(meta), options.clone())? + .schema() + .clone(); + let projected = columns + .iter() + .map(|name| Projected::resolve(&schema, name)) + .collect::, _>>()?; + let fields = projected + .iter() + .map(|column| Arc::clone(&column.field)) + .collect::>(); + let types = fields + .iter() + .map(|field| canonical_type(field)) + .collect::, _>>()?; + + if let Some(codec) = unsupported_codec(meta, &plan.row_groups, &projected) { + return Err(codec); + } + + let descr = meta.file_metadata().schema_descr(); + let leaves = (0..descr.num_columns()) + .filter(|leaf| { + projected + .iter() + .any(|column| column.reads_leaf(descr.column(*leaf).path().parts())) + }) + .collect::>(); + let fetch = ChunkFetch::new(source, size, meta, &plan.row_groups, &leaves); + let meta = if plan.skip > 0 { + let mut reader = ParquetMetaDataReader::new_with_metadata(meta.as_ref().clone()) + .with_page_index_policy(PageIndexPolicy::Optional); + reader.read_page_indexes(&fetch)?; + ArrowReaderMetadata::try_new(Arc::new(reader.finish()?), options)? + } else { + ArrowReaderMetadata::try_new(Arc::clone(meta), options)? + }; + let mask = ProjectionMask::leaves(meta.metadata().file_metadata().schema_descr(), leaves); + let mut builder = ParquetRecordBatchReaderBuilder::new_with_metadata(fetch, meta) + .with_row_groups(plan.row_groups) + .with_projection(mask) + .with_batch_size(batch_size); + + if plan.skip > 0 { + builder = builder.with_offset(plan.skip); + } + + if let Some(limit) = plan.limit { + builder = builder.with_limit(limit); + } + + let batches = builder.build()?; + let schema = batches.schema(); + let positions = projected + .into_iter() + .map(|mut column| { + let root = schema + .index_of(&column.path[0]) + .map_err(|_| Error::MissingColumn(column.field.name().clone()))?; + column.path.remove(0); + + Ok((root, column.path)) + }) + .collect::, Error>>()?; + + Ok(Self { + batches, + fields, + types, + positions, + }) + } + + /// The canonical type each of `fields()` is read as. + pub fn types(&self) -> &[DataType] { + &self.types + } + + pub fn fields(&self) -> &[FieldRef] { + &self.fields + } + + pub fn next(&mut self) -> Option), Error>> { + let batch = match self.batches.next()? { + Ok(batch) => batch, + Err(error) => return Some(Err(error.into())), + }; + + Some( + self.fields + .iter() + .zip(&self.positions) + .map(|(field, (root, children))| canonical(field, &child(batch.column(*root), children)?)) + .collect::, _>>() + .map(|columns| (batch.num_rows(), columns)), + ) + } +} + +/// The struct child at `path` under `array`, null wherever a struct above it is. +fn child(array: &ArrayRef, path: &[String]) -> Result { + let mut array = Arc::clone(array); + + for name in path { + let structure = array.as_struct(); + let child = structure + .column_by_name(name) + .ok_or_else(|| Error::MissingColumn(name.clone()))?; + + array = match NullBuffer::union(structure.nulls(), child.nulls()) { + Some(nulls) if structure.nulls().is_some() => { + make_array(child.to_data().into_builder().nulls(Some(nulls)).build()?) + } + _ => Arc::clone(child), + }; + } + + Ok(array) +} + +/// The inferred schema with every INT96 leaf read as `Timestamp(µs, "UTC")` - exact over years 1 to 9999, where +/// arrow-rs's default nanoseconds wrap outside 1677-2262; `None` without INT96 leaves. +fn int96_hint(schema: &SchemaRef, descr: &SchemaDescriptor) -> Option { + let mut leaf = 0; + let mut hinted = false; + let fields = schema + .fields() + .iter() + .map(|field| int96_hinted(field, descr, &mut leaf, &mut hinted)) + .collect::(); + + hinted.then(|| Arc::new(Schema::new_with_metadata(fields, schema.metadata().clone()))) +} + +/// `field` with its INT96 leaves hinted; `leaf` counts the Parquet leaves `field`'s arrow leaves stand for, in order. +fn int96_hinted(field: &FieldRef, descr: &SchemaDescriptor, leaf: &mut usize, hinted: &mut bool) -> FieldRef { + let mut hint = |field: &FieldRef| int96_hinted(field, descr, leaf, hinted); + let data_type = match field.data_type() { + DataType::Struct(children) => DataType::Struct(children.iter().map(&mut hint).collect()), + DataType::List(element) => DataType::List(hint(element)), + DataType::LargeList(element) => DataType::LargeList(hint(element)), + DataType::FixedSizeList(element, size) => DataType::FixedSizeList(hint(element), *size), + DataType::Map(entries, sorted) => DataType::Map(hint(entries), *sorted), + _ => { + let index = *leaf; + *leaf += 1; + + if index >= descr.num_columns() || descr.column(index).physical_type() != Type::INT96 { + return Arc::clone(field); + } + + *hinted = true; + DataType::Timestamp(TimeUnit::Microsecond, Some("UTC".into())) + } + }; + + Arc::new(Field::clone(field).with_data_type(data_type)) +} + +/// A codec the parquet crate is built without, in a projected column chunk of the planned row groups. +fn unsupported_codec(meta: &ParquetMetaData, row_groups: &[usize], columns: &[Projected]) -> Option { + row_groups.iter().find_map(|group| { + meta.row_group(*group).columns().iter().find_map(|chunk| { + let column = columns + .iter() + .find(|column| column.reads_leaf(chunk.column_path().parts()))?; + + (chunk.compression() == Compression::LZO).then(|| Error::Unsupported { + column: column.field.name().clone(), + parquet: "LZO".to_string(), + }) + }) + }) +} + +#[cfg(test)] +mod tests { + use std::sync::Arc; + + use arrow_array::builder::{Int64Builder, MapBuilder, StringBuilder}; + use arrow_array::cast::AsArray; + use arrow_array::types::TimestampMicrosecondType; + use arrow_array::{Array, ArrayRef, Int64Array, ListArray, RecordBatch, StringArray, StructArray}; + use arrow_buffer::NullBuffer; + use arrow_schema::{DataType, Field, Fields, Schema, TimeUnit}; + use bytes::Bytes; + use parquet::arrow::ArrowWriter; + use parquet::basic::Compression; + use parquet::data_type::{Int64Type, Int96, Int96Type}; + use parquet::file::metadata::{ParquetMetaData, ParquetMetaDataReader}; + use parquet::file::properties::WriterProperties; + use parquet::file::writer::SerializedFileWriter; + use parquet::schema::parser::parse_message_type; + use parquet::schema::types::SchemaDescriptor; + + use super::{int96_hint, unsupported_codec, ParquetReader, Projected, ReadPlan}; + use crate::parquet::error::Error; + use crate::parquet::footer::Footer; + use crate::parquet::source::fake::Counting; + + fn eight_groups_of_8192() -> parquet::file::metadata::ParquetMetaData { + let schema = Arc::new(Schema::new(vec![Field::new("id", DataType::Int64, false)])); + let mut file = Vec::new(); + let mut writer = ArrowWriter::try_new( + &mut file, + Arc::clone(&schema), + Some( + WriterProperties::builder() + .set_max_row_group_row_count(Some(8_192)) + .build(), + ), + ) + .unwrap(); + writer + .write(&RecordBatch::try_new(schema, vec![Arc::new(Int64Array::from_iter_values(0..65_536))]).unwrap()) + .unwrap(); + writer.close().unwrap(); + + ParquetMetaDataReader::new() + .parse_and_finish(&Bytes::from(file)) + .unwrap() + } + + fn plan(offset: u64, limit: u64) -> (Vec, usize) { + let plan = ReadPlan::new(&eight_groups_of_8192(), offset, Some(limit)); + + assert_eq!(plan.limit, Some(limit as usize)); + + (plan.row_groups, plan.skip) + } + + #[test] + fn plan_reads_the_groups_a_window_overlaps() { + assert_eq!(eight_groups_of_8192().row_groups().len(), 8); + assert_eq!(plan(0, 10), (vec![0], 0)); + assert_eq!(plan(5_399, 3), (vec![0], 5_399)); + assert_eq!(plan(20_000, 15_000), (vec![2, 3, 4], 3_616)); + assert_eq!(plan(65_000, 100), (vec![7], 7_656)); + assert_eq!(plan(65_046, 1), (vec![7], 7_702)); + } + + #[test] + fn plan_without_limit_reads_every_group_from_the_offset() { + let plan = ReadPlan::new(&eight_groups_of_8192(), 16_384, None); + + assert_eq!( + plan, + ReadPlan { + row_groups: vec![2, 3, 4, 5, 6, 7], + skip: 0, + limit: None + } + ); + } + + fn with_lzo_names() -> ParquetMetaData { + let schema = Arc::new(Schema::new(vec![ + Field::new("id", DataType::Int64, false), + Field::new("name", DataType::Utf8, true), + ])); + let mut file = Vec::new(); + let mut writer = ArrowWriter::try_new(&mut file, Arc::clone(&schema), None).unwrap(); + writer + .write( + &RecordBatch::try_new( + schema, + vec![ + Arc::new(Int64Array::from(vec![1])), + Arc::new(StringArray::from(vec!["a"])), + ], + ) + .unwrap(), + ) + .unwrap(); + writer.close().unwrap(); + + let mut builder = ParquetMetaDataReader::new() + .parse_and_finish(&Bytes::from(file)) + .unwrap() + .into_builder(); + let row_groups = builder + .take_row_groups() + .into_iter() + .map(|group| { + let columns = group + .columns() + .iter() + .map(|chunk| match chunk.column_path().string().as_str() { + "name" => chunk + .clone() + .into_builder() + .set_compression(Compression::LZO) + .build() + .unwrap(), + _ => chunk.clone(), + }) + .collect(); + + group.into_builder().set_column_metadata(columns).build().unwrap() + }) + .collect(); + + builder.set_row_groups(row_groups).build() + } + + #[test] + fn an_lzo_column_chunk_is_refused_when_projected() { + let meta = with_lzo_names(); + let schema = Schema::new(vec![ + Field::new("id", DataType::Int64, false), + Field::new("name", DataType::Utf8, true), + ]); + let column = |name: &str| Projected::resolve(&schema, name).unwrap(); + + assert!(matches!( + unsupported_codec(&meta, &[0], &[column("id"), column("name")]), + Some(Error::Unsupported { column, parquet }) if column == "name" && parquet == "LZO" + )); + assert!(unsupported_codec(&meta, &[0], &[column("id")]).is_none()); + } + + fn open( + file: Vec, + columns: &[&str], + offset: u64, + limit: Option, + ) -> Result<(ParquetReader, Arc), Error> { + let footer = Footer::read(&Bytes::from(file.clone()))?; + let source = Arc::new(Counting::new(file)); + let size = source.data.len() as u64; + let columns = columns.iter().map(|column| column.to_string()).collect::>(); + let plan = ReadPlan::new(footer.meta(), offset, limit); + + Ok(( + ParquetReader::open(Arc::clone(&source), size, footer.meta(), &columns, plan, 1_024)?, + source, + )) + } + + fn read(file: Vec, columns: &[&str]) -> Result, Error> { + let (mut reader, _) = open(file, columns, 0, None)?; + let (_, arrays) = reader.next().unwrap()?; + + assert!(reader.next().is_none()); + + Ok(arrays) + } + + fn int96(days_since_epoch: i64, nanos_of_day: i64) -> Int96 { + let mut value = Int96::new(); + value.set_data( + nanos_of_day as u32, + (nanos_of_day >> 32) as u32, + (days_since_epoch + 2_440_588) as u32, + ); + + value + } + + /// `ts` INT96 flat, `nanos` INT64 TIMESTAMP(NANOS), `st.ts` INT96 in a struct, `l` a list of INT96. + fn int96_file() -> Vec { + let schema = Arc::new( + parse_message_type( + "message s { optional int96 ts; optional int64 nanos (TIMESTAMP(NANOS,false)); \ + optional group st { optional int96 ts; } \ + optional group l (LIST) { repeated group list { optional int96 element; } } }", + ) + .unwrap(), + ); + let mut file = Vec::new(); + let mut writer = + SerializedFileWriter::new(&mut file, schema, Arc::new(WriterProperties::builder().build())).unwrap(); + let mut group = writer.next_row_group().unwrap(); + let first = int96(-719_162, 1_000); + let last = int96(-3_653, 86_399_999_999_999); + + let mut column = group.next_column().unwrap().unwrap(); + column + .typed::() + .write_batch(&[first, last], Some(&[1, 1]), None) + .unwrap(); + column.close().unwrap(); + let mut column = group.next_column().unwrap().unwrap(); + column + .typed::() + .write_batch(&[1_500, -1_500], Some(&[1, 1]), None) + .unwrap(); + column.close().unwrap(); + let mut column = group.next_column().unwrap().unwrap(); + column + .typed::() + .write_batch(&[first, last], Some(&[2, 2]), None) + .unwrap(); + column.close().unwrap(); + let mut column = group.next_column().unwrap().unwrap(); + column + .typed::() + .write_batch(&[first, last], Some(&[3, 3]), Some(&[0, 0])) + .unwrap(); + column.close().unwrap(); + group.close().unwrap(); + writer.close().unwrap(); + + file + } + + fn micros(array: &ArrayRef) -> Vec> { + array.as_primitive::().iter().collect() + } + + #[test] + fn int96_leaves_are_read_as_exact_microseconds() { + let arrays = read(int96_file(), &["ts", "nanos", "st", "l"]).unwrap(); + let expected = vec![Some(-62_135_596_799_999_999), Some(-315_532_800_000_001)]; + + assert_eq!(micros(&arrays[0]), expected); + assert_eq!(micros(&arrays[1]), vec![Some(1), Some(-2)]); + assert_eq!(micros(arrays[2].as_struct().column(0)), expected); + assert_eq!(micros(arrays[3].as_list::().values()), expected); + } + + #[test] + fn int96_is_not_hinted_without_int96_leaves() { + let schema = parse_message_type("message s { optional int64 nanos (TIMESTAMP(NANOS,false)); }").unwrap(); + let descr = SchemaDescriptor::new(Arc::new(schema)); + let inferred = Arc::new(Schema::new(vec![Field::new( + "nanos", + DataType::Timestamp(TimeUnit::Nanosecond, None), + true, + )])); + + assert!(int96_hint(&inferred, &descr).is_none()); + } + + fn struct_file() -> Vec { + let children = Fields::from(vec![ + Field::new("int64", DataType::Int64, true), + Field::new("name", DataType::Utf8, false), + ]); + let structure = StructArray::new( + children.clone(), + vec![ + Arc::new(Int64Array::from(vec![Some(1), Some(2), None])), + Arc::new(StringArray::from(vec!["a", "b", "c"])), + ], + Some(NullBuffer::from(vec![true, false, true])), + ); + let list = ListArray::from_iter_primitive::(vec![Some(vec![Some(1)]); 3]); + let mut map = MapBuilder::new(None, StringBuilder::new(), Int64Builder::new()); + for _ in 0..3 { + map.keys().append_value("k"); + map.values().append_value(1); + map.append(true).unwrap(); + } + let map = map.finish(); + let schema = Arc::new(Schema::new(vec![ + Field::new("structure", DataType::Struct(children), true), + Field::new("list", list.data_type().clone(), true), + Field::new("map", map.data_type().clone(), true), + ])); + let mut file = Vec::new(); + let mut writer = ArrowWriter::try_new(&mut file, Arc::clone(&schema), None).unwrap(); + writer + .write(&RecordBatch::try_new(schema, vec![Arc::new(structure), Arc::new(list), Arc::new(map)]).unwrap()) + .unwrap(); + writer.close().unwrap(); + + file + } + + #[test] + fn a_struct_path_reads_the_child_null_under_a_null_struct() { + let arrays = read(struct_file(), &["structure.int64", "structure.name"]).unwrap(); + + assert_eq!( + arrays[0] + .as_primitive::() + .iter() + .collect::>(), + vec![Some(1), None, None] + ); + assert_eq!( + arrays[1].as_binary::().iter().collect::>(), + vec![Some(&b"a"[..]), None, Some(&b"c"[..])] + ); + } + + #[test] + fn a_struct_path_is_named_as_asked() { + let (reader, _) = open(struct_file(), &["structure.name"], 0, None).unwrap(); + + assert_eq!(reader.fields()[0].name(), "structure.name"); + assert!(reader.fields()[0].is_nullable()); + assert_eq!(reader.types(), &[DataType::Binary]); + } + + #[test] + fn a_path_into_a_list_or_a_map_is_refused() { + for path in ["list.list.element", "list.item", "map.key_value.key", "map.entries"] { + assert!(matches!( + read(struct_file(), &[path]), + Err(Error::Unsupported { column, parquet }) if column == path && parquet == "a path into LIST/MAP" + )); + } + } + + #[test] + fn a_path_that_names_nothing_is_missing() { + for path in ["structure.missing", "missing.int64", "structure.int64.deeper"] { + assert!(matches!(read(struct_file(), &[path]), Err(Error::MissingColumn(column)) if column == path)); + } + } + + /// Columns `a`, `b`, `c` (Int64) in four row groups of 100 rows. + fn four_groups() -> Vec { + let schema = Arc::new(Schema::new(vec![ + Field::new("a", DataType::Int64, false), + Field::new("b", DataType::Int64, false), + Field::new("c", DataType::Int64, false), + ])); + let column = || Arc::new(Int64Array::from_iter_values(0..400)) as ArrayRef; + let mut file = Vec::new(); + let mut writer = ArrowWriter::try_new( + &mut file, + Arc::clone(&schema), + Some( + WriterProperties::builder() + .set_max_row_group_row_count(Some(100)) + .build(), + ), + ) + .unwrap(); + writer + .write(&RecordBatch::try_new(schema, vec![column(), column(), column()]).unwrap()) + .unwrap(); + writer.close().unwrap(); + + file + } + + /// The compressed bytes of `columns`' chunks in `groups`. + fn chunk_bytes(file: &[u8], groups: &[usize], columns: &[usize]) -> usize { + let meta = Footer::read(&Bytes::from(file.to_vec())).unwrap(); + + groups + .iter() + .flat_map(|group| columns.iter().map(move |column| (*group, *column))) + .map(|(group, column)| meta.meta().row_group(group).column(column).byte_range().1 as usize) + .sum() + } + + fn drained(mut reader: ParquetReader) -> usize { + let mut rows = 0; + + while let Some(batch) = reader.next() { + rows += batch.unwrap().0; + } + + rows + } + + #[test] + fn a_full_read_fetches_each_row_group_once() { + let file = four_groups(); + let (reader, source) = open(file.clone(), &["a", "b", "c"], 0, None).unwrap(); + + assert_eq!(drained(reader), 400); + assert_eq!(source.counted(), (4, chunk_bytes(&file, &[0, 1, 2, 3], &[0, 1, 2]))); + } + + #[test] + fn a_projected_read_fetches_only_the_projected_chunks() { + let file = four_groups(); + let (reader, source) = open(file.clone(), &["b"], 0, None).unwrap(); + + assert_eq!(drained(reader), 400); + assert_eq!(source.counted(), (4, chunk_bytes(&file, &[0, 1, 2, 3], &[1]))); + } + + #[test] + fn an_offset_read_fetches_only_the_planned_row_group_and_the_page_index() { + let file = four_groups(); + let (reader, source) = open(file.clone(), &["a", "c"], 250, Some(10)).unwrap(); + let meta = Footer::read(&Bytes::from(file.clone())).unwrap(); + let index = meta + .meta() + .row_groups() + .iter() + .flat_map(|group| group.columns()) + .map(|chunk| { + chunk.column_index_length().unwrap_or(0) as usize + chunk.offset_index_length().unwrap_or(0) as usize + }) + .sum::(); + + assert_eq!(drained(reader), 10); + + let (reads, bytes) = source.counted(); + + assert!(reads <= 3, "{reads} reads"); + assert!(bytes >= chunk_bytes(&file, &[2], &[0, 2]), "{bytes} bytes"); + assert!(bytes <= chunk_bytes(&file, &[2], &[0, 1, 2]) + index, "{bytes} bytes"); + } + + #[test] + fn plan_past_the_end_reads_nothing() { + assert_eq!(plan(65_536, 10), (vec![], 0)); + assert_eq!(plan(0, 0), (vec![], 0)); + } +} diff --git a/src/extension/arrow-ext/src/parquet/reader.rs b/src/extension/arrow-ext/src/parquet/reader.rs deleted file mode 100644 index f948fd4143..0000000000 --- a/src/extension/arrow-ext/src/parquet/reader.rs +++ /dev/null @@ -1,424 +0,0 @@ -use arrow_schema::{DataType, TimeUnit}; -use ext_php_rs::boxed::ZBox; -use ext_php_rs::prelude::*; -use ext_php_rs::types::{ZendHashTable, ZendObject}; -use parquet::arrow::arrow_reader::{ - ArrowReaderMetadata, ArrowReaderOptions, ParquetRecordBatchReader, - ParquetRecordBatchReaderBuilder, -}; -use parquet::arrow::ProjectionMask; - -use crate::parquet::exception::parquet_exception; -use crate::parquet::type_converter; -use crate::stream::php_source_stream::PhpSourceStream; - -#[php_class] -#[php(name = "Flow\\Arrow\\Parquet\\Reader")] -#[derive(Default)] -pub struct Reader { - stream: Option, - reader_metadata: Option, - num_row_groups: usize, - current_row_group: usize, - batch_size: Option, - active_batch_reader: Option, - active_columns: Vec, -} - -impl Reader { - fn ensure_open(&self) -> PhpResult<(&PhpSourceStream, &ArrowReaderMetadata)> { - match (&self.stream, &self.reader_metadata) { - (Some(s), Some(m)) => Ok((s, m)), - _ => Err(parquet_exception("Reader is closed")), - } - } - - fn batch_to_php( - batch: &arrow_array::RecordBatch, - columns: &[String], - ) -> PhpResult> { - let mut result = ZendHashTable::new(); - let schema = batch.schema(); - - for col_name in columns { - let column = batch.column_by_name(col_name).ok_or_else(|| { - parquet_exception(format!("Column '{}' not found in batch", col_name)) - })?; - let field = schema.field_with_name(col_name).ok(); - - let values = type_converter::arrow_array_to_php_values(column.as_ref(), field)?; - - let mut php_array = ZendHashTable::with_capacity(values.len() as u32); - - for zv in values { - php_array - .push(zv) - .map_err(|_| parquet_exception("Failed to build column array"))?; - } - - result - .insert(col_name.as_str(), php_array) - .map_err(|_| parquet_exception("Failed to build result array"))?; - } - - Ok(result) - } - - fn consume_all_batches( - reader: ParquetRecordBatchReader, - columns: &[String], - ) -> PhpResult>> { - let mut column_data: Vec> = - columns.iter().map(|_| Vec::new()).collect(); - - for batch_result in reader { - let batch = batch_result - .map_err(|e| parquet_exception(format!("Failed to read batch: {}", e)))?; - let schema = batch.schema(); - - for (idx, col_name) in columns.iter().enumerate() { - let column = batch.column_by_name(col_name).ok_or_else(|| { - parquet_exception(format!("Column '{}' not found in batch", col_name)) - })?; - let field = schema.field_with_name(col_name).ok(); - - let mut values = type_converter::arrow_array_to_php_values(column.as_ref(), field)?; - column_data[idx].append(&mut values); - } - } - - let mut result = ZendHashTable::new(); - - for (idx, col_name) in columns.iter().enumerate() { - let values = std::mem::take(&mut column_data[idx]); - let mut php_array = ZendHashTable::with_capacity(values.len() as u32); - - for zv in values { - php_array - .push(zv) - .map_err(|_| parquet_exception("Failed to build column array"))?; - } - - result - .insert(col_name.as_str(), php_array) - .map_err(|_| parquet_exception("Failed to build result array"))?; - } - - Ok(Some(result)) - } -} - -fn time_unit_name(unit: &TimeUnit) -> &'static str { - match unit { - // Parquet has no second unit; arrow-rs stores seconds as MILLIS. - TimeUnit::Second | TimeUnit::Millisecond => "MILLIS", - TimeUnit::Microsecond => "MICROS", - TimeUnit::Nanosecond => "NANOS", - } -} - -fn field_to_php_schema(field: &arrow_schema::Field) -> PhpResult> { - let mut entry = ZendHashTable::new(); - entry - .insert("name", field.name().as_str()) - .map_err(|_| parquet_exception("Failed to build schema entry"))?; - - let type_str = match field.data_type() { - DataType::Boolean => "BOOLEAN", - DataType::Int8 => "INT8", - DataType::Int16 => "INT16", - DataType::Int32 => "INT32", - DataType::Int64 => "INT64", - DataType::UInt8 => "UINT8", - DataType::UInt16 => "UINT16", - DataType::UInt32 => "UINT32", - DataType::UInt64 => "UINT64", - DataType::Float32 => "FLOAT", - DataType::Float64 => "DOUBLE", - DataType::Utf8 | DataType::LargeUtf8 => "STRING", - DataType::Binary | DataType::LargeBinary => "BINARY", - DataType::Date32 | DataType::Date64 => "DATE", - DataType::Timestamp(_, _) => "TIMESTAMP", - DataType::Time32(_) | DataType::Time64(_) => "TIME", - DataType::Decimal128(_, _) | DataType::Decimal256(_, _) => "DECIMAL", - DataType::FixedSizeBinary(_) => "FIXED_SIZE_BINARY", - DataType::List(_) | DataType::LargeList(_) => "LIST", - DataType::Struct(_) => "STRUCT", - DataType::Map(_, _) => "MAP", - other => { - return Err(parquet_exception(format!( - "Unsupported Arrow type in schema: {:?}", - other - ))); - } - }; - - entry - .insert("type", type_str) - .map_err(|_| parquet_exception("Failed to build schema entry"))?; - entry - .insert("optional", field.is_nullable()) - .map_err(|_| parquet_exception("Failed to build schema entry"))?; - - match field.data_type() { - DataType::Decimal128(precision, scale) | DataType::Decimal256(precision, scale) => { - entry - .insert("precision", *precision as i64) - .map_err(|_| parquet_exception("Failed to build schema entry"))?; - entry - .insert("scale", *scale as i64) - .map_err(|_| parquet_exception("Failed to build schema entry"))?; - } - DataType::Timestamp(unit, tz) => { - entry - .insert("unit", time_unit_name(unit)) - .map_err(|_| parquet_exception("Failed to build schema entry"))?; - entry - .insert("utc", tz.is_some()) - .map_err(|_| parquet_exception("Failed to build schema entry"))?; - } - DataType::Time32(unit) | DataType::Time64(unit) => { - entry - .insert("unit", time_unit_name(unit)) - .map_err(|_| parquet_exception("Failed to build schema entry"))?; - } - DataType::FixedSizeBinary(n) => { - entry - .insert("length", *n as i64) - .map_err(|_| parquet_exception("Failed to build schema entry"))?; - } - DataType::List(child_field) | DataType::LargeList(child_field) => { - let mut children = ZendHashTable::new(); - let child_schema = field_to_php_schema(child_field)?; - children - .push(child_schema) - .map_err(|_| parquet_exception("Failed to build schema children"))?; - entry - .insert("children", children) - .map_err(|_| parquet_exception("Failed to build schema entry"))?; - } - DataType::Struct(fields) => { - let mut children = ZendHashTable::new(); - for f in fields { - let child_schema = field_to_php_schema(f)?; - children - .push(child_schema) - .map_err(|_| parquet_exception("Failed to build schema children"))?; - } - entry - .insert("children", children) - .map_err(|_| parquet_exception("Failed to build schema entry"))?; - } - DataType::Map(struct_field, _) => { - if let DataType::Struct(fields) = struct_field.data_type() { - let mut children = ZendHashTable::new(); - for f in fields { - let child_schema = field_to_php_schema(f)?; - children - .push(child_schema) - .map_err(|_| parquet_exception("Failed to build schema children"))?; - } - entry - .insert("children", children) - .map_err(|_| parquet_exception("Failed to build schema entry"))?; - } - } - _ => {} - } - - Ok(entry) -} - -#[php_impl] -impl Reader { - pub fn __construct( - source: &mut ZendObject, - options: Option<&ZendHashTable>, - ) -> PhpResult { - let stream = PhpSourceStream::new(source).map_err(parquet_exception)?; - - let reader_metadata = ArrowReaderMetadata::load(&stream, ArrowReaderOptions::default()) - .map_err(|e| parquet_exception(format!("Failed to open Parquet file: {}", e)))?; - - let num_row_groups = reader_metadata.metadata().num_row_groups(); - - let batch_size = options - .and_then(|opts| opts.get("BATCH_SIZE")) - .and_then(|z| z.long()) - .map(|v| v as usize); - - Ok(Self { - stream: Some(stream), - reader_metadata: Some(reader_metadata), - num_row_groups, - current_row_group: 0, - batch_size, - active_batch_reader: None, - active_columns: Vec::new(), - }) - } - - pub fn schema(&self) -> PhpResult> { - let (_, reader_metadata) = self.ensure_open()?; - let arrow_schema = reader_metadata.schema(); - - let mut result = ZendHashTable::new(); - - for field in arrow_schema.fields() { - let entry = field_to_php_schema(field)?; - result - .push(entry) - .map_err(|_| parquet_exception("Failed to build schema array"))?; - } - - Ok(result) - } - - pub fn metadata(&self) -> PhpResult> { - let (_, reader_metadata) = self.ensure_open()?; - let metadata = reader_metadata.metadata(); - - let mut result = ZendHashTable::new(); - result - .insert("rows", metadata.file_metadata().num_rows()) - .map_err(|_| parquet_exception("Failed to build metadata"))?; - result - .insert("row_groups", metadata.num_row_groups() as i64) - .map_err(|_| parquet_exception("Failed to build metadata"))?; - - match metadata.file_metadata().created_by() { - Some(created_by) => { - result - .insert("created_by", created_by) - .map_err(|_| parquet_exception("Failed to build metadata"))?; - } - None => { - result - .insert("created_by", ()) - .map_err(|_| parquet_exception("Failed to build metadata"))?; - } - } - - let mut kv_metadata = ZendHashTable::new(); - - if let Some(kv_pairs) = metadata.file_metadata().key_value_metadata() { - for kv in kv_pairs { - if let Some(ref value) = kv.value { - kv_metadata - .insert(kv.key.as_str(), value.as_str()) - .map_err(|_| parquet_exception("Failed to build metadata"))?; - } - } - } - - result - .insert("metadata", kv_metadata) - .map_err(|_| parquet_exception("Failed to build metadata"))?; - - Ok(result) - } - - #[allow(non_snake_case)] - pub fn readRowGroup( - &mut self, - columns: Option>, - ) -> PhpResult>> { - loop { - if let Some(ref mut batch_reader) = self.active_batch_reader { - match batch_reader.next() { - Some(Ok(batch)) => { - return Ok(Some(Self::batch_to_php(&batch, &self.active_columns)?)); - } - Some(Err(e)) => { - self.active_batch_reader = None; - return Err(parquet_exception(format!("Failed to read batch: {}", e))); - } - None => { - self.active_batch_reader = None; - self.current_row_group += 1; - continue; - } - } - } - - if self.current_row_group >= self.num_row_groups { - return Ok(None); - } - - let (stream, reader_metadata) = self.ensure_open()?; - let arrow_schema = reader_metadata.schema().clone(); - let parquet_metadata = reader_metadata.metadata().clone(); - let cloned_stream = stream.clone(); - let cloned_metadata = reader_metadata.clone(); - - let selected_columns: Vec = if let Some(ref cols) = columns { - for col_name in cols { - if arrow_schema.field_with_name(col_name).is_err() { - return Err(parquet_exception(format!( - "Column '{}' not found in schema", - col_name - ))); - } - } - cols.clone() - } else { - arrow_schema - .fields() - .iter() - .map(|f| f.name().clone()) - .collect() - }; - - let parquet_schema = parquet_metadata.file_metadata().schema_descr(); - - let mut builder = - ParquetRecordBatchReaderBuilder::new_with_metadata(cloned_stream, cloned_metadata); - builder = builder.with_row_groups(vec![self.current_row_group]); - - if columns.is_some() { - let indices: Result, _> = selected_columns - .iter() - .map(|name| { - arrow_schema - .fields() - .iter() - .position(|f| f.name() == name) - .ok_or_else(|| { - parquet_exception(format!("Column '{}' not found in schema", name)) - }) - }) - .collect(); - let indices = indices?; - let mask = ProjectionMask::roots(parquet_schema, indices); - builder = builder.with_projection(mask); - } - - if let Some(bs) = self.batch_size { - builder = builder.with_batch_size(bs); - } - - let reader = builder - .build() - .map_err(|e| parquet_exception(format!("Failed to build batch reader: {}", e)))?; - - self.active_columns = selected_columns; - - if self.batch_size.is_some() { - self.active_batch_reader = Some(reader); - continue; - } else { - let result = Self::consume_all_batches(reader, &self.active_columns)?; - self.current_row_group += 1; - self.active_columns = Vec::new(); - return Ok(result); - } - } - } - - pub fn close(&mut self) { - self.active_batch_reader = None; - self.stream = None; - self.reader_metadata = None; - self.active_columns = Vec::new(); - } -} diff --git a/src/extension/arrow-ext/src/parquet/sink.rs b/src/extension/arrow-ext/src/parquet/sink.rs new file mode 100644 index 0000000000..647b7ce0dc --- /dev/null +++ b/src/extension/arrow-ext/src/parquet/sink.rs @@ -0,0 +1,31 @@ +//! Parquet bytes to a `Flow\Filesystem\DestinationStream` (`append(string $data)`). + +use std::io::{self, Write}; +use std::sync::Arc; + +use crate::parquet::source::PhpStream; +use crate::php::zval_str; + +pub struct PhpSink { + stream: Arc, +} + +impl PhpSink { + pub fn new(stream: Arc) -> Self { + Self { stream } + } +} + +impl Write for PhpSink { + fn write(&mut self, buf: &[u8]) -> io::Result { + self.stream + .call("append", &mut [zval_str(buf)]) + .ok_or_else(|| io::Error::other("append() threw"))?; + + Ok(buf.len()) + } + + fn flush(&mut self) -> io::Result<()> { + Ok(()) + } +} diff --git a/src/extension/arrow-ext/src/parquet/source.rs b/src/extension/arrow-ext/src/parquet/source.rs new file mode 100644 index 0000000000..d0248331b3 --- /dev/null +++ b/src/extension/arrow-ext/src/parquet/source.rs @@ -0,0 +1,311 @@ +//! Parquet bytes from a `Flow\Filesystem\SourceStream` (`read(int $length, int $offset): string`, `size(): ?int`), and +//! the PHP stream object both the source and the sink call into. + +use std::cell::{Cell, RefCell}; +use std::io::{self, Read}; +use std::sync::Arc; + +use bytes::{Buf, Bytes}; +use ext_php_rs::boxed::ZBox; +use ext_php_rs::exception::PhpException; +use ext_php_rs::types::{ZendObject, Zval}; +use parquet::errors::ParquetError; +use parquet::file::reader::{ChunkReader, Length}; + +use crate::parquet::error::Error; +use crate::php::{call_handle_catching, ce_method_ref, expect_object, zval_long}; + +/// What `get_read()` pulls from the stream per `read()` call. +pub const WINDOW: usize = 64 * 1024; + +/// A PHP stream object. An exception its method throws is kept here, for the surface to rethrow as itself once the +/// Parquet call it broke returns. +pub struct PhpStream { + stream: Zval, + thrown: RefCell>>, + detached: Cell, +} + +// SAFETY: the stream belongs to the request thread that built it; the parquet crate requires Send + Sync of its +// readers and writers but calls them only from that thread, synchronously. +unsafe impl Send for PhpStream {} +unsafe impl Sync for PhpStream {} + +impl PhpStream { + pub fn new(stream: &Zval) -> Result { + expect_object(stream, "a stream")?; + + Ok(Self { + stream: stream.shallow_clone(), + thrown: RefCell::new(None), + detached: Cell::new(false), + }) + } + + /// `$stream->$method(...$args)`; `None` when it threw, or once the stream is detached. + pub fn call(&self, method: &str, args: &mut [Zval]) -> Option { + if self.detached.get() { + return None; + } + + let object = self.stream.object()?; + let function = ce_method_ref(unsafe { object.ce.as_ref() }?, method).ok()?; + + match call_handle_catching(function, Some(object), args) { + Ok(value) => Some(value), + Err(exception) => { + self.thrown.replace(Some(exception)); + + None + } + } + } + + /// No PHP call from here on: what an abandoned Parquet writer flushes while it is dropped - possibly while a PHP + /// exception unwinds, which a call would take - never reaches the stream. + pub fn detach(&self) { + self.detached.set(true); + } + + /// The exception a call threw, taken. + pub fn thrown(&self) -> Option> { + self.thrown.take() + } +} + +/// Bytes at an offset of a source: the one call `WindowedRead` and `get_bytes()` make. +pub trait ReadAt { + fn read_at(&self, offset: u64, length: usize) -> Result; +} + +impl ReadAt for PhpStream { + fn read_at(&self, offset: u64, length: usize) -> Result { + let read = self + .call("read", &mut [zval_long(length as i64), zval_long(offset as i64)]) + .ok_or_else(|| ParquetError::External("read() threw".into()))?; + + Ok(Bytes::copy_from_slice( + read.zend_str() + .ok_or_else(|| ParquetError::External("read() must return a string".into()))? + .as_bytes(), + )) + } +} + +#[derive(Clone)] +pub struct PhpSource { + stream: Arc, + size: u64, +} + +impl PhpSource { + pub fn new(stream: Arc) -> Result { + let size = stream + .call("size", &mut []) + .ok_or_else(|| Error::Stream("size() threw".to_string()))? + .long() + .ok_or_else(|| Error::Stream("size() must return the stream size".to_string()))?; + + Ok(Self { + stream, + size: u64::try_from(size).map_err(|_| Error::Stream(format!("size() returned {size}")))?, + }) + } +} + +impl Length for PhpSource { + fn len(&self) -> u64 { + self.size + } +} + +impl ChunkReader for PhpSource { + type T = WindowedRead; + + fn get_read(&self, start: u64) -> Result { + Ok(WindowedRead::new(Arc::clone(&self.stream), start, self.size)) + } + + fn get_bytes(&self, start: u64, length: usize) -> Result { + exact(self.stream.as_ref(), start, length) + } +} + +/// Exactly `length` bytes at `start`, in one read. +pub fn exact(source: &R, start: u64, length: usize) -> Result { + if length == 0 { + return Ok(Bytes::new()); + } + + let bytes = source.read_at(start, length)?; + + if bytes.len() != length { + return Err(ParquetError::EOF(format!( + "expected {length} bytes at offset {start}, read {}", + bytes.len() + ))); + } + + Ok(bytes) +} + +/// A `Read` from `position` to `end` that pulls `WINDOW` bytes at a time, only when the previous window is used up. +pub struct WindowedRead { + source: Arc, + position: u64, + end: u64, + window: Bytes, +} + +impl WindowedRead { + pub fn new(source: Arc, position: u64, end: u64) -> Self { + Self { + source, + position, + end, + window: Bytes::new(), + } + } +} + +impl Read for WindowedRead { + fn read(&mut self, buf: &mut [u8]) -> io::Result { + if self.window.is_empty() { + if self.position >= self.end { + return Ok(0); + } + + let length = (self.end - self.position).min(WINDOW as u64) as usize; + self.window = self.source.read_at(self.position, length).map_err(io::Error::other)?; + self.position += self.window.len() as u64; + } + + let read = buf.len().min(self.window.len()); + buf[..read].copy_from_slice(&self.window[..read]); + self.window.advance(read); + + Ok(read) + } +} + +/// A `ReadAt` over bytes in memory that counts the reads made through it and the bytes they returned. +#[cfg(test)] +pub mod fake { + use std::sync::atomic::{AtomicUsize, Ordering}; + + use bytes::Bytes; + use parquet::errors::ParquetError; + + use super::ReadAt; + + pub struct Counting { + pub data: Bytes, + reads: AtomicUsize, + bytes: AtomicUsize, + } + + impl Counting { + pub fn new(data: Vec) -> Self { + Self { + data: Bytes::from(data), + reads: AtomicUsize::new(0), + bytes: AtomicUsize::new(0), + } + } + + /// (reads, bytes) so far. + pub fn counted(&self) -> (usize, usize) { + (self.reads.load(Ordering::Relaxed), self.bytes.load(Ordering::Relaxed)) + } + } + + impl ReadAt for Counting { + fn read_at(&self, offset: u64, length: usize) -> Result { + let start = (offset as usize).min(self.data.len()); + let end = (start + length).min(self.data.len()); + self.reads.fetch_add(1, Ordering::Relaxed); + self.bytes.fetch_add(end - start, Ordering::Relaxed); + + Ok(self.data.slice(start..end)) + } + } +} + +#[cfg(test)] +mod tests { + use std::io::Read; + use std::sync::Arc; + + use parquet::errors::ParquetError; + + use super::fake::Counting; + use super::{exact, WindowedRead, WINDOW}; + + fn sized(size: usize) -> Counting { + Counting::new((0..size).map(|i| i as u8).collect()) + } + + #[test] + fn windowed_read_pulls_one_window_for_a_small_read() { + let source = Arc::new(sized(10 * WINDOW)); + let mut read = WindowedRead::new(Arc::clone(&source), 100, source.data.len() as u64); + let mut header = [0u8; 32]; + + read.read_exact(&mut header).unwrap(); + + assert_eq!(header.to_vec(), source.data[100..132].to_vec()); + assert_eq!(source.counted(), (1, WINDOW)); + } + + #[test] + fn windowed_read_pulls_the_next_window_only_when_the_first_is_used_up() { + let source = Arc::new(sized(10 * WINDOW)); + let mut read = WindowedRead::new(Arc::clone(&source), 0, source.data.len() as u64); + let mut buffer = vec![0u8; WINDOW + 1]; + + read.read_exact(&mut buffer[..WINDOW]).unwrap(); + assert_eq!(source.counted(), (1, WINDOW)); + + read.read_exact(&mut buffer[WINDOW..]).unwrap(); + assert_eq!(buffer, source.data[..WINDOW + 1].to_vec()); + assert_eq!(source.counted(), (2, 2 * WINDOW)); + } + + #[test] + fn windowed_read_never_reads_past_the_end() { + let source = Arc::new(sized(3 * WINDOW)); + let mut read = WindowedRead::new(Arc::clone(&source), 3 * WINDOW as u64 - 10, 3 * WINDOW as u64); + let mut rest = Vec::new(); + + read.read_to_end(&mut rest).unwrap(); + + assert_eq!(rest, source.data[3 * WINDOW - 10..].to_vec()); + assert_eq!(source.counted(), (1, 10)); + } + + #[test] + fn exact_reads_the_length_asked_in_one_read() { + let source = sized(10 * WINDOW); + + assert_eq!( + exact(&source, 5, 3 * WINDOW).unwrap().to_vec(), + source.data[5..5 + 3 * WINDOW].to_vec() + ); + assert_eq!(source.counted(), (1, 3 * WINDOW)); + } + + #[test] + fn exact_refuses_a_short_read() { + let source = sized(100); + + assert!(matches!(exact(&source, 90, 20), Err(ParquetError::EOF(_)))); + } + + #[test] + fn exact_reads_nothing_for_zero_bytes() { + let source = sized(100); + + assert!(exact(&source, 0, 0).unwrap().is_empty()); + assert_eq!(source.counted().0, 0); + } +} diff --git a/src/extension/arrow-ext/src/parquet/type_converter.rs b/src/extension/arrow-ext/src/parquet/type_converter.rs deleted file mode 100644 index 323e7c9e71..0000000000 --- a/src/extension/arrow-ext/src/parquet/type_converter.rs +++ /dev/null @@ -1,1608 +0,0 @@ -use arrow_array::builder::{ - BinaryBuilder, BooleanBuilder, Date32Builder, Decimal128Builder, FixedSizeBinaryBuilder, - Float32Builder, Float64Builder, Int16Builder, Int32Builder, Int64Builder, Int8Builder, - StringBuilder, UInt16Builder, UInt32Builder, UInt64Builder, UInt8Builder, -}; -use arrow_array::types::{Decimal128Type, Decimal256Type, DecimalType}; -use arrow_array::*; -use arrow_buffer::{NullBuffer, OffsetBuffer, ScalarBuffer}; -use arrow_schema::{ - extension::{Json, Uuid}, - DataType, Field, Fields, Schema, TimeUnit, -}; -use ext_php_rs::convert::IntoZvalDyn; -use ext_php_rs::prelude::*; -use ext_php_rs::types::{ArrayKey, ZendHashTable, ZendObject, Zval}; -use ext_php_rs::zend::ClassEntry; -use std::sync::Arc; - -use crate::parquet::exception::parquet_exception; - -fn split_epoch(value: i64, units_per_second: i64) -> (i64, i64) { - ( - value.div_euclid(units_per_second), - value.rem_euclid(units_per_second) * (1_000_000 / units_per_second), - ) -} - -fn create_datetime_immutable(seconds: i64, microseconds: i64) -> PhpResult { - let mut callable_ht = ZendHashTable::with_capacity(2); - let mut class_zv = Zval::new(); - class_zv - .set_string("DateTimeImmutable", false) - .map_err(|_| parquet_exception("Failed to create DateTimeImmutable callable"))?; - callable_ht - .push(class_zv) - .map_err(|_| parquet_exception("Failed to create DateTimeImmutable callable"))?; - let mut method_zv = Zval::new(); - method_zv - .set_string("createFromFormat", false) - .map_err(|_| parquet_exception("Failed to create DateTimeImmutable callable"))?; - callable_ht - .push(method_zv) - .map_err(|_| parquet_exception("Failed to create DateTimeImmutable callable"))?; - - let mut callable = Zval::new(); - callable.set_hashtable(callable_ht); - - let format = "U.u".to_string(); - let value = format!("{}.{:06}", seconds, microseconds); - - callable - .try_call(vec![ - &format as &dyn IntoZvalDyn, - &value as &dyn IntoZvalDyn, - ]) - .map_err(|e| { - parquet_exception(format!( - "Failed to call DateTimeImmutable::createFromFormat: {:?}", - e - )) - }) -} - -fn create_date_interval(total_microseconds: i64) -> PhpResult { - let hours = total_microseconds / 3_600_000_000; - let minutes = (total_microseconds / 60_000_000) % 60; - let secs = (total_microseconds / 1_000_000) % 60; - let remaining_us = total_microseconds % 1_000_000; - - let spec = format!("PT{}H{}M{}S", hours, minutes, secs); - - let ce = ClassEntry::try_find("DateInterval") - .ok_or_else(|| parquet_exception("DateInterval class not found"))?; - let mut obj = ZendObject::new(ce); - - obj.try_call_method("__construct", vec![&spec as &dyn IntoZvalDyn]) - .map_err(|e| parquet_exception(format!("Failed to construct DateInterval: {:?}", e)))?; - - if remaining_us > 0 { - let fraction = remaining_us as f64 / 1_000_000.0; - obj.set_property("f", fraction).map_err(|e| { - parquet_exception(format!("Failed to set DateInterval microseconds: {:?}", e)) - })?; - } - - let mut zv = Zval::new(); - zv.set_object(&mut obj); - Ok(zv) -} - -fn extract_datetime_timestamp(zv: &Zval, column_name: &str) -> Result<(i64, i64), String> { - let obj = zv - .object() - .ok_or_else(|| format!("Column '{}': expected object", column_name))?; - - let dt_ce = ClassEntry::try_find("DateTimeInterface") - .ok_or_else(|| format!("Column '{}': DateTimeInterface not found", column_name))?; - - if !obj.instance_of(dt_ce) { - return Err(format!( - "Column '{}': expected DateTimeInterface or int, got object", - column_name - )); - } - - let ts_zv = obj - .try_call_method("getTimestamp", vec![]) - .map_err(|e| format!("Column '{}': getTimestamp() failed: {:?}", column_name, e))?; - let seconds = ts_zv.long().ok_or_else(|| { - format!( - "Column '{}': getTimestamp() did not return int", - column_name - ) - })?; - - let format_u = "u".to_string(); - let us_zv = obj - .try_call_method("format", vec![&format_u as &dyn IntoZvalDyn]) - .map_err(|e| format!("Column '{}': format('u') failed: {:?}", column_name, e))?; - let us_str = us_zv.str().ok_or_else(|| { - format!( - "Column '{}': format('u') did not return string", - column_name - ) - })?; - let microseconds: i64 = us_str.parse().unwrap_or(0); - - Ok((seconds, microseconds)) -} - -fn extract_datetime_offset(zv: &Zval, column_name: &str) -> Result { - let obj = zv - .object() - .ok_or_else(|| format!("Column '{}': expected object", column_name))?; - - obj.try_call_method("getOffset", vec![]) - .map_err(|e| format!("Column '{}': getOffset() failed: {:?}", column_name, e))? - .long() - .ok_or_else(|| format!("Column '{}': getOffset() did not return int", column_name)) -} - -fn extract_date_interval_microseconds(zv: &Zval, column_name: &str) -> Result { - let obj = zv - .object() - .ok_or_else(|| format!("Column '{}': expected object", column_name))?; - - let di_ce = ClassEntry::try_find("DateInterval") - .ok_or_else(|| format!("Column '{}': DateInterval not found", column_name))?; - - if !obj.instance_of(di_ce) { - return Err(format!( - "Column '{}': expected DateInterval or int, got object", - column_name - )); - } - - let h: i64 = obj.get_property::("h").unwrap_or(0); - let i: i64 = obj.get_property::("i").unwrap_or(0); - let s: i64 = obj.get_property::("s").unwrap_or(0); - let f: f64 = obj.get_property::("f").unwrap_or(0.0); - - Ok(h * 3_600_000_000 + i * 60_000_000 + s * 1_000_000 + (f * 1_000_000.0) as i64) -} - -fn decimal_to_zval(value: T::Native, precision: u8, scale: i8) -> PhpResult { - let str_val = T::format_decimal(value, precision, scale); - let float_val: f64 = str_val.parse().map_err(|e| { - parquet_exception(format!( - "Failed to parse decimal value '{}': {}", - str_val, e - )) - })?; - let mut zv = Zval::new(); - zv.set_double(float_val); - Ok(zv) -} - -fn downcast_array<'a, T: 'static>(array: &'a dyn Array, type_name: &str) -> PhpResult<&'a T> { - array - .as_any() - .downcast_ref::() - .ok_or_else(|| parquet_exception(format!("Internal error: expected {}", type_name))) -} - -fn convert_primitive_values( - arr: &PrimitiveArray, - convert: impl Fn(T::Native) -> PhpResult, -) -> PhpResult> { - let len = arr.len(); - let mut result = Vec::with_capacity(len); - for i in 0..len { - if arr.is_null(i) { - let mut zv = Zval::new(); - zv.set_null(); - result.push(zv); - } else { - result.push(convert(arr.value(i))?); - } - } - Ok(result) -} - -fn convert_string_values(arr: &GenericStringArray) -> PhpResult> { - let len = arr.len(); - let mut result = Vec::with_capacity(len); - for i in 0..len { - let mut zv = Zval::new(); - if arr.is_null(i) { - zv.set_null(); - } else { - zv.set_string(arr.value(i), false) - .map_err(|_| parquet_exception("Failed to set string value"))?; - } - result.push(zv); - } - Ok(result) -} - -fn convert_binary_values(arr: &GenericBinaryArray) -> Vec { - let len = arr.len(); - let mut result = Vec::with_capacity(len); - for i in 0..len { - let mut zv = Zval::new(); - if arr.is_null(i) { - zv.set_null(); - } else { - zv.set_binary(arr.value(i).to_vec()); - } - result.push(zv); - } - result -} - -fn uuid_to_string(b: &[u8]) -> String { - format!( - "{:02x}{:02x}{:02x}{:02x}-{:02x}{:02x}-{:02x}{:02x}-{:02x}{:02x}-{:02x}{:02x}{:02x}{:02x}{:02x}{:02x}", - b[0], b[1], b[2], b[3], b[4], b[5], b[6], b[7], - b[8], b[9], b[10], b[11], b[12], b[13], b[14], b[15] - ) -} - -fn is_uuid_field(field: Option<&Field>) -> bool { - field.is_some_and(|f| f.try_extension_type::().is_ok()) -} - -fn convert_single_arrow_value(array: &dyn Array, index: usize, is_uuid: bool) -> PhpResult { - if array.is_null(index) { - let mut zv = Zval::new(); - zv.set_null(); - return Ok(zv); - } - - match array.data_type() { - DataType::Boolean => { - let arr = downcast_array::(array, "BooleanArray")?; - let mut zv = Zval::new(); - zv.set_bool(arr.value(index)); - Ok(zv) - } - DataType::Int8 => { - let arr = downcast_array::(array, "Int8Array")?; - let mut zv = Zval::new(); - zv.set_long(arr.value(index) as i64); - Ok(zv) - } - DataType::Int16 => { - let arr = downcast_array::(array, "Int16Array")?; - let mut zv = Zval::new(); - zv.set_long(arr.value(index) as i64); - Ok(zv) - } - DataType::Int32 => { - let arr = downcast_array::(array, "Int32Array")?; - let mut zv = Zval::new(); - zv.set_long(arr.value(index) as i64); - Ok(zv) - } - DataType::Int64 => { - let arr = downcast_array::(array, "Int64Array")?; - let mut zv = Zval::new(); - zv.set_long(arr.value(index)); - Ok(zv) - } - DataType::UInt8 => { - let arr = downcast_array::(array, "UInt8Array")?; - let mut zv = Zval::new(); - zv.set_long(arr.value(index) as i64); - Ok(zv) - } - DataType::UInt16 => { - let arr = downcast_array::(array, "UInt16Array")?; - let mut zv = Zval::new(); - zv.set_long(arr.value(index) as i64); - Ok(zv) - } - DataType::UInt32 => { - let arr = downcast_array::(array, "UInt32Array")?; - let mut zv = Zval::new(); - zv.set_long(arr.value(index) as i64); - Ok(zv) - } - DataType::UInt64 => { - let arr = downcast_array::(array, "UInt64Array")?; - let val = arr.value(index); - let mut zv = Zval::new(); - if val > i64::MAX as u64 { - zv.set_string(&val.to_string(), false) - .map_err(|_| parquet_exception("Failed to set uint64 string value"))?; - } else { - zv.set_long(val as i64); - } - Ok(zv) - } - DataType::Float32 => { - let arr = downcast_array::(array, "Float32Array")?; - let mut zv = Zval::new(); - zv.set_double(arr.value(index) as f64); - Ok(zv) - } - DataType::Float64 => { - let arr = downcast_array::(array, "Float64Array")?; - let mut zv = Zval::new(); - zv.set_double(arr.value(index)); - Ok(zv) - } - DataType::Utf8 => { - let arr = downcast_array::(array, "StringArray")?; - let mut zv = Zval::new(); - zv.set_string(arr.value(index), false) - .map_err(|_| parquet_exception("Failed to set string value"))?; - Ok(zv) - } - DataType::LargeUtf8 => { - let arr = downcast_array::(array, "LargeStringArray")?; - let mut zv = Zval::new(); - zv.set_string(arr.value(index), false) - .map_err(|_| parquet_exception("Failed to set string value"))?; - Ok(zv) - } - DataType::Binary => { - let arr = downcast_array::(array, "BinaryArray")?; - let mut zv = Zval::new(); - zv.set_binary(arr.value(index).to_vec()); - Ok(zv) - } - DataType::LargeBinary => { - let arr = downcast_array::(array, "LargeBinaryArray")?; - let mut zv = Zval::new(); - zv.set_binary(arr.value(index).to_vec()); - Ok(zv) - } - DataType::FixedSizeBinary(_) => { - let arr = downcast_array::(array, "FixedSizeBinaryArray")?; - let mut zv = Zval::new(); - if is_uuid && arr.value(index).len() == 16 { - zv.set_string(&uuid_to_string(arr.value(index)), false) - .map_err(|_| parquet_exception("Failed to set UUID string value"))?; - } else { - zv.set_binary(arr.value(index).to_vec()); - } - Ok(zv) - } - DataType::Date32 => { - let arr = downcast_array::(array, "Date32Array")?; - create_datetime_immutable(arr.value(index) as i64 * 86400, 0) - } - DataType::Date64 => { - let arr = downcast_array::(array, "Date64Array")?; - let (s, us) = split_epoch(arr.value(index), 1_000); - create_datetime_immutable(s, us) - } - DataType::Timestamp(TimeUnit::Second, _) => { - let arr = downcast_array::(array, "TimestampSecondArray")?; - create_datetime_immutable(arr.value(index), 0) - } - DataType::Timestamp(TimeUnit::Millisecond, _) => { - let arr = - downcast_array::(array, "TimestampMillisecondArray")?; - let (s, us) = split_epoch(arr.value(index), 1_000); - create_datetime_immutable(s, us) - } - DataType::Timestamp(TimeUnit::Microsecond, _) => { - let arr = - downcast_array::(array, "TimestampMicrosecondArray")?; - let (s, us) = split_epoch(arr.value(index), 1_000_000); - create_datetime_immutable(s, us) - } - DataType::Timestamp(TimeUnit::Nanosecond, _) => { - let arr = - downcast_array::(array, "TimestampNanosecondArray")?; - let (s, us) = split_epoch(arr.value(index).div_euclid(1_000), 1_000_000); - create_datetime_immutable(s, us) - } - DataType::Time32(TimeUnit::Second) => { - let arr = downcast_array::(array, "Time32SecondArray")?; - create_date_interval(arr.value(index) as i64 * 1_000_000) - } - DataType::Time32(TimeUnit::Millisecond) => { - let arr = downcast_array::(array, "Time32MillisecondArray")?; - create_date_interval(arr.value(index) as i64 * 1000) - } - DataType::Time64(TimeUnit::Microsecond) => { - let arr = downcast_array::(array, "Time64MicrosecondArray")?; - create_date_interval(arr.value(index)) - } - DataType::Time64(TimeUnit::Nanosecond) => { - let arr = downcast_array::(array, "Time64NanosecondArray")?; - create_date_interval(arr.value(index).div_euclid(1_000)) - } - DataType::Decimal128(precision, scale) => { - let arr = downcast_array::(array, "Decimal128Array")?; - decimal_to_zval::(arr.value(index), *precision, *scale) - } - DataType::Decimal256(precision, scale) => { - let arr = downcast_array::(array, "Decimal256Array")?; - decimal_to_zval::(arr.value(index), *precision, *scale) - } - DataType::List(element_field) => { - let list_array = downcast_array::(array, "ListArray")?; - let element_is_uuid = is_uuid_field(Some(element_field)); - let values = list_array.value(index); - let mut ht = ZendHashTable::with_capacity(values.len() as u32); - for j in 0..values.len() { - let child_zv = convert_single_arrow_value(values.as_ref(), j, element_is_uuid)?; - ht.push(child_zv) - .map_err(|_| parquet_exception("Failed to build list element"))?; - } - let mut zv = Zval::new(); - zv.set_hashtable(ht); - Ok(zv) - } - DataType::LargeList(element_field) => { - let list_array = downcast_array::(array, "LargeListArray")?; - let element_is_uuid = is_uuid_field(Some(element_field)); - let values = list_array.value(index); - let mut ht = ZendHashTable::with_capacity(values.len() as u32); - for j in 0..values.len() { - let child_zv = convert_single_arrow_value(values.as_ref(), j, element_is_uuid)?; - ht.push(child_zv) - .map_err(|_| parquet_exception("Failed to build list element"))?; - } - let mut zv = Zval::new(); - zv.set_hashtable(ht); - Ok(zv) - } - DataType::Struct(fields) => { - let struct_array = downcast_array::(array, "StructArray")?; - let mut ht = ZendHashTable::with_capacity(fields.len() as u32); - for (col_idx, field) in fields.iter().enumerate() { - let child_array = struct_array.column(col_idx); - let child_zv = convert_single_arrow_value( - child_array.as_ref(), - index, - is_uuid_field(Some(field)), - )?; - ht.insert(field.name().as_str(), child_zv) - .map_err(|_| parquet_exception("Failed to build struct field"))?; - } - let mut zv = Zval::new(); - zv.set_hashtable(ht); - Ok(zv) - } - DataType::Map(entries_field, _) => { - let map_array = downcast_array::(array, "MapArray")?; - let (key_is_uuid, val_is_uuid) = match entries_field.data_type() { - DataType::Struct(entry_fields) if entry_fields.len() == 2 => ( - is_uuid_field(Some(&entry_fields[0])), - is_uuid_field(Some(&entry_fields[1])), - ), - _ => (false, false), - }; - let entries = map_array.value(index); - let keys_array = entries.column(0); - let values_array = entries.column(1); - let mut ht = ZendHashTable::with_capacity(entries.len() as u32); - for j in 0..entries.len() { - let key_zv = convert_single_arrow_value(keys_array.as_ref(), j, key_is_uuid)?; - let val_zv = convert_single_arrow_value(values_array.as_ref(), j, val_is_uuid)?; - let key_str = if let Some(s) = key_zv.str() { - s.to_string() - } else if let Some(l) = key_zv.long() { - l.to_string() - } else { - j.to_string() - }; - ht.insert(key_str.as_str(), val_zv) - .map_err(|_| parquet_exception("Failed to build map entry"))?; - } - let mut zv = Zval::new(); - zv.set_hashtable(ht); - Ok(zv) - } - other => Err(parquet_exception(format!( - "Unsupported Arrow type in nested conversion: {:?}", - other - ))), - } -} - -pub fn arrow_array_to_php_values(array: &dyn Array, field: Option<&Field>) -> PhpResult> { - match array.data_type() { - DataType::Boolean => { - let arr = downcast_array::(array, "BooleanArray")?; - let len = arr.len(); - let mut result = Vec::with_capacity(len); - for i in 0..len { - let mut zv = Zval::new(); - if arr.is_null(i) { - zv.set_null(); - } else { - zv.set_bool(arr.value(i)); - } - result.push(zv); - } - Ok(result) - } - - DataType::Int8 => { - let arr = downcast_array::(array, "Int8Array")?; - convert_primitive_values(arr, |v| { - let mut zv = Zval::new(); - zv.set_long(v as i64); - Ok(zv) - }) - } - DataType::Int16 => { - let arr = downcast_array::(array, "Int16Array")?; - convert_primitive_values(arr, |v| { - let mut zv = Zval::new(); - zv.set_long(v as i64); - Ok(zv) - }) - } - DataType::Int32 => { - let arr = downcast_array::(array, "Int32Array")?; - convert_primitive_values(arr, |v| { - let mut zv = Zval::new(); - zv.set_long(v as i64); - Ok(zv) - }) - } - DataType::Int64 => { - let arr = downcast_array::(array, "Int64Array")?; - convert_primitive_values(arr, |v| { - let mut zv = Zval::new(); - zv.set_long(v); - Ok(zv) - }) - } - DataType::UInt8 => { - let arr = downcast_array::(array, "UInt8Array")?; - convert_primitive_values(arr, |v| { - let mut zv = Zval::new(); - zv.set_long(v as i64); - Ok(zv) - }) - } - DataType::UInt16 => { - let arr = downcast_array::(array, "UInt16Array")?; - convert_primitive_values(arr, |v| { - let mut zv = Zval::new(); - zv.set_long(v as i64); - Ok(zv) - }) - } - DataType::UInt32 => { - let arr = downcast_array::(array, "UInt32Array")?; - convert_primitive_values(arr, |v| { - let mut zv = Zval::new(); - zv.set_long(v as i64); - Ok(zv) - }) - } - - DataType::UInt64 => { - let arr = downcast_array::(array, "UInt64Array")?; - convert_primitive_values(arr, |val| { - let mut zv = Zval::new(); - if val > i64::MAX as u64 { - zv.set_string(&val.to_string(), false) - .map_err(|_| parquet_exception("Failed to set uint64 string value"))?; - } else { - zv.set_long(val as i64); - } - Ok(zv) - }) - } - - DataType::Float32 => { - let arr = downcast_array::(array, "Float32Array")?; - convert_primitive_values(arr, |v| { - let mut zv = Zval::new(); - zv.set_double(v as f64); - Ok(zv) - }) - } - DataType::Float64 => { - let arr = downcast_array::(array, "Float64Array")?; - convert_primitive_values(arr, |v| { - let mut zv = Zval::new(); - zv.set_double(v); - Ok(zv) - }) - } - - DataType::Utf8 => { - let arr = downcast_array::(array, "StringArray")?; - convert_string_values(arr) - } - DataType::LargeUtf8 => { - let arr = downcast_array::(array, "LargeStringArray")?; - convert_string_values(arr) - } - - DataType::Binary => { - let arr = downcast_array::(array, "BinaryArray")?; - Ok(convert_binary_values(arr)) - } - DataType::LargeBinary => { - let arr = downcast_array::(array, "LargeBinaryArray")?; - Ok(convert_binary_values(arr)) - } - - DataType::FixedSizeBinary(_) => { - let arr = downcast_array::(array, "FixedSizeBinaryArray")?; - let is_uuid = is_uuid_field(field); - let len = arr.len(); - let mut result = Vec::with_capacity(len); - for i in 0..len { - let mut zv = Zval::new(); - if arr.is_null(i) { - zv.set_null(); - } else if is_uuid && arr.value(i).len() == 16 { - zv.set_string(&uuid_to_string(arr.value(i)), false) - .map_err(|_| parquet_exception("Failed to set UUID string value"))?; - } else { - zv.set_binary(arr.value(i).to_vec()); - } - result.push(zv); - } - Ok(result) - } - - DataType::Date32 => { - let arr = downcast_array::(array, "Date32Array")?; - convert_primitive_values(arr, |v| create_datetime_immutable(v as i64 * 86400, 0)) - } - DataType::Date64 => { - let arr = downcast_array::(array, "Date64Array")?; - convert_primitive_values(arr, |v| { - let (s, us) = split_epoch(v, 1_000); - create_datetime_immutable(s, us) - }) - } - - DataType::Timestamp(TimeUnit::Second, _) => { - let arr = downcast_array::(array, "TimestampSecondArray")?; - convert_primitive_values(arr, |v| create_datetime_immutable(v, 0)) - } - DataType::Timestamp(TimeUnit::Millisecond, _) => { - let arr = - downcast_array::(array, "TimestampMillisecondArray")?; - convert_primitive_values(arr, |v| { - let (s, us) = split_epoch(v, 1_000); - create_datetime_immutable(s, us) - }) - } - DataType::Timestamp(TimeUnit::Microsecond, _) => { - let arr = - downcast_array::(array, "TimestampMicrosecondArray")?; - convert_primitive_values(arr, |v| { - let (s, us) = split_epoch(v, 1_000_000); - create_datetime_immutable(s, us) - }) - } - DataType::Timestamp(TimeUnit::Nanosecond, _) => { - let arr = - downcast_array::(array, "TimestampNanosecondArray")?; - convert_primitive_values(arr, |v| { - let (s, us) = split_epoch(v.div_euclid(1_000), 1_000_000); - create_datetime_immutable(s, us) - }) - } - - DataType::Time32(TimeUnit::Second) => { - let arr = downcast_array::(array, "Time32SecondArray")?; - convert_primitive_values(arr, |v| create_date_interval(v as i64 * 1_000_000)) - } - DataType::Time32(TimeUnit::Millisecond) => { - let arr = downcast_array::(array, "Time32MillisecondArray")?; - convert_primitive_values(arr, |v| create_date_interval(v as i64 * 1000)) - } - DataType::Time64(TimeUnit::Microsecond) => { - let arr = downcast_array::(array, "Time64MicrosecondArray")?; - convert_primitive_values(arr, create_date_interval) - } - DataType::Time64(TimeUnit::Nanosecond) => { - let arr = downcast_array::(array, "Time64NanosecondArray")?; - convert_primitive_values(arr, |v| create_date_interval(v.div_euclid(1_000))) - } - - DataType::Decimal128(precision, scale) => { - let arr = downcast_array::(array, "Decimal128Array")?; - convert_primitive_values(arr, |v| { - decimal_to_zval::(v, *precision, *scale) - }) - } - - DataType::Decimal256(precision, scale) => { - let arr = downcast_array::(array, "Decimal256Array")?; - convert_primitive_values(arr, |v| { - decimal_to_zval::(v, *precision, *scale) - }) - } - - DataType::List(_) | DataType::LargeList(_) | DataType::Struct(_) | DataType::Map(_, _) => { - let len = array.len(); - let is_uuid = is_uuid_field(field); - let mut result = Vec::with_capacity(len); - for i in 0..len { - result.push(convert_single_arrow_value(array, i, is_uuid)?); - } - Ok(result) - } - - other => Err(parquet_exception(format!( - "Unsupported Arrow array type: {:?}", - other - ))), - } -} - -fn zval_type_name(zv: &Zval) -> &'static str { - if zv.is_null() { - "null" - } else if zv.is_bool() { - "bool" - } else if zv.is_long() { - "int" - } else if zv.is_double() { - "float" - } else if zv.is_string() { - "string" - } else if zv.is_array() { - "array" - } else if zv.is_object() { - "object" - } else { - "unknown" - } -} - -fn round_decimal_half_away(repr: &str, scale: i8) -> String { - let (sign, digits) = match repr.strip_prefix('-') { - Some(stripped) => ("-", stripped), - None => ("", repr), - }; - let (integer_part, fractional_part) = match digits.find('.') { - Some(dot_pos) => (&digits[..dot_pos], &digits[dot_pos + 1..]), - None => (digits, ""), - }; - let scale = scale.max(0) as usize; - - if fractional_part.len() <= scale { - return format!( - "{}{}.{:0 = format!("{}{}", integer_part, &fractional_part[..scale]).into_bytes(); - - if fractional_part.as_bytes()[scale] >= b'5' { - let mut i = kept.len(); - loop { - if i == 0 { - kept.insert(0, b'1'); - break; - } - i -= 1; - if kept[i] == b'9' { - kept[i] = b'0'; - } else { - kept[i] += 1; - break; - } - } - } - - let kept = String::from_utf8(kept).unwrap_or_default(); - let split = kept.len() - scale; - - format!("{}{}.{}", sign, &kept[..split], &kept[split..]) -} - -fn parse_decimal_string(s: &str, scale: i8) -> Result { - let trimmed = s.trim(); - if trimmed.is_empty() { - return Err("Empty decimal string".into()); - } - - let (negative, digits_str) = if let Some(stripped) = trimmed.strip_prefix('-') { - (true, stripped) - } else if let Some(stripped) = trimmed.strip_prefix('+') { - (false, stripped) - } else { - (false, trimmed) - }; - - let (integer_part, fractional_part) = if let Some(dot_pos) = digits_str.find('.') { - (&digits_str[..dot_pos], &digits_str[dot_pos + 1..]) - } else { - (digits_str, "") - }; - - if !integer_part.chars().all(|c| c.is_ascii_digit()) { - return Err(format!("Invalid decimal integer part: '{}'", integer_part)); - } - if !fractional_part.chars().all(|c| c.is_ascii_digit()) { - return Err(format!( - "Invalid decimal fractional part: '{}'", - fractional_part - )); - } - - let scale_usize = scale as usize; - let adjusted_frac = if fractional_part.len() >= scale_usize { - fractional_part[..scale_usize].to_string() - } else { - format!("{:0 {{ - let mut builder = <$builder_type>::with_capacity($values.len()); - for zv in $values { - if zv.is_null() { - builder.append_null(); - } else if let Some(v) = zv.long() { - let converted = <$cast_type>::try_from(v).map_err(|_| { - format!( - "Column '{}': value {} out of range for {} (valid range: {} to {})", - $column_name, - v, - stringify!($cast_type), - <$cast_type>::MIN, - <$cast_type>::MAX - ) - })?; - builder.append_value(converted); - } else { - return Err(format!( - "Column '{}': expected int, got {}", - $column_name, - zval_type_name(zv) - )); - } - } - Ok(Arc::new(builder.finish()) as ArrayRef) - }}; -} - -macro_rules! build_float_array { - ($values:expr, $column_name:expr, $builder_type:ty, $cast_type:ty) => {{ - let mut builder = <$builder_type>::with_capacity($values.len()); - for zv in $values { - if zv.is_null() { - builder.append_null(); - } else if let Some(v) = zv.double() { - let converted = v as $cast_type; - if converted.is_infinite() && !v.is_infinite() { - return Err(format!( - "Column '{}': value {} overflows {}", - $column_name, - v, - stringify!($cast_type) - )); - } - builder.append_value(converted); - } else if let Some(v) = zv.long() { - builder.append_value(v as $cast_type); - } else { - return Err(format!( - "Column '{}': expected float or int, got {}", - $column_name, - zval_type_name(zv) - )); - } - } - Ok(Arc::new(builder.finish()) as ArrayRef) - }}; -} - -fn parse_time_unit(entry_ht: &ZendHashTable, name: &str) -> Result { - match entry_ht.get("unit").and_then(|z| z.str()) { - None | Some("MICROS") => Ok(TimeUnit::Microsecond), - Some("MILLIS") => Ok(TimeUnit::Millisecond), - Some("NANOS") => Ok(TimeUnit::Nanosecond), - Some(s) => Err(format!( - "Column '{}': unsupported unit '{}', expected MILLIS, MICROS or NANOS", - name, s - )), - } -} - -fn php_schema_entry_to_field(entry_ht: &ZendHashTable) -> Result { - let name = entry_ht - .get("name") - .and_then(|z| z.str()) - .ok_or_else(|| "Schema entry must have a 'name' string key".to_string())? - .to_string(); - - let type_str = entry_ht - .get("type") - .and_then(|z| z.str()) - .ok_or_else(|| "Schema entry must have a 'type' string key".to_string())? - .to_string(); - - let nullable = entry_ht - .get("optional") - .and_then(|z| z.bool()) - .unwrap_or(true); - - let data_type = match type_str.as_str() { - "BOOLEAN" => DataType::Boolean, - "INT8" => DataType::Int8, - "INT16" => DataType::Int16, - "INT32" => DataType::Int32, - "INT64" => DataType::Int64, - "UINT8" => DataType::UInt8, - "UINT16" => DataType::UInt16, - "UINT32" => DataType::UInt32, - "UINT64" => DataType::UInt64, - "FLOAT" => DataType::Float32, - "DOUBLE" => DataType::Float64, - "STRING" => DataType::Utf8, - "BINARY" => DataType::Binary, - "DATE" => DataType::Date32, - "TIMESTAMP" => { - let unit = parse_time_unit(entry_ht, &name)?; - let utc = entry_ht.get("utc").and_then(|z| z.bool()).unwrap_or(true); - DataType::Timestamp(unit, if utc { Some("UTC".into()) } else { None }) - } - "TIME" => match parse_time_unit(entry_ht, &name)? { - TimeUnit::Millisecond => DataType::Time32(TimeUnit::Millisecond), - unit => DataType::Time64(unit), - }, - "DECIMAL" => { - let precision = entry_ht - .get("precision") - .and_then(|z| z.long()) - .ok_or_else(|| { - format!( - "Column '{}': DECIMAL type requires 'precision' integer key", - name - ) - })? as u8; - let scale = entry_ht - .get("scale") - .and_then(|z| z.long()) - .ok_or_else(|| { - format!( - "Column '{}': DECIMAL type requires 'scale' integer key", - name - ) - })? as i8; - DataType::Decimal128(precision, scale) - } - "UUID" => { - let mut field = Field::new(&name, DataType::FixedSizeBinary(16), nullable); - field.try_with_extension_type(Uuid).map_err(|e| { - format!( - "Column '{}': failed to set UUID extension type: {}", - name, e - ) - })?; - return Ok(field); - } - "JSON" => { - let mut field = Field::new(&name, DataType::Utf8, nullable); - field - .try_with_extension_type(Json::default()) - .map_err(|e| { - format!( - "Column '{}': failed to set JSON extension type: {}", - name, e - ) - })?; - return Ok(field); - } - "FIXED_SIZE_BINARY" => { - let length = entry_ht - .get("length") - .and_then(|z| z.long()) - .ok_or_else(|| { - format!( - "Column '{}': FIXED_SIZE_BINARY type requires 'length' integer key", - name - ) - })? as i32; - DataType::FixedSizeBinary(length) - } - "LIST" => { - let children = entry_ht - .get("children") - .and_then(|z| z.array()) - .ok_or_else(|| format!("Column '{}': LIST type requires 'children' array", name))?; - let child_zv = children.values().next().ok_or_else(|| { - format!("Column '{}': LIST type requires exactly one child", name) - })?; - let child_ht = child_zv - .array() - .ok_or_else(|| format!("Column '{}': LIST child must be an array", name))?; - let child_field = php_schema_entry_to_field(child_ht)?; - DataType::List(Arc::new(child_field)) - } - "STRUCT" => { - let children = entry_ht - .get("children") - .and_then(|z| z.array()) - .ok_or_else(|| { - format!("Column '{}': STRUCT type requires 'children' array", name) - })?; - let fields: Result, String> = children - .values() - .map(|child_zv| { - let child_ht = child_zv.array().ok_or_else(|| { - format!("Column '{}': STRUCT child must be an array", name) - })?; - php_schema_entry_to_field(child_ht) - }) - .collect(); - DataType::Struct(Fields::from(fields?)) - } - "MAP" => { - let children = entry_ht - .get("children") - .and_then(|z| z.array()) - .ok_or_else(|| format!("Column '{}': MAP type requires 'children' array", name))?; - let mut child_iter = children.values(); - let key_zv = child_iter.next().ok_or_else(|| { - format!( - "Column '{}': MAP type requires key and value children", - name - ) - })?; - let val_zv = child_iter.next().ok_or_else(|| { - format!( - "Column '{}': MAP type requires key and value children", - name - ) - })?; - let key_field = - php_schema_entry_to_field(key_zv.array().ok_or_else(|| { - format!("Column '{}': MAP key child must be an array", name) - })?)?; - let val_field = - php_schema_entry_to_field(val_zv.array().ok_or_else(|| { - format!("Column '{}': MAP value child must be an array", name) - })?)?; - let entries_struct = DataType::Struct(Fields::from(vec![key_field, val_field])); - DataType::Map( - Arc::new(Field::new("entries", entries_struct, false)), - false, - ) - } - other => { - return Err(format!("Column '{}': Unsupported type '{}'", name, other)); - } - }; - - Ok(Field::new(&name, data_type, nullable)) -} - -pub fn php_schema_to_arrow(schema: &ZendHashTable) -> Result { - if schema.is_empty() { - return Err("Schema must have at least one column".into()); - } - - let mut fields = Vec::with_capacity(schema.len()); - - for (_key, zv) in schema.iter() { - let entry_ht = zv - .array() - .ok_or_else(|| "Schema entry must be an array".to_string())?; - fields.push(php_schema_entry_to_field(entry_ht)?); - } - - Ok(Schema::new(fields)) -} - -pub fn php_array_to_arrow( - values: &[&Zval], - data_type: &DataType, - column_name: &str, -) -> Result { - match data_type { - DataType::Boolean => { - let mut builder = BooleanBuilder::with_capacity(values.len()); - for zv in values { - if zv.is_null() { - builder.append_null(); - } else if let Some(v) = zv.bool() { - builder.append_value(v); - } else { - return Err(format!( - "Column '{}': expected bool, got {}", - column_name, - zval_type_name(zv) - )); - } - } - Ok(Arc::new(builder.finish()) as ArrayRef) - } - DataType::Int8 => build_int_array!(values, column_name, Int8Builder, i8), - DataType::Int16 => build_int_array!(values, column_name, Int16Builder, i16), - DataType::Int32 => build_int_array!(values, column_name, Int32Builder, i32), - DataType::Int64 => build_int_array!(values, column_name, Int64Builder, i64), - DataType::UInt8 => build_int_array!(values, column_name, UInt8Builder, u8), - DataType::UInt16 => build_int_array!(values, column_name, UInt16Builder, u16), - DataType::UInt32 => build_int_array!(values, column_name, UInt32Builder, u32), - DataType::UInt64 => { - let mut builder = UInt64Builder::with_capacity(values.len()); - for zv in values { - if zv.is_null() { - builder.append_null(); - } else if let Some(v) = zv.long() { - let converted = - u64::try_from(v).map_err(|_| { - format!( - "Column '{}': value {} out of range for u64 (valid range: {} to {})", - column_name, v, u64::MIN, u64::MAX - ) - })?; - builder.append_value(converted); - } else if let Some(s) = zv.str() { - let parsed: u64 = s.parse().map_err(|e| { - format!( - "Column '{}': failed to parse '{}' as u64: {}", - column_name, s, e - ) - })?; - builder.append_value(parsed); - } else { - return Err(format!( - "Column '{}': expected int or string, got {}", - column_name, - zval_type_name(zv) - )); - } - } - Ok(Arc::new(builder.finish()) as ArrayRef) - } - DataType::Float32 => build_float_array!(values, column_name, Float32Builder, f32), - DataType::Float64 => build_float_array!(values, column_name, Float64Builder, f64), - DataType::Utf8 => { - let mut builder = StringBuilder::with_capacity(values.len(), values.len() * 32); - for zv in values { - if zv.is_null() { - builder.append_null(); - } else if let Some(s) = zv.str() { - builder.append_value(s); - } else { - return Err(format!( - "Column '{}': expected string, got {}", - column_name, - zval_type_name(zv) - )); - } - } - Ok(Arc::new(builder.finish()) as ArrayRef) - } - DataType::Binary => { - let mut builder = BinaryBuilder::with_capacity(values.len(), values.len() * 32); - for zv in values { - if zv.is_null() { - builder.append_null(); - } else if let Some(zend_str) = zv.zend_str() { - builder.append_value(zend_str.as_bytes()); - } else { - return Err(format!( - "Column '{}': expected string (binary), got {}", - column_name, - zval_type_name(zv) - )); - } - } - Ok(Arc::new(builder.finish()) as ArrayRef) - } - DataType::FixedSizeBinary(n) => { - let mut builder = FixedSizeBinaryBuilder::with_capacity(values.len(), *n); - for zv in values { - if zv.is_null() { - builder.append_null(); - } else if let Some(zend_str) = zv.zend_str() { - let bytes = zend_str.as_bytes(); - if bytes.len() == *n as usize { - builder - .append_value(bytes) - .map_err(|e| format!("Column '{}': {}", column_name, e))?; - } else if *n == 16 && (bytes.len() == 36 || bytes.len() == 32) { - let hex: String = bytes - .iter() - .filter(|b| **b != b'-') - .map(|b| *b as char) - .collect(); - if hex.len() != 32 { - return Err(format!("Column '{}': invalid UUID string", column_name)); - } - let mut uuid_bytes = [0u8; 16]; - for i in 0..16 { - uuid_bytes[i] = u8::from_str_radix(&hex[i * 2..i * 2 + 2], 16) - .map_err(|e| { - format!("Column '{}': invalid UUID hex: {}", column_name, e) - })?; - } - builder - .append_value(uuid_bytes) - .map_err(|e| format!("Column '{}': {}", column_name, e))?; - } else { - return Err(format!( - "Column '{}': expected {} bytes for FIXED_SIZE_BINARY, got {}", - column_name, - n, - bytes.len() - )); - } - } else { - return Err(format!( - "Column '{}': expected string (binary), got {}", - column_name, - zval_type_name(zv) - )); - } - } - Ok(Arc::new(builder.finish()) as ArrayRef) - } - DataType::Date32 => { - let mut builder = Date32Builder::with_capacity(values.len()); - for zv in values { - if zv.is_null() { - builder.append_null(); - } else if let Some(v) = zv.long() { - builder.append_value(i32::try_from(v).map_err(|_| { - format!( - "Column '{}': value {} out of range for DATE days", - column_name, v - ) - })?); - } else if zv.is_object() { - let (seconds, _) = extract_datetime_timestamp(zv, column_name)?; - let offset = extract_datetime_offset(zv, column_name)?; - builder.append_value((seconds + offset).div_euclid(86_400) as i32); - } else { - return Err(format!( - "Column '{}': expected DateTimeInterface or int, got {}", - column_name, - zval_type_name(zv) - )); - } - } - Ok(Arc::new(builder.finish()) as ArrayRef) - } - DataType::Timestamp(unit, tz) => { - let mut result: Vec> = Vec::with_capacity(values.len()); - for zv in values { - if zv.is_null() { - result.push(None); - } else if let Some(v) = zv.long() { - result.push(Some(v)); - } else if zv.is_object() { - let (seconds, microseconds) = extract_datetime_timestamp(zv, column_name)?; - let total = seconds * 1_000_000 + microseconds; - result.push(Some(match unit { - TimeUnit::Second => total.div_euclid(1_000_000), - TimeUnit::Millisecond => total.div_euclid(1_000), - TimeUnit::Microsecond => total, - TimeUnit::Nanosecond => total.checked_mul(1_000).ok_or_else(|| { - format!( - "Column '{}': DateTime outside the TIMESTAMP(NANOS) range", - column_name - ) - })?, - })); - } else { - return Err(format!( - "Column '{}': expected DateTimeInterface or int, got {}", - column_name, - zval_type_name(zv) - )); - } - } - Ok(match unit { - TimeUnit::Second => { - Arc::new(TimestampSecondArray::from(result).with_timezone_opt(tz.clone())) - as ArrayRef - } - TimeUnit::Millisecond => { - Arc::new(TimestampMillisecondArray::from(result).with_timezone_opt(tz.clone())) - as ArrayRef - } - TimeUnit::Microsecond => { - Arc::new(TimestampMicrosecondArray::from(result).with_timezone_opt(tz.clone())) - as ArrayRef - } - TimeUnit::Nanosecond => { - Arc::new(TimestampNanosecondArray::from(result).with_timezone_opt(tz.clone())) - as ArrayRef - } - }) - } - DataType::Time32(TimeUnit::Millisecond) => { - let mut result: Vec> = Vec::with_capacity(values.len()); - for zv in values { - if zv.is_null() { - result.push(None); - } else if let Some(v) = zv.long() { - result.push(Some(i32::try_from(v).map_err(|_| { - format!( - "Column '{}': value {} out of range for TIME(MILLIS)", - column_name, v - ) - })?)); - } else if zv.is_object() { - result.push(Some( - extract_date_interval_microseconds(zv, column_name)?.div_euclid(1_000) - as i32, - )); - } else { - return Err(format!( - "Column '{}': expected DateInterval or int, got {}", - column_name, - zval_type_name(zv) - )); - } - } - Ok(Arc::new(Time32MillisecondArray::from(result)) as ArrayRef) - } - DataType::Time64(unit @ (TimeUnit::Microsecond | TimeUnit::Nanosecond)) => { - let mut result: Vec> = Vec::with_capacity(values.len()); - for zv in values { - if zv.is_null() { - result.push(None); - } else if let Some(v) = zv.long() { - result.push(Some(v)); - } else if zv.is_object() { - let micros = extract_date_interval_microseconds(zv, column_name)?; - result.push(Some(if *unit == TimeUnit::Nanosecond { - micros * 1_000 - } else { - micros - })); - } else { - return Err(format!( - "Column '{}': expected DateInterval or int, got {}", - column_name, - zval_type_name(zv) - )); - } - } - Ok(if *unit == TimeUnit::Nanosecond { - Arc::new(Time64NanosecondArray::from(result)) as ArrayRef - } else { - Arc::new(Time64MicrosecondArray::from(result)) as ArrayRef - }) - } - DataType::Decimal128(p, s) => { - let mut builder = Decimal128Builder::with_capacity(values.len()); - for zv in values { - if zv.is_null() { - builder.append_null(); - } else if let Some(str_val) = zv.str() { - let unscaled = parse_decimal_string(str_val, *s)?; - builder.append_value(unscaled); - } else if let Some(v) = zv.double() { - let str_val = round_decimal_half_away(&format!("{}", v), *s); - let unscaled = parse_decimal_string(&str_val, *s)?; - builder.append_value(unscaled); - } else if let Some(v) = zv.long() { - let str_val = format!("{}.{}", v, "0".repeat(*s as usize)); - let unscaled = parse_decimal_string(&str_val, *s)?; - builder.append_value(unscaled); - } else { - return Err(format!( - "Column '{}': expected string, float, or int for DECIMAL, got {}", - column_name, - zval_type_name(zv) - )); - } - } - let array = builder - .finish() - .with_precision_and_scale(*p, *s) - .map_err(|e| format!("Column '{}': {}", column_name, e))?; - Ok(Arc::new(array) as ArrayRef) - } - DataType::List(child_field) => { - let mut offsets = vec![0i32]; - let mut null_bits: Vec = Vec::with_capacity(values.len()); - let mut child_refs: Vec<&Zval> = Vec::new(); - - for zv in values { - if zv.is_null() { - null_bits.push(false); - offsets.push(*offsets.last().unwrap()); - } else { - null_bits.push(true); - let arr = zv.array().ok_or_else(|| { - format!( - "Column '{}': expected array for LIST, got {}", - column_name, - zval_type_name(zv) - ) - })?; - for (_key, elem) in arr.iter() { - child_refs.push(elem); - } - offsets.push(child_refs.len() as i32); - } - } - - let child_array = php_array_to_arrow( - &child_refs, - child_field.data_type(), - &format!("{}.item", column_name), - )?; - - let offsets_buffer = OffsetBuffer::new(ScalarBuffer::from(offsets)); - let null_buffer = NullBuffer::from(null_bits); - - Ok(Arc::new( - ListArray::try_new( - child_field.clone(), - offsets_buffer, - child_array, - Some(null_buffer), - ) - .map_err(|e| format!("Column '{}': {}", column_name, e))?, - ) as ArrayRef) - } - DataType::Struct(fields) => { - let null_zval = Zval::new(); - - for (row_idx, zv) in values.iter().enumerate() { - if !zv.is_null() { - let arr = zv.array().ok_or_else(|| { - format!( - "Column '{}': expected array for STRUCT at row {}, got {}", - column_name, - row_idx, - zval_type_name(zv) - ) - })?; - for field in fields.iter() { - if !field.is_nullable() { - let child_val = arr.get(field.name().as_str()); - if child_val.is_none() || child_val.unwrap().is_null() { - return Err(format!( - "Column '{}': non-nullable field '{}' is missing/null at row {}", - column_name, - field.name(), - row_idx - )); - } - } - } - } - } - - let null_bits: Vec = values.iter().map(|zv| !zv.is_null()).collect(); - let null_buffer = NullBuffer::from(null_bits); - - let child_arrays: Result, String> = fields - .iter() - .map(|field| { - let child_refs: Vec<&Zval> = values - .iter() - .map(|zv| { - if zv.is_null() { - &null_zval - } else { - zv.array() - .and_then(|arr| arr.get(field.name().as_str())) - .unwrap_or(&null_zval) - } - }) - .collect(); - - php_array_to_arrow( - &child_refs, - field.data_type(), - &format!("{}.{}", column_name, field.name()), - ) - }) - .collect(); - - Ok(Arc::new( - StructArray::try_new(fields.clone(), child_arrays?, Some(null_buffer)) - .map_err(|e| format!("Column '{}': {}", column_name, e))?, - ) as ArrayRef) - } - DataType::Map(entries_field, _) => { - let (key_field, val_field) = match entries_field.data_type() { - DataType::Struct(fields) if fields.len() == 2 => (&fields[0], &fields[1]), - _ => { - return Err(format!( - "Column '{}': MAP entries must be Struct with key and value", - column_name - )) - } - }; - - let mut offsets = vec![0i32]; - let mut null_bits: Vec = Vec::with_capacity(values.len()); - let mut key_zvals: Vec = Vec::new(); - let mut val_refs: Vec<&Zval> = Vec::new(); - - for zv in values { - if zv.is_null() { - null_bits.push(false); - offsets.push(*offsets.last().unwrap()); - } else { - null_bits.push(true); - let arr = zv.array().ok_or_else(|| { - format!( - "Column '{}': expected array for MAP, got {}", - column_name, - zval_type_name(zv) - ) - })?; - for (key, val) in arr.iter() { - let mut key_zv = Zval::new(); - let set_key = |zv: &mut Zval, s: &str| { - zv.set_string(s, false).map_err(|_| { - format!("Column '{}': failed to set MAP key string", column_name) - }) - }; - match key { - ArrayKey::Long(n) => key_zv.set_long(n), - ArrayKey::String(ref s) => set_key(&mut key_zv, s)?, - ArrayKey::Str(s) => set_key(&mut key_zv, s)?, - ArrayKey::ZendString(zs) => { - let s = zs.as_str().map_err(|_| { - format!( - "Column '{}': failed to read MAP key string", - column_name - ) - })?; - set_key(&mut key_zv, s)?; - } - } - key_zvals.push(key_zv); - val_refs.push(val); - } - offsets.push(val_refs.len() as i32); - } - } - - let key_refs: Vec<&Zval> = key_zvals.iter().collect(); - let key_array = php_array_to_arrow( - &key_refs, - key_field.data_type(), - &format!("{}.key", column_name), - )?; - let val_array = php_array_to_arrow( - &val_refs, - val_field.data_type(), - &format!("{}.value", column_name), - )?; - - let entries_struct = StructArray::try_new( - Fields::from(vec![key_field.clone(), val_field.clone()]), - vec![key_array, val_array], - None, - ) - .map_err(|e| format!("Column '{}': {}", column_name, e))?; - - let offsets_buffer = OffsetBuffer::new(ScalarBuffer::from(offsets)); - let null_buffer = NullBuffer::from(null_bits); - - let map_array = MapArray::try_new( - entries_field.clone(), - offsets_buffer, - entries_struct, - Some(null_buffer), - false, - ) - .map_err(|e| format!("Column '{}': {}", column_name, e))?; - - Ok(Arc::new(map_array) as ArrayRef) - } - _ => Err(format!( - "Column '{}': unsupported data type {:?} in writer", - column_name, data_type - )), - } -} diff --git a/src/extension/arrow-ext/src/parquet/write.rs b/src/extension/arrow-ext/src/parquet/write.rs new file mode 100644 index 0000000000..4eaf234486 --- /dev/null +++ b/src/extension/arrow-ext/src/parquet/write.rs @@ -0,0 +1,55 @@ +//! Canonical arrays to Parquet: each batch cast to the writer schema's types (`to_parquet`) and written by arrow-rs. + +use std::sync::Arc; + +use arrow_array::{ArrayRef, RecordBatch, RecordBatchOptions}; +use arrow_schema::{Schema, SchemaRef}; +use parquet::arrow::ArrowWriter; +use parquet::file::properties::WriterProperties; + +use crate::parquet::canonical::to_parquet; +use crate::parquet::error::Error; +use crate::parquet::sink::PhpSink; + +pub struct ParquetWriter { + writer: ArrowWriter, + schema: SchemaRef, +} + +impl ParquetWriter { + pub fn open(sink: PhpSink, schema: Schema, props: WriterProperties) -> Result { + let schema = Arc::new(schema); + + Ok(Self { + writer: ArrowWriter::try_new(sink, Arc::clone(&schema), Some(props))?, + schema, + }) + } + + pub fn schema(&self) -> &SchemaRef { + &self.schema + } + + /// `columns`: canonical, one per writer schema field, in its order. + pub fn write(&mut self, rows: usize, columns: &[ArrayRef]) -> Result<(), Error> { + let arrays = columns + .iter() + .zip(self.schema.fields()) + .map(|(column, field)| to_parquet(column, field.data_type()).map_err(|e| e.in_column(field.name()))) + .collect::, _>>()?; + + self.writer.write(&RecordBatch::try_new_with_options( + Arc::clone(&self.schema), + arrays, + &RecordBatchOptions::new().with_row_count(Some(rows)), + )?)?; + + Ok(()) + } + + pub fn close(self) -> Result<(), Error> { + self.writer.close()?; + + Ok(()) + } +} diff --git a/src/extension/arrow-ext/src/parquet/writer.rs b/src/extension/arrow-ext/src/parquet/writer.rs deleted file mode 100644 index be26ae0578..0000000000 --- a/src/extension/arrow-ext/src/parquet/writer.rs +++ /dev/null @@ -1,269 +0,0 @@ -use std::str::FromStr; -use std::sync::Arc; - -use arrow_array::RecordBatch; -use arrow_schema::Schema as ArrowSchema; -use ext_php_rs::prelude::*; -use ext_php_rs::types::{ArrayKey, ZendHashTable, ZendObject}; -use parquet::arrow::ArrowWriter; -use parquet::basic::{BrotliLevel, Compression, Encoding, GzipLevel, ZstdLevel}; -use parquet::file::properties::{WriterProperties, WriterVersion}; -use parquet::schema::types::ColumnPath; - -use crate::parquet::exception::parquet_exception; -use crate::parquet::type_converter; -use crate::stream::php_destination_stream::PhpDestinationStream; - -#[php_class] -#[php(name = "Flow\\Arrow\\Parquet\\Writer")] -#[derive(Default)] -pub struct Writer { - writer: Option>, - schema: Option>, -} - -fn parse_compression( - s: &str, - gzip_level: Option, - brotli_level: Option, - zstd_level: Option, -) -> Result { - match s.to_uppercase().as_str() { - "UNCOMPRESSED" => Ok(Compression::UNCOMPRESSED), - "SNAPPY" => Ok(Compression::SNAPPY), - "GZIP" => { - let level = match gzip_level { - Some(l) => GzipLevel::try_new(l) - .map_err(|e| format!("Invalid GZIP compression level: {}", e))?, - None => Default::default(), - }; - Ok(Compression::GZIP(level)) - } - "ZSTD" => { - let level = match zstd_level { - Some(l) => ZstdLevel::try_new(l) - .map_err(|e| format!("Invalid ZSTD compression level: {}", e))?, - None => Default::default(), - }; - Ok(Compression::ZSTD(level)) - } - "LZ4" | "LZ4_RAW" => Ok(Compression::LZ4_RAW), - "BROTLI" => { - let level = match brotli_level { - Some(l) => BrotliLevel::try_new(l) - .map_err(|e| format!("Invalid BROTLI compression level: {}", e))?, - None => Default::default(), - }; - Ok(Compression::BROTLI(level)) - } - other => Err(format!( - "Unsupported compression codec '{}'. Supported: UNCOMPRESSED, SNAPPY, GZIP, ZSTD, LZ4, LZ4_RAW, BROTLI", - other - )), - } -} - -fn parse_encoding(s: &str) -> Result { - Encoding::from_str(&s.to_uppercase()) - .map_err(|e| format!("Unsupported encoding '{}': {}", s, e)) -} - -fn apply_writer_options( - mut builder: parquet::file::properties::WriterPropertiesBuilder, - options: &ZendHashTable, - compression_str: &str, -) -> Result { - let gzip_level = options - .get("GZIP_COMPRESSION_LEVEL") - .and_then(|z| z.long()) - .map(|v| v as u32); - - let brotli_level = options - .get("BROTLI_COMPRESSION_LEVEL") - .and_then(|z| z.long()) - .map(|v| v as u32); - - let zstd_level = options - .get("ZSTD_COMPRESSION_LEVEL") - .and_then(|z| z.long()) - .map(|v| v as i32); - - let codec = parse_compression(compression_str, gzip_level, brotli_level, zstd_level)?; - builder = builder.set_compression(codec); - - if let Some(val) = options.get("ROW_GROUP_SIZE_BYTES").and_then(|z| z.long()) { - builder = builder.set_max_row_group_bytes(Some(val as usize)); - } - - if let Some(val) = options.get("PAGE_SIZE_BYTES").and_then(|z| z.long()) { - builder = builder.set_data_page_size_limit(val as usize); - } - - if let Some(val) = options.get("DICTIONARY_PAGE_SIZE").and_then(|z| z.long()) { - builder = builder.set_dictionary_page_size_limit(val as usize); - } - - if let Some(val) = options.get("WRITER_VERSION").and_then(|z| z.long()) { - let version = match val { - 1 => WriterVersion::PARQUET_1_0, - 2 => WriterVersion::PARQUET_2_0, - other => { - return Err(format!( - "Invalid WRITER_VERSION '{}'. Must be 1 or 2", - other - )) - } - }; - builder = builder.set_writer_version(version); - } - - if let Some(col_comp_zval) = options.get("COLUMNS_COMPRESSIONS") { - if let Some(col_comp_ht) = col_comp_zval.array() { - for (key, val) in col_comp_ht.iter() { - let col_path_str = match key { - ArrayKey::String(ref s) => s.clone(), - ArrayKey::Str(s) => s.to_string(), - _ => continue, - }; - if let Some(codec_str) = val.str() { - let parts: Vec = - col_path_str.split('.').map(|s| s.to_string()).collect(); - let col_path = ColumnPath::new(parts); - let col_codec = - parse_compression(codec_str, gzip_level, brotli_level, zstd_level)?; - builder = builder.set_column_compression(col_path, col_codec); - } - } - } - } - - if let Some(col_enc_zval) = options.get("COLUMNS_ENCODINGS") { - if let Some(col_enc_ht) = col_enc_zval.array() { - for (key, val) in col_enc_ht.iter() { - let col_path_str = match key { - ArrayKey::String(ref s) => s.clone(), - ArrayKey::Str(s) => s.to_string(), - _ => continue, - }; - if let Some(enc_str) = val.str() { - let parts: Vec = - col_path_str.split('.').map(|s| s.to_string()).collect(); - let col_path = ColumnPath::new(parts); - let encoding = parse_encoding(enc_str)?; - builder = builder.set_column_encoding(col_path, encoding); - } - } - } - } - - Ok(builder) -} - -impl Drop for Writer { - fn drop(&mut self) { - if let Some(writer) = self.writer.take() { - let _ = writer.close(); - } - } -} - -#[php_impl] -impl Writer { - pub fn __construct( - stream: &mut ZendObject, - schema: &ZendHashTable, - compression: Option, - options: Option<&ZendHashTable>, - ) -> PhpResult { - let dest = PhpDestinationStream::new(stream).map_err(parquet_exception)?; - - let arrow_schema = - type_converter::php_schema_to_arrow(schema).map_err(parquet_exception)?; - - let compression_str = compression.as_deref().unwrap_or("SNAPPY"); - - let props = if let Some(opts) = options { - if !opts.is_empty() { - let builder = WriterProperties::builder(); - apply_writer_options(builder, opts, compression_str) - .map_err(parquet_exception)? - .build() - } else { - let codec = parse_compression(compression_str, None, None, None) - .map_err(parquet_exception)?; - WriterProperties::builder().set_compression(codec).build() - } - } else { - let codec = - parse_compression(compression_str, None, None, None).map_err(parquet_exception)?; - WriterProperties::builder().set_compression(codec).build() - }; - - let schema_arc = Arc::new(arrow_schema); - - let writer = ArrowWriter::try_new(dest, schema_arc.clone(), Some(props)) - .map_err(|e| parquet_exception(format!("Failed to create Parquet writer: {}", e)))?; - - Ok(Self { - writer: Some(writer), - schema: Some(schema_arc), - }) - } - - #[allow(non_snake_case)] - pub fn writeBatch(&mut self, batch: &ZendHashTable) -> PhpResult<()> { - let schema = self - .schema - .clone() - .ok_or_else(|| parquet_exception("Writer is closed"))?; - - let writer = self - .writer - .as_mut() - .ok_or_else(|| parquet_exception("Writer is closed"))?; - - let mut columns = Vec::with_capacity(schema.fields().len()); - - for field in schema.fields() { - let col_zval = batch.get(field.name().as_str()).ok_or_else(|| { - parquet_exception(format!("Batch is missing column '{}'", field.name())) - })?; - - let php_ht = col_zval.array().ok_or_else(|| { - parquet_exception(format!("Column '{}' must be an array", field.name())) - })?; - - let values: Vec<&ext_php_rs::types::Zval> = php_ht.values().collect(); - - let array = - type_converter::php_array_to_arrow(&values, field.data_type(), field.name()) - .map_err(parquet_exception)?; - - columns.push(array); - } - - let record_batch = RecordBatch::try_new(schema, columns) - .map_err(|e| parquet_exception(format!("Failed to create record batch: {}", e)))?; - - writer - .write(&record_batch) - .map_err(|e| parquet_exception(format!("Failed to write batch: {}", e)))?; - - Ok(()) - } - - pub fn close(&mut self) -> PhpResult<()> { - let writer = self - .writer - .take() - .ok_or_else(|| parquet_exception("Writer is already closed"))?; - - self.schema = None; - - writer - .close() - .map_err(|e| parquet_exception(format!("Failed to close Parquet writer: {}", e)))?; - - Ok(()) - } -} diff --git a/src/extension/arrow-ext/src/php.rs b/src/extension/arrow-ext/src/php.rs new file mode 100644 index 0000000000..f320eb8a90 --- /dev/null +++ b/src/extension/arrow-ext/src/php.rs @@ -0,0 +1,465 @@ +//! PHP-engine plumbing: class-entry/slot lookups, hashtable helpers, calls, and the request-scoped cache of datetime +//! handles, timezones and thrift specs. + +use std::collections::HashMap; +use std::rc::Rc; + +use ext_php_rs::boxed::ZBox; +use ext_php_rs::convert::FromZval; +use ext_php_rs::exception::PhpException; +use ext_php_rs::ffi::{ + _zend_property_info, zend_call_known_function, zend_hash_index_update, zend_hash_str_find, zend_hash_str_update, + zend_ulong, +}; +use ext_php_rs::flags::DataType; +use ext_php_rs::prelude::PhpResult; +use ext_php_rs::types::{ZendHashTable, ZendObject, ZendStr, Zval}; +use ext_php_rs::zend::{ce, ClassEntry, ExecutorGlobals, Function, ModuleGlobal, ModuleGlobals}; + +use crate::exception::ext_exception; +use crate::thrift::Specs; + +/// PHP array-key coercion (`_zend_handle_numeric_str`): parse + canonical +/// re-format equality rejects "+5", "-0", leading zeros and whitespace exactly +/// like the engine. +fn array_key_index(key: &[u8]) -> Option { + let string = std::str::from_utf8(key).ok()?; + let value: i64 = string.parse().ok()?; + + (value.to_string().as_bytes() == key).then_some(value) +} + +/// Inserts with PHP array-assignment semantics; ownership of `value` moves +/// into the table. +pub fn ht_insert(ht: &mut ZendHashTable, key: &[u8], mut value: Zval) { + unsafe { + match array_key_index(key) { + Some(index) => { + zend_hash_index_update(ht, index as u64, std::ptr::from_mut(&mut value)); + } + None => { + zend_hash_str_update(ht, key.as_ptr().cast(), key.len(), std::ptr::from_mut(&mut value)); + } + } + } + + std::mem::forget(value); +} + +pub fn ht_insert_long(ht: &mut ZendHashTable, index: i64, mut value: Zval) { + unsafe { + zend_hash_index_update(ht, index as u64, std::ptr::from_mut(&mut value)); + } + + std::mem::forget(value); +} + +/// `$array[$key]` with PHP array-key coercion. +pub fn ht_get<'a>(ht: &'a ZendHashTable, key: &[u8]) -> Option<&'a Zval> { + match array_key_index(key) { + Some(index) => ht.get_index(index), + None => unsafe { zend_hash_str_find(ht, key.as_ptr().cast(), key.len()).as_ref() }, + } +} + +pub fn zval_str(bytes: &[u8]) -> Zval { + let mut zv = Zval::new(); + zv.set_zend_string(ZendStr::new(bytes, false)); + zv +} + +pub fn zval_long(value: i64) -> Zval { + let mut zv = Zval::new(); + zv.set_long(value); + zv +} + +pub fn find_class(name: &str) -> Result<&'static ClassEntry, PhpException> { + ClassEntry::try_find(name) + .ok_or_else(|| ext_exception(format!("arrow requires class \"{name}\" to be autoloadable"))) +} + +/// Fails when the last engine call left an exception pending, so PHP-level +/// failures are never silently ignored. +pub fn ensure_no_pending_exception(context: &str) -> Result<(), PhpException> { + if ExecutorGlobals::get().exception.is_null() { + return Ok(()); + } + + Err(ext_exception(format!("arrow failed to {context}"))) +} + +/// Resolves a declared property's property-table slot offset; keyed by the +/// plain (unmangled) name for private properties too. +pub fn property_offset(ce: &ClassEntry, name: &str) -> Result { + let properties_info: &ZendHashTable = &ce.properties_info; + + let info = properties_info + .get(name) + .and_then(|zv| unsafe { zv.ptr::<_zend_property_info>() }) + .ok_or_else(|| { + ext_exception(format!( + "arrow failed to resolve property \"{name}\" on class \"{}\"", + ce.name().unwrap_or_default() + )) + })?; + + Ok(unsafe { (*info).offset }) +} + +/// Reads one property with `BP_VAR_R` semantics; `ZendObject::get_property` passes `BP_VAR_W`, which trips the +/// readonly-property guard. +pub fn read_property(obj: &ZendObject, name: &str) -> Result { + let handler = unsafe { obj.handlers.as_ref() } + .and_then(|handlers| handlers.read_property) + .ok_or_else(|| ext_exception("arrow failed to resolve a property read handler"))?; + + let mut name_zstr = ZendStr::new(name, false); + let mut rv = Zval::new(); + + let value = unsafe { + handler( + std::ptr::from_ref(obj).cast_mut(), + std::ptr::from_mut(&mut *name_zstr), + 0, + std::ptr::null_mut(), + std::ptr::from_mut(&mut rv), + ) + .as_ref() + } + .ok_or_else(|| ext_exception(format!("arrow failed to read property \"{name}\"")))?; + ensure_no_pending_exception(&format!("read property \"{name}\""))?; + + Ok(value.shallow_clone()) +} + +pub fn ce_method_ref(ce: &ClassEntry, method: &str) -> Result<&'static Function, PhpException> { + let function = unsafe { + ext_php_rs::ffi::zend_hash_str_find_ptr_lc(&raw const ce.function_table, method.as_ptr().cast(), method.len()) + .cast::() + .as_ref() + }; + + function.ok_or_else(|| { + ext_exception(format!( + "arrow failed to resolve method {}::{method}", + ce.name().unwrap_or_default() + )) + }) +} + +/// `$object->$method(...$args)`; an exception the method throws surfaces as itself. +pub fn call_method(object: &Zval, method: &str, args: &mut [Zval]) -> Result { + let object = expect_object(object, "a method receiver")?; + let ce = unsafe { object.ce.as_ref() }.ok_or_else(|| ext_exception("arrow failed to resolve an object class"))?; + + call_handle_transparent(ce_method_ref(ce, method)?, Some(object), args) +} + +/// Calls a pre-resolved function handle and hands a PHP exception thrown by the callee back to the caller as the +/// exception OBJECT, so the caller can surface it verbatim or replace it with one of its own. Taking it also clears +/// the pending exception, leaving the engine ready for the next call. +pub fn call_handle_catching( + func: &Function, + object: Option<&ZendObject>, + args: &mut [Zval], +) -> Result> { + let mut retval = Zval::new(); + + let (object_ptr, called_scope) = match object { + Some(obj) => (std::ptr::from_ref(obj).cast_mut(), obj.ce), + None => (std::ptr::null_mut(), unsafe { func.common.scope }), + }; + + unsafe { + zend_call_known_function( + std::ptr::from_ref(func).cast_mut(), + object_ptr, + called_scope, + std::ptr::from_mut(&mut retval), + args.len() as u32, + args.as_mut_ptr(), + std::ptr::null_mut(), + ); + } + + match ExecutorGlobals::take_exception() { + Some(exception) => Err(exception), + None => Ok(retval), + } +} + +/// Surfaces a PHP exception object as ITSELF rather than wrapped, so the class and message PHP sees are the ones +/// thrown. +pub fn transparent_exception(exception: &mut ZendObject) -> PhpException { + let mut zv = Zval::new(); + zv.set_object(exception); + + PhpException::from_message(String::new()).with_object(zv) +} + +/// [`call_handle_catching`] with the exception object already re-raised as itself. +pub fn call_handle_transparent( + func: &Function, + object: Option<&ZendObject>, + args: &mut [Zval], +) -> Result { + call_handle_catching(func, object, args).map_err(|mut exception| transparent_exception(&mut exception)) +} + +/// Calls a pre-resolved function handle. Argument zvals stay caller-owned +/// (the engine copies what it keeps), so cached zvals pass as shallow clones. +pub fn call_handle( + func: &Function, + object: Option<&mut ZendObject>, + args: &mut [Zval], + context: &str, +) -> Result { + let mut retval = Zval::new(); + + let (object_ptr, called_scope) = match object { + Some(obj) => { + let scope = obj.ce; + (std::ptr::from_mut(obj), scope) + } + None => (std::ptr::null_mut(), unsafe { func.common.scope }), + }; + + unsafe { + zend_call_known_function( + std::ptr::from_ref(func).cast_mut(), + object_ptr, + called_scope, + std::ptr::from_mut(&mut retval), + args.len() as u32, + args.as_mut_ptr(), + std::ptr::null_mut(), + ); + } + + ensure_no_pending_exception(context)?; + + Ok(retval) +} + +/// `HASH_FLAG_PACKED` from zend_types.h. +const HASH_FLAG_PACKED: u8 = 1 << 2; + +/// Raw order-preserving iteration over packed and bucket layouts, binary-safe +/// for arbitrary keys (ext-php-rs's `iter()` panics on non-UTF-8 string keys). +pub fn ht_for_each<'a>( + ht: &'a ZendHashTable, + mut f: impl FnMut(Option<&'a ZendStr>, zend_ulong, &'a Zval) -> Result<(), PhpException>, +) -> Result<(), PhpException> { + let packed = unsafe { ht.u.v.flags } & HASH_FLAG_PACKED != 0; + + for index in 0..ht.nNumUsed { + if packed { + let value = unsafe { &*ht.__bindgen_anon_1.arPacked.add(index as usize) }; + + if value.get_type() == ext_php_rs::flags::DataType::Undef { + continue; + } + + f(None, zend_ulong::from(index), value)?; + } else { + let bucket = unsafe { &*ht.__bindgen_anon_1.arData.add(index as usize) }; + + if bucket.val.get_type() == ext_php_rs::flags::DataType::Undef { + continue; + } + + let key = unsafe { bucket.key.as_ref() }; + f(key, bucket.h, &bucket.val)?; + } + } + + Ok(()) +} + +pub fn expect_object<'a>(zv: &'a Zval, context: &str) -> Result<&'a ZendObject, PhpException> { + zv.object() + .ok_or_else(|| ext_exception(format!("arrow expected {context} to be an object"))) +} + +/// Request-scoped caches of class entries, function handles and value zvals, reached only through [`with_ctx`]: +/// every function below reads and writes the cache inside the closure and calls PHP (autoload, constructors) +/// outside it, so PHP code re-entering arrow never meets a borrowed `Ctx`. +#[derive(Default)] +pub struct Ctx { + timezone: Option, + datetime_immutable: Option, + timezones: HashMap, Zval>, + /// The `$_TSPEC`s reachable from a root thrift class, by that class. + pub thrift_specs: HashMap>, +} + +#[derive(Clone, Copy)] +pub struct DateTimeFns { + pub ce: &'static ClassEntry, + pub set_timezone: &'static Function, +} + +#[derive(Clone, Copy)] +struct TimezoneFns { + ce: &'static ClassEntry, + construct: &'static Function, +} + +/// A cached handle: read inside the context, resolved outside it on a miss, then stored. +fn cached( + read: impl FnOnce(&Ctx) -> Option, + resolve: impl FnOnce() -> Result, + store: impl FnOnce(&mut Ctx, &T), +) -> Result { + if let Some(value) = with_ctx(|ctx| Ok(read(ctx)))? { + return Ok(value); + } + + let value = resolve()?; + with_ctx(|ctx| { + store(ctx, &value); + Ok(()) + })?; + + Ok(value) +} + +pub fn datetime_immutable() -> Result { + cached( + |ctx| ctx.datetime_immutable, + || { + let ce = find_class("DateTimeImmutable")?; + Ok(DateTimeFns { + ce, + set_timezone: ce_method_ref(ce, "setTimezone")?, + }) + }, + |ctx, fns| ctx.datetime_immutable = Some(*fns), + ) +} + +fn timezone_fns() -> Result { + cached( + |ctx| ctx.timezone, + || { + let ce = find_class("DateTimeZone")?; + Ok(TimezoneFns { + ce, + construct: ce_method_ref(ce, "__construct")?, + }) + }, + |ctx, fns| ctx.timezone = Some(*fns), + ) +} + +/// `new DateTimeZone($name)`, one instance per name. +pub fn timezone(name: &[u8]) -> Result { + cached( + |ctx| ctx.timezones.get(name).map(Zval::shallow_clone), + || { + let fns = timezone_fns()?; + let mut timezone = ZendObject::new(fns.ce); + call_handle( + fns.construct, + Some(&mut timezone), + &mut [zval_str(name)], + "construct DateTimeZone", + )?; + + let mut zv = Zval::new(); + zv.set_object(&mut timezone); + + Ok(zv) + }, + |ctx, zv| { + ctx.timezones.insert(name.to_vec(), zv.shallow_clone()); + }, + ) +} + +/// Per-request state: the `Ctx` cache lives in module globals (per thread on ZTS), is built lazily and is dropped in +/// RSHUTDOWN, while the executor still runs; objects freed after RSHUTDOWN never touch it. +#[derive(Default)] +pub struct ArrowGlobals { + ctx: Option, + closed: bool, +} + +impl ModuleGlobal for ArrowGlobals {} + +pub static GLOBALS: ModuleGlobals = ModuleGlobals::new(); + +pub extern "C" fn request_startup(_type: i32, _module_number: i32) -> i32 { + unsafe { GLOBALS.get_mut() }.closed = false; + + 0 +} + +pub extern "C" fn request_shutdown(_type: i32, _module_number: i32) -> i32 { + let globals = unsafe { GLOBALS.get_mut() }; + globals.ctx = None; + globals.closed = true; + + 0 +} + +/// The only access to `Ctx`. `f` must not call PHP code: PHP re-entering arrow would alias the `&mut Ctx`. +pub fn with_ctx(f: impl FnOnce(&mut Ctx) -> PhpResult) -> PhpResult { + let globals = unsafe { GLOBALS.get_mut() }; + + if globals.closed { + return Err(ext_exception("arrow used after request shutdown")); + } + + f(globals.ctx.get_or_insert_with(Ctx::default)) +} + +/// An `iterable` argument kept as its zval. ext-php-rs's `Iterable` opens a `zend_object_iterator` for a `Traversable` +/// and 0.16 never frees it (no `zend_iterator_dtor`): the iterator and its generator lived until the request ended. +pub struct IterableArg<'a>(pub &'a Zval); + +impl<'a> FromZval<'a> for IterableArg<'a> { + const TYPE: DataType = DataType::Iterable; + + fn from_zval(zval: &'a Zval) -> Option { + (zval.is_array() || zval.is_traversable()).then_some(Self(zval)) + } +} + +/// Every value of `$iterable` in order, keys ignored: an `Iterator` through `rewind()` / `valid()` / `current()` / +/// `next()` as `foreach` does, an `IteratorAggregate` through `getIterator()`. +pub fn for_each_value(iterable: &Zval, mut f: impl FnMut(&Zval) -> PhpResult<()>) -> PhpResult<()> { + if let Some(array) = iterable.array() { + for value in array.values() { + f(value)?; + } + + return Ok(()); + } + + let mut iterator = iterable.shallow_clone(); + + while iterator + .object() + .is_some_and(|object| object.instance_of(ce::aggregate())) + { + iterator = call_method(&iterator, "getIterator", &mut [])?; + } + + if !iterator + .object() + .is_some_and(|object| object.instance_of(ce::iterator())) + { + return Err(ext_exception("arrow expected getIterator() to return a Traversable")); + } + + call_method(&iterator, "rewind", &mut [])?; + + while call_method(&iterator, "valid", &mut [])?.bool().unwrap_or(false) { + f(&call_method(&iterator, "current", &mut [])?)?; + call_method(&iterator, "next", &mut [])?; + } + + Ok(()) +} diff --git a/src/extension/arrow-ext/src/render.rs b/src/extension/arrow-ext/src/render.rs new file mode 100644 index 0000000000..030ec2f9ed --- /dev/null +++ b/src/extension/arrow-ext/src/render.rs @@ -0,0 +1,94 @@ +//! Parquet core refusals rendered as the `Flow\Parquet` exceptions, for the readers and the writer. + +use ext_php_rs::exception::PhpException; + +use crate::parquet::error::Error; +use crate::parquet::source::PhpStream; +use crate::php::{find_class, transparent_exception}; + +const RUNTIME: &str = "Flow\\Parquet\\Exception\\RuntimeException"; +const VALIDATION: &str = "Flow\\Parquet\\Exception\\ValidationException"; +const INVALID_ARGUMENT: &str = "Flow\\Parquet\\Exception\\InvalidArgumentException"; + +pub fn exception(class: &str, message: String) -> PhpException { + match find_class(class) { + Ok(ce) => PhpException::new(message, 0, ce), + Err(e) => e, + } +} + +/// Which Parquet surface refused: the same core refusal names what to do about it per side. +pub enum Side { + Read, + Write, +} + +/// A Parquet core refusal; an exception the stream threw while the core called it surfaces as itself. +pub fn parquet_exception(error: Error, stream: &PhpStream, side: Side) -> PhpException { + if let Some(mut thrown) = stream.thrown() { + return transparent_exception(&mut thrown); + } + + let failure = |message: String| exception(RUNTIME, message); + + match (error, side) { + (Error::Unsupported { column, parquet }, Side::Read) => exception( + RUNTIME, + format!( + "Parquet column \"{column}\" ({parquet}) is not supported by the arrow Parquet reader; read the file \ + with \\Flow\\Parquet\\Reader::php()" + ), + ), + (Error::Unsupported { column, parquet }, Side::Write) => exception( + RUNTIME, + format!( + "Parquet column \"{column}\" ({parquet}) is not supported by the arrow Parquet writer; write it with \ + \\Flow\\Parquet\\Writer::php()" + ), + ), + (Error::Overflow { column, row }, _) => exception( + RUNTIME, + format!("Parquet column \"{column}\" row {row} holds a value out of the range arrow stores it in"), + ), + (Error::InvalidUtf8 { column, row }, _) => exception( + RUNTIME, + format!( + "Parquet column \"{column}\" row {row} holds a string that is not valid UTF-8; Parquet STRING columns \ + require UTF-8" + ), + ), + (Error::Length { column, row, expected }, _) => exception( + RUNTIME, + format!("Parquet column \"{column}\" row {row} holds a value that is not {expected} bytes long"), + ), + ( + Error::Value { + column, + row, + expected, + got, + }, + _, + ) => exception( + VALIDATION, + format!("Column \"{column}\" row {row}: expected {expected}, got {got}"), + ), + (Error::MapKey { column, key }, _) => exception( + INVALID_ARGUMENT, + format!("Map key of Parquet column \"{column}\" must be int or string, got {key}"), + ), + (Error::MissingColumn(name), _) => { + exception(INVALID_ARGUMENT, format!("Parquet file has no column \"{name}\"")) + } + (Error::Options(message), _) => exception(INVALID_ARGUMENT, message), + (Error::NotParquet(message), _) => exception( + INVALID_ARGUMENT, + format!("Given file is not valid Parquet file: {message}"), + ), + (Error::Stream(message), _) => failure(format!("arrow Parquet stream {message}")), + (Error::Parquet(error), Side::Read) => failure(format!("arrow failed to read Parquet: {error}")), + (Error::Arrow(error), Side::Read) => failure(format!("arrow failed to read Parquet: {error}")), + (Error::Parquet(error), Side::Write) => failure(format!("arrow failed to write Parquet: {error}")), + (Error::Arrow(error), Side::Write) => failure(format!("arrow failed to write Parquet: {error}")), + } +} diff --git a/src/extension/arrow-ext/src/stream/mod.rs b/src/extension/arrow-ext/src/stream/mod.rs deleted file mode 100644 index 89278b895d..0000000000 --- a/src/extension/arrow-ext/src/stream/mod.rs +++ /dev/null @@ -1,4 +0,0 @@ -pub mod output_stream; -pub mod php_destination_stream; -pub mod php_source_stream; -pub mod random_access_file; diff --git a/src/extension/arrow-ext/src/stream/output_stream.rs b/src/extension/arrow-ext/src/stream/output_stream.rs deleted file mode 100644 index b2c53a254d..0000000000 --- a/src/extension/arrow-ext/src/stream/output_stream.rs +++ /dev/null @@ -1,22 +0,0 @@ -use ext_php_rs::args::Arg; -use ext_php_rs::builders::{ClassBuilder, FunctionBuilder}; -use ext_php_rs::error::Result; -use ext_php_rs::flags::{ClassFlags, DataType, MethodFlags}; -use ext_php_rs::zend::ClassEntry; - -fn noop(_ce: &'static mut ClassEntry) {} - -pub fn register() -> Result<()> { - ClassBuilder::new("Flow\\Arrow\\OutputStream") - .flags(ClassFlags::Interface) - .registration(noop) - .method( - FunctionBuilder::new_abstract("append") - .arg(Arg::new("data", DataType::String)) - .returns(DataType::Object(Some("self")), false, false), - MethodFlags::Public | MethodFlags::Abstract, - ) - .register()?; - - Ok(()) -} diff --git a/src/extension/arrow-ext/src/stream/php_destination_stream.rs b/src/extension/arrow-ext/src/stream/php_destination_stream.rs deleted file mode 100644 index da8569c012..0000000000 --- a/src/extension/arrow-ext/src/stream/php_destination_stream.rs +++ /dev/null @@ -1,53 +0,0 @@ -use ext_php_rs::boxed::ZBox; -use ext_php_rs::convert::IntoZvalDyn; -use ext_php_rs::types::{ZendObject, Zval}; -use ext_php_rs::zend::ClassEntry; -use std::io::{self, Write}; - -pub struct PhpDestinationStream { - obj: ZBox, -} - -// SAFETY: PhpDestinationStream wraps a PHP ZendObject that belongs to the -// current request thread. In NTS mode PHP is single-threaded; in ZTS mode each -// thread owns its own request context. The parquet crate requires Send+Sync for -// ArrowWriter but never sends across threads in synchronous usage. -unsafe impl Send for PhpDestinationStream {} -unsafe impl Sync for PhpDestinationStream {} - -impl PhpDestinationStream { - pub fn new(obj: &mut ZendObject) -> Result { - let ce = ClassEntry::try_find("Flow\\Arrow\\OutputStream").ok_or_else(|| { - "Interface Flow\\Arrow\\OutputStream not loaded. Did you require the file?".to_string() - })?; - if !obj.instance_of(ce) { - return Err("Destination must implement Flow\\Arrow\\OutputStream".to_string()); - } - - let ptr: *mut ZendObject = obj; - unsafe { (*ptr).gc.refcount += 1 }; - Ok(Self { - obj: unsafe { ZBox::from_raw(ptr) }, - }) - } -} - -impl Write for PhpDestinationStream { - fn write(&mut self, buf: &[u8]) -> io::Result { - let mut data = Zval::new(); - data.set_binary(buf.to_vec()); - - let _result = self - .obj - .try_call_method("append", vec![&data as &dyn IntoZvalDyn]) - .map_err(|e| io::Error::other(format!("Failed to call append(): {:?}", e)))?; - - // OutputStream::append() is all-or-nothing: it returns self on success - // or throws on failure (caught above). No partial write path exists. - Ok(buf.len()) - } - - fn flush(&mut self) -> io::Result<()> { - Ok(()) - } -} diff --git a/src/extension/arrow-ext/src/stream/php_source_stream.rs b/src/extension/arrow-ext/src/stream/php_source_stream.rs deleted file mode 100644 index 5d070a3762..0000000000 --- a/src/extension/arrow-ext/src/stream/php_source_stream.rs +++ /dev/null @@ -1,104 +0,0 @@ -use bytes::Bytes; -use ext_php_rs::boxed::ZBox; -use ext_php_rs::convert::IntoZvalDyn; -use ext_php_rs::types::ZendObject; -use ext_php_rs::zend::ClassEntry; -use parquet::errors::ParquetError; -use parquet::file::reader::{ChunkReader, Length}; -use std::io::Cursor; -use std::sync::Arc; - -struct PhpSourceStreamInner { - obj: ZBox, - len: u64, -} - -// SAFETY: PhpSourceStream wraps a PHP ZendObject that belongs to the current -// request thread. In NTS mode PHP is single-threaded; in ZTS mode each thread -// owns its own request context. The parquet crate requires Send+Sync for -// ChunkReader but never sends across threads in synchronous usage. -unsafe impl Send for PhpSourceStreamInner {} -unsafe impl Sync for PhpSourceStreamInner {} - -#[derive(Clone)] -pub struct PhpSourceStream { - inner: Arc, -} - -impl PhpSourceStream { - pub fn new(obj: &mut ZendObject) -> Result { - let ce = ClassEntry::try_find("Flow\\Arrow\\RandomAccessFile").ok_or_else(|| { - "Interface Flow\\Arrow\\RandomAccessFile not loaded. Did you require the file?" - .to_string() - })?; - if !obj.instance_of(ce) { - return Err("Source must implement Flow\\Arrow\\RandomAccessFile".to_string()); - } - - let ptr: *mut ZendObject = obj; - unsafe { (*ptr).gc.refcount += 1 }; - let owned = unsafe { ZBox::from_raw(ptr) }; - - let zval = owned - .try_call_method("size", vec![]) - .map_err(|e| format!("Failed to call size(): {:?}", e))?; - - if zval.is_null() { - return Err("size() returned null; file size is required for Parquet reading".into()); - } - - let size = zval - .long() - .ok_or_else(|| "size() must return an integer".to_string())?; - - Ok(Self { - inner: Arc::new(PhpSourceStreamInner { - obj: owned, - len: size as u64, - }), - }) - } -} - -impl Length for PhpSourceStream { - fn len(&self) -> u64 { - self.inner.len - } -} - -impl ChunkReader for PhpSourceStream { - type T = Cursor>; - - fn get_read(&self, start: u64) -> parquet::errors::Result { - let length = self.inner.len.saturating_sub(start) as usize; - let bytes = self.get_bytes(start, length)?; - Ok(Cursor::new(bytes.into())) - } - - fn get_bytes(&self, start: u64, length: usize) -> parquet::errors::Result { - if length == 0 { - return Ok(Bytes::new()); - } - - let length_arg = length as i64; - let offset_arg = start as i64; - - let result = self - .inner - .obj - .try_call_method( - "read", - vec![ - &length_arg as &dyn IntoZvalDyn, - &offset_arg as &dyn IntoZvalDyn, - ], - ) - .map_err(|e| ParquetError::General(format!("Failed to call read(): {:?}", e)))?; - - let zend_str = result - .zend_str() - .ok_or_else(|| ParquetError::General("read() must return a string".into()))?; - - Ok(Bytes::copy_from_slice(zend_str.as_bytes())) - } -} diff --git a/src/extension/arrow-ext/src/stream/random_access_file.rs b/src/extension/arrow-ext/src/stream/random_access_file.rs deleted file mode 100644 index cfd7a5f899..0000000000 --- a/src/extension/arrow-ext/src/stream/random_access_file.rs +++ /dev/null @@ -1,27 +0,0 @@ -use ext_php_rs::args::Arg; -use ext_php_rs::builders::{ClassBuilder, FunctionBuilder}; -use ext_php_rs::error::Result; -use ext_php_rs::flags::{ClassFlags, DataType, MethodFlags}; -use ext_php_rs::zend::ClassEntry; - -fn noop(_ce: &'static mut ClassEntry) {} - -pub fn register() -> Result<()> { - ClassBuilder::new("Flow\\Arrow\\RandomAccessFile") - .flags(ClassFlags::Interface) - .registration(noop) - .method( - FunctionBuilder::new_abstract("read") - .arg(Arg::new("length", DataType::Long)) - .arg(Arg::new("offset", DataType::Long)) - .returns(DataType::String, false, false), - MethodFlags::Public | MethodFlags::Abstract, - ) - .method( - FunctionBuilder::new_abstract("size").returns(DataType::Long, false, true), - MethodFlags::Public | MethodFlags::Abstract, - ) - .register()?; - - Ok(()) -} diff --git a/src/extension/arrow-ext/src/thrift.rs b/src/extension/arrow-ext/src/thrift.rs new file mode 100644 index 0000000000..b00d6d31ef --- /dev/null +++ b/src/extension/arrow-ext/src/thrift.rs @@ -0,0 +1,469 @@ +//! Thrift compact-protocol bytes as the PHP objects of the classes the Thrift compiler generates: each class's static +//! `$_TSPEC` says which field id is which property and of which type, as the generated `read()` does. No Parquet +//! knowledge here. + +use std::collections::HashMap; +use std::rc::Rc; + +use ext_php_rs::exception::PhpException; +use ext_php_rs::types::{ArrayKey, ZendHashTable, ZendObject, Zval}; +use ext_php_rs::zend::ClassEntry; + +use crate::exception::ext_exception; +use crate::php::{find_class, property_offset, with_ctx, zval_long, zval_str}; + +/// Nesting deeper than this is refused instead of recursing further. +const DEPTH: usize = 64; + +// the compact protocol's type ids (field headers, list headers) +const BOOLEAN_TRUE: u8 = 1; +const BOOLEAN_FALSE: u8 = 2; +const BYTE: u8 = 3; +const I16: u8 = 4; +const I32: u8 = 5; +const I64: u8 = 6; +const DOUBLE: u8 = 7; +const BINARY: u8 = 8; +const LIST: u8 = 9; +const SET: u8 = 10; +const MAP: u8 = 11; +const STRUCT: u8 = 12; + +/// A `$_TSPEC` field type; a struct is the index of its class in `Specs`. +enum Value { + Bool, + Byte, + I16, + I32, + I64, + Double, + String, + Struct(usize), + List(Box), +} + +impl Value { + /// Whether a field of this type arrives as compact type `wire`; any other is skipped, as the generated `read()`. + fn arrives_as(&self, wire: u8) -> bool { + match self { + Value::Bool => wire == BOOLEAN_TRUE || wire == BOOLEAN_FALSE, + Value::Byte => wire == BYTE, + Value::I16 => wire == I16, + Value::I32 => wire == I32, + Value::I64 => wire == I64, + Value::Double => wire == DOUBLE, + Value::String => wire == BINARY, + Value::Struct(_) => wire == STRUCT, + Value::List(_) => wire == LIST, + } + } +} + +struct Class { + ce: &'static ClassEntry, + /// By field id: the property's slot offset and its type. + fields: Vec>, +} + +/// Every class reachable from a root class, by index. +pub struct Specs { + classes: Vec, +} + +impl Specs { + /// The root class and every class its `$_TSPEC` reaches, root at index 0. + fn build(root: &str) -> Result { + let mut specs = Self { classes: Vec::new() }; + specs.class(root, &mut HashMap::new())?; + + Ok(specs) + } + + fn class(&mut self, name: &str, seen: &mut HashMap) -> Result { + let name = name.trim_start_matches('\\'); + + if let Some(index) = seen.get(name) { + return Ok(*index); + } + + let ce = find_class(name)?; + let index = self.classes.len(); + seen.insert(name.to_string(), index); + self.classes.push(Class { ce, fields: Vec::new() }); + + let spec = ce + .get_static_property::<&ZendHashTable>("_TSPEC") + .ok_or_else(|| ext_exception(format!("arrow requires {name}::$_TSPEC to be an array")))?; + let mut fields: Vec> = Vec::new(); + + for (key, entry) in spec.iter() { + let ArrayKey::Long(id) = key else { + return Err(ext_exception(format!( + "arrow requires {name}::$_TSPEC to be keyed by field id" + ))); + }; + let entry = entry + .array() + .ok_or_else(|| ext_exception(format!("arrow requires {name}::$_TSPEC[{id}] to be an array")))?; + let var = entry + .get("var") + .and_then(Zval::str) + .ok_or_else(|| ext_exception(format!("arrow requires {name}::$_TSPEC[{id}]['var']")))?; + let offset = property_offset(ce, var)?; + let value = self.value(entry, name, seen)?; + let id = usize::try_from(id) + .ok() + .filter(|id| *id <= i16::MAX as usize) + .ok_or_else(|| ext_exception(format!("arrow requires {name}::$_TSPEC field ids in 0..=32767")))?; + + if fields.len() <= id { + fields.resize_with(id + 1, || None); + } + + fields[id] = Some((offset, value)); + } + + self.classes[index].fields = fields; + + Ok(index) + } + + /// A `$_TSPEC` entry's type (`'type'`, with `'class'` for a struct and `'elem'` for a list's element). + fn value( + &mut self, + entry: &ZendHashTable, + class: &str, + seen: &mut HashMap, + ) -> Result { + let unsupported = |what: String| ext_exception(format!("arrow cannot decode {class}::$_TSPEC {what}")); + + // Thrift\Type\TType + Ok(match entry.get("type").and_then(Zval::long) { + Some(2) => Value::Bool, + Some(3) => Value::Byte, + Some(4) => Value::Double, + Some(6) => Value::I16, + Some(8) => Value::I32, + Some(10) => Value::I64, + Some(11) => Value::String, + Some(12) => Value::Struct( + self.class( + entry + .get("class") + .and_then(Zval::str) + .ok_or_else(|| unsupported("struct without a class".to_string()))?, + seen, + )?, + ), + Some(15) => Value::List(Box::new( + self.value( + entry + .get("elem") + .and_then(Zval::array) + .ok_or_else(|| unsupported("list without an element".to_string()))?, + class, + seen, + )?, + )), + other => return Err(unsupported(format!("type {other:?}"))), + }) + } +} + +fn thrift_specs(class: &str) -> Result>, PhpException> { + with_ctx(|ctx| Ok(ctx.thrift_specs.get(class).cloned())) +} + +fn store_thrift_specs(class: &str, specs: &Rc) -> Result<(), PhpException> { + with_ctx(|ctx| { + ctx.thrift_specs.insert(class.to_string(), Rc::clone(specs)); + Ok(()) + }) +} + +/// `class`'s instance decoded from `bytes`; with `field`, only that field of the root is decoded and returned (null +/// when absent), the fields before it skipped and the ones after it never read. STRUCT → object (created without its constructor, public properties set by +/// `var`), LIST → list, BOOL/BYTE/I16/I32/I64/DOUBLE/STRING (binary) → scalars; an unknown id, or a known one of another +/// type, is skipped by the compact skip rules. +pub fn decode(bytes: &[u8], class: &str, field: Option) -> Result { + let specs = match thrift_specs(class)? { + Some(specs) => specs, + None => { + let specs = Rc::new(Specs::build(class)?); + store_thrift_specs(class, &specs)?; + specs + } + }; + let mut reader = Reader { bytes, position: 0 }; + + match field { + None => reader.object(&specs, 0, 0), + Some(field) => reader.field(&specs, field), + } +} + +struct Reader<'a> { + bytes: &'a [u8], + position: usize, +} + +fn truncated() -> PhpException { + ext_exception("arrow failed to decode thrift: the bytes end inside a value") +} + +impl Reader<'_> { + fn byte(&mut self) -> Result { + let byte = *self.bytes.get(self.position).ok_or_else(truncated)?; + self.position += 1; + + Ok(byte) + } + + fn take(&mut self, length: usize) -> Result<&[u8], PhpException> { + let end = self + .position + .checked_add(length) + .filter(|end| *end <= self.bytes.len()) + .ok_or_else(truncated)?; + let taken = &self.bytes[self.position..end]; + self.position = end; + + Ok(taken) + } + + fn varint(&mut self) -> Result { + let mut value = 0u64; + + for shift in (0..70).step_by(7) { + let byte = self.byte()?; + value |= u64::from(byte & 0x7f) << shift; + + if byte & 0x80 == 0 { + return Ok(value); + } + } + + Err(ext_exception( + "arrow failed to decode thrift: a varint longer than 10 bytes", + )) + } + + fn zigzag(&mut self) -> Result { + let value = self.varint()?; + + Ok((value >> 1) as i64 ^ -((value & 1) as i64)) + } + + /// The next field's id and compact type, `None` at the struct's STOP. + fn field_header(&mut self, last: &mut i16) -> Result, PhpException> { + let header = self.byte()?; + + if header == 0 { + return Ok(None); + } + + let delta = header >> 4; + *last = if delta == 0 { + i16::try_from(self.zigzag()?).map_err(|_| ext_exception("arrow failed to decode thrift: a field id"))? + } else { + last.wrapping_add(i16::from(delta)) + }; + + Ok(Some((*last, header & 0x0f))) + } + + /// A list or set header: its size and element type. + fn list_header(&mut self) -> Result<(usize, u8), PhpException> { + let header = self.byte()?; + let size = match header >> 4 { + 15 => usize::try_from(self.varint()?).map_err(|_| truncated())?, + size => usize::from(size), + }; + + Ok((size, header & 0x0f)) + } + + fn skip(&mut self, wire: u8, depth: usize) -> Result<(), PhpException> { + if depth > DEPTH { + return Err(ext_exception("arrow failed to decode thrift: nested deeper than 64")); + } + + match wire { + BOOLEAN_TRUE | BOOLEAN_FALSE => {} + BYTE => { + self.byte()?; + } + I16 | I32 | I64 => { + self.varint()?; + } + DOUBLE => { + self.take(8)?; + } + BINARY => { + let length = usize::try_from(self.varint()?).map_err(|_| truncated())?; + self.take(length)?; + } + LIST | SET => { + let (size, element) = self.list_header()?; + + for _ in 0..size { + self.skip_element(element, depth + 1)?; + } + } + MAP => { + let size = usize::try_from(self.varint()?).map_err(|_| truncated())?; + + if size > 0 { + let types = self.byte()?; + + for _ in 0..size { + self.skip_element(types >> 4, depth + 1)?; + self.skip_element(types & 0x0f, depth + 1)?; + } + } + } + STRUCT => { + let mut last = 0; + + while let Some((_, wire)) = self.field_header(&mut last)? { + self.skip(wire, depth + 1)?; + } + } + other => { + return Err(ext_exception(format!( + "arrow failed to decode thrift: compact type {other}" + ))) + } + } + + Ok(()) + } + + /// A container element: a bool is one byte there, not a field-header type. + fn skip_element(&mut self, wire: u8, depth: usize) -> Result<(), PhpException> { + match wire { + BOOLEAN_TRUE | BOOLEAN_FALSE => self.byte().map(|_| ()), + wire => self.skip(wire, depth), + } + } + + /// A value of type `value` that arrived as `wire`; `element`: inside a list, where a bool is a byte. + fn value( + &mut self, + specs: &Specs, + value: &Value, + wire: u8, + element: bool, + depth: usize, + ) -> Result, PhpException> { + let mut zv = Zval::new(); + + match value { + Value::Bool => zv.set_bool(if element { + self.byte()? == BOOLEAN_TRUE + } else { + wire == BOOLEAN_TRUE + }), + Value::Byte => zv = zval_long(i64::from(self.byte()? as i8)), + Value::I16 | Value::I32 | Value::I64 => zv = zval_long(self.zigzag()?), + Value::Double => { + let bytes = self.take(8)?; + zv.set_double(f64::from_le_bytes(bytes.try_into().map_err(|_| truncated())?)); + } + Value::String => { + let length = usize::try_from(self.varint()?).map_err(|_| truncated())?; + zv = zval_str(self.take(length)?); + } + Value::Struct(class) => zv = self.object(specs, *class, depth + 1)?, + Value::List(item) => { + let (size, wire) = self.list_header()?; + + if !item.arrives_as(wire) { + for _ in 0..size { + self.skip_element(wire, depth + 1)?; + } + + return Ok(None); + } + + let mut list = ZendHashTable::with_capacity(u32::try_from(size).unwrap_or(u32::MAX)); + + for _ in 0..size { + let value = self.value(specs, item, wire, true, depth + 1)?.unwrap_or_default(); + list.push(value) + .map_err(|e| ext_exception(format!("arrow failed to collect a thrift list: {e:?}")))?; + } + + zv.set_hashtable(list); + } + } + + Ok(Some(zv)) + } + + /// An instance of `specs`' class `class`, its properties set from the fields that arrive. + fn object(&mut self, specs: &Specs, class: usize, depth: usize) -> Result { + if depth > DEPTH { + return Err(ext_exception("arrow failed to decode thrift: nested deeper than 64")); + } + + let class = &specs.classes[class]; + let mut object = ZendObject::new(class.ce); + let mut last = 0; + + while let Some((id, wire)) = self.field_header(&mut last)? { + match field_of(class, id) { + Some((offset, value)) if value.arrives_as(wire) => { + if let Some(decoded) = self.value(specs, value, wire, false, depth)? { + replace_slot(&mut object, *offset, decoded); + } + } + _ => self.skip(wire, depth)?, + } + } + + let mut zv = Zval::new(); + zv.set_object(&mut object); + + Ok(zv) + } + + /// Field `field` of the root class, the fields before it skipped; null when it never arrives. + fn field(&mut self, specs: &Specs, field: i16) -> Result { + let class = &specs.classes[0]; + let mut last = 0; + + while let Some((id, wire)) = self.field_header(&mut last)? { + match field_of(class, id) { + Some((_, value)) if id == field && value.arrives_as(wire) => { + if let Some(decoded) = self.value(specs, value, wire, false, 0)? { + return Ok(decoded); + } + } + _ => self.skip(wire, 0)?, + } + } + + Ok(Zval::new()) + } +} + +fn field_of(class: &Class, id: i16) -> Option<&(u32, Value)> { + usize::try_from(id) + .ok() + .and_then(|id| class.fields.get(id)) + .and_then(Option::as_ref) +} + +/// Moves `value` into a declared-property slot, releasing what the slot held (null on a fresh object, an earlier +/// value when a field id repeats). +fn replace_slot(object: &mut ZendObject, offset: u32, value: Zval) { + let slot = unsafe { + std::ptr::from_mut(object) + .cast::() + .add(offset as usize) + .cast::() + }; + + drop(unsafe { std::ptr::replace(slot, value) }); +} diff --git a/src/extension/arrow-ext/src/values.rs b/src/extension/arrow-ext/src/values.rs new file mode 100644 index 0000000000..4bb8fb3020 --- /dev/null +++ b/src/extension/arrow-ext/src/values.rs @@ -0,0 +1,257 @@ +//! PHP value construction through the engine's C date API. + +use std::ffi::{c_char, c_int, c_uint, c_void}; + +use ext_php_rs::exception::PhpException; +use ext_php_rs::types::{ZendObject, Zval}; + +use crate::exception::ext_exception; +use crate::php::{self, call_handle, ensure_no_pending_exception}; + +extern "C" { + fn php_date_instantiate(pce: *mut ext_php_rs::zend::ClassEntry, object: *mut Zval) -> *mut Zval; + + #[cfg(php84)] + fn php_date_initialize_from_ts_long(dateobj: *mut c_void, sec: i64, usec: c_int); + + fn php_date_initialize( + dateobj: *mut c_void, + time_str: *const c_char, + time_str_len: usize, + format: *const c_char, + timezone_object: *mut Zval, + flags: c_int, + ) -> bool; +} + +const PHP_DATE_OBJ_STD_OFFSET: usize = std::mem::size_of::<*const c_void>(); + +/// `php_date_obj_from_obj`: the `php_date_obj` a datetime `zend_object` is embedded in. +fn php_date_obj(object: &mut ZendObject) -> *mut c_void { + std::ptr::from_mut(object) + .cast::() + .wrapping_sub(PHP_DATE_OBJ_STD_OFFSET) + .cast::() +} + +// `timelib_time` up to `tim_uptodate` (ext/date/lib/timelib.h, identical in the 2022.14-2022.17 timelib of PHP 8.3-8.5) +#[repr(C)] +struct TimelibSpecial { + kind: c_uint, + amount: i64, +} + +#[repr(C)] +struct TimelibRelTime { + y: i64, + m: i64, + d: i64, + h: i64, + i: i64, + s: i64, + us: i64, + weekday: c_int, + weekday_behavior: c_int, + first_last_day_of: c_int, + invert: c_int, + days: i64, + special: TimelibSpecial, + have_weekday_relative: c_uint, + have_special_relative: c_uint, +} + +#[repr(C)] +struct TimelibTime { + y: i64, + m: i64, + d: i64, + h: i64, + i: i64, + s: i64, + us: i64, + z: c_int, + tz_abbr: *mut c_char, + tz_info: *mut c_void, + dst: c_int, + relative: TimelibRelTime, + sse: i64, + have_time: c_uint, + have_date: c_uint, + have_zone: c_uint, + have_relative: c_uint, + have_weeknr_day: c_uint, + sse_uptodate: c_uint, + tim_uptodate: c_uint, +} + +/// The up-to-date `timelib_time` of a `DateTimeImmutable` (or subclass) object. +fn timelib_time(datetime: &ZendObject) -> Option<&TimelibTime> { + let dateobj = std::ptr::from_ref(datetime) + .cast::() + .wrapping_sub(PHP_DATE_OBJ_STD_OFFSET) + .cast::<*const TimelibTime>(); + let time = unsafe { (*dateobj).as_ref() }?; + + (time.sse_uptodate != 0 && time.tim_uptodate != 0).then_some(time) +} + +/// `getTimestamp() * 1_000_000 + format('u')`, `None` on i64 overflow. +pub fn datetime_micros(datetime: &ZendObject) -> Option { + let time = timelib_time(datetime)?; + + time.sse.checked_mul(1_000_000)?.checked_add(time.us) +} + +/// Days since the epoch of the value's own calendar day, `None` outside i32. +pub fn datetime_days(datetime: &ZendObject) -> Option { + let time = timelib_time(datetime)?; + let days = days_from_civil(time.y, time.m, time.d)?; + + i32::try_from(days).is_ok().then_some(days) +} + +/// Days since 1970-01-01 of a proleptic Gregorian date, `None` on i64 overflow. +fn days_from_civil(year: i64, month: i64, day: i64) -> Option { + let year = year - i64::from(month <= 2); + let era = (if year >= 0 { year } else { year.checked_sub(399)? }) / 400; + let year_of_era = year - era * 400; + let day_of_year = (153 * (if month > 2 { month - 3 } else { month + 9 }) + 2) / 5 + day - 1; + let day_of_era = year_of_era * 365 + year_of_era / 4 - year_of_era / 100 + day_of_year; + + era.checked_mul(146_097)?.checked_add(day_of_era - 719_468) +} + +/// A fresh `DateTimeImmutable` initialised by `init` over its `php_date_obj`, then moved into `zone`. +fn datetime_in( + zone: &[u8], + init: impl FnOnce(*mut c_void) -> bool, + failure: impl FnOnce() -> PhpException, +) -> Result { + let fns = php::datetime_immutable()?; + let mut datetime = Zval::new(); + + unsafe { + php_date_instantiate(std::ptr::from_ref(fns.ce).cast_mut(), std::ptr::from_mut(&mut datetime)); + } + + let datetime_obj = datetime + .object_mut() + .ok_or_else(|| ext_exception("arrow failed to instantiate a datetime object"))?; + let initialized = init(php_date_obj(datetime_obj)); + + ensure_no_pending_exception("restore a datetime value")?; + + if !initialized { + return Err(failure()); + } + + let datetime = call_handle( + fns.set_timezone, + Some(datetime_obj), + &mut [php::timezone(zone)?], + "restore a datetime timezone", + )?; + + if !datetime.is_object() { + return Err(ext_exception("arrow expected setTimezone to return a datetime object")); + } + + Ok(datetime) +} + +/// The instant `micros` in `zone`. +pub fn datetime_from_micros(micros: i64, zone: &[u8]) -> Result { + let seconds = micros.div_euclid(1_000_000); + let fraction = micros.rem_euclid(1_000_000); + + datetime_in( + zone, + |dateobj| { + // the createFromTimestamp() initializer, exported from PHP 8.4 + #[cfg(php84)] + { + unsafe { php_date_initialize_from_ts_long(dateobj, seconds, fraction as c_int) }; + + true + } + + #[cfg(not(php84))] + { + // timelib reads the byte AFTER the consumed input, so the buffer must be NUL-terminated like the + // zend_strings PHP hands it (length excludes the NUL) + let mut time_str = format!("{seconds}.{fraction:06}\0"); + + unsafe { + php_date_initialize( + dateobj, + time_str.as_mut_ptr().cast::(), + time_str.len() - 1, + c"U.u".as_ptr(), + std::ptr::null_mut(), + // PHP_DATE_INIT_FORMAT - the flags createFromFormat passes + 0x02, + ) + } + } + }, + || { + ext_exception(format!( + "arrow failed to restore datetime from \"{micros}\" microseconds" + )) + }, + ) +} + +/// `new DateTimeImmutable('@' . ($days * 86_400))` in UTC. +pub fn date_from_days(days: i64) -> Result { + let mut time_str = format!("@{}\0", days * 86_400); + + datetime_in( + b"UTC", + |dateobj| unsafe { + php_date_initialize( + dateobj, + time_str.as_mut_ptr().cast::(), + time_str.len() - 1, + std::ptr::null(), + std::ptr::null_mut(), + 0, + ) + }, + || ext_exception(format!("arrow failed to restore date from \"{days}\" days")), + ) +} + +/// The lowercase 36-char form of 16 uuid bytes. +pub fn uuid_text(bytes: &[u8]) -> Vec { + const HEX: &[u8; 16] = b"0123456789abcdef"; + let mut text = Vec::with_capacity(36); + + for (index, byte) in bytes.iter().enumerate() { + if matches!(index, 4 | 6 | 8 | 10) { + text.push(b'-'); + } + + text.push(HEX[usize::from(byte >> 4)]); + text.push(HEX[usize::from(byte & 0x0f)]); + } + + text +} + +/// `hex2bin()` of a 36-char uuid with its dashes removed. +pub fn uuid_bytes(text: &[u8]) -> Option<[u8; 16]> { + let mut bytes = [0u8; 16]; + let mut digits = text + .iter() + .filter(|byte| **byte != b'-') + .map(|byte| (*byte as char).to_digit(16)); + + for byte in &mut bytes { + let high = digits.next()??; + let low = digits.next()??; + *byte = (high * 16 + low) as u8; + } + + digits.next().is_none().then_some(bytes) +} diff --git a/src/extension/arrow-ext/tests/fixtures/all_flat_types.parquet b/src/extension/arrow-ext/tests/fixtures/all_flat_types.parquet deleted file mode 100644 index ba93367e43..0000000000 Binary files a/src/extension/arrow-ext/tests/fixtures/all_flat_types.parquet and /dev/null differ diff --git a/src/extension/arrow-ext/tests/fixtures/decimals_fixed_len.parquet b/src/extension/arrow-ext/tests/fixtures/decimals_fixed_len.parquet deleted file mode 100644 index 33508eb43c..0000000000 Binary files a/src/extension/arrow-ext/tests/fixtures/decimals_fixed_len.parquet and /dev/null differ diff --git a/src/extension/arrow-ext/tests/fixtures/deeply_nested.parquet b/src/extension/arrow-ext/tests/fixtures/deeply_nested.parquet deleted file mode 100644 index 99b577527e..0000000000 Binary files a/src/extension/arrow-ext/tests/fixtures/deeply_nested.parquet and /dev/null differ diff --git a/src/extension/arrow-ext/tests/fixtures/nested.parquet b/src/extension/arrow-ext/tests/fixtures/nested.parquet deleted file mode 100644 index 47a6e60fbc..0000000000 Binary files a/src/extension/arrow-ext/tests/fixtures/nested.parquet and /dev/null differ diff --git a/src/extension/arrow-ext/tests/fixtures/simple.parquet b/src/extension/arrow-ext/tests/fixtures/simple.parquet deleted file mode 100644 index 2d3a7196ba..0000000000 Binary files a/src/extension/arrow-ext/tests/fixtures/simple.parquet and /dev/null differ diff --git a/src/extension/arrow-ext/tests/phpt/001_extension_loaded.phpt b/src/extension/arrow-ext/tests/phpt/001_extension_loaded.phpt index 8eaf6e756c..80f841ba90 100644 --- a/src/extension/arrow-ext/tests/phpt/001_extension_loaded.phpt +++ b/src/extension/arrow-ext/tests/phpt/001_extension_loaded.phpt @@ -1,14 +1,39 @@ --TEST-- -arrow extension is loaded and classes are registered +arrow extension is loaded and the Parquet and Arrow C Data classes are registered --SKIPIF-- --FILE-- isInternal() ? 'internal' : 'userland', ', ', $r->isFinal() ? 'final' : 'open', "\n"; +} + +foreach (['Flow\Arrow\Parquet\Reader', 'Flow\Arrow\Parquet\Writer', 'Flow\Arrow\Parquet\Exception'] as $class) { + var_dump(class_exists($class, false)); +} + +foreach (['Flow\Arrow\RustParquetBatch', 'Flow\Arrow\RustArrowSchema'] as $class) { + try { + new $class(); + } catch (Throwable $e) { + echo $class, ': ', $e->getMessage(), "\n"; + } +} ?> --EXPECT-- bool(true) -bool(true) -bool(true) +Flow\Parquet\Engine\RustParquetEngine: internal, final +Flow\Parquet\Engine\RustParquetFileReader: internal, final +Flow\Arrow\Parquet\RustColumnsReader: internal, final +Flow\Parquet\Engine\RustParquetFileWriter: internal, final +Flow\Arrow\Parquet\RustBatchReader: internal, final +Flow\Arrow\RustParquetBatch: internal, final +Flow\Arrow\RustArrowSchema: internal, final +bool(false) +bool(false) +bool(false) +Flow\Arrow\RustParquetBatch: You cannot instantiate this class from PHP. +Flow\Arrow\RustArrowSchema: You cannot instantiate this class from PHP. diff --git a/src/extension/arrow-ext/tests/phpt/002_reader_construct.phpt b/src/extension/arrow-ext/tests/phpt/002_reader_construct.phpt deleted file mode 100644 index 9c13a05cdd..0000000000 --- a/src/extension/arrow-ext/tests/phpt/002_reader_construct.phpt +++ /dev/null @@ -1,35 +0,0 @@ ---TEST-- -Reader constructor opens a Parquet file ---SKIPIF-- - ---FILE-- -data = file_get_contents($filePath); - } - - public function read(int $length, int $offset): string { - return substr($this->data, $offset, $length); - } - - public function size(): ?int { - return strlen($this->data); - } -} - -$stream = new TestSourceStream(__DIR__ . '/../fixtures/simple.parquet'); -$reader = new Flow\Arrow\Parquet\Reader($stream); -echo "Reader created\n"; -$reader->close(); -echo "Reader closed\n"; -?> ---EXPECT-- -Reader created -Reader closed diff --git a/src/extension/arrow-ext/tests/phpt/003_reader_null_size.phpt b/src/extension/arrow-ext/tests/phpt/003_reader_null_size.phpt deleted file mode 100644 index 6923332331..0000000000 --- a/src/extension/arrow-ext/tests/phpt/003_reader_null_size.phpt +++ /dev/null @@ -1,35 +0,0 @@ ---TEST-- -Reader constructor throws exception when size() returns null ---SKIPIF-- - ---FILE-- -getMessage(), 'null')); -} catch (\Exception $e) { - echo "Caught Exception\n"; - var_dump(str_contains($e->getMessage(), 'null')); -} -?> ---EXPECT-- -Caught Flow\Arrow\Parquet\Exception -bool(true) diff --git a/src/extension/arrow-ext/tests/phpt/004_reader_schema.phpt b/src/extension/arrow-ext/tests/phpt/004_reader_schema.phpt deleted file mode 100644 index 53ff6b36c8..0000000000 --- a/src/extension/arrow-ext/tests/phpt/004_reader_schema.phpt +++ /dev/null @@ -1,47 +0,0 @@ ---TEST-- -Reader schema returns column descriptors ---SKIPIF-- - ---FILE-- -data = file_get_contents($filePath); - } - - public function read(int $length, int $offset): string { - return substr($this->data, $offset, $length); - } - - public function size(): ?int { - return strlen($this->data); - } -} - -$stream = new TestSourceStream(__DIR__ . '/../fixtures/simple.parquet'); -$reader = new Flow\Arrow\Parquet\Reader($stream); -$schema = $reader->schema(); - -var_dump(count($schema)); -var_dump($schema[0]['name']); -var_dump($schema[0]['type']); -var_dump($schema[0]['optional']); -var_dump($schema[1]['name']); -var_dump($schema[1]['type']); -var_dump($schema[1]['optional']); -$reader->close(); -?> ---EXPECT-- -int(2) -string(2) "id" -string(5) "INT64" -bool(false) -string(4) "name" -string(6) "STRING" -bool(false) diff --git a/src/extension/arrow-ext/tests/phpt/005_reader_metadata.phpt b/src/extension/arrow-ext/tests/phpt/005_reader_metadata.phpt deleted file mode 100644 index fbfa6549ff..0000000000 --- a/src/extension/arrow-ext/tests/phpt/005_reader_metadata.phpt +++ /dev/null @@ -1,41 +0,0 @@ ---TEST-- -Reader metadata returns file info ---SKIPIF-- - ---FILE-- -data = file_get_contents($filePath); - } - - public function read(int $length, int $offset): string { - return substr($this->data, $offset, $length); - } - - public function size(): ?int { - return strlen($this->data); - } -} - -$stream = new TestSourceStream(__DIR__ . '/../fixtures/simple.parquet'); -$reader = new Flow\Arrow\Parquet\Reader($stream); -$meta = $reader->metadata(); - -var_dump($meta['rows']); -var_dump($meta['row_groups']); -var_dump(is_string($meta['created_by'])); -var_dump(is_array($meta['metadata'])); -$reader->close(); -?> ---EXPECT-- -int(3) -int(1) -bool(true) -bool(true) diff --git a/src/extension/arrow-ext/tests/phpt/006_reader_flat_columns.phpt b/src/extension/arrow-ext/tests/phpt/006_reader_flat_columns.phpt deleted file mode 100644 index 8d626a541e..0000000000 --- a/src/extension/arrow-ext/tests/phpt/006_reader_flat_columns.phpt +++ /dev/null @@ -1,51 +0,0 @@ ---TEST-- -Reader reads flat columns from simple.parquet ---SKIPIF-- - ---FILE-- -data = file_get_contents($filePath); - } - - public function read(int $length, int $offset): string { - return substr($this->data, $offset, $length); - } - - public function size(): ?int { - return strlen($this->data); - } -} - -$stream = new TestSourceStream(__DIR__ . '/../fixtures/simple.parquet'); -$reader = new Flow\Arrow\Parquet\Reader($stream); -$data = $reader->readRowGroup(); - -var_dump($data['id']); -var_dump($data['name']); -$reader->close(); -?> ---EXPECT-- -array(3) { - [0]=> - int(1) - [1]=> - int(2) - [2]=> - int(3) -} -array(3) { - [0]=> - string(5) "Alice" - [1]=> - string(3) "Bob" - [2]=> - string(7) "Charlie" -} diff --git a/src/extension/arrow-ext/tests/phpt/007_reader_all_flat_types.phpt b/src/extension/arrow-ext/tests/phpt/007_reader_all_flat_types.phpt deleted file mode 100644 index e14c36ccba..0000000000 --- a/src/extension/arrow-ext/tests/phpt/007_reader_all_flat_types.phpt +++ /dev/null @@ -1,106 +0,0 @@ ---TEST-- -Reader converts all flat Arrow types correctly ---SKIPIF-- - ---FILE-- -data = file_get_contents($filePath); - } - - public function read(int $length, int $offset): string { - return substr($this->data, $offset, $length); - } - - public function size(): ?int { - return strlen($this->data); - } -} - -$stream = new TestSourceStream(__DIR__ . '/../fixtures/all_flat_types.parquet'); -$reader = new Flow\Arrow\Parquet\Reader($stream); -$data = $reader->readRowGroup(); - -// Boolean -echo "bool: "; -var_dump($data['col_bool'][0] === true && $data['col_bool'][1] === false && $data['col_bool'][2] === null); - -// Integers -echo "int8: "; -var_dump($data['col_int8'][0] === 1 && $data['col_int8'][1] === -128 && $data['col_int8'][2] === null); - -echo "int16: "; -var_dump($data['col_int16'][0] === 256 && $data['col_int16'][1] === -32768 && $data['col_int16'][2] === null); - -echo "int32: "; -var_dump($data['col_int32'][0] === 100000 && $data['col_int32'][1] === -100000 && $data['col_int32'][2] === null); - -echo "int64: "; -var_dump($data['col_int64'][0] === 1000000000 && $data['col_int64'][1] === -1000000000 && $data['col_int64'][2] === null); - -// Unsigned integers -echo "uint8: "; -var_dump($data['col_uint8'][0] === 0 && $data['col_uint8'][1] === 255 && $data['col_uint8'][2] === null); - -echo "uint16: "; -var_dump($data['col_uint16'][0] === 0 && $data['col_uint16'][1] === 65535 && $data['col_uint16'][2] === null); - -echo "uint32: "; -var_dump($data['col_uint32'][0] === 0 && $data['col_uint32'][1] === 4294967295 && $data['col_uint32'][2] === null); - -echo "uint64: "; -var_dump($data['col_uint64'][0] === 0 && $data['col_uint64'][1] === "18446744073709551615" && $data['col_uint64'][2] === null); - -// Floats -echo "float: "; -var_dump(is_float($data['col_float'][0]) && is_float($data['col_float'][1]) && $data['col_float'][2] === null); - -echo "double: "; -var_dump($data['col_double'][0] === 1.23456 && $data['col_double'][1] === -7.89012 && $data['col_double'][2] === null); - -// String -echo "string: "; -var_dump($data['col_string'][0] === "hello" && $data['col_string'][1] === "world" && $data['col_string'][2] === null); - -// Binary -echo "binary: "; -var_dump($data['col_binary'][0] === "\x00\x01\x02" && $data['col_binary'][1] === "\xff\xfe" && $data['col_binary'][2] === null); - -// Date (19000 days since epoch = 2021-12-24) -echo "date: "; -var_dump($data['col_date'][0] instanceof DateTimeImmutable && $data['col_date'][0]->getTimestamp() === 1641600000 && $data['col_date'][1] instanceof DateTimeImmutable && $data['col_date'][1]->getTimestamp() === 0 && $data['col_date'][2] === null); - -// Timestamp (1700000000000000 microseconds = 2023-11-14T22:13:20) -echo "timestamp: "; -var_dump($data['col_timestamp'][0] instanceof DateTimeImmutable && $data['col_timestamp'][0]->getTimestamp() === 1700000000 && $data['col_timestamp'][1] instanceof DateTimeImmutable && $data['col_timestamp'][1]->getTimestamp() === 0 && $data['col_timestamp'][2] === null); - -// Decimal (string representation) -echo "decimal: "; -var_dump($data['col_decimal'][0] === 123.45 && $data['col_decimal'][1] === -999.99 && $data['col_decimal'][2] === null); - -$reader->close(); -?> ---EXPECT-- -bool: bool(true) -int8: bool(true) -int16: bool(true) -int32: bool(true) -int64: bool(true) -uint8: bool(true) -uint16: bool(true) -uint32: bool(true) -uint64: bool(true) -float: bool(true) -double: bool(true) -string: bool(true) -binary: bool(true) -date: bool(true) -timestamp: bool(true) -decimal: bool(true) diff --git a/src/extension/arrow-ext/tests/phpt/008_reader_column_projection.phpt b/src/extension/arrow-ext/tests/phpt/008_reader_column_projection.phpt deleted file mode 100644 index 33732a94fb..0000000000 --- a/src/extension/arrow-ext/tests/phpt/008_reader_column_projection.phpt +++ /dev/null @@ -1,56 +0,0 @@ ---TEST-- -Reader returns only selected columns with projection ---SKIPIF-- - ---FILE-- -data = file_get_contents($filePath); - } - - public function read(int $length, int $offset): string { - return substr($this->data, $offset, $length); - } - - public function size(): ?int { - return strlen($this->data); - } -} - -$stream = new TestSourceStream(__DIR__ . '/../fixtures/simple.parquet'); -$reader = new Flow\Arrow\Parquet\Reader($stream); -$data = $reader->readRowGroup(['name']); - -echo "column count: "; -var_dump(count($data)); - -echo "has name: "; -var_dump(array_key_exists('name', $data)); - -echo "has id: "; -var_dump(array_key_exists('id', $data)); - -echo "name values: "; -var_dump($data['name']); - -$reader->close(); -?> ---EXPECT-- -column count: int(1) -has name: bool(true) -has id: bool(false) -name values: array(3) { - [0]=> - string(5) "Alice" - [1]=> - string(3) "Bob" - [2]=> - string(7) "Charlie" -} diff --git a/src/extension/arrow-ext/tests/phpt/009_reader_exhausted.phpt b/src/extension/arrow-ext/tests/phpt/009_reader_exhausted.phpt deleted file mode 100644 index 21d4db9781..0000000000 --- a/src/extension/arrow-ext/tests/phpt/009_reader_exhausted.phpt +++ /dev/null @@ -1,47 +0,0 @@ ---TEST-- -Reader returns null when all row groups are consumed ---SKIPIF-- - ---FILE-- -data = file_get_contents($filePath); - } - - public function read(int $length, int $offset): string { - return substr($this->data, $offset, $length); - } - - public function size(): ?int { - return strlen($this->data); - } -} - -$stream = new TestSourceStream(__DIR__ . '/../fixtures/simple.parquet'); -$reader = new Flow\Arrow\Parquet\Reader($stream); - -$rg1 = $reader->readRowGroup(); -echo "first: "; -var_dump(is_array($rg1)); - -$rg2 = $reader->readRowGroup(); -echo "second: "; -var_dump($rg2); - -$rg3 = $reader->readRowGroup(); -echo "third: "; -var_dump($rg3); - -$reader->close(); -?> ---EXPECT-- -first: bool(true) -second: NULL -third: NULL diff --git a/src/extension/arrow-ext/tests/phpt/010_reader_list.phpt b/src/extension/arrow-ext/tests/phpt/010_reader_list.phpt deleted file mode 100644 index 0f1291bdc5..0000000000 --- a/src/extension/arrow-ext/tests/phpt/010_reader_list.phpt +++ /dev/null @@ -1,57 +0,0 @@ ---TEST-- -Reader reads LIST columns from nested.parquet ---SKIPIF-- - ---FILE-- -data = file_get_contents($filePath); - } - - public function read(int $length, int $offset): string { - return substr($this->data, $offset, $length); - } - - public function size(): ?int { - return strlen($this->data); - } -} - -$stream = new TestSourceStream(__DIR__ . '/../fixtures/nested.parquet'); -$reader = new Flow\Arrow\Parquet\Reader($stream); -$data = $reader->readRowGroup(); - -// Row 0: ["a", "b", "c"] -echo "row0 is array: "; -var_dump(is_array($data['tags'][0])); -echo "row0 count: "; -var_dump(count($data['tags'][0])); -echo "row0 values: "; -var_dump($data['tags'][0][0] === "a" && $data['tags'][0][1] === "b" && $data['tags'][0][2] === "c"); - -// Row 1: [] (empty list) -echo "row1 is array: "; -var_dump(is_array($data['tags'][1])); -echo "row1 count: "; -var_dump(count($data['tags'][1])); - -// Row 2: null -echo "row2 is null: "; -var_dump($data['tags'][2] === null); - -$reader->close(); -?> ---EXPECT-- -row0 is array: bool(true) -row0 count: int(3) -row0 values: bool(true) -row1 is array: bool(true) -row1 count: int(0) -row2 is null: bool(true) diff --git a/src/extension/arrow-ext/tests/phpt/011_reader_struct.phpt b/src/extension/arrow-ext/tests/phpt/011_reader_struct.phpt deleted file mode 100644 index c2ac1fa15b..0000000000 --- a/src/extension/arrow-ext/tests/phpt/011_reader_struct.phpt +++ /dev/null @@ -1,57 +0,0 @@ ---TEST-- -Reader reads STRUCT columns from nested.parquet ---SKIPIF-- - ---FILE-- -data = file_get_contents($filePath); - } - - public function read(int $length, int $offset): string { - return substr($this->data, $offset, $length); - } - - public function size(): ?int { - return strlen($this->data); - } -} - -$stream = new TestSourceStream(__DIR__ . '/../fixtures/nested.parquet'); -$reader = new Flow\Arrow\Parquet\Reader($stream); -$data = $reader->readRowGroup(); - -// Row 0: {key: "x", value: 10} -echo "row0 is array: "; -var_dump(is_array($data['metadata'][0])); -echo "row0 key: "; -var_dump($data['metadata'][0]['key']); -echo "row0 value: "; -var_dump($data['metadata'][0]['value']); - -// Row 1: null -echo "row1 is null: "; -var_dump($data['metadata'][1] === null); - -// Row 2: {key: "y", value: 20} -echo "row2 key: "; -var_dump($data['metadata'][2]['key']); -echo "row2 value: "; -var_dump($data['metadata'][2]['value']); - -$reader->close(); -?> ---EXPECT-- -row0 is array: bool(true) -row0 key: string(1) "x" -row0 value: int(10) -row1 is null: bool(true) -row2 key: string(1) "y" -row2 value: int(20) diff --git a/src/extension/arrow-ext/tests/phpt/012_reader_map.phpt b/src/extension/arrow-ext/tests/phpt/012_reader_map.phpt deleted file mode 100644 index d8189f8064..0000000000 --- a/src/extension/arrow-ext/tests/phpt/012_reader_map.phpt +++ /dev/null @@ -1,54 +0,0 @@ ---TEST-- -Reader reads MAP columns from nested.parquet ---SKIPIF-- - ---FILE-- -data = file_get_contents($filePath); - } - - public function read(int $length, int $offset): string { - return substr($this->data, $offset, $length); - } - - public function size(): ?int { - return strlen($this->data); - } -} - -$stream = new TestSourceStream(__DIR__ . '/../fixtures/nested.parquet'); -$reader = new Flow\Arrow\Parquet\Reader($stream); -$data = $reader->readRowGroup(); - -// Row 0: {"p1": 100, "p2": 200} -echo "row0 is array: "; -var_dump(is_array($data['props'][0])); -echo "row0 p1: "; -var_dump($data['props'][0]['p1']); -echo "row0 p2: "; -var_dump($data['props'][0]['p2']); - -// Row 1: {"p3": 300} -echo "row1 p3: "; -var_dump($data['props'][1]['p3']); - -// Row 2: null -echo "row2 is null: "; -var_dump($data['props'][2] === null); - -$reader->close(); -?> ---EXPECT-- -row0 is array: bool(true) -row0 p1: int(100) -row0 p2: int(200) -row1 p3: int(300) -row2 is null: bool(true) diff --git a/src/extension/arrow-ext/tests/phpt/013_reader_deeply_nested.phpt b/src/extension/arrow-ext/tests/phpt/013_reader_deeply_nested.phpt deleted file mode 100644 index 95bb994c5e..0000000000 --- a/src/extension/arrow-ext/tests/phpt/013_reader_deeply_nested.phpt +++ /dev/null @@ -1,66 +0,0 @@ ---TEST-- -Reader reads deeply nested columns from deeply_nested.parquet ---SKIPIF-- - ---FILE-- -data = file_get_contents($filePath); - } - - public function read(int $length, int $offset): string { - return substr($this->data, $offset, $length); - } - - public function size(): ?int { - return strlen($this->data); - } -} - -$stream = new TestSourceStream(__DIR__ . '/../fixtures/deeply_nested.parquet'); -$reader = new Flow\Arrow\Parquet\Reader($stream); -$data = $reader->readRowGroup(); - -// Row 0: [{name:"alpha", scores:[10,20]}, {name:"beta", scores:[30]}] -echo "row0 count: "; -var_dump(count($data['groups'][0])); -echo "row0[0] name: "; -var_dump($data['groups'][0][0]['name']); -echo "row0[0] scores: "; -var_dump($data['groups'][0][0]['scores'][0] === 10 && $data['groups'][0][0]['scores'][1] === 20); -echo "row0[1] name: "; -var_dump($data['groups'][0][1]['name']); -echo "row0[1] scores: "; -var_dump($data['groups'][0][1]['scores'][0] === 30); - -// Row 1: [{name:"gamma", scores:[]}] -echo "row1 count: "; -var_dump(count($data['groups'][1])); -echo "row1[0] name: "; -var_dump($data['groups'][1][0]['name']); -echo "row1[0] scores count: "; -var_dump(count($data['groups'][1][0]['scores'])); - -// Row 2: null -echo "row2 is null: "; -var_dump($data['groups'][2] === null); - -$reader->close(); -?> ---EXPECT-- -row0 count: int(2) -row0[0] name: string(5) "alpha" -row0[0] scores: bool(true) -row0[1] name: string(4) "beta" -row0[1] scores: bool(true) -row1 count: int(1) -row1[0] name: string(5) "gamma" -row1[0] scores count: int(0) -row2 is null: bool(true) diff --git a/src/extension/arrow-ext/tests/phpt/014_reader_null_nested.phpt b/src/extension/arrow-ext/tests/phpt/014_reader_null_nested.phpt deleted file mode 100644 index 0c80e70b3c..0000000000 --- a/src/extension/arrow-ext/tests/phpt/014_reader_null_nested.phpt +++ /dev/null @@ -1,62 +0,0 @@ ---TEST-- -Reader propagates nulls correctly in nested columns ---SKIPIF-- - ---FILE-- -data = file_get_contents($filePath); - } - - public function read(int $length, int $offset): string { - return substr($this->data, $offset, $length); - } - - public function size(): ?int { - return strlen($this->data); - } -} - -$stream = new TestSourceStream(__DIR__ . '/../fixtures/nested.parquet'); -$reader = new Flow\Arrow\Parquet\Reader($stream); -$data = $reader->readRowGroup(); - -// null list (row 2 tags) is PHP null, NOT empty array -echo "null list is null: "; -var_dump($data['tags'][2] === null); -echo "null list is not array: "; -var_dump(!is_array($data['tags'][2])); - -// empty list (row 1 tags) is PHP empty array, NOT null -echo "empty list is array: "; -var_dump(is_array($data['tags'][1])); -echo "empty list is not null: "; -var_dump($data['tags'][1] !== null); -echo "empty list count: "; -var_dump(count($data['tags'][1])); - -// null struct (row 1 metadata) is PHP null -echo "null struct is null: "; -var_dump($data['metadata'][1] === null); - -// null map (row 2 props) is PHP null -echo "null map is null: "; -var_dump($data['props'][2] === null); - -$reader->close(); -?> ---EXPECT-- -null list is null: bool(true) -null list is not array: bool(true) -empty list is array: bool(true) -empty list is not null: bool(true) -empty list count: int(0) -null struct is null: bool(true) -null map is null: bool(true) diff --git a/src/extension/arrow-ext/tests/phpt/015_writer_construct.phpt b/src/extension/arrow-ext/tests/phpt/015_writer_construct.phpt deleted file mode 100644 index 0ad24dcbc6..0000000000 --- a/src/extension/arrow-ext/tests/phpt/015_writer_construct.phpt +++ /dev/null @@ -1,44 +0,0 @@ ---TEST-- -Writer constructs, writes a minimal batch, and produces valid Parquet ---SKIPIF-- - ---FILE-- -data .= $data; - return $this; - } -} - -$dest = new TestDestinationStream(); -$schema = [ - ['name' => 'id', 'type' => 'INT64', 'optional' => false], -]; - -$writer = new Flow\Arrow\Parquet\Writer($dest, $schema); -$writer->writeBatch([ - 'id' => [1, 2, 3], -]); -$writer->close(); - -// Parquet files start and end with magic bytes "PAR1" -$magic = substr($dest->data, 0, 4); -$footer = substr($dest->data, -4); - -echo "starts_with_PAR1: "; -var_dump($magic === "PAR1"); -echo "ends_with_PAR1: "; -var_dump($footer === "PAR1"); -echo "size_gt_zero: "; -var_dump(strlen($dest->data) > 0); -?> ---EXPECT-- -starts_with_PAR1: bool(true) -ends_with_PAR1: bool(true) -size_gt_zero: bool(true) diff --git a/src/extension/arrow-ext/tests/phpt/016_writer_flat_columns.phpt b/src/extension/arrow-ext/tests/phpt/016_writer_flat_columns.phpt deleted file mode 100644 index 0306249dba..0000000000 --- a/src/extension/arrow-ext/tests/phpt/016_writer_flat_columns.phpt +++ /dev/null @@ -1,71 +0,0 @@ ---TEST-- -Writer writes INT64+STRING columns and Reader reads them back ---SKIPIF-- - ---FILE-- -data .= $data; - return $this; - } -} - -class TestSourceStream implements Flow\Arrow\RandomAccessFile { - private string $data; - public function __construct(string $data) { - $this->data = $data; - } - public function read(int $length, int $offset): string { - return substr($this->data, $offset, $length); - } - public function size(): ?int { - return strlen($this->data); - } -} - -$dest = new TestDestinationStream(); -$schema = [ - ['name' => 'id', 'type' => 'INT64', 'optional' => false], - ['name' => 'name', 'type' => 'STRING', 'optional' => false], -]; - -$writer = new Flow\Arrow\Parquet\Writer($dest, $schema, 'UNCOMPRESSED'); -$writer->writeBatch([ - 'id' => [1, 2, 3], - 'name' => ['Alice', 'Bob', 'Charlie'], -]); -$writer->close(); - -$source = new TestSourceStream($dest->data); -$reader = new Flow\Arrow\Parquet\Reader($source); -$data = $reader->readRowGroup(); - -var_dump($data['id']); -var_dump($data['name']); - -$reader->close(); -?> ---EXPECT-- -array(3) { - [0]=> - int(1) - [1]=> - int(2) - [2]=> - int(3) -} -array(3) { - [0]=> - string(5) "Alice" - [1]=> - string(3) "Bob" - [2]=> - string(7) "Charlie" -} diff --git a/src/extension/arrow-ext/tests/phpt/017_writer_roundtrip.phpt b/src/extension/arrow-ext/tests/phpt/017_writer_roundtrip.phpt deleted file mode 100644 index 84b221e8eb..0000000000 --- a/src/extension/arrow-ext/tests/phpt/017_writer_roundtrip.phpt +++ /dev/null @@ -1,144 +0,0 @@ ---TEST-- -Writer roundtrips all 16 flat types through Writer then Reader ---SKIPIF-- - ---FILE-- -data .= $data; - return $this; - } -} - -class TestSourceStream implements Flow\Arrow\RandomAccessFile { - private string $data; - public function __construct(string $data) { - $this->data = $data; - } - public function read(int $length, int $offset): string { - return substr($this->data, $offset, $length); - } - public function size(): ?int { - return strlen($this->data); - } -} - -$dest = new TestDestinationStream(); -$schema = [ - ['name' => 'col_bool', 'type' => 'BOOLEAN', 'optional' => true], - ['name' => 'col_int8', 'type' => 'INT8', 'optional' => true], - ['name' => 'col_int16', 'type' => 'INT16', 'optional' => true], - ['name' => 'col_int32', 'type' => 'INT32', 'optional' => true], - ['name' => 'col_int64', 'type' => 'INT64', 'optional' => true], - ['name' => 'col_uint8', 'type' => 'UINT8', 'optional' => true], - ['name' => 'col_uint16', 'type' => 'UINT16', 'optional' => true], - ['name' => 'col_uint32', 'type' => 'UINT32', 'optional' => true], - ['name' => 'col_uint64', 'type' => 'UINT64', 'optional' => true], - ['name' => 'col_float', 'type' => 'FLOAT', 'optional' => true], - ['name' => 'col_double', 'type' => 'DOUBLE', 'optional' => true], - ['name' => 'col_string', 'type' => 'STRING', 'optional' => true], - ['name' => 'col_binary', 'type' => 'BINARY', 'optional' => true], - ['name' => 'col_date', 'type' => 'DATE', 'optional' => true], - ['name' => 'col_timestamp', 'type' => 'TIMESTAMP', 'optional' => true], - ['name' => 'col_decimal', 'type' => 'DECIMAL', 'optional' => true, 'precision' => 10, 'scale' => 2], -]; - -$writer = new Flow\Arrow\Parquet\Writer($dest, $schema, 'UNCOMPRESSED'); -$writer->writeBatch([ - 'col_bool' => [true, false, null], - 'col_int8' => [1, -128, null], - 'col_int16' => [256, -32768, null], - 'col_int32' => [100000, -100000, null], - 'col_int64' => [1000000000, -1000000000, null], - 'col_uint8' => [0, 255, null], - 'col_uint16' => [0, 65535, null], - 'col_uint32' => [0, 4294967295, null], - 'col_uint64' => [0, "18446744073709551615", null], - 'col_float' => [1.5, -2.5, null], - 'col_double' => [3.14159, -2.71828, null], - 'col_string' => ['hello', 'world', null], - 'col_binary' => ["\x00\x01\x02", "\xff\xfe", null], - 'col_date' => [19000, 0, null], - 'col_timestamp' => [1700000000000000, 0, null], - 'col_decimal' => ['123.45', '-999.99', null], -]); -$writer->close(); - -$source = new TestSourceStream($dest->data); -$reader = new Flow\Arrow\Parquet\Reader($source); -$data = $reader->readRowGroup(); - -echo "bool: "; -var_dump($data['col_bool'][0] === true && $data['col_bool'][1] === false && $data['col_bool'][2] === null); - -echo "int8: "; -var_dump($data['col_int8'][0] === 1 && $data['col_int8'][1] === -128 && $data['col_int8'][2] === null); - -echo "int16: "; -var_dump($data['col_int16'][0] === 256 && $data['col_int16'][1] === -32768 && $data['col_int16'][2] === null); - -echo "int32: "; -var_dump($data['col_int32'][0] === 100000 && $data['col_int32'][1] === -100000 && $data['col_int32'][2] === null); - -echo "int64: "; -var_dump($data['col_int64'][0] === 1000000000 && $data['col_int64'][1] === -1000000000 && $data['col_int64'][2] === null); - -echo "uint8: "; -var_dump($data['col_uint8'][0] === 0 && $data['col_uint8'][1] === 255 && $data['col_uint8'][2] === null); - -echo "uint16: "; -var_dump($data['col_uint16'][0] === 0 && $data['col_uint16'][1] === 65535 && $data['col_uint16'][2] === null); - -echo "uint32: "; -var_dump($data['col_uint32'][0] === 0 && $data['col_uint32'][1] === 4294967295 && $data['col_uint32'][2] === null); - -echo "uint64: "; -var_dump($data['col_uint64'][0] === 0 && $data['col_uint64'][1] === "18446744073709551615" && $data['col_uint64'][2] === null); - -echo "float: "; -var_dump(is_float($data['col_float'][0]) && is_float($data['col_float'][1]) && $data['col_float'][2] === null); - -echo "double: "; -var_dump($data['col_double'][0] === 3.14159 && $data['col_double'][1] === -2.71828 && $data['col_double'][2] === null); - -echo "string: "; -var_dump($data['col_string'][0] === "hello" && $data['col_string'][1] === "world" && $data['col_string'][2] === null); - -echo "binary: "; -var_dump($data['col_binary'][0] === "\x00\x01\x02" && $data['col_binary'][1] === "\xff\xfe" && $data['col_binary'][2] === null); - -echo "date: "; -var_dump($data['col_date'][0] instanceof DateTimeImmutable && $data['col_date'][0]->getTimestamp() === 1641600000 && $data['col_date'][1] instanceof DateTimeImmutable && $data['col_date'][1]->getTimestamp() === 0 && $data['col_date'][2] === null); - -echo "timestamp: "; -var_dump($data['col_timestamp'][0] instanceof DateTimeImmutable && $data['col_timestamp'][0]->getTimestamp() === 1700000000 && $data['col_timestamp'][1] instanceof DateTimeImmutable && $data['col_timestamp'][1]->getTimestamp() === 0 && $data['col_timestamp'][2] === null); - -echo "decimal: "; -var_dump($data['col_decimal'][0] === 123.45 && $data['col_decimal'][1] === -999.99 && $data['col_decimal'][2] === null); - -$reader->close(); -?> ---EXPECT-- -bool: bool(true) -int8: bool(true) -int16: bool(true) -int32: bool(true) -int64: bool(true) -uint8: bool(true) -uint16: bool(true) -uint32: bool(true) -uint64: bool(true) -float: bool(true) -double: bool(true) -string: bool(true) -binary: bool(true) -date: bool(true) -timestamp: bool(true) -decimal: bool(true) diff --git a/src/extension/arrow-ext/tests/phpt/018_writer_compression.phpt b/src/extension/arrow-ext/tests/phpt/018_writer_compression.phpt deleted file mode 100644 index a1b809aff9..0000000000 --- a/src/extension/arrow-ext/tests/phpt/018_writer_compression.phpt +++ /dev/null @@ -1,87 +0,0 @@ ---TEST-- -Writer supports all compression codecs and defaults to SNAPPY ---SKIPIF-- - ---FILE-- -data .= $data; - return $this; - } -} - -class TestSourceStream implements Flow\Arrow\RandomAccessFile { - private string $data; - public function __construct(string $data) { - $this->data = $data; - } - public function read(int $length, int $offset): string { - return substr($this->data, $offset, $length); - } - public function size(): ?int { - return strlen($this->data); - } -} - -$schema = [ - ['name' => 'id', 'type' => 'INT64', 'optional' => false], - ['name' => 'name', 'type' => 'STRING', 'optional' => false], -]; -$batch = [ - 'id' => [1, 2, 3], - 'name' => ['Alice', 'Bob', 'Charlie'], -]; - -$codecs = ['UNCOMPRESSED', 'SNAPPY', 'GZIP', 'ZSTD', 'LZ4', 'BROTLI']; - -foreach ($codecs as $codec) { - $dest = new TestDestinationStream(); - $writer = new Flow\Arrow\Parquet\Writer($dest, $schema, $codec); - $writer->writeBatch($batch); - $writer->close(); - - $source = new TestSourceStream($dest->data); - $reader = new Flow\Arrow\Parquet\Reader($source); - $data = $reader->readRowGroup(); - - $ok = $data['id'][0] === 1 - && $data['id'][1] === 2 - && $data['id'][2] === 3 - && $data['name'][0] === 'Alice' - && $data['name'][1] === 'Bob' - && $data['name'][2] === 'Charlie'; - - echo "$codec: "; - var_dump($ok); - $reader->close(); -} - -// Default compression (no arg) should also work (SNAPPY) -$dest = new TestDestinationStream(); -$writer = new Flow\Arrow\Parquet\Writer($dest, $schema); -$writer->writeBatch($batch); -$writer->close(); - -$source = new TestSourceStream($dest->data); -$reader = new Flow\Arrow\Parquet\Reader($source); -$data = $reader->readRowGroup(); - -echo "DEFAULT: "; -var_dump($data['id'][0] === 1 && $data['name'][2] === 'Charlie'); -$reader->close(); -?> ---EXPECT-- -UNCOMPRESSED: bool(true) -SNAPPY: bool(true) -GZIP: bool(true) -ZSTD: bool(true) -LZ4: bool(true) -BROTLI: bool(true) -DEFAULT: bool(true) diff --git a/src/extension/arrow-ext/tests/phpt/019_writer_null_values.phpt b/src/extension/arrow-ext/tests/phpt/019_writer_null_values.phpt deleted file mode 100644 index 64c612e23f..0000000000 --- a/src/extension/arrow-ext/tests/phpt/019_writer_null_values.phpt +++ /dev/null @@ -1,108 +0,0 @@ ---TEST-- -Writer preserves null values for multiple column types ---SKIPIF-- - ---FILE-- -data .= $data; - return $this; - } -} - -class TestSourceStream implements Flow\Arrow\RandomAccessFile { - private string $data; - public function __construct(string $data) { - $this->data = $data; - } - public function read(int $length, int $offset): string { - return substr($this->data, $offset, $length); - } - public function size(): ?int { - return strlen($this->data); - } -} - -$dest = new TestDestinationStream(); -$schema = [ - ['name' => 'col_int64', 'type' => 'INT64', 'optional' => true], - ['name' => 'col_string', 'type' => 'STRING', 'optional' => true], - ['name' => 'col_bool', 'type' => 'BOOLEAN', 'optional' => true], - ['name' => 'col_double', 'type' => 'DOUBLE', 'optional' => true], - ['name' => 'col_decimal', 'type' => 'DECIMAL', 'optional' => true, 'precision' => 10, 'scale' => 2], -]; - -$writer = new Flow\Arrow\Parquet\Writer($dest, $schema, 'UNCOMPRESSED'); -$writer->writeBatch([ - 'col_int64' => [null, 42, null, 99, null], - 'col_string' => ['hello', null, null, 'world', null], - 'col_bool' => [true, null, false, null, null], - 'col_double' => [null, null, 3.14, null, 2.71], - 'col_decimal' => ['1.23', null, null, '-4.56', null], -]); -$writer->close(); - -$source = new TestSourceStream($dest->data); -$reader = new Flow\Arrow\Parquet\Reader($source); -$data = $reader->readRowGroup(); - -echo "int64: "; -var_dump( - $data['col_int64'][0] === null - && $data['col_int64'][1] === 42 - && $data['col_int64'][2] === null - && $data['col_int64'][3] === 99 - && $data['col_int64'][4] === null -); - -echo "string: "; -var_dump( - $data['col_string'][0] === 'hello' - && $data['col_string'][1] === null - && $data['col_string'][2] === null - && $data['col_string'][3] === 'world' - && $data['col_string'][4] === null -); - -echo "bool: "; -var_dump( - $data['col_bool'][0] === true - && $data['col_bool'][1] === null - && $data['col_bool'][2] === false - && $data['col_bool'][3] === null - && $data['col_bool'][4] === null -); - -echo "double: "; -var_dump( - $data['col_double'][0] === null - && $data['col_double'][1] === null - && $data['col_double'][2] === 3.14 - && $data['col_double'][3] === null - && $data['col_double'][4] === 2.71 -); - -echo "decimal: "; -var_dump( - $data['col_decimal'][0] === 1.23 - && $data['col_decimal'][1] === null - && $data['col_decimal'][2] === null - && $data['col_decimal'][3] === -4.56 - && $data['col_decimal'][4] === null -); - -$reader->close(); -?> ---EXPECT-- -int64: bool(true) -string: bool(true) -bool: bool(true) -double: bool(true) -decimal: bool(true) diff --git a/src/extension/arrow-ext/tests/phpt/020_writer_list.phpt b/src/extension/arrow-ext/tests/phpt/020_writer_list.phpt deleted file mode 100644 index b9b69775ac..0000000000 --- a/src/extension/arrow-ext/tests/phpt/020_writer_list.phpt +++ /dev/null @@ -1,83 +0,0 @@ ---TEST-- -Writer roundtrips LIST(STRING) columns through Writer then Reader ---SKIPIF-- - ---FILE-- -data .= $data; - return $this; - } -} - -class TestSourceStream implements Flow\Arrow\RandomAccessFile { - private string $data; - public function __construct(string $data) { - $this->data = $data; - } - public function read(int $length, int $offset): string { - return substr($this->data, $offset, $length); - } - public function size(): ?int { - return strlen($this->data); - } -} - -$dest = new TestDestinationStream(); -$schema = [ - ['name' => 'id', 'type' => 'INT64', 'optional' => false], - ['name' => 'tags', 'type' => 'LIST', 'optional' => true, 'children' => [ - ['name' => 'item', 'type' => 'STRING', 'optional' => true] - ]], -]; - -$writer = new Flow\Arrow\Parquet\Writer($dest, $schema, 'UNCOMPRESSED'); -$writer->writeBatch([ - 'id' => [1, 2, 3], - 'tags' => [["a", "b", "c"], [], null], -]); -$writer->close(); - -$source = new TestSourceStream($dest->data); -$reader = new Flow\Arrow\Parquet\Reader($source); -$data = $reader->readRowGroup(); - -// Row 0: ["a", "b", "c"] -echo "row0 is array: "; -var_dump(is_array($data['tags'][0])); -echo "row0 count: "; -var_dump(count($data['tags'][0])); -echo "row0 values: "; -var_dump($data['tags'][0][0] === "a" && $data['tags'][0][1] === "b" && $data['tags'][0][2] === "c"); - -// Row 1: [] (empty list) -echo "row1 is array: "; -var_dump(is_array($data['tags'][1])); -echo "row1 count: "; -var_dump(count($data['tags'][1])); - -// Row 2: null -echo "row2 is null: "; -var_dump($data['tags'][2] === null); - -// Verify id column -echo "ids: "; -var_dump($data['id'][0] === 1 && $data['id'][1] === 2 && $data['id'][2] === 3); - -$reader->close(); -?> ---EXPECT-- -row0 is array: bool(true) -row0 count: int(3) -row0 values: bool(true) -row1 is array: bool(true) -row1 count: int(0) -row2 is null: bool(true) -ids: bool(true) diff --git a/src/extension/arrow-ext/tests/phpt/021_writer_struct.phpt b/src/extension/arrow-ext/tests/phpt/021_writer_struct.phpt deleted file mode 100644 index 55aed2c1bc..0000000000 --- a/src/extension/arrow-ext/tests/phpt/021_writer_struct.phpt +++ /dev/null @@ -1,79 +0,0 @@ ---TEST-- -Writer roundtrips STRUCT columns through Writer then Reader ---SKIPIF-- - ---FILE-- -data .= $data; - return $this; - } -} - -class TestSourceStream implements Flow\Arrow\RandomAccessFile { - private string $data; - public function __construct(string $data) { - $this->data = $data; - } - public function read(int $length, int $offset): string { - return substr($this->data, $offset, $length); - } - public function size(): ?int { - return strlen($this->data); - } -} - -$dest = new TestDestinationStream(); -$schema = [ - ['name' => 'id', 'type' => 'INT64', 'optional' => false], - ['name' => 'metadata', 'type' => 'STRUCT', 'optional' => true, 'children' => [ - ['name' => 'key', 'type' => 'STRING', 'optional' => false], - ['name' => 'value', 'type' => 'INT64', 'optional' => false], - ]], -]; - -$writer = new Flow\Arrow\Parquet\Writer($dest, $schema, 'UNCOMPRESSED'); -$writer->writeBatch([ - 'id' => [1, 2, 3], - 'metadata' => [['key' => 'x', 'value' => 10], null, ['key' => 'y', 'value' => 20]], -]); -$writer->close(); - -$source = new TestSourceStream($dest->data); -$reader = new Flow\Arrow\Parquet\Reader($source); -$data = $reader->readRowGroup(); - -// Row 0: {key: "x", value: 10} -echo "row0 is array: "; -var_dump(is_array($data['metadata'][0])); -echo "row0 key: "; -var_dump($data['metadata'][0]['key']); -echo "row0 value: "; -var_dump($data['metadata'][0]['value']); - -// Row 1: null -echo "row1 is null: "; -var_dump($data['metadata'][1] === null); - -// Row 2: {key: "y", value: 20} -echo "row2 key: "; -var_dump($data['metadata'][2]['key']); -echo "row2 value: "; -var_dump($data['metadata'][2]['value']); - -$reader->close(); -?> ---EXPECT-- -row0 is array: bool(true) -row0 key: string(1) "x" -row0 value: int(10) -row1 is null: bool(true) -row2 key: string(1) "y" -row2 value: int(20) diff --git a/src/extension/arrow-ext/tests/phpt/022_writer_map.phpt b/src/extension/arrow-ext/tests/phpt/022_writer_map.phpt deleted file mode 100644 index 080f20ec66..0000000000 --- a/src/extension/arrow-ext/tests/phpt/022_writer_map.phpt +++ /dev/null @@ -1,76 +0,0 @@ ---TEST-- -Writer roundtrips MAP(STRING, INT64) columns through Writer then Reader ---SKIPIF-- - ---FILE-- -data .= $data; - return $this; - } -} - -class TestSourceStream implements Flow\Arrow\RandomAccessFile { - private string $data; - public function __construct(string $data) { - $this->data = $data; - } - public function read(int $length, int $offset): string { - return substr($this->data, $offset, $length); - } - public function size(): ?int { - return strlen($this->data); - } -} - -$dest = new TestDestinationStream(); -$schema = [ - ['name' => 'id', 'type' => 'INT64', 'optional' => false], - ['name' => 'props', 'type' => 'MAP', 'optional' => true, 'children' => [ - ['name' => 'keys', 'type' => 'STRING', 'optional' => false], - ['name' => 'values', 'type' => 'INT64', 'optional' => true], - ]], -]; - -$writer = new Flow\Arrow\Parquet\Writer($dest, $schema, 'UNCOMPRESSED'); -$writer->writeBatch([ - 'id' => [1, 2, 3], - 'props' => [['p1' => 100, 'p2' => 200], ['p3' => 300], null], -]); -$writer->close(); - -$source = new TestSourceStream($dest->data); -$reader = new Flow\Arrow\Parquet\Reader($source); -$data = $reader->readRowGroup(); - -// Row 0: {"p1": 100, "p2": 200} -echo "row0 is array: "; -var_dump(is_array($data['props'][0])); -echo "row0 p1: "; -var_dump($data['props'][0]['p1']); -echo "row0 p2: "; -var_dump($data['props'][0]['p2']); - -// Row 1: {"p3": 300} -echo "row1 p3: "; -var_dump($data['props'][1]['p3']); - -// Row 2: null -echo "row2 is null: "; -var_dump($data['props'][2] === null); - -$reader->close(); -?> ---EXPECT-- -row0 is array: bool(true) -row0 p1: int(100) -row0 p2: int(200) -row1 p3: int(300) -row2 is null: bool(true) diff --git a/src/extension/arrow-ext/tests/phpt/023_writer_deeply_nested.phpt b/src/extension/arrow-ext/tests/phpt/023_writer_deeply_nested.phpt deleted file mode 100644 index b1cc90682d..0000000000 --- a/src/extension/arrow-ext/tests/phpt/023_writer_deeply_nested.phpt +++ /dev/null @@ -1,97 +0,0 @@ ---TEST-- -Writer roundtrips deeply nested LIST(STRUCT(STRING, LIST(INT64))) through Writer then Reader ---SKIPIF-- - ---FILE-- -data .= $data; - return $this; - } -} - -class TestSourceStream implements Flow\Arrow\RandomAccessFile { - private string $data; - public function __construct(string $data) { - $this->data = $data; - } - public function read(int $length, int $offset): string { - return substr($this->data, $offset, $length); - } - public function size(): ?int { - return strlen($this->data); - } -} - -$dest = new TestDestinationStream(); -$schema = [ - ['name' => 'groups', 'type' => 'LIST', 'optional' => true, 'children' => [ - ['name' => 'item', 'type' => 'STRUCT', 'optional' => true, 'children' => [ - ['name' => 'name', 'type' => 'STRING', 'optional' => false], - ['name' => 'scores', 'type' => 'LIST', 'optional' => true, 'children' => [ - ['name' => 'item', 'type' => 'INT64', 'optional' => true] - ]], - ]], - ]], -]; - -$writer = new Flow\Arrow\Parquet\Writer($dest, $schema, 'UNCOMPRESSED'); -$writer->writeBatch([ - 'groups' => [ - // Row 0: [{name:"alpha", scores:[10,20]}, {name:"beta", scores:[30]}] - [['name' => 'alpha', 'scores' => [10, 20]], ['name' => 'beta', 'scores' => [30]]], - // Row 1: [{name:"gamma", scores:[]}] - [['name' => 'gamma', 'scores' => []]], - // Row 2: null - null, - ], -]); -$writer->close(); - -$source = new TestSourceStream($dest->data); -$reader = new Flow\Arrow\Parquet\Reader($source); -$data = $reader->readRowGroup(); - -// Row 0: [{name:"alpha", scores:[10,20]}, {name:"beta", scores:[30]}] -echo "row0 count: "; -var_dump(count($data['groups'][0])); -echo "row0[0] name: "; -var_dump($data['groups'][0][0]['name']); -echo "row0[0] scores: "; -var_dump($data['groups'][0][0]['scores'][0] === 10 && $data['groups'][0][0]['scores'][1] === 20); -echo "row0[1] name: "; -var_dump($data['groups'][0][1]['name']); -echo "row0[1] scores: "; -var_dump($data['groups'][0][1]['scores'][0] === 30); - -// Row 1: [{name:"gamma", scores:[]}] -echo "row1 count: "; -var_dump(count($data['groups'][1])); -echo "row1[0] name: "; -var_dump($data['groups'][1][0]['name']); -echo "row1[0] scores count: "; -var_dump(count($data['groups'][1][0]['scores'])); - -// Row 2: null -echo "row2 is null: "; -var_dump($data['groups'][2] === null); - -$reader->close(); -?> ---EXPECT-- -row0 count: int(2) -row0[0] name: string(5) "alpha" -row0[0] scores: bool(true) -row0[1] name: string(4) "beta" -row0[1] scores: bool(true) -row1 count: int(1) -row1[0] name: string(5) "gamma" -row1[0] scores count: int(0) -row2 is null: bool(true) diff --git a/src/extension/arrow-ext/tests/phpt/024_writer_roundtrip_all_types.phpt b/src/extension/arrow-ext/tests/phpt/024_writer_roundtrip_all_types.phpt deleted file mode 100644 index ecbc7aa8ed..0000000000 --- a/src/extension/arrow-ext/tests/phpt/024_writer_roundtrip_all_types.phpt +++ /dev/null @@ -1,115 +0,0 @@ ---TEST-- -Writer roundtrips flat and nested types combined in a single batch ---SKIPIF-- - ---FILE-- -data .= $data; - return $this; - } -} - -class TestSourceStream implements Flow\Arrow\RandomAccessFile { - private string $data; - public function __construct(string $data) { - $this->data = $data; - } - public function read(int $length, int $offset): string { - return substr($this->data, $offset, $length); - } - public function size(): ?int { - return strlen($this->data); - } -} - -$dest = new TestDestinationStream(); -$schema = [ - ['name' => 'id', 'type' => 'INT64', 'optional' => false], - ['name' => 'name', 'type' => 'STRING', 'optional' => true], - ['name' => 'score', 'type' => 'DOUBLE', 'optional' => true], - ['name' => 'tags', 'type' => 'LIST', 'optional' => true, 'children' => [ - ['name' => 'item', 'type' => 'STRING', 'optional' => true] - ]], - ['name' => 'meta', 'type' => 'STRUCT', 'optional' => true, 'children' => [ - ['name' => 'key', 'type' => 'STRING', 'optional' => false], - ['name' => 'val', 'type' => 'INT64', 'optional' => true], - ]], - ['name' => 'attrs', 'type' => 'MAP', 'optional' => true, 'children' => [ - ['name' => 'keys', 'type' => 'STRING', 'optional' => false], - ['name' => 'values', 'type' => 'STRING', 'optional' => true], - ]], -]; - -$writer = new Flow\Arrow\Parquet\Writer($dest, $schema, 'UNCOMPRESSED'); -$writer->writeBatch([ - 'id' => [1, 2, 3], - 'name' => ['Alice', null, 'Charlie'], - 'score' => [9.5, 8.0, null], - 'tags' => [['php', 'rust'], [], null], - 'meta' => [['key' => 'role', 'val' => 1], null, ['key' => 'team', 'val' => null]], - 'attrs' => [['color' => 'red', 'size' => 'large'], ['color' => 'blue'], null], -]); -$writer->close(); - -$source = new TestSourceStream($dest->data); -$reader = new Flow\Arrow\Parquet\Reader($source); -$data = $reader->readRowGroup(); - -// Flat columns -echo "id: "; -var_dump($data['id'][0] === 1 && $data['id'][1] === 2 && $data['id'][2] === 3); -echo "name: "; -var_dump($data['name'][0] === 'Alice' && $data['name'][1] === null && $data['name'][2] === 'Charlie'); -echo "score: "; -var_dump($data['score'][0] === 9.5 && $data['score'][1] === 8.0 && $data['score'][2] === null); - -// LIST -echo "tags0: "; -var_dump($data['tags'][0][0] === 'php' && $data['tags'][0][1] === 'rust'); -echo "tags1: "; -var_dump(is_array($data['tags'][1]) && count($data['tags'][1]) === 0); -echo "tags2: "; -var_dump($data['tags'][2] === null); - -// STRUCT -echo "meta0: "; -var_dump($data['meta'][0]['key'] === 'role' && $data['meta'][0]['val'] === 1); -echo "meta1: "; -var_dump($data['meta'][1] === null); -echo "meta2 key: "; -var_dump($data['meta'][2]['key'] === 'team'); -echo "meta2 val: "; -var_dump($data['meta'][2]['val'] === null); - -// MAP -echo "attrs0: "; -var_dump($data['attrs'][0]['color'] === 'red' && $data['attrs'][0]['size'] === 'large'); -echo "attrs1: "; -var_dump($data['attrs'][1]['color'] === 'blue'); -echo "attrs2: "; -var_dump($data['attrs'][2] === null); - -$reader->close(); -?> ---EXPECT-- -id: bool(true) -name: bool(true) -score: bool(true) -tags0: bool(true) -tags1: bool(true) -tags2: bool(true) -meta0: bool(true) -meta1: bool(true) -meta2 key: bool(true) -meta2 val: bool(true) -attrs0: bool(true) -attrs1: bool(true) -attrs2: bool(true) diff --git a/src/extension/arrow-ext/tests/phpt/025_writer_options_row_group_size.phpt b/src/extension/arrow-ext/tests/phpt/025_writer_options_row_group_size.phpt deleted file mode 100644 index 4b718445c9..0000000000 --- a/src/extension/arrow-ext/tests/phpt/025_writer_options_row_group_size.phpt +++ /dev/null @@ -1,63 +0,0 @@ ---TEST-- -Writer respects ROW_GROUP_SIZE_BYTES option to produce multiple row groups ---SKIPIF-- - ---FILE-- -data .= $data; - return $this; - } -} - -class TestSourceStream implements Flow\Arrow\RandomAccessFile { - private string $data; - public function __construct(string $data) { - $this->data = $data; - } - public function read(int $length, int $offset): string { - return substr($this->data, $offset, $length); - } - public function size(): ?int { - return strlen($this->data); - } -} - -$schema = [ - ['name' => 'id', 'type' => 'INT64', 'optional' => false], - ['name' => 'name', 'type' => 'STRING', 'optional' => false], -]; - -$dest = new TestDestinationStream(); -$options = ['ROW_GROUP_SIZE_BYTES' => 500]; -$writer = new Flow\Arrow\Parquet\Writer($dest, $schema, 'UNCOMPRESSED', $options); -for ($i = 0; $i < 100; $i++) { - $batch = [ - 'id' => [($i * 10) + 1, ($i * 10) + 2, ($i * 10) + 3, ($i * 10) + 4, ($i * 10) + 5, - ($i * 10) + 6, ($i * 10) + 7, ($i * 10) + 8, ($i * 10) + 9, ($i * 10) + 10], - 'name' => [str_repeat('x', 100), str_repeat('y', 100), str_repeat('z', 100), - str_repeat('a', 100), str_repeat('b', 100), str_repeat('c', 100), - str_repeat('d', 100), str_repeat('e', 100), str_repeat('f', 100), - str_repeat('g', 100)], - ]; - $writer->writeBatch($batch); -} -$writer->close(); - -$source = new TestSourceStream($dest->data); -$reader = new Flow\Arrow\Parquet\Reader($source); -$meta = $reader->metadata(); - -echo "multiple_row_groups: "; -var_dump($meta['row_groups'] > 1); -$reader->close(); -?> ---EXPECT-- -multiple_row_groups: bool(true) diff --git a/src/extension/arrow-ext/tests/phpt/026_writer_options_compression_level.phpt b/src/extension/arrow-ext/tests/phpt/026_writer_options_compression_level.phpt deleted file mode 100644 index 7ab7adfb3e..0000000000 --- a/src/extension/arrow-ext/tests/phpt/026_writer_options_compression_level.phpt +++ /dev/null @@ -1,73 +0,0 @@ ---TEST-- -Writer respects compression level options for GZIP, ZSTD, and BROTLI ---SKIPIF-- - ---FILE-- -data .= $data; - return $this; - } -} - -class TestSourceStream implements Flow\Arrow\RandomAccessFile { - private string $data; - public function __construct(string $data) { - $this->data = $data; - } - public function read(int $length, int $offset): string { - return substr($this->data, $offset, $length); - } - public function size(): ?int { - return strlen($this->data); - } -} - -$schema = [ - ['name' => 'id', 'type' => 'INT64', 'optional' => false], - ['name' => 'name', 'type' => 'STRING', 'optional' => false], -]; -$batch = [ - 'id' => [1, 2, 3], - 'name' => ['Alice', 'Bob', 'Charlie'], -]; - -$codecs = [ - 'GZIP' => ['GZIP_COMPRESSION_LEVEL' => 1], - 'ZSTD' => ['ZSTD_COMPRESSION_LEVEL' => 1], - 'BROTLI' => ['BROTLI_COMPRESSION_LEVEL' => 1], -]; - -foreach ($codecs as $codec => $options) { - $dest = new TestDestinationStream(); - $writer = new Flow\Arrow\Parquet\Writer($dest, $schema, $codec, $options); - $writer->writeBatch($batch); - $writer->close(); - - $source = new TestSourceStream($dest->data); - $reader = new Flow\Arrow\Parquet\Reader($source); - $data = $reader->readRowGroup(); - - $ok = $data['id'][0] === 1 - && $data['id'][1] === 2 - && $data['id'][2] === 3 - && $data['name'][0] === 'Alice' - && $data['name'][1] === 'Bob' - && $data['name'][2] === 'Charlie'; - - echo "$codec: "; - var_dump($ok); - $reader->close(); -} -?> ---EXPECT-- -GZIP: bool(true) -ZSTD: bool(true) -BROTLI: bool(true) diff --git a/src/extension/arrow-ext/tests/phpt/027_writer_options_per_column.phpt b/src/extension/arrow-ext/tests/phpt/027_writer_options_per_column.phpt deleted file mode 100644 index 93cfc1f820..0000000000 --- a/src/extension/arrow-ext/tests/phpt/027_writer_options_per_column.phpt +++ /dev/null @@ -1,69 +0,0 @@ ---TEST-- -Writer respects per-column compression via COLUMNS_COMPRESSIONS option ---SKIPIF-- - ---FILE-- -data .= $data; - return $this; - } -} - -class TestSourceStream implements Flow\Arrow\RandomAccessFile { - private string $data; - public function __construct(string $data) { - $this->data = $data; - } - public function read(int $length, int $offset): string { - return substr($this->data, $offset, $length); - } - public function size(): ?int { - return strlen($this->data); - } -} - -$schema = [ - ['name' => 'id', 'type' => 'INT64', 'optional' => false], - ['name' => 'name', 'type' => 'STRING', 'optional' => false], -]; -$batch = [ - 'id' => [1, 2, 3], - 'name' => ['Alice', 'Bob', 'Charlie'], -]; - -$dest = new TestDestinationStream(); -$options = [ - 'COLUMNS_COMPRESSIONS' => [ - 'id' => 'GZIP', - 'name' => 'ZSTD', - ], -]; -$writer = new Flow\Arrow\Parquet\Writer($dest, $schema, 'SNAPPY', $options); -$writer->writeBatch($batch); -$writer->close(); - -$source = new TestSourceStream($dest->data); -$reader = new Flow\Arrow\Parquet\Reader($source); -$data = $reader->readRowGroup(); - -$ok = $data['id'][0] === 1 - && $data['id'][1] === 2 - && $data['id'][2] === 3 - && $data['name'][0] === 'Alice' - && $data['name'][1] === 'Bob' - && $data['name'][2] === 'Charlie'; - -echo "per_column_compression: "; -var_dump($ok); -$reader->close(); -?> ---EXPECT-- -per_column_compression: bool(true) diff --git a/src/extension/arrow-ext/tests/phpt/028_writer_options_writer_version.phpt b/src/extension/arrow-ext/tests/phpt/028_writer_options_writer_version.phpt deleted file mode 100644 index ecfc1860e3..0000000000 --- a/src/extension/arrow-ext/tests/phpt/028_writer_options_writer_version.phpt +++ /dev/null @@ -1,67 +0,0 @@ ---TEST-- -Writer respects WRITER_VERSION option for v1 and v2 ---SKIPIF-- - ---FILE-- -data .= $data; - return $this; - } -} - -class TestSourceStream implements Flow\Arrow\RandomAccessFile { - private string $data; - public function __construct(string $data) { - $this->data = $data; - } - public function read(int $length, int $offset): string { - return substr($this->data, $offset, $length); - } - public function size(): ?int { - return strlen($this->data); - } -} - -$schema = [ - ['name' => 'id', 'type' => 'INT64', 'optional' => false], - ['name' => 'name', 'type' => 'STRING', 'optional' => false], -]; -$batch = [ - 'id' => [1, 2, 3], - 'name' => ['Alice', 'Bob', 'Charlie'], -]; - -foreach ([1, 2] as $version) { - $dest = new TestDestinationStream(); - $options = ['WRITER_VERSION' => $version]; - $writer = new Flow\Arrow\Parquet\Writer($dest, $schema, 'SNAPPY', $options); - $writer->writeBatch($batch); - $writer->close(); - - $source = new TestSourceStream($dest->data); - $reader = new Flow\Arrow\Parquet\Reader($source); - $data = $reader->readRowGroup(); - - $ok = $data['id'][0] === 1 - && $data['id'][1] === 2 - && $data['id'][2] === 3 - && $data['name'][0] === 'Alice' - && $data['name'][1] === 'Bob' - && $data['name'][2] === 'Charlie'; - - echo "v$version: "; - var_dump($ok); - $reader->close(); -} -?> ---EXPECT-- -v1: bool(true) -v2: bool(true) diff --git a/src/extension/arrow-ext/tests/phpt/029_reader_options_unknown_ignored.phpt b/src/extension/arrow-ext/tests/phpt/029_reader_options_unknown_ignored.phpt deleted file mode 100644 index e4e8245acb..0000000000 --- a/src/extension/arrow-ext/tests/phpt/029_reader_options_unknown_ignored.phpt +++ /dev/null @@ -1,58 +0,0 @@ ---TEST-- -Reader silently ignores unknown options ---SKIPIF-- - ---FILE-- -data .= $data; - return $this; - } -} - -class TestSourceStream implements Flow\Arrow\RandomAccessFile { - private string $data; - public function __construct(string $data) { - $this->data = $data; - } - public function read(int $length, int $offset): string { - return substr($this->data, $offset, $length); - } - public function size(): ?int { - return strlen($this->data); - } -} - -$schema = [ - ['name' => 'id', 'type' => 'INT64', 'optional' => false], -]; -$batch = [ - 'id' => [1, 2, 3], -]; - -$dest = new TestDestinationStream(); -$writer = new Flow\Arrow\Parquet\Writer($dest, $schema); -$writer->writeBatch($batch); -$writer->close(); - -$source = new TestSourceStream($dest->data); -$options = [ - 'SOME_UNKNOWN_OPTION' => 'whatever', - 'ANOTHER_ONE' => 42, -]; -$reader = new Flow\Arrow\Parquet\Reader($source, $options); -$data = $reader->readRowGroup(); - -echo "read_ok: "; -var_dump($data['id'][0] === 1 && $data['id'][1] === 2 && $data['id'][2] === 3); -$reader->close(); -?> ---EXPECT-- -read_ok: bool(true) diff --git a/src/extension/arrow-ext/tests/phpt/030_reader_batch_size.phpt b/src/extension/arrow-ext/tests/phpt/030_reader_batch_size.phpt deleted file mode 100644 index 4970bb5ab2..0000000000 --- a/src/extension/arrow-ext/tests/phpt/030_reader_batch_size.phpt +++ /dev/null @@ -1,67 +0,0 @@ ---TEST-- -Reader BATCH_SIZE option returns rows in batches ---SKIPIF-- - ---FILE-- -data .= $data; - return $this; - } -} - -class TestSourceStream implements Flow\Arrow\RandomAccessFile { - private string $data; - public function __construct(string $data) { - $this->data = $data; - } - public function read(int $length, int $offset): string { - return substr($this->data, $offset, $length); - } - public function size(): ?int { - return strlen($this->data); - } -} - -$schema = [ - ['name' => 'id', 'type' => 'INT64', 'optional' => false], -]; - -$dest = new TestDestinationStream(); -$writer = new Flow\Arrow\Parquet\Writer($dest, $schema); -$writer->writeBatch([ - 'id' => [1, 2, 3, 4, 5], -]); -$writer->close(); - -$source = new TestSourceStream($dest->data); -$options = ['BATCH_SIZE' => 2]; -$reader = new Flow\Arrow\Parquet\Reader($source, $options); - -$batch1 = $reader->readRowGroup(); -echo "batch1_count: " . count($batch1['id']) . "\n"; - -$batch2 = $reader->readRowGroup(); -echo "batch2_count: " . count($batch2['id']) . "\n"; - -$batch3 = $reader->readRowGroup(); -echo "batch3_count: " . count($batch3['id']) . "\n"; - -$batch4 = $reader->readRowGroup(); -echo "batch4_null: "; -var_dump($batch4 === null); - -$reader->close(); -?> ---EXPECT-- -batch1_count: 2 -batch2_count: 2 -batch3_count: 1 -batch4_null: bool(true) diff --git a/src/extension/arrow-ext/tests/phpt/031_reader_batch_size_across_row_groups.phpt b/src/extension/arrow-ext/tests/phpt/031_reader_batch_size_across_row_groups.phpt deleted file mode 100644 index 8cd98ba0f7..0000000000 --- a/src/extension/arrow-ext/tests/phpt/031_reader_batch_size_across_row_groups.phpt +++ /dev/null @@ -1,72 +0,0 @@ ---TEST-- -Reader BATCH_SIZE streams transparently across row group boundaries ---SKIPIF-- - ---FILE-- -data .= $data; - return $this; - } -} - -class TestSourceStream implements Flow\Arrow\RandomAccessFile { - private string $data; - public function __construct(string $data) { - $this->data = $data; - } - public function read(int $length, int $offset): string { - return substr($this->data, $offset, $length); - } - public function size(): ?int { - return strlen($this->data); - } -} - -$schema = [ - ['name' => 'id', 'type' => 'INT64', 'optional' => false], - ['name' => 'name', 'type' => 'STRING', 'optional' => false], -]; - -$dest = new TestDestinationStream(); -$writerOptions = ['ROW_GROUP_SIZE_BYTES' => 500]; -$writer = new Flow\Arrow\Parquet\Writer($dest, $schema, 'UNCOMPRESSED', $writerOptions); - -$ids = []; -$names = []; -for ($i = 0; $i < 100; $i++) { - $ids[] = $i; - $names[] = "name_" . $i; -} -$writer->writeBatch(['id' => $ids, 'name' => $names]); -$writer->close(); - -$source = new TestSourceStream($dest->data); -$readerOptions = ['BATCH_SIZE' => 3]; -$reader = new Flow\Arrow\Parquet\Reader($source, $readerOptions); - -$totalRows = 0; -$batchCount = 0; -while (null !== ($batch = $reader->readRowGroup())) { - $totalRows += count($batch['id']); - $batchCount++; -} -$reader->close(); - -echo "total_rows: " . $totalRows . "\n"; -echo "multiple_batches: "; -var_dump($batchCount > 1); -echo "all_rows_read: "; -var_dump($totalRows === 100); -?> ---EXPECT-- -total_rows: 100 -multiple_batches: bool(true) -all_rows_read: bool(true) diff --git a/src/extension/arrow-ext/tests/phpt/032_reader_batch_size_null.phpt b/src/extension/arrow-ext/tests/phpt/032_reader_batch_size_null.phpt deleted file mode 100644 index 1e9fd69fa4..0000000000 --- a/src/extension/arrow-ext/tests/phpt/032_reader_batch_size_null.phpt +++ /dev/null @@ -1,62 +0,0 @@ ---TEST-- -Reader with BATCH_SIZE null returns full row group in one call ---SKIPIF-- - ---FILE-- -data .= $data; - return $this; - } -} - -class TestSourceStream implements Flow\Arrow\RandomAccessFile { - private string $data; - public function __construct(string $data) { - $this->data = $data; - } - public function read(int $length, int $offset): string { - return substr($this->data, $offset, $length); - } - public function size(): ?int { - return strlen($this->data); - } -} - -$schema = [ - ['name' => 'id', 'type' => 'INT64', 'optional' => false], -]; - -$dest = new TestDestinationStream(); -$writer = new Flow\Arrow\Parquet\Writer($dest, $schema); -$writer->writeBatch([ - 'id' => [1, 2, 3, 4, 5], -]); -$writer->close(); - -$source = new TestSourceStream($dest->data); -$options = ['BATCH_SIZE' => null]; -$reader = new Flow\Arrow\Parquet\Reader($source, $options); - -$batch = $reader->readRowGroup(); -echo "count: " . count($batch['id']) . "\n"; -echo "values_ok: "; -var_dump($batch['id'] === [1, 2, 3, 4, 5]); - -$next = $reader->readRowGroup(); -echo "exhausted: "; -var_dump($next === null); - -$reader->close(); -?> ---EXPECT-- -count: 5 -values_ok: bool(true) -exhausted: bool(true) diff --git a/src/extension/arrow-ext/tests/phpt/033_writer_int_overflow.phpt b/src/extension/arrow-ext/tests/phpt/033_writer_int_overflow.phpt deleted file mode 100644 index 3ef2a911b6..0000000000 --- a/src/extension/arrow-ext/tests/phpt/033_writer_int_overflow.phpt +++ /dev/null @@ -1,60 +0,0 @@ ---TEST-- -Writer throws exception when integer value overflows target type ---SKIPIF-- - ---FILE-- -data .= $data; - return $this; - } -} - -$dest = new TestDestinationStream(); -$schema = [ - ['name' => 'col_int8', 'type' => 'INT8'], -]; - -$writer = new Flow\Arrow\Parquet\Writer($dest, $schema, 'UNCOMPRESSED'); - -try { - $writer->writeBatch([ - 'col_int8' => [256], - ]); - echo "ERROR: No exception thrown\n"; -} catch (Flow\Arrow\Parquet\Exception $e) { - echo "Caught expected exception\n"; - echo (str_contains($e->getMessage(), 'out of range') ? "Message contains 'out of range'" : "ERROR: unexpected message: " . $e->getMessage()) . "\n"; - echo (str_contains($e->getMessage(), '256') ? "Message contains the value" : "ERROR: message missing value") . "\n"; -} - -// Also test negative value in unsigned type -$dest2 = new TestDestinationStream(); -$schema2 = [ - ['name' => 'col_uint64', 'type' => 'UINT64'], -]; - -$writer2 = new Flow\Arrow\Parquet\Writer($dest2, $schema2, 'UNCOMPRESSED'); - -try { - $writer2->writeBatch([ - 'col_uint64' => [-1], - ]); - echo "ERROR: No exception thrown\n"; -} catch (Flow\Arrow\Parquet\Exception $e) { - echo "Caught expected exception for negative uint64\n"; - echo (str_contains($e->getMessage(), 'out of range') ? "Message contains 'out of range'" : "ERROR: unexpected message: " . $e->getMessage()) . "\n"; -} -?> ---EXPECT-- -Caught expected exception -Message contains 'out of range' -Message contains the value -Caught expected exception for negative uint64 -Message contains 'out of range' diff --git a/src/extension/arrow-ext/tests/phpt/034_writer_float_overflow.phpt b/src/extension/arrow-ext/tests/phpt/034_writer_float_overflow.phpt deleted file mode 100644 index d731e80dec..0000000000 --- a/src/extension/arrow-ext/tests/phpt/034_writer_float_overflow.phpt +++ /dev/null @@ -1,38 +0,0 @@ ---TEST-- -Writer throws exception when float value overflows Float32 ---SKIPIF-- - ---FILE-- -data .= $data; - return $this; - } -} - -$dest = new TestDestinationStream(); -$schema = [ - ['name' => 'col_float', 'type' => 'FLOAT'], -]; - -$writer = new Flow\Arrow\Parquet\Writer($dest, $schema, 'UNCOMPRESSED'); - -try { - $writer->writeBatch([ - 'col_float' => [1.0e39], - ]); - echo "ERROR: No exception thrown\n"; -} catch (Flow\Arrow\Parquet\Exception $e) { - echo "Caught expected exception\n"; - echo (str_contains($e->getMessage(), 'overflows') ? "Message contains 'overflows'" : "ERROR: unexpected message: " . $e->getMessage()) . "\n"; -} -?> ---EXPECT-- -Caught expected exception -Message contains 'overflows' diff --git a/src/extension/arrow-ext/tests/phpt/036_interfaces_and_exception_registered.phpt b/src/extension/arrow-ext/tests/phpt/036_interfaces_and_exception_registered.phpt deleted file mode 100644 index 13923e021b..0000000000 --- a/src/extension/arrow-ext/tests/phpt/036_interfaces_and_exception_registered.phpt +++ /dev/null @@ -1,30 +0,0 @@ ---TEST-- -arrow extension registers OutputStream, RandomAccessFile interfaces and Parquet\Exception class ---SKIPIF-- - ---FILE-- -isInterface()); -var_dump($r->hasMethod('append')); - -$r = new ReflectionClass('Flow\Arrow\RandomAccessFile'); -var_dump($r->isInterface()); -var_dump($r->hasMethod('read')); -var_dump($r->hasMethod('size')); -?> ---EXPECT-- -bool(true) -bool(true) -bool(true) -bool(true) -bool(true) -bool(true) -bool(true) -bool(true) -bool(true) diff --git a/src/extension/arrow-ext/tests/phpt/037_writer_roundtrip_nested_uuid_json.phpt b/src/extension/arrow-ext/tests/phpt/037_writer_roundtrip_nested_uuid_json.phpt deleted file mode 100644 index 817564b7aa..0000000000 --- a/src/extension/arrow-ext/tests/phpt/037_writer_roundtrip_nested_uuid_json.phpt +++ /dev/null @@ -1,94 +0,0 @@ ---TEST-- -Writer roundtrips uuid and json nested in struct, list and map; reader returns canonical uuid strings at every depth ---SKIPIF-- - ---FILE-- -data .= $data; - return $this; - } -} - -class TestSourceStream implements Flow\Arrow\RandomAccessFile { - private string $data; - public function __construct(string $data) { - $this->data = $data; - } - public function read(int $length, int $offset): string { - return substr($this->data, $offset, $length); - } - public function size(): ?int { - return strlen($this->data); - } -} - -$uuid = 'f6d6e0e8-4b7e-4b0e-8d7a-ff0a0c9c9a5a'; - -$dest = new TestDestinationStream(); -$schema = [ - ['name' => 'top_uuid', 'type' => 'UUID', 'optional' => true], - ['name' => 'body', 'type' => 'STRUCT', 'optional' => true, 'children' => [ - ['name' => 'id', 'type' => 'UUID', 'optional' => true], - ['name' => 'data', 'type' => 'JSON', 'optional' => true], - ['name' => 'inner', 'type' => 'STRUCT', 'optional' => true, 'children' => [ - ['name' => 'deep', 'type' => 'UUID', 'optional' => true], - ]], - ]], - ['name' => 'uuid_list', 'type' => 'LIST', 'optional' => true, 'children' => [ - ['name' => 'element', 'type' => 'UUID', 'optional' => true], - ]], - ['name' => 'uuid_map', 'type' => 'MAP', 'optional' => true, 'children' => [ - ['name' => 'key', 'type' => 'STRING', 'optional' => false], - ['name' => 'value', 'type' => 'UUID', 'optional' => true], - ]], - ['name' => 'json_map', 'type' => 'MAP', 'optional' => true, 'children' => [ - ['name' => 'key', 'type' => 'STRING', 'optional' => false], - ['name' => 'value', 'type' => 'JSON', 'optional' => true], - ]], -]; - -$writer = new Flow\Arrow\Parquet\Writer($dest, $schema, 'UNCOMPRESSED'); -$writer->writeBatch([ - 'top_uuid' => [$uuid], - 'body' => [['id' => $uuid, 'data' => '{"a":1}', 'inner' => ['deep' => $uuid]]], - 'uuid_list' => [[$uuid, $uuid, null]], - 'uuid_map' => [['k' => $uuid]], - 'json_map' => [['k' => '{"d":4}']], -]); -$writer->close(); - -$source = new TestSourceStream($dest->data); -$reader = new Flow\Arrow\Parquet\Reader($source); -$data = $reader->readRowGroup(); - -echo "top_uuid: "; -var_dump($data['top_uuid'][0] === $uuid); -echo "struct id: "; -var_dump($data['body'][0]['id'] === $uuid); -echo "struct json: "; -var_dump($data['body'][0]['data'] === '{"a":1}'); -echo "deep struct uuid: "; -var_dump($data['body'][0]['inner']['deep'] === $uuid); -echo "list: "; -var_dump($data['uuid_list'][0] === [$uuid, $uuid, null]); -echo "uuid map: "; -var_dump($data['uuid_map'][0] === ['k' => $uuid]); -echo "json map: "; -var_dump($data['json_map'][0] === ['k' => '{"d":4}']); -?> ---EXPECT-- -top_uuid: bool(true) -struct id: bool(true) -struct json: bool(true) -deep struct uuid: bool(true) -list: bool(true) -uuid map: bool(true) -json map: bool(true) diff --git a/src/extension/arrow-ext/tests/phpt/038_extension_version.phpt b/src/extension/arrow-ext/tests/phpt/038_extension_version.phpt index 6f1f935c11..2bfd90d11c 100644 --- a/src/extension/arrow-ext/tests/phpt/038_extension_version.phpt +++ b/src/extension/arrow-ext/tests/phpt/038_extension_version.phpt @@ -1,8 +1,8 @@ --TEST-- -extension version is the release tag, or the last tag with commit distance and hash +extension version is the release tag, or the next minor's dev pre-release with commit distance and hash --FILE-- --EXPECT-- int(1) diff --git a/src/extension/arrow-ext/tests/phpt/039_reader_pre_epoch_timestamps.phpt b/src/extension/arrow-ext/tests/phpt/039_reader_pre_epoch_timestamps.phpt deleted file mode 100644 index 511e71049c..0000000000 --- a/src/extension/arrow-ext/tests/phpt/039_reader_pre_epoch_timestamps.phpt +++ /dev/null @@ -1,64 +0,0 @@ ---TEST-- -Reader floors pre-epoch timestamps with a sub-second fraction ---SKIPIF-- - ---FILE-- -data .= $data; - return $this; - } -} - -class TestSourceStream implements Flow\Arrow\RandomAccessFile { - private string $data; - public function __construct(string $data) { - $this->data = $data; - } - public function read(int $length, int $offset): string { - return substr($this->data, $offset, $length); - } - public function size(): ?int { - return strlen($this->data); - } -} - -$dest = new TestDestinationStream(); -$writer = new Flow\Arrow\Parquet\Writer($dest, [ - ['name' => 'ts', 'type' => 'TIMESTAMP', 'optional' => true], - ['name' => 'ts_list', 'type' => 'LIST', 'optional' => true, 'children' => [ - ['name' => 'item', 'type' => 'TIMESTAMP', 'optional' => true], - ]], -], 'UNCOMPRESSED'); -$writer->writeBatch([ - 'ts' => [-1500000, -1, 1577934245678901], - 'ts_list' => [[-1500000, -1, 1577934245678901], null, null], -]); -$writer->close(); - -$source = new TestSourceStream($dest->data); -$reader = new Flow\Arrow\Parquet\Reader($source); -$data = $reader->readRowGroup(); - -foreach ($data['ts'] as $value) { - echo $value->format('Y-m-d H:i:s.u'), "\n"; -} - -foreach ($data['ts_list'][0] as $value) { - echo $value->format('Y-m-d H:i:s.u'), "\n"; -} -?> ---EXPECT-- -1969-12-31 23:59:58.500000 -1969-12-31 23:59:59.999999 -2020-01-02 03:04:05.678901 -1969-12-31 23:59:58.500000 -1969-12-31 23:59:59.999999 -2020-01-02 03:04:05.678901 diff --git a/src/extension/arrow-ext/tests/phpt/040_reader_decimal256.phpt b/src/extension/arrow-ext/tests/phpt/040_reader_decimal256.phpt deleted file mode 100644 index 183d118b01..0000000000 --- a/src/extension/arrow-ext/tests/phpt/040_reader_decimal256.phpt +++ /dev/null @@ -1,51 +0,0 @@ ---TEST-- -Reader converts FIXED_LEN_BYTE_ARRAY decimals of every width, Decimal256 included ---SKIPIF-- - ---FILE-- -data .= $data; - return $this; - } -} - -class TestSourceStream implements Flow\Arrow\RandomAccessFile { - private string $data; - public function __construct(string $data) { - $this->data = $data; - } - public function read(int $length, int $offset): string { - return substr($this->data, $offset, $length); - } - public function size(): ?int { - return strlen($this->data); - } -} - -$source = new TestSourceStream(file_get_contents(__DIR__ . '/../fixtures/decimals_fixed_len.parquet')); -$reader = new Flow\Arrow\Parquet\Reader($source); -$data = $reader->readRowGroup(); - -foreach ([0, 1] as $row) { - foreach (['dec9', 'dec18', 'dec38', 'dec50'] as $column) { - var_dump($data[$column][$row]); - } -} -?> ---EXPECT-- -float(12345.67) -float(1234567890123456.8) -float(1.2345678901234568E+18) -float(1.2345678901234568E+39) -float(-12345.67) -float(-0.01) -float(-12345.67) -float(-12345.67) diff --git a/src/extension/arrow-ext/tests/phpt/041_writer_date_lanes.phpt b/src/extension/arrow-ext/tests/phpt/041_writer_date_lanes.phpt deleted file mode 100644 index fd437a5352..0000000000 --- a/src/extension/arrow-ext/tests/phpt/041_writer_date_lanes.phpt +++ /dev/null @@ -1,61 +0,0 @@ ---TEST-- -Writer DATE int lane is days since epoch, objects write their wall-clock day ---SKIPIF-- - ---FILE-- -data .= $data; - return $this; - } -} - -class TestSourceStream implements Flow\Arrow\RandomAccessFile { - private string $data; - public function __construct(string $data) { - $this->data = $data; - } - public function read(int $length, int $offset): string { - return substr($this->data, $offset, $length); - } - public function size(): ?int { - return strlen($this->data); - } -} - -$dest = new TestDestinationStream(); -$writer = new Flow\Arrow\Parquet\Writer($dest, [ - ['name' => 'd_int', 'type' => 'DATE', 'optional' => true], - ['name' => 'd_object', 'type' => 'DATE', 'optional' => true], -], 'UNCOMPRESSED'); -$writer->writeBatch([ - 'd_int' => [18263, -1], - 'd_object' => [ - new DateTimeImmutable('2024-01-01 00:00:00', new DateTimeZone('Europe/Warsaw')), - new DateTimeImmutable('1969-12-31 12:00:00', new DateTimeZone('UTC')), - ], -]); -$writer->close(); - -$source = new TestSourceStream($dest->data); -$reader = new Flow\Arrow\Parquet\Reader($source); -$data = $reader->readRowGroup(); - -foreach (['d_int', 'd_object'] as $column) { - foreach ($data[$column] as $value) { - echo $value->format('Y-m-d'), "\n"; - } -} -?> ---EXPECT-- -2020-01-02 -1969-12-31 -2024-01-01 -1969-12-31 diff --git a/src/extension/arrow-ext/tests/phpt/042_writer_temporal_units.phpt b/src/extension/arrow-ext/tests/phpt/042_writer_temporal_units.phpt deleted file mode 100644 index 863631e446..0000000000 --- a/src/extension/arrow-ext/tests/phpt/042_writer_temporal_units.phpt +++ /dev/null @@ -1,74 +0,0 @@ ---TEST-- -Writer honours the declared TIMESTAMP and TIME units ---SKIPIF-- - ---FILE-- -data .= $data; - return $this; - } -} - -class TestSourceStream implements Flow\Arrow\RandomAccessFile { - private string $data; - public function __construct(string $data) { - $this->data = $data; - } - public function read(int $length, int $offset): string { - return substr($this->data, $offset, $length); - } - public function size(): ?int { - return strlen($this->data); - } -} - -$time = new DateInterval('PT3H4M5S'); -$time->f = 0.678901; - -$dest = new TestDestinationStream(); -$writer = new Flow\Arrow\Parquet\Writer($dest, [ - ['name' => 'ts_ms', 'type' => 'TIMESTAMP', 'optional' => true, 'unit' => 'MILLIS', 'utc' => false], - ['name' => 'ts_ns', 'type' => 'TIMESTAMP', 'optional' => true, 'unit' => 'NANOS'], - ['name' => 't_ms', 'type' => 'TIME', 'optional' => true, 'unit' => 'MILLIS'], - ['name' => 't_ns', 'type' => 'TIME', 'optional' => true, 'unit' => 'NANOS'], -], 'UNCOMPRESSED'); -$writer->writeBatch([ - 'ts_ms' => [new DateTimeImmutable('2020-01-02 03:04:05.678901 UTC')], - 'ts_ns' => [new DateTimeImmutable('2020-01-02 03:04:05.678901 UTC')], - 't_ms' => [$time], - 't_ns' => [$time], -]); -$writer->close(); - -$source = new TestSourceStream($dest->data); -$reader = new Flow\Arrow\Parquet\Reader($source); -$data = $reader->readRowGroup(); - -echo $data['ts_ms'][0]->format('Y-m-d H:i:s.u'), "\n"; -echo $data['ts_ns'][0]->format('Y-m-d H:i:s.u'), "\n"; -echo $data['t_ms'][0]->format('%H:%I:%S.%F'), "\n"; -echo $data['t_ns'][0]->format('%H:%I:%S.%F'), "\n"; - -try { - $invalid = new TestDestinationStream(); - new Flow\Arrow\Parquet\Writer($invalid, [ - ['name' => 'ts', 'type' => 'TIMESTAMP', 'optional' => true, 'unit' => 'SECONDS'], - ], 'UNCOMPRESSED'); -} catch (Flow\Arrow\Parquet\Exception $e) { - echo $e->getMessage(), "\n"; -} -?> ---EXPECT-- -2020-01-02 03:04:05.678000 -2020-01-02 03:04:05.678901 -03:04:05.678000 -03:04:05.678901 -Column 'ts': unsupported unit 'SECONDS', expected MILLIS, MICROS or NANOS diff --git a/src/extension/arrow-ext/tests/phpt/043_writer_decimal_rounding.phpt b/src/extension/arrow-ext/tests/phpt/043_writer_decimal_rounding.phpt deleted file mode 100644 index af4032e9f2..0000000000 --- a/src/extension/arrow-ext/tests/phpt/043_writer_decimal_rounding.phpt +++ /dev/null @@ -1,53 +0,0 @@ ---TEST-- -Writer rounds float DECIMAL values half away from zero ---SKIPIF-- - ---FILE-- -data .= $data; - return $this; - } -} - -class TestSourceStream implements Flow\Arrow\RandomAccessFile { - private string $data; - public function __construct(string $data) { - $this->data = $data; - } - public function read(int $length, int $offset): string { - return substr($this->data, $offset, $length); - } - public function size(): ?int { - return strlen($this->data); - } -} - -$dest = new TestDestinationStream(); -$writer = new Flow\Arrow\Parquet\Writer($dest, [ - ['name' => 'd', 'type' => 'DECIMAL', 'optional' => true, 'precision' => 9, 'scale' => 2], -], 'UNCOMPRESSED'); -$writer->writeBatch(['d' => [0.125, 2.675, -0.125, 1.005, 9.995, -99.995]]); -$writer->close(); - -$source = new TestSourceStream($dest->data); -$reader = new Flow\Arrow\Parquet\Reader($source); - -foreach ($reader->readRowGroup()['d'] as $value) { - var_dump($value); -} -?> ---EXPECT-- -float(0.13) -float(2.68) -float(-0.13) -float(1.01) -float(10) -float(-100) diff --git a/src/extension/arrow-ext/tests/phpt/044_reader_schema_units_and_decimal256.phpt b/src/extension/arrow-ext/tests/phpt/044_reader_schema_units_and_decimal256.phpt deleted file mode 100644 index 2a8e110a96..0000000000 --- a/src/extension/arrow-ext/tests/phpt/044_reader_schema_units_and_decimal256.phpt +++ /dev/null @@ -1,67 +0,0 @@ ---TEST-- -Reader schema describes temporal units, the UTC flag and Decimal256 columns ---SKIPIF-- - ---FILE-- -data .= $data; - return $this; - } -} - -class TestSourceStream implements Flow\Arrow\RandomAccessFile { - private string $data; - public function __construct(string $data) { - $this->data = $data; - } - public function read(int $length, int $offset): string { - return substr($this->data, $offset, $length); - } - public function size(): ?int { - return strlen($this->data); - } -} - -$dest = new TestDestinationStream(); -$writer = new Flow\Arrow\Parquet\Writer($dest, [ - ['name' => 'ts', 'type' => 'TIMESTAMP', 'optional' => true], - ['name' => 'ts_ms', 'type' => 'TIMESTAMP', 'optional' => true, 'unit' => 'MILLIS', 'utc' => false], - ['name' => 'ts_ns', 'type' => 'TIMESTAMP', 'optional' => true, 'unit' => 'NANOS'], - ['name' => 't_ms', 'type' => 'TIME', 'optional' => true, 'unit' => 'MILLIS'], - ['name' => 't_ns', 'type' => 'TIME', 'optional' => true, 'unit' => 'NANOS'], -], 'UNCOMPRESSED'); -$writer->writeBatch(['ts' => [0], 'ts_ms' => [0], 'ts_ns' => [0], 't_ms' => [0], 't_ns' => [0]]); -$writer->close(); - -$source = new TestSourceStream($dest->data); -$reader = new Flow\Arrow\Parquet\Reader($source); - -foreach ($reader->schema() as $column) { - echo $column['name'], ' ', $column['type'], ' ', $column['unit'], ' ', var_export($column['utc'] ?? null, true), "\n"; -} - -$fixture = new TestSourceStream(file_get_contents(__DIR__ . '/../fixtures/decimals_fixed_len.parquet')); -$decimals = new Flow\Arrow\Parquet\Reader($fixture); - -foreach ($decimals->schema() as $column) { - echo $column['name'], ' ', $column['type'], ' ', $column['precision'], ' ', $column['scale'], "\n"; -} -?> ---EXPECT-- -ts TIMESTAMP MICROS true -ts_ms TIMESTAMP MILLIS false -ts_ns TIMESTAMP NANOS true -t_ms TIME MILLIS NULL -t_ns TIME NANOS NULL -dec9 DECIMAL 9 2 -dec18 DECIMAL 18 2 -dec38 DECIMAL 38 10 -dec50 DECIMAL 50 10 diff --git a/src/extension/arrow-ext/tests/phpt/045_parquet_lib_read_parity.phpt b/src/extension/arrow-ext/tests/phpt/045_parquet_lib_read_parity.phpt new file mode 100644 index 0000000000..a7ce94a2ff --- /dev/null +++ b/src/extension/arrow-ext/tests/phpt/045_parquet_lib_read_parity.phpt @@ -0,0 +1,98 @@ +--TEST-- +new Reader(engine: new RustParquetEngine()) on arrow reads every lib Parquet fixture as Reader::php() does, both refuse it, or it refuses naming Reader::php() +--SKIPIF-- + +--FILE-- + ParquetRows::read(Reader::php()->read($file), $batchSize)); + $native = arrow_outcome(static fn() => ParquetRows::read((new Reader(engine: new RustParquetEngine()))->read($file), $batchSize)); + $name = substr($file, strlen($fixtures) + 1) . ' ' . $batchSize; + + echo match (true) { + arrow_refused($php) && arrow_refused($native) => "{$name}: both refuse\n", + $php === $native => "{$name}: identical\n", + arrow_refused($native) && str_contains($native, 'Reader::php()') => "{$name}: {$native}\n", + default => "{$name}: DIFFER\n php: " . substr($php, 0, 300) . "\n native: " . substr($native, 0, 300) . "\n", + }; + } +} + +$interval = $fixtures . '/EdgeCases/interval.parquet'; +echo 'interval.parquet [id]: ', arrow_outcome(static fn() => ParquetRows::read(Reader::php()->read($interval), 10, ['id'])) === arrow_outcome(static fn() => ParquetRows::read((new Reader(engine: new RustParquetEngine()))->read($interval), 10, ['id'])) ? 'identical' : 'DIFFER', "\n"; + +$everyType = MemoryParquetFile::written(Writer::php(), EveryType::schema(), EveryType::rows()); +echo 'every type: ', arrow_outcome(static fn() => ParquetRows::read(MemoryParquetFile::read(Reader::php(), $everyType), 2)) === arrow_outcome(static fn() => ParquetRows::read(MemoryParquetFile::read(new Reader(engine: new RustParquetEngine()), $everyType), 2)) ? 'identical' : 'DIFFER', "\n"; + +// pre-epoch timestamps with a sub-second fraction are floored, flat and inside a list +$preEpoch = MemoryParquetFile::written( + Writer::php(), + Schema::with(FlatColumn::dateTime('ts'), NestedColumn::list('ts_list', ListElement::datetime())), + array_map( + static fn(DateTimeImmutable $at): array => ['ts' => $at, 'ts_list' => [$at]], + [(new DateTimeImmutable('@0'))->modify('-1500000 microseconds'), (new DateTimeImmutable('@0'))->modify('-1 microsecond'), new DateTimeImmutable('2020-01-02 03:04:05.678901 UTC')], + ), +); +echo 'pre-epoch timestamps: ', arrow_outcome(static fn() => ParquetRows::read(MemoryParquetFile::read(Reader::php(), $preEpoch), 3)) === arrow_outcome(static fn() => ParquetRows::read(MemoryParquetFile::read(new Reader(engine: new RustParquetEngine()), $preEpoch), 3)) ? 'identical' : 'DIFFER', "\n"; + +foreach (ParquetRows::read(MemoryParquetFile::read(new Reader(engine: new RustParquetEngine()), $preEpoch), 3) as $row) { + echo $row['ts']->format('Y-m-d H:i:s.u'), ' ', $row['ts_list'][0]->format('Y-m-d H:i:s.u'), "\n"; +} +?> +--EXPECT-- +columns.required.parquet 100: identical +columns.required.parquet 10000: identical +decimals_fixed_len.parquet 100: identical +decimals_fixed_len.parquet 10000: identical +delta_binary_acked_encoded_integers.parquet 100: identical +delta_binary_acked_encoded_integers.parquet 10000: identical +lists.parquet 100: identical +lists.parquet 10000: identical +logical_types.parquet 100: identical +logical_types.parquet 10000: identical +maps.parquet 100: identical +maps.parquet 10000: identical +multiple_pages.parquet 100: identical +multiple_pages.parquet 10000: identical +pagination_row_group_1kb_5k_rows.snappy.parquet 100: identical +pagination_row_group_1kb_5k_rows.snappy.parquet 10000: identical +primitives.parquet 100: identical +primitives.parquet 10000: identical +structs.parquet 100: identical +structs.parquet 10000: identical +EdgeCases/datapage_v2.snappy.parquet 100: identical +EdgeCases/datapage_v2.snappy.parquet 10000: identical +EdgeCases/int96.parquet 100: identical +EdgeCases/int96.parquet 10000: identical +EdgeCases/interval.parquet 100: Flow\Parquet\Exception\RuntimeException: Parquet column "iv" (Interval(DayTime)) is not supported by the arrow Parquet reader; read the file with \Flow\Parquet\Reader::php() +EdgeCases/interval.parquet 10000: Flow\Parquet\Exception\RuntimeException: Parquet column "iv" (Interval(DayTime)) is not supported by the arrow Parquet reader; read the file with \Flow\Parquet\Reader::php() +EdgeCases/map_keys.parquet 100: both refuse +EdgeCases/map_keys.parquet 10000: both refuse +EdgeCases/nonnullable.impala.parquet 100: identical +EdgeCases/nonnullable.impala.parquet 10000: identical +EdgeCases/null_list.parquet 100: identical +EdgeCases/null_list.parquet 10000: identical +EdgeCases/uint64_overflow.parquet 100: both refuse +EdgeCases/uint64_overflow.parquet 10000: both refuse +EdgeCases/unsigned.parquet 100: identical +EdgeCases/unsigned.parquet 10000: identical +interval.parquet [id]: identical +every type: identical +pre-epoch timestamps: identical +1969-12-31 23:59:58.500000 1969-12-31 23:59:58.500000 +1969-12-31 23:59:59.999999 1969-12-31 23:59:59.999999 +2020-01-02 03:04:05.678901 2020-01-02 03:04:05.678901 diff --git a/src/extension/arrow-ext/tests/phpt/046_parquet_lib_write_parity.phpt b/src/extension/arrow-ext/tests/phpt/046_parquet_lib_write_parity.phpt new file mode 100644 index 0000000000..a5da1e1e67 --- /dev/null +++ b/src/extension/arrow-ext/tests/phpt/046_parquet_lib_write_parity.phpt @@ -0,0 +1,43 @@ +--TEST-- +new Writer(engine: new RustParquetEngine()) on arrow accepts and refuses what Writer::php() does per target type, and what both accept reads back the same +--SKIPIF-- + +--FILE-- + arrow_outcome(static fn() => ParquetRows::read(MemoryParquetFile::read( + Reader::php(), + MemoryParquetFile::written($writer, Schema::with(WriteAcceptance::target($target)), [['c' => WriteAcceptance::input($input)]]), + ), 1)); + $php = $read(Writer::php()); + $native = $read(new Writer(engine: new RustParquetEngine())); + $cells++; + + match (true) { + arrow_refused($php) && arrow_refused($native) => $refused++, + !arrow_refused($php) && $php === $native => $accepted++, + default => print("{$target} <- {$input}: DIFFER\n php: " . substr($php, 0, 200) . "\n native: " . substr($native, 0, 200) . "\n"), + }; + } +} + +echo "{$cells} cells: {$accepted} accepted by both, {$refused} refused by both\n"; +echo 'every type: ', arrow_outcome(static fn() => ParquetRows::read(MemoryParquetFile::read(Reader::php(), MemoryParquetFile::written(Writer::php(), EveryType::schema(), EveryType::rows())), 3)) === arrow_outcome(static fn() => ParquetRows::read(MemoryParquetFile::read(Reader::php(), MemoryParquetFile::written(new Writer(engine: new RustParquetEngine()), EveryType::schema(), EveryType::rows())), 3)) ? 'identical' : 'DIFFER', "\n"; +?> +--EXPECT-- +624 cells: 102 accepted by both, 522 refused by both +every type: identical diff --git a/src/extension/arrow-ext/tests/phpt/047_parquet_footer_thrift_parity.phpt b/src/extension/arrow-ext/tests/phpt/047_parquet_footer_thrift_parity.phpt new file mode 100644 index 0000000000..ee944b0629 --- /dev/null +++ b/src/extension/arrow-ext/tests/phpt/047_parquet_footer_thrift_parity.phpt @@ -0,0 +1,84 @@ +--TEST-- +RustParquetFileReader decodes the footer to the FileMetaData and schema the PHP thrift reader builds, from one footer read +--SKIPIF-- + +--FILE-- +read(new CompactProtocol(new MemoryBuffer(substr($bytes, -8 - $length, $length)))); + + $stream = new ReadCountingSourceStream(NativeLocalSourceStream::open(path_real($path))); + $file = new RustParquetFileReader($stream); + $reads = $stream->reads; + + printf( + "%s: thrift %s, schema %s, footer reads %d\n", + substr($path, strlen($fixtures) + 1), + serialize($file->thrift()) === serialize($php) ? 'identical' : 'DIFFER', + serialize($file->schema()) === serialize(Schema::fromThrift($php->schema)) ? 'identical' : 'DIFFER', + $reads, + ); + $file->close(); +} + +// the temporal units and the UTC flag of every TIMESTAMP / TIME unit +$units = MemoryParquetFile::written(Writer::php(), Schema::with( + FlatColumn::dateTime('ts'), + new FlatColumn('ts_ms', PhysicalType::INT64, logicalType: new LogicalType(LogicalType::TIMESTAMP, timestamp: new Timestamp(false, TimeUnit::MILLISECONDS))), + new FlatColumn('ts_ns', PhysicalType::INT64, logicalType: new LogicalType(LogicalType::TIMESTAMP, timestamp: new Timestamp(true, TimeUnit::NANOSECONDS))), + new FlatColumn('t_ms', PhysicalType::INT32, logicalType: new LogicalType(LogicalType::TIME, time: new Time(false, TimeUnit::MILLISECONDS))), + new FlatColumn('t_ns', PhysicalType::INT64, logicalType: new LogicalType(LogicalType::TIME, time: new Time(false, TimeUnit::NANOSECONDS))), +), [['ts' => null, 'ts_ms' => null, 'ts_ns' => null, 't_ms' => null, 't_ns' => null]]); +$length = unpack('V', substr($units, -8, 4))[1]; +$php = new FileMetaData(); +$php->read(new CompactProtocol(new MemoryBuffer(substr($units, -8 - $length, $length)))); +$file = new RustParquetFileReader(new MemorySourceStream($units)); + +printf( + "temporal units: thrift %s, schema %s\n", + serialize($file->thrift()) === serialize($php) ? 'identical' : 'DIFFER', + serialize($file->schema()) === serialize(Schema::fromThrift($php->schema)) ? 'identical' : 'DIFFER', +); +$file->close(); +?> +--EXPECT-- +columns.required.parquet: thrift identical, schema identical, footer reads 1 +decimals_fixed_len.parquet: thrift identical, schema identical, footer reads 1 +delta_binary_acked_encoded_integers.parquet: thrift identical, schema identical, footer reads 1 +lists.parquet: thrift identical, schema identical, footer reads 1 +logical_types.parquet: thrift identical, schema identical, footer reads 1 +maps.parquet: thrift identical, schema identical, footer reads 1 +multiple_pages.parquet: thrift identical, schema identical, footer reads 1 +pagination_row_group_1kb_5k_rows.snappy.parquet: thrift identical, schema identical, footer reads 2 +primitives.parquet: thrift identical, schema identical, footer reads 1 +structs.parquet: thrift identical, schema identical, footer reads 1 +EdgeCases/datapage_v2.snappy.parquet: thrift identical, schema identical, footer reads 1 +EdgeCases/int96.parquet: thrift identical, schema identical, footer reads 1 +EdgeCases/interval.parquet: thrift identical, schema identical, footer reads 1 +EdgeCases/map_keys.parquet: thrift identical, schema identical, footer reads 1 +EdgeCases/nonnullable.impala.parquet: thrift identical, schema identical, footer reads 1 +EdgeCases/null_list.parquet: thrift identical, schema identical, footer reads 1 +EdgeCases/uint64_overflow.parquet: thrift identical, schema identical, footer reads 1 +EdgeCases/unsigned.parquet: thrift identical, schema identical, footer reads 1 +temporal units: thrift identical, schema identical diff --git a/src/extension/arrow-ext/tests/phpt/048_parquet_lib_no_leaks.phpt b/src/extension/arrow-ext/tests/phpt/048_parquet_lib_no_leaks.phpt new file mode 100644 index 0000000000..d59a7583b7 --- /dev/null +++ b/src/extension/arrow-ext/tests/phpt/048_parquet_lib_no_leaks.phpt @@ -0,0 +1,79 @@ +--TEST-- +RustParquetFileReader, RustColumnsReader and RustParquetFileWriter leak no PHP memory, refusals and Traversable batches included +--SKIPIF-- + +--FILE-- + $column->name(), EveryType::schema()->columns()); + +$cycle = static function () use ($schema, $options, $rows, $columns): void { + $filesystem = memory_filesystem(); + $writer = new RustParquetFileWriter($filesystem->writeTo(path('memory://leaks.parquet')), $schema, Compressions::SNAPPY, $options, 2); + $writer->writeRows($rows); + $writer->writeRow($rows[0]); + $writer->writeBatch((static function () use ($rows): Generator { + yield from $rows; + })()); + $writer->writeBatch(new ArrayIterator($rows)); + + try { + $writer->writeRow(['int32' => 'not an int']); + throw new LogicException('a string was written into INT32'); + } catch (Flow\Parquet\Exception\ValidationException) { + } + + $writer->close(); + + $file = new RustParquetFileReader($filesystem->readFrom(path('memory://leaks.parquet'))); + $file->thrift(); + $file->schema(); + $file->metadata(); + + foreach (new RustColumnsReader($file, $columns, 2, 1, 3) as $chunk) { + } + + foreach ($file->readColumns($columns, 2, 3, 1) as $chunk) { + } + + try { + new RustColumnsReader($file, ['missing'], 2, null, null); + throw new LogicException('a missing column was read'); + } catch (Flow\Parquet\Exception\InvalidArgumentException) { + } + + $file->close(); + + $abandoned = new RustParquetFileWriter($filesystem->writeTo(path('memory://abandoned.parquet')), $schema, Compressions::SNAPPY, $options, 2); + $abandoned->writeRows($rows); +}; + +// Schema::fromThrift() / Metadata::fromThrift() warm the allocator up once, whoever calls them +for ($i = 0; $i < 100; $i++) { + $cycle(); +} +gc_collect_cycles(); +$baseline = memory_get_usage(false); + +for ($i = 0; $i < 200; $i++) { + $cycle(); +} +gc_collect_cycles(); + +var_dump(memory_get_usage(false) <= $baseline); +?> +--EXPECT-- +bool(true) diff --git a/src/extension/arrow-ext/tests/phpt/049_parquet_write_columns_parity.phpt b/src/extension/arrow-ext/tests/phpt/049_parquet_write_columns_parity.phpt new file mode 100644 index 0000000000..7b1543897a --- /dev/null +++ b/src/extension/arrow-ext/tests/phpt/049_parquet_write_columns_parity.phpt @@ -0,0 +1,102 @@ +--TEST-- +RustParquetFileWriter::writeColumns() accepts, refuses and keeps what writeRows() does, for every write-acceptance cell and across flushed batches +--SKIPIF-- + +--FILE-- +set(Option::ARROW_WRITE_BATCH_SIZE, $batchSize)), $stream, $schema); + $refusal = 'written'; + + try { + $door === 'rows' ? $file->writeBatch($rows) : $file->writeColumns(ColumnDoor::columns($rows)); + } catch (Throwable $e) { + $refusal = $e::class . ': ' . $e->getMessage(); + } + + $file->close(); + + return $refusal . ' ' . var_export(ColumnDoor::read($stream), true); +}; + +$identical = 0; +$refused = 0; +$cells = 0; + +foreach (WriteAcceptance::targets() as $target) { + foreach (WriteAcceptance::inputs() as $input) { + $schema = Schema::with(WriteAcceptance::target($target)); + // an accepted row before the cell: a refusal keeps it + $rows = [['c' => null], ['c' => WriteAcceptance::input($input)]]; + $cells++; + + try { + $rowsOutcome = $outcome($schema, $rows, 'rows'); + $columnsOutcome = $outcome($schema, $rows, 'columns'); + } catch (Throwable $e) { + // a REQUIRED target refuses the null row before the cell is reached, through both doors + $rowsOutcome = $columnsOutcome = $e::class; + } + + $refused += (int) !str_starts_with($rowsOutcome, 'written'); + + if ($rowsOutcome === $columnsOutcome) { + $identical++; + } else { + echo "{$target} <- {$input}\n rows: {$rowsOutcome}\n columns: {$columnsOutcome}\n"; + } + } +} + +echo "{$identical} of {$cells} cells identical, ", $refused > 0 ? 'some refused' : 'none refused', "\n"; + +$schema = Schema::with(FlatColumn::int32('a'), FlatColumn::int32('b'), FlatColumn::int32('c')); +$rows = array_map(static fn(int $i): array => ['a' => $i, 'b' => $i, 'c' => $i], range(0, 9)); +$rows[7]['a'] = 'seven'; +$rows[4]['c'] = 'four'; +$rows[4]['b'] = 'four'; + +foreach ([1_000, 3, 2, 1] as $batchSize) { + $rowsOutcome = $outcome($schema, $rows, 'rows', $batchSize); + echo "batch size {$batchSize}: ", $rowsOutcome === $outcome($schema, $rows, 'columns', $batchSize) ? strtok($rowsOutcome, "\n") : 'DIFFERENT', "\n"; +} + +$columns = ColumnDoor::columns(ColumnDoor::rows(50)); +$batch = ColumnDoor::stream(); +$file = ColumnDoor::open(new RustParquetEngine(Options::default()->set(Option::ARROW_WRITE_BATCH_SIZE, 7)), $batch, ColumnDoor::schema()); +$file->writeBatch(ColumnDoor::rows(50)); +$file->close(); +$door = ColumnDoor::stream(); +$file = ColumnDoor::open(new RustParquetEngine(Options::default()->set(Option::ARROW_WRITE_BATCH_SIZE, 7)), $door, ColumnDoor::schema()); +$file->writeColumns($columns); +$file->close(); +var_dump($batch->content() === $door->content()); + +foreach ([['a' => [1, 2], 'b' => [1]], ['a' => 'no list']] as $columns) { + try { + ColumnDoor::open(new RustParquetEngine(), ColumnDoor::stream(), $schema)->writeColumns($columns); + } catch (Throwable $e) { + echo $e::class, ': ', $e->getMessage(), "\n"; + } +} +?> +--EXPECT-- +624 of 624 cells identical, some refused +batch size 1000: Flow\Parquet\Exception\ValidationException: Column "b" row 4: expected int, got string array ( +batch size 3: Flow\Parquet\Exception\ValidationException: Column "b" row 1: expected int, got string array ( +batch size 2: Flow\Parquet\Exception\ValidationException: Column "b" row 0: expected int, got string array ( +batch size 1: Flow\Parquet\Exception\ValidationException: Column "b" row 0: expected int, got string array ( +bool(true) +Flow\Parquet\Exception\InvalidArgumentException: writeColumns() takes lists of one length, got "a": 2, "b": 1 +Flow\Parquet\Exception\InvalidArgumentException: arrow Parquet writer columns must be arrays diff --git a/src/extension/arrow-ext/tests/phpt/050_parquet_batch_roundtrip.phpt b/src/extension/arrow-ext/tests/phpt/050_parquet_batch_roundtrip.phpt new file mode 100644 index 0000000000..95aa5e661f --- /dev/null +++ b/src/extension/arrow-ext/tests/phpt/050_parquet_batch_roundtrip.phpt @@ -0,0 +1,94 @@ +--TEST-- +RustBatchReader batches written back through RustParquetFileWriter::writeArrowBatch() read as the file; writer columns the batch lacks are nulls, children the writer lacks are ignored, a batch imports once, userland objects are refused, nothing leaks +--SKIPIF-- + +--FILE-- + ['id' => $i, 'name' => $i % 3 ? "name {$i}" : null, 'tags' => $i % 4 ? [$i, $i + 1] : null], range(0, 249)); + +$roundTrip = static function (Filesystem $filesystem) use ($source, $target, $options, $rows): array { + $writer = new RustParquetFileWriter($filesystem->writeTo(path('memory://source.parquet')), SchemaConverter::toExtension($source), Compressions::SNAPPY, $options, 100); + $writer->writeRows($rows); + $writer->close(); + + $batches = new RustBatchReader(new RustParquetFileReader($filesystem->readFrom(path('memory://source.parquet'))), ['id', 'name', 'tags'], 100, null, null); + $schema = $batches->schema(); + $copy = new RustParquetFileWriter($filesystem->writeTo(path('memory://copy.parquet')), SchemaConverter::toExtension($target), Compressions::SNAPPY, $options, 100); + $counts = []; + + while (($batch = $batches->next()) !== null) { + $counts[] = $batch->count(); + $copy->writeArrowBatch($batch); + } + + $copy->close(); + + return [$schema, $counts]; +}; + +$filesystem = memory_filesystem(); +[$schema, $counts] = $roundTrip($filesystem); +$expected = array_map(static fn(array $row): array => ['id' => $row['id'], 'name' => $row['name'], 'missing' => null], $rows); + +echo get_class($schema), ' ', $schema->arrowSchemaAddress() > 0 ? 'has an address' : 'NO ADDRESS', "\n"; +echo 'batches: ', implode(',', $counts), "\n"; +echo 'round trip: ', iterator_to_array(Reader::php()->readStream($filesystem->readFrom(path('memory://copy.parquet')))->values(), false) === $expected ? 'identical' : 'DIFFER', "\n"; + +$batches = new RustBatchReader(new RustParquetFileReader($filesystem->readFrom(path('memory://source.parquet'))), ['id', 'name', 'tags'], 100, null, null); +$batch = $batches->next(); +$twice = new RustParquetFileWriter($filesystem->writeTo(path('memory://twice.parquet')), SchemaConverter::toExtension($target), Compressions::SNAPPY, $options, 100); +$twice->writeArrowBatch($batch); +echo arrow_outcome(static fn() => $twice->writeArrowBatch($batch)), "\n"; +echo arrow_outcome(static fn() => $twice->writeArrowBatch(new UserlandBatch())), "\n"; +$twice->close(); + +for ($i = 0; $i < 10; $i++) { + $roundTrip(memory_filesystem()); +} +gc_collect_cycles(); +$baseline = memory_get_usage(false); + +for ($i = 0; $i < 100; $i++) { + $roundTrip(memory_filesystem()); +} +gc_collect_cycles(); + +var_dump(memory_get_usage(false) <= $baseline); +?> +--EXPECT-- +Flow\Arrow\RustArrowSchema has an address +batches: 100,100,50 +round trip: identical +Flow\Parquet\Exception\InvalidArgumentException: Arrow C Data batch is already imported +Flow\Parquet\Exception\InvalidArgumentException: Arrow C Data batch must be an extension's class, got UserlandBatch +bool(true) diff --git a/src/extension/arrow-ext/tests/phpt/051_parquet_interfaces_reflection.phpt b/src/extension/arrow-ext/tests/phpt/051_parquet_interfaces_reflection.phpt new file mode 100644 index 0000000000..e99a778453 --- /dev/null +++ b/src/extension/arrow-ext/tests/phpt/051_parquet_interfaces_reflection.phpt @@ -0,0 +1,26 @@ +--TEST-- +arrow registers the flow-php/parquet interfaces reflection-identical to the package files +--SKIPIF-- + +--FILE-- +getParameters()[$position]->getType(), "\n"; +} +?> +--EXPECT-- +library reflected +identical +Flow\Parquet\ParquetEngine::writeRows: iterable +Flow\Parquet\ParquetFileWriter::writeBatch: iterable diff --git a/src/extension/arrow-ext/tests/phpt/052_parquet_file_reader_writer_contracts.phpt b/src/extension/arrow-ext/tests/phpt/052_parquet_file_reader_writer_contracts.phpt new file mode 100644 index 0000000000..35320738f6 --- /dev/null +++ b/src/extension/arrow-ext/tests/phpt/052_parquet_file_reader_writer_contracts.phpt @@ -0,0 +1,201 @@ +--TEST-- +RustParquetFileReader implements ParquetFileReader and RustParquetFileWriter implements ParquetFileWriter: the lib's metadata and schema, RustColumnsReader as an Iterator that streams once, INT96 and LZO refusals, closed reader and writer texts +--SKIPIF-- + +--FILE-- + new RustParquetFileReader( + NativeLocalSourceStream::open(path_real("{$fixtures}/{$name}")), + $int96AsDatetime, +); + +echo "-- reader\n"; +$file = $open('structs.parquet'); +$metadata = $file->metadata(); +$bytes = array_sum(array_map(static fn($group): int => $group->totalByteSize(), $metadata->rowGroups()->all())); +var_dump($file instanceof ParquetFileReader); +var_dump(serialize($metadata) === serialize(Reader::php()->read("{$fixtures}/structs.parquet")->metadata())); +var_dump(serialize($open('structs.parquet')->schema()) === serialize($metadata->schema()), $file->schema() === $metadata->schema()); +var_dump($file->rowsNumber() === $metadata->rowsNumber(), $file->totalByteSize() === $bytes); + +$chunks = $file->readColumns(['struct_flat', 'struct_flat.int'], 30, 70, 5); +$php = (new PhpParquetEngine(ByteOrder::LITTLE_ENDIAN, new Options())) + ->openForRead(NativeLocalSourceStream::open(path_real("{$fixtures}/structs.parquet"))) + ->readColumns(['struct_flat', 'struct_flat.int'], 30, 70, 5); +var_dump($chunks instanceof Iterator); +$read = []; + +foreach ($chunks as $key => $chunk) { + $read[$key] = $chunk; +} + +echo json_encode(array_map(static fn(array $chunk): int => count($chunk['struct_flat.int']), $read)), "\n"; +var_dump($read == iterator_to_array($php)); +echo arrow_outcome(static fn() => $chunks->rewind()), "\n"; +echo arrow_outcome(static fn() => iterator_to_array($open('EdgeCases/int96.parquet', false)->readColumns(['ts'], 10, null, null))), "\n"; +var_dump(count(iterator_to_array($open('EdgeCases/int96.parquet')->readColumns(['ts'], 10, null, null))) > 0); +echo arrow_outcome(static fn() => $open('lists.parquet')->readColumns(['list.list.element'], 10, null, null)), "\n"; +echo arrow_outcome(static fn() => new RustParquetFileReader(new MemorySourceStream('{"not": "parquet"}'))), "\n"; + +$file->close(); + +foreach ([ + static fn() => $file->metadata(), + static fn() => $file->schema(), + static fn() => $file->rowsNumber(), + static fn() => $file->totalByteSize(), + static fn() => $file->thrift(), + static fn() => $file->readColumns(['struct_flat'], 1, null, null), + static fn() => $file->close(), +] as $call) { + echo arrow_outcome($call), "\n"; +} + +echo "-- writer\n"; +$filesystem = memory_filesystem(); +$schema = Schema::with(FlatColumn::int32('id'), FlatColumn::string('name')); +$writer = static fn($stream, Compressions $compression = Compressions::SNAPPY, int $batchSize = 1_000): RustParquetFileWriter => new RustParquetFileWriter( + $stream, + SchemaConverter::toExtension($schema), + $compression, + OptionsConverter::toExtension(new Options()), + $batchSize, +); +$values = static fn(string $uri): string => json_encode(iterator_to_array(Reader::php()->readStream($filesystem->readFrom(path($uri)))->values(), false)); + +$file = $writer($filesystem->writeTo(path('memory://doors.parquet')), Compressions::GZIP, 2); +var_dump($file instanceof ParquetFileWriter); +$file->writeRow(['id' => 1]); +$file->writeBatch([['id' => 2], 'key' => ['id' => 3, 'name' => 'c']]); +$file->writeBatch((static function () { + yield ['id' => 4]; + yield 'key' => ['id' => 5]; +})()); +echo arrow_outcome(static fn() => $file->writeRow(['id' => 'six'])), "\n"; +$file->writeBatch(new ArrayIterator([['id' => 6]])); +$file->writeColumns(['id' => [7, 8], 'unknown' => ['x', 'y']]); +echo arrow_outcome(static fn() => $file->writeBatch((static function () { + yield ['id' => 9]; + + throw new LogicException('the rows ran out'); +})())), "\n"; +$file->close(); +echo $values('memory://doors.parquet'), "\n"; + +foreach ([ + static fn() => $file->writeRow(['id' => 1]), + static fn() => $file->writeRows([['id' => 1]]), + static fn() => $file->writeBatch([]), + static fn() => $file->writeBatch(new ArrayIterator([['id' => 1]])), + static fn() => $file->writeColumns(['id' => [1]]), + static fn() => $file->close(), +] as $call) { + echo arrow_outcome($call), "\n"; +} + +$failing = $writer(new FailingCloseDestinationStream($filesystem->writeTo(path('memory://failing.parquet')))); +echo arrow_outcome(static fn() => $failing->close()), "\n"; +echo arrow_outcome(static fn() => $failing->close()), "\n"; +// a refused flush still closes the stream; the flush's refusal is the one thrown +$inner = new class($filesystem->writeTo(path('memory://append.parquet'))) implements Flow\Filesystem\DestinationStream { + public bool $closed = false; + + public function __construct(private Flow\Filesystem\DestinationStream $stream) {} + + public function append(string $data): self + { + $this->stream->append($data); + + return $this; + } + + public function close(): void + { + $this->closed = true; + } + + public function fromResource($resource): self + { + return $this; + } + + public function isOpen(): bool + { + return !$this->closed; + } + + public function path(): Flow\Filesystem\Path + { + return $this->stream->path(); + } +}; +$appending = $writer(new FailingAppendDestinationStream($inner)); +$appending->writeRow(['id' => 1, 'name' => 'a']); +echo arrow_outcome(static fn() => $appending->close()), "\n"; +var_dump($inner->closed); +echo arrow_outcome(static fn() => $writer($filesystem->writeTo(path('memory://lzo.parquet')), Compressions::LZO)), "\n"; +?> +--EXPECT-- +-- reader +bool(true) +bool(true) +bool(true) +bool(true) +bool(true) +bool(true) +bool(true) +[30,30,10] +bool(true) +Flow\Parquet\Exception\RuntimeException: RustColumnsReader cannot rewind +Flow\Parquet\Exception\InvalidArgumentException: Parquet column "ts" holds INT96, which arrow reads only as a datetime (Option::INT_96_AS_DATETIME = true); read the file with \Flow\Parquet\Reader::php() +bool(true) +Flow\Parquet\Exception\RuntimeException: Parquet column "list.list.element" (a path into LIST/MAP) is not supported by the arrow Parquet reader; read the file with \Flow\Parquet\Reader::php() +Flow\Parquet\Exception\InvalidArgumentException: Given file is not valid Parquet file: Invalid Parquet file. Corrupt footer +Flow\Parquet\Exception\RuntimeException: Reader is not open +Flow\Parquet\Exception\RuntimeException: Reader is not open +Flow\Parquet\Exception\RuntimeException: Reader is not open +Flow\Parquet\Exception\RuntimeException: Reader is not open +Flow\Parquet\Exception\RuntimeException: Reader is not open +Flow\Parquet\Exception\RuntimeException: Reader is not open +Flow\Parquet\Exception\RuntimeException: Reader is not open +-- writer +bool(true) +Flow\Parquet\Exception\ValidationException: Column "id" row 1: expected int, got string +LogicException: the rows ran out +[{"id":1,"name":null},{"id":2,"name":null},{"id":3,"name":"c"},{"id":4,"name":null},{"id":5,"name":null},{"id":6,"name":null},{"id":7,"name":null},{"id":8,"name":null},{"id":9,"name":null}] +Flow\Parquet\Exception\RuntimeException: Writer is not open +Flow\Parquet\Exception\RuntimeException: Writer is not open +Flow\Parquet\Exception\RuntimeException: Writer is not open +Flow\Parquet\Exception\RuntimeException: Writer is not open +Flow\Parquet\Exception\RuntimeException: Writer is not open +Flow\Parquet\Exception\RuntimeException: Writer is not open +Flow\Filesystem\Exception\RuntimeException: Closing "memory://failing.parquet" failed +Flow\Parquet\Exception\RuntimeException: Writer is not open +Flow\Filesystem\Exception\RuntimeException: Appending to "memory://append.parquet" failed +bool(true) +Flow\Parquet\Exception\RuntimeException: LZO compression is not supported by the Arrow engine diff --git a/src/extension/arrow-ext/tests/phpt/053_columns_reader_refusal_ends.phpt b/src/extension/arrow-ext/tests/phpt/053_columns_reader_refusal_ends.phpt new file mode 100644 index 0000000000..100fc339a7 --- /dev/null +++ b/src/extension/arrow-ext/tests/phpt/053_columns_reader_refusal_ends.phpt @@ -0,0 +1,74 @@ +--TEST-- +RustColumnsReader behaves as a Generator: it starts on first use, a refused chunk ends it for good (valid() false, key() and current() null, next() a no-op) and rewind() is refused only after next() +--SKIPIF-- + +--FILE-- +set(Option::ROW_GROUP_SIZE_BYTES, 1)->set(Option::ROW_GROUP_SIZE_CHECK_INTERVAL, 2); +$writer = new Writer(options: $options, engine: new PhpParquetEngine(options: $options)); +$writer->openForStream($filesystem->writeTo(path('memory://groups.parquet')), Schema::with(FlatColumn::int64('id'))); +$writer->writeBatch([['id' => 1], ['id' => 2], ['id' => 3], ['id' => 4]]); +$writer->close(); + +$bytes = $filesystem->readFrom(path('memory://groups.parquet'))->content(); +$open = static fn(string $bytes): RustParquetFileReader => new RustParquetFileReader(new StringSourceStream(path('memory://groups.parquet'), $bytes)); +$chunk = $open($bytes)->metadata()->rowGroups()->all()[1]->columnChunks()[0]; +$corrupt = substr_replace($bytes, str_repeat("\xFF", $chunk->totalCompressedSize()), $chunk->pageOffset(), $chunk->totalCompressedSize()); + +$step = static function (string $label, callable $call): void { + try { + $result = json_encode($call()); + } catch (Throwable) { + $result = 'threw'; + } + + echo $label, ': ', $result, "\n"; +}; + +$reader = new RustColumnsReader($open($corrupt), ['id'], 2, null, null); +$step('key before rewind', static fn() => $reader->key()); +$step('current', static fn() => $reader->current()); +$step('rewind', static fn() => $reader->rewind()); +$step('next (chunk refused)', static fn() => $reader->next()); +$step('valid', static fn() => $reader->valid()); +$step('key', static fn() => $reader->key()); +$step('current', static fn() => $reader->current()); +$step('next', static fn() => $reader->next()); +$step('rewind', static fn() => $reader->rewind()); + +// refused before the first chunk: rewind() throws once, then the reader is empty and rewinds freely +$corrupt = substr_replace($bytes, str_repeat("\xFF", 8), 4, 8); +$reader = new RustColumnsReader($open($corrupt), ['id'], 2, null, null); +$step('rewind (chunk refused)', static fn() => $reader->rewind()); +$step('valid', static fn() => $reader->valid()); +$step('rewind', static fn() => $reader->rewind()); +?> +--EXPECT-- +key before rewind: 0 +current: {"id":[1,2]} +rewind: null +next (chunk refused): threw +valid: false +key: null +current: null +next: null +rewind: threw +rewind (chunk refused): threw +valid: false +rewind: null diff --git a/src/extension/arrow-ext/tests/phpt/054_abi_matches_flow_php_parquet.phpt b/src/extension/arrow-ext/tests/phpt/054_abi_matches_flow_php_parquet.phpt new file mode 100644 index 0000000000..a9dd504db2 --- /dev/null +++ b/src/extension/arrow-ext/tests/phpt/054_abi_matches_flow_php_parquet.phpt @@ -0,0 +1,35 @@ +--TEST-- +arrow registers FLOW_ARROW_ABI equal to the ABI flow-php/parquet expects, so AdaptiveParquetEngine takes the Rust lane +--SKIPIF-- + +--FILE-- +available()); + +$writer = (new AdaptiveParquetEngine())->openForWrite( + new StringDestinationStream(path('memory://out.parquet')), + Schema::with(FlatColumn::int32('id')), + Compressions::UNCOMPRESSED, + new Options(), +); +var_dump($writer instanceof RustParquetFileWriter); +$writer->close(); +?> +--EXPECT-- +bool(true) +bool(true) +bool(true) diff --git a/src/extension/arrow-ext/tests/phpt/bootstrap.php b/src/extension/arrow-ext/tests/phpt/bootstrap.php new file mode 100644 index 0000000000..000169488f --- /dev/null +++ b/src/extension/arrow-ext/tests/phpt/bootstrap.php @@ -0,0 +1,64 @@ +getPrevious(); + + return ( + get_class($e) + . ': ' + . $e->getMessage() + . ($previous === null ? '' : ' <- ' . get_class($previous) . ': ' . $previous->getMessage()) + ); + } +} + +/** + * Whether an arrow_outcome() is a refusal: the class and message of what the callable threw. + */ +function arrow_refused(string $outcome): bool +{ + return (bool) preg_match('/^[A-Za-z\\\\]+(Exception|Error|Overflow): /', $outcome); +} + +/** + * JSON of the interfaces arrow registers as reflection sees them: methods, parameters, types, optional, variadic, by-ref. + */ +function arrow_interfaces_reflection(): string +{ + $interfaces = []; + + foreach ([ + 'Flow\\Parquet\\ParquetEngine', + 'Flow\\Parquet\\ParquetFileReader', + 'Flow\\Parquet\\ParquetFileWriter', + ] as $name) { + $class = new ReflectionClass($name); + $interfaces[$name] = [ + $class->isInterface(), + array_map(static fn(ReflectionMethod $method): array => [ + $method->getName(), + (string) $method->getReturnType(), + array_map(static fn(ReflectionParameter $parameter): array => [ + $parameter->getName(), + (string) $parameter->getType(), + $parameter->isOptional(), + $parameter->isVariadic(), + $parameter->isPassedByReference(), + ], $method->getParameters()), + ], $class->getMethods()), + ]; + } + + return (string) json_encode($interfaces); +} diff --git a/src/extension/flow-php-ext/Cargo.lock b/src/extension/flow-php-ext/Cargo.lock index cf93be0665..3366a1782a 100644 --- a/src/extension/flow-php-ext/Cargo.lock +++ b/src/extension/flow-php-ext/Cargo.lock @@ -19,6 +19,20 @@ dependencies = [ "cpufeatures", ] +[[package]] +name = "ahash" +version = "0.8.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a15f179cd60c4584b8a8c596927aadc462e27f2ca70c04e0071964a73ba7a75" +dependencies = [ + "cfg-if", + "const-random", + "getrandom 0.3.4", + "once_cell", + "version_check", + "zerocopy", +] + [[package]] name = "aho-corasick" version = "1.1.4" @@ -28,12 +42,96 @@ dependencies = [ "memchr", ] +[[package]] +name = "android_system_properties" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae221649c9976a6f6c56ae1facf410f3ddb33cc661c4b7b61020a912d4237fbc" +dependencies = [ + "libc", +] + [[package]] name = "anyhow" version = "1.0.103" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2a4385e2e34eb35d6b3efe798b9eb88096925d87726c0798709bf56d9ed84af3" +[[package]] +name = "arrow-array" +version = "59.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e5f6adeffdf587d7a31db5d2266189624b526730cd3627f9ff9fedae97ad584" +dependencies = [ + "ahash", + "arrow-buffer", + "arrow-data", + "arrow-schema", + "chrono", + "half", + "hashbrown", + "libc", + "num-complex", + "num-integer", + "num-traits", +] + +[[package]] +name = "arrow-buffer" +version = "59.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "097d193003ce7995d5d087089069ec2a6e0187faf5a6f8c9f38af2645d987182" +dependencies = [ + "bytes", + "half", + "num-bigint", + "num-traits", +] + +[[package]] +name = "arrow-data" +version = "59.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ba2f832eaeca24b8f26143dba750e42ee4ab51cf7d65e701ca9607cfda9f358" +dependencies = [ + "arrow-buffer", + "arrow-schema", + "half", + "num-integer", + "num-traits", +] + +[[package]] +name = "arrow-schema" +version = "59.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "10fab8d4563491417ba801fab29d205104d20d4bdf37bda6cd1cf425cff598cd" +dependencies = [ + "bitflags", + "serde_core", + "serde_json", +] + +[[package]] +name = "arrow-select" +version = "59.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc58569193c2525915f3cc6310edba3792f1200f65d6e9ed330aa33e691493b8" +dependencies = [ + "ahash", + "arrow-array", + "arrow-buffer", + "arrow-data", + "arrow-schema", + "num-traits", +] + +[[package]] +name = "autocfg" +version = "1.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2032f911046de80f0a198e0901378627c33f59ea0ac00e363d481118bd70a53" + [[package]] name = "base64" version = "0.22.1" @@ -46,6 +144,24 @@ version = "1.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2af50177e190e07a26ab74f8b1efbfe2ef87da2116221318cb1c2e82baf7de06" +[[package]] +name = "bindgen" +version = "0.73.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "787ef8ef523575546b106a58213d6e6b06198a05c2f757258c68a74273670cfa" +dependencies = [ + "bitflags", + "cexpr", + "clang-sys", + "log", + "proc-macro2", + "quote", + "regex", + "rustc-hash", + "shlex", + "syn 2.0.118", +] + [[package]] name = "bitflags" version = "2.13.0" @@ -129,7 +245,7 @@ dependencies = [ "find-msvc-tools", "jobserver", "libc", - "shlex 2.0.1", + "shlex", ] [[package]] @@ -147,6 +263,17 @@ version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" +[[package]] +name = "chrono" +version = "0.4.45" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1aa79e62e7697b8e29b513a68abacf485adcd1fe8284a4316c5ae868e6633327" +dependencies = [ + "iana-time-zone", + "num-traits", + "windows-link", +] + [[package]] name = "cipher" version = "0.5.2" @@ -180,6 +307,26 @@ version = "0.10.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a6ef517f0926dd24a1582492c791b6a4818a4d94e789a334894aa15b0d12f55c" +[[package]] +name = "const-random" +version = "0.1.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "87e00182fe74b066627d63b85fd550ac2998d4b0bd86bfed477a0ae4c7c71359" +dependencies = [ + "const-random-macro", +] + +[[package]] +name = "const-random-macro" +version = "0.1.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f9d839f2a20b0aee515dc581a6172f2321f96cab76c1a38a4c584a194955390e" +dependencies = [ + "getrandom 0.2.17", + "once_cell", + "tiny-keccak", +] + [[package]] name = "constant_time_eq" version = "0.4.2" @@ -188,9 +335,9 @@ checksum = "3d52eff69cd5e647efe296129160853a42795992097e8af39800e1060caeea9b" [[package]] name = "convert_case" -version = "0.11.0" +version = "0.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "affbf0190ed2caf063e3def54ff444b449371d55c58e513a95ab98eca50adb49" +checksum = "1af709f1f33454bf52eadfc8c78b3b9ef9cb26fb54d16dc9cd9a7299f899fd1b" dependencies = [ "unicode-segmentation", ] @@ -235,6 +382,12 @@ dependencies = [ "cfg-if", ] +[[package]] +name = "crunchy" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "460fbee9c2c2f33933d720630a6a0bac33ba7053db5344fac858d4b8952d77d5" + [[package]] name = "crypto-common" version = "0.2.2" @@ -255,9 +408,9 @@ dependencies = [ [[package]] name = "darling" -version = "0.23.0" +version = "0.24.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "25ae13da2f202d56bd7f91c25fba009e7717a1e4a1cc98a76d844b65ae912e9d" +checksum = "ed17f5901b6630b993ca003def43f2f8ef4014fc13b047b57aad617ff32bc2ec" dependencies = [ "darling_core", "darling_macro", @@ -265,26 +418,26 @@ dependencies = [ [[package]] name = "darling_core" -version = "0.23.0" +version = "0.24.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9865a50f7c335f53564bb694ef660825eb8610e0a53d3e11bf1b0d3df31e03b0" +checksum = "6837e2cf7485aaae18f86181d2f0e9a7ed297a025e220aeabf63fdebd3a2ddff" dependencies = [ "ident_case", "proc-macro2", "quote", "strsim", - "syn", + "syn 3.0.6", ] [[package]] name = "darling_macro" -version = "0.23.0" +version = "0.24.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ac3984ec7bd6cfa798e62b4a642426a5be0e68f9401cfc2a01e3fa9ea2fcdb8d" +checksum = "2ac7135c3ef02b2f7833bbeb1be5ba7f966dcde8a87c6b87f65a778d71a02785" dependencies = [ "darling_core", "quote", - "syn", + "syn 3.0.6", ] [[package]] @@ -355,46 +508,28 @@ dependencies = [ [[package]] name = "ext-php-rs" -version = "0.15.15" +version = "0.16.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "abe62f25cd053f5f95dc7bbf94e5b41818ea3cf39b36cc25de8ca6b4de68a0eb" +checksum = "530651e90f34e23c956cbd1a33ec4878c05e19ea6b50dd13926b94dd6a2f2134" dependencies = [ "anyhow", + "bindgen", "bitflags", "cc", "cfg-if", - "ext-php-rs-bindgen", "ext-php-rs-build", "ext-php-rs-derive", + "ext-php-rs-introspection", "inventory", "native-tls", "once_cell", "parking_lot", "skeptic", + "thiserror", "ureq", "zip", ] -[[package]] -name = "ext-php-rs-bindgen" -version = "0.72.1-extphprs.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "32ceb73da3b8b18a3424765dd6926dbc29464a0a7bc31ab491600bfa02adc4bc" -dependencies = [ - "bitflags", - "cexpr", - "clang-sys", - "itertools", - "log", - "prettyplease", - "proc-macro2", - "quote", - "regex", - "rustc-hash", - "shlex 1.3.0", - "syn", -] - [[package]] name = "ext-php-rs-build" version = "0.1.1" @@ -406,18 +541,24 @@ dependencies = [ [[package]] name = "ext-php-rs-derive" -version = "0.11.14" +version = "0.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f57a075f878ad51bcc56745fb21efe601429a509204f8a385bbe526566597320" +checksum = "cb3ec2dc5b2228bffa41f5a569b5656e48f964c39cd71633032eebae51e15ef1" dependencies = [ "convert_case", "darling", "itertools", "proc-macro2", "quote", - "syn", + "syn 3.0.6", ] +[[package]] +name = "ext-php-rs-introspection" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3a31bb462ea3b026fc419c2c4a38283b270980ac34fed3888dd5afd635c3ef94" + [[package]] name = "fastrand" version = "2.4.1" @@ -441,12 +582,31 @@ dependencies = [ "zlib-rs", ] +[[package]] +name = "flow-batch-frame" +version = "0.1.0" +dependencies = [ + "arrow-array", + "arrow-buffer", + "arrow-data", + "arrow-schema", + "serde", + "serde_json", +] + [[package]] name = "flow_php" version = "0.0.0" dependencies = [ + "arrow-array", + "arrow-buffer", + "arrow-data", + "arrow-schema", + "arrow-select", + "bytes", "ext-php-rs", "ext-php-rs-build", + "flow-batch-frame", "memchr", "serde", "serde_json", @@ -491,6 +651,17 @@ dependencies = [ "slab", ] +[[package]] +name = "getrandom" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff2abc00be7fca6ebc474524697ae276ad847ad0a6b3faa4bcb027e9a4614ad0" +dependencies = [ + "cfg-if", + "libc", + "wasi", +] + [[package]] name = "getrandom" version = "0.3.4" @@ -522,6 +693,18 @@ version = "0.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0cc23270f6e1808e30a928bdc84dea0b9b4136a8bc82338574f23baf47bbd280" +[[package]] +name = "half" +version = "2.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ea2d84b969582b4b1864a92dc5d27cd2b77b622a8d79306834f1be5ba20d84b" +dependencies = [ + "cfg-if", + "crunchy", + "num-traits", + "zerocopy", +] + [[package]] name = "hashbrown" version = "0.17.1" @@ -562,6 +745,30 @@ dependencies = [ "typenum", ] +[[package]] +name = "iana-time-zone" +version = "0.1.65" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e31bc9ad994ba00e440a8aa5c9ef0ec67d5cb5e5cb0cc7f8b744a35b389cc470" +dependencies = [ + "android_system_properties", + "core-foundation-sys", + "iana-time-zone-haiku", + "js-sys", + "log", + "wasm-bindgen", + "windows-core", +] + +[[package]] +name = "iana-time-zone-haiku" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f31827a206f56af32e590ba56d5d2d085f558508192593743f16b2306495269f" +dependencies = [ + "cc", +] + [[package]] name = "ident_case" version = "1.0.1" @@ -598,9 +805,9 @@ dependencies = [ [[package]] name = "itertools" -version = "0.14.0" +version = "0.15.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2b192c782037fadd9cfa75548310488aabdbf3d2da73885b31bd0abd03351285" +checksum = "8b4baf93f58d4425749ca49a51c50ebab072c5df6994d08fed93541c331481dc" dependencies = [ "either", ] @@ -654,6 +861,12 @@ dependencies = [ "windows-link", ] +[[package]] +name = "libm" +version = "0.2.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6d2cec3eae94f9f509c767b45932f1ada8350c4bdb85af2fcab4a3c14807981" + [[package]] name = "linux-raw-sys" version = "0.12.1" @@ -733,12 +946,50 @@ dependencies = [ "minimal-lexical", ] +[[package]] +name = "num-bigint" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93e7820bc0a80a0238e650327316f929ba18d5be054b647490a3a6a339f3e7c0" +dependencies = [ + "num-integer", + "num-traits", +] + +[[package]] +name = "num-complex" +version = "0.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "73f88a1307638156682bada9d7604135552957b7818057dcef22705b4d509495" +dependencies = [ + "num-traits", +] + [[package]] name = "num-conv" version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "521739c6d2bac4aa25192232afe6841231376b2b26d4d9fae5ecf8ca5772e441" +[[package]] +name = "num-integer" +version = "0.1.47" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7ce2d95d4b3734dc35aa2f45e1aa22cd416814592a4f9d9205e11affd5b8e10b" +dependencies = [ + "num-traits", +] + +[[package]] +name = "num-traits" +version = "0.2.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" +dependencies = [ + "autocfg", + "libm", +] + [[package]] name = "once_cell" version = "1.21.4" @@ -767,7 +1018,7 @@ checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.118", ] [[package]] @@ -860,16 +1111,6 @@ version = "1.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "efca4c95a19a79d1c98f791f10aebd5c1363b473244630bb7dbde1dc98455a24" -[[package]] -name = "prettyplease" -version = "0.2.37" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "479ca8adacdd7ce8f1fb39ce9ecccbfe93a3f1344b3d0d97f20bc0196208f62b" -dependencies = [ - "proc-macro2", - "syn", -] - [[package]] name = "proc-macro2" version = "1.0.106" @@ -1067,7 +1308,7 @@ checksum = "d540f220d3187173da220f885ab66608367b6574e925011a9353e4badda91d79" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.118", ] [[package]] @@ -1105,12 +1346,6 @@ dependencies = [ "digest", ] -[[package]] -name = "shlex" -version = "1.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64" - [[package]] name = "shlex" version = "2.0.1" @@ -1167,6 +1402,17 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "syn" +version = "3.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8593e8e72159ed2257d083c7a454a85cbf854f37a0966d8d483aff8c8a3ebcee" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + [[package]] name = "tempfile" version = "3.27.0" @@ -1180,6 +1426,26 @@ dependencies = [ "windows-sys", ] +[[package]] +name = "thiserror" +version = "2.0.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09e52cb86a36cede5cb101bf8908837b3e4c6e5e59fe7fd85c23fb56200d189e" +dependencies = [ + "thiserror-impl", +] + +[[package]] +name = "thiserror-impl" +version = "2.0.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fe5197923287db20a58125f0bc85c062f7f2c892de97b18c356f9efb14b28524" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.6", +] + [[package]] name = "time" version = "0.3.53" @@ -1200,6 +1466,15 @@ version = "0.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9e1c906769ad99c88eaa54e728060edef082f8e358ff32030cb7c7d315e81109" +[[package]] +name = "tiny-keccak" +version = "2.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2c9d3793400a45f954c52e73d068316d76b6f4e36977e3fcebb13a2721e80237" +dependencies = [ + "crunchy", +] + [[package]] name = "typed-path" version = "0.12.3" @@ -1288,6 +1563,12 @@ dependencies = [ "winapi-util", ] +[[package]] +name = "wasi" +version = "0.11.1+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b" + [[package]] name = "wasip2" version = "1.0.4+wasi-0.2.12" @@ -1329,7 +1610,7 @@ dependencies = [ "bumpalo", "proc-macro2", "quote", - "syn", + "syn 2.0.118", "wasm-bindgen-shared", ] @@ -1360,12 +1641,65 @@ dependencies = [ "windows-sys", ] +[[package]] +name = "windows-core" +version = "0.62.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8e83a14d34d0623b51dce9581199302a221863196a1dde71a7663a4c2be9deb" +dependencies = [ + "windows-implement", + "windows-interface", + "windows-link", + "windows-result", + "windows-strings", +] + +[[package]] +name = "windows-implement" +version = "0.60.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "053e2e040ab57b9dc951b72c264860db7eb3b0200ba345b4e4c3b14f67855ddf" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.118", +] + +[[package]] +name = "windows-interface" +version = "0.59.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f316c4a2570ba26bbec722032c4099d8c8bc095efccdc15688708623367e358" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.118", +] + [[package]] name = "windows-link" version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" +[[package]] +name = "windows-result" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7781fa89eaf60850ac3d2da7af8e5242a5ea78d1a11c49bf2910bb5a73853eb5" +dependencies = [ + "windows-link", +] + +[[package]] +name = "windows-strings" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7837d08f69c77cf6b07689544538e017c1bfcf57e34b4c0ff58e6c2cd3b37091" +dependencies = [ + "windows-link", +] + [[package]] name = "windows-sys" version = "0.61.2" @@ -1381,6 +1715,26 @@ version = "0.57.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1ebf944e87a7c253233ad6766e082e3cd714b5d03812acc24c318f549614536e" +[[package]] +name = "zerocopy" +version = "0.8.59" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6df92bf3d9227be3d53173901ddbffac2babc27ae50f397776ffd6dc33f800cb" +dependencies = [ + "zerocopy-derive", +] + +[[package]] +name = "zerocopy-derive" +version = "0.8.59" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac4f328cf2f05d084e496c3e9c3f33ed0a183656a16e1fcec4d464d8373aec82" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.118", +] + [[package]] name = "zeroize" version = "1.9.0" diff --git a/src/extension/flow-php-ext/Cargo.toml b/src/extension/flow-php-ext/Cargo.toml index 26a2ef0c97..6ad5666997 100644 --- a/src/extension/flow-php-ext/Cargo.toml +++ b/src/extension/flow-php-ext/Cargo.toml @@ -7,10 +7,20 @@ publish = false crate-type = ["cdylib"] [dependencies] -ext-php-rs = "0.15" +arrow-array = { version = "59", features = ["ffi"] } +arrow-buffer = "59" +arrow-data = "59" +arrow-schema = { version = "59", features = ["canonical_extension_types"] } +arrow-select = "59" +bytes = "1" +ext-php-rs = "0.16" +flow-batch-frame = { path = "crates/flow-batch-frame" } memchr = "2" serde = { version = "1", features = ["derive"] } -serde_json = { version = "1", features = ["raw_value"] } +serde_json = { version = "1", features = ["raw_value", "unbounded_depth"] } [build-dependencies] ext-php-rs-build = "0.1" + +[profile.release] +strip = true diff --git a/src/extension/flow-php-ext/Makefile b/src/extension/flow-php-ext/Makefile index f307e63ba1..11a4e308ce 100644 --- a/src/extension/flow-php-ext/Makefile +++ b/src/extension/flow-php-ext/Makefile @@ -16,6 +16,10 @@ endif CARGO_OUTPUT := target/release/libflow_php.$(LIB_EXT) EXTENSION_SO := $(MODULE_DIR)/flow_php.so +# arrow-ext's built extension: loaded before flow_php in every test run, so the Parquet ETL phpts run instead of skip +ARROW_SO ?= +LOAD_ARROW := $(if $(ARROW_SO),-d extension=$(realpath $(ARROW_SO)),) + .PHONY: all build clean test install rebuild all: build @@ -40,7 +44,7 @@ test: build sed -n '/^--SKIPIF--$$/,/^--FILE--$$/p' "$$f" | sed '1d;$$d' > "$$skipif"; \ skip_out=""; \ if [ -s "$$skipif" ]; then \ - skip_out=$$($(PHP) -d extension=$$(realpath $(EXTENSION_SO)) "$$skipif" 2>&1) || true; \ + skip_out=$$($(PHP) $(LOAD_ARROW) -d extension=$$(realpath $(EXTENSION_SO)) "$$skipif" 2>&1) || true; \ fi; \ rm -f "$$skipif"; \ case "$$skip_out" in skip*) \ @@ -50,7 +54,7 @@ test: build esac; \ sed -n '/^--FILE--$$/,/^--EXPECT/p' "$$f" | sed '1d;$$d' > "$$tmp"; \ expected=$$(sed -n '/^--EXPECT\(F\)\{0,1\}--$$/,$$p' "$$f" | sed '1d' | tr -d '\r'); \ - actual=$$($(PHP) -d extension=$$(realpath $(EXTENSION_SO)) "$$tmp" 2>&1) || true; \ + actual=$$($(PHP) $(LOAD_ARROW) -d extension=$$(realpath $(EXTENSION_SO)) "$$tmp" 2>&1) || true; \ actual=$$(printf '%s' "$$actual" | tr -d '\r'); \ rm -f "$$tmp"; \ if [ "$$actual" = "$$expected" ]; then \ diff --git a/src/extension/flow-php-ext/README.md b/src/extension/flow-php-ext/README.md index e56869c4de..ca4c8fa1cd 100644 --- a/src/extension/flow-php-ext/README.md +++ b/src/extension/flow-php-ext/README.md @@ -1,10 +1,7 @@ # Extension: Flow PHP -A Rust-powered PHP extension using [ext-php-rs](https://github.com/extphprs/ext-php-rs) providing native -implementations for performance-critical parts of the Flow DataFrame framework: encoding and decoding of the -Floe binary format frames and row hydration/casting against a schema. The extension is optional — the pure-PHP -implementations in `flow-php/etl` are the canonical behavior reference, and Flow routes to the native ones -automatically when the extension is loaded. +A Rust-powered PHP extension using [ext-php-rs](https://github.com/extphprs/ext-php-rs) providing native CSV reading +and string-type narrowing; the pure-PHP implementations in `flow-php/etl` are the canonical behaviour reference. > [!IMPORTANT] > This repository is a subtree split from our monorepo. If you'd like to contribute, please visit our main monorepo [flow-php/flow](https://github.com/flow-php/flow). diff --git a/src/extension/flow-php-ext/build.rs b/src/extension/flow-php-ext/build.rs index 24b1b85636..3c3391bc2c 100644 --- a/src/extension/flow-php-ext/build.rs +++ b/src/extension/flow-php-ext/build.rs @@ -12,7 +12,9 @@ fn main() { /// The `php84`/`php85` cfgs ext-php-rs sets for itself, for the same PHP (`PHP` first, then `PATH`): the extension /// binary is bound to the PHP minor it is built against, so version gates are compile-time. fn emit_php_version_cfg() { - use ext_php_rs_build::{emit_check_cfg, emit_php_cfg_flags, emit_rerun_if_env_changed, find_php, ApiVersion, PHPInfo}; + use ext_php_rs_build::{ + emit_check_cfg, emit_php_cfg_flags, emit_rerun_if_env_changed, find_php, ApiVersion, PHPInfo, + }; emit_rerun_if_env_changed(); emit_check_cfg(); @@ -80,7 +82,8 @@ fn git(args: &[&str]) -> Option { .filter(|stdout| !stdout.is_empty()) } -/// `0.43.0-107-gf998a45d0` (git describe) becomes `0.43.0+107.gf998a45d0`, semver build metadata Composer reads as 0.43.0. +/// `0.44.1-65-g4525c4adc` (git describe) becomes `0.45.0-dev+65.g4525c4adc`: commits past a tag are a pre-release of the +/// next minor, outside Composer's `<0.45` conflict of the libraries. A tag, or any other string, passes through. fn as_semver(describe: &str) -> String { let mut parts = describe.rsplitn(3, '-'); @@ -88,9 +91,17 @@ fn as_semver(describe: &str) -> String { (Some(hash), Some(distance), Some(tag)) if hash.starts_with('g') && !distance.is_empty() - && distance.bytes().all(|b| b.is_ascii_digit()) => + && distance.bytes().all(|b| b.is_ascii_digit()) + && distance != "0" => { - format!("{tag}+{distance}.{hash}") + let mut numbers = tag.split('.').map(|number| number.parse::().ok()); + + match (numbers.next().flatten(), numbers.next().flatten()) { + (Some(major), Some(minor)) => { + format!("{major}.{}.0-dev+{distance}.{hash}", minor + 1) + } + _ => describe.to_string(), + } } _ => describe.to_string(), } diff --git a/src/extension/flow-php-ext/composer.json b/src/extension/flow-php-ext/composer.json index c21e562074..196e5d1785 100644 --- a/src/extension/flow-php-ext/composer.json +++ b/src/extension/flow-php-ext/composer.json @@ -1,7 +1,7 @@ { "name": "flow-php/flow-php-ext", "type": "php-ext", - "description": "Flow PHP native extension (Rust) - Floe frame-body encoder/decoder for DataFrame Rows", + "description": "Flow PHP native extension (Rust) - native column backend, CSV/JSON readers, Parquet ETL through arrow-ext", "keywords": ["serializer", "binary", "extension", "rust", "native", "dataframe"], "license": "MIT", "require": { @@ -9,9 +9,7 @@ }, "autoload": { "psr-4": { - "Flow\\ETL\\Adapter\\CSV\\": "php/Flow/ETL/Adapter/CSV/", - "Flow\\ETL\\Row\\": "php/Flow/ETL/Row/", - "Flow\\Floe\\": "php/Flow/Floe/" + "Flow\\ETL\\": "php/Flow/ETL/" } }, "scripts": { diff --git a/src/extension/flow-php-ext/crates/flow-batch-frame/.gitattributes b/src/extension/flow-php-ext/crates/flow-batch-frame/.gitattributes new file mode 100644 index 0000000000..360f09ff9c --- /dev/null +++ b/src/extension/flow-php-ext/crates/flow-batch-frame/.gitattributes @@ -0,0 +1,5 @@ +# Exclude development files from distribution archives +/tests export-ignore +/target export-ignore +/.gitattributes export-ignore +/.gitignore export-ignore diff --git a/src/extension/flow-php-ext/crates/flow-batch-frame/.gitignore b/src/extension/flow-php-ext/crates/flow-batch-frame/.gitignore new file mode 100644 index 0000000000..b83d22266a --- /dev/null +++ b/src/extension/flow-php-ext/crates/flow-batch-frame/.gitignore @@ -0,0 +1 @@ +/target/ diff --git a/src/extension/flow-php-ext/crates/flow-batch-frame/Cargo.lock b/src/extension/flow-php-ext/crates/flow-batch-frame/Cargo.lock new file mode 100644 index 0000000000..bddb682b52 --- /dev/null +++ b/src/extension/flow-php-ext/crates/flow-batch-frame/Cargo.lock @@ -0,0 +1,629 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "ahash" +version = "0.8.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a15f179cd60c4584b8a8c596927aadc462e27f2ca70c04e0071964a73ba7a75" +dependencies = [ + "cfg-if", + "const-random", + "getrandom 0.3.4", + "once_cell", + "version_check", + "zerocopy", +] + +[[package]] +name = "android_system_properties" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae221649c9976a6f6c56ae1facf410f3ddb33cc661c4b7b61020a912d4237fbc" +dependencies = [ + "libc", +] + +[[package]] +name = "arrow-array" +version = "59.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e5f6adeffdf587d7a31db5d2266189624b526730cd3627f9ff9fedae97ad584" +dependencies = [ + "ahash", + "arrow-buffer", + "arrow-data", + "arrow-schema", + "chrono", + "half", + "hashbrown", + "num-complex", + "num-integer", + "num-traits", +] + +[[package]] +name = "arrow-buffer" +version = "59.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "097d193003ce7995d5d087089069ec2a6e0187faf5a6f8c9f38af2645d987182" +dependencies = [ + "bytes", + "half", + "num-bigint", + "num-traits", +] + +[[package]] +name = "arrow-data" +version = "59.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ba2f832eaeca24b8f26143dba750e42ee4ab51cf7d65e701ca9607cfda9f358" +dependencies = [ + "arrow-buffer", + "arrow-schema", + "half", + "num-integer", + "num-traits", +] + +[[package]] +name = "arrow-schema" +version = "59.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "10fab8d4563491417ba801fab29d205104d20d4bdf37bda6cd1cf425cff598cd" + +[[package]] +name = "autocfg" +version = "1.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2032f911046de80f0a198e0901378627c33f59ea0ac00e363d481118bd70a53" + +[[package]] +name = "bumpalo" +version = "3.20.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72f5acc6cb2ba439de613abc23857ec3d78374d8ed5ac84e9d11336e87da8649" + +[[package]] +name = "bytes" +version = "1.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc652a48c352aef3ea3aed32080501cf3ef6ed5da78602a020c991775b0aff04" + +[[package]] +name = "cc" +version = "1.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f360145194ee8e21db5ee7f3fcd4fe52210864c75c985dae33218202c8bbe040" +dependencies = [ + "find-msvc-tools", + "shlex", +] + +[[package]] +name = "cfg-if" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4e7648175b45a9a48536d676f68d918270699102aa8dab5496df06904c914600" + +[[package]] +name = "chrono" +version = "0.4.45" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1aa79e62e7697b8e29b513a68abacf485adcd1fe8284a4316c5ae868e6633327" +dependencies = [ + "iana-time-zone", + "num-traits", + "windows-link", +] + +[[package]] +name = "const-random" +version = "0.1.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "87e00182fe74b066627d63b85fd550ac2998d4b0bd86bfed477a0ae4c7c71359" +dependencies = [ + "const-random-macro", +] + +[[package]] +name = "const-random-macro" +version = "0.1.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f9d839f2a20b0aee515dc581a6172f2321f96cab76c1a38a4c584a194955390e" +dependencies = [ + "getrandom 0.2.17", + "once_cell", + "tiny-keccak", +] + +[[package]] +name = "core-foundation-sys" +version = "0.8.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" + +[[package]] +name = "crunchy" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "460fbee9c2c2f33933d720630a6a0bac33ba7053db5344fac858d4b8952d77d5" + +[[package]] +name = "find-msvc-tools" +version = "0.1.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aedcfb3409746eddb02b9e19ebda1c3394f759a152e48ee875a0844d1b955484" + +[[package]] +name = "flow-batch-frame" +version = "0.1.0" +dependencies = [ + "arrow-array", + "arrow-buffer", + "arrow-data", + "arrow-schema", + "serde", + "serde_json", +] + +[[package]] +name = "futures-core" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92d699e522242e69e3003b94ecc1f960f3a5e015aa7c5d7486e65ad01dd94f5e" + +[[package]] +name = "futures-task" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cd417de3d1d015fc3bfd2b1ea46dfc7bab72ef86f1cc7cc9c78e728b34a6d1fd" + +[[package]] +name = "futures-util" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0d50a92467f8ba5dd6e3ee5d4bd04d73ab2e4e1c44474a0674821dfce14b79bc" +dependencies = [ + "futures-core", + "futures-task", + "pin-project-lite", + "slab", +] + +[[package]] +name = "getrandom" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff2abc00be7fca6ebc474524697ae276ad847ad0a6b3faa4bcb027e9a4614ad0" +dependencies = [ + "cfg-if", + "libc", + "wasi", +] + +[[package]] +name = "getrandom" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "899def5c37c4fd7b2664648c28120ecec138e4d395b459e5ca34f9cce2dd77fd" +dependencies = [ + "cfg-if", + "libc", + "r-efi", + "wasip2", +] + +[[package]] +name = "half" +version = "2.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ea2d84b969582b4b1864a92dc5d27cd2b77b622a8d79306834f1be5ba20d84b" +dependencies = [ + "cfg-if", + "crunchy", + "num-traits", + "zerocopy", +] + +[[package]] +name = "hashbrown" +version = "0.17.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed5909b6e89a2db4456e54cd5f673791d7eca6732202bbf2a9cc504fe2f9b84a" + +[[package]] +name = "iana-time-zone" +version = "0.1.65" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e31bc9ad994ba00e440a8aa5c9ef0ec67d5cb5e5cb0cc7f8b744a35b389cc470" +dependencies = [ + "android_system_properties", + "core-foundation-sys", + "iana-time-zone-haiku", + "js-sys", + "log", + "wasm-bindgen", + "windows-core", +] + +[[package]] +name = "iana-time-zone-haiku" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f31827a206f56af32e590ba56d5d2d085f558508192593743f16b2306495269f" +dependencies = [ + "cc", +] + +[[package]] +name = "itoa" +version = "1.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" + +[[package]] +name = "js-sys" +version = "0.3.106" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7883d941dae510fb2d978fc3fe018c71c9e2892fd38854de3e8b92c2e5ad9cc5" +dependencies = [ + "cfg-if", + "futures-util", + "wasm-bindgen", +] + +[[package]] +name = "libc" +version = "0.2.189" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3eaf3ede3fee6db1a4c2ee091bf8a8b4dccdc6d17f656fb07896ee72867612f2" + +[[package]] +name = "libm" +version = "0.2.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6d2cec3eae94f9f509c767b45932f1ada8350c4bdb85af2fcab4a3c14807981" + +[[package]] +name = "log" +version = "0.4.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f9f8bd3e56ce4dfc153cf470fffbfa98c7620958b312ca5c3a4b8d5181fd13c6" + +[[package]] +name = "memchr" +version = "2.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98" + +[[package]] +name = "num-bigint" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93e7820bc0a80a0238e650327316f929ba18d5be054b647490a3a6a339f3e7c0" +dependencies = [ + "num-integer", + "num-traits", +] + +[[package]] +name = "num-complex" +version = "0.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "73f88a1307638156682bada9d7604135552957b7818057dcef22705b4d509495" +dependencies = [ + "num-traits", +] + +[[package]] +name = "num-integer" +version = "0.1.47" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7ce2d95d4b3734dc35aa2f45e1aa22cd416814592a4f9d9205e11affd5b8e10b" +dependencies = [ + "num-traits", +] + +[[package]] +name = "num-traits" +version = "0.2.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" +dependencies = [ + "autocfg", + "libm", +] + +[[package]] +name = "once_cell" +version = "1.21.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" + +[[package]] +name = "pin-project-lite" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a89322df9ebe1c1578d689c92318e070967d1042b512afbe49518723f4e6d5cd" + +[[package]] +name = "proc-macro2" +version = "1.0.107" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "985e7ec9bb745e6ce6535b544d84d6cd6f7ad8bd711c398938ae983b91a766d9" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "quote" +version = "1.0.47" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fbf4db142a473a8d80c26bbf18454ed458bf8d26c8219c331daecfdbd079001" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "r-efi" +version = "5.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "69cdb34c158ceb288df11e18b4bd39de994f6657d83847bdffdbd7f346754b0f" + +[[package]] +name = "rustversion" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf54715a573b99ac80df0bc206da022bcd442c974952c7b9720069370852e21f" + +[[package]] +name = "serde" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4148590afebada386688f18773da617792bf2ef03ffc1e4cbd2b1d45b023e0ba" +dependencies = [ + "serde_core", + "serde_derive", +] + +[[package]] +name = "serde_core" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67dca2c9c51e58a4791a4b1ed58308b39c64224d349a935ab5039aa360942a48" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7a5d71263a5a7d47b41f6b3f06ba276f10cc18b0931f1799f710578e2309348" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.6", +] + +[[package]] +name = "serde_json" +version = "1.0.151" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c841b55ecdae098c80dcae9cf767f6f8a0c2cdb3416bbef72181df4d0fe73f14" +dependencies = [ + "itoa", + "memchr", + "serde", + "serde_core", + "zmij", +] + +[[package]] +name = "shlex" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8fadd59c855ef2080decdef8ff161eb6661b86933c9d82e5ba29dc602a55aba" + +[[package]] +name = "slab" +version = "0.4.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c790de23124f9ab44544d7ac05d60440adc586479ce501c1d6d7da3cd8c9cf5" + +[[package]] +name = "syn" +version = "2.0.119" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "872831b642d1a07999a962a351ed35b955ea2cfc8f3862091e2a240a84f17297" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "syn" +version = "3.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8593e8e72159ed2257d083c7a454a85cbf854f37a0966d8d483aff8c8a3ebcee" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "tiny-keccak" +version = "2.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2c9d3793400a45f954c52e73d068316d76b6f4e36977e3fcebb13a2721e80237" +dependencies = [ + "crunchy", +] + +[[package]] +name = "unicode-ident" +version = "1.0.26" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d245f478577f809a851594d02313b640fb437e0bb33866753cff937863096954" + +[[package]] +name = "version_check" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" + +[[package]] +name = "wasi" +version = "0.11.1+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b" + +[[package]] +name = "wasip2" +version = "1.0.4+wasi-0.2.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b67efb37e106e55ce722a510d6b5f9c17f083e5fc79afc2badeb12cc313d9487" +dependencies = [ + "wit-bindgen", +] + +[[package]] +name = "wasm-bindgen" +version = "0.2.129" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9bb54f33acc68fd454578d9820b0bde1a1a3d17aa17bb7b6595806d02886d409" +dependencies = [ + "cfg-if", + "once_cell", + "rustversion", + "wasm-bindgen-macro", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-macro" +version = "0.2.129" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2e29d0c35b16e224a7eeb5cd2d25e3e1968fbd65604117b44d3b789d00ee8535" +dependencies = [ + "quote", + "wasm-bindgen-macro-support", +] + +[[package]] +name = "wasm-bindgen-macro-support" +version = "0.2.129" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6f501a8bc3719dba86ef8ae4728879c08001bea749eb1333ac5b91e040e2a6b7" +dependencies = [ + "bumpalo", + "proc-macro2", + "quote", + "syn 3.0.6", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-shared" +version = "0.2.129" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "23f0c9c52aa7cd7d77769a4cfe2a9adb1b331f489a41d912ce14513d5ab995c6" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "windows-core" +version = "0.62.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8e83a14d34d0623b51dce9581199302a221863196a1dde71a7663a4c2be9deb" +dependencies = [ + "windows-implement", + "windows-interface", + "windows-link", + "windows-result", + "windows-strings", +] + +[[package]] +name = "windows-implement" +version = "0.60.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "053e2e040ab57b9dc951b72c264860db7eb3b0200ba345b4e4c3b14f67855ddf" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "windows-interface" +version = "0.59.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f316c4a2570ba26bbec722032c4099d8c8bc095efccdc15688708623367e358" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "windows-link" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" + +[[package]] +name = "windows-result" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7781fa89eaf60850ac3d2da7af8e5242a5ea78d1a11c49bf2910bb5a73853eb5" +dependencies = [ + "windows-link", +] + +[[package]] +name = "windows-strings" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7837d08f69c77cf6b07689544538e017c1bfcf57e34b4c0ff58e6c2cd3b37091" +dependencies = [ + "windows-link", +] + +[[package]] +name = "wit-bindgen" +version = "0.57.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ebf944e87a7c253233ad6766e082e3cd714b5d03812acc24c318f549614536e" + +[[package]] +name = "zerocopy" +version = "0.8.59" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6df92bf3d9227be3d53173901ddbffac2babc27ae50f397776ffd6dc33f800cb" +dependencies = [ + "zerocopy-derive", +] + +[[package]] +name = "zerocopy-derive" +version = "0.8.59" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac4f328cf2f05d084e496c3e9c3f33ed0a183656a16e1fcec4d464d8373aec82" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "zmij" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29666d0abbfad1e3dc4dcf6144730dd3a3ab225bbbdac83319345b1b44ccfc1b" diff --git a/src/extension/flow-php-ext/crates/flow-batch-frame/Cargo.toml b/src/extension/flow-php-ext/crates/flow-batch-frame/Cargo.toml new file mode 100644 index 0000000000..492a33515e --- /dev/null +++ b/src/extension/flow-php-ext/crates/flow-batch-frame/Cargo.toml @@ -0,0 +1,15 @@ +[package] +name = "flow-batch-frame" +version = "0.1.0" +edition = "2021" +publish = false + +[lib] + +[dependencies] +arrow-array = "59" +arrow-buffer = "59" +arrow-data = "59" +arrow-schema = "59" +serde = { version = "1", features = ["derive"] } +serde_json = "1" diff --git a/src/extension/flow-php-ext/crates/flow-batch-frame/src/column.rs b/src/extension/flow-php-ext/crates/flow-batch-frame/src/column.rs new file mode 100644 index 0000000000..66fcb2c567 --- /dev/null +++ b/src/extension/flow-php-ext/crates/flow-batch-frame/src/column.rs @@ -0,0 +1,535 @@ +//! One column's buffers in the canonical form of `Column::encode()`, and back (`Backend::decode()`). +//! +//! Canonical: validity '' when the rows hold no null, else packed LSB-first from bit 0 with zero padding; a null slot +//! zeroed (0, 0.0, bit 0, 16 zero bytes, offset not advanced, structure children null under a null parent); offsets +//! from 0 over exactly the rows' bytes and children, whatever slice or selection the array is. + +use arrow_array::cast::AsArray; +use arrow_array::types::{Date32Type, DurationMicrosecondType, Float64Type, Int64Type, TimestampMicrosecondType}; +use arrow_array::{make_array, Array, ArrowPrimitiveType}; +use arrow_buffer::{Buffer, ToByteSlice}; +use arrow_data::ArrayData; +use arrow_schema::DataType; + +use crate::error::{Error, Of, Role, Values}; +use crate::kind::{data_type, Kind}; +use crate::layout::{buffer_count, node_count, nodes_at, null_count_of}; + +pub fn encode(data: &ArrayData, kind: &Kind) -> Result>, Error> { + let expected = data_type(kind); + + if data.data_type() != &expected { + return Err(Error::DataTypeMismatch { + expected, + actual: data.data_type().clone(), + }); + } + + let array = make_array(data.clone()); + let mut out = Vec::with_capacity(buffer_count(kind)); + encode_node( + array.as_ref(), + kind, + &Rows::Range { + start: 0, + len: array.len(), + }, + None, + &mut out, + ); + + Ok(out) +} + +/// `ColumnDecoder::decode()` + `PhpBackend::decode()`: the same refusals in the same order. The result's data type is +/// `data_type(kind)`. A top-level null under NOT NULL is not refused here (`Rows::fromColumns` does). +pub fn decode(kind: &Kind, buffers: &[&[u8]], len: u32, null_count: u32) -> Result { + let mut cursor = Cursor::new(buffers); + let data = decode_node(kind, 0, &mut cursor, len, null_count)?; + + if cursor.remaining() != 0 { + return Err(Error::LeftoverBuffers { + count: cursor.remaining(), + }); + } + + Ok(data) +} + +/// The rows of an array a node encodes, as logical indexes: the whole array or a slice, or the compacted children of +/// the valid list/map slots. +enum Rows { + Range { start: usize, len: usize }, + Indices(Vec), +} + +impl Rows { + fn len(&self) -> usize { + match self { + Rows::Range { len, .. } => *len, + Rows::Indices(indices) => indices.len(), + } + } + + fn at(&self, j: usize) -> usize { + match self { + Rows::Range { start, .. } => start + j, + Rows::Indices(indices) => indices[j], + } + } +} + +fn encode_node(array: &dyn Array, kind: &Kind, rows: &Rows, parent: Option<&[bool]>, out: &mut Vec>) { + if let Kind::Null = kind { + return; + } + + let valid = effective_validity(array, rows, parent); + out.push(valid.as_deref().map_or_else(Vec::new, pack_bits)); + let valid = valid.as_deref(); + + match kind { + Kind::Null => unreachable!("returned above"), + Kind::Int64 => out.push(fixed::(array, rows, valid)), + Kind::Timestamp => out.push(fixed::(array, rows, valid)), + Kind::Duration => out.push(fixed::(array, rows, valid)), + Kind::Int32 => out.push(fixed::(array, rows, valid)), + Kind::Float64 => out.push(fixed::(array, rows, valid)), + Kind::Boolean => { + let values = array.as_boolean().values(); + let mut bits = vec![0u8; rows.len().div_ceil(8)]; + + for j in 0..rows.len() { + if is_valid(valid, j) && values.value(rows.at(j)) { + bits[j >> 3] |= 1 << (j & 7); + } + } + + out.push(bits); + } + Kind::Uuid => { + let values = array.as_fixed_size_binary(); + let mut bytes = Vec::with_capacity(rows.len() * 16); + + for j in 0..rows.len() { + if is_valid(valid, j) { + bytes.extend_from_slice(values.value(rows.at(j))); + } else { + bytes.extend_from_slice(&[0; 16]); + } + } + + out.push(bytes); + } + Kind::Bytes => { + let values = array.as_binary::(); + let offsets = values.value_offsets(); + let data = values.value_data(); + + if let (Rows::Range { start, len }, None) = (rows, valid) { + out.push(rebased(&offsets[*start..=start + len])); + out.push(data[offsets[*start] as usize..offsets[start + len] as usize].to_vec()); + } else { + let mut packed = Vec::with_capacity((rows.len() + 1) * 4); + let mut bytes = Vec::new(); + packed.extend_from_slice(&0u32.to_le_bytes()); + + for j in 0..rows.len() { + if is_valid(valid, j) { + let i = rows.at(j); + bytes.extend_from_slice(&data[offsets[i] as usize..offsets[i + 1] as usize]); + } + + packed.extend_from_slice(&(bytes.len() as u32).to_le_bytes()); + } + + out.push(packed); + out.push(bytes); + } + } + Kind::List(element) => { + let list = array.as_list::(); + let (offsets, children) = compact(list.value_offsets(), rows, valid); + out.push(offsets); + encode_node(list.values().as_ref(), &element.kind, &children, None, out); + } + Kind::Map(key, value) => { + let map = array.as_map(); + let (offsets, children) = compact(map.value_offsets(), rows, valid); + out.push(offsets); + out.push(Vec::new()); + encode_node(map.keys().as_ref(), &key.kind, &children, None, out); + encode_node(map.values().as_ref(), &value.kind, &children, None, out); + } + Kind::Struct(fields) => { + let structure = array.as_struct(); + + for (i, field) in fields.iter().enumerate() { + encode_node(structure.column(i).as_ref(), &field.kind, rows, valid, out); + } + } + } +} + +/// The null of a row is its own or its parent structure's; `None` when every row is valid. +fn effective_validity(array: &dyn Array, rows: &Rows, parent: Option<&[bool]>) -> Option> { + let nulls = array.nulls(); + + if nulls.is_none() && parent.is_none() { + return None; + } + + let valid: Vec = (0..rows.len()) + .map(|j| parent.is_none_or(|p| p[j]) && nulls.is_none_or(|n| n.is_valid(rows.at(j)))) + .collect(); + + if valid.iter().all(|v| *v) { + None + } else { + Some(valid) + } +} + +fn is_valid(valid: Option<&[bool]>, j: usize) -> bool { + valid.is_none_or(|v| v[j]) +} + +fn pack_bits(valid: &[bool]) -> Vec { + let mut bits = vec![0u8; valid.len().div_ceil(8)]; + + for (j, _) in valid.iter().enumerate().filter(|(_, v)| **v) { + bits[j >> 3] |= 1 << (j & 7); + } + + bits +} + +fn fixed(array: &dyn Array, rows: &Rows, valid: Option<&[bool]>) -> Vec { + let values = array.as_primitive::().values(); + + if let (Rows::Range { start, len }, None) = (rows, valid) { + return values[*start..start + len].to_byte_slice().to_vec(); + } + + let width = std::mem::size_of::(); + let mut bytes = Vec::with_capacity(rows.len() * width); + + for j in 0..rows.len() { + if is_valid(valid, j) { + bytes.extend_from_slice(values[rows.at(j)].to_byte_slice()); + } else { + bytes.resize(bytes.len() + width, 0); + } + } + + bytes +} + +/// Offsets from 0 over the valid rows' ranges (a null row's range is dropped), and the child rows those ranges select. +/// Compaction only shrinks an i32 array's ranges, so the offsets stay within i32. +fn compact(offsets: &[i32], rows: &Rows, valid: Option<&[bool]>) -> (Vec, Rows) { + if let (Rows::Range { start, len }, None) = (rows, valid) { + return ( + rebased(&offsets[*start..=start + len]), + Rows::Range { + start: offsets[*start] as usize, + len: (offsets[start + len] - offsets[*start]) as usize, + }, + ); + } + + let mut packed = Vec::with_capacity((rows.len() + 1) * 4); + let mut children = Vec::new(); + packed.extend_from_slice(&0u32.to_le_bytes()); + + for j in 0..rows.len() { + if is_valid(valid, j) { + let i = rows.at(j); + children.extend(offsets[i] as usize..offsets[i + 1] as usize); + } + + packed.extend_from_slice(&(children.len() as u32).to_le_bytes()); + } + + (packed, Rows::Indices(children)) +} + +fn rebased(offsets: &[i32]) -> Vec { + offsets + .iter() + .flat_map(|offset| ((offset - offsets[0]) as u32).to_le_bytes()) + .collect() +} + +struct Cursor<'b, 'a> { + buffers: &'b [&'a [u8]], + position: usize, +} + +impl<'b, 'a> Cursor<'b, 'a> { + fn new(buffers: &'b [&'a [u8]]) -> Self { + Self { buffers, position: 0 } + } + + fn next(&mut self) -> Result<&'a [u8], Error> { + let buffer = self.buffers.get(self.position).ok_or(Error::BuffersExhausted)?; + self.position += 1; + + Ok(buffer) + } + + fn remaining(&self) -> usize { + self.buffers.len() - self.position + } +} + +fn decode_node( + kind: &Kind, + node: usize, + cursor: &mut Cursor<'_, '_>, + len: u32, + null_count: u32, +) -> Result { + if let Kind::List(_) | Kind::Map(_, _) | Kind::Struct(_) = kind { + let mut subtree = Vec::with_capacity(buffer_count(kind)); + + for _ in 0..buffer_count(kind) { + subtree.push(cursor.next()?); + } + + let nodes = nodes_at(kind, node, len, null_count, &subtree)?; + + return decode_nested(kind, node, &mut Cursor::new(&subtree), len, null_count, &nodes); + } + + if let Kind::Null = kind { + if null_count != len { + return Err(Error::NullKindNullCount { + rows: len as u64, + null_count: null_count as u64, + }); + } + + return Ok(ArrayData::new_null(&DataType::Null, len as usize)); + } + + let nulls = validity(cursor.next()?, len, null_count)?; + let builder = ArrayData::builder(data_type(kind)) + .len(len as usize) + .null_bit_buffer(nulls); + let builder = match kind { + Kind::Int64 | Kind::Timestamp | Kind::Duration => { + builder.add_buffer(fixed_values(cursor.next()?, 8, len, Values::Int64)?) + } + Kind::Int32 => builder.add_buffer(fixed_values(cursor.next()?, 4, len, Values::Int32)?), + Kind::Float64 => builder.add_buffer(fixed_values(cursor.next()?, 8, len, Values::Float64)?), + Kind::Uuid => builder.add_buffer(fixed_values(cursor.next()?, 16, len, Values::FixedBinary16)?), + Kind::Boolean => { + let buffer = cursor.next()?; + let expected = (len as u64).div_ceil(8); + + if (buffer.len() as u64) < expected { + return Err(Error::ValuesLength { + values: Values::Boolean, + bytes: buffer.len() as u64, + expected, + rows: len as u64, + }); + } + + builder.add_buffer(Buffer::from_slice_ref(buffer)) + } + Kind::Bytes => { + let offsets = cursor.next()?; + let last = check_offsets(offsets, len, Of::Utf8)?; + let data = cursor.next()?; + + if last as u64 != data.len() as u64 { + return Err(Error::Utf8DataLength { + bytes: data.len() as u64, + last_offset: last as u64, + }); + } + + fits_i32(last)?; + + builder + .add_buffer(Buffer::from_slice_ref(offsets)) + .add_buffer(Buffer::from_slice_ref(data)) + } + Kind::Null | Kind::List(_) | Kind::Map(_, _) | Kind::Struct(_) => { + unreachable!("returned above") + } + }; + + builder.build().map_err(Error::Arrow) +} + +fn decode_nested( + kind: &Kind, + node: usize, + own: &mut Cursor<'_, '_>, + len: u32, + null_count: u32, + nodes: &[(u32, u32)], +) -> Result { + let nulls = validity(own.next()?, len, null_count)?; + let builder = ArrayData::builder(data_type(kind)) + .len(len as usize) + .null_bit_buffer(nulls); + + let builder = match kind { + Kind::List(element) => { + let offsets = own.next()?; + let last = check_offsets(offsets, len, Of::List)?; + let (length, nulls) = nodes[1]; + + if nulls > 0 && !element.optional && element.kind != Kind::Null { + return Err(Error::NestedNulls { + node: node + 1, + role: Role::ListElement, + nulls: nulls as u64, + }); + } + + let child = decode_node(&element.kind, node + 1, own, length, nulls)?; + fits_i32(last)?; + + builder + .add_buffer(Buffer::from_slice_ref(offsets)) + .add_child_data(child) + } + Kind::Map(key, value) => { + let offsets = own.next()?; + let last = check_offsets(offsets, len, Of::Map)?; + own.next()?; + let key_node = node + 2; + let value_node = key_node + node_count(&key.kind); + let (length, key_nulls) = nodes[key_node - node]; + let (_, value_nulls) = nodes[value_node - node]; + + if key_nulls > 0 { + return Err(Error::NestedNulls { + node: key_node, + role: Role::MapKey, + nulls: key_nulls as u64, + }); + } + + if value_nulls > 0 && !value.optional && value.kind != Kind::Null { + return Err(Error::NestedNulls { + node: value_node, + role: Role::MapValue, + nulls: value_nulls as u64, + }); + } + + let keys = decode_node(&key.kind, key_node, own, length, key_nulls)?; + let values = decode_node(&value.kind, value_node, own, length, value_nulls)?; + fits_i32(last)?; + let DataType::Map(entries, _) = data_type(kind) else { + unreachable!("a map kind has a map data type") + }; + let entries = ArrayData::builder(entries.data_type().clone()) + .len(length as usize) + .add_child_data(keys) + .add_child_data(values) + .build() + .map_err(Error::Arrow)?; + + builder + .add_buffer(Buffer::from_slice_ref(offsets)) + .add_child_data(entries) + } + Kind::Struct(fields) => { + let mut child_node = node + 1; + let mut children = Vec::with_capacity(fields.len()); + + for field in fields { + let (length, nulls) = nodes[child_node - node]; + children.push(decode_node(&field.kind, child_node, own, length, nulls)?); + child_node += node_count(&field.kind); + } + + builder.child_data(children) + } + _ => unreachable!("decode_nested takes a nested kind"), + }; + + builder.build().map_err(Error::Arrow) +} + +/// `Some` bitmap only when it holds a null: the canonical form omits it otherwise. +fn validity(bitmap: &[u8], len: u32, declared: u32) -> Result, Error> { + let derived = null_count_of(bitmap, len)?; + + if declared != derived { + return Err(Error::NullCountMismatch { + declared: declared as u64, + derived: derived as u64, + rows: len as u64, + }); + } + + Ok((derived > 0).then(|| Buffer::from_slice_ref(bitmap))) +} + +fn fixed_values(buffer: &[u8], width: u64, len: u32, values: Values) -> Result { + let expected = width * len as u64; + + if buffer.len() as u64 != expected { + return Err(Error::ValuesLength { + values, + bytes: buffer.len() as u64, + expected, + rows: len as u64, + }); + } + + Ok(Buffer::from_slice_ref(buffer)) +} + +/// `Offsets::unpack()`: exactly (len + 1) * 4 bytes, from 0, monotonic. Returns the last offset. +fn check_offsets(buffer: &[u8], len: u32, of: Of) -> Result { + let expected = (len as u64 + 1) * 4; + + if buffer.len() as u64 != expected { + return Err(Error::OffsetsLength { + of, + bytes: buffer.len() as u64, + expected, + rows: len as u64, + }); + } + + let offset = |i: usize| u32::from_le_bytes(buffer[i * 4..i * 4 + 4].try_into().expect("4-byte slice")); + + if offset(0) != 0 { + return Err(Error::OffsetsStart { + of, + first: offset(0) as u64, + }); + } + + for i in 1..=len as usize { + if offset(i) < offset(i - 1) { + return Err(Error::OffsetsNotMonotonic { + of, + index: i as u64, + previous: offset(i - 1) as u64, + current: offset(i) as u64, + }); + } + } + + Ok(offset(len as usize)) +} + +/// Native offsets are i32 (D5): the PHP decoder takes them unsigned, so a last offset past i32::MAX is refused here. +fn fits_i32(last: u32) -> Result<(), Error> { + if last > i32::MAX as u32 { + return Err(Error::OffsetOverflow { last: last as u64 }); + } + + Ok(()) +} diff --git a/src/extension/flow-php-ext/crates/flow-batch-frame/src/error.rs b/src/extension/flow-php-ext/crates/flow-batch-frame/src/error.rs new file mode 100644 index 0000000000..afc40edcc1 --- /dev/null +++ b/src/extension/flow-php-ext/crates/flow-batch-frame/src/error.rs @@ -0,0 +1,168 @@ +//! Every refusal the crate returns, as data. The wording belongs to the caller: flow_php renders each variant into the +//! message its pure-PHP twin throws, so a message change never moves this crate. + +use arrow_schema::{ArrowError, DataType}; + +/// The layout an offsets buffer belongs to. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Of { + List, + Map, + Utf8, +} + +/// The layout a fixed-width values buffer belongs to. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Values { + Int64, + Int32, + Float64, + Boolean, + FixedBinary16, +} + +/// The child of a list or map that holds nulls it may not. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Role { + ListElement, + MapKey, + MapValue, +} + +/// The part of a nested type JSON that is missing. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Part { + Element, + Key, + Value, + Base, +} + +/// `node` is a pre-order index in one column's node tree (the map entries node counts); `column` and `buffer` are +/// indexes in a frame body. +#[derive(Debug)] +pub enum Error { + SchemaJsonUtf8, + SchemaJson(serde_json::Error), + TypeJsonUtf8, + TypeJson(serde_json::Error), + MissingPart { + type_name: String, + part: Part, + }, + UnknownType { + type_name: String, + }, + BufferCount { + node: usize, + buffers: usize, + expected: usize, + }, + OffsetsLength { + of: Of, + bytes: u64, + expected: u64, + rows: u64, + }, + MapEntriesValidity { + bytes: u64, + }, + ValidityTooShort { + bytes: u64, + rows: u64, + }, + BuffersExhausted, + LeftoverBuffers { + count: usize, + }, + NullKindNullCount { + rows: u64, + null_count: u64, + }, + NullCountMismatch { + declared: u64, + derived: u64, + rows: u64, + }, + NestedNulls { + node: usize, + role: Role, + nulls: u64, + }, + OffsetsStart { + of: Of, + first: u64, + }, + OffsetsNotMonotonic { + of: Of, + index: u64, + previous: u64, + current: u64, + }, + ValuesLength { + values: Values, + bytes: u64, + expected: u64, + rows: u64, + }, + Utf8DataLength { + bytes: u64, + last_offset: u64, + }, + OffsetOverflow { + last: u64, + }, + DataTypeMismatch { + expected: DataType, + actual: DataType, + }, + Arrow(ArrowError), + FrameTooLarge { + bytes: u64, + }, + DirectoryTruncated, + FrameNodeCount { + frame: u64, + schema: u64, + }, + FrameBufferCount { + frame: u64, + schema: u64, + }, + ColumnRows { + column: usize, + rows: u64, + frame_rows: u64, + }, + ColumnNulls { + column: usize, + nulls: u64, + rows: u64, + }, + BufferOutsideArea { + buffer: usize, + area: u64, + }, + BufferShorterThanPrefix { + buffer: usize, + }, + CompressedBuffer { + buffer: usize, + prefix: i64, + }, + NodesDisagree { + column: usize, + }, + MalformedColumn { + column: usize, + error: Box, + }, +} + +impl std::fmt::Display for Error { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + std::fmt::Debug::fmt(self, f) + } +} + +impl std::error::Error for Error {} diff --git a/src/extension/flow-php-ext/crates/flow-batch-frame/src/kind.rs b/src/extension/flow-php-ext/crates/flow-batch-frame/src/kind.rs new file mode 100644 index 0000000000..fbc24068a9 --- /dev/null +++ b/src/extension/flow-php-ext/crates/flow-batch-frame/src/kind.rs @@ -0,0 +1,206 @@ +//! Flow schema and type JSON (`Schema::normalize()`, `Type::normalize()`) and the column kind a type is stored as. + +use std::sync::Arc; + +use arrow_schema::{DataType, Field as ArrowField, FieldRef, Fields, TimeUnit}; +use serde::Deserialize; + +use crate::error::{Error, Part}; + +#[derive(Deserialize, Default)] +pub struct TypeJson { + #[serde(rename = "type")] + pub type_: String, + element: Option>, + key: Option>, + value: Option>, + base: Option>, + #[serde(default)] + fields: Vec, + #[serde(default)] + zone: Option, + #[serde(default)] + class: Option, +} + +#[derive(Deserialize)] +pub struct StructureElementJson { + // ALWAYS a JSON string - PHP casts the name on normalize(). A JSON number here + // would fail parse_schema for the WHOLE schema, not just this field. + pub name: String, + #[serde(rename = "type")] + pub type_: TypeJson, + #[serde(default)] + pub optional: bool, +} + +impl TypeJson { + pub fn element(&self) -> Option<&TypeJson> { + self.element.as_deref() + } + + pub fn key(&self) -> Option<&TypeJson> { + self.key.as_deref() + } + + pub fn value(&self) -> Option<&TypeJson> { + self.value.as_deref() + } + + pub fn base(&self) -> Option<&TypeJson> { + self.base.as_deref() + } + + pub fn fields(&self) -> &[StructureElementJson] { + &self.fields + } + + pub fn zone(&self) -> Option<&[u8]> { + self.zone.as_deref().map(str::as_bytes) + } + + pub fn class(&self) -> Option<&[u8]> { + self.class.as_deref().map(str::as_bytes) + } + + fn is_optional(&self) -> bool { + self.type_ == "optional" + } +} + +#[derive(Deserialize)] +pub struct NormalizedDefinition { + #[serde(rename = "ref")] + pub name: String, + #[serde(rename = "type")] + pub type_: TypeJson, + pub nullable: bool, +} + +pub fn parse_schema(schema_json: &[u8]) -> Result, Error> { + let json = std::str::from_utf8(schema_json).map_err(|_| Error::SchemaJsonUtf8)?; + + serde_json::from_str(json).map_err(Error::SchemaJson) +} + +pub fn parse_type(type_json: &[u8]) -> Result { + let json = std::str::from_utf8(type_json).map_err(|_| Error::TypeJsonUtf8)?; + + serde_json::from_str(json).map_err(Error::TypeJson) +} + +/// How a column of a Flow type is stored. `Bytes` holds every string-like type (string, non-empty and numeric +/// strings, class strings, json, enum, timezone, xml, xml_element, html, html_element), `Timestamp` a datetime, +/// `Duration` a time and `Int32` a date. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum Kind { + Null, + Boolean, + Int64, + Int32, + Float64, + Uuid, + Bytes, + Timestamp, + Duration, + List(Box), + Map(Box, Box), + Struct(Vec), +} + +/// A nested child. `optional` is the structure element's flag for a structure child, and whether the child type is +/// optional for a list element, map key or map value. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Field { + pub name: String, + pub kind: Kind, + pub optional: bool, +} + +pub fn kind(type_json: &[u8]) -> Result { + kind_of(&parse_type(type_json)?) +} + +pub fn kind_of(parsed: &TypeJson) -> Result { + let missing = |part: Part| Error::MissingPart { + type_name: parsed.type_.clone(), + part, + }; + let child = |name: &str, type_json: &TypeJson| -> Result, Error> { + Ok(Box::new(Field { + name: name.to_owned(), + kind: kind_of(type_json)?, + optional: type_json.is_optional(), + })) + }; + + Ok(match parsed.type_.as_str() { + "integer" | "positive_integer" => Kind::Int64, + "float" => Kind::Float64, + "boolean" => Kind::Boolean, + "string" | "non_empty_string" | "numeric-string" | "class_string" | "json" | "enum" | "timezone" | "xml" + | "xml_element" | "html" | "html_element" => Kind::Bytes, + "datetime" => Kind::Timestamp, + "date" => Kind::Int32, + "time" => Kind::Duration, + "uuid" => Kind::Uuid, + "null" => Kind::Null, + "list" => Kind::List(child("item", parsed.element().ok_or_else(|| missing(Part::Element))?)?), + "map" => Kind::Map( + child("key", parsed.key().ok_or_else(|| missing(Part::Key))?)?, + child("value", parsed.value().ok_or_else(|| missing(Part::Value))?)?, + ), + "structure_v2" => Kind::Struct( + parsed + .fields() + .iter() + .map(|element| { + Ok(Field { + name: element.name.clone(), + kind: kind_of(&element.type_)?, + optional: element.optional, + }) + }) + .collect::>()?, + ), + "optional" => kind_of(parsed.base().ok_or_else(|| missing(Part::Base))?)?, + other => { + return Err(Error::UnknownType { + type_name: other.to_owned(), + }) + } + }) +} + +// Must equal arrow-ext's parquet/canonical.rs canonical_type(), nested names and nullability included - pinned by phpt 091. +pub fn data_type(kind: &Kind) -> DataType { + match kind { + Kind::Null => DataType::Null, + Kind::Boolean => DataType::Boolean, + Kind::Int64 => DataType::Int64, + Kind::Int32 => DataType::Date32, + Kind::Float64 => DataType::Float64, + Kind::Uuid => DataType::FixedSizeBinary(16), + Kind::Bytes => DataType::Binary, + Kind::Timestamp => DataType::Timestamp(TimeUnit::Microsecond, Some("UTC".into())), + Kind::Duration => DataType::Duration(TimeUnit::Microsecond), + Kind::List(element) => DataType::List(field("item", &element.kind)), + Kind::Map(key, value) => DataType::Map( + Arc::new(ArrowField::new( + "entries", + DataType::Struct(Fields::from(vec![ + ArrowField::new("key", data_type(&key.kind), false), + ArrowField::new("value", data_type(&value.kind), true), + ])), + false, + )), + false, + ), + Kind::Struct(fields) => DataType::Struct(fields.iter().map(|child| field(&child.name, &child.kind)).collect()), + } +} + +/// Nullable: only map entries and keys are not, and data_type() builds those itself. +pub fn field(name: &str, kind: &Kind) -> FieldRef { + Arc::new(ArrowField::new(name, data_type(kind), true)) +} diff --git a/src/extension/flow-php-ext/crates/flow-batch-frame/src/layout.rs b/src/extension/flow-php-ext/crates/flow-batch-frame/src/layout.rs new file mode 100644 index 0000000000..a39ee8dbdd --- /dev/null +++ b/src/extension/flow-php-ext/crates/flow-batch-frame/src/layout.rs @@ -0,0 +1,143 @@ +//! The one node/buffer layout of a column (`Flow\ETL\Column\BufferLayout`): the pre-order nodes and buffers a kind +//! spans, and the child nodes derived from a column's buffers. + +use crate::error::{Error, Of}; +use crate::kind::Kind; + +pub fn node_count(kind: &Kind) -> usize { + match kind { + Kind::List(element) => 1 + node_count(&element.kind), + Kind::Map(key, value) => 2 + node_count(&key.kind) + node_count(&value.kind), + Kind::Struct(fields) => 1 + fields.iter().map(|field| node_count(&field.kind)).sum::(), + _ => 1, + } +} + +pub fn buffer_count(kind: &Kind) -> usize { + match kind { + Kind::Null => 0, + Kind::List(element) => 2 + buffer_count(&element.kind), + Kind::Map(key, value) => 3 + buffer_count(&key.kind) + buffer_count(&value.kind), + Kind::Struct(fields) => 1 + fields.iter().map(|field| buffer_count(&field.kind)).sum::(), + Kind::Bytes => 3, + _ => 2, + } +} + +/// Pre-order `(length, null count)` of one column's tree: the first node as given, every child derived from the +/// buffers - a null kind is `(length, length)`, list and map children span the last offset, a map entries node holds +/// no nulls and structure children span their parent. +pub fn nodes(kind: &Kind, len: u32, null_count: u32, buffers: &[&[u8]]) -> Result, Error> { + nodes_at(kind, 0, len, null_count, buffers) +} + +pub(crate) fn nodes_at( + kind: &Kind, + node: usize, + len: u32, + null_count: u32, + buffers: &[&[u8]], +) -> Result, Error> { + let expected = buffer_count(kind); + + if buffers.len() != expected { + return Err(Error::BufferCount { + node, + buffers: buffers.len(), + expected, + }); + } + + let mut nodes = vec![(len, null_count)]; + let mut children: Vec<(&Kind, u32)> = Vec::new(); + let mut position = 1; + + match kind { + Kind::List(element) => { + children.push((&element.kind, last_offset(buffers[1], len, Of::List)?)); + position = 2; + } + Kind::Map(key, value) => { + let child_len = last_offset(buffers[1], len, Of::Map)?; + + if !buffers[2].is_empty() { + return Err(Error::MapEntriesValidity { + bytes: buffers[2].len() as u64, + }); + } + + nodes.push((child_len, 0)); + children.push((&key.kind, child_len)); + children.push((&value.kind, child_len)); + position = 3; + } + Kind::Struct(fields) => { + children.extend(fields.iter().map(|field| (&field.kind, len))); + } + _ => {} + } + + for (child, child_len) in children { + let child_buffers = &buffers[position..position + buffer_count(child)]; + position += child_buffers.len(); + let child_nulls = match child { + Kind::Null => child_len, + _ => null_count_of(child_buffers[0], child_len)?, + }; + + nodes.extend(nodes_at( + child, + node + nodes.len(), + child_len, + child_nulls, + child_buffers, + )?); + } + + Ok(nodes) +} + +/// `Validity::nullCount()`: '' holds no nulls; a bitmap may be longer than its rows. +pub fn null_count_of(bitmap: &[u8], len: u32) -> Result { + if bitmap.is_empty() { + return Ok(0); + } + + let needed = (len as usize).div_ceil(8); + + if bitmap.len() < needed { + return Err(Error::ValidityTooShort { + bytes: bitmap.len() as u64, + rows: len as u64, + }); + } + + let full = len as usize / 8; + let mut valid: u32 = bitmap[..full].iter().map(|byte| byte.count_ones()).sum(); + let tail = len % 8; + + if tail != 0 { + valid += (bitmap[full] & ((1u8 << tail) - 1)).count_ones(); + } + + Ok(len - valid) +} + +fn last_offset(offsets: &[u8], len: u32, of: Of) -> Result { + let expected = (len as u64 + 1) * 4; + + if offsets.len() as u64 != expected { + return Err(Error::OffsetsLength { + of, + bytes: offsets.len() as u64, + expected, + rows: len as u64, + }); + } + + let at = len as usize * 4; + + Ok(u32::from_le_bytes( + offsets[at..at + 4].try_into().expect("4-byte slice"), + )) +} diff --git a/src/extension/flow-php-ext/crates/flow-batch-frame/src/lib.rs b/src/extension/flow-php-ext/crates/flow-batch-frame/src/lib.rs new file mode 100644 index 0000000000..68d82efcc0 --- /dev/null +++ b/src/extension/flow-php-ext/crates/flow-batch-frame/src/lib.rs @@ -0,0 +1,14 @@ +//! Flow's batch layout in Rust: one column's buffers in the canonical form of `Flow\ETL\Column\Column::encode()` and +//! back (`column`). The pure-PHP classes in flow-php/etl are the reference; this crate holds no PHP types and no message +//! text. + +// Floe is little-endian by construction; a big-endian host could never match its bytes. +#[cfg(target_endian = "big")] +compile_error!("flow-batch-frame only supports little-endian targets"); + +pub mod column; +pub mod error; +pub mod kind; +pub mod layout; + +pub use error::Error; diff --git a/src/extension/flow-php-ext/crates/flow-batch-frame/tests/column.rs b/src/extension/flow-php-ext/crates/flow-batch-frame/tests/column.rs new file mode 100644 index 0000000000..36ff0d5b92 --- /dev/null +++ b/src/extension/flow-php-ext/crates/flow-batch-frame/tests/column.rs @@ -0,0 +1,424 @@ +mod common; + +use std::sync::Arc; + +use arrow_array::builder::{BinaryBuilder, Int64Builder, MapBuilder, MapFieldNames}; +use arrow_array::types::Int64Type; +use arrow_array::{ + Array, ArrayRef, BinaryArray, BooleanArray, Date32Array, FixedSizeBinaryArray, Float64Array, Int64Array, ListArray, + NullArray, StructArray, TimestampMicrosecondArray, +}; +use arrow_buffer::NullBuffer; +use arrow_data::ArrayData; +use common::{hex, unhex}; +use flow_batch_frame::column::{decode, encode}; +use flow_batch_frame::error::{Error, Of, Role, Values}; +use flow_batch_frame::kind::{data_type, kind, Kind}; + +fn kind_of(type_json: &str) -> Kind { + kind(type_json.as_bytes()).expect("a valid type JSON") +} + +fn encoded_hex(data: &ArrayData, kind: &Kind) -> Vec { + encode(data, kind) + .expect("an encodable column") + .iter() + .map(|buffer| hex(buffer)) + .collect() +} + +fn decoded(kind: &Kind, buffers: &[&str], len: u32, null_count: u32) -> Result { + let buffers: Vec> = buffers.iter().map(|buffer| unhex(buffer)).collect(); + let buffers: Vec<&[u8]> = buffers.iter().map(Vec::as_slice).collect(); + + decode(kind, &buffers, len, null_count) +} + +fn raw(kind: &Kind, buffers: &[&[u8]], len: u32, null_count: u32) -> Error { + decode(kind, buffers, len, null_count).expect_err("a refused column") +} + +fn structure(fields: Vec<(&str, ArrayRef)>, nulls: Option>) -> ArrayData { + let arrow_fields = fields + .iter() + .map(|(name, array)| Arc::new(arrow_schema::Field::new(*name, array.data_type().clone(), true))) + .collect::>(); + + StructArray::try_new( + arrow_fields.into(), + fields.into_iter().map(|(_, array)| array).collect(), + nulls.map(NullBuffer::from), + ) + .expect("a valid struct") + .into_data() +} + +/// ColumnEncodeTest.php:40-105, and each case decodes back to itself. +fn cases() -> Vec<(&'static str, Kind, ArrayData, Vec<&'static str>)> { + let list = kind_of("{\"type\":\"list\",\"element\":{\"type\":\"integer\"}}"); + let list_rows = ListArray::from_iter_primitive::(vec![ + Some(vec![Some(1)]), + Some(vec![Some(2), Some(3)]), + None, + ]); + let mut map = MapBuilder::new( + Some(MapFieldNames { + entry: "entries".into(), + key: "key".into(), + value: "value".into(), + }), + BinaryBuilder::new(), + Int64Builder::new(), + ); + map.keys().append_value(b"a"); + map.values().append_value(1); + map.append(true).expect("a map row"); + + vec![ + ( + "integer", + kind_of("{\"type\":\"integer\"}"), + Int64Array::from(vec![1, 2]).into_data(), + vec!["", "01000000000000000200000000000000"], + ), + ( + "nullable string", + kind_of("{\"type\":\"string\"}"), + BinaryArray::from(vec![Some(b"ab".as_ref()), None]).into_data(), + vec!["01", "000000000200000002000000", "6162"], + ), + ( + "float", + kind_of("{\"type\":\"float\"}"), + Float64Array::from(vec![1.5]).into_data(), + vec!["", "000000000000f83f"], + ), + ( + "date before the epoch", + kind_of("{\"type\":\"date\"}"), + Date32Array::from(vec![-3]).into_data(), + vec!["", "fdffffff"], + ), + ( + "boolean", + kind_of("{\"type\":\"boolean\"}"), + BooleanArray::from(vec![true, false, true]).into_data(), + vec!["", "05"], + ), + ( + "nullable boolean with a null slot at 0", + kind_of("{\"type\":\"boolean\"}"), + BooleanArray::from(vec![Some(true), None, Some(true)]).into_data(), + vec!["05", "05"], + ), + ( + "uuid", + kind_of("{\"type\":\"uuid\"}"), + FixedSizeBinaryArray::try_from_iter(vec![unhex("6c2f1d4e8b3a4c5d9e6f0a1b2c3d4e5f")].into_iter()) + .expect("16 bytes") + .into_data(), + vec!["", "6c2f1d4e8b3a4c5d9e6f0a1b2c3d4e5f"], + ), + ( + "list with a null slot", + list.clone(), + ListArray::from_iter_primitive::(vec![ + Some(vec![Some(1)]), + None, + Some(vec![Some(2), Some(3)]), + ]) + .into_data(), + vec!["05", "00000000010000000100000003000000", "", "010000000000000002000000000000000300000000000000"], + ), + ( + "map", + kind_of("{\"type\":\"map\",\"key\":{\"type\":\"string\"},\"value\":{\"type\":\"integer\"}}"), + map.finish().into_data(), + vec!["", "0000000001000000", "", "", "0000000001000000", "61", "", "0100000000000000"], + ), + ( + "structure: the child bit is 0 under a null parent", + kind_of("{\"type\":\"structure_v2\",\"fields\":[{\"name\":\"x\",\"type\":{\"type\":\"integer\"}}]}"), + structure(vec![("x", Arc::new(Int64Array::from(vec![1, 7])))], Some(vec![true, false])), + vec!["01", "01", "01000000000000000000000000000000"], + ), + ( + "a null column has no buffers", + kind_of("{\"type\":\"null\"}"), + NullArray::new(3).into_data(), + vec![], + ), + ( + "a null element of a structure has no buffers", + kind_of( + "{\"type\":\"structure_v2\",\"fields\":[{\"name\":\"x\",\"type\":{\"type\":\"integer\"}},{\"name\":\"n\",\"type\":{\"type\":\"null\"}}]}", + ), + structure( + vec![("x", Arc::new(Int64Array::from(vec![1]))), ("n", Arc::new(NullArray::new(1)))], + None, + ), + vec!["", "", "0100000000000000"], + ), + ( + "an optional null element of a structure has no buffers", + kind_of( + "{\"type\":\"structure_v2\",\"fields\":[{\"name\":\"x\",\"type\":{\"type\":\"integer\"}},{\"name\":\"n\",\"type\":{\"type\":\"optional\",\"base\":{\"type\":\"null\"}}}]}", + ), + structure( + vec![("x", Arc::new(Int64Array::from(vec![1]))), ("n", Arc::new(NullArray::new(1)))], + None, + ), + vec!["", "", "0100000000000000"], + ), + ( + "a slice re-bases offsets and re-packs validity from bit 0", + list, + list_rows.slice(1, 2).into_data(), + vec!["01", "000000000200000002000000", "", "02000000000000000300000000000000"], + ), + ] +} + +#[test] +fn encodes_the_column_encode_test_cases() { + for (name, kind, data, expected) in cases() { + assert_eq!(expected, encoded_hex(&data, &kind), "{name}"); + } +} + +#[test] +fn decodes_the_column_encode_test_cases_back() { + for (name, kind, data, expected) in cases() { + let null_count = match kind { + Kind::Null => data.len() as u32, + _ => data.nulls().map_or(0, |nulls| nulls.null_count() as u32), + }; + let decoded = decoded(&kind, &expected, data.len() as u32, null_count).expect(name); + + assert_eq!(&data_type(&kind), decoded.data_type(), "{name}"); + assert_eq!(expected, encoded_hex(&decoded, &kind), "{name}"); + } +} + +#[test] +fn a_selection_under_a_null_list_slot_is_compacted() { + let kind = kind_of("{\"type\":\"list\",\"element\":{\"type\":\"string\"}}"); + let strings = BinaryArray::from(vec![b"a".as_ref(), b"bc", b"d"]); + let offsets = arrow_buffer::OffsetBuffer::new(vec![0, 1, 2, 3].into()); + let field = Arc::new(arrow_schema::Field::new("item", arrow_schema::DataType::Binary, true)); + let list = ListArray::new( + field, + offsets, + Arc::new(strings), + Some(NullBuffer::from(vec![true, false, true])), + ); + + assert_eq!( + vec![ + "05", + "00000000010000000100000002000000", + "", + "000000000100000002000000", + "6164" + ], + encoded_hex(&list.into_data(), &kind), + ); +} + +#[test] +fn bytes_that_are_not_utf8_round_trip() { + let kind = kind_of("{\"type\":\"string\"}"); + let data = BinaryArray::from(vec![b"a\xFFb".as_ref()]).into_data(); + let buffers = encoded_hex(&data, &kind); + let buffers: Vec<&str> = buffers.iter().map(String::as_str).collect(); + + assert_eq!(vec!["", "0000000003000000", "61ff62"], buffers); + assert_eq!( + b"a\xFFb", + decoded(&kind, &buffers, 1, 0).expect("a binary column").buffers()[1].as_slice(), + ); +} + +#[test] +fn encode_refuses_a_foreign_data_type() { + let kind = kind_of("{\"type\":\"datetime\",\"zone\":\"UTC\"}"); + + assert!(matches!( + encode(&TimestampMicrosecondArray::from(vec![1]).into_data(), &kind), + Err(Error::DataTypeMismatch { .. }), + )); +} + +/// PhpBackendTest.php:454-556. +#[test] +fn decode_refuses_corrupt_buffers() { + let int = kind_of("{\"type\":\"integer\"}"); + let string = kind_of("{\"type\":\"string\"}"); + let map = kind_of("{\"type\":\"map\",\"key\":{\"type\":\"string\"},\"value\":{\"type\":\"integer\"}}"); + let null = kind_of("{\"type\":\"null\"}"); + + assert!(matches!(raw(&int, &[b""], 1, 0), Error::BuffersExhausted)); + assert!(matches!( + raw(&int, &[b"", b"\x01\x00"], 1, 0), + Error::ValuesLength { + values: Values::Int64, + bytes: 2, + expected: 8, + rows: 1 + }, + )); + assert!(matches!( + raw(&string, &[b"", b"\x01\x00\x00\x00\x02\x00\x00\x00", b"ab"], 1, 0), + Error::OffsetsStart { of: Of::Utf8, first: 1 }, + )); + assert!(matches!( + raw( + &string, + &[b"", b"\x00\x00\x00\x00\x02\x00\x00\x00\x01\x00\x00\x00", b"ab"], + 2, + 0 + ), + Error::OffsetsNotMonotonic { + of: Of::Utf8, + index: 2, + previous: 2, + current: 1 + }, + )); + assert!(matches!( + raw(&string, &[b"", b"\x00\x00\x00\x00\x03\x00\x00\x00", b"ab"], 1, 0), + Error::Utf8DataLength { + bytes: 2, + last_offset: 3 + }, + )); + assert!(matches!( + raw( + &map, + &[ + b"", + b"\x00\x00\x00\x00\x00\x00\x00\x00", + b"\x01", + b"", + b"\x00\x00\x00\x00", + b"", + b"", + b"" + ], + 1, + 0 + ), + Error::MapEntriesValidity { bytes: 1 }, + )); + assert!(matches!( + raw(&int, &[b"\x01", &[0; 16]], 2, 0), + Error::NullCountMismatch { + declared: 0, + derived: 1, + rows: 2 + }, + )); + assert!(matches!( + raw(&int, &[b"", b""], 9, 0), + Error::ValuesLength { + values: Values::Int64, + bytes: 0, + expected: 72, + rows: 9 + }, + )); + assert!(matches!( + raw(&null, &[], 2, 1), + Error::NullKindNullCount { rows: 2, null_count: 1 } + )); + assert!(matches!( + raw(&int, &[b"", b"\x01\x00\x00\x00\x00\x00\x00\x00", b""], 1, 0), + Error::LeftoverBuffers { count: 1 }, + )); + assert!(matches!( + raw(&int, &[b"\x01", &[0; 72]], 9, 0), + Error::ValidityTooShort { bytes: 1, rows: 9 }, + )); + assert!(matches!( + raw(&string, &[b"", b"\x00\x00", b""], 1, 0), + Error::OffsetsLength { + of: Of::Utf8, + bytes: 2, + expected: 8, + rows: 1 + }, + )); +} + +/// ColumnDecoderTest.php: a list element or map value holds nulls only when optional or of the null kind, a map key +/// never, a structure child always may. +#[test] +fn decode_refuses_nested_nulls_where_the_type_forbids_them() { + let one_null_child = [b"".as_ref(), b"\x00\x00\x00\x00\x01\x00\x00\x00", b"\x00", &[0; 8]]; + + assert!(matches!( + raw( + &kind_of("{\"type\":\"list\",\"element\":{\"type\":\"integer\"}}"), + &one_null_child, + 1, + 0 + ), + Error::NestedNulls { + node: 1, + role: Role::ListElement, + nulls: 1 + }, + )); + assert!(decode( + &kind_of("{\"type\":\"list\",\"element\":{\"type\":\"optional\",\"base\":{\"type\":\"integer\"}}}"), + &one_null_child, + 1, + 0, + ) + .is_ok()); + assert!(decode( + &kind_of("{\"type\":\"list\",\"element\":{\"type\":\"null\"}}"), + &[b"".as_ref(), b"\x00\x00\x00\x00\x02\x00\x00\x00"], + 1, + 0, + ) + .is_ok()); + + let entries = |key_validity: &'static [u8], value_validity: &'static [u8]| -> [&'static [u8]; 7] { + [ + b"", + b"\x00\x00\x00\x00\x01\x00\x00\x00", + b"", + key_validity, + &[0; 8], + value_validity, + &[0; 8], + ] + }; + let map = kind_of("{\"type\":\"map\",\"key\":{\"type\":\"integer\"},\"value\":{\"type\":\"integer\"}}"); + let optional_values = + kind_of("{\"type\":\"map\",\"key\":{\"type\":\"integer\"},\"value\":{\"type\":\"optional\",\"base\":{\"type\":\"integer\"}}}"); + + assert!(matches!( + raw(&map, &entries(b"\x00", b""), 1, 0), + Error::NestedNulls { + node: 2, + role: Role::MapKey, + nulls: 1 + }, + )); + assert!(matches!( + raw(&map, &entries(b"", b"\x00"), 1, 0), + Error::NestedNulls { + node: 3, + role: Role::MapValue, + nulls: 1 + }, + )); + assert!(decode(&optional_values, &entries(b"", b"\x00"), 1, 0).is_ok()); + + let structure = + kind_of("{\"type\":\"structure_v2\",\"fields\":[{\"name\":\"x\",\"type\":{\"type\":\"integer\"}}]}"); + + assert!(decode(&structure, &[b"".as_ref(), b"\x00", &[0; 8]], 1, 0).is_ok()); +} diff --git a/src/extension/flow-php-ext/crates/flow-batch-frame/tests/common/mod.rs b/src/extension/flow-php-ext/crates/flow-batch-frame/tests/common/mod.rs new file mode 100644 index 0000000000..a9dfa5a5ca --- /dev/null +++ b/src/extension/flow-php-ext/crates/flow-batch-frame/tests/common/mod.rs @@ -0,0 +1,10 @@ +pub fn hex(bytes: &[u8]) -> String { + bytes.iter().map(|byte| format!("{byte:02x}")).collect() +} + +pub fn unhex(hex: &str) -> Vec { + (0..hex.len()) + .step_by(2) + .map(|i| u8::from_str_radix(&hex[i..i + 2], 16).expect("hex digit pair")) + .collect() +} diff --git a/src/extension/flow-php-ext/crates/flow-batch-frame/tests/kind.rs b/src/extension/flow-php-ext/crates/flow-batch-frame/tests/kind.rs new file mode 100644 index 0000000000..f6610f5508 --- /dev/null +++ b/src/extension/flow-php-ext/crates/flow-batch-frame/tests/kind.rs @@ -0,0 +1,127 @@ +use arrow_data::ArrayData; +use flow_batch_frame::column::{decode, encode}; +use flow_batch_frame::error::{Error, Part}; +use flow_batch_frame::kind::{data_type, kind, kind_of, parse_schema, parse_type, Field, Kind}; + +/// The footer schema of FloeGoldenTest's `v2/all-entry-types.floe`. +const GOLDEN_SCHEMA: &str = r#"[{"ref":"int","type":{"type":"integer"},"nullable":false,"metadata":[]},{"ref":"int_min","type":{"type":"integer"},"nullable":false,"metadata":[]},{"ref":"int_max","type":{"type":"integer"},"nullable":false,"metadata":[]},{"ref":"float","type":{"type":"float"},"nullable":false,"metadata":[]},{"ref":"bool","type":{"type":"boolean"},"nullable":false,"metadata":[]},{"ref":"string","type":{"type":"string"},"nullable":false,"metadata":[]},{"ref":"string_binary","type":{"type":"string"},"nullable":false,"metadata":[]},{"ref":"string_unicode","type":{"type":"string"},"nullable":false,"metadata":[]},{"ref":"string_null","type":{"type":"string"},"nullable":true,"metadata":[]},{"ref":"string_from_null","type":{"type":"null"},"nullable":true,"metadata":[]},{"ref":"datetime_immutable","type":{"type":"datetime","zone":"Europe\/Warsaw"},"nullable":false,"metadata":[]},{"ref":"datetime_mutable","type":{"type":"datetime","zone":"America\/New_York"},"nullable":false,"metadata":[]},{"ref":"datetime_before_epoch","type":{"type":"datetime","zone":"UTC"},"nullable":false,"metadata":[]},{"ref":"date","type":{"type":"date"},"nullable":false,"metadata":[]},{"ref":"time","type":{"type":"time"},"nullable":false,"metadata":[]},{"ref":"uuid","type":{"type":"uuid"},"nullable":false,"metadata":[]},{"ref":"json","type":{"type":"json"},"nullable":false,"metadata":[]},{"ref":"json_object","type":{"type":"json"},"nullable":false,"metadata":[]},{"ref":"enum_backed","type":{"type":"enum","class":"Flow\\ETL\\Tests\\Fixtures\\Enum\\BackedStringEnum"},"nullable":false,"metadata":[]},{"ref":"enum_basic","type":{"type":"enum","class":"Flow\\ETL\\Tests\\Fixtures\\Enum\\BasicEnum"},"nullable":false,"metadata":[]},{"ref":"list","type":{"type":"list","element":{"type":"integer"}},"nullable":false,"metadata":[]},{"ref":"map","type":{"type":"map","key":{"type":"string"},"value":{"type":"integer"}},"nullable":false,"metadata":[]},{"ref":"structure","type":{"type":"structure_v2","fields":[{"name":"street","type":{"type":"string"},"optional":false},{"name":"nested","type":{"type":"structure_v2","fields":[{"name":"count","type":{"type":"integer"},"optional":false},{"name":"tags","type":{"type":"list","element":{"type":"string"}},"optional":false}]},"optional":false}]},"nullable":false,"metadata":[]},{"ref":"structure_interleaved","type":{"type":"structure_v2","fields":[{"name":"z","type":{"type":"integer"},"optional":false},{"name":"a","type":{"type":"string"},"optional":true},{"name":"b","type":{"type":"string"},"optional":false}]},"nullable":false,"metadata":[]},{"ref":"xml","type":{"type":"xml"},"nullable":false,"metadata":[]},{"ref":"xml_element","type":{"type":"xml_element"},"nullable":false,"metadata":[]}]"#; + +fn field(name: &str, kind: Kind, optional: bool) -> Field { + Field { + name: name.to_owned(), + kind, + optional, + } +} + +#[test] +fn parses_a_golden_footer_schema() { + let definitions = parse_schema(GOLDEN_SCHEMA.as_bytes()).expect("the golden schema"); + + assert_eq!(26, definitions.len()); + assert_eq!("string_null", definitions[8].name); + assert!(definitions[8].nullable); + assert_eq!(Some(b"Europe/Warsaw".as_ref()), definitions[10].type_.zone()); + assert_eq!( + Some(br"Flow\ETL\Tests\Fixtures\Enum\BackedStringEnum".as_ref()), + definitions[18].type_.class() + ); + assert_eq!( + Kind::Struct(vec![ + field("z", Kind::Int64, false), + field("a", Kind::Bytes, true), + field("b", Kind::Bytes, false), + ]), + kind_of(&definitions[23].type_).expect("a structure"), + ); +} + +#[test] +fn refuses_schema_and_type_json_it_cannot_read() { + assert!(matches!(parse_schema(b"[\xFF]"), Err(Error::SchemaJsonUtf8))); + assert!(matches!(parse_schema(b"[{"), Err(Error::SchemaJson(_)))); + assert!(matches!(parse_type(b"\xFF"), Err(Error::TypeJsonUtf8))); + assert!(matches!(parse_type(b"{}"), Err(Error::TypeJson(_)))); +} + +#[test] +fn refuses_a_type_without_a_column_kind() { + assert!(matches!( + kind(br#"{"type":"mixed"}"#), + Err(Error::UnknownType { type_name }) if type_name == "mixed", + )); + assert!(matches!( + kind(br#"{"type":"list"}"#), + Err(Error::MissingPart { type_name, part: Part::Element }) if type_name == "list", + )); + assert!(matches!( + kind(br#"{"type":"map","key":{"type":"string"}}"#), + Err(Error::MissingPart { part: Part::Value, .. }), + )); + assert!(matches!( + kind(br#"{"type":"optional"}"#), + Err(Error::MissingPart { part: Part::Base, .. }) + )); +} + +/// Every Definition's `Schema::normalize()` type JSON. +fn definitions() -> Vec<(&'static str, Kind)> { + vec![ + (r#"{"type":"integer"}"#, Kind::Int64), + (r#"{"type":"float"}"#, Kind::Float64), + (r#"{"type":"boolean"}"#, Kind::Boolean), + (r#"{"type":"string"}"#, Kind::Bytes), + (r#"{"type":"datetime","zone":"UTC"}"#, Kind::Timestamp), + (r#"{"type":"date"}"#, Kind::Int32), + (r#"{"type":"time"}"#, Kind::Duration), + (r#"{"type":"uuid"}"#, Kind::Uuid), + (r#"{"type":"json"}"#, Kind::Bytes), + (r#"{"type":"enum","class":"E"}"#, Kind::Bytes), + (r#"{"type":"timezone"}"#, Kind::Bytes), + (r#"{"type":"xml"}"#, Kind::Bytes), + (r#"{"type":"xml_element"}"#, Kind::Bytes), + (r#"{"type":"html"}"#, Kind::Bytes), + (r#"{"type":"html_element"}"#, Kind::Bytes), + ( + r#"{"type":"list","element":{"type":"optional","base":{"type":"integer"}}}"#, + Kind::List(Box::new(field("item", Kind::Int64, true))), + ), + ( + r#"{"type":"map","key":{"type":"string"},"value":{"type":"optional","base":{"type":"integer"}}}"#, + Kind::Map( + Box::new(field("key", Kind::Bytes, false)), + Box::new(field("value", Kind::Int64, true)), + ), + ), + ( + r#"{"type":"structure_v2","fields":[{"name":"a","type":{"type":"integer"},"optional":false},{"name":"b","type":{"type":"optional","base":{"type":"string"}},"optional":false},{"name":"c","type":{"type":"string"},"optional":true}]}"#, + Kind::Struct(vec![ + field("a", Kind::Int64, false), + field("b", Kind::Bytes, false), + field("c", Kind::Bytes, true), + ]), + ), + (r#"{"type":"null"}"#, Kind::Null), + ] +} + +#[test] +fn every_definition_has_its_kind() { + for (type_json, expected) in definitions() { + assert_eq!(expected, kind(type_json.as_bytes()).expect(type_json), "{type_json}"); + } +} + +#[test] +fn decode_builds_the_data_type_of_its_kind() { + for (type_json, kind) in definitions() { + let empty = encode(&ArrayData::new_empty(&data_type(&kind)), &kind).expect(type_json); + let buffers: Vec<&[u8]> = empty.iter().map(Vec::as_slice).collect(); + + assert_eq!( + &data_type(&kind), + decode(&kind, &buffers, 0, 0).expect(type_json).data_type(), + "{type_json}", + ); + } +} diff --git a/src/extension/flow-php-ext/php/Flow/ETL/Adapter/CSV/RustCSVEncoder.php b/src/extension/flow-php-ext/php/Flow/ETL/Adapter/CSV/RustCSVEncoder.php new file mode 100644 index 0000000000..af254d42d9 --- /dev/null +++ b/src/extension/flow-php-ext/php/Flow/ETL/Adapter/CSV/RustCSVEncoder.php @@ -0,0 +1,45 @@ + $headers + */ + public function encodeHeader(array $headers): string + { + throw new RuntimeException('flow_php extension is not loaded'); + } +} diff --git a/src/extension/flow-php-ext/php/Flow/ETL/Adapter/CSV/RustCSVOpenSource.php b/src/extension/flow-php-ext/php/Flow/ETL/Adapter/CSV/RustCSVOpenSource.php new file mode 100644 index 0000000000..a3f41b99a7 --- /dev/null +++ b/src/extension/flow-php-ext/php/Flow/ETL/Adapter/CSV/RustCSVOpenSource.php @@ -0,0 +1,83 @@ + + */ + public function batches(Schema $schema, int $batchSize, Backend $backend): RustIterator + { + throw new RuntimeException('flow_php extension is not loaded'); + } + + public function close(): void + { + throw new RuntimeException('flow_php extension is not loaded'); + } + + public function columns(): array + { + throw new RuntimeException('flow_php extension is not loaded'); + } + + public function headers(): array + { + throw new RuntimeException('flow_php extension is not loaded'); + } + + public function producedBytes(): int + { + throw new RuntimeException('flow_php extension is not loaded'); + } + + public function producedRows(): int + { + throw new RuntimeException('flow_php extension is not loaded'); + } + + /** + * @return RustIterator> + */ + public function records(): RustIterator + { + throw new RuntimeException('flow_php extension is not loaded'); + } + + public function sniff(array $names, int $rowBudget, SchemaInference $inference, TypeNarrower $typer): ColumnTypes + { + throw new RuntimeException('flow_php extension is not loaded'); + } +} diff --git a/src/extension/flow-php-ext/php/Flow/ETL/Adapter/CSV/RustCSVReaderNative.php b/src/extension/flow-php-ext/php/Flow/ETL/Adapter/CSV/RustCSVReaderNative.php deleted file mode 100644 index 1a35add1bd..0000000000 --- a/src/extension/flow-php-ext/php/Flow/ETL/Adapter/CSV/RustCSVReaderNative.php +++ /dev/null @@ -1,77 +0,0 @@ - - */ - public function consumedBytes(): int - { - throw new RuntimeException('flow_php extension is not loaded'); - } - - public function feed(string $chunk): void - { - throw new RuntimeException('flow_php extension is not loaded'); - } - - public function finish(): void - { - throw new RuntimeException('flow_php extension is not loaded'); - } - - /** - * @param int<0, max>|-1 $limit - * - * @return int<0, max> - */ - public function fold(RustColumnFoldNative $fold, int $limit): int - { - throw new RuntimeException('flow_php extension is not loaded'); - } - - /** - * @return list - */ - public function headers(): array - { - throw new RuntimeException('flow_php extension is not loaded'); - } - - /** - * @param int<1, max> $batchSize - * - * @return list - */ - public function next(int $batchSize): array - { - throw new RuntimeException('flow_php extension is not loaded'); - } -} diff --git a/src/extension/flow-php-ext/php/Flow/ETL/Adapter/CSV/RustColumnFoldNative.php b/src/extension/flow-php-ext/php/Flow/ETL/Adapter/CSV/RustColumnFoldNative.php deleted file mode 100644 index 57e98214ee..0000000000 --- a/src/extension/flow-php-ext/php/Flow/ETL/Adapter/CSV/RustColumnFoldNative.php +++ /dev/null @@ -1,46 +0,0 @@ - $names - * @param list $candidates - the candidate types' toString() - */ - public function __construct(array $names, array $candidates) - { - throw new RuntimeException('flow_php extension is not loaded'); - } - - public function narrowOne(string $value): string - { - throw new RuntimeException('flow_php extension is not loaded'); - } - - /** - * @return int<0, max> - */ - public function rows(): int - { - throw new RuntimeException('flow_php extension is not loaded'); - } - - /** - * @return array - */ - public function types(): array - { - throw new RuntimeException('flow_php extension is not loaded'); - } -} diff --git a/src/extension/flow-php-ext/php/Flow/ETL/Adapter/JSON/RustJsonEncoder.php b/src/extension/flow-php-ext/php/Flow/ETL/Adapter/JSON/RustJsonEncoder.php new file mode 100644 index 0000000000..09f3c63807 --- /dev/null +++ b/src/extension/flow-php-ext/php/Flow/ETL/Adapter/JSON/RustJsonEncoder.php @@ -0,0 +1,33 @@ + $offset + */ + public function __construct(SourceStream $stream, bool $lines, string $uri, string $head, int $offset) + { + throw new RuntimeException('flow_php extension is not loaded'); + } + + /** + * @return RustIterator + */ + public function batches(Schema $schema, int $batchSize, Backend $backend): RustIterator + { + throw new RuntimeException('flow_php extension is not loaded'); + } + + public function close(): void + { + throw new RuntimeException('flow_php extension is not loaded'); + } +} diff --git a/src/extension/flow-php-ext/php/Flow/ETL/Adapter/Parquet/RustParquetOpenSink.php b/src/extension/flow-php-ext/php/Flow/ETL/Adapter/Parquet/RustParquetOpenSink.php new file mode 100644 index 0000000000..b6ff2c2628 --- /dev/null +++ b/src/extension/flow-php-ext/php/Flow/ETL/Adapter/Parquet/RustParquetOpenSink.php @@ -0,0 +1,42 @@ + + */ + public function batches(Schema $schema, int $batchSize, ?int $offset, ?int $limit, Backend $backend): RustIterator + { + throw new RuntimeException('flow_php extension is not loaded'); + } + + public function close(): void + { + throw new RuntimeException('flow_php extension is not loaded'); + } +} diff --git a/src/extension/flow-php-ext/php/Flow/ETL/Column/RustBackend.php b/src/extension/flow-php-ext/php/Flow/ETL/Column/RustBackend.php new file mode 100644 index 0000000000..5decae27bb --- /dev/null +++ b/src/extension/flow-php-ext/php/Flow/ETL/Column/RustBackend.php @@ -0,0 +1,42 @@ + $batch - */ - public function hydrate(array $batch, Schema $schema): Rows - { - throw new RuntimeException('flow_php extension is not loaded'); - } - - /** - * @return list - */ - public function dehydrate(Rows $rows): array - { - throw new RuntimeException('flow_php extension is not loaded'); - } - - /** - * Present in every extension whose casts and decoders apply the datetime column zone. - */ - public function datetimeZones(): bool - { - throw new RuntimeException('flow_php extension is not loaded'); - } -} diff --git a/src/extension/flow-php-ext/php/Flow/ETL/RustIterator.php b/src/extension/flow-php-ext/php/Flow/ETL/RustIterator.php new file mode 100644 index 0000000000..5839796311 --- /dev/null +++ b/src/extension/flow-php-ext/php/Flow/ETL/RustIterator.php @@ -0,0 +1,52 @@ + + */ +final class RustIterator implements Iterator +{ + /** + * @return T + */ + public function current(): mixed + { + throw new RuntimeException('flow_php extension is not loaded'); + } + + public function key(): ?int + { + throw new RuntimeException('flow_php extension is not loaded'); + } + + public function next(): void + { + throw new RuntimeException('flow_php extension is not loaded'); + } + + public function rewind(): void + { + throw new RuntimeException('flow_php extension is not loaded'); + } + + public function valid(): bool + { + throw new RuntimeException('flow_php extension is not loaded'); + } +} diff --git a/src/extension/flow-php-ext/php/Flow/Floe/Exception/ExtensionException.php b/src/extension/flow-php-ext/php/Flow/Floe/Exception/ExtensionException.php deleted file mode 100644 index 28972d4f34..0000000000 --- a/src/extension/flow-php-ext/php/Flow/Floe/Exception/ExtensionException.php +++ /dev/null @@ -1,15 +0,0 @@ - $batch - * @param string $schemaBody SCHEMA frame body (JSON list of normalized definitions) - * @param Schema $schema the same definitions as objects, so a refusal is raised through the PHP factory - * - * @throws ExtensionException - * - * @return list bare ROW frame bodies - */ - public function encode(array $batch, string $schemaBody, Schema $schema): array - { - throw new RuntimeException('flow_php extension is not loaded'); - } - - /** - * @param list $frameBodies - * @param string $schemaBody SCHEMA frame body (JSON list of normalized definitions) - * - * @throws ExtensionException - * - * @return list - */ - public function decode(array $frameBodies, string $schemaBody): array - { - throw new RuntimeException('flow_php extension is not loaded'); - } - - /** - * `RustRowHydratorNative::hydrate($this->decode($frameBodies, $schemaBody), $schema)` in one pass. - * - * @param list $frameBodies - * @param string $schemaBody SCHEMA frame body (JSON list of normalized definitions) - * - * @throws ExtensionException - */ - public function decodeRows(array $frameBodies, string $schemaBody, Schema $schema): Rows - { - throw new RuntimeException('flow_php extension is not loaded'); - } - - /** - * `Format::frame(Format::FRAME_ROW, ...)` over `$this->encode(RustRowHydratorNative::dehydrate($rows), ...)`. - * - * @param string $schemaBody SCHEMA frame body (JSON list of normalized definitions) - * - * @throws ExtensionException - */ - public function encodeFrames(Rows $rows, string $schemaBody, Schema $schema): string - { - throw new RuntimeException('flow_php extension is not loaded'); - } - - /** - * Present in every extension whose casts and decoders apply the datetime column zone. - */ - public function datetimeZones(): bool - { - throw new RuntimeException('flow_php extension is not loaded'); - } -} diff --git a/src/extension/flow-php-ext/rustfmt.toml b/src/extension/flow-php-ext/rustfmt.toml new file mode 100644 index 0000000000..7530651796 --- /dev/null +++ b/src/extension/flow-php-ext/rustfmt.toml @@ -0,0 +1 @@ +max_width = 120 diff --git a/src/extension/flow-php-ext/src/alloc.rs b/src/extension/flow-php-ext/src/alloc.rs new file mode 100644 index 0000000000..30e252c8f8 --- /dev/null +++ b/src/extension/flow-php-ext/src/alloc.rs @@ -0,0 +1,65 @@ +//! The global allocator: `System`, counting the bytes it holds for `RustBackend::allocatedBytes()`, plus the bytes +//! of the Arrow C Data batches imported from another extension's allocator while they live. + +use std::alloc::{GlobalAlloc, Layout, System}; +use std::sync::atomic::{AtomicI64, Ordering}; + +static ALLOCATED: AtomicI64 = AtomicI64::new(0); +static IMPORTED: AtomicI64 = AtomicI64::new(0); + +struct Counting; + +unsafe impl GlobalAlloc for Counting { + unsafe fn alloc(&self, layout: Layout) -> *mut u8 { + let ptr = System.alloc(layout); + + if !ptr.is_null() { + ALLOCATED.fetch_add(layout.size() as i64, Ordering::Relaxed); + } + + ptr + } + + unsafe fn alloc_zeroed(&self, layout: Layout) -> *mut u8 { + let ptr = System.alloc_zeroed(layout); + + if !ptr.is_null() { + ALLOCATED.fetch_add(layout.size() as i64, Ordering::Relaxed); + } + + ptr + } + + unsafe fn dealloc(&self, ptr: *mut u8, layout: Layout) { + System.dealloc(ptr, layout); + ALLOCATED.fetch_sub(layout.size() as i64, Ordering::Relaxed); + } + + unsafe fn realloc(&self, ptr: *mut u8, layout: Layout, new_size: usize) -> *mut u8 { + let new = System.realloc(ptr, layout, new_size); + + if !new.is_null() { + ALLOCATED.fetch_add(new_size as i64 - layout.size() as i64, Ordering::Relaxed); + } + + new + } +} + +#[global_allocator] +static GLOBAL: Counting = Counting; + +/// Bytes of imported batches: added on import, subtracted when the batch is released. +pub fn imported(delta: i64) { + IMPORTED.fetch_add(delta, Ordering::Relaxed); +} + +#[cfg(test)] +pub fn imported_bytes() -> i64 { + IMPORTED.load(Ordering::Relaxed) +} + +/// Process-wide: on ZTS every thread's allocations. +pub fn allocated_bytes() -> i64 { + ALLOCATED.load(Ordering::Relaxed) + IMPORTED.load(Ordering::Relaxed) +} diff --git a/src/extension/flow-php-ext/src/arrow_c.rs b/src/extension/flow-php-ext/src/arrow_c.rs new file mode 100644 index 0000000000..a18ecf5885 --- /dev/null +++ b/src/extension/flow-php-ext/src/arrow_c.rs @@ -0,0 +1,462 @@ +//! Batches across the Arrow C Data Interface, to and from another extension (arrow-ext): one batch = one struct-typed +//! `FFI_ArrowSchema` (one child per column) plus one struct-typed `FFI_ArrowArray`. Any extension's internal class +//! exposing the two addresses is accepted. An import moves the producer's array behind a release forwarder that counts +//! its buffers in `allocatedBytes()` for as long as they live. + +use std::ffi::{c_char, CStr}; +use std::rc::Rc; + +use arrow_array::ffi::{from_ffi, to_ffi, FFI_ArrowArray, FFI_ArrowSchema}; +use arrow_array::{Array, ArrayRef, StructArray}; +use arrow_schema::{ArrowError, DataType, Field, Fields}; +use ext_php_rs::exception::PhpException; +use ext_php_rs::flags::ClassFlags; +use ext_php_rs::prelude::*; +use ext_php_rs::types::{ZendObject, Zval}; +use ext_php_rs::zend::ClassEntry; +use flow_batch_frame::kind::data_type; + +use crate::alloc; +use crate::ctx::{call_handle_transparent, call_method, ce_method_ref}; +use crate::plan::TypePlan; +use crate::render::invalid_argument; + +/// Child field metadata: the arrow type of the file column the canonical child was read from. +const SOURCE_TYPE: &str = "parquet.source_type"; + +/// `ZEND_INTERNAL_CLASS` (zend_compile.h): a class an extension registered, which userland can neither declare nor +/// extend once final. +const ZEND_INTERNAL_CLASS: c_char = 1; + +/// One exported batch, until its consumer moves the array out. +#[php_class] +#[php(name = "Flow\\ETL\\Column\\RustColumnsBatch", flags = ClassFlags::Final)] +pub struct RustColumnsBatch { + schema: FFI_ArrowSchema, + array: FFI_ArrowArray, + rows: usize, +} + +#[php_impl] +impl RustColumnsBatch { + pub fn count(&self) -> i64 { + self.rows as i64 + } + + #[php(name = "arrowSchemaAddress")] + pub fn arrow_schema_address(&self) -> i64 { + std::ptr::from_ref(&self.schema) as i64 + } + + #[php(name = "arrowArrayAddress")] + pub fn arrow_array_address(&self) -> i64 { + std::ptr::from_ref(&self.array) as i64 + } +} + +/// What an import refuses; nothing was taken from the producer. +#[derive(Debug)] +enum Refusal { + NotStruct, + Count { + children: i64, + columns: usize, + }, + Name { + child: String, + column: usize, + }, + /// Column `column` arrives as `child`, which its plan does not store. + Type { + column: usize, + child: Field, + }, + Schema(ArrowError), + Released, + Length { + children: i64, + schema: i64, + }, + Invalid(ArrowError), +} + +/// The struct schema `schema`, its children named and typed as `columns` (name, stored type) are. +fn check(schema: &FFI_ArrowSchema, columns: &[(&[u8], DataType)]) -> Result<(), Refusal> { + if schema.format.is_null() || unsafe { CStr::from_ptr(schema.format) }.to_bytes() != b"+s" { + return Err(Refusal::NotStruct); + } + + if schema.n_children != columns.len() as i64 { + return Err(Refusal::Count { + children: schema.n_children, + columns: columns.len(), + }); + } + + for (column, (child, (name, stored))) in schema.children().zip(columns).enumerate() { + let child = Field::try_from(child).map_err(Refusal::Schema)?; + + if child.name().as_bytes() != *name { + return Err(Refusal::Name { + child: child.name().clone(), + column, + }); + } + + if child.data_type() != stored { + return Err(Refusal::Type { column, child }); + } + } + + Ok(()) +} + +/// An imported array: the producer's, moved here, and the bytes it counts while arrow-rs holds its buffers. +struct Counted { + inner: FFI_ArrowArray, + bytes: i64, +} + +/// The forwarder's release: arrow-rs calls it once, when the batch's last buffer drops; dropping `inner` runs the +/// producer's release. +unsafe extern "C" fn counted_release(array: *mut FFI_ArrowArray) { + let Some(array) = (unsafe { array.as_mut() }) else { + return; + }; + let Counted { inner, bytes } = *unsafe { Box::from_raw(array.private_data.cast::()) }; + + alloc::imported(-bytes); + drop(inner); + array.release = None; +} + +/// `source` moved out (its release left NULL) behind the counting forwarder, as the struct `schema` describes, after +/// [`check`]: the row count and one array per column. A refusal before the move takes nothing; after it the forwarder +/// owns the producer's array and releases it. +fn take( + schema: &FFI_ArrowSchema, + source: &mut FFI_ArrowArray, + columns: &[(&[u8], DataType)], +) -> Result<(usize, Vec), Refusal> { + check(schema, columns)?; + + if source.release.is_none() { + return Err(Refusal::Released); + } + + if source.n_children != schema.n_children || source.length < 0 { + return Err(Refusal::Length { + children: source.n_children, + schema: schema.n_children, + }); + } + + let moved = unsafe { std::ptr::read(source) }; + source.release = None; + + let counted = Box::into_raw(Box::new(Counted { inner: moved, bytes: 0 })); + let inner = unsafe { &(*counted).inner }; + let forward = FFI_ArrowArray { + length: inner.length, + null_count: inner.null_count, + offset: inner.offset, + n_buffers: inner.n_buffers, + n_children: inner.n_children, + buffers: inner.buffers, + children: inner.children, + dictionary: inner.dictionary, + release: Some(counted_release), + private_data: counted.cast(), + }; + + let data = unsafe { from_ffi(forward, schema) } + .and_then(|data| data.validate().map(|()| data)) + .map_err(Refusal::Invalid)?; + let bytes = data.get_buffer_memory_size() as i64; + + // the forwarder lives in arrow-rs's owner of `data`'s buffers, so `counted` does too + unsafe { (*counted).bytes = bytes }; + alloc::imported(bytes); + + let batch = StructArray::from(data); + + Ok((batch.len(), batch.into_parts().1)) +} + +fn refused(refusal: Refusal, columns: &[(Vec, Rc)]) -> PhpException { + let message = match refusal { + Refusal::NotStruct => "Arrow C Data batch must be a struct".to_string(), + Refusal::Count { children, columns } => { + format!("Arrow C Data batch of {children} children does not match the {columns} columns read") + } + Refusal::Name { child, column } => format!( + "Arrow C Data batch child \"{child}\" is not the column \"{}\"", + String::from_utf8_lossy(&columns[column].0) + ), + Refusal::Type { column, child } => { + let (name, plan) = &columns[column]; + let type_name = match call_method(&plan.type_zv, "toString", &mut []) { + Ok(type_name) => type_name, + Err(exception) => return exception, + }; + + format!( + "Parquet column \"{}\" ({}) is read as {}, its schema type {} stores {}", + String::from_utf8_lossy(name), + child.metadata().get(SOURCE_TYPE).map_or("", String::as_str), + child.data_type(), + String::from_utf8_lossy(type_name.zend_str().map_or(&b""[..], |name| name.as_bytes())), + data_type(&plan.kind), + ) + } + Refusal::Schema(error) => format!("Arrow C Data schema is refused: {error}"), + Refusal::Released => "Arrow C Data batch is already imported".to_string(), + Refusal::Length { children, schema } => { + format!("Arrow C Data batch of {children} children does not match its schema of {schema}") + } + Refusal::Invalid(error) => { + format!("Arrow C Data batch is not a valid struct array: {error}") + } + }; + + invalid_argument(message) +} + +/// The object of an exporter's internal class. +fn exporter(carrier: &Zval) -> PhpResult<(&ZendObject, &ClassEntry)> { + let object = carrier + .object() + .ok_or_else(|| invalid_argument("Arrow C Data carrier must be an object".to_string()))?; + let ce = unsafe { object.ce.as_ref() } + .ok_or_else(|| invalid_argument("Arrow C Data carrier has no class".to_string()))?; + + if ce.type_ != ZEND_INTERNAL_CLASS { + return Err(invalid_argument(format!( + "Arrow C Data carrier must be an extension's class, got {}", + ce.name().unwrap_or_default() + ))); + } + + Ok((object, ce)) +} + +/// `$object->$method()` of an address method: a pointer the exporter owns. +fn address(object: &ZendObject, ce: &ClassEntry, method: &str) -> PhpResult { + let address = call_handle_transparent(ce_method_ref(ce, method)?, Some(object), &mut [])?; + + address + .long() + .and_then(|address| usize::try_from(address).ok()) + .filter(|address| *address != 0) + .ok_or_else(|| invalid_argument(format!("Arrow C Data {method}() must return a non-null address"))) +} + +fn stored(columns: &[(Vec, Rc)]) -> Vec<(&[u8], DataType)> { + columns + .iter() + .map(|(name, plan)| (name.as_slice(), data_type(&plan.kind))) + .collect() +} + +/// Refuses a schema carrier (`arrowSchemaAddress()`) whose struct children are not `columns`, by name and stored type. +pub fn check_schema(schema: &Zval, columns: &[(Vec, Rc)]) -> PhpResult<()> { + let (object, ce) = exporter(schema)?; + let schema = unsafe { &*(address(object, ce, "arrowSchemaAddress")? as *const FFI_ArrowSchema) }; + + check(schema, &stored(columns)).map_err(|refusal| refused(refusal, columns)) +} + +/// The batch `batch` carries (`arrowSchemaAddress()` read by reference, `arrowArrayAddress()` moved): the row count +/// and one array per column of `columns`. +pub fn import(batch: &Zval, columns: &[(Vec, Rc)]) -> PhpResult<(usize, Vec)> { + let (object, ce) = exporter(batch)?; + let schema = unsafe { &*(address(object, ce, "arrowSchemaAddress")? as *const FFI_ArrowSchema) }; + let source = unsafe { &mut *(address(object, ce, "arrowArrayAddress")? as *mut FFI_ArrowArray) }; + + take(schema, source, &stored(columns)).map_err(|refusal| refused(refusal, columns)) +} + +/// `columns` as one struct batch of `rows` rows, every child nullable. +pub fn export(rows: usize, columns: Vec<(String, ArrayRef)>) -> PhpResult { + let fields = columns + .iter() + .map(|(name, array)| Field::new(name, array.data_type().clone(), true)) + .collect::(); + let arrays = columns.into_iter().map(|(_, array)| array).collect(); + let (array, schema) = StructArray::try_new_with_length(fields, arrays, None, rows) + .and_then(|batch| to_ffi(&batch.to_data())) + .map_err(|error| invalid_argument(format!("flow_php failed to export an Arrow C Data batch: {error}")))?; + + Ok(RustColumnsBatch { schema, array, rows }) +} + +#[cfg(test)] +mod tests { + use std::ffi::c_void; + use std::sync::atomic::{AtomicUsize, Ordering}; + use std::sync::{Arc, Mutex}; + + use arrow_array::ffi::{to_ffi, FFI_ArrowArray, FFI_ArrowSchema}; + use arrow_array::types::Int64Type; + use arrow_array::{Array, ArrayRef, Int64Array, ListArray, StructArray}; + use arrow_schema::{DataType, Field, Fields}; + + use super::{take, Refusal}; + use crate::alloc; + + /// `IMPORTED` is process-wide: the tests that import run one at a time. + static SERIAL: Mutex<()> = Mutex::new(()); + + static RELEASES: AtomicUsize = AtomicUsize::new(0); + + /// The producer: arrow-rs's export, wrapped so its release counts how often it ran. + unsafe extern "C" fn counting_release(array: *mut FFI_ArrowArray) { + let array = unsafe { &mut *array }; + drop(unsafe { Box::from_raw(array.private_data.cast::()) }); + array.release = None; + RELEASES.fetch_add(1, Ordering::SeqCst); + } + + fn columns() -> Vec<(String, ArrayRef)> { + vec![ + ( + "id".to_string(), + Arc::new(Int64Array::from_iter((0..1000).map(|i| (i % 3 != 0).then_some(i)))) as ArrayRef, + ), + ( + "list".to_string(), + Arc::new(ListArray::from_iter_primitive::( + (0..1000).map(|i| (i % 5 != 0).then(|| vec![Some(i), None])), + )) as ArrayRef, + ), + ] + } + + fn produced() -> (FFI_ArrowArray, FFI_ArrowSchema) { + let fields = columns() + .iter() + .map(|(name, array)| Field::new(name, array.data_type().clone(), true)) + .collect::(); + let batch = StructArray::new(fields, columns().into_iter().map(|(_, array)| array).collect(), None); + let (inner, schema) = to_ffi(&batch.to_data()).unwrap(); + let inner = Box::into_raw(Box::new(inner)); + let exported = unsafe { &*inner }; + let array = FFI_ArrowArray { + length: exported.length, + null_count: exported.null_count, + offset: exported.offset, + n_buffers: exported.n_buffers, + n_children: exported.n_children, + buffers: exported.buffers, + children: exported.children, + dictionary: exported.dictionary, + release: Some(counting_release), + private_data: inner.cast::(), + }; + + (array, schema) + } + + fn expected() -> Vec<(&'static [u8], DataType)> { + vec![(b"id", DataType::Int64), (b"list", columns()[1].1.data_type().clone())] + } + + fn imported() -> i64 { + alloc::imported_bytes() + } + + #[test] + fn an_imported_batch_counts_its_buffers_until_the_last_one_drops() { + let _serial = SERIAL.lock().unwrap(); + let (mut array, schema) = produced(); + let releases = RELEASES.load(Ordering::SeqCst); + + let (rows, arrays) = take(&schema, &mut array, &expected()).unwrap(); + + assert!(array.release.is_none()); + assert_eq!(rows, 1000); + assert_eq!(arrays.len(), 2); + assert_eq!(arrays[0].as_ref(), columns()[0].1.as_ref()); + assert_eq!(arrays[1].as_ref(), columns()[1].1.as_ref()); + assert_eq!( + imported(), + arrays + .iter() + .map(|array| array.to_data().get_buffer_memory_size() as i64) + .sum::() + ); + + let first = arrays.into_iter().next().unwrap(); + assert!(imported() > 0); + assert_eq!(RELEASES.load(Ordering::SeqCst), releases); + + drop(first); + assert_eq!(imported(), 0); + assert_eq!(RELEASES.load(Ordering::SeqCst), releases + 1); + + drop(array); + drop(schema); + assert_eq!(RELEASES.load(Ordering::SeqCst), releases + 1); + } + + fn refused(columns: &[(&[u8], DataType)], refusal: impl Fn(&Refusal) -> bool) { + let _serial = SERIAL.lock().unwrap(); + let (mut array, schema) = produced(); + let releases = RELEASES.load(Ordering::SeqCst); + + assert!(refusal(&take(&schema, &mut array, columns).unwrap_err())); + assert_eq!(imported(), 0); + assert!(array.release.is_some()); + assert_eq!(RELEASES.load(Ordering::SeqCst), releases); + + drop(array); + assert_eq!(RELEASES.load(Ordering::SeqCst), releases + 1); + } + + #[test] + fn a_child_of_another_type_is_refused_before_the_move() { + refused( + &[(b"id", DataType::Float64), (b"list", expected()[1].1.clone())], + |refusal| matches!(refusal, Refusal::Type { column: 0, .. }), + ); + } + + #[test] + fn a_child_of_another_name_is_refused_before_the_move() { + refused( + &[(b"id", DataType::Int64), (b"other", expected()[1].1.clone())], + |refusal| matches!(refusal, Refusal::Name { column: 1, .. }), + ); + } + + #[test] + fn another_child_count_is_refused_before_the_move() { + refused(&[(b"id", DataType::Int64)], |refusal| { + matches!( + refusal, + Refusal::Count { + children: 2, + columns: 1 + } + ) + }); + } + + #[test] + fn an_already_imported_batch_is_refused() { + let _serial = SERIAL.lock().unwrap(); + let (mut array, schema) = produced(); + + let (_, arrays) = take(&schema, &mut array, &expected()).unwrap(); + let counted = imported(); + + assert!(matches!( + take(&schema, &mut array, &expected()).unwrap_err(), + Refusal::Released + )); + assert_eq!(imported(), counted); + + drop(arrays); + assert_eq!(imported(), 0); + } +} diff --git a/src/extension/flow-php-ext/src/backend.rs b/src/extension/flow-php-ext/src/backend.rs new file mode 100644 index 0000000000..d2bf5b04fb --- /dev/null +++ b/src/extension/flow-php-ext/src/backend.rs @@ -0,0 +1,218 @@ +//! `Flow\ETL\Column\RustBackend`: every column is a `RustColumn`. + +// a parameter's Rust name is its PHP name, and `Backend::decode()` names it `$nullCount` +#![allow(non_snake_case)] + +use std::rc::Rc; + +use arrow_array::ArrayRef; + +use ext_php_rs::convert::IntoZval; +use ext_php_rs::error::Result as ZvalResult; +use ext_php_rs::flags::{ClassFlags, DataType}; +use ext_php_rs::prelude::*; +use ext_php_rs::types::{ZendHashTable, Zval}; +use flow_batch_frame::column::decode; +use flow_batch_frame::layout::buffer_count; + +use crate::batch_columns::HeldBatch; +use crate::builder::{throw, RustColumnBuilder}; +use crate::column::RustColumn; +use crate::ctx::{ + call_method, call_static, column_key, expect_object, find_class, ht_for_each, ht_get, ht_insert, zval_long, +}; +use crate::exception::ext_exception; +use crate::interfaces::backend_ce; +use crate::plan::{physical_for_definition, type_plan, TypePlan}; +use crate::render::{decode_exception, invalid_argument}; + +const COLUMN_MISMATCH: &str = "Flow\\ETL\\Exception\\ColumnMismatchException"; +const MIXED_TYPE: &str = "Flow\\Types\\Type\\Native\\MixedType"; + +/// A return typed `Flow\ETL\Column\Column`: `adopt()` hands back its argument itself. +pub struct ColumnValue(pub Zval); + +impl IntoZval for ColumnValue { + const TYPE: DataType = DataType::object("Flow\\ETL\\Column\\Column"); + const NULLABLE: bool = false; + + fn set_zval(self, zv: &mut Zval, _persistent: bool) -> ZvalResult<()> { + *zv = self.0; + + Ok(()) + } +} + +fn definition_plan(definition: &Zval) -> PhpResult> { + type_plan(&call_method(definition, "type", &mut [])?) +} + +fn rows(count: i64, name: &str) -> PhpResult { + u32::try_from(count).map_err(|_| invalid_argument(format!("flow_php {name} must be between 0 and {}", u32::MAX))) +} + +#[php_class] +#[php( + name = "Flow\\ETL\\Column\\RustBackend", + flags = ClassFlags::Final, + implements(ce = backend_ce, stub = "Flow\\ETL\\Column\\Backend") +)] +pub struct RustBackend; + +#[php_impl] +impl RustBackend { + pub fn __construct() -> Self { + RustBackend + } + + pub fn builder(&self, definition: &Zval) -> PhpResult { + physical_for_definition(definition)?; + + RustColumnBuilder::for_definition(definition, definition_plan(definition)?) + } + + pub fn constant(&self, definition: &Zval, value: &Zval, count: i64) -> PhpResult { + let mut builder = self.builder(definition)?; + let pending = builder.pending(value)?; + + builder.append_repeated(pending, rows(count, "constant count")?)?; + builder.finish_column() + } + + pub fn decode( + &self, + definition: &Zval, + buffers: &ZendHashTable, + count: i64, + nullCount: i64, + ) -> PhpResult { + let plan = definition_plan(definition)?; + let bytes = buffers + .values() + .map(|buffer| { + buffer + .zend_str() + .map(|buffer| buffer.as_bytes()) + .ok_or_else(|| invalid_argument("flow_php column buffers must be strings".to_string())) + }) + .collect::, _>>()?; + + let needed = buffer_count(&plan.kind).min(bytes.len()); + let data = decode( + &plan.kind, + &bytes[..needed], + rows(count, "row count")?, + rows(nullCount, "null count")?, + ) + .map_err(|e| decode_exception(e, &plan.type_zv, &plan.kind))?; + + if bytes.len() > needed { + let entry = call_method(definition, "entry", &mut [])?; + let name = call_method(&entry, "name", &mut [])?; + + return Err(invalid_argument(format!( + "Column \"{}\": {} buffers left after decoding", + String::from_utf8_lossy(name.zend_str().map_or(&b""[..], |name| name.as_bytes())), + bytes.len() - needed + ))); + } + + Ok(RustColumn::new(arrow_array::make_array(data), plan)) + } + + pub fn adopt(&self, definition: &Zval, column: &Zval) -> PhpResult { + if column.extract::<&RustColumn>().is_some() { + return Ok(ColumnValue(column.shallow_clone())); + } + + if expect_object(&call_method(column, "type", &mut [])?, "a Type")?.instance_of(find_class(MIXED_TYPE)?) { + return Err(throw(call_static( + COLUMN_MISMATCH, + "untypedColumn", + &mut [definition.shallow_clone()], + )?)); + } + + let mut builder = self.builder(definition)?; + let count = call_method(column, "count", &mut [])? + .long() + .ok_or_else(|| ext_exception("flow_php expected Column::count() to return an int"))?; + + if count > 0 { + let mut indices = ZendHashTable::with_capacity(count as u32); + + for i in 0..count { + indices + .push(zval_long(i)) + .map_err(|e| ext_exception(format!("flow_php failed to collect row indices: {e:?}")))?; + } + + builder.append_take(column, &indices)?; + } + + let mut adopted = builder.finish_column()?.into_zval(false)?; + + Ok(ColumnValue(std::mem::take(&mut adopted))) + } + + #[php(name = "allocatedBytes")] + pub fn allocated_bytes(&self) -> i64 { + crate::alloc::allocated_bytes() + } +} + +/// `$held` as `Rows` with every column adopted by `$backend` - what a Rust open source yields: the configured backend +/// keeps its own columns and copies the native ones. +pub fn adopt_batch(schema: &Zval, backend: &Zval, held: HeldBatch) -> PhpResult { + let definitions = call_method(schema, "definitions", &mut [])?; + let definitions = definitions + .array() + .ok_or_else(|| ext_exception("flow_php expected Schema::definitions() to return an array"))?; + let mut columns = ZendHashTable::with_capacity(definitions.len() as u32); + + ht_for_each(definitions, |name, index, definition| { + let key = column_key(name, index); + let column = ht_get(&held.columns, &key).ok_or_else(|| { + ext_exception(format!( + "flow_php expected the batch to hold the column \"{}\"", + String::from_utf8_lossy(&key) + )) + })?; + ht_insert( + &mut columns, + &key, + call_method( + backend, + "adopt", + &mut [definition.shallow_clone(), column.shallow_clone()], + )?, + ); + + Ok(()) + })?; + + let mut columns_zv = Zval::new(); + columns_zv.set_hashtable(columns); + + call_static( + "Flow\\ETL\\Rows", + "fromColumns", + &mut [schema.shallow_clone(), columns_zv, zval_long(held.count as i64)], + ) +} + +/// `$column` as a `RustColumn`'s plan and data: itself, or `RustBackend`'s adoption of it under `$definition`. +pub fn native_column(definition: &Zval, column: &Zval) -> PhpResult<(Rc, ArrayRef)> { + let adopted; + let native = match column.extract::<&RustColumn>() { + Some(native) => native, + None => { + adopted = RustBackend.adopt(definition, column)?.0; + adopted + .extract::<&RustColumn>() + .ok_or_else(|| ext_exception("flow_php expected RustBackend::adopt() to build a RustColumn"))? + } + }; + + Ok((Rc::clone(native.plan()), ArrayRef::clone(native.data()))) +} diff --git a/src/extension/flow-php-ext/src/batch_columns.rs b/src/extension/flow-php-ext/src/batch_columns.rs new file mode 100644 index 0000000000..c0af6311eb --- /dev/null +++ b/src/extension/flow-php-ext/src/batch_columns.rs @@ -0,0 +1,276 @@ +//! A batch of native columns, one builder per schema definition, with `RowsBuilder::appendRows()`'s refusals: each +//! column keeps its first, the lowest refused row wins (then the earliest definition), and a NOT NULL column a row +//! lacks is refused only when nothing else was. The readers differ only in where a cell comes from. + +use std::rc::Rc; + +use ext_php_rs::boxed::ZBox; +use ext_php_rs::exception::PhpException; +use ext_php_rs::types::{ZendHashTable, Zval}; +use flow_batch_frame::kind::Kind; + +use crate::builder::{definition_nullable, schema_mismatch}; +use crate::cast::{ + bool_from_str, float_from_str, integer_from_str, json_gate, parse_iso_date, parse_iso_datetime, CastKind, +}; +use crate::column::RustColumn; +use crate::ctx::{self, call_method, call_static, ht_for_each, ht_insert, ht_insert_long, zval_long}; +use crate::date_check::iso_instant_micros; +use crate::exception::ext_exception; +use crate::json_check::json_valid; +use crate::kind_builder::KindBuilder; +use crate::physical::overflow; +use crate::plan::{type_plan, TypePlan}; +use crate::render::invalid_argument; +use crate::uuid_check::is_uuid; +use crate::values::{datetime_days, datetime_micros, uuid_bytes}; + +/// A `Schema::definitions()` key: an int for a numeric name, as PHP keys it. +pub enum Key { + Index(i64), + Name(Vec), +} + +impl Key { + pub fn name(&self) -> Vec { + match self { + Key::Index(index) => index.to_string().into_bytes(), + Key::Name(name) => name.clone(), + } + } +} + +pub struct BatchColumn { + pub key: Key, + pub definition: Zval, + pub nullable: bool, + pub plan: Rc, + pub values: KindBuilder, + /// The first refused row and its `ColumnMismatchException`. + pub refusal: Option<(usize, Zval)>, + /// The first row lacking this NOT NULL column. + pub absent: Option, +} + +/// The batch being built, and the schema and batch size it is built for. +pub struct BatchColumns { + schema: Zval, + schema_json: Vec, + pub batch_size: usize, + pub columns: Vec, + pub rows: usize, +} + +/// A PHP `$batchSize` as a row count: refused unless greater than 0. +pub fn batch_size_of(batch_size: i64, what: &str) -> Result { + usize::try_from(batch_size) + .ok() + .filter(|size| *size > 0) + .ok_or_else(|| ext_exception(format!("flow_php {what} batch size must be greater than 0"))) +} + +fn schema_json(schema: &Zval) -> Result, PhpException> { + let normalized = call_method(schema, "normalize", &mut [])?; + let json = ctx::call_handle(ctx::json_encode()?, None, &mut [normalized], "encode a schema as JSON")?; + + Ok(json + .zend_str() + .ok_or_else(|| ext_exception("flow_php expected schema JSON to be a string"))? + .as_bytes() + .to_vec()) +} + +impl BatchColumns { + fn new(schema: &Zval, schema_json: Vec, batch_size: usize) -> Result { + let definitions = call_method(schema, "definitions", &mut [])?; + let definitions = definitions + .array() + .ok_or_else(|| ext_exception("flow_php expected Schema::definitions() to return an array"))?; + let mut columns = Vec::with_capacity(definitions.len()); + + ht_for_each(definitions, |name, index, definition| { + let plan = type_plan(&call_method(definition, "type", &mut [])?)?; + + columns.push(BatchColumn { + key: match name { + Some(name) => Key::Name(name.as_bytes().to_vec()), + None => Key::Index(index as i64), + }, + definition: definition.shallow_clone(), + nullable: definition_nullable(definition)?, + values: KindBuilder::new(&plan.kind), + plan, + refusal: None, + absent: None, + }); + + Ok(()) + })?; + + Ok(Self { + schema: schema.shallow_clone(), + schema_json, + batch_size, + columns, + rows: 0, + }) + } + + /// The state for `schema` and `batch_size`: the same object or the same `normalize()` JSON keeps it (`false`), any + /// other schema rebuilds it (`true`); changing the schema or the batch size of a batch with rows pending throws. + pub fn prepare<'a>( + pending: &'a mut Option, + schema: &Zval, + batch_size: usize, + what: &str, + ) -> Result<(&'a mut Self, bool), PhpException> { + let same_object = pending.as_ref().is_some_and(|state| { + state.schema.object().map(|object| object.handle) == schema.object().map(|object| object.handle) + }); + let mut rebuilt = false; + + if !same_object { + let json = schema_json(schema)?; + + match pending { + Some(state) if state.schema_json == json => state.schema = schema.shallow_clone(), + Some(state) if state.rows > 0 => { + return Err(ext_exception(format!( + "flow_php cannot change the schema of a pending {what} batch" + ))); + } + _ => { + *pending = Some(Self::new(schema, json, batch_size)?); + rebuilt = true; + } + } + } + + let state = pending.as_mut().expect("built above"); + + if state.batch_size != batch_size { + if state.rows > 0 { + return Err(ext_exception(format!( + "flow_php cannot change the batch size of a pending {what} batch" + ))); + } + + state.batch_size = batch_size; + } + + Ok((state, rebuilt)) + } + + pub fn reset(&mut self) { + self.rows = 0; + + for column in &mut self.columns { + column.values = KindBuilder::new(&column.plan.kind); + column.refusal = None; + column.absent = None; + } + } + + /// The batch's columns, or its refusal: the lowest refused row (ties: the earliest definition), else the lowest + /// absent row (ties: the earliest definition) as `missingColumn`. The state is reset either way. + pub fn finish(&mut self) -> Result { + let batch = self.build(); + self.reset(); + + batch + } + + fn build(&self) -> Result { + // min_by_key() keeps the first of equal rows: the earliest definition + if let Some((row, cause)) = self + .columns + .iter() + .filter_map(|column| column.refusal.as_ref()) + .min_by_key(|(row, _)| *row) + { + return Err(schema_mismatch(zval_long(*row as i64), cause.shallow_clone())); + } + + if let Some((row, column)) = self + .columns + .iter() + .filter_map(|column| Some((column.absent?, column))) + .min_by_key(|(row, _)| *row) + { + let cause = call_static( + "Flow\\ETL\\Exception\\ColumnMismatchException", + "missingColumn", + &mut [column.definition.shallow_clone()], + )?; + + return Err(schema_mismatch(zval_long(row as i64), cause)); + } + + let mut columns = ZendHashTable::with_capacity(self.columns.len() as u32); + + for column in &self.columns { + let data = column + .values + .finish(&column.plan.kind) + .map_err(|e| invalid_argument(format!("flow_php built an invalid column: {e}")))?; + let mut native = Zval::new(); + ext_php_rs::convert::IntoZval::set_zval( + RustColumn::new(arrow_array::make_array(data), Rc::clone(&column.plan)), + &mut native, + false, + )?; + + match &column.key { + Key::Index(index) => ht_insert_long(&mut columns, *index, native), + Key::Name(name) => ht_insert(&mut columns, name, native), + } + } + + Ok(HeldBatch { + columns, + count: self.rows, + }) + } +} + +/// A batch's native columns keyed as its schema's definitions, before the configured backend adopts them. +pub struct HeldBatch { + pub columns: ZBox, + pub count: usize, +} + +/// A text value `cast.rs` casts natively, appended in its physical form; `false` hands it to the PHP lane. +pub fn native_text(values: &mut KindBuilder, kind: &Kind, cast: &CastKind, bytes: &[u8]) -> Result { + match (kind, cast) { + (Kind::Int64, CastKind::Integer) => Ok(integer_from_str(bytes) + .map(|value| values.append_fixed(&value.to_le_bytes())) + .is_some()), + (Kind::Float64, CastKind::Float) => Ok(float_from_str(bytes) + .map(|value| values.append_fixed(&value.to_le_bytes())) + .is_some()), + (Kind::Boolean, CastKind::Boolean) => Ok(bool_from_str(bytes).map(|value| values.append_bool(value)).is_some()), + (Kind::Bytes, CastKind::String) => values.append_bytes(bytes).map(|()| true).map_err(overflow), + (Kind::Bytes, CastKind::NonEmptyString) if !bytes.is_empty() => { + values.append_bytes(bytes).map(|()| true).map_err(overflow) + } + (Kind::Bytes, CastKind::Json) if json_gate(bytes) && json_valid(bytes) => { + values.append_bytes(bytes).map(|()| true).map_err(overflow) + } + (Kind::Uuid, CastKind::Uuid) if is_uuid(bytes) => { + Ok(uuid_bytes(bytes).map(|uuid| values.append_fixed(&uuid)).is_some()) + } + (Kind::Timestamp, CastKind::DateTime(zone)) => Ok(match iso_instant_micros(bytes) { + Some(micros) => Some(micros), + None => { + parse_iso_datetime(bytes, zone)?.and_then(|(_, instant)| instant.object().and_then(datetime_micros)) + } + } + .map(|micros| values.append_fixed(µs.to_le_bytes())) + .is_some()), + (Kind::Int32, CastKind::Date) => Ok(parse_iso_date(bytes)? + .and_then(|instant| instant.object().and_then(datetime_days)) + .map(|days| values.append_fixed(&(days as i32).to_le_bytes())) + .is_some()), + _ => Ok(false), + } +} diff --git a/src/extension/flow-php-ext/src/builder.rs b/src/extension/flow-php-ext/src/builder.rs new file mode 100644 index 0000000000..277b021658 --- /dev/null +++ b/src/extension/flow-php-ext/src/builder.rs @@ -0,0 +1,647 @@ +//! `Flow\ETL\Column\RustColumnBuilder`: `CastingColumnBuilder` over arrow storage. A value takes the native lane +//! where `cast.rs` casts it natively; every other value, and every value the native lane refuses, runs PHP's +//! `$type->cast()` then `$physical->toPhysical()`, so results and refusals are PHP's own. + +// `appendPhysicals()`'s `$nullCount`: ext-php-rs names a PHP parameter after its Rust identifier and binds it in a +// generated handler, which an `allow` on the method, the impl block or the parameter does not reach +#![allow(non_snake_case)] + +use std::rc::Rc; + +use arrow_array::Array; +use ext_php_rs::boxed::ZBox; +use ext_php_rs::exception::PhpException; +use ext_php_rs::flags::ClassFlags; +use ext_php_rs::prelude::*; +use ext_php_rs::types::{ZendHashTable, ZendObject, Zval}; +use flow_batch_frame::kind::Kind; + +use crate::cast::{cast_value, CastKind, MapKeyKind}; +use crate::column::RustColumn; +use crate::ctx::{ + self, call_handle_catching, call_method, call_static, construct, expect_object, find_class, ht_for_each, ht_get, + null_zval, read_slot, transparent_exception, zval_long, +}; +use crate::date_check::iso_instant_micros; +use crate::exception::ext_exception; +use crate::interfaces::column_builder_ce; +use crate::kind_builder::KindBuilder; +use crate::physical::{append_physical, overflow}; +use crate::plan::TypePlan; +use crate::render::invalid_argument; +use crate::values::{datetime_days, datetime_micros, uuid_bytes}; + +const COLUMN_MISMATCH: &str = "Flow\\ETL\\Exception\\ColumnMismatchException"; +const SCHEMA_MISMATCH: &str = "Flow\\ETL\\Exception\\SchemaMismatchException"; +const TYPES_EXCEPTION: &str = "Flow\\Types\\Exception\\Exception"; +const NULL_DEFINITION: &str = "Flow\\ETL\\Schema\\Definition\\NullDefinition"; + +/// A value in its physical form, produced by the native lane without a PHP call. +pub enum Native { + Fixed([u8; 8]), + Days(i32), + Bool(bool), + Uuid([u8; 16]), + Bytes(Zval), +} + +/// One value ready to append, for `RustBackend::constant()`. +pub enum Pending { + Null, + Native(Native), + Physical(Zval), +} + +/// Why the PHP lane refused a value: its cast, or its `toPhysical()`. +pub enum Refusal { + Cast(ZBox), + Physical(ZBox), +} + +/// What `append_one()` did with a value. +pub enum Append { + Done, + /// `cause` is the `ColumnMismatchException` a positioned append wraps; `raised` the `Flow\Types` exception an + /// unpositioned append rethrows (`None` for a NOT NULL null). + Refused { + cause: Zval, + raised: Option>, + }, +} + +fn string_slot(object: &ZendObject, slot: u32) -> Option { + let value = read_slot(object, slot); + + value.is_string().then(|| value.shallow_clone()) +} + +/// The physical form of `value` for `plan`, when `cast.rs` casts it natively; `None` hands it to the PHP lane. +pub fn native_lane(plan: &TypePlan, value: &Zval) -> Result, PhpException> { + native_leaf(&plan.kind, &plan.cast, value) +} + +pub(crate) fn native_leaf(kind: &Kind, cast_kind: &CastKind, value: &Zval) -> Result, PhpException> { + if let (Kind::Timestamp, CastKind::DateTime(_)) = (kind, cast_kind) { + if let Some(micros) = value + .zend_str() + .and_then(|string| iso_instant_micros(string.as_bytes())) + { + return Ok(Some(Native::Fixed(micros.to_le_bytes()))); + } + } + + if nested(kind) { + return Ok(None); + } + + let Some(cast) = cast_value(cast_kind, value)? else { + return Ok(None); + }; + + Ok(match (kind, cast_kind) { + (Kind::Int64, CastKind::Integer | CastKind::PositiveInteger) => { + cast.long().map(|long| Native::Fixed(long.to_le_bytes())) + } + (Kind::Float64, CastKind::Float) => cast.double().map(|double| Native::Fixed(double.to_le_bytes())), + (Kind::Boolean, CastKind::Boolean) => cast.bool().map(Native::Bool), + (Kind::Bytes, CastKind::String | CastKind::NonEmptyString) => cast.is_string().then_some(Native::Bytes(cast)), + (Kind::Timestamp, CastKind::DateTime(_)) => cast + .object() + .and_then(datetime_micros) + .map(|micros| Native::Fixed(micros.to_le_bytes())), + (Kind::Int32, CastKind::Date) => cast + .object() + .and_then(datetime_days) + .map(|days| Native::Days(days as i32)), + (Kind::Uuid, CastKind::Uuid) => { + let (_, value_slot) = ctx::uuid()?; + + cast.object() + .and_then(|uuid| string_slot(uuid, value_slot)) + .and_then(|text| uuid_bytes(text.zend_str()?.as_bytes())) + .map(Native::Uuid) + } + (Kind::Bytes, CastKind::Json) => { + let (_, value_slot, _) = ctx::json()?; + + cast.object() + .and_then(|json| string_slot(json, value_slot)) + .map(Native::Bytes) + } + _ => None, + }) +} + +fn nested(kind: &Kind) -> bool { + matches!(kind, Kind::List(_) | Kind::Map(..) | Kind::Struct(_)) +} + +/// Casts `value` straight into `builder`, element by element. `Ok(false)`: Rust does not cast this value, and the row +/// is left half-appended for the caller to truncate before the PHP lane casts the whole value. +fn append_cast( + builder: &mut KindBuilder, + kind: &Kind, + cast_kind: &CastKind, + value: &Zval, +) -> Result { + let value = value.dereference(); + + match (kind, cast_kind) { + (_, CastKind::Optional(inner)) => { + if value.is_null() { + builder.append_null(); + + return Ok(true); + } + + append_cast(builder, kind, inner, value) + } + (Kind::List(element), CastKind::List(inner)) => { + let Some(values) = value.array() else { + return Ok(false); + }; + let mut expected = 0; + let mut cast = true; + + ht_for_each(values, |string_key, index, item| { + cast = cast + && string_key.is_none() + && index == expected + && append_cast(builder.list_element(), &element.kind, inner, item)?; + expected += 1; + + Ok(()) + })?; + + if cast { + builder.end_entries().map_err(overflow)?; + } + + Ok(cast) + } + (Kind::Map(key, item_kind), CastKind::Map(key_cast, inner)) => { + let Some(values) = value.array() else { + return Ok(false); + }; + let mut cast = true; + + ht_for_each(values, |string_key, index, item| { + if !cast { + return Ok(()); + } + + let (keys, items) = builder.map_entries(); + + cast = match (key_cast, string_key, &key.kind) { + (MapKeyKind::Int, None, Kind::Int64) => { + keys.append_fixed(&(index as i64).to_le_bytes()); + + true + } + (MapKeyKind::Str, Some(name), Kind::Bytes) => { + keys.append_bytes(name.as_bytes()).map_err(overflow)?; + + true + } + _ => false, + } && append_cast(items, &item_kind.kind, inner, item)?; + + Ok(()) + })?; + + if cast { + builder.end_entries().map_err(overflow)?; + } + + Ok(cast) + } + (Kind::Struct(fields), CastKind::Structure(elements)) => { + let Some(values) = value.array() else { + return Ok(false); + }; + + for ((child, field), element) in builder.struct_children().iter_mut().zip(fields).zip(elements) { + match ht_get(values, field.name.as_bytes()) { + None if element.required => return Ok(false), + None => child.append_null(), + Some(item) if item.dereference().is_null() && !matches!(element.kind, CastKind::Optional(_)) => { + return Ok(false); + } + Some(item) => { + if !append_cast(child, &field.kind, &element.kind, item)? { + return Ok(false); + } + } + } + } + + builder.end_struct(); + + Ok(true) + } + _ => match native_leaf(kind, cast_kind, value)? { + Some(native) => { + append_native(builder, native)?; + + Ok(true) + } + None => Ok(false), + }, + } +} + +/// One value, as `RustColumnBuilder::appendMany()` appends it: a null is refused under NOT NULL; a nested value +/// takes `append_cast()`, a leaf the native lane, and what neither casts the PHP lane. Exceptions other than +/// `Flow\Types` ones escape as thrown. +pub fn append_one( + values: &mut KindBuilder, + plan: &TypePlan, + definition: &Zval, + nullable: bool, + value: &Zval, +) -> Result { + let value = value.dereference(); + + if value.is_null() { + if !nullable { + return Ok(Append::Refused { + cause: value_does_not_match(definition, null_zval(), None)?, + raised: None, + }); + } + + values.append_null(); + + return Ok(Append::Done); + } + + if nested(&plan.kind) { + let row = values.len(); + + if append_cast(values, &plan.kind, &plan.cast, value)? { + return Ok(Append::Done); + } + + values.truncate(row); + } else if let Some(native) = native_lane(plan, value)? { + append_native(values, native)?; + + return Ok(Append::Done); + } + + match php_lane(plan, value) { + Ok(physical) => { + append_physical(values, &plan.kind, &physical)?; + + Ok(Append::Done) + } + Err(Refusal::Cast(mut exception) | Refusal::Physical(mut exception)) if !is_types_exception(&exception)? => { + Err(transparent_exception(&mut exception)) + } + // the cast's own reason is dropped, as CastingColumnBuilder::appendMany drops it + Err(Refusal::Cast(exception)) => Ok(Append::Refused { + cause: value_does_not_match(definition, value.shallow_clone(), None)?, + raised: Some(exception), + }), + Err(Refusal::Physical(mut exception)) => { + let mut reason = Zval::new(); + reason.set_object(&mut exception); + + // the value cast, so only the physical form's reason (a range) tells the user why it was refused + Ok(Append::Refused { + cause: value_does_not_match(definition, value.shallow_clone(), Some(reason))?, + raised: Some(exception), + }) + } + } +} + +/// `$type->cast($value)`, then `$physical->toPhysical($cast)` unless the physical is the identity. +pub fn php_lane(plan: &TypePlan, value: &Zval) -> Result { + let type_obj = plan.type_zv.object().expect("a plan's type is an object"); + let cast = + call_handle_catching(plan.cast_fn, Some(type_obj), &mut [value.shallow_clone()]).map_err(Refusal::Cast)?; + + if plan.identity { + return Ok(cast); + } + + let physical = plan.physical.object().expect("a plan's physical is an object"); + + call_handle_catching(plan.to_physical, Some(physical), &mut [cast]).map_err(Refusal::Physical) +} + +pub(crate) fn append_native(builder: &mut KindBuilder, native: Native) -> Result<(), PhpException> { + match native { + Native::Fixed(bytes) => builder.append_fixed(&bytes), + Native::Days(days) => builder.append_fixed(&days.to_le_bytes()), + Native::Bool(value) => builder.append_bool(value), + Native::Uuid(bytes) => builder.append_fixed(&bytes), + Native::Bytes(string) => builder + .append_bytes(string.zend_str().expect("a native string").as_bytes()) + .map_err(overflow)?, + } + + Ok(()) +} + +/// `ColumnMismatchException::valueDoesNotMatch($definition, $value[, $reason])`. +pub fn value_does_not_match(definition: &Zval, value: Zval, reason: Option) -> Result { + let mut args = vec![definition.shallow_clone(), value]; + args.extend(reason); + + call_static(COLUMN_MISMATCH, "valueDoesNotMatch", &mut args) +} + +pub fn throw(mut exception: Zval) -> PhpException { + match exception.object_mut() { + Some(object) => transparent_exception(object), + None => ext_exception("flow_php expected an exception object"), + } +} + +/// `new SchemaMismatchException($position, $cause)`. +pub fn schema_mismatch(position: Zval, cause: Zval) -> PhpException { + let built = find_class(SCHEMA_MISMATCH).and_then(|ce| construct(ce, &mut [position, cause])); + + match built { + Ok(mut exception) => transparent_exception(&mut exception), + Err(e) => e, + } +} + +/// Whether `definition` admits a null row: `isNullable()`, or a `NullDefinition`. +pub fn definition_nullable(definition: &Zval) -> Result { + Ok(call_method(definition, "isNullable", &mut [])?.bool().unwrap_or(false) + || expect_object(definition, "a Definition")?.instance_of(find_class(NULL_DEFINITION)?)) +} + +pub fn is_types_exception(exception: &ZendObject) -> Result { + Ok(exception.instance_of(find_class(TYPES_EXCEPTION)?)) +} + +#[php_class] +#[php( + name = "Flow\\ETL\\Column\\RustColumnBuilder", + flags = ClassFlags::Final, + implements(ce = column_builder_ce, stub = "Flow\\ETL\\Column\\ColumnBuilder") +)] +pub struct RustColumnBuilder { + definition: Zval, + nullable: bool, + plan: Rc, + values: KindBuilder, +} + +impl RustColumnBuilder { + pub fn for_definition(definition: &Zval, plan: Rc) -> Result { + Ok(Self { + definition: definition.shallow_clone(), + nullable: definition_nullable(definition)?, + values: KindBuilder::new(&plan.kind), + plan, + }) + } + + /// One value for `RustBackend::constant()`: refusals escape as PHP threw them. + pub fn pending(&self, value: &Zval) -> Result { + let value = value.dereference(); + + if value.is_null() { + if !self.nullable { + return Err(throw(value_does_not_match(&self.definition, null_zval(), None)?)); + } + + return Ok(Pending::Null); + } + + if let Some(native) = native_lane(&self.plan, value)? { + return Ok(Pending::Native(native)); + } + + match php_lane(&self.plan, value) { + Ok(physical) => Ok(Pending::Physical(physical)), + Err(Refusal::Cast(mut exception) | Refusal::Physical(mut exception)) => { + Err(transparent_exception(&mut exception)) + } + } + } + + /// One value cast and appended. With a `position`, refusals are `appendMany()`'s `SchemaMismatchException` at it; + /// without, they escape as PHP threw them, as `append()` lets them. + fn append_value(&mut self, value: &Zval, position: Option<&dyn Fn() -> Zval>) -> Result<(), PhpException> { + match append_one(&mut self.values, &self.plan, &self.definition, self.nullable, value)? { + Append::Done => Ok(()), + Append::Refused { cause, raised } => Err(match (position, raised) { + (Some(position), _) => schema_mismatch(position(), cause), + (None, Some(mut raised)) => transparent_exception(&mut raised), + (None, None) => throw(cause), + }), + } + } + + fn not_null(&self, is_null: bool) -> Result<(), PhpException> { + if is_null && !self.nullable { + return Err(throw(value_does_not_match(&self.definition, null_zval(), None)?)); + } + + Ok(()) + } + + fn same_kind<'a>(&self, column: &'a Zval) -> Option<&'a RustColumn> { + column + .extract::<&RustColumn>() + .filter(|native| native.data().data_type() == &flow_batch_frame::kind::data_type(&self.plan.kind)) + } + + /// `count` copies of one value, for `RustBackend::constant()`. + pub fn append_repeated(&mut self, pending: Pending, count: u32) -> Result<(), PhpException> { + match pending { + Pending::Null => (0..count).for_each(|_| self.values.append_null()), + Pending::Physical(physical) => { + for _ in 0..count { + append_physical(&mut self.values, &self.plan.kind, &physical)?; + } + } + Pending::Native(native) => { + let mut single = KindBuilder::new(&self.plan.kind); + append_native(&mut single, native)?; + let one = arrow_array::make_array( + single + .finish(&self.plan.kind) + .map_err(|e| invalid_argument(format!("flow_php built an invalid column: {e}")))?, + ); + + for _ in 0..count { + self.values + .append_from(one.as_ref(), &self.plan.kind, 0) + .map_err(overflow)?; + } + } + } + + Ok(()) + } + + pub fn finish_column(&self) -> Result { + let data = self + .values + .finish(&self.plan.kind) + .map_err(|e| invalid_argument(format!("flow_php built an invalid column: {e}")))?; + + Ok(RustColumn::new(arrow_array::make_array(data), Rc::clone(&self.plan))) + } +} + +fn index(i: i64, count: usize) -> Result { + usize::try_from(i) + .ok() + .filter(|i| *i < count) + .ok_or_else(|| invalid_argument(format!("flow_php column index {i} is outside [0, {count})"))) +} + +#[php_impl] +impl RustColumnBuilder { + pub fn __construct() -> PhpResult { + Err(ext_exception( + "Flow\\ETL\\Column\\RustColumnBuilder is built by RustBackend", + )) + } + + pub fn append(&mut self, value: &Zval) -> PhpResult<()> { + let start = self.values.len(); + let result = self.append_value(value, None); + + if result.is_err() { + self.values.truncate(start); + } + + result + } + + #[php(name = "appendMany")] + pub fn append_many(&mut self, values: &ZendHashTable) -> PhpResult<()> { + let start = self.values.len(); + let result = ht_for_each(values, |string_key, index, value| { + self.append_value( + value, + Some(&|| match string_key { + Some(name) => crate::ctx::zval_str(name.as_bytes()), + None => zval_long(index as i64), + }), + ) + }); + + if result.is_err() { + self.values.truncate(start); + } + + result + } + + /// `ColumnBuilder::appendPhysicals()`: physical values as they are. A null under NOT NULL is refused at its + /// position, a physical of another kind as `append_physical()` refuses it; `$nullCount` is the PHP builders' hint, + /// this builder counts nulls as it appends. + #[php(name = "appendPhysicals")] + pub fn append_physicals(&mut self, physicals: &ZendHashTable, nullCount: Option) -> PhpResult<()> { + let _ = nullCount; + let start = self.values.len(); + let result = ht_for_each(physicals, |_, index, physical| { + if physical.is_null() && !self.nullable { + return Err(schema_mismatch( + zval_long(index as i64), + value_does_not_match(&self.definition, null_zval(), None)?, + )); + } + + append_physical(&mut self.values, &self.plan.kind, physical) + }); + + if result.is_err() { + self.values.truncate(start); + } + + result + } + + #[php(name = "appendFrom")] + pub fn append_from(&mut self, column: &Zval, i: i64) -> PhpResult<()> { + if let Some(native) = self.same_kind(column) { + let i = index(i, native.data().len())?; + self.not_null(matches!(self.plan.kind, Kind::Null) || native.data().is_null(i))?; + + return self + .values + .append_from(native.data().as_ref(), &self.plan.kind, i) + .map_err(overflow); + } + + self.not_null( + call_method(column, "isNull", &mut [zval_long(i)])? + .bool() + .unwrap_or(false), + )?; + let physical = call_method(column, "at", &mut [zval_long(i)])?; + + append_physical(&mut self.values, &self.plan.kind, &physical) + } + + #[php(name = "appendTake")] + pub fn append_take(&mut self, column: &Zval, indices: &ZendHashTable) -> PhpResult<()> { + if let Some(native) = self.same_kind(column) { + let data = native.data(); + let mut rows = Vec::with_capacity(indices.len()); + + ht_for_each(indices, |_, _, i| { + let row = index( + i.long() + .ok_or_else(|| invalid_argument("flow_php column indices must be ints".into()))?, + data.len(), + )?; + self.not_null(matches!(self.plan.kind, Kind::Null) || data.is_null(row))?; + rows.push(row); + + Ok(()) + })?; + + for row in rows { + self.values + .append_from(data.as_ref(), &self.plan.kind, row) + .map_err(overflow)?; + } + + return Ok(()); + } + + let all = call_method(column, "physicals", &mut [])?; + let all = all + .array() + .ok_or_else(|| ext_exception("flow_php expected Column::physicals() to return an array"))?; + let mut physicals = Vec::with_capacity(indices.len()); + + ht_for_each(indices, |_, _, i| { + let physical = i + .long() + .and_then(|i| all.get_index(i)) + .map_or_else(null_zval, Zval::shallow_clone); + self.not_null(physical.is_null())?; + physicals.push(physical); + + Ok(()) + })?; + + for physical in physicals { + append_physical(&mut self.values, &self.plan.kind, &physical)?; + } + + Ok(()) + } + + pub fn count(&self) -> i64 { + self.values.len() as i64 + } + + pub fn finish(&self) -> PhpResult { + self.finish_column() + } +} diff --git a/src/extension/flow-php-ext/src/cast.rs b/src/extension/flow-php-ext/src/cast.rs index 119e589967..46280c4a43 100644 --- a/src/extension/flow-php-ext/src/cast.rs +++ b/src/extension/flow-php-ext/src/cast.rs @@ -2,27 +2,18 @@ use std::os::raw::{c_char, c_int}; use ext_php_rs::boxed::ZBox; use ext_php_rs::exception::PhpException; -use ext_php_rs::flags::DataType; -use ext_php_rs::types::{ZendHashTable, ZendObject, ZendStr, Zval}; -use ext_php_rs::zend::Function; +use ext_php_rs::types::{ZendObject, ZendStr, Zval}; +use ext_php_rs::zend::{ClassEntry, Function}; use crate::ctx::{ - array_key_index, call_handle, call_handle_catching, call_handle_on, call_handle_transparent, - ce_method_ref, construct_with_zvals, ht_find_key, ht_insert, ht_insert_key, null_zval, - schema_mismatch, transparent_exception, write_slot, zval_long, Ctx, HtKey, + self, array_key_index, call_handle_on, call_handle_transparent, expect_object, null_zval, write_slot, zval_long, }; use crate::date_check::{iso_date_gate, iso_date_time_gate, IsoSuffix}; -use crate::encode::{expect_object, ht_for_each, read_slot}; use crate::exception::ext_exception; -use crate::format::Reader; -use crate::hydrate::{ - build_hydrate_plan, fold_metadata_into_schema, fold_pending_metadata, read_cell, AssemblyClasses, - HydrateColumn, HydratePlan, RowValuesClass, -}; use crate::json_check::json_valid; -use crate::plan::{parse_schema_json, Plan, TypeJson}; use crate::uuid_check::is_uuid; use crate::values::date_from_free_form; +use flow_batch_frame::kind::TypeJson; extern "C" { fn zval_get_long_func(op: *const Zval, is_strict: bool) -> i64; @@ -39,18 +30,18 @@ extern "C" { ) -> u8; } -pub(crate) enum MapKeyKind { +pub enum MapKeyKind { Int, Str, } -pub(crate) struct StructureElement { - name: String, - kind: CastKind, - required: bool, +pub struct StructureElement { + pub name: String, + pub kind: CastKind, + pub required: bool, } -pub(crate) enum CastKind { +pub enum CastKind { Integer, PositiveInteger, Float, @@ -92,7 +83,7 @@ fn build_structure_kind(type_json: &TypeJson) -> CastKind { CastKind::Structure(elements) } -fn build_cast_kind(type_json: &TypeJson) -> CastKind { +pub(crate) fn build_cast_kind(type_json: &TypeJson) -> CastKind { match type_json.type_.as_str() { "integer" => CastKind::Integer, "positive_integer" => CastKind::PositiveInteger, @@ -128,158 +119,6 @@ fn build_cast_kind(type_json: &TypeJson) -> CastKind { } } -struct CastColumn { - kind: CastKind, - /// The retained `$definition->type()` object - the fallback receiver. - type_zv: Zval, - /// `Type::cast` resolved on the concrete type class. - cast_fn: &'static Function, -} - -pub struct CastPlan { - hydrate: HydratePlan, - columns: Vec, -} - -fn build_cast_plan(schema: &Zval, ctx: &mut Ctx) -> Result { - let hydrate = build_hydrate_plan(schema)?; - - let schema_obj = expect_object(schema, "a Schema")?; - let schema_ce = unsafe { schema_obj.ce.as_ref() } - .ok_or_else(|| ext_exception("flow_php failed to resolve the Schema class"))?; - let normalize_fn = ce_method_ref(schema_ce, "normalize")?; - let normalized = call_handle_on(normalize_fn, schema_obj, &mut [], "normalize a schema")?; - - let json = { - let json_encode = ctx.json_encode()?; - call_handle( - json_encode, - None, - &mut [normalized.shallow_clone()], - "encode a schema as JSON", - )? - }; - let json_bytes = json - .zend_str() - .ok_or_else(|| ext_exception("flow_php expected schema JSON to be a string"))? - .as_bytes(); - - let definitions = parse_schema_json(json_bytes)?; - - if definitions.len() != hydrate.columns.len() { - return Err(ext_exception( - "flow_php normalized schema JSON does not match the schema definitions", - )); - } - - let mut columns = Vec::with_capacity(definitions.len()); - - for (definition, column) in definitions.iter().zip(&hydrate.columns) { - if column.name_zv.zend_str().map(ZendStr::as_bytes) != Some(definition.name.as_bytes()) { - return Err(ext_exception( - "flow_php normalized schema JSON does not match the schema definitions", - )); - } - - let def_obj = column - .base_def - .object() - .ok_or_else(|| ext_exception("flow_php expected a Definition object"))?; - let def_ce = unsafe { def_obj.ce.as_ref() } - .ok_or_else(|| ext_exception("flow_php failed to resolve a Definition class"))?; - - let type_fn = ce_method_ref(def_ce, "type")?; - let type_zv = call_handle_on(type_fn, def_obj, &mut [], "read a definition type")?; - let type_obj = type_zv - .object() - .ok_or_else(|| ext_exception("flow_php expected a definition type to be an object"))?; - let type_ce = unsafe { type_obj.ce.as_ref() } - .ok_or_else(|| ext_exception("flow_php failed to resolve a Type class"))?; - - columns.push(CastColumn { - kind: build_cast_kind(&definition.type_), - cast_fn: ce_method_ref(type_ce, "cast")?, - type_zv, - }); - } - - Ok(CastPlan { hydrate, columns }) -} - -/// The Schema `definitions` array address identifies the definition set; -/// retaining it (refcount++) prevents address reuse. -fn ensure_cast_plan( - plan: &mut Option, - schema: &Zval, - ctx: &mut Ctx, -) -> Result<(), PhpException> { - let schema_obj = expect_object(schema, "a Schema")?; - - let current = plan.as_ref().and_then(|p| { - read_slot(schema_obj, p.hydrate.definitions_slot) - .array() - .zip(p.hydrate.definitions_retained.array()) - }); - - if current.is_some_and(|(definitions, retained)| std::ptr::eq(definitions, retained)) { - return Ok(()); - } - - *plan = Some(build_cast_plan(schema, ctx)?); - - Ok(()) -} - -/// `HASH_FLAG_PACKED` from zend_types.h. -const HASH_FLAG_PACKED: u8 = 1 << 2; - -/// In-order hashtable walk yielding `(string key, numeric key, value)` - the -/// early-exit sibling of `encode::ht_for_each` for the container cast checks. -struct HtEntries<'a> { - ht: &'a ZendHashTable, - packed: bool, - index: u32, -} - -fn ht_entries(ht: &ZendHashTable) -> HtEntries<'_> { - HtEntries { - ht, - packed: unsafe { ht.u.v.flags } & HASH_FLAG_PACKED != 0, - index: 0, - } -} - -impl<'a> Iterator for HtEntries<'a> { - type Item = (Option<&'a ZendStr>, u64, &'a Zval); - - fn next(&mut self) -> Option { - while self.index < self.ht.nNumUsed { - let position = self.index as usize; - self.index += 1; - - if self.packed { - let value = unsafe { &*self.ht.__bindgen_anon_1.arPacked.add(position) }; - - if value.get_type() == DataType::Undef { - continue; - } - - return Some((None, position as u64, value)); - } - - let bucket = unsafe { &*self.ht.__bindgen_anon_1.arData.add(position) }; - - if bucket.val.get_type() == DataType::Undef { - continue; - } - - return Some((unsafe { bucket.key.as_ref() }, bucket.h, &bucket.val)); - } - - None - } -} - fn engine_long(value: &Zval) -> i64 { unsafe { zval_get_long_func(std::ptr::from_ref(value), false) } } @@ -298,7 +137,7 @@ fn engine_string(value: &Zval) -> Result, PhpException> { /// `mb_strtolower` + table match from `BooleanType::cast`. ASCII lowering /// suffices: no non-ASCII uppercase code point lowercases onto these words. -fn bool_from_str(bytes: &[u8]) -> Option { +pub(crate) fn bool_from_str(bytes: &[u8]) -> Option { if bytes.is_empty() || bytes.len() > 5 { return None; } @@ -318,21 +157,21 @@ fn bool_from_str(bytes: &[u8]) -> Option { } /// The cheap prefix of `Json::isValid`: non-empty + matching `{}`/`[]` pair. -fn json_gate(bytes: &[u8]) -> bool { +pub(crate) fn json_gate(bytes: &[u8]) -> bool { bytes.len() >= 2 && ((bytes[0] == b'{' && bytes[bytes.len() - 1] == b'}') || (bytes[0] == b'[' && bytes[bytes.len() - 1] == b']')) } /// `source` is a gated JSON string; the Json shares its zend_string instead of copying the bytes. -fn json_object_from(source: &Zval, ctx: &mut Ctx) -> Result { +fn json_object_from(source: &Zval) -> Result { let bytes = source .zend_str() .ok_or_else(|| ext_exception("flow_php expected a JSON string"))? .as_bytes(); let is_object = bytes[0] == b'{' && bytes[bytes.len() - 1] == b'}'; - let (json_ce, value_slot, is_object_slot) = ctx.json()?; + let (json_ce, value_slot, is_object_slot) = ctx::json()?; let mut json = ZendObject::new(json_ce); write_slot(&mut json, value_slot, source.shallow_clone()); @@ -355,13 +194,27 @@ const IS_DOUBLE: u8 = 5; /// reproducing the grammar and drifting from it. Yields the type code together /// with the value `_is_numeric_string_ex` already computed, so the integer cast /// reads one parse instead of parsing the same bytes a second time. +/// +/// The engine scans past `length` until a non-digit (it expects a zend_string's +/// NUL terminator), so a CSV cell - a slice of the record buffer - is copied +/// NUL-terminated first; unterminated, `"1"` followed by the next field's +/// `"2..."` reads as trailing data and is refused. fn is_numeric_str(bytes: &[u8]) -> (u8, i64, f64) { let mut lval: i64 = 0; let mut dval: f64 = 0.0; + let mut short = [0u8; 64]; + let long; + let terminated: &[u8] = if bytes.len() < short.len() { + short[..bytes.len()].copy_from_slice(bytes); + &short + } else { + long = [bytes, &[0]].concat(); + &long + }; let code = unsafe { _is_numeric_string_ex( - bytes.as_ptr().cast::(), + terminated.as_ptr().cast::(), bytes.len(), &mut lval, &mut dval, @@ -410,12 +263,96 @@ fn numeric_scalar(value: &Zval) -> bool { } } -/// Casts one value natively; `Ok(None)` bails the WHOLE column value to the +/// `IntegerType::cast` of a string: `None` for any string it refuses or that needs its round-trip branch. A form feed is +/// whitespace to `is_numeric()` but not to `trim()`, so PHP's round-trip refuses it: it bails. +pub(crate) fn integer_from_str(bytes: &[u8]) -> Option { + match is_numeric_str(bytes) { + (IS_LONG, long, _) if !bytes.contains(&0x0C) => Some(long), + (IS_DOUBLE, _, _) if integer_shaped(bytes) => None, + (IS_DOUBLE, _, double) if fits_i64(double) => Some(double as i64), + _ => None, + } +} + +/// `StringType::cast` of a non-finite float: the spelling the text writers produce, built without PHP's `(string)`, +/// which warns on NAN. +fn non_finite_text(value: f64) -> Option<&'static [u8]> { + match value { + value if value.is_nan() => Some(b"NAN"), + f64::INFINITY => Some(b"INF"), + f64::NEG_INFINITY => Some(b"-INF"), + _ => None, + } +} + +/// The three non-finite spellings the text writers produce and `FloatType::cast` reads back, case-sensitive. +fn named_float(bytes: &[u8]) -> Option { + match bytes { + b"NAN" => Some(f64::NAN), + b"INF" => Some(f64::INFINITY), + b"-INF" => Some(f64::NEG_INFINITY), + _ => None, + } +} + +/// `FloatType::cast` of a numeric string: the engine's `(float)` of it, which `is_numeric()` already computed; an integer +/// zero keeps the sign of its text, as `(float) '-0'` is `-0.0`. +pub(crate) fn float_from_str(bytes: &[u8]) -> Option { + if let Some(named) = named_float(bytes) { + return Some(named); + } + + match is_numeric_str(bytes) { + (IS_LONG, 0, _) if bytes.iter().find(|byte| !byte.is_ascii_whitespace() && **byte != 0x0B) == Some(&b'-') => { + Some(-0.0) + } + (IS_LONG, long, _) => Some(long as f64), + (IS_DOUBLE, _, double) => Some(double), + _ => None, + } +} + +/// The instant an ISO string names, parsed as `DateTimeType::cast` parses it (before moving it into the column zone); +/// any other string is gated on StringTemporalParts, a PHP class, so it bails. +pub(crate) fn parse_iso_datetime(bytes: &[u8], zone: &[u8]) -> Result, PhpException> { + Ok( + match iso_date_time_gate(bytes).or_else(|| iso_date_gate(bytes).then_some(IsoSuffix::Naive)) { + Some(IsoSuffix::Zulu) => { + // mirrors DateTimeType::cast's Z branch + let local = bytes.strip_suffix(b"\n").unwrap_or(bytes); + let local = local.strip_suffix(b"Z").unwrap_or(local); + let mut utc = ctx::timezone(b"UTC")?; + + date_from_free_form(local, Some(&mut utc))?.map(|instant| (IsoSuffix::Zulu, instant)) + } + Some(IsoSuffix::Offset) => date_from_free_form(bytes, None)?.map(|instant| (IsoSuffix::Offset, instant)), + Some(IsoSuffix::Naive) => { + let mut column = ctx::timezone(zone)?; + + date_from_free_form(bytes, Some(&mut column))?.map(|instant| (IsoSuffix::Naive, instant)) + } + None => None, + }, + ) +} + +/// `DateType::cast` of an ISO date string: midnight UTC. +pub(crate) fn parse_iso_date(bytes: &[u8]) -> Result, PhpException> { + if !iso_date_gate(bytes) { + return Ok(None); + } + + let mut utc = ctx::timezone(b"UTC")?; + + date_from_free_form(bytes, Some(&mut utc)) +} + +/// Casts one leaf value natively (containers: `builder.rs` `append_cast`); `Ok(None)` bails the WHOLE column value to the /// PHP fallback (`Type::cast`), which reproduces results, exception classes, /// messages and `previous` chains by re-running the canonical implementation. /// PHP calls made inside a branch discard their own thrown exceptions and bail /// instead - mirroring the `catch (Throwable)` wrappers in the PHP casts. -fn cast_value(kind: &CastKind, value: &Zval, ctx: &mut Ctx) -> Result, PhpException> { +pub(crate) fn cast_value(kind: &CastKind, value: &Zval) -> Result, PhpException> { // Type::cast receives its argument by value; a branch that returns `value` itself must not hand // back the caller's reference let value = value.dereference(); @@ -433,14 +370,7 @@ fn cast_value(kind: &CastKind, value: &Zval, ctx: &mut Ctx) -> Result Some(zval_long(long)), - (IS_DOUBLE, _, _) if integer_shaped(bytes) => None, - (IS_DOUBLE, _, double) if fits_i64(double) => Some(zval_long(double as i64)), - _ => None, - } + integer_from_str(string.as_bytes()).map(zval_long) } else if value.is_bool() { Some(zval_long(engine_long(value))) } else { @@ -454,6 +384,10 @@ fn cast_value(kind: &CastKind, value: &Zval, ctx: &mut Ctx) -> Result { if value.is_double() { Some(value.shallow_clone()) + } else if let Some(named) = value.zend_str().and_then(|string| named_float(string.as_bytes())) { + let mut zv = Zval::new(); + zv.set_double(named); + Some(zv) } else if value.is_long() || numeric_scalar(value) { let mut zv = Zval::new(); zv.set_double(engine_double(value)); @@ -472,6 +406,8 @@ fn cast_value(kind: &CastKind, value: &Zval, ctx: &mut Ctx) -> Result Result { if value.is_string() { Some(value.shallow_clone()) + } else if let Some(text) = value.double().and_then(non_finite_text) { + Some(ctx::zval_str(text)) } else if value.is_long() || value.is_double() { let mut zv = Zval::new(); zv.set_zend_string(engine_string(value)?); Some(zv) } else if value.is_true() || value.is_false() { - Some(ctx.bool_str(value.is_true())?.shallow_clone()) + Some(ctx::bool_str(value.is_true())?) } else { None } @@ -496,66 +434,49 @@ fn cast_value(kind: &CastKind, value: &Zval, ctx: &mut Ctx) -> Result { if value.zend_str().is_some_and(|s| !s.as_bytes().is_empty()) { Some(value.shallow_clone()) + } else if let Some(text) = value.double().and_then(non_finite_text) { + Some(ctx::zval_str(text)) } else if value.is_long() || value.is_double() { // engine string form of a number is never empty let mut zv = Zval::new(); zv.set_zend_string(engine_string(value)?); Some(zv) } else if value.is_true() || value.is_false() { - Some(ctx.bool_str(value.is_true())?.shallow_clone()) + Some(ctx::bool_str(value.is_true())?) } else { None } } CastKind::DateTime(zone) => { if let Some(object) = value.object() { - let immutable_ce = ctx.datetime_fns(false)?.ce; + let immutable_ce = ctx::datetime_immutable()?.ce; if !object.instance_of(immutable_ce) { None - } else if zone_name(object, ctx)? == zone.as_slice() { + } else if zone_name(object)? == zone.as_slice() { Some(value.shallow_clone()) } else { - Some(in_zone(value, zone, ctx)?) + Some(in_zone(value, zone)?) } } else if let Some(string) = value.zend_str() { - // any other string is gated on StringTemporalParts, a PHP class, so it bails - let bytes = string.as_bytes(); - - match iso_date_time_gate(bytes).or_else(|| iso_date_gate(bytes).then_some(IsoSuffix::Naive)) { - Some(IsoSuffix::Zulu) => { - // mirrors DateTimeType::cast's Z branch - let local = bytes.strip_suffix(b"\n").unwrap_or(bytes); - let local = local.strip_suffix(b"Z").unwrap_or(local); - let mut utc = ctx.timezone(b"UTC")?.shallow_clone(); - - match date_from_free_form(local, Some(&mut utc), ctx)? { - Some(instant) if zone.as_slice() != b"UTC" => Some(in_zone(&instant, zone, ctx)?), - instant => instant, - } - } - Some(IsoSuffix::Offset) => date_from_free_form(bytes, None, ctx)? - .map(|instant| in_zone(&instant, zone, ctx)) - .transpose()?, - Some(IsoSuffix::Naive) => { - let mut column = ctx.timezone(zone)?.shallow_clone(); - - date_from_free_form(bytes, Some(&mut column), ctx)? - } + match parse_iso_datetime(string.as_bytes(), zone)? { + Some((IsoSuffix::Zulu, instant)) if zone.as_slice() != b"UTC" => Some(in_zone(&instant, zone)?), + Some((IsoSuffix::Offset, instant)) => Some(in_zone(&instant, zone)?), + Some((_, instant)) => Some(instant), None => None, } - } else if value.is_long() || value.is_double() { - date_from_free_form(×tamp_string(value)?, None, ctx)? - .map(|instant| in_zone(&instant, zone, ctx)) + } else if value.is_long() || value.double().is_some_and(f64::is_finite) { + date_from_free_form(×tamp_string(value)?, None)? + .map(|instant| in_zone(&instant, zone)) .transpose()? } else { None } } - CastKind::Date => cast_date(value, ctx)?, + CastKind::Date => cast_date(value)?, CastKind::Uuid => { if let Some(object) = value.object() { - let (uuid_ce, _) = ctx.uuid()?; + let (uuid_ce, _) = ctx::uuid()?; if object.instance_of(uuid_ce) { Some(value.shallow_clone()) @@ -564,7 +485,7 @@ fn cast_value(kind: &CastKind, value: &Zval, ctx: &mut Ctx) -> Result Result cast_json(value, ctx)?, - CastKind::List(inner) => { - let Some(values) = value.array() else { - return Ok(None); - }; - - let mut out = ZendHashTable::with_capacity(values.len() as u32); - - for (expected, (string_key, index, item)) in ht_entries(values).enumerate() { - if string_key.is_some() || index != expected as u64 { - return Ok(None); - } - - let Some(casted) = cast_value(inner, item, ctx)? else { - return Ok(None); - }; - - out.push(casted).map_err(|e| { - ext_exception(format!("flow_php failed to collect list values: {e:?}")) - })?; - } - - let mut zv = Zval::new(); - zv.set_hashtable(out); - Some(zv) - } - CastKind::Map(key_kind, value_kind) => { - let Some(values) = value.array() else { - return Ok(None); - }; - - let mut out = ZendHashTable::with_capacity(values.len() as u32); - - for (string_key, index, item) in ht_entries(values) { - let key = match (key_kind, string_key) { - (MapKeyKind::Int, None) => HtKey::Index(index as i64), - (MapKeyKind::Str, Some(name)) => HtKey::Str(name), - _ => return Ok(None), - }; - - let Some(casted) = cast_value(value_kind, item, ctx)? else { - return Ok(None); - }; - - ht_insert_key(&mut out, &key, casted); - } - - let mut zv = Zval::new(); - zv.set_hashtable(out); - Some(zv) - } - CastKind::Structure(elements) => { - let Some(values) = value.array() else { - return Ok(None); - }; - - let mut out = ZendHashTable::with_capacity(elements.len() as u32); - let mut matched = 0usize; - - for element in elements { - let Some(item) = values.get(element.name.as_str()) else { - if element.required { - // PHP throws MissingElementCastingException for absent required elements - return Ok(None); - } - - continue; - }; - - if item.is_null() && !matches!(element.kind, CastKind::Optional(_)) { - // PHP throws MissingElementCastingException for present-null elements - // whose type rejects null - required and structure-level optional alike - return Ok(None); - } - - let Some(casted) = cast_value(&element.kind, item, ctx)? else { - return Ok(None); - }; - - ht_insert(&mut out, element.name.as_bytes(), casted); - matched += 1; - } - - if matched == 0 { - // an all-optional structure with no matched keys fails PHP's assert - return Ok(None); - } - - let mut zv = Zval::new(); - zv.set_hashtable(out); - Some(zv) - } + CastKind::Json => cast_json(value)?, CastKind::Optional(inner) => { if value.is_null() { Some(null_zval()) } else { - cast_value(inner, value, ctx)? + cast_value(inner, value)? } } - CastKind::Fallback => None, + CastKind::List(_) | CastKind::Map(..) | CastKind::Structure(_) | CastKind::Fallback => None, }) } +fn object_ce(object: &ZendObject) -> Result<&'static ClassEntry, PhpException> { + unsafe { object.ce.as_ref() }.ok_or_else(|| ext_exception("flow_php failed to resolve an object class")) +} + /// `"@" . $value` with the engine's number-to-string conversion. fn timestamp_string(value: &Zval) -> Result, PhpException> { let number = engine_string(value)?; @@ -692,12 +526,12 @@ fn timestamp_string(value: &Zval) -> Result, PhpException> { } /// `$datetime->getTimezone()->getName()` of a DateTimeImmutable. -fn zone_name(datetime: &ZendObject, ctx: &mut Ctx) -> Result, PhpException> { - let immutable_ce = ctx.datetime_fns(false)?.ce; - let get_timezone = ctx.datetime_encode_fns(std::ptr::from_ref(immutable_ce))?.get_timezone; +fn zone_name(datetime: &ZendObject) -> Result, PhpException> { + let immutable_ce = ctx::datetime_immutable()?.ce; + let get_timezone = ctx::datetime_get_timezone(immutable_ce)?; let timezone_zv = call_handle_on(get_timezone, datetime, &mut [], "read a timezone")?; let timezone = expect_object(&timezone_zv, "a timezone")?; - let name = call_handle_on(ctx.timezone_get_name()?, timezone, &mut [], "read a timezone name")?; + let name = call_handle_on(ctx::timezone_get_name()?, timezone, &mut [], "read a timezone name")?; Ok(name .zend_str() @@ -707,29 +541,29 @@ fn zone_name(datetime: &ZendObject, ctx: &mut Ctx) -> Result, PhpExcepti } /// `$datetime->setTimezone(new DateTimeZone($zone))` of a DateTimeImmutable. -fn in_zone(datetime: &Zval, zone: &[u8], ctx: &mut Ctx) -> Result { - let set_timezone = ctx.datetime_fns(false)?.set_timezone; - let timezone = ctx.timezone(zone)?.shallow_clone(); +fn in_zone(datetime: &Zval, zone: &[u8]) -> Result { + let set_timezone = ctx::datetime_immutable()?.set_timezone; + let timezone = ctx::timezone(zone)?; call_handle_on( - &set_timezone, + set_timezone, expect_object(datetime, "a datetime value")?, &mut [timezone], "move a datetime into its column zone", ) } -fn cast_date(value: &Zval, ctx: &mut Ctx) -> Result, PhpException> { +fn cast_date(value: &Zval) -> Result, PhpException> { if let Some(object) = value.object() { - let immutable_ce = ctx.datetime_fns(false)?.ce; + let immutable_ce = ctx::datetime_immutable()?.ce; if !object.instance_of(immutable_ce) { return Ok(None); } - let fns = ctx.datetime_cast_fns(object.ce.cast_const())?; + let fns = ctx::datetime_cast_fns(object_ce(object)?)?; let (set_time, format) = (fns.set_time, fns.format); - let his = ctx.format_his()?.shallow_clone(); + let his = ctx::string(b"H:i:s.u")?; // isValid passthrough: midnight keeps the SAME object (serialize // back-reference topology); a thrown format() bails like PHP's catch @@ -745,17 +579,11 @@ fn cast_date(value: &Zval, ctx: &mut Ctx) -> Result, PhpException> } if let Some(string) = value.zend_str() { - return if iso_date_gate(string.as_bytes()) { - let mut utc = ctx.timezone(b"UTC")?.shallow_clone(); - - date_from_free_form(string.as_bytes(), Some(&mut utc), ctx) - } else { - Ok(None) - }; + return parse_iso_date(string.as_bytes()); } - let parsed = if value.is_long() || value.is_double() { - date_from_free_form(×tamp_string(value)?, None, ctx)? + let parsed = if value.is_long() || value.double().is_some_and(f64::is_finite) { + date_from_free_form(×tamp_string(value)?, None)? } else { None }; @@ -763,19 +591,16 @@ fn cast_date(value: &Zval, ctx: &mut Ctx) -> Result, PhpException> let Some(instant) = parsed else { return Ok(None); }; - let datetime = in_zone(&instant, b"UTC", ctx)?; + let datetime = in_zone(&instant, b"UTC")?; let object = datetime .object() .ok_or_else(|| ext_exception("flow_php expected a datetime object"))?; - let set_time = ctx.datetime_cast_fns(object.ce.cast_const())?.set_time; + let set_time = ctx::datetime_cast_fns(object_ce(object)?)?.set_time; set_midnight(set_time, object) } -fn set_midnight( - set_time: &'static Function, - object: &ZendObject, -) -> Result, PhpException> { +fn set_midnight(set_time: &'static Function, object: &ZendObject) -> Result, PhpException> { let mut args = [zval_long(0), zval_long(0), zval_long(0), zval_long(0)]; let Ok(result) = call_handle_transparent(set_time, Some(object), &mut args) else { @@ -789,9 +614,9 @@ fn set_midnight( Ok(Some(result)) } -fn cast_json(value: &Zval, ctx: &mut Ctx) -> Result, PhpException> { +fn cast_json(value: &Zval) -> Result, PhpException> { if let Some(object) = value.object() { - let (json_ce, _, _) = ctx.json()?; + let (json_ce, _, _) = ctx::json()?; if object.instance_of(json_ce) { return Ok(Some(value.shallow_clone())); @@ -810,15 +635,13 @@ fn cast_json(value: &Zval, ctx: &mut Ctx) -> Result, PhpException> return Ok(None); } - return Ok(Some(json_object_from(value, ctx)?)); + return Ok(Some(json_object_from(value)?)); } if value.is_array() { // Json::fromArray's json_encode, in its non-throwing flavor: `false` // (or a thrown JsonSerializable) bails to the PHP cast - let json_encode = ctx.json_encode()?; - let Ok(encoded) = call_handle_transparent(json_encode, None, &mut [value.shallow_clone()]) - else { + let Ok(encoded) = call_handle_transparent(ctx::json_encode()?, None, &mut [value.shallow_clone()]) else { return Ok(None); }; @@ -830,238 +653,8 @@ fn cast_json(value: &Zval, ctx: &mut Ctx) -> Result, PhpException> return Ok(None); } - return Ok(Some(json_object_from(&encoded, ctx)?)); + return Ok(Some(json_object_from(&encoded)?)); } Ok(None) } - -/// One value of a row against its column: a present null under NOT NULL is refused, a null passes, anything else is -/// cast natively where proven identical and otherwise through the retained PHP `Type::cast`. -fn cast_cell( - column: &HydrateColumn, - cast_column: &CastColumn, - value: &Zval, - row_index: u64, - assembly: &AssemblyClasses, - ctx: &mut Ctx, -) -> Result { - // a present null is a DIFFERENT refusal from an absence: valueDoesNotMatch, not - // missingColumn, and 027 pins the two messages apart - if value.is_null() && !column.nullable { - return Err(schema_mismatch( - assembly.schema_mismatch_ce, - assembly.value_does_not_match, - row_index, - &mut [column.base_def.shallow_clone(), null_zval()], - )?); - } - - if value.is_null() { - return Ok(null_zval()); - } - - if let Some(casted) = cast_value(&cast_column.kind, value, ctx)? { - return Ok(casted); - } - - let type_obj = cast_column - .type_zv - .object() - .ok_or_else(|| ext_exception("flow_php expected a Type object"))?; - - match call_handle_catching(cast_column.cast_fn, Some(type_obj), &mut [value.shallow_clone()]) { - Ok(casted) => Ok(casted), - Err(mut refusal) => { - if !refusal.instance_of(assembly.types_exception_ce) { - return Err(transparent_exception(&mut refusal)); - } - - // The declared type refused the value. Its own exception is dropped rather than chained, exactly as - // HydratedBatch's guard drops it - 027 compares the two hydrators' class and message byte for byte. - Err(schema_mismatch( - assembly.schema_mismatch_ce, - assembly.value_does_not_match, - row_index, - &mut [column.base_def.shallow_clone(), value.shallow_clone()], - )?) - } - } -} - -/// Builds one `Row` through its PHP constructor and appends it to the batch. -fn push_row( - rows_ht: &mut ZendHashTable, - row_values: ZBox, - assembly: &AssemblyClasses, -) -> Result<(), PhpException> { - let mut values_zv = Zval::new(); - values_zv.set_hashtable(row_values); - - let mut row = construct_with_zvals(assembly.row_ce, &mut [values_zv], "a Row")?; - let mut row_zv = Zval::new(); - row_zv.set_object(&mut row); - - rows_ht - .push(row_zv) - .map_err(|e| ext_exception(format!("flow_php failed to collect hydrated rows: {e:?}"))) -} - -/// Every non-null value was cast to its column's type, so the batch takes the shape-only door - the one -/// HydratedBatch returns through. Row::conform() returns $this for a row holding exactly the Schema's keys, in -/// order, with no null under NOT NULL - which every cast row is once no column was absent. An absence keeps the -/// PHP door: padding, missingColumn and the failing row's index are conform()'s to decide. -fn assemble_rows( - rows_ht: ZBox, - schema_zv: Zval, - every_column_present: bool, - assembly: &AssemblyClasses, -) -> Result { - let mut rows_zv = Zval::new(); - rows_zv.set_hashtable(rows_ht); - - let door = if every_column_present { assembly.rows_trusted } else { assembly.rows_conformed }; - - // conform()'s refusal surfaces as the SchemaMismatchException PHP threw, like the cast refusals above - call_handle_transparent(door, None, &mut [schema_zv, rows_zv]) -} - -/// Native `PhpRowHydrator::hydrate`: raw scalars cast against a `Schema` and -/// assembled into `Flow\ETL\Rows` in one pass. Column values cast natively -/// where proven identical, otherwise per value through the retained PHP -/// `Type::cast`; a null under a nullable declaration passes through, and both an -/// absence and a present null under NOT NULL are refused - each with its own cause. -pub fn cast_rows( - batch: &Zval, - schema: &Zval, - plan_slot: &mut Option, - raw_class: &RowValuesClass, - assembly: &AssemblyClasses, - ctx: &mut Ctx, -) -> Result { - ensure_cast_plan(plan_slot, schema, ctx)?; - let plan = plan_slot.as_ref().expect("plan built above"); - - let batch_ht = batch - .array() - .ok_or_else(|| ext_exception("flow_php expected a list of raw row values"))?; - - let schema_zv = fold_metadata_into_schema(schema, batch_ht, raw_class, ctx)?; - let mut rows_ht = ZendHashTable::with_capacity(batch_ht.len() as u32); - let mut every_column_present = true; - - ht_for_each(batch_ht, |_, row_index, rv_zv| { - let rv = expect_object(rv_zv, "a RawRowValues")?; - let values_ht = read_slot(rv, raw_class.values_slot) - .array() - .ok_or_else(|| ext_exception("flow_php expected RawRowValues::values to be an array"))?; - - let mut row_values = ZendHashTable::with_capacity(plan.hydrate.columns.len() as u32); - - for (column, cast_column) in plan.hydrate.columns.iter().zip(&plan.columns) { - let key = column.key(); - - // An absent column is simply not inserted, mirroring HydratedBatch's `continue`: - // Rows::__construct runs Row::matchTo(), which pads a declared-nullable absence and - // raises missingColumn for a NOT-NULL one. Re-implementing that here instead would - // refuse the absence mid-row, so a later cast refusal in the same batch would be - // reported by PHP and pre-empted by native - the two paths would name different - // columns and different rows for the same input. - let Some(value) = ht_find_key(values_ht, &key).map(Zval::dereference) else { - every_column_present = false; - continue; - }; - - let casted = cast_cell(column, cast_column, value, row_index, assembly, ctx)?; - ht_insert_key(&mut row_values, &key, casted); - } - - push_row(&mut rows_ht, row_values, assembly) - })?; - - assemble_rows(rows_ht, schema_zv, every_column_present, assembly) -} - -/// `cast_rows(decode(...))` in one pass: each ROW frame body is decoded straight into the cast, so no -/// `RawRowValues` is built. A Schema column takes the decoded column of the same name - the lookup `cast_rows` does -/// in `RawRowValues::values` - and a Schema column the frames do not carry is an absence, exactly as there. -pub fn decode_rows( - frame_bodies: &Zval, - decode_plan: &Plan, - schema: &Zval, - plan_slot: &mut Option, - assembly: &AssemblyClasses, - ctx: &mut Ctx, -) -> Result { - ensure_cast_plan(plan_slot, schema, ctx)?; - let plan = plan_slot.as_ref().expect("plan built above"); - - let bodies_ht = frame_bodies - .array() - .ok_or_else(|| ext_exception("flow_php expected a list of row frame bodies"))?; - - let decoded_names: Vec<&[u8]> = decode_plan.columns.iter().map(|column| column.name.as_bytes()).collect(); - // RawRowValues::values refuses a name decoded twice at the second occurrence - let duplicated = (0..decoded_names.len()).find(|&index| decoded_names[..index].contains(&decoded_names[index])); - let sources: Vec> = plan - .hydrate - .columns - .iter() - .map(|column| { - let name = column.name_zv.zend_str().map(ZendStr::as_bytes); - decoded_names.iter().position(|decoded| Some(*decoded) == name) - }) - .collect(); - - let mut decoded: Vec = (0..decoded_names.len()).map(|_| Zval::new()).collect(); - let mut metadata: Vec<(Vec, Zval)> = Vec::new(); - let mut rows_ht = ZendHashTable::with_capacity(bodies_ht.len() as u32); - let every_column_present = sources.iter().all(Option::is_some); - - ht_for_each(bodies_ht, |_, row_index, body_zv| { - let bytes = body_zv - .zend_str() - .ok_or_else(|| ext_exception("flow_php expected a row frame body to be a string"))? - .as_bytes(); - - let mut reader = Reader::new(bytes); - - for (index, column) in decode_plan.columns.iter().enumerate() { - let (value, value_metadata) = read_cell(&column.decoder, &mut reader, ctx)?; - - if let Some(value_metadata) = value_metadata { - metadata.push((column.name.as_bytes().to_vec(), value_metadata)); - } - - if duplicated == Some(index) { - return Err(ext_exception(format!( - "flow_php found duplicated entry name \"{}\" in a row frame", - column.name - ))); - } - - decoded[index] = value; - } - - if !reader.is_eof() { - return Err(ext_exception("flow_php row frame length does not match its content")); - } - - let mut row_values = ZendHashTable::with_capacity(plan.hydrate.columns.len() as u32); - - for ((column, cast_column), source) in plan.hydrate.columns.iter().zip(&plan.columns).zip(&sources) { - let Some(source) = source else { - continue; - }; - - let casted = cast_cell(column, cast_column, &decoded[*source], row_index, assembly, ctx)?; - ht_insert_key(&mut row_values, &column.key(), casted); - } - - push_row(&mut rows_ht, row_values, assembly) - })?; - - let schema_zv = fold_pending_metadata(schema, metadata, ctx)?; - - assemble_rows(rows_ht, schema_zv, every_column_present, assembly) -} diff --git a/src/extension/flow-php-ext/src/column.rs b/src/extension/flow-php-ext/src/column.rs new file mode 100644 index 0000000000..8b1f7f5c30 --- /dev/null +++ b/src/extension/flow-php-ext/src/column.rs @@ -0,0 +1,338 @@ +//! `Flow\ETL\Column\RustColumn`: one arrow array of the column's kind, read as the PHP columns read. + +use std::rc::Rc; + +use arrow_array::cast::AsArray; +use arrow_array::{Array, ArrayRef, UInt32Array}; +use arrow_schema::ArrowError; +use ext_php_rs::boxed::ZBox; +use ext_php_rs::convert::IntoZval; +use ext_php_rs::error::Result as ZvalResult; +use ext_php_rs::exception::PhpException; +use ext_php_rs::flags::{ClassFlags, DataType}; +use ext_php_rs::prelude::*; +use ext_php_rs::types::{ZendHashTable, Zval}; +use flow_batch_frame::column::encode; +use flow_batch_frame::kind::{data_type, Kind}; + +use crate::ctx::{call_method, expect_object, find_class, ht_for_each, instance, read_property, zval_long, zval_str}; +use crate::exception::ext_exception; +use crate::interfaces::column_ce; +use crate::physical::{physical_at, value_at}; +use crate::plan::{type_plan, TypePlan}; +use crate::render::{invalid_argument, offset_overflow}; + +/// A return typed `Flow\Types\Type`. +pub struct TypeObject(Zval); + +impl IntoZval for TypeObject { + const TYPE: DataType = DataType::object("Flow\\Types\\Type"); + const NULLABLE: bool = false; + + fn set_zval(self, zv: &mut Zval, _persistent: bool) -> ZvalResult<()> { + *zv = self.0; + + Ok(()) + } +} + +#[php_class] +#[php( + name = "Flow\\ETL\\Column\\RustColumn", + flags = ClassFlags::Final, + implements(ce = column_ce, stub = "Flow\\ETL\\Column\\Column") +)] +pub struct RustColumn { + data: ArrayRef, + plan: Rc, +} + +impl RustColumn { + pub fn new(data: ArrayRef, plan: Rc) -> Self { + debug_assert_eq!(data.data_type(), &data_type(&plan.kind)); + + Self { data, plan } + } + + pub fn data(&self) -> &ArrayRef { + &self.data + } + + pub fn plan(&self) -> &Rc { + &self.plan + } + + fn kind(&self) -> &Kind { + &self.plan.kind + } + + fn row(&self, i: i64) -> Result { + usize::try_from(i) + .ok() + .filter(|i| *i < self.data.len()) + .ok_or_else(|| invalid_argument(format!("flow_php column row {i} is outside [0, {})", self.data.len()))) + } + + fn rows(&self, read: impl Fn(usize) -> Result) -> Result, PhpException> { + let mut rows = ZendHashTable::with_capacity(self.data.len() as u32); + + for i in 0..self.data.len() { + rows.push(read(i)?) + .map_err(|e| ext_exception(format!("flow_php failed to collect column rows: {e:?}")))?; + } + + Ok(rows) + } +} + +fn null_count(array: &dyn Array, kind: &Kind) -> usize { + match kind { + Kind::Null => array.len(), + _ => array.null_count(), + } +} + +/// The child rows of a list or map slice, compacted as the PHP columns hold them. +fn entries(offsets: &[i32], child: &ArrayRef) -> ArrayRef { + let start = offsets[0] as usize; + + child.slice(start, offsets[offsets.len() - 1] as usize - start) +} + +fn base(type_zv: Zval) -> Result { + if expect_object(&type_zv, "a Type")?.instance_of(find_class("Flow\\Types\\Type\\Logical\\OptionalType")?) { + return call_method(&type_zv, "base", &mut []); + } + + Ok(type_zv) +} + +/// `Column::withType()` of the PHP columns over the same storage: `Retype::assert()` at every level, and a required +/// structure element refuses the rows its optional one left absent. +fn retype(from: Zval, to: Zval, array: &ArrayRef, kind: &Kind) -> Result<(), PhpException> { + call_method( + &instance("Flow\\ETL\\Column\\Retype")?, + "assert", + &mut [ + from.shallow_clone(), + to.shallow_clone(), + zval_long(null_count(array.as_ref(), kind) as i64), + ], + )?; + + match kind { + Kind::List(element) => { + let list = array.as_list::(); + + retype( + call_method(&base(from)?, "element", &mut [])?, + call_method(&base(to)?, "element", &mut [])?, + &entries(list.value_offsets(), list.values()), + &element.kind, + ) + } + Kind::Map(key, value) => { + let map = array.as_map(); + let (from, to) = (base(from)?, base(to)?); + + retype( + call_method(&from, "key", &mut [])?, + call_method(&to, "key", &mut [])?, + &entries(map.value_offsets(), map.keys()), + &key.kind, + )?; + retype( + call_method(&from, "value", &mut [])?, + call_method(&to, "value", &mut [])?, + &entries(map.value_offsets(), map.values()), + &value.kind, + ) + } + Kind::Struct(fields) => { + let structure = array.as_struct(); + let from_elements = call_method(&base(from)?, "elements", &mut [])?; + let to_elements = call_method(&base(to)?, "elements", &mut [])?; + let from_elements: Vec<&Zval> = from_elements.array().map(|a| a.values().collect()).unwrap_or_default(); + let to_elements: Vec<&Zval> = to_elements.array().map(|a| a.values().collect()).unwrap_or_default(); + + for (((field, child), from_element), to_element) in fields + .iter() + .zip(structure.columns()) + .zip(from_elements) + .zip(to_elements) + { + let from_element = expect_object(from_element, "a StructureElement")?; + let to_element = expect_object(to_element, "a StructureElement")?; + let optional = |element| -> Result { + Ok(read_property(element, "optional")?.bool().unwrap_or(false)) + }; + + if optional(from_element)? && !optional(to_element)? { + let absent = (0..child.len()) + .filter(|i| !array.is_null(*i) && (matches!(field.kind, Kind::Null) || child.is_null(*i))) + .count(); + + if absent > 0 { + return Err(invalid_argument(format!( + "{absent} absent values under the required element \"{}\"", + field.name + ))); + } + } + + retype( + read_property(from_element, "type")?, + read_property(to_element, "type")?, + child, + &field.kind, + )?; + } + + Ok(()) + } + _ => Ok(()), + } +} + +fn class_name(zv: &Zval) -> String { + zv.object() + .and_then(|object| object.get_class_name().ok()) + .unwrap_or_else(|| zv.get_type().to_string()) +} + +#[php_impl] +impl RustColumn { + pub fn __construct() -> PhpResult { + Err(ext_exception("Flow\\ETL\\Column\\RustColumn is built by RustBackend")) + } + + #[php(name = "type")] + pub fn column_type(&self) -> TypeObject { + TypeObject(self.plan.type_zv.shallow_clone()) + } + + pub fn count(&self) -> i64 { + self.data.len() as i64 + } + + #[php(name = "isNull")] + pub fn is_null(&self, i: i64) -> PhpResult { + let row = self.row(i)?; + + Ok(matches!(self.kind(), Kind::Null) || self.data.is_null(row)) + } + + #[php(name = "nullCount")] + pub fn null_count(&self) -> i64 { + null_count(self.data.as_ref(), self.kind()) as i64 + } + + pub fn at(&self, i: i64) -> PhpResult { + physical_at(self.data.as_ref(), self.kind(), self.row(i)?) + } + + pub fn physicals(&self) -> PhpResult> { + self.rows(|i| physical_at(self.data.as_ref(), self.kind(), i)) + } + + pub fn value(&self, i: i64) -> PhpResult { + value_at(self.data.as_ref(), self.kind(), &self.plan.values, self.row(i)?) + } + + pub fn values(&self) -> PhpResult> { + self.rows(|i| value_at(self.data.as_ref(), self.kind(), &self.plan.values, i)) + } + + pub fn slice(&self, offset: i64, length: i64) -> PhpResult { + let (Ok(offset), Ok(length)) = (usize::try_from(offset), usize::try_from(length)) else { + return Err(invalid_argument(format!( + "flow_php cannot slice {length} rows from row {offset}" + ))); + }; + + if offset.checked_add(length).is_none_or(|end| end > self.data.len()) { + return Err(invalid_argument(format!( + "flow_php cannot slice {length} rows from row {offset} of {}", + self.data.len() + ))); + } + + Ok(RustColumn::new(self.data.slice(offset, length), Rc::clone(&self.plan))) + } + + pub fn take(&self, indices: &ZendHashTable) -> PhpResult { + let mut rows = Vec::with_capacity(indices.len()); + + ht_for_each(indices, |_, _, index| { + let row = index + .long() + .ok_or_else(|| invalid_argument("flow_php column indices must be ints".to_string()))?; + rows.push(self.row(row)? as u32); + + Ok(()) + })?; + + let data = arrow_select::take::take(self.data.as_ref(), &UInt32Array::from(rows), None) + .map_err(|e| invalid_argument(format!("flow_php failed to take column rows: {e}")))?; + + Ok(RustColumn::new(data, Rc::clone(&self.plan))) + } + + #[php(name = "withType")] + pub fn with_type(&self, r#type: &Zval) -> PhpResult { + retype( + self.plan.type_zv.shallow_clone(), + r#type.shallow_clone(), + &self.data, + self.kind(), + )?; + let plan = type_plan(r#type)?; + + if data_type(&plan.kind) != data_type(self.kind()) { + return Err(invalid_argument(format!( + "flow_php cannot store {:?} in a {:?} column", + plan.kind, + self.kind() + ))); + } + + Ok(RustColumn::new(self.data.clone(), plan)) + } + + pub fn concat(&self, others: &[&Zval]) -> PhpResult { + let mut arrays: Vec<&dyn Array> = vec![self.data.as_ref()]; + + for other in others { + let native = other + .extract::<&RustColumn>() + .filter(|native| native.data.data_type() == self.data.data_type()) + .ok_or_else(|| { + invalid_argument(format!( + "Flow\\ETL\\Column\\RustColumn cannot concat {}", + class_name(other) + )) + })?; + arrays.push(native.data.as_ref()); + } + + let data = arrow_select::concat::concat(&arrays).map_err(|e| match e { + ArrowError::OffsetOverflowError(last) => offset_overflow(last as u64), + e => invalid_argument(format!("flow_php failed to concat columns: {e}")), + })?; + + Ok(RustColumn::new(data, Rc::clone(&self.plan))) + } + + pub fn encode(&self) -> PhpResult> { + let buffers = encode(&self.data.to_data(), self.kind()) + .map_err(|e| invalid_argument(format!("flow_php failed to encode a column: {e}")))?; + let mut list = ZendHashTable::with_capacity(buffers.len() as u32); + + for buffer in buffers { + list.push(zval_str(&buffer)) + .map_err(|e| ext_exception(format!("flow_php failed to collect column buffers: {e:?}")))?; + } + + Ok(list) + } +} diff --git a/src/extension/flow-php-ext/src/csv/columns.rs b/src/extension/flow-php-ext/src/csv/columns.rs new file mode 100644 index 0000000000..b9f0ed0808 --- /dev/null +++ b/src/extension/flow-php-ext/src/csv/columns.rs @@ -0,0 +1,120 @@ +//! `RustCSVOpenSource::batches()`: CSV cells straight into native columns through `BatchColumns`; a NOT NULL +//! column absent from the header is absent from every row. + +use ext_php_rs::exception::PhpException; +use ext_php_rs::types::Zval; + +use crate::batch_columns::{native_text, BatchColumn, BatchColumns, HeldBatch}; +use crate::builder::{is_types_exception, php_lane, value_does_not_match, Refusal}; +use crate::csv::CsvReader; +use crate::ctx::{null_zval, transparent_exception, zval_str}; +use crate::physical::append_physical; + +/// The batch being built, and the field of the header each of its columns reads. +#[derive(Default)] +pub struct CsvColumns { + batch: Option, + /// `None`: the header has no column of this name. + fields: Vec>, +} + +/// One cell into its column: a NOT NULL null or a PHP-lane `Flow\Types` refusal becomes the column's refusal, any +/// other exception escapes as thrown. +fn append_cell(column: &mut BatchColumn, row: usize, cell: Option<&[u8]>) -> Result<(), PhpException> { + let Some(bytes) = cell else { + if !column.nullable { + column.refusal = Some((row, value_does_not_match(&column.definition, null_zval(), None)?)); + + return Ok(()); + } + + column.values.append_null(); + + return Ok(()); + }; + + if native_text(&mut column.values, &column.plan.kind, &column.plan.cast, bytes)? { + return Ok(()); + } + + let value = zval_str(bytes); + + match php_lane(&column.plan, &value) { + Ok(physical) => append_physical(&mut column.values, &column.plan.kind, &physical), + Err(Refusal::Cast(mut exception)) => { + if !is_types_exception(&exception)? { + return Err(transparent_exception(&mut exception)); + } + + column.refusal = Some((row, value_does_not_match(&column.definition, value, None)?)); + + Ok(()) + } + Err(Refusal::Physical(mut exception)) => { + if !is_types_exception(&exception)? { + return Err(transparent_exception(&mut exception)); + } + + let mut reason = Zval::new(); + reason.set_object(&mut exception); + column.refusal = Some((row, value_does_not_match(&column.definition, value, Some(reason))?)); + + Ok(()) + } + } +} + +pub fn next_columns( + reader: &mut CsvReader, + pending: &mut CsvColumns, + schema: &Zval, + batch_size: usize, +) -> Result, PhpException> { + if reader.headers().is_empty() { + return Ok(None); + } + + let (state, rebuilt) = BatchColumns::prepare(&mut pending.batch, schema, batch_size, "CSV")?; + + if rebuilt { + pending.fields = state + .columns + .iter() + .map(|column| reader.field_of(&column.key.name())) + .collect(); + } + + while state.rows < state.batch_size && reader.next_record() { + let row = state.rows; + + for (column, field) in state.columns.iter_mut().zip(&pending.fields) { + if column.refusal.is_some() { + continue; + } + + let Some(field) = field else { + if !column.nullable { + column.absent = column.absent.or(Some(row)); + } + + column.values.append_null(); + + continue; + }; + + if let Err(e) = append_cell(column, row, reader.cell(*field)) { + state.reset(); + + return Err(e); + } + } + + state.rows += 1; + } + + if state.rows == 0 || (state.rows < state.batch_size && !reader.is_finished()) { + return Ok(None); + } + + state.finish().map(Some) +} diff --git a/src/extension/flow-php-ext/src/csv/fold.rs b/src/extension/flow-php-ext/src/csv/fold.rs index 21131eb326..05a37309ab 100644 --- a/src/extension/flow-php-ext/src/csv/fold.rs +++ b/src/extension/flow-php-ext/src/csv/fold.rs @@ -9,9 +9,9 @@ use std::collections::HashMap; use ext_php_rs::exception::PhpException; use ext_php_rs::types::{ZendHashTable, Zval}; -use crate::ctx::{call_handle, zval_str, Ctx}; use crate::csv::php_trim; use crate::csv::tokenizer::is_space; +use crate::ctx::{self, call_handle, zval_str}; use crate::date_check::{checkdate, iso_date_gate, iso_date_time_gate}; use crate::exception::ext_exception; use crate::json_check::json_valid; @@ -129,15 +129,11 @@ impl Candidates { /// `StringTypeNarrower::narrow()` for a string cell, without the HTML and XML rungs. pub struct Narrower { candidates: Candidates, - ctx: Ctx, } impl Narrower { fn new(candidates: Candidates) -> Result { - Ok(Self { - candidates, - ctx: Ctx::new()?, - }) + Ok(Self { candidates }) } pub fn narrow(&mut self, value: &[u8]) -> Result { @@ -192,7 +188,12 @@ impl Narrower { } // same one-sided rule as the cast: only PHP may say "not JSON" - let valid = call_handle(self.ctx.json_validate()?, None, &mut [zval_str(value)], "validate a JSON cell")?; + let valid = call_handle( + ctx::json_validate()?, + None, + &mut [zval_str(value)], + "validate a JSON cell", + )?; Ok(valid.bool().unwrap_or(false)) } @@ -208,7 +209,12 @@ impl Narrower { return Ok(Some(Leaf::Date)); } - let parts_zv = call_handle(self.ctx.date_parse()?, None, &mut [zval_str(value)], "parse a temporal cell")?; + let parts_zv = call_handle( + ctx::date_parse()?, + None, + &mut [zval_str(value)], + "parse a temporal cell", + )?; let parts = parts_zv .array() .ok_or_else(|| ext_exception("flow_php expected date_parse() to return an array"))?; @@ -242,11 +248,11 @@ impl Narrower { } fn is_timezone(&mut self, value: &[u8]) -> Result { - if self.ctx.timezone_identifiers()?.contains(value) { + if ctx::is_timezone_identifier(value)? { return Ok(true); } - Ok(is_offset(value) && self.ctx.timezone_accepts(value)?) + Ok(is_offset(value) && ctx::timezone_accepts(value)?) } } @@ -261,14 +267,22 @@ fn is_null(value: &[u8]) -> bool { /// `Json::isValid()`'s shape gate before `json_validate()`. fn is_json_shaped(value: &[u8]) -> bool { - matches!((value.first(), value.last()), (Some(b'{'), Some(b'}')) | (Some(b'['), Some(b']'))) + matches!( + (value.first(), value.last()), + (Some(b'{'), Some(b'}')) | (Some(b'['), Some(b']')) + ) } /// PHP 8 `is_numeric()` on a string: optional leading and trailing whitespace, optional sign, /// `digits[.digits] | .digits | digits.`, optional exponent. No hex, no `INF`/`NAN`. fn is_numeric(value: &[u8]) -> bool { let mut position = value.iter().take_while(|byte| is_space(**byte)).count(); - let end = value.len() - value[position..].iter().rev().take_while(|byte| is_space(**byte)).count(); + let end = value.len() + - value[position..] + .iter() + .rev() + .take_while(|byte| is_space(**byte)) + .count(); if matches!(value.get(position), Some(b'+' | b'-')) { position += 1; @@ -338,7 +352,10 @@ fn has_explicit_day(value: &[u8]) -> bool { return true; } - groups >= 2 && value.windows(3).any(|window| window.iter().all(u8::is_ascii_alphabetic)) + groups >= 2 + && value + .windows(3) + .any(|window| window.iter().all(u8::is_ascii_alphabetic)) } /// `StringTypeNarrower::isTimeZone()`'s `/^[+-]\d{2}:\d{2}$/` on a trimmed value. @@ -354,7 +371,7 @@ struct Column { /// `ColumnTypes::observe()` over whole rows: names seeded `null` (then widened), names met only in rows start at /// their first observed type; a `string` / `?string` column skips non-null cells. pub struct Fold { - pub narrower: Narrower, + narrower: Narrower, columns: Vec, index: HashMap, usize>, rows: u64, @@ -383,7 +400,12 @@ impl Fold { } /// One cell of the current row; `None` is a null cell. `position` caches the cell's column across rows. - pub fn observe(&mut self, name: &[u8], position: &mut Option, value: Option<&[u8]>) -> Result<(), PhpException> { + pub fn observe( + &mut self, + name: &[u8], + position: &mut Option, + value: Option<&[u8]>, + ) -> Result<(), PhpException> { if position.is_none() { *position = self.index.get(name).copied(); } @@ -426,7 +448,9 @@ impl Fold { /// Column name => type code, first-seen order. pub fn types(&self) -> impl Iterator { - self.columns.iter().map(|column| (column.name.as_slice(), column.kind.code())) + self.columns + .iter() + .map(|column| (column.name.as_slice(), column.kind.code())) } } diff --git a/src/extension/flow-php-ext/src/csv/mod.rs b/src/extension/flow-php-ext/src/csv/mod.rs index 6b7fc4e6ae..534a4f614a 100644 --- a/src/extension/flow-php-ext/src/csv/mod.rs +++ b/src/extension/flow-php-ext/src/csv/mod.rs @@ -1,16 +1,22 @@ //! Native CSV reading: records and fields from `tokenizer`, rows shaped exactly like -//! `CSVEncoder::decode()` + `CSVRowNormalizer::normalize()` shape them. +//! `CSVDecoder::decode()` (`Flow\ETL\Extractor\Records`) shapes them. +pub mod columns; pub mod fold; +#[allow( + non_snake_case, + reason = "PHP takes a parameter name from its Rust identifier: the contracts' are camelCase" +)] +pub mod source; pub mod tokenizer; +pub mod write; use ext_php_rs::boxed::ZBox; use ext_php_rs::exception::PhpException; -use ext_php_rs::types::{ZendHashTable, ZendObject, ZendStr, Zval}; +use ext_php_rs::types::{ZendHashTable, ZendStr, Zval}; -use crate::ctx::{array_key_index, ht_insert_key, null_zval, write_slot, zval_str, HtKey}; +use crate::ctx::{array_key_index, ht_insert_key, null_zval, zval_str, HtKey}; use crate::exception::ext_exception; -use crate::hydrate::RowValuesClass; use tokenizer::{Dialect, Field, Record, Tokenizer}; /// A header name resolved once into the array key `array_combine()` would use for it. @@ -42,7 +48,7 @@ impl Header { } } -/// One key of `RawRowValues::$values`: `array_combine()` keeps a duplicated header at its first position with the +/// One key of a row: `array_combine()` keeps a duplicated header at its first position with the /// value of its last one. struct Cell { header_index: usize, @@ -58,6 +64,7 @@ pub struct CsvReader { record: Record, first_row_pending: bool, consumed_bytes: u64, + finished: bool, } impl CsvReader { @@ -97,6 +104,7 @@ impl CsvReader { record: Record::default(), first_row_pending: false, consumed_bytes: 0, + finished: false, }) } @@ -106,6 +114,41 @@ impl CsvReader { pub fn finish(&mut self) { self.tokenizer.finish(); + self.finished = true; + } + + pub fn is_finished(&self) -> bool { + self.finished + } + + /// The field of the header named `name` (the last of duplicates, as `array_combine()` keeps it); `None` before the + /// header resolved or when no header has that name. + pub fn field_of(&mut self, name: &[u8]) -> Option { + self.resolve_headers(); + let headers = self.headers.as_ref()?; + + self.cells + .iter() + .find(|cell| headers[cell.header_index].name == name) + .map(|cell| cell.field_index) + } + + /// Advances to the next complete row, counting its bytes; `false` when none is buffered. + pub fn next_record(&mut self) -> bool { + self.resolve_headers(); + + self.headers.is_some() + && next_row( + &mut self.first_row_pending, + &mut self.tokenizer, + &mut self.record, + &mut self.consumed_bytes, + ) + } + + /// The current row's cell of `field`, as `Records::decode()` shapes it; `None` is null. + pub fn cell(&self, field: usize) -> Option<&[u8]> { + cell_value(&self.record, field, self.empty_to_null) } /// The bytes, as read, of every row `next` and `fold` produced so far: line endings included, the header @@ -124,8 +167,8 @@ impl CsvReader { .unwrap_or_default() } - /// Up to `batch_size` `RawRowValues`; an empty list when no complete record is buffered. - pub fn next(&mut self, batch_size: usize, class: &RowValuesClass) -> Result, PhpException> { + /// Up to `batch_size` rows keyed by header; an empty list when no complete record is buffered. + pub fn next(&mut self, batch_size: usize) -> Result, PhpException> { self.resolve_headers(); let mut batch = ZendHashTable::new(); @@ -135,12 +178,17 @@ impl CsvReader { }; while batch.len() < batch_size { - if !next_row(&mut self.first_row_pending, &mut self.tokenizer, &mut self.record, &mut self.consumed_bytes) { + if !next_row( + &mut self.first_row_pending, + &mut self.tokenizer, + &mut self.record, + &mut self.consumed_bytes, + ) { break; } batch - .push(row_values(headers, &self.record, self.empty_to_null, class)) + .push(row_values(headers, &self.record, self.empty_to_null)) .map_err(|e| ext_exception(format!("flow_php failed to collect CSV row values: {e:?}")))?; } @@ -160,7 +208,12 @@ impl CsvReader { let mut positions = vec![None; self.cells.len()]; while limit.is_none_or(|limit| folded < limit) { - if !next_row(&mut self.first_row_pending, &mut self.tokenizer, &mut self.record, &mut self.consumed_bytes) { + if !next_row( + &mut self.first_row_pending, + &mut self.tokenizer, + &mut self.record, + &mut self.consumed_bytes, + ) { break; } @@ -179,7 +232,7 @@ impl CsvReader { Ok(folded) } - /// `CSVEncoder::decode()`'s first line: the header when `withHeader`, else `e00, e01, ...` sized by + /// `Records::decode()`'s first record: the header when `withHeader`, else `e00, e01, ...` sized by /// the first record, which then stays pending as the first row. fn resolve_headers(&mut self) { if self.headers.is_some() || !self.tokenizer.next(&mut self.record) { @@ -206,13 +259,19 @@ impl CsvReader { } else { self.first_row_pending = true; - (0..record.len()).map(|index| Header::new(format!("e{index:02}").into_bytes())).collect() + (0..record.len()) + .map(|index| Header::new(format!("e{index:02}").into_bytes())) + .collect() }); let headers = self.headers.as_ref().expect("resolved above"); for (index, header) in headers.iter().enumerate() { - match self.cells.iter_mut().find(|cell| headers[cell.header_index].name == header.name) { + match self + .cells + .iter_mut() + .find(|cell| headers[cell.header_index].name == header.name) + { Some(cell) => cell.field_index = index, None => self.cells.push(Cell { header_index: index, @@ -240,18 +299,18 @@ fn next_row( true } -/// `CSVRowNormalizer::normalize()` for one cell: padding past the record's end, `emptyToNull`; `None` is null. +/// `Records::decode()` for one cell: a cell past the record's end is null, a present `''` is null under `emptyToNull`; +/// `None` is null. fn cell_value(record: &Record, index: usize, empty_to_null: bool) -> Option<&[u8]> { match record.field(index) { Field::Value(value) if !(empty_to_null && value.is_empty()) => Some(value), - Field::Value(_) | Field::Null => None, - Field::Missing if empty_to_null => None, - Field::Missing => Some(b""), + Field::Value(_) | Field::Null | Field::Missing => None, } } -/// `new RawRowValues(array_combine($headers, $normalizer->normalize($fields, count($headers))))`. -fn row_values(headers: &[Header], record: &Record, empty_to_null: bool, class: &RowValuesClass) -> ZBox { +/// `Records::decode()` for one record: one value per header, a duplicated header keeps its first position and its +/// last value. +fn row_values(headers: &[Header], record: &Record, empty_to_null: bool) -> Zval { let mut values = ZendHashTable::with_capacity(headers.len() as u32); for (index, header) in headers.iter().enumerate() { @@ -260,24 +319,20 @@ fn row_values(headers: &[Header], record: &Record, empty_to_null: bool, class: & ht_insert_key(&mut values, &header.key(), value); } - let mut row_values = ZendObject::new(class.ce); - let mut values_zv = Zval::new(); values_zv.set_hashtable(values); - write_slot(&mut row_values, class.values_slot, values_zv); - - let mut metadata_zv = Zval::new(); - metadata_zv.set_hashtable(ZendHashTable::new()); - write_slot(&mut row_values, class.metadata_slot, metadata_zv); - row_values + values_zv } /// PHP's `trim()`: `" \t\n\r\0\x0B"`, nothing else. pub fn php_trim(bytes: &[u8]) -> &[u8] { let is_trimmed = |byte: &u8| matches!(byte, b' ' | b'\t' | b'\n' | b'\r' | b'\0' | 0x0B); let start = bytes.iter().position(|byte| !is_trimmed(byte)).unwrap_or(bytes.len()); - let end = bytes.iter().rposition(|byte| !is_trimmed(byte)).map_or(start, |end| end + 1); + let end = bytes + .iter() + .rposition(|byte| !is_trimmed(byte)) + .map_or(start, |end| end + 1); &bytes[start..end] } diff --git a/src/extension/flow-php-ext/src/csv/source.rs b/src/extension/flow-php-ext/src/csv/source.rs new file mode 100644 index 0000000000..cf994f1a7e --- /dev/null +++ b/src/extension/flow-php-ext/src/csv/source.rs @@ -0,0 +1,434 @@ +//! `Flow\ETL\Adapter\CSV\RustCSVOpenSource`: one CSV file read through the native tokenizer - the stream fed in chunks +//! (a `NativeLocalSourceStream` in `CHUNK`s, any other stream in `$charactersReadInLine`), batches as native columns +//! adopted by the configured backend, records as PHP arrays, the type sniff folded natively. + +use std::cell::{Cell, RefCell}; +use std::collections::VecDeque; +use std::rc::Rc; + +use ext_php_rs::binary_slice::BinarySlice; +use ext_php_rs::convert::IntoZval; +use ext_php_rs::error::Result as ZvalResult; +use ext_php_rs::flags::{ClassFlags, DataType}; +use ext_php_rs::prelude::*; +use ext_php_rs::types::{ZendHashTable, Zval}; + +use crate::batch_columns::{batch_size_of, HeldBatch}; +use crate::csv::columns::{next_columns, CsvColumns}; +use crate::csv::fold::{string_values, Fold}; +use crate::csv::CsvReader; +use crate::ctx::{call_function, call_method, call_static, construct, find_class, ht_insert, zval_long, zval_str}; +use crate::exception::ext_exception; +use crate::interfaces::csv_open_source_ce; +use crate::iterator::RustIterator; +use crate::source::{self, reading, Fed, CHUNK}; + +const CLASS: &str = "RustCSVOpenSource"; +const RECORDS_BATCH: usize = 1000; +const NATIVE_LOCAL_SOURCE_STREAM: &str = "Flow\\Filesystem\\Stream\\NativeLocalSourceStream"; +const STRING_TYPE_NARROWER: &str = "Flow\\Types\\Type\\Native\\String\\StringTypeNarrower"; +const SCHEMA_INFERRER: &str = "Flow\\ETL\\Schema\\Inference\\SchemaInferrer"; +/// The candidates the native fold narrows to, in `StringTypeNarrower`'s order; HTML and XML are never folded natively. +const FOLDED: [&str; 8] = [ + "Flow\\Types\\DSL\\type_json", + "Flow\\Types\\DSL\\type_uuid", + "Flow\\Types\\DSL\\type_float", + "Flow\\Types\\DSL\\type_integer", + "Flow\\Types\\DSL\\type_datetime", + "Flow\\Types\\DSL\\type_date", + "Flow\\Types\\DSL\\type_boolean", + "Flow\\Types\\DSL\\type_time_zone", +]; + +/// A contract method's `array` return. +pub struct ArrayValue(Zval); + +impl IntoZval for ArrayValue { + const TYPE: DataType = DataType::Array; + const NULLABLE: bool = false; + + fn set_zval(self, zv: &mut Zval, _persistent: bool) -> ZvalResult<()> { + *zv = self.0; + + Ok(()) + } +} + +/// `CSVOpenSource::sniff()`'s return, typed `Flow\ETL\Schema\Inference\ColumnTypes`. +pub struct ColumnTypesValue(Zval); + +impl IntoZval for ColumnTypesValue { + const TYPE: DataType = DataType::object("Flow\\ETL\\Schema\\Inference\\ColumnTypes"); + const NULLABLE: bool = false; + + fn set_zval(self, zv: &mut Zval, _persistent: bool) -> ZvalResult<()> { + *zv = self.0; + + Ok(()) + } +} + +struct Reading { + reader: CsvReader, + columns: CsvColumns, +} + +struct Shared { + stream: Zval, + chunk_size: i64, + reading: RefCell, + produced_rows: Cell, + produced_bytes: Cell, +} + +impl Shared { + /// `f` over the reader, no PHP stream call inside; `producedBytes()` follows what it consumed. + fn with(&self, f: impl FnOnce(&mut Reading) -> PhpResult) -> PhpResult { + let mut reading = reading(&self.reading, CLASS)?; + let result = f(&mut reading); + self.produced_bytes.set(reading.reader.consumed_bytes()); + + result + } + + fn produced(&self, rows: u64) { + self.produced_rows.set(self.produced_rows.get() + rows); + } + + /// `$stream->iterate($chunkSize)`, the generator one consuming method feeds from. + fn chunks(&self) -> PhpResult { + call_method(&self.stream, "iterate", &mut [zval_long(self.chunk_size)]) + } + + /// The next chunk of `chunks` fed to the reader; false once the generator is exhausted. The generator runs with no + /// borrow held: the stream may call back into the source. + fn feed(&self, chunks: &Zval) -> PhpResult { + if !call_method(chunks, "valid", &mut [])?.bool().unwrap_or(false) { + return Ok(false); + } + + let chunk = call_method(chunks, "current", &mut [])?; + let bytes = chunk + .zend_str() + .ok_or_else(|| ext_exception("flow_php expected SourceStream::iterate() to yield strings"))? + .as_bytes(); + self.with(|reading| { + reading.reader.feed(bytes); + + Ok(()) + })?; + call_method(chunks, "next", &mut [])?; + + Ok(true) + } + + fn finish(&self) -> PhpResult<()> { + self.with(|reading| { + reading.reader.finish(); + + Ok(()) + }) + } +} + +/// One `batches()` call over the shared reader. +struct CsvFeed { + shared: Rc, + chunks: Zval, + schema: Zval, + batch_size: usize, +} + +impl Fed for CsvFeed { + fn next_batch(&self) -> PhpResult> { + let batch = self + .shared + .with(|reading| next_columns(&mut reading.reader, &mut reading.columns, &self.schema, self.batch_size))?; + + if let Some(batch) = &batch { + self.shared.produced(batch.count as u64); + } + + Ok(batch) + } + + fn feed(&self) -> PhpResult { + self.shared.feed(&self.chunks) + } + + fn finish(&self) -> PhpResult<()> { + self.shared.finish() + } +} + +#[php_class] +#[php( + name = "Flow\\ETL\\Adapter\\CSV\\RustCSVOpenSource", + flags = ClassFlags::Final, + implements(ce = csv_open_source_ce, stub = "Flow\\ETL\\Adapter\\CSV\\CSVOpenSource") +)] +pub struct RustCSVOpenSource { + shared: Rc, +} + +impl RustCSVOpenSource { + fn records_iterator(&self) -> PhpResult { + let shared = Rc::clone(&self.shared); + let chunks = shared.chunks()?; + let mut pending: VecDeque = VecDeque::new(); + let mut finished = false; + + Ok(RustIterator::new(Box::new(move || loop { + if let Some(record) = pending.pop_front() { + return Ok(Some(record)); + } + + let batch = shared.with(|reading| reading.reader.next(RECORDS_BATCH))?; + + if !batch.is_empty() { + shared.produced(batch.len() as u64); + pending.extend(batch.values().map(Zval::shallow_clone)); + + continue; + } + + if finished { + return Ok(None); + } + + if !shared.feed(&chunks)? { + shared.finish()?; + finished = true; + } + }))) + } +} + +#[php_impl] +impl RustCSVOpenSource { + #[allow(clippy::too_many_arguments)] + pub fn __construct( + stream: &Zval, + separator: BinarySlice, + enclosure: BinarySlice, + escape: BinarySlice, + withHeader: bool, + emptyToNull: bool, + removeBOM: bool, + charactersReadInLine: Option, + ) -> PhpResult { + // a local file ignores the read length, as NativeLocalSourceStream::readLines() does on the PHP path; for + // S3/Azure it is the range-request size - and from_csv() defaults it to 10 MB, which would be the chunk + let local = match ( + stream.object(), + ext_php_rs::zend::ClassEntry::try_find(NATIVE_LOCAL_SOURCE_STREAM), + ) { + (Some(object), Some(ce)) => object.instance_of(ce), + _ => false, + }; + + Ok(Self { + shared: Rc::new(Shared { + stream: stream.shallow_clone(), + chunk_size: if local { + CHUNK + } else { + charactersReadInLine.unwrap_or(CHUNK) + }, + reading: RefCell::new(Reading { + reader: CsvReader::new(&separator, &enclosure, &escape, withHeader, emptyToNull, removeBOM)?, + columns: CsvColumns::default(), + }), + produced_rows: Cell::new(0), + produced_bytes: Cell::new(0), + }), + }) + } + + pub fn close(&self) -> PhpResult<()> { + call_method(&self.shared.stream, "close", &mut [])?; + + Ok(()) + } + + #[php(name = "producedBytes")] + pub fn produced_bytes(&self) -> i64 { + self.shared.produced_bytes.get() as i64 + } + + #[php(name = "producedRows")] + pub fn produced_rows(&self) -> i64 { + self.shared.produced_rows.get() as i64 + } + + pub fn columns(&self) -> PhpResult { + let chunks = self.shared.chunks()?; + + while self.shared.feed(&chunks)? { + if self.shared.with(|reading| Ok(!reading.reader.headers().is_empty()))? { + return self.headers(); + } + } + + self.shared.finish()?; + + self.headers() + } + + /// Exactly `$batchSize` rows keyed and ordered by `$schema` (the last may be shorter), every column adopted by + /// `$backend`; refusals are `RowsBuilder::appendRows()`'s, row indexes relative to the batch. + pub fn batches(&self, schema: &Zval, batchSize: i64, backend: &Zval) -> PhpResult { + let fed = CsvFeed { + shared: Rc::clone(&self.shared), + chunks: self.shared.chunks()?, + schema: schema.shallow_clone(), + batch_size: batch_size_of(batchSize, "CSV")?, + }; + + Ok(source::batches(fed, schema, backend)) + } + + pub fn headers(&self) -> PhpResult { + self.shared.with(|reading| headers(&mut reading.reader)).map(ArrayValue) + } + + pub fn records(&self) -> PhpResult { + self.records_iterator() + } + + /// `SchemaInferrer::sniff()` over `records()`; a `StringTypeNarrower` without its HTML and XML rungs folds natively. + pub fn sniff( + &self, + names: &ZendHashTable, + rowBudget: i64, + inference: &Zval, + typer: &Zval, + ) -> PhpResult { + if rowBudget < -1 { + return Err(ext_exception("flow_php CSV fold limit must be -1 or at least 0")); + } + + if !folds_natively(typer)? { + let mut records = Zval::new(); + self.records_iterator()?.set_zval(&mut records, false)?; + let mut names_zv = Zval::new(); + names_zv.set_hashtable(names.to_owned()); + let inferrer = construct( + find_class(SCHEMA_INFERRER)?, + &mut [inference.shallow_clone(), typer.shallow_clone()], + )? + .into_zval(false)?; + + return call_method(&inferrer, "sniff", &mut [names_zv, records, zval_long(rowBudget)]) + .map(ColumnTypesValue); + } + + let mut candidates = Vec::new(); + + for function in FOLDED { + let candidate = call_function(function, &mut [])?; + + if call_method(typer, "emitsType", &mut [candidate.shallow_clone()])? + .bool() + .unwrap_or(false) + { + candidates.push( + call_method(&candidate, "toString", &mut [])? + .zend_str() + .ok_or_else(|| ext_exception("flow_php expected Type::toString() to return a string"))? + .as_bytes() + .to_vec(), + ); + } + } + + let mut fold = Fold::new(string_values(names), &candidates)?; + let remaining = |fold: &Fold| match rowBudget { + -1 => None, + budget => Some((budget as u64).saturating_sub(fold.rows())), + }; + let chunks = self.shared.chunks()?; + + while self.shared.feed(&chunks)? { + let limit = remaining(&fold); + let folded = self.shared.with(|reading| reading.reader.fold(&mut fold, limit))?; + self.shared.produced(folded); + + if rowBudget != -1 && fold.rows() >= rowBudget as u64 { + break; + } + } + + if rowBudget == -1 || fold.rows() < rowBudget as u64 { + self.shared.finish()?; + let limit = remaining(&fold); + let folded = self.shared.with(|reading| reading.reader.fold(&mut fold, limit))?; + self.shared.produced(folded); + } + + let mut types = ZendHashTable::new(); + + for (name, code) in fold.types() { + ht_insert( + &mut types, + name, + call_static( + "Flow\\Types\\Type\\TypeFactory", + "fromString", + &mut [zval_str(code.as_bytes())], + )?, + ); + } + + let mut types_zv = Zval::new(); + types_zv.set_hashtable(types); + + call_static( + "Flow\\ETL\\Schema\\Inference\\ColumnTypes", + "fromColumnTypes", + &mut [types_zv, zval_long(fold.rows() as i64), typer.shallow_clone()], + ) + .map(ColumnTypesValue) + } +} + +/// The Rust fold is `StringTypeNarrower` without its HTML and XML rungs - a DOMDocument per cell is not worth porting. +fn folds_natively(typer: &Zval) -> PhpResult { + let narrower = match ( + typer.object(), + ext_php_rs::zend::ClassEntry::try_find(STRING_TYPE_NARROWER), + ) { + (Some(object), Some(ce)) => object.instance_of(ce), + _ => false, + }; + + if !narrower { + return Ok(false); + } + + for markup in ["Flow\\Types\\DSL\\type_html", "Flow\\Types\\DSL\\type_xml"] { + if call_method(typer, "emitsType", &mut [call_function(markup, &mut [])?])? + .bool() + .unwrap_or(false) + { + return Ok(false); + } + } + + Ok(true) +} + +fn headers(reader: &mut CsvReader) -> PhpResult { + let headers = reader.headers(); + let mut list = ZendHashTable::with_capacity(headers.len() as u32); + + for header in headers { + list.push(zval_str(header)) + .map_err(|e| ext_exception(format!("flow_php failed to collect a CSV header: {e:?}")))?; + } + + let mut zv = Zval::new(); + zv.set_hashtable(list); + + Ok(zv) +} diff --git a/src/extension/flow-php-ext/src/csv/tokenizer.rs b/src/extension/flow-php-ext/src/csv/tokenizer.rs index 1596d0ef35..54c959404f 100644 --- a/src/extension/flow-php-ext/src/csv/tokenizer.rs +++ b/src/extension/flow-php-ext/src/csv/tokenizer.rs @@ -156,7 +156,11 @@ impl Tokenizer { /// Advances the scan over unseen bytes; returns the position of the `\n` that ends the record. fn scan_to_record_end(&mut self) -> Option { - let Dialect { separator, enclosure, escape } = self.dialect; + let Dialect { + separator, + enclosure, + escape, + } = self.dialect; let bytes = &self.buffer; let mut position = self.scanned; let mut scan = self.scan; @@ -234,7 +238,11 @@ impl Tokenizer { /// `str_getcsv($record, ...)` into `record`. fn split(&self, range: Range, record: &mut Record) { - let Dialect { separator, enclosure, escape } = self.dialect; + let Dialect { + separator, + enclosure, + escape, + } = self.dialect; let escape = escape.filter(|escape| *escape != enclosure); let line = &self.buffer[range]; // next_record() already stripped every trailing \r and \n, so php_fgetcsv()'s line end is always empty here @@ -320,7 +328,9 @@ impl Tokenizer { } else { let delimiter = memchr(separator, &line[position..limit]).map(|offset| position + offset); - record.bytes.extend_from_slice(&line[position..delimiter.unwrap_or(limit)]); + record + .bytes + .extend_from_slice(&line[position..delimiter.unwrap_or(limit)]); let trimmed = field_start + trailing_line_end(&record.bytes[field_start..]); record.bytes.truncate(trimmed); diff --git a/src/extension/flow-php-ext/src/csv/write.rs b/src/extension/flow-php-ext/src/csv/write.rs new file mode 100644 index 0000000000..c09b9dca13 --- /dev/null +++ b/src/extension/flow-php-ext/src/csv/write.rs @@ -0,0 +1,217 @@ +//! `Flow\ETL\Adapter\CSV\RustCSVEncoder`: a `Rows` batch as the bytes `PhpCSVEncoder::encode()` returns, rendered +//! from the arrow arrays; the columns it does not render are the cells the held `PhpCSVEncoder::cells()` produces. + +use ext_php_rs::binary::Binary; +use ext_php_rs::binary_slice::BinarySlice; +use ext_php_rs::exception::PhpException; +use ext_php_rs::flags::ClassFlags; +use ext_php_rs::prelude::*; +use ext_php_rs::types::{ZendHashTable, Zval}; + +use crate::ctx::{call_method, find_class}; +use crate::exception::ext_exception; +use crate::interfaces::csv_encoder_ce; +use crate::render::invalid_argument; +use crate::text::batch::{batch, cells, renderer, Unrendered}; +use crate::text::value::Formats; +use crate::text::Error; + +/// `json_encode()`'s refusal of a nested cell, as `JSON_THROW_ON_ERROR` throws it. +fn json_exception(error: Error) -> PhpException { + match find_class("JsonException") { + Ok(ce) => PhpException::new(error.message().to_string(), error.code(), ce), + Err(e) => e, + } +} + +#[php_class] +#[php( + name = "Flow\\ETL\\Adapter\\CSV\\RustCSVEncoder", + flags = ClassFlags::Final, + implements(ce = csv_encoder_ce, stub = "Flow\\ETL\\Adapter\\CSV\\CSVEncoder") +)] +pub struct RustCSVEncoder { + separator: u8, + enclosure: u8, + escape: Option, + new_line: Vec, + /// The bytes that make `fputcsv()` enclose a field. + enclosing: [bool; 256], + /// The separator, the enclosure and the escape character cannot occur in a number, a boolean or a uuid. + plain_kinds: bool, + formats: Formats, + /// The `PhpCSVEncoder` that renders the unrendered columns and the header. + php: Zval, + unrendered: Unrendered, +} + +impl RustCSVEncoder { + /// One field as `fputcsv()` writes it (ext/standard/file.c `php_fputcsv()`). + fn field(&self, out: &mut Vec, text: &[u8]) { + if !text.iter().any(|byte| self.enclosing[usize::from(*byte)]) { + out.extend_from_slice(text); + + return; + } + + let mut escaped = false; + + out.push(self.enclosure); + + for byte in text { + if self.escape == Some(*byte) { + escaped = true; + } else if !escaped && *byte == self.enclosure { + out.push(self.enclosure); + } else { + escaped = false; + } + + out.push(*byte); + } + + out.push(self.enclosure); + } +} + +#[php_impl] +impl RustCSVEncoder { + pub fn __construct( + separator: BinarySlice, + enclosure: BinarySlice, + escape: BinarySlice, + new_line_separator: BinarySlice, + date_time_format: BinarySlice, + date_format: BinarySlice, + php: &Zval, + ) -> PhpResult { + if separator.len() != 1 { + return Err(invalid_argument("Separator must be a single character".to_string())); + } + + if enclosure.len() != 1 { + return Err(invalid_argument("Enclosure must be a single character".to_string())); + } + + if escape.len() > 1 { + return Err(invalid_argument( + "Escape must be empty or a single character".to_string(), + )); + } + + let (separator, enclosure, escape) = (separator[0], enclosure[0], escape.first().copied()); + let mut enclosing = [false; 256]; + + for byte in [ + Some(separator), + Some(enclosure), + escape, + Some(b'\n'), + Some(b'\r'), + Some(b'\t'), + Some(b' '), + ] + .into_iter() + .flatten() + { + enclosing[usize::from(byte)] = true; + } + + Ok(Self { + separator, + enclosure, + escape, + new_line: new_line_separator.to_vec(), + enclosing, + plain_kinds: ![Some(separator), Some(enclosure), escape] + .into_iter() + .flatten() + .any(|byte| byte.is_ascii_alphanumeric() || b"_+./:-".contains(&byte) || byte >= 0x80), + formats: Formats::parse(&date_time_format, &date_format), + php: php.shallow_clone(), + unrendered: Unrendered::default(), + }) + } + + /// The rows as `PhpCSVEncoder::encode()` writes them; the unrendered columns through the held encoder's `cells()`. + pub fn encode(&mut self, rows: &Zval) -> PhpResult> { + let cells = self.unrendered.render(rows, &self.php, "cells", &self.formats, true)?; + + self.render_rows(rows, &cells) + } + + #[php(name = "encodeHeader")] + pub fn encode_header(&self, headers: &Zval) -> PhpResult> { + call_method(&self.php, "encodeHeader", &mut [headers.shallow_clone()])? + .zend_str() + .map(|header| Binary::new(header.as_bytes())) + .ok_or_else(|| ext_exception("flow_php expected PhpCSVEncoder::encodeHeader() to return a string")) + } +} + +impl RustCSVEncoder { + /// `$cells_of`: `PhpCSVEncoder::cells()` of the unrendered columns, each of `$rows->count()` values. + fn render_rows(&self, rows: &Zval, cells_of: &ZendHashTable) -> PhpResult> { + let batch = batch(rows, &self.formats, true)?; + let mut fields = Vec::with_capacity(batch.columns.len()); + let mut scratch = Vec::new(); + + for column in &batch.columns { + let mut arena = Vec::with_capacity(batch.count * 16); + let mut ends = Vec::with_capacity(batch.count); + + match &column.native { + Some((plan, data)) => { + let renderer = renderer(plan, data, &self.formats)?; + let plain = self.plain_kinds && renderer.is_plain(); + + for i in 0..batch.count { + if !renderer.is_null(i) { + if plain { + renderer.text(&mut arena, i).map_err(json_exception)?; + } else if let Some(text) = renderer.bytes(i) { + self.field(&mut arena, text); + } else { + scratch.clear(); + renderer.text(&mut scratch, i).map_err(json_exception)?; + self.field(&mut arena, &scratch); + } + } + + ends.push(arena.len()); + } + } + None => { + for cell in cells(cells_of, &column.name, batch.count, "cells")? { + if let Some(text) = cell { + self.field(&mut arena, text); + } + + ends.push(arena.len()); + } + } + } + + fields.push((arena, ends)); + } + + let mut out = Vec::with_capacity( + fields.iter().map(|(arena, _)| arena.len()).sum::() + + batch.count * (fields.len() + self.new_line.len()), + ); + + for i in 0..batch.count { + for (position, (arena, ends)) in fields.iter().enumerate() { + if position > 0 { + out.push(self.separator); + } + + out.extend_from_slice(&arena[if i == 0 { 0 } else { ends[i - 1] }..ends[i]]); + } + + out.extend_from_slice(&self.new_line); + } + + Ok(Binary::new(out)) + } +} diff --git a/src/extension/flow-php-ext/src/ctx.rs b/src/extension/flow-php-ext/src/ctx.rs index ebca596f0d..f57a522d9b 100644 --- a/src/extension/flow-php-ext/src/ctx.rs +++ b/src/extension/flow-php-ext/src/ctx.rs @@ -1,33 +1,22 @@ //! PHP-engine plumbing: class-entry/slot lookups, hashtable helpers, calls and -//! per-instance caches (timezones, enums, callables). +//! request-scoped caches (timezones, enums, callables). use std::collections::{HashMap, HashSet}; +use std::rc::Rc; use ext_php_rs::boxed::ZBox; -use ext_php_rs::convert::IntoZvalDyn; use ext_php_rs::exception::PhpException; use ext_php_rs::ffi::{ - _zend_property_info, zend_call_known_function, zend_hash_find, zend_hash_index_update, - zend_hash_str_update, zend_hash_update, zend_ulong, + _zend_property_info, zend_call_known_function, zend_hash_index_update, zend_hash_str_find, zend_hash_str_update, + zend_hash_update, zend_ulong, }; use ext_php_rs::flags::ClassFlags; use ext_php_rs::types::{ZendHashTable, ZendObject, ZendStr, Zval}; use ext_php_rs::zend::{ClassEntry, ExecutorGlobals, Function}; use crate::exception::ext_exception; - -// Not in ext-php-rs's allowed bindings; ZEND_API symbols with the same ABI as -// the bound zend_hash_*_update functions. -extern "C" { - fn zend_hash_str_add( - ht: *mut ZendHashTable, - key: *const std::ffi::c_char, - len: usize, - data: *mut Zval, - ) -> *mut Zval; - - fn zend_hash_index_add(ht: *mut ZendHashTable, h: zend_ulong, data: *mut Zval) -> *mut Zval; -} +use crate::globals::with_ctx; +use crate::plan::TypePlan; /// PHP array-key coercion (`_zend_handle_numeric_str`): parse + canonical /// re-format equality rejects "+5", "-0", leading zeros and whitespace exactly @@ -48,12 +37,7 @@ pub fn ht_insert(ht: &mut ZendHashTable, key: &[u8], mut value: Zval) { zend_hash_index_update(ht, index as u64, std::ptr::from_mut(&mut value)); } None => { - zend_hash_str_update( - ht, - key.as_ptr().cast(), - key.len(), - std::ptr::from_mut(&mut value), - ); + zend_hash_str_update(ht, key.as_ptr().cast(), key.len(), std::ptr::from_mut(&mut value)); } } } @@ -70,15 +54,6 @@ pub enum HtKey<'a> { Str(&'a ZendStr), } -pub fn ht_find_key<'a>(ht: &'a ZendHashTable, key: &HtKey<'_>) -> Option<&'a Zval> { - match key { - HtKey::Index(index) => ht.get_index(*index), - HtKey::Str(name) => unsafe { - zend_hash_find(ht, std::ptr::from_ref(*name).cast_mut()).as_ref() - }, - } -} - /// [`ht_insert`] for a pre-resolved key; ownership of `value` moves into the /// table. pub fn ht_insert_key(ht: &mut ZendHashTable, key: &HtKey<'_>, mut value: Zval) { @@ -88,11 +63,7 @@ pub fn ht_insert_key(ht: &mut ZendHashTable, key: &HtKey<'_>, mut value: Zval) { zend_hash_index_update(ht, *index as u64, std::ptr::from_mut(&mut value)); } HtKey::Str(name) => { - zend_hash_update( - ht, - std::ptr::from_ref(*name).cast_mut(), - std::ptr::from_mut(&mut value), - ); + zend_hash_update(ht, std::ptr::from_ref(*name).cast_mut(), std::ptr::from_mut(&mut value)); } } } @@ -100,7 +71,7 @@ pub fn ht_insert_key(ht: &mut ZendHashTable, key: &HtKey<'_>, mut value: Zval) { std::mem::forget(value); } -pub fn ht_insert_index(ht: &mut ZendHashTable, index: i64, mut value: Zval) { +pub fn ht_insert_long(ht: &mut ZendHashTable, index: i64, mut value: Zval) { unsafe { zend_hash_index_update(ht, index as u64, std::ptr::from_mut(&mut value)); } @@ -108,29 +79,12 @@ pub fn ht_insert_index(ht: &mut ZendHashTable, index: i64, mut value: Zval) { std::mem::forget(value); } -/// Inserts only when the key is absent (PHP array-key coercion applies) and -/// reports whether it did; on `false` the value is dropped, not inserted. -pub fn ht_add(ht: &mut ZendHashTable, key: &[u8], mut value: Zval) -> bool { - let added = unsafe { - match array_key_index(key) { - Some(index) => { - !zend_hash_index_add(ht, index as u64, std::ptr::from_mut(&mut value)).is_null() - } - None => !zend_hash_str_add( - ht, - key.as_ptr().cast(), - key.len(), - std::ptr::from_mut(&mut value), - ) - .is_null(), - } - }; - - if added { - std::mem::forget(value); +/// `$array[$key]` with PHP array-key coercion. +pub fn ht_get<'a>(ht: &'a ZendHashTable, key: &[u8]) -> Option<&'a Zval> { + match array_key_index(key) { + Some(index) => ht.get_index(index), + None => unsafe { zend_hash_str_find(ht, key.as_ptr().cast(), key.len()).as_ref() }, } - - added } pub fn zval_str(bytes: &[u8]) -> Zval { @@ -151,30 +105,9 @@ pub fn null_zval() -> Zval { zv } -/// `throw new SchemaMismatchException($rowIndex, )`, where the cause comes from a -/// `ColumnMismatchException` factory - the PHP side authors every schema-mismatch message. -pub fn schema_mismatch( - schema_mismatch_ce: &'static ClassEntry, - factory: &'static Function, - row_index: u64, - args: &mut [Zval], -) -> Result { - let cause = call_handle_transparent(factory, None, args)?; - let mut wrapped = construct_with_zvals( - schema_mismatch_ce, - &mut [zval_long(row_index as i64), cause], - "a SchemaMismatchException", - )?; - - Ok(transparent_exception(&mut wrapped)) -} - pub fn find_class(name: &str) -> Result<&'static ClassEntry, PhpException> { - ClassEntry::try_find(name).ok_or_else(|| { - ext_exception(format!( - "flow_php requires class \"{name}\" to be autoloadable" - )) - }) + ClassEntry::try_find(name) + .ok_or_else(|| ext_exception(format!("flow_php requires class \"{name}\" to be autoloadable"))) } /// Fails when the last engine call left an exception pending, so PHP-level @@ -189,11 +122,7 @@ pub fn ensure_no_pending_exception(context: &str) -> Result<(), PhpException> { /// Writes one property through the object's write handler, which copies the /// value with assign semantics - the caller keeps ownership of `value`. -pub fn write_property_raw( - obj: &mut ZendObject, - name: &mut ZendStr, - mut value: Zval, -) -> Result<(), PhpException> { +pub fn write_property_raw(obj: &mut ZendObject, name: &mut ZendStr, mut value: Zval) -> Result<(), PhpException> { let handler = unsafe { obj.handlers.as_ref() } .and_then(|handlers| handlers.write_property) .ok_or_else(|| ext_exception("flow_php failed to resolve a property write handler"))?; @@ -233,16 +162,13 @@ pub fn property_offset(ce: &ClassEntry, name: &str) -> Result /// a plain userland class when the zval matches the declared type exactly. pub fn write_slot(obj: &mut ZendObject, offset: u32, value: Zval) { unsafe { - let slot = std::ptr::from_mut(obj) - .cast::() - .add(offset as usize) - .cast::(); + let slot = std::ptr::from_mut(obj).cast::().add(offset as usize).cast::(); std::ptr::write(slot, value); } } -/// Reads one property with `BP_VAR_R` semantics; `ZendObject::get_property` -/// passes `BP_VAR_W`, which trips the readonly-property guard on `HydratorColumn`. +/// Reads one property with `BP_VAR_R` semantics; `ZendObject::get_property` passes `BP_VAR_W`, which trips the +/// readonly-property guard. pub fn read_property(obj: &ZendObject, name: &str) -> Result { let handler = unsafe { obj.handlers.as_ref() } .and_then(|handlers| handlers.read_property) @@ -267,70 +193,38 @@ pub fn read_property(obj: &ZendObject, name: &str) -> Result Ok(value.shallow_clone()) } -pub fn construct_object( - ce: &'static ClassEntry, - args: Vec<&dyn IntoZvalDyn>, - context: &str, -) -> Result, PhpException> { - let obj = ZendObject::new(ce); - - obj.try_call_method("__construct", args) - .map_err(|e| ext_exception(format!("flow_php failed to construct {context}: {e:?}")))?; - ensure_no_pending_exception(&format!("construct {context}"))?; +/// `$object->$method(...$args)`; an exception the method throws surfaces as itself. +pub fn call_method(object: &Zval, method: &str, args: &mut [Zval]) -> Result { + let object = expect_object(object, "a method receiver")?; + let ce = + unsafe { object.ce.as_ref() }.ok_or_else(|| ext_exception("flow_php failed to resolve an object class"))?; - Ok(obj) + call_handle_transparent(ce_method_ref(ce, method)?, Some(object), args) } -/// Constructs a fresh object and calls its `__construct` with already-owned -/// zvals (what `construct_object` can't express - it takes `IntoZvalDyn`). Used -/// to call the canonical `Row`/`Rows` constructors from row-object graphs. -pub fn construct_with_zvals( - ce: &'static ClassEntry, - args: &mut [Zval], - context: &str, -) -> Result, PhpException> { - let mut obj = ZendObject::new(ce); - let constructor = ce_method_ref(ce, "__construct")?; - - call_handle( - constructor, - Some(&mut obj), - args, - &format!("construct {context}"), - )?; - - Ok(obj) +/// `$class::$method(...$args)`; an exception the method throws surfaces as itself. +pub fn call_static(class: &str, method: &str, args: &mut [Zval]) -> Result { + call_handle_transparent(ce_method_ref(find_class(class)?, method)?, None, args) } -fn method_handle(class: &str, method: &str) -> Result { - Function::try_from_method(class, method).ok_or_else(|| { - ext_exception(format!( - "flow_php failed to resolve method {class}::{method}" - )) - }) -} +/// `new $ce(...$args)` with already-owned zvals; an exception the constructor throws surfaces as itself. +pub fn construct(ce: &'static ClassEntry, args: &mut [Zval]) -> Result, PhpException> { + let object = ZendObject::new(ce); + call_handle_transparent(ce_method_ref(ce, "__construct")?, Some(&object), args)?; -fn function_handle(name: &str) -> Result { - Function::try_from_function(name) - .ok_or_else(|| ext_exception(format!("flow_php failed to resolve function {name}"))) + Ok(object) } -/// Resolves a method to a reference into the class function table WITHOUT -/// copying the zend_function: the engine writes into userland op_arrays -/// (run-time cache), and a copy discards those writes on every call. -fn method_handle_ref(class: &str, method: &str) -> Result<&'static Function, PhpException> { - ce_method_ref(find_class(class)?, method) +/// A hashtable key as a column name: an int key is a numeric column name. +pub fn column_key(name: Option<&ZendStr>, index: zend_ulong) -> Vec { + name.map_or_else(|| index.to_string().into_bytes(), |name| name.as_bytes().to_vec()) } pub fn ce_method_ref(ce: &ClassEntry, method: &str) -> Result<&'static Function, PhpException> { let function = unsafe { - ext_php_rs::ffi::zend_hash_str_find_ptr_lc( - &raw const ce.function_table, - method.as_ptr().cast(), - method.len(), - ) - .cast::() - .as_ref() + ext_php_rs::ffi::zend_hash_str_find_ptr_lc(&raw const ce.function_table, method.as_ptr().cast(), method.len()) + .cast::() + .as_ref() }; function.ok_or_else(|| { @@ -402,13 +296,13 @@ pub fn call_handle_catching( } } -/// Surfaces a PHP exception object as ITSELF rather than wrapped in an -/// `ExtensionException`, so the class and message PHP sees are the ones thrown. +/// Surfaces a PHP exception object as ITSELF rather than wrapped in a +/// `RuntimeException`, so the class and message PHP sees are the ones thrown. pub fn transparent_exception(exception: &mut ZendObject) -> PhpException { let mut zv = Zval::new(); zv.set_object(exception); - PhpException::default(String::new()).with_object(zv) + PhpException::from_message(String::new()).with_object(zv) } /// [`call_handle_catching`] with the exception object already re-raised as @@ -420,8 +314,7 @@ pub fn call_handle_transparent( object: Option<&ZendObject>, args: &mut [Zval], ) -> Result { - call_handle_catching(func, object, args) - .map_err(|mut exception| transparent_exception(&mut exception)) + call_handle_catching(func, object, args).map_err(|mut exception| transparent_exception(&mut exception)) } /// Calls a pre-resolved function handle. Argument zvals stay caller-owned @@ -459,513 +352,452 @@ pub fn call_handle( Ok(retval) } -pub struct DateTimeFns { - pub ce: &'static ClassEntry, - pub set_timezone: Function, -} - -pub struct IntervalFns { - pub ce: &'static ClassEntry, - pub construct: Function, - pub spec: Zval, - pub names: [ZBox; 8], -} - +/// Request-scoped caches of class entries, function handles and value zvals, reached only through [`with_ctx`]: +/// every function below reads and writes the cache inside the closure and calls PHP (autoload, constructors, +/// functions) outside it, so PHP code re-entering flow_php never meets a borrowed `Ctx`. +#[derive(Default)] pub struct Ctx { - uuid_ce: Option<(&'static ClassEntry, u32)>, - json_ce: Option<(&'static ClassEntry, u32, u32)>, + uuid: Option<(&'static ClassEntry, u32)>, + json: Option<(&'static ClassEntry, u32, u32)>, interval: Option, - timezone_ce: Option<&'static ClassEntry>, + timezone: Option, datetime_immutable: Option, - datetime_mutable: Option, + datetime_get_timezone: HashMap, + datetime_cast: HashMap, timezones: HashMap, Zval>, enums: HashMap, Zval>, - fn_defined: Option, - fn_constant: Option, - value_decoder_statics: HashMap<&'static str, &'static Function>, - value_encoder_statics: HashMap<&'static str, &'static Function>, - fn_json_encode: Option, - fn_json_decode: Option, - fn_json_validate: Option, - fn_date_parse: Option, + functions: HashMap<&'static str, &'static Function>, timezone_identifiers: Option>>, - timezone_construct: Option<&'static Function>, - metadata_from_array: Option<&'static Function>, - metadata_map_slot: Option, - typed_row_values_slots: Option<(u32, u32)>, - schema_set_metadata: Option<&'static Function>, - schema_find_definition: Option<&'static Function>, - timezone_get_name: Option<&'static Function>, - datetime_encode: HashMap, - datetime_cast: HashMap, - save_html: HashMap, - format_u: Option, - format_his: Option, - str_true: Option, - str_false: Option, + strings: HashMap<&'static [u8], Zval>, + instances: HashMap<&'static str, Zval>, + plans: HashMap, Rc>, + /// `plans` by Type object handle; each entry holds its object, so the handle is not reused while cached. + type_plans: HashMap)>, } -pub struct DateTimeEncFns { - pub get_timestamp: &'static Function, - pub format: &'static Function, - pub get_timezone: &'static Function, +/// Type objects `type_plans` holds before it starts over. +const TYPE_PLANS: usize = 1024; + +#[derive(Clone, Copy)] +pub struct DateTimeFns { + pub ce: &'static ClassEntry, + pub set_timezone: &'static Function, +} + +#[derive(Clone, Copy)] +pub struct IntervalFns { + pub ce: &'static ClassEntry, + pub construct: &'static Function, } +#[derive(Clone, Copy)] +pub struct TimezoneFns { + pub ce: &'static ClassEntry, + pub construct: &'static Function, + pub get_name: &'static Function, +} + +#[derive(Clone, Copy)] pub struct DateTimeCastFns { pub set_time: &'static Function, pub format: &'static Function, } -impl Ctx { - pub fn new() -> Result { - Ok(Self { - uuid_ce: None, - json_ce: None, - interval: None, - timezone_ce: None, - datetime_immutable: None, - datetime_mutable: None, - timezones: HashMap::new(), - enums: HashMap::new(), - fn_defined: None, - fn_constant: None, - value_decoder_statics: HashMap::new(), - value_encoder_statics: HashMap::new(), - fn_json_encode: None, - fn_json_decode: None, - fn_json_validate: None, - fn_date_parse: None, - timezone_identifiers: None, - timezone_construct: None, - metadata_from_array: None, - metadata_map_slot: None, - typed_row_values_slots: None, - schema_set_metadata: None, - schema_find_definition: None, - timezone_get_name: None, - datetime_encode: HashMap::new(), - datetime_cast: HashMap::new(), - save_html: HashMap::new(), - format_u: None, - format_his: None, - str_true: None, - str_false: None, - }) - } +/// One instance of a stateless PHP class (a `Physical`, `PhysicalFor`), constructed without arguments. +pub fn instance(class: &'static str) -> Result { + cached( + |ctx| ctx.instances.get(class).map(Zval::shallow_clone), + || { + let ce = find_class(class)?; + let mut object = ZendObject::new(ce); - /// Property-table slot offsets `(values, metadata)` on `Flow\ETL\Row\TypedRowValues`. - pub fn typed_row_values_slots(&mut self) -> Result<(u32, u32), PhpException> { - if self.typed_row_values_slots.is_none() { - let ce = find_class("Flow\\ETL\\Row\\TypedRowValues")?; - self.typed_row_values_slots = - Some((property_offset(ce, "values")?, property_offset(ce, "metadata")?)); - } + if !ce.constructor.is_null() { + call_handle_transparent(ce_method_ref(ce, "__construct")?, Some(&object), &mut [])?; + } - Ok(self.typed_row_values_slots.expect("just initialized")) - } + let mut zv = Zval::new(); + zv.set_object(&mut object); - /// `Schema::setMetadata` / `Schema::findDefinition` - the native hydrate and cast paths fold - /// `RawRowValues::metadata` into the batch Schema exactly like `HydratedBatch` does in PHP. - pub fn schema_set_metadata(&mut self) -> Result<&'static Function, PhpException> { - if self.schema_set_metadata.is_none() { - self.schema_set_metadata = Some(method_handle_ref("Flow\\ETL\\Schema", "setMetadata")?); - } + Ok(zv) + }, + |ctx, zv| { + ctx.instances.insert(class, zv.shallow_clone()); + }, + ) +} - Ok(self.schema_set_metadata.expect("just initialized")) - } +pub fn plan(json: &[u8]) -> Result>, PhpException> { + with_ctx(|ctx| Ok(ctx.plans.get(json).cloned())) +} - pub fn schema_find_definition(&mut self) -> Result<&'static Function, PhpException> { - if self.schema_find_definition.is_none() { - self.schema_find_definition = - Some(method_handle_ref("Flow\\ETL\\Schema", "findDefinition")?); - } +pub fn store_plan(json: Vec, plan: &Rc) -> Result<(), PhpException> { + with_ctx(|ctx| { + ctx.plans.insert(json, Rc::clone(plan)); + Ok(()) + }) +} - Ok(self.schema_find_definition.expect("just initialized")) - } +/// The plan of a Type object seen before; Types are immutable, so the object names its plan. +pub fn type_plan_of(type_zv: &Zval) -> Result>, PhpException> { + let Some(handle) = type_zv.object().map(|object| object.handle) else { + return Ok(None); + }; - pub fn metadata_map_slot(&mut self) -> Result { - if self.metadata_map_slot.is_none() { - self.metadata_map_slot = Some(property_offset( - find_class("Flow\\ETL\\Schema\\Metadata")?, - "map", - )?); - } + with_ctx(|ctx| Ok(ctx.type_plans.get(&handle).map(|(_, plan)| Rc::clone(plan)))) +} - Ok(self.metadata_map_slot.expect("just initialized")) - } +pub fn store_type_plan(type_zv: &Zval, plan: &Rc) -> Result<(), PhpException> { + let Some(handle) = type_zv.object().map(|object| object.handle) else { + return Ok(()); + }; - pub fn timezone_get_name(&mut self) -> Result<&'static Function, PhpException> { - if self.timezone_get_name.is_none() { - self.timezone_get_name = Some(method_handle_ref("DateTimeZone", "getName")?); - } + // the evicted objects are released outside the context: freeing one may run PHP code + let _evicted = with_ctx(|ctx| { + let evicted = if ctx.type_plans.len() >= TYPE_PLANS { + std::mem::take(&mut ctx.type_plans) + } else { + HashMap::new() + }; + ctx.type_plans + .insert(handle, (type_zv.shallow_clone(), Rc::clone(plan))); - Ok(self.timezone_get_name.expect("just initialized")) - } + Ok(evicted) + })?; - /// Per-datetime-class encode handles (getTimestamp/format/getTimezone). - pub fn datetime_encode_fns( - &mut self, - ce: *const ClassEntry, - ) -> Result<&DateTimeEncFns, PhpException> { - let key = ce as usize; - - if let std::collections::hash_map::Entry::Vacant(entry) = self.datetime_encode.entry(key) { - let ce_ref = unsafe { ce.as_ref() } - .ok_or_else(|| ext_exception("flow_php failed to resolve a datetime class"))?; - - entry.insert(DateTimeEncFns { - get_timestamp: ce_method_ref(ce_ref, "getTimestamp")?, - format: ce_method_ref(ce_ref, "format")?, - get_timezone: ce_method_ref(ce_ref, "getTimezone")?, - }); - } + Ok(()) +} - Ok(self.datetime_encode.get(&key).expect("just inserted")) +/// A cached handle: read inside the context, resolved outside it on a miss, then stored. +fn cached( + read: impl FnOnce(&Ctx) -> Option, + resolve: impl FnOnce() -> Result, + store: impl FnOnce(&mut Ctx, &T), +) -> Result { + if let Some(value) = with_ctx(|ctx| Ok(read(ctx)))? { + return Ok(value); } - pub fn save_html(&mut self, ce: *const ClassEntry) -> Result<&'static Function, PhpException> { - let key = ce as usize; + let value = resolve()?; + with_ctx(|ctx| { + store(ctx, &value); + Ok(()) + })?; - if let std::collections::hash_map::Entry::Vacant(entry) = self.save_html.entry(key) { - let ce_ref = unsafe { ce.as_ref() } - .ok_or_else(|| ext_exception("flow_php failed to resolve an HTML class"))?; - entry.insert(ce_method_ref(ce_ref, "saveHtml")?); - } - - Ok(self.save_html.get(&key).expect("just inserted")) - } + Ok(value) +} - /// Cached `"u"` format-string zval for microsecond reads. - pub fn format_u(&mut self) -> Result<&Zval, PhpException> { - if self.format_u.is_none() { - self.format_u = Some(zval_str(b"u")); - } +/// An internal PHP function; the engine's function table outlives every request. +fn function(name: &'static str) -> Result<&'static Function, PhpException> { + cached( + |ctx| ctx.functions.get(name).copied(), + || { + let table = ExecutorGlobals::get().function_table; - Ok(self.format_u.as_ref().expect("just initialized")) - } + let found = unsafe { ext_php_rs::ffi::zend_hash_str_find_ptr_lc(table, name.as_ptr().cast(), name.len()) } + .cast::(); - /// Cached `"H:i:s.u"` format-string zval for the date midnight check. - pub fn format_his(&mut self) -> Result<&Zval, PhpException> { - if self.format_his.is_none() { - self.format_his = Some(zval_str(b"H:i:s.u")); - } + unsafe { found.as_ref() } + .ok_or_else(|| ext_exception(format!("flow_php failed to resolve function {name}"))) + }, + |ctx, function| { + ctx.functions.insert(name, function); + }, + ) +} - Ok(self.format_his.as_ref().expect("just initialized")) - } +/// `$name(...$args)` of a PHP function; an exception it throws surfaces as itself. +pub fn call_function(name: &'static str, args: &mut [Zval]) -> Result { + call_handle_transparent(function(name)?, None, args) +} - /// Cached `"true"`/`"false"` string zvals for bool-to-string casts. - pub fn bool_str(&mut self, value: bool) -> Result<&Zval, PhpException> { - let slot = if value { - &mut self.str_true - } else { - &mut self.str_false - }; +pub fn json_encode() -> Result<&'static Function, PhpException> { + function("json_encode") +} - if slot.is_none() { - *slot = Some(zval_str(if value { b"true" } else { b"false" })); - } +pub fn json_decode() -> Result<&'static Function, PhpException> { + function("json_decode") +} - Ok(slot.as_ref().expect("just initialized")) - } +pub fn json_validate() -> Result<&'static Function, PhpException> { + function("json_validate") +} - /// Per-datetime-class cast handles (setTime/format), subclass-safe. - pub fn datetime_cast_fns( - &mut self, - ce: *const ClassEntry, - ) -> Result<&DateTimeCastFns, PhpException> { - let key = ce as usize; - - if let std::collections::hash_map::Entry::Vacant(entry) = self.datetime_cast.entry(key) { - let ce_ref = unsafe { ce.as_ref() } - .ok_or_else(|| ext_exception("flow_php failed to resolve a datetime class"))?; - - entry.insert(DateTimeCastFns { - set_time: ce_method_ref(ce_ref, "setTime")?, - format: ce_method_ref(ce_ref, "format")?, - }); - } +pub fn date_parse() -> Result<&'static Function, PhpException> { + function("date_parse") +} - Ok(self.datetime_cast.get(&key).expect("just inserted")) - } +/// A cached string zval, e.g. a format string. +pub fn string(bytes: &'static [u8]) -> Result { + cached( + |ctx| ctx.strings.get(bytes).map(Zval::shallow_clone), + || Ok(zval_str(bytes)), + |ctx, zv| { + ctx.strings.insert(bytes, zv.shallow_clone()); + }, + ) +} - /// Cached handles to `ValueEncoder`'s public static markup serializers. - pub fn value_encoder_static( - &mut self, - method: &'static str, - ) -> Result<&'static Function, PhpException> { - if !self.value_encoder_statics.contains_key(method) { - let handle = method_handle_ref("Flow\\Floe\\ValueEncoder", method)?; - self.value_encoder_statics.insert(method, handle); - } +/// `"true"`/`"false"` for bool-to-string casts. +pub fn bool_str(value: bool) -> Result { + string(if value { b"true" } else { b"false" }) +} - Ok(self - .value_encoder_statics - .get(method) - .expect("just inserted")) - } +pub fn uuid() -> Result<(&'static ClassEntry, u32), PhpException> { + cached( + |ctx| ctx.uuid, + || { + let ce = find_class("Flow\\Types\\Value\\Uuid")?; + Ok((ce, property_offset(ce, "value")?)) + }, + |ctx, uuid| ctx.uuid = Some(*uuid), + ) +} - pub fn json(&mut self) -> Result<(&'static ClassEntry, u32, u32), PhpException> { - if self.json_ce.is_none() { +pub fn json() -> Result<(&'static ClassEntry, u32, u32), PhpException> { + cached( + |ctx| ctx.json, + || { let ce = find_class("Flow\\Types\\Value\\Json")?; - self.json_ce = Some(( - ce, - property_offset(ce, "value")?, - property_offset(ce, "isObject")?, - )); - } - - Ok(self.json_ce.expect("just initialized")) - } - - /// Enum cases cached per `Class::Case`, resolved with the same validation - /// and errors as `ValueDecoder` (enum_exists + defined + constant). - pub fn enum_case(&mut self, class: &[u8], case: &[u8]) -> Result { - let mut cache_key = Vec::with_capacity(class.len() + case.len() + 2); - cache_key.extend_from_slice(class); - cache_key.extend_from_slice(b"::"); - cache_key.extend_from_slice(case); - - if let Some(cached) = self.enums.get(&cache_key) { - return Ok(cached.shallow_clone()); - } - - let constant_name = std::str::from_utf8(&cache_key) - .map_err(|_| ext_exception("flow_php expected an enum name to be UTF-8"))? - .to_string(); - let class_name = std::str::from_utf8(class) - .expect("validated above") - .to_string(); - let case_name = std::str::from_utf8(case) - .expect("validated above") - .to_string(); - - let is_enum = ClassEntry::try_find(&class_name) - .is_some_and(|ce| ce.flags().contains(ClassFlags::Enum)); - - if !is_enum { - return Err(ext_exception(format!( - "flow_php cannot restore enum of class \"{class_name}\", enum not found" - ))); - } + Ok((ce, property_offset(ce, "value")?, property_offset(ce, "isObject")?)) + }, + |ctx, json| ctx.json = Some(*json), + ) +} - if self.fn_defined.is_none() { - self.fn_defined = Some(function_handle("defined")?); - self.fn_constant = Some(function_handle("constant")?); - } +pub fn interval() -> Result { + cached( + |ctx| ctx.interval, + || { + let ce = find_class("DateInterval")?; + Ok(IntervalFns { + ce, + construct: ce_method_ref(ce, "__construct")?, + }) + }, + |ctx, interval| ctx.interval = Some(*interval), + ) +} - let constant_name_zv = zval_str(constant_name.as_bytes()); +pub fn datetime_immutable() -> Result { + cached( + |ctx| ctx.datetime_immutable, + || { + let ce = find_class("DateTimeImmutable")?; + Ok(DateTimeFns { + ce, + set_timezone: ce_method_ref(ce, "setTimezone")?, + }) + }, + |ctx, fns| ctx.datetime_immutable = Some(*fns), + ) +} - let defined = call_handle( - &self.fn_defined.expect("just initialized"), - None, - &mut [constant_name_zv.shallow_clone()], - "check an enum case", - )?; +fn timezone_fns() -> Result { + cached( + |ctx| ctx.timezone, + || { + let ce = find_class("DateTimeZone")?; + Ok(TimezoneFns { + ce, + construct: ce_method_ref(ce, "__construct")?, + get_name: ce_method_ref(ce, "getName")?, + }) + }, + |ctx, fns| ctx.timezone = Some(*fns), + ) +} - if !defined.bool().unwrap_or(false) { - return Err(ext_exception(format!( - "flow_php cannot restore enum case \"{class_name}::{case_name}\"" - ))); - } +pub fn timezone_get_name() -> Result<&'static Function, PhpException> { + Ok(timezone_fns()?.get_name) +} - let case_zv = call_handle( - &self.fn_constant.expect("just initialized"), - None, - &mut [constant_name_zv], - "restore an enum case", - )?; - - if !case_zv.is_object() { - return Err(ext_exception(format!( - "flow_php cannot restore enum case \"{class_name}::{case_name}\"" - ))); - } +/// `getTimezone` of a datetime class, subclass-safe. +pub fn datetime_get_timezone(ce: &'static ClassEntry) -> Result<&'static Function, PhpException> { + let key = std::ptr::from_ref(ce) as usize; - let result = case_zv.shallow_clone(); - self.enums.insert(cache_key, case_zv); + cached( + |ctx| ctx.datetime_get_timezone.get(&key).copied(), + || ce_method_ref(ce, "getTimezone"), + |ctx, function| { + ctx.datetime_get_timezone.insert(key, function); + }, + ) +} - Ok(result) - } +/// `setTime`/`format` of a datetime class, subclass-safe. +pub fn datetime_cast_fns(ce: &'static ClassEntry) -> Result { + let key = std::ptr::from_ref(ce) as usize; - /// Cached handles to `ValueDecoder`'s public static markup restorers. - pub fn value_decoder_static( - &mut self, - method: &'static str, - ) -> Result<&'static Function, PhpException> { - if !self.value_decoder_statics.contains_key(method) { - let handle = method_handle_ref("Flow\\Floe\\ValueDecoder", method)?; - self.value_decoder_statics.insert(method, handle); - } + cached( + |ctx| ctx.datetime_cast.get(&key).copied(), + || { + Ok(DateTimeCastFns { + set_time: ce_method_ref(ce, "setTime")?, + format: ce_method_ref(ce, "format")?, + }) + }, + |ctx, fns| { + ctx.datetime_cast.insert(key, *fns); + }, + ) +} + +/// `new DateTimeZone($name)`, one instance per name. +pub fn timezone(name: &[u8]) -> Result { + cached( + |ctx| ctx.timezones.get(name).map(Zval::shallow_clone), + || { + let fns = timezone_fns()?; + let mut timezone = ZendObject::new(fns.ce); + call_handle( + fns.construct, + Some(&mut timezone), + &mut [zval_str(name)], + "construct DateTimeZone", + )?; - Ok(self - .value_decoder_statics - .get(method) - .expect("just inserted")) - } + let mut zv = Zval::new(); + zv.set_object(&mut timezone); - pub fn uuid(&mut self) -> Result<(&'static ClassEntry, u32), PhpException> { - if self.uuid_ce.is_none() { - let ce = find_class("Flow\\Types\\Value\\Uuid")?; - self.uuid_ce = Some((ce, property_offset(ce, "value")?)); - } + Ok(zv) + }, + |ctx, zv| { + ctx.timezones.insert(name.to_vec(), zv.shallow_clone()); + }, + ) +} - Ok(self.uuid_ce.expect("just initialized")) - } +/// Whether `new DateTimeZone($name)` succeeds; the exception it throws otherwise is discarded. +pub fn timezone_accepts(name: &[u8]) -> Result { + let fns = timezone_fns()?; + let timezone = ZendObject::new(fns.ce); - pub fn interval(&mut self) -> Result<&mut IntervalFns, PhpException> { - if self.interval.is_none() { - self.interval = Some(IntervalFns { - ce: find_class("DateInterval")?, - construct: method_handle("DateInterval", "__construct")?, - spec: zval_str(b"PT0S"), - names: [ - ZendStr::new("y", false), - ZendStr::new("m", false), - ZendStr::new("d", false), - ZendStr::new("h", false), - ZendStr::new("i", false), - ZendStr::new("s", false), - ZendStr::new("invert", false), - ZendStr::new("f", false), - ], - }); - } + Ok(call_handle_catching(fns.construct, Some(&timezone), &mut [zval_str(name)]).is_ok()) +} - Ok(self.interval.as_mut().expect("just initialized")) +/// Whether `$name` is in `DateTimeZone::listIdentifiers()`, fetched once per request. +pub fn is_timezone_identifier(name: &[u8]) -> Result { + if let Some(known) = with_ctx(|ctx| Ok(ctx.timezone_identifiers.as_ref().map(|ids| ids.contains(name))))? { + return Ok(known); } - /// Datetime class entry + pre-resolved setTimezone handle per class. - pub fn datetime_fns(&mut self, mutable: bool) -> Result<&DateTimeFns, PhpException> { - let slot = if mutable { - &mut self.datetime_mutable - } else { - &mut self.datetime_immutable - }; + let list = call_handle( + ce_method_ref(timezone_fns()?.ce, "listIdentifiers")?, + None, + &mut [], + "list timezone identifiers", + )?; + let identifiers: HashSet> = list + .array() + .ok_or_else(|| ext_exception("flow_php expected DateTimeZone::listIdentifiers() to return an array"))? + .values() + .filter_map(|zv| zv.zend_str().map(|name| name.as_bytes().to_vec())) + .collect(); + let known = identifiers.contains(name); + + with_ctx(|ctx| { + ctx.timezone_identifiers = Some(identifiers); + Ok(()) + })?; + + Ok(known) +} + +/// An enum case, cached per `Class::Case`, resolved with `enum_exists` + `defined` + `constant`. +pub fn enum_case(class: &[u8], case: &[u8]) -> Result { + let mut constant_name = Vec::with_capacity(class.len() + case.len() + 2); + constant_name.extend_from_slice(class); + constant_name.extend_from_slice(b"::"); + constant_name.extend_from_slice(case); + + cached( + |ctx| ctx.enums.get(&constant_name).map(Zval::shallow_clone), + || { + let class_name = String::from_utf8_lossy(class); + let case_name = String::from_utf8_lossy(case); + let is_enum = std::str::from_utf8(class) + .ok() + .and_then(ClassEntry::try_find) + .is_some_and(|ce| ce.flags().contains(ClassFlags::Enum)); + + if !is_enum { + return Err(ext_exception(format!( + "flow_php cannot restore enum of class \"{class_name}\", enum not found" + ))); + } - if slot.is_none() { - let class = if mutable { - "DateTime" + let constant_name_zv = zval_str(&constant_name); + let defined = call_handle( + function("defined")?, + None, + &mut [constant_name_zv.shallow_clone()], + "check an enum case", + )?; + let case_zv = if defined.bool().unwrap_or(false) { + call_handle( + function("constant")?, + None, + &mut [constant_name_zv], + "restore an enum case", + )? } else { - "DateTimeImmutable" + null_zval() }; - *slot = Some(DateTimeFns { - ce: find_class(class)?, - set_timezone: method_handle(class, "setTimezone")?, - }); - } - - Ok(slot.as_ref().expect("just initialized")) - } - /// `DateTimeZone` instances cached per timezone name, mirroring - /// `ValueDecoder::$timezones`. - pub fn timezone(&mut self, name: &[u8]) -> Result<&Zval, PhpException> { - if !self.timezones.contains_key(name) { - if self.timezone_ce.is_none() { - self.timezone_ce = Some(find_class("DateTimeZone")?); + if !case_zv.is_object() { + return Err(ext_exception(format!( + "flow_php cannot restore enum case \"{class_name}::{case_name}\"" + ))); } - let name_string = std::str::from_utf8(name) - .map_err(|_| ext_exception("flow_php found a non UTF-8 timezone name"))? - .to_string(); - let mut timezone = construct_object( - self.timezone_ce.expect("just initialized"), - vec![&name_string as &dyn IntoZvalDyn], - "DateTimeZone", - )?; - - let mut zv = Zval::new(); - zv.set_object(&mut timezone); - self.timezones.insert(name.to_vec(), zv); - } - - Ok(self.timezones.get(name).expect("just inserted")) - } - - pub fn json_encode(&mut self) -> Result<&Function, PhpException> { - if self.fn_json_encode.is_none() { - self.fn_json_encode = Some(function_handle("json_encode")?); - } - - Ok(self.fn_json_encode.as_ref().expect("just initialized")) - } + Ok(case_zv) + }, + |ctx, zv| { + ctx.enums.insert(constant_name.clone(), zv.shallow_clone()); + }, + ) +} - pub fn json_decode(&mut self) -> Result<&Function, PhpException> { - if self.fn_json_decode.is_none() { - self.fn_json_decode = Some(function_handle("json_decode")?); - } +/// `HASH_FLAG_PACKED` from zend_types.h. +pub const HASH_FLAG_PACKED: u8 = 1 << 2; - Ok(self.fn_json_decode.as_ref().expect("just initialized")) - } +/// Raw order-preserving iteration over packed and bucket layouts, binary-safe +/// for arbitrary keys (ext-php-rs's `iter()` panics on non-UTF-8 string keys). +pub fn ht_for_each( + ht: &ZendHashTable, + mut f: impl FnMut(Option<&ZendStr>, zend_ulong, &Zval) -> Result<(), PhpException>, +) -> Result<(), PhpException> { + let packed = unsafe { ht.u.v.flags } & HASH_FLAG_PACKED != 0; - pub fn json_validate(&mut self) -> Result<&Function, PhpException> { - if self.fn_json_validate.is_none() { - self.fn_json_validate = Some(function_handle("json_validate")?); - } + for index in 0..ht.nNumUsed { + if packed { + let value = unsafe { &*ht.__bindgen_anon_1.arPacked.add(index as usize) }; - Ok(self.fn_json_validate.as_ref().expect("just initialized")) - } + if value.get_type() == ext_php_rs::flags::DataType::Undef { + continue; + } - pub fn date_parse(&mut self) -> Result<&Function, PhpException> { - if self.fn_date_parse.is_none() { - self.fn_date_parse = Some(function_handle("date_parse")?); - } + f(None, zend_ulong::from(index), value)?; + } else { + let bucket = unsafe { &*ht.__bindgen_anon_1.arData.add(index as usize) }; - Ok(self.fn_date_parse.as_ref().expect("just initialized")) - } + if bucket.val.get_type() == ext_php_rs::flags::DataType::Undef { + continue; + } - /// `DateTimeZone::listIdentifiers()`, fetched once per `Ctx`. - pub fn timezone_identifiers(&mut self) -> Result<&HashSet>, PhpException> { - if self.timezone_identifiers.is_none() { - let list = call_handle( - method_handle_ref("DateTimeZone", "listIdentifiers")?, - None, - &mut [], - "list timezone identifiers", - )?; - let identifiers = list - .array() - .ok_or_else(|| ext_exception("flow_php expected DateTimeZone::listIdentifiers() to return an array"))? - .values() - .filter_map(|zv| zv.zend_str().map(|name| name.as_bytes().to_vec())) - .collect(); - - self.timezone_identifiers = Some(identifiers); + let key = unsafe { bucket.key.as_ref() }; + f(key, bucket.h, &bucket.val)?; } - - Ok(self.timezone_identifiers.as_ref().expect("just initialized")) } - /// Whether `new DateTimeZone($name)` succeeds; the exception it throws otherwise is discarded. - pub fn timezone_accepts(&mut self, name: &[u8]) -> Result { - if self.timezone_ce.is_none() { - self.timezone_ce = Some(find_class("DateTimeZone")?); - } - - let ce = self.timezone_ce.expect("just initialized"); - - if self.timezone_construct.is_none() { - self.timezone_construct = Some(ce_method_ref(ce, "__construct")?); - } - - let timezone = ZendObject::new(ce); - - Ok(call_handle_catching( - self.timezone_construct.expect("just initialized"), - Some(&timezone), - &mut [zval_str(name)], - ) - .is_ok()) - } + Ok(()) +} - /// Static `Flow\ETL\Schema\Metadata::fromArray` handle - builds a Metadata - /// value object from a decoded per-value metadata map (the rare path). - pub fn metadata_from_array(&mut self) -> Result<&'static Function, PhpException> { - if self.metadata_from_array.is_none() { - self.metadata_from_array = - Some(method_handle_ref("Flow\\ETL\\Schema\\Metadata", "fromArray")?); - } +pub fn read_slot(obj: &ZendObject, offset: u32) -> &Zval { + unsafe { &*std::ptr::from_ref(obj).cast::().add(offset as usize).cast::() } +} - Ok(self.metadata_from_array.expect("just initialized")) - } +pub fn expect_object<'a>(zv: &'a Zval, context: &str) -> Result<&'a ZendObject, PhpException> { + zv.object() + .ok_or_else(|| ext_exception(format!("flow_php expected {context} to be an object"))) } diff --git a/src/extension/flow-php-ext/src/date_check.rs b/src/extension/flow-php-ext/src/date_check.rs index eae53983e6..ab941fc99d 100644 --- a/src/extension/flow-php-ext/src/date_check.rs +++ b/src/extension/flow-php-ext/src/date_check.rs @@ -114,3 +114,81 @@ pub fn checkdate(month: i64, day: i64, year: i64) -> bool { day <= days } + +/// Epoch microseconds of `YYYY-MM-DDTHH:MM:SS[.f]` (1–6 fraction digits) + `Z` or `±HH:MM`, every field in range - the +/// instant `DateTimeType::cast` yields, computed without timelib. `None` for every other string, which timelib parses. +pub fn iso_instant_micros(bytes: &[u8]) -> Option { + let byte_at = |at: usize| bytes.get(at).copied(); + let number = |at: usize, count: usize| -> Option { + bytes.get(at..at + count)?.iter().try_fold(0i64, |value, digit| { + digit.is_ascii_digit().then(|| value * 10 + i64::from(digit - b'0')) + }) + }; + + if [(4, b'-'), (7, b'-'), (10, b'T'), (13, b':'), (16, b':')] + .iter() + .any(|(at, separator)| byte_at(*at) != Some(*separator)) + { + return None; + } + + let (year, month, day) = (number(0, 4)?, number(5, 2)?, number(8, 2)?); + let (hour, minute, second) = (number(11, 2)?, number(14, 2)?, number(17, 2)?); + + if year > 9999 || !checkdate(month, day, year) || hour > 23 || minute > 59 || second > 59 { + return None; + } + + let mut at = 19; + let mut fraction = 0; + + if byte_at(at) == Some(b'.') { + let digits = bytes[at + 1..].iter().take_while(|byte| byte.is_ascii_digit()).count(); + + if !(1..=6).contains(&digits) { + return None; + } + + fraction = number(at + 1, digits)? * 10i64.pow(6 - digits as u32); + at += 1 + digits; + } + + let offset = match byte_at(at) { + Some(b'Z') => { + at += 1; + + 0 + } + Some(sign @ (b'+' | b'-')) if byte_at(at + 3) == Some(b':') => { + let (hours, minutes) = (number(at + 1, 2)?, number(at + 4, 2)?); + + if hours > 23 || minutes > 59 { + return None; + } + + at += 6; + + (hours * 3600 + minutes * 60) * if sign == b'-' { -1 } else { 1 } + } + _ => return None, + }; + + if at != bytes.len() { + return None; + } + + let seconds = days_from_civil(year, month, day)? * 86_400 + hour * 3600 + minute * 60 + second - offset; + + Some(seconds * 1_000_000 + fraction) +} + +/// `DaysFromCivil::of()`: days since 1970-01-01 of a proleptic Gregorian date, `None` on i64 overflow. +pub(crate) fn days_from_civil(year: i64, month: i64, day: i64) -> Option { + let year = year - i64::from(month <= 2); + let era = (if year >= 0 { year } else { year.checked_sub(399)? }) / 400; + let year_of_era = year - era * 400; + let day_of_year = (153 * (if month > 2 { month - 3 } else { month + 9 }) + 2) / 5 + day - 1; + let day_of_era = year_of_era * 365 + year_of_era / 4 - year_of_era / 100 + day_of_year; + + era.checked_mul(146_097)?.checked_add(day_of_era - 719_468) +} diff --git a/src/extension/flow-php-ext/src/encode.rs b/src/extension/flow-php-ext/src/encode.rs deleted file mode 100644 index 1c4a5bea04..0000000000 --- a/src/extension/flow-php-ext/src/encode.rs +++ /dev/null @@ -1,795 +0,0 @@ -//! Floe ROW frame-body encoder mirroring `Flow\Floe\PhpFloeEncoder::encode` -//! byte-for-byte; values and per-value metadata are read from a `TypedRowValues`. - -use ext_php_rs::exception::PhpException; -use ext_php_rs::ffi::zend_ulong; -use ext_php_rs::types::{ZendHashTable, ZendObject, ZendStr, Zval}; -use ext_php_rs::zend::{ClassEntry, Function}; - -use crate::ctx::{ - call_handle, call_handle_on, ce_method_ref, find_class, null_zval, read_property, - schema_mismatch, zval_str, Ctx, -}; -use crate::exception::ext_exception; -use crate::format::{ - write_u32, FRAME_ROW, VALUE_ABSENT, VALUE_NULL, VALUE_NULL_WITH_META, VALUE_PRESENT, VALUE_PRESENT_WITH_META, -}; -use crate::plan::{parse_schema_json, TypeJson}; - -enum EncodeMapKey { - Integer, - String, -} - -/// Declared structure element key after PHP's array-key coercion, mirroring -/// `ValueEncoder::structureEncoder`. -enum DeclaredKey { - Index(i64), - Str(Vec), -} - -/// Recursive value encoder mirroring `ValueEncoder::encoderFor`. -enum Encoder { - Integer, - Float, - Boolean, - String, - Null, - DateTime, - Interval, - Uuid, - Json, - TimeZone, - Enum, - Xml, - XmlElement, - Html, - HtmlElement, - List(Box), - Map(EncodeMapKey, Box), - Structure(Vec<(DeclaredKey, Vec, Encoder)>), - Optional(Box), -} - -fn build_encoder(type_json: &TypeJson) -> Result { - let missing = |part: &str| { - ext_exception(format!( - "flow_php schema JSON for type \"{}\" is missing its {part}", - type_json.type_ - )) - }; - - Ok(match type_json.type_.as_str() { - "integer" | "positive_integer" => Encoder::Integer, - "float" => Encoder::Float, - "boolean" => Encoder::Boolean, - "string" | "non_empty_string" | "numeric-string" | "class_string" => Encoder::String, - "null" => Encoder::Null, - "datetime" | "date" => Encoder::DateTime, - "time" => Encoder::Interval, - "uuid" => Encoder::Uuid, - "json" => Encoder::Json, - "timezone" => Encoder::TimeZone, - "enum" => Encoder::Enum, - "xml" => Encoder::Xml, - "xml_element" => Encoder::XmlElement, - "html" => Encoder::Html, - "html_element" => Encoder::HtmlElement, - "list" => Encoder::List(Box::new(build_encoder( - type_json.element().ok_or_else(|| missing("element"))?, - )?)), - "map" => { - let key = match type_json - .key() - .ok_or_else(|| missing("key"))? - .type_ - .as_str() - { - "integer" => EncodeMapKey::Integer, - "string" => EncodeMapKey::String, - other => { - return Err(ext_exception(format!( - "flow_php does not support map keys of type \"{other}\"" - ))); - } - }; - - Encoder::Map( - key, - Box::new(build_encoder( - type_json.value().ok_or_else(|| missing("value"))?, - )?), - ) - } - "structure_v2" => { - if type_json.fields().is_empty() { - return Err(ext_exception( - "flow_php read a structure type with no fields; the loaded flow_php extension and the \ - flow-php/etl in use disagree on the structure schema format - reinstall one to match the \ - other, or set the Floe engine to FloeEngine::php", - )); - } - - let mut elements = Vec::new(); - - for field in type_json.fields() { - let bytes = field.name.clone().into_bytes(); - let declared = match crate::ctx::array_key_index(&bytes) { - Some(index) => DeclaredKey::Index(index), - None => DeclaredKey::Str(bytes.clone()), - }; - elements.push((declared, bytes, build_encoder(&field.type_)?)); - } - - Encoder::Structure(elements) - } - "optional" => Encoder::Optional(Box::new(build_encoder( - type_json.base().ok_or_else(|| missing("base"))?, - )?)), - other => { - return Err(ext_exception(format!( - "flow_php does not support values of type \"{other}\" in this build" - ))); - } - }) -} - -pub(crate) struct EncodeColumn { - pub(crate) name: Vec, - encoder: Encoder, - /// `Definition::isNullable()`, read once at plan build - never per row - nullable: bool, - /// retained (refcount++) so a refusal can be raised through the PHP factory, - /// which is the only place a schema-mismatch message is ever authored - base_def: Zval, - /// Canonical PHP `json_encode` of this column's section-schema metadata - the - /// divergence reference. An entry whose metadata JSON differs rides its own - /// metadata beside the value. - plan_metadata_json: Vec, - plan_metadata_empty: bool, -} - -pub struct EncodePlan { - pub(crate) columns: Vec, - schema_mismatch_ce: &'static ClassEntry, - value_does_not_match: &'static Function, -} - -/// `HASH_FLAG_PACKED` from zend_types.h. -const HASH_FLAG_PACKED: u8 = 1 << 2; - -/// Raw order-preserving iteration over packed and bucket layouts, binary-safe -/// for arbitrary keys (ext-php-rs's `iter()` panics on non-UTF-8 string keys). -pub fn ht_for_each( - ht: &ZendHashTable, - mut f: impl FnMut(Option<&ZendStr>, zend_ulong, &Zval) -> Result<(), PhpException>, -) -> Result<(), PhpException> { - let packed = unsafe { ht.u.v.flags } & HASH_FLAG_PACKED != 0; - - for index in 0..ht.nNumUsed { - if packed { - let value = unsafe { &*ht.__bindgen_anon_1.arPacked.add(index as usize) }; - - if value.get_type() == ext_php_rs::flags::DataType::Undef { - continue; - } - - f(None, zend_ulong::from(index), value)?; - } else { - let bucket = unsafe { &*ht.__bindgen_anon_1.arData.add(index as usize) }; - - if bucket.val.get_type() == ext_php_rs::flags::DataType::Undef { - continue; - } - - let key = unsafe { bucket.key.as_ref() }; - f(key, bucket.h, &bucket.val)?; - } - } - - Ok(()) -} - -pub(crate) fn read_slot(obj: &ZendObject, offset: u32) -> &Zval { - unsafe { - &*std::ptr::from_ref(obj) - .cast::() - .add(offset as usize) - .cast::() - } -} - -pub(crate) fn expect_object<'a>( - zv: &'a Zval, - context: &str, -) -> Result<&'a ZendObject, PhpException> { - zv.object() - .ok_or_else(|| ext_exception(format!("flow_php expected {context} to be an object"))) -} - -fn write_len_prefixed(out: &mut Vec, bytes: &[u8]) { - write_u32(out, bytes.len() as u32); - out.extend_from_slice(bytes); -} - -/// Builds the encode plan from a SCHEMA frame body (the JSON the PHP -/// `SchemaEncoder` emits). Each column's section-schema metadata is captured as -/// its canonical PHP `json_encode` so a diverging entry can be detected without -/// re-parsing the schema per row. -pub fn build_encode_plan( - schema_json: &[u8], - schema: &Zval, - ctx: &mut Ctx, -) -> Result { - let definitions = parse_schema_json(schema_json)?; - let base_defs = schema_definitions(schema)?; - - let mut assoc_zv = Zval::new(); - assoc_zv.set_bool(true); - let decoded = { - let json_decode = ctx.json_decode()?; - call_handle( - json_decode, - None, - &mut [zval_str(schema_json), assoc_zv], - "decode a schema frame for metadata", - )? - }; - let decoded_ht = decoded - .array() - .ok_or_else(|| ext_exception("flow_php expected a schema frame to decode to a list"))?; - - let mut columns = Vec::with_capacity(definitions.len()); - - for (index, definition) in definitions.iter().enumerate() { - let plan_metadata_json = column_metadata_json(decoded_ht, index, ctx)?; - - let base_def = base_defs - .get(index) - .ok_or_else(|| ext_exception("flow_php expected a Definition for every schema column"))? - .shallow_clone(); - - columns.push(EncodeColumn { - name: definition.name.clone().into_bytes(), - encoder: build_encoder(&definition.type_)?, - nullable: definition.nullable, - base_def, - plan_metadata_empty: plan_metadata_json == b"[]", - plan_metadata_json, - }); - } - - Ok(EncodePlan { - columns, - schema_mismatch_ce: find_class("Flow\\ETL\\Exception\\SchemaMismatchException")?, - value_does_not_match: ce_method_ref( - find_class("Flow\\ETL\\Exception\\ColumnMismatchException")?, - "valueDoesNotMatch", - )?, - }) -} - -/// Canonical PHP `json_encode` of the `metadata` map of the `index`th decoded -/// schema definition (`[]` when absent). -fn column_metadata_json( - decoded_ht: &ZendHashTable, - index: usize, - ctx: &mut Ctx, -) -> Result, PhpException> { - let definition_ht = decoded_ht - .get_index(index as i64) - .and_then(Zval::array) - .ok_or_else(|| ext_exception("flow_php expected a schema definition to be a map"))?; - - let metadata_zv = match definition_ht.get("metadata") { - Some(zv) => zv.shallow_clone(), - None => return Ok(b"[]".to_vec()), - }; - - let json_encode = ctx.json_encode()?; - let encoded = call_handle( - json_encode, - None, - &mut [metadata_zv], - "encode section-schema metadata", - )?; - - encoded - .zend_str() - .map(|s| s.as_bytes().to_vec()) - .ok_or_else(|| ext_exception("flow_php expected json_encode to return a string")) -} - -/// The `Schema`'s own `Definition` objects in declaration order - the same walk -/// `build_hydrate_plan` does, and the same order `parse_schema_json` yields. -/// Runs once per schema rebind, never per row. -fn schema_definitions(schema: &Zval) -> Result, PhpException> { - let schema_obj = expect_object(schema, "a Schema")?; - let schema_ce = unsafe { schema_obj.ce.as_ref() } - .ok_or_else(|| ext_exception("flow_php failed to resolve the Schema class"))?; - let definitions = call_handle_on( - ce_method_ref(schema_ce, "definitions")?, - schema_obj, - &mut [], - "read schema definitions", - )?; - let definitions_ht = definitions - .array() - .ok_or_else(|| ext_exception("flow_php expected Schema::definitions to return an array"))?; - - let mut base_defs = Vec::with_capacity(definitions_ht.len()); - - ht_for_each(definitions_ht, |_, _, def_zv| { - base_defs.push(def_zv.shallow_clone()); - - Ok(()) - })?; - - Ok(base_defs) -} - -/// Appends one `Flow\ETL\Row\TypedRowValues` (its `values` + `metadata` maps) to `out` -/// as a bare ROW frame body (no length prefix, no frame type). Byte-identical -/// to `Flow\Floe\PhpFloeEncoder::encode` for the same row. -pub fn encode_typed_row( - plan: &EncodePlan, - row_index: u64, - values_ht: &ZendHashTable, - metadata_ht: &ZendHashTable, - out: &mut Vec, - ctx: &mut Ctx, -) -> Result<(), PhpException> { - for column in &plan.columns { - let Some(value) = ht_find(values_ht, &column.name) else { - return Err(ext_exception(format!( - "flow_php found a row that does not carry the declared column \"{}\"", - String::from_utf8_lossy(&column.name) - ))); - }; - - let (diverges, entry_metadata_json) = typed_metadata(column, metadata_ht, ctx)?; - - if value.is_null() { - if !column.nullable { - return Err(schema_mismatch( - plan.schema_mismatch_ce, - plan.value_does_not_match, - row_index, - &mut [column.base_def.shallow_clone(), null_zval()], - )?); - } - - if diverges { - out.push(VALUE_NULL_WITH_META); - write_len_prefixed(out, &entry_metadata_json); - } else { - out.push(VALUE_NULL); - } - - continue; - } - - if diverges { - out.push(VALUE_PRESENT_WITH_META); - write_len_prefixed(out, &entry_metadata_json); - } else { - out.push(VALUE_PRESENT); - } - - encode_value(&column.encoder, value, out, ctx)?; - } - - Ok(()) -} - -/// `Format::frame(Format::FRAME_ROW, $body)` over `encode_typed_row` for every `Row` of a batch: the rows' own -/// values tables and the batch's one `TypedRowValues::metadata` map, complete ROW frames in one buffer. -pub fn encode_frames( - plan: &EncodePlan, - rows_ht: &ZendHashTable, - row_values_slot: u32, - metadata_ht: &ZendHashTable, - ctx: &mut Ctx, -) -> Result, PhpException> { - let mut out = Vec::with_capacity(rows_ht.len() * 256); - - ht_for_each(rows_ht, |_, row_index, row_zv| { - let values_ht = read_slot(expect_object(row_zv, "a Row")?, row_values_slot) - .array() - .ok_or_else(|| ext_exception("flow_php expected Row::values to be an array"))?; - - out.push(FRAME_ROW); - let length_at = out.len(); - out.extend_from_slice(&[0; 4]); - - encode_typed_row(plan, row_index, values_ht, metadata_ht, &mut out, ctx)?; - - let length = u32::try_from(out.len() - length_at - 4) - .map_err(|_| ext_exception("flow_php encoded a row frame longer than 4 GiB"))?; - out[length_at..length_at + 4].copy_from_slice(&length.to_le_bytes()); - - Ok(()) - })?; - - Ok(out) -} - -/// Whether a column's per-value metadata diverges from the section-schema -/// reference and, if so, its canonical JSON (the beside-value blob). `metadata_ht` -/// is the `TypedRowValues::metadata` map, which carries only non-empty entries -/// (mirroring `PhpRowHydrator::dehydrate`); an absent key is empty metadata. Fast -/// path: an empty entry against an empty-metadata column never diverges - no PHP call. -fn typed_metadata( - column: &EncodeColumn, - metadata_ht: &ZendHashTable, - ctx: &mut Ctx, -) -> Result<(bool, Vec), PhpException> { - let metadata_zv = ht_find(metadata_ht, &column.name); - - let entry_empty = match metadata_zv { - None => true, - Some(zv) => { - let metadata = expect_object(zv, "per-value metadata")?; - let map_slot = ctx.metadata_map_slot()?; - read_slot(metadata, map_slot) - .array() - .is_none_or(|map| map.is_empty()) - } - }; - - if entry_empty && column.plan_metadata_empty { - return Ok((false, Vec::new())); - } - - let entry_metadata_json = if entry_empty { - b"[]".to_vec() - } else { - let metadata = expect_object( - metadata_zv.expect("Some when the entry is not empty"), - "per-value metadata", - )?; - let map_slot = ctx.metadata_map_slot()?; - let map_owned = read_slot(metadata, map_slot).shallow_clone(); - let json_encode = ctx.json_encode()?; - let encoded = call_handle( - json_encode, - None, - &mut [map_owned], - "encode per-value metadata", - )?; - - encoded - .zend_str() - .map(|s| s.as_bytes().to_vec()) - .ok_or_else(|| ext_exception("flow_php expected json_encode to return a string"))? - }; - - let diverges = entry_metadata_json != column.plan_metadata_json; - - Ok((diverges, entry_metadata_json)) -} - -fn encode_value( - encoder: &Encoder, - value: &Zval, - out: &mut Vec, - ctx: &mut Ctx, -) -> Result<(), PhpException> { - match encoder { - Encoder::Integer => out.extend_from_slice( - &value - .long() - .ok_or_else(|| ext_exception("flow_php expected an integer value"))? - .to_le_bytes(), - ), - Encoder::Float => out.extend_from_slice( - &php_float(value) - .ok_or_else(|| ext_exception("flow_php expected a float value"))? - .to_le_bytes(), - ), - Encoder::Boolean => { - out.push(u8::from(value.bool().ok_or_else(|| { - ext_exception("flow_php expected a boolean value") - })?)) - } - Encoder::String => write_len_prefixed( - out, - value - .zend_str() - .ok_or_else(|| ext_exception("flow_php expected a string value"))? - .as_bytes(), - ), - Encoder::Null => {} - Encoder::DateTime => encode_datetime(expect_object(value, "a datetime value")?, out, ctx)?, - Encoder::Interval => encode_interval(expect_object(value, "a time value")?, out, ctx)?, - Encoder::Uuid => { - let uuid = expect_object(value, "a uuid value")?; - let (uuid_ce, value_slot) = ctx.uuid()?; - - if !std::ptr::eq(uuid.ce.cast_const(), std::ptr::from_ref(uuid_ce)) { - return Err(ext_exception("flow_php expected a uuid value to be an object")); - } - - let bytes = read_slot(uuid, value_slot) - .zend_str() - .ok_or_else(|| ext_exception("flow_php expected Uuid to hold a string"))?; - write_len_prefixed(out, bytes.as_bytes()); - } - Encoder::Json => encode_json(expect_object(value, "a json value")?, out, ctx)?, - Encoder::TimeZone => { - let timezone = expect_object(value, "a timezone value")?; - let name = call_handle_on( - ctx.timezone_get_name()?, - timezone, - &mut [], - "read a timezone name", - )?; - write_len_prefixed( - out, - name.zend_str() - .ok_or_else(|| ext_exception("flow_php expected a timezone name string"))? - .as_bytes(), - ); - } - Encoder::Enum => { - let case = expect_object(value, "an enum value")?; - let class = unsafe { case.ce.as_ref() } - .and_then(ClassEntry::name) - .ok_or_else(|| ext_exception("flow_php failed to resolve an enum class"))?; - write_len_prefixed(out, class.as_bytes()); - - let name = read_property(case, "name")?; - write_len_prefixed( - out, - name.zend_str() - .ok_or_else(|| ext_exception("flow_php expected an enum case name"))? - .as_bytes(), - ); - } - Encoder::Xml => encode_via_static(value, out, ctx, "xmlDocumentToString")?, - Encoder::XmlElement => encode_via_static(value, out, ctx, "xmlElementToString")?, - Encoder::HtmlElement => encode_via_static(value, out, ctx, "htmlElementToString")?, - Encoder::Html => { - let document = expect_object(value, "an html value")?; - let html = call_handle_on( - ctx.save_html(document.ce.cast_const())?, - document, - &mut [], - "convert an HTML document", - )?; - write_len_prefixed( - out, - html.zend_str() - .ok_or_else(|| ext_exception("flow_php expected saveHtml to return a string"))? - .as_bytes(), - ); - } - Encoder::List(element) => { - let values = value - .array() - .ok_or_else(|| ext_exception("flow_php expected a list value"))?; - write_u32(out, values.len() as u32); - ht_for_each(values, |_, _, item| encode_value(element, item, out, ctx))?; - } - Encoder::Map(key, element) => { - let values = value - .array() - .ok_or_else(|| ext_exception("flow_php expected a map value"))?; - write_u32(out, values.len() as u32); - ht_for_each(values, |map_key, index, item| { - match key { - EncodeMapKey::Integer => { - let int_key = match map_key { - None => index as i64, - Some(_) => { - return Err(ext_exception("flow_php expected an integer map key")); - } - }; - out.extend_from_slice(&int_key.to_le_bytes()); - } - EncodeMapKey::String => match map_key { - // numeric-string keys coerced to int by the PHP array - // are cast back, like ValueEncoder - None => write_len_prefixed(out, (index as i64).to_string().as_bytes()), - Some(key) => write_len_prefixed(out, key.as_bytes()), - }, - } - - encode_value(element, item, out, ctx) - })?; - } - Encoder::Structure(elements) => { - let values = value - .array() - .ok_or_else(|| ext_exception("flow_php expected a structure value"))?; - - for (declared, _, element) in elements { - let item = match declared { - DeclaredKey::Index(index) => values.get_index(*index), - DeclaredKey::Str(key) => ht_find(values, key), - }; - - match item { - None => out.push(VALUE_ABSENT), - Some(item) if item.is_null() => out.push(VALUE_NULL), - Some(item) => { - out.push(VALUE_PRESENT); - encode_value(element, item, out, ctx)?; - } - } - } - } - Encoder::Optional(base) => { - if value.is_null() { - out.push(VALUE_NULL); - } else { - out.push(VALUE_PRESENT); - encode_value(base, value, out, ctx)?; - } - } - } - - Ok(()) -} - -/// Accepts ints where FloatType is declared, permissive like `pack('e')`. -fn php_float(value: &Zval) -> Option { - value.double().or_else(|| value.long().map(|l| l as f64)) -} - -fn ht_find<'a>(ht: &'a ZendHashTable, key: &[u8]) -> Option<&'a Zval> { - match crate::ctx::array_key_index(key) { - Some(index) => ht.get_index(index), - None => unsafe { - ext_php_rs::ffi::zend_hash_str_find( - std::ptr::from_ref(ht).cast_mut(), - key.as_ptr().cast(), - key.len(), - ) - .as_ref() - }, - } -} - -/// Mirrors `ValueEncoder::encodeDateTime`. -fn encode_datetime( - value: &ZendObject, - out: &mut Vec, - ctx: &mut Ctx, -) -> Result<(), PhpException> { - let ce = value.ce.cast_const(); - let immutable_ce = std::ptr::from_ref(ctx.datetime_fns(false)?.ce); - let mutable_ce = std::ptr::from_ref(ctx.datetime_fns(true)?.ce); - - // the class is not stored - a datetime column always hydrates to DateTimeImmutable - but a - // custom subclass is still refused, because its extra state would be dropped silently - if !std::ptr::eq(ce, immutable_ce) && !std::ptr::eq(ce, mutable_ce) { - let class = unsafe { ce.as_ref() } - .and_then(ClassEntry::name) - .ok_or_else(|| ext_exception("flow_php failed to resolve a datetime class"))?; - - return Err(ext_exception(format!( - "Floe supports only DateTime and DateTimeImmutable, got {class} - convert custom datetime instances before writing" - ))); - } - - let enc = ctx.datetime_encode_fns(ce)?; - let get_timestamp = enc.get_timestamp; - let format = enc.format; - let get_timezone = enc.get_timezone; - - let timestamp = call_handle_on(get_timestamp, value, &mut [], "read a timestamp")? - .long() - .ok_or_else(|| ext_exception("flow_php expected getTimestamp to return an int"))?; - - let format_u = ctx.format_u()?.shallow_clone(); - let microseconds_zv = - call_handle_on(format, value, &mut [format_u], "read datetime microseconds")?; - let microseconds: u32 = microseconds_zv - .zend_str() - .and_then(|s| std::str::from_utf8(s.as_bytes()).ok()) - .and_then(|s| s.parse().ok()) - .ok_or_else(|| ext_exception("flow_php expected format('u') to return digits"))?; - - let timezone_zv = call_handle_on(get_timezone, value, &mut [], "read a timezone")?; - let timezone = expect_object(&timezone_zv, "a timezone")?; - let name = call_handle_on( - ctx.timezone_get_name()?, - timezone, - &mut [], - "read a timezone name", - )?; - - out.extend_from_slice(×tamp.to_le_bytes()); - write_u32(out, microseconds); - write_len_prefixed( - out, - name.zend_str() - .ok_or_else(|| ext_exception("flow_php expected a timezone name string"))? - .as_bytes(), - ); - - Ok(()) -} - -/// Mirrors `ValueEncoder::encodeInterval`. -fn encode_interval( - value: &ZendObject, - out: &mut Vec, - _ctx: &mut Ctx, -) -> Result<(), PhpException> { - let read = |name: &str| -> Result { read_property(value, name) }; - - let invert = read("invert")? - .long() - .ok_or_else(|| ext_exception("flow_php expected DateInterval::invert to be an int"))?; - out.push(invert as u8); - - for field in ["y", "m", "d", "h", "i", "s"] { - let field_value = read(field)? - .long() - .ok_or_else(|| ext_exception("flow_php expected DateInterval fields to be ints"))?; - write_u32(out, field_value as u32); - } - - let fraction = read("f")?; - let fraction = php_float(&fraction) - .ok_or_else(|| ext_exception("flow_php expected DateInterval::f to be a float"))?; - out.extend_from_slice(&fraction.to_le_bytes()); - - Ok(()) -} - -fn encode_json(value: &ZendObject, out: &mut Vec, ctx: &mut Ctx) -> Result<(), PhpException> { - let (json_ce, value_slot, is_object_slot) = ctx.json()?; - - if !std::ptr::eq(value.ce.cast_const(), std::ptr::from_ref(json_ce)) { - return Err(ext_exception("flow_php expected a json value to be an object")); - } - - write_len_prefixed( - out, - read_slot(value, value_slot) - .zend_str() - .ok_or_else(|| ext_exception("flow_php expected Json to hold a string"))? - .as_bytes(), - ); - out.push(u8::from( - read_slot(value, is_object_slot) - .bool() - .ok_or_else(|| ext_exception("flow_php expected Json::isObject to be a bool"))?, - )); - - Ok(()) -} - -/// Markup serialization delegated to the pure-PHP `ValueEncoder` public -/// statics - the exact code its closures run. -fn encode_via_static( - value: &Zval, - out: &mut Vec, - ctx: &mut Ctx, - method: &'static str, -) -> Result<(), PhpException> { - let handle = ctx.value_encoder_static(method)?; - let string = call_handle( - handle, - None, - &mut [value.shallow_clone()], - "serialize a markup value", - )?; - - write_len_prefixed( - out, - string - .zend_str() - .ok_or_else(|| { - ext_exception(format!( - "flow_php expected ValueEncoder::{method} to return a string" - )) - })? - .as_bytes(), - ); - - Ok(()) -} diff --git a/src/extension/flow-php-ext/src/exception.rs b/src/extension/flow-php-ext/src/exception.rs index 29bbb12c3c..2e2b28b3db 100644 --- a/src/extension/flow-php-ext/src/exception.rs +++ b/src/extension/flow-php-ext/src/exception.rs @@ -1,27 +1,39 @@ -use ext_php_rs::builders::ClassBuilder; -use ext_php_rs::error::Result; use ext_php_rs::exception::PhpException; -use ext_php_rs::flags::ClassFlags; use ext_php_rs::zend::{ce, ClassEntry}; +use flow_batch_frame::error::Part; +use flow_batch_frame::Error; -const EXCEPTION_CLASS: &str = "Flow\\Floe\\Exception\\ExtensionException"; +use crate::render::RUNTIME; -fn noop(_ce: &'static mut ClassEntry) {} - -pub fn register() -> Result<()> { - ClassBuilder::new(EXCEPTION_CLASS) - .extends((ce::exception, "Exception")) - .flags(ClassFlags::Final) - .registration(noop) - .register()?; - - Ok(()) -} - -fn exception_ce() -> &'static ClassEntry { - ClassEntry::try_find(EXCEPTION_CLASS).unwrap_or_else(|| ce::exception()) +/// Never through `ctx::find_class()`: its not-found branch calls this function, so a missing class would recurse +/// until the stack overflows; without `flow-php/etl` the failure is a plain `\Exception`. +pub fn ext_exception(message: impl Into) -> PhpException { + PhpException::new( + message.into(), + 0, + ClassEntry::try_find(RUNTIME).unwrap_or_else(ce::exception), + ) } -pub fn ext_exception(message: impl Into) -> PhpException { - PhpException::new(message.into(), 0, exception_ce()) +/// A flow-batch-frame refusal of schema or type JSON, which only `kind` returns. +pub fn json_exception(error: Error) -> PhpException { + ext_exception(match error { + Error::SchemaJsonUtf8 => "flow_php failed to decode schema JSON: invalid UTF-8".to_string(), + Error::SchemaJson(e) => format!("flow_php failed to decode schema JSON: {e}"), + Error::TypeJsonUtf8 => "flow_php failed to decode type JSON: invalid UTF-8".to_string(), + Error::TypeJson(e) => format!("flow_php failed to decode type JSON: {e}"), + Error::MissingPart { type_name, part } => format!( + "flow_php schema JSON for type \"{type_name}\" is missing its {}", + match part { + Part::Element => "element", + Part::Key => "key", + Part::Value => "value", + Part::Base => "base", + } + ), + Error::UnknownType { type_name } => { + format!("flow_php cannot build a column for type \"{type_name}\"") + } + error => unreachable!("kind returns JSON refusals only, got {error}"), + }) } diff --git a/src/extension/flow-php-ext/src/format.rs b/src/extension/flow-php-ext/src/format.rs deleted file mode 100644 index 5fbaa072c9..0000000000 --- a/src/extension/flow-php-ext/src/format.rs +++ /dev/null @@ -1,81 +0,0 @@ -//! Floe frame-body wire constants and a bounds-checked byte reader. The canonical -//! definitions live in the pure-PHP `Flow\Floe\Format`; mirror any change in the same PR. - -use ext_php_rs::exception::PhpException; - -use crate::exception::ext_exception; - -// Floe is little-endian by construction; a big-endian host could never match its bytes. -#[cfg(target_endian = "big")] -compile_error!("flow_php only supports little-endian targets"); - -pub const FRAME_ROW: u8 = 0x02; - -pub const VALUE_NULL: u8 = 0x00; -pub const VALUE_PRESENT: u8 = 0x01; -pub const VALUE_NULL_WITH_META: u8 = 0x02; -pub const VALUE_ABSENT: u8 = 0x03; -pub const VALUE_PRESENT_WITH_META: u8 = 0x04; - -pub struct Reader<'a> { - data: &'a [u8], - pos: usize, -} - -impl<'a> Reader<'a> { - pub fn new(data: &'a [u8]) -> Self { - Self { data, pos: 0 } - } - - pub fn is_eof(&self) -> bool { - self.pos >= self.data.len() - } - - fn take(&mut self, len: usize, context: &str) -> Result<&'a [u8], PhpException> { - let end = self - .pos - .checked_add(len) - .filter(|end| *end <= self.data.len()); - - match end { - Some(end) => { - let slice = &self.data[self.pos..end]; - self.pos = end; - Ok(slice) - } - None => Err(ext_exception(format!( - "flow_php frame body is truncated, {context} is incomplete" - ))), - } - } - - pub fn u8(&mut self, context: &str) -> Result { - Ok(self.take(1, context)?[0]) - } - - pub fn u32(&mut self, context: &str) -> Result { - let bytes = self.take(4, context)?; - - Ok(u32::from_le_bytes(bytes.try_into().expect("4-byte slice"))) - } - - pub fn i64(&mut self, context: &str) -> Result { - let bytes = self.take(8, context)?; - - Ok(i64::from_le_bytes(bytes.try_into().expect("8-byte slice"))) - } - - pub fn f64(&mut self, context: &str) -> Result { - let bytes = self.take(8, context)?; - - Ok(f64::from_le_bytes(bytes.try_into().expect("8-byte slice"))) - } - - pub fn bytes(&mut self, len: usize, context: &str) -> Result<&'a [u8], PhpException> { - self.take(len, context) - } -} - -pub fn write_u32(out: &mut Vec, value: u32) { - out.extend_from_slice(&value.to_le_bytes()); -} diff --git a/src/extension/flow-php-ext/src/globals.rs b/src/extension/flow-php-ext/src/globals.rs new file mode 100644 index 0000000000..aafa44dbad --- /dev/null +++ b/src/extension/flow-php-ext/src/globals.rs @@ -0,0 +1,43 @@ +//! Per-request state: the `Ctx` cache lives in module globals (per thread on ZTS), is built lazily and is dropped in +//! RSHUTDOWN, while the executor still runs; objects freed after RSHUTDOWN never touch it. + +use ext_php_rs::prelude::PhpResult; +use ext_php_rs::zend::{ModuleGlobal, ModuleGlobals}; + +use crate::ctx::Ctx; +use crate::exception::ext_exception; + +#[derive(Default)] +pub struct FlowGlobals { + ctx: Option, + closed: bool, +} + +impl ModuleGlobal for FlowGlobals {} + +pub static GLOBALS: ModuleGlobals = ModuleGlobals::new(); + +pub extern "C" fn request_startup(_type: i32, _module_number: i32) -> i32 { + unsafe { GLOBALS.get_mut() }.closed = false; + + 0 +} + +pub extern "C" fn request_shutdown(_type: i32, _module_number: i32) -> i32 { + let globals = unsafe { GLOBALS.get_mut() }; + globals.ctx = None; + globals.closed = true; + + 0 +} + +/// The only access to `Ctx`. `f` must not call PHP code: PHP re-entering flow_php would alias the `&mut Ctx`. +pub fn with_ctx(f: impl FnOnce(&mut Ctx) -> PhpResult) -> PhpResult { + let globals = unsafe { GLOBALS.get_mut() }; + + if globals.closed { + return Err(ext_exception("flow_php used after request shutdown")); + } + + f(globals.ctx.get_or_insert_with(Ctx::default)) +} diff --git a/src/extension/flow-php-ext/src/hydrate.rs b/src/extension/flow-php-ext/src/hydrate.rs deleted file mode 100644 index 767e9123cf..0000000000 --- a/src/extension/flow-php-ext/src/hydrate.rs +++ /dev/null @@ -1,576 +0,0 @@ -//! Object-graph assembly mirroring `Flow\ETL\Row\PhpRowHydrator` and -//! `EntryInstantiator`: constructor-less instantiation with direct -//! property-slot writes. - -use std::collections::HashMap; - -use ext_php_rs::boxed::ZBox; -use ext_php_rs::exception::PhpException; -use ext_php_rs::types::{ZendHashTable, ZendObject, Zval}; - -use ext_php_rs::zend::{ClassEntry, Function}; - -use crate::ctx::{ - array_key_index, call_handle, call_handle_on, ce_method_ref, - find_class, ht_add, ht_find_key, ht_insert, ht_insert_key, - property_offset, write_slot, zval_str, Ctx, HtKey, -}; -use crate::encode::{expect_object, ht_for_each, read_slot}; -use crate::exception::ext_exception; -use crate::format::{ - Reader, VALUE_NULL, VALUE_NULL_WITH_META, VALUE_PRESENT, VALUE_PRESENT_WITH_META, -}; -use crate::plan::{Decoder, Plan}; -use crate::values::decode_value; - -/// Resolved `Flow\ETL\Row\RawRowValues` class handles, shared by the binary decoder and the hydrator. -pub struct RowValuesClass { - pub ce: &'static ClassEntry, - pub values_slot: u32, - pub metadata_slot: u32, -} - -impl RowValuesClass { - pub fn resolve() -> Result { - let ce = find_class("Flow\\ETL\\Row\\RawRowValues")?; - - Ok(Self { - ce, - values_slot: property_offset(ce, "values")?, - metadata_slot: property_offset(ce, "metadata")?, - }) - } -} - -/// Reads one per-value metadata blob (`u32 len` + JSON) and rebuilds the -/// `Flow\ETL\Schema\Metadata` value object via the canonical PHP -/// `json_decode` + `Metadata::fromArray` - the rare diverging path only. -fn read_metadata(reader: &mut Reader, ctx: &mut Ctx) -> Result { - let length = reader.u32("per-value metadata length")? as usize; - let blob = reader.bytes(length, "per-value metadata")?; - - let json_zv = zval_str(blob); - let mut assoc_zv = Zval::new(); - assoc_zv.set_bool(true); - - let map = { - let json_decode = ctx.json_decode()?; - call_handle( - json_decode, - None, - &mut [json_zv.shallow_clone(), assoc_zv], - "decode per-value metadata", - )? - }; - - let from_array = ctx.metadata_from_array()?; - call_handle(from_array, None, &mut [map], "build per-value metadata") -} - -/// One column of a ROW frame body: its value, and the per-value metadata when the flag carries one. -pub(crate) fn read_cell( - decoder: &Decoder, - reader: &mut Reader, - ctx: &mut Ctx, -) -> Result<(Zval, Option), PhpException> { - Ok(match reader.u8("row value flag")? { - VALUE_PRESENT => (decode_value(decoder, reader, ctx)?, None), - VALUE_NULL => (Zval::new(), None), - VALUE_PRESENT_WITH_META => { - let metadata = read_metadata(reader, ctx)?; - - (decode_value(decoder, reader, ctx)?, Some(metadata)) - } - VALUE_NULL_WITH_META => (Zval::new(), Some(read_metadata(reader, ctx)?)), - other => { - return Err(ext_exception(format!( - "flow_php found unknown value flag 0x{other:02X}" - ))); - } - }) -} - -/// Decodes one ROW frame body into a `Flow\ETL\Row\RawRowValues` object - values keyed by -/// column name, absent columns omitted, diverging per-value metadata recorded in `metadata`. -pub fn decode_row_values( - plan: &Plan, - reader: &mut Reader, - ctx: &mut Ctx, - class: &RowValuesClass, -) -> Result, PhpException> { - let mut values_ht = ZendHashTable::with_capacity(plan.columns.len() as u32); - let mut metadata_ht = ZendHashTable::new(); - - for column in &plan.columns { - let (value, metadata) = read_cell(&column.decoder, reader, ctx)?; - - if let Some(metadata) = metadata { - ht_insert(&mut metadata_ht, column.name.as_bytes(), metadata); - } - - if !ht_add(&mut values_ht, column.name.as_bytes(), value) { - return Err(ext_exception(format!( - "flow_php found duplicated entry name \"{}\" in a row frame", - column.name - ))); - } - } - - let mut row_values = ZendObject::new(class.ce); - - let mut values_zv = Zval::new(); - values_zv.set_hashtable(values_ht); - write_slot(&mut row_values, class.values_slot, values_zv); - - let mut metadata_zv = Zval::new(); - metadata_zv.set_hashtable(metadata_ht); - write_slot(&mut row_values, class.metadata_slot, metadata_zv); - - Ok(row_values) -} - -/// Resolved property-table slot offsets for the `Rows -> Schema` / `Rows -> Row` -/// graph traversed by `dehydrate`. The Schema carries the column types now, so -/// they are read once per batch instead of once per cell. -pub struct RowsClasses { - pub rows_slot: u32, - pub schema_slot: u32, - pub row_values_slot: u32, - pub definitions_slot: u32, -} - -impl RowsClasses { - pub fn resolve() -> Result { - let rows_ce = find_class("Flow\\ETL\\Rows")?; - - Ok(Self { - rows_slot: property_offset(rows_ce, "rows")?, - schema_slot: property_offset(rows_ce, "schema")?, - row_values_slot: property_offset(find_class("Flow\\ETL\\Row")?, "values")?, - definitions_slot: property_offset(find_class("Flow\\ETL\\Schema")?, "definitions")?, - }) - } -} - -/// Resolved `Flow\ETL\Row\TypedRowValues` class handle + its three slot offsets. -pub struct TypedRowValuesClass { - pub ce: &'static ClassEntry, - pub values_slot: u32, - pub types_slot: u32, - pub metadata_slot: u32, -} - -impl TypedRowValuesClass { - pub fn resolve() -> Result { - let ce = find_class("Flow\\ETL\\Row\\TypedRowValues")?; - - Ok(Self { - ce, - values_slot: property_offset(ce, "values")?, - types_slot: property_offset(ce, "types")?, - metadata_slot: property_offset(ce, "metadata")?, - }) - } -} - -/// `(type(), metadata())` method handles cached per definition class. -pub type DefFnCache = HashMap; - -pub(crate) fn def_dehydrate_fns( - cache: &mut DefFnCache, - ce: &ClassEntry, -) -> Result<(&'static Function, &'static Function), PhpException> { - let key = std::ptr::from_ref(ce) as usize; - - if let Some(fns) = cache.get(&key) { - return Ok(*fns); - } - - let fns = (ce_method_ref(ce, "type")?, ce_method_ref(ce, "metadata")?); - cache.insert(key, fns); - - Ok(fns) -} - -/// One column of a batch's dehydrate plan: the key, and the `type()` / -/// `metadata()` zvals read once from the batch Schema. -pub(crate) struct DehydrateColumn { - numeric_key: Option, - name_zv: Zval, - type_zv: Zval, - metadata_zv: Option, -} - -impl DehydrateColumn { - fn key(&self) -> HtKey<'_> { - match self.numeric_key { - Some(index) => HtKey::Index(index), - None => HtKey::Str(self.name_zv.zend_str().expect("column names are strings")), - } - } -} - -/// Mirrors `PhpRowHydrator::dehydrate` for one row: values come from -/// `Row::values`, types and metadata from the batch Schema. -fn dehydrate_row( - values_ht: &ZendHashTable, - columns: &[DehydrateColumn], - class: &TypedRowValuesClass, -) -> Result, PhpException> { - let mut values_ht_out = ZendHashTable::with_capacity(columns.len() as u32); - let mut types_ht = ZendHashTable::with_capacity(columns.len() as u32); - let mut metadata_ht = ZendHashTable::new(); - - for column in columns { - let key = column.key(); - - let Some(value) = ht_find_key(values_ht, &key) else { - return Err(ext_exception(format!( - "flow_php found a row that does not carry the declared column \"{}\"", - column.name_zv.str().unwrap_or("?") - ))); - }; - - ht_insert_key(&mut values_ht_out, &key, value.shallow_clone()); - ht_insert_key(&mut types_ht, &key, column.type_zv.shallow_clone()); - - if let Some(metadata) = &column.metadata_zv { - ht_insert_key(&mut metadata_ht, &key, metadata.shallow_clone()); - } - } - - let mut row_values = ZendObject::new(class.ce); - - let mut values_zv = Zval::new(); - values_zv.set_hashtable(values_ht_out); - write_slot(&mut row_values, class.values_slot, values_zv); - - let mut types_zv = Zval::new(); - types_zv.set_hashtable(types_ht); - write_slot(&mut row_values, class.types_slot, types_zv); - - let mut metadata_zv = Zval::new(); - metadata_zv.set_hashtable(metadata_ht); - write_slot(&mut row_values, class.metadata_slot, metadata_zv); - - Ok(row_values) -} - -/// The batch behind a `Flow\ETL\Rows`: its `rows` table and one dehydrate column per Schema definition - the -/// `type()`/`metadata()` calls made once per batch, not once per cell. -pub(crate) fn dehydrate_batch<'a>( - rows: &'a Zval, - rows_classes: &RowsClasses, - def_fn_cache: &mut DefFnCache, - ctx: &mut Ctx, -) -> Result<(&'a ZendHashTable, Vec), PhpException> { - let rows_obj = expect_object(rows, "Rows")?; - let rows_ht = read_slot(rows_obj, rows_classes.rows_slot) - .array() - .ok_or_else(|| ext_exception("flow_php expected Rows::rows to be an array"))?; - - let schema = read_slot(rows_obj, rows_classes.schema_slot) - .object() - .ok_or_else(|| ext_exception("flow_php expected Rows::schema to be an object"))?; - let definitions_ht = read_slot(schema, rows_classes.definitions_slot) - .array() - .ok_or_else(|| ext_exception("flow_php expected Schema::definitions to be an array"))?; - - let metadata_map_slot = ctx.metadata_map_slot()?; - - let mut columns: Vec = Vec::with_capacity(definitions_ht.len()); - - ht_for_each(definitions_ht, |_, _, def_zv| { - let definition = expect_object(def_zv, "a Definition")?; - let definition_ce = unsafe { definition.ce.as_ref() } - .ok_or_else(|| ext_exception("flow_php failed to resolve a definition class"))?; - - let entry_fn = ce_method_ref(definition_ce, "entry")?; - let reference = call_handle_on(entry_fn, definition, &mut [], "read a definition reference")?; - let reference_obj = expect_object(&reference, "a definition reference")?; - let reference_ce = unsafe { reference_obj.ce.as_ref() } - .ok_or_else(|| ext_exception("flow_php failed to resolve a Reference class"))?; - let name_fn = ce_method_ref(reference_ce, "name")?; - let name_zv = call_handle_on(name_fn, reference_obj, &mut [], "read a definition name")?; - let numeric_key = array_key_index( - name_zv - .zend_str() - .ok_or_else(|| ext_exception("flow_php expected a definition name to be a string"))? - .as_bytes(), - ); - - let (type_fn, metadata_fn) = def_dehydrate_fns(def_fn_cache, definition_ce)?; - - let type_zv = call_handle_on(type_fn, definition, &mut [], "read a definition type")?; - let metadata_zv = - call_handle_on(metadata_fn, definition, &mut [], "read definition metadata")?; - - let metadata = expect_object(&metadata_zv, "definition metadata")?; - let non_empty = read_slot(metadata, metadata_map_slot) - .array() - .is_some_and(|map| !map.is_empty()); - - columns.push(DehydrateColumn { - numeric_key, - name_zv, - type_zv, - metadata_zv: if non_empty { Some(metadata_zv) } else { None }, - }); - - Ok(()) - })?; - - Ok((rows_ht, columns)) -} - -/// `TypedRowValues::metadata` as `dehydrate_row` builds it - the batch's columns with non-empty metadata, the same -/// map for every row of the batch. -pub(crate) fn batch_metadata(columns: &[DehydrateColumn]) -> ZBox { - let mut metadata_ht = ZendHashTable::new(); - - for column in columns { - if let Some(metadata) = &column.metadata_zv { - ht_insert_key(&mut metadata_ht, &column.key(), metadata.shallow_clone()); - } - } - - metadata_ht -} - -/// Native `PhpRowHydrator::dehydrate`: turns a `Flow\ETL\Rows` into a list of -/// `Flow\ETL\Row\TypedRowValues`, value zvals moved verbatim (no casting). -pub fn dehydrate_rows( - rows: &Zval, - rows_classes: &RowsClasses, - class: &TypedRowValuesClass, - def_fn_cache: &mut DefFnCache, - ctx: &mut Ctx, -) -> Result { - let (rows_ht, columns) = dehydrate_batch(rows, rows_classes, def_fn_cache, ctx)?; - - let mut out = ZendHashTable::with_capacity(rows_ht.len() as u32); - - ht_for_each(rows_ht, |_, _, row_zv| { - let row = expect_object(row_zv, "a Row")?; - let values_ht = read_slot(row, rows_classes.row_values_slot) - .array() - .ok_or_else(|| ext_exception("flow_php expected Row::values to be an array"))?; - - let row_values = dehydrate_row(values_ht, &columns, class)?; - - out.push(row_values) - .map_err(|e| ext_exception(format!("flow_php failed to collect row values: {e:?}")))?; - - Ok(()) - })?; - - let mut zv = Zval::new(); - zv.set_hashtable(out); - - Ok(zv) -} - -/// The `Row`/`Rows` class entries used to assemble a hydrated `Rows` through -/// the canonical PHP `Row` constructor and `Rows::conformed()` door, plus the -/// handles the cast path re-raises a refused value through. -pub struct AssemblyClasses { - pub row_ce: &'static ClassEntry, - /// `Rows::conformed()` - the shape-only door `HydratedBatch` returns through, - /// so both hydrators assemble the batch by the same rule - pub rows_conformed: &'static Function, - /// `Rows::trusted()` - taken instead of `rows_conformed` when conform() would return every row unchanged - pub rows_trusted: &'static Function, - pub schema_mismatch_ce: &'static ClassEntry, - /// the base `PhpRowHydrator`'s guard catches, so a refusal is recognised on - /// both paths by the same rule and anything else stays the caller's exception - pub types_exception_ce: &'static ClassEntry, - pub value_does_not_match: &'static Function, -} - -impl AssemblyClasses { - pub fn resolve() -> Result { - Ok(Self { - row_ce: find_class("Flow\\ETL\\Row")?, - rows_conformed: ce_method_ref(find_class("Flow\\ETL\\Rows")?, "conformed")?, - rows_trusted: ce_method_ref(find_class("Flow\\ETL\\Rows")?, "trusted")?, - schema_mismatch_ce: find_class("Flow\\ETL\\Exception\\SchemaMismatchException")?, - types_exception_ce: find_class("Flow\\Types\\Exception\\Exception")?, - value_does_not_match: ce_method_ref( - find_class("Flow\\ETL\\Exception\\ColumnMismatchException")?, - "valueDoesNotMatch", - )?, - }) - } -} - -/// A per-column hydrate plan entry. Storage is name-keyed and the Schema owns -/// the column's type, so a column is just its key. -pub(crate) struct HydrateColumn { - numeric_key: Option, - pub(crate) name_zv: Zval, - /// retained (refcount++) so the cast plan can read `type()` once per column - pub(crate) base_def: Zval, - /// `Definition::isNullable()`, read once at plan build - never per row - pub(crate) nullable: bool, -} - -impl HydrateColumn { - pub(crate) fn key(&self) -> HtKey<'_> { - match self.numeric_key { - Some(index) => HtKey::Index(index), - None => HtKey::Str(self.name_zv.zend_str().expect("column names are strings")), - } - } -} - -/// `Schema` is immutable - `add()`/`keep()`/`makeNullable()` return a new -/// instance carrying a freshly built `definitions` array - so the array's -/// address identifies the definition set; retaining it (refcount++) prevents -/// address reuse. Keying on the array rather than on `Schema` identity also -/// survives a caller that hands the same definitions to a new `Schema`. -/// Per-value `setMetadata` returns a new `Definition` and leaves the retained -/// ones untouched, so it needs no rebuild. -pub struct HydratePlan { - pub(crate) definitions_slot: u32, - pub(crate) definitions_retained: Zval, - pub(crate) columns: Vec, -} - -/// Builds the per-column hydrate plan from a `Schema` object. Storage is -/// name-keyed and the Schema owns the column type, so a column is just its key. -/// Method calls are fine here - this runs once per schema, never per row. -pub(crate) fn build_hydrate_plan(schema: &Zval) -> Result { - let schema_obj = expect_object(schema, "a Schema")?; - let schema_ce = unsafe { schema_obj.ce.as_ref() } - .ok_or_else(|| ext_exception("flow_php failed to resolve the Schema class"))?; - - let definitions_slot = property_offset(schema_ce, "definitions")?; - let definitions_fn = ce_method_ref(schema_ce, "definitions")?; - let definitions = call_handle_on( - definitions_fn, - schema_obj, - &mut [], - "read schema definitions", - )?; - let definitions_ht = definitions - .array() - .ok_or_else(|| ext_exception("flow_php expected Schema::definitions to return an array"))?; - - let mut columns = Vec::with_capacity(definitions_ht.len()); - - ht_for_each(definitions_ht, |_, _, def_zv| { - let def_obj = expect_object(def_zv, "a Definition")?; - let def_ce = unsafe { def_obj.ce.as_ref() } - .ok_or_else(|| ext_exception("flow_php failed to resolve a Definition class"))?; - - let entry_fn = ce_method_ref(def_ce, "entry")?; - let reference = call_handle_on(entry_fn, def_obj, &mut [], "read a definition reference")?; - let reference_obj = expect_object(&reference, "a definition reference")?; - let reference_ce = unsafe { reference_obj.ce.as_ref() } - .ok_or_else(|| ext_exception("flow_php failed to resolve a Reference class"))?; - let name_fn = ce_method_ref(reference_ce, "name")?; - let name_zv = call_handle_on(name_fn, reference_obj, &mut [], "read a definition name")?; - let numeric_key = array_key_index( - name_zv - .zend_str() - .ok_or_else(|| ext_exception("flow_php expected a definition name to be a string"))? - .as_bytes(), - ); - - let nullable = call_handle_on( - ce_method_ref(def_ce, "isNullable")?, - def_obj, - &mut [], - "read a definition nullability", - )? - .bool() - .ok_or_else(|| ext_exception("flow_php expected Definition::isNullable to return a bool"))?; - - columns.push(HydrateColumn { - numeric_key, - name_zv, - base_def: def_zv.shallow_clone(), - nullable, - }); - - Ok(()) - })?; - - Ok(HydratePlan { - definitions_slot, - definitions_retained: definitions, - columns, - }) -} - -/// `HydratedBatch` folds `RawRowValues::metadata` into the batch Schema before building rows, so a -/// column carries one Metadata rather than a per-row divergent one. The native hydrate and cast -/// paths have to fold identically or a natively hydrated batch silently loses column metadata that -/// the pure-PHP one keeps. Last write wins, matching the PHP loop order. -pub fn fold_metadata_into_schema( - schema: &Zval, - batch_ht: &ZendHashTable, - raw_class: &RowValuesClass, - ctx: &mut Ctx, -) -> Result { - let mut pending: Vec<(Vec, Zval)> = Vec::new(); - - ht_for_each(batch_ht, |_, _, rv_zv| { - let rv = expect_object(rv_zv, "a RawRowValues")?; - - let Some(metadata_ht) = read_slot(rv, raw_class.metadata_slot).array() else { - return Ok(()); - }; - - ht_for_each(metadata_ht, |key, index, metadata_zv| { - // PHP stores a numeric column name as an integer key; HydratedBatch casts it back to the name - let name = key.map_or_else(|| (index as i64).to_string().into_bytes(), |name| name.as_bytes().to_vec()); - pending.push((name, metadata_zv.shallow_clone())); - - Ok(()) - }) - })?; - - fold_pending_metadata(schema, pending, ctx) -} - -/// Folds `(column name, Metadata)` pairs, in the order the rows carried them, into the Schema. -pub fn fold_pending_metadata( - schema: &Zval, - pending: Vec<(Vec, Zval)>, - ctx: &mut Ctx, -) -> Result { - if pending.is_empty() { - return Ok(schema.shallow_clone()); - } - - let find_definition = ctx.schema_find_definition()?; - let set_metadata = ctx.schema_set_metadata()?; - let mut current = schema.shallow_clone(); - - for (name, metadata) in pending { - let object = current - .object() - .ok_or_else(|| ext_exception("flow_php expected a Schema object"))?; - - let found = call_handle_on( - find_definition, - object, - &mut [zval_str(&name)], - "Flow\\ETL\\Schema::findDefinition", - )?; - - if found.is_null() { - continue; - } - - current = call_handle_on( - set_metadata, - object, - &mut [zval_str(&name), metadata], - "Flow\\ETL\\Schema::setMetadata", - )?; - } - - Ok(current) -} diff --git a/src/extension/flow-php-ext/src/interfaces.rs b/src/extension/flow-php-ext/src/interfaces.rs new file mode 100644 index 0000000000..d98a248442 --- /dev/null +++ b/src/extension/flow-php-ext/src/interfaces.rs @@ -0,0 +1,330 @@ +//! `Flow\ETL\Column\{Column, ColumnBuilder, Backend}` and the CSV / JSON / Parquet adapters' encoder, open source and +//! open sink contracts: registered at MINIT with the package +//! files' exact signatures, so the extension's classes implement them and the package files (autoloaded only without +//! flow_php) stay unused. + +use std::sync::atomic::{AtomicPtr, Ordering}; + +use ext_php_rs::args::{Arg, ArgInfoTables}; +use ext_php_rs::builders::{ClassBuilder, FunctionBuilder}; +use ext_php_rs::error::Result; +use ext_php_rs::flags::{ClassFlags, DataType, MethodFlags}; +use ext_php_rs::zend::ClassEntry; + +const DEFINITION: DataType = DataType::object("Flow\\ETL\\Schema\\Definition"); +const TYPE: DataType = DataType::object("Flow\\Types\\Type"); +const COLUMN: DataType = DataType::object("Flow\\ETL\\Column\\Column"); +const COLUMN_BUILDER: DataType = DataType::object("Flow\\ETL\\Column\\ColumnBuilder"); +const SELF: DataType = DataType::object("self"); +const ROWS: DataType = DataType::object("Flow\\ETL\\Rows"); +const SCHEMA: DataType = DataType::object("Flow\\ETL\\Schema"); +const BACKEND: DataType = DataType::object("Flow\\ETL\\Column\\Backend"); +const ITERATOR: DataType = DataType::object("Iterator"); +const COLUMN_TYPES: DataType = DataType::object("Flow\\ETL\\Schema\\Inference\\ColumnTypes"); +const SCHEMA_INFERENCE: DataType = DataType::object("Flow\\ETL\\Schema\\Inference\\SchemaInference"); +const TYPE_NARROWER: DataType = DataType::object("Flow\\Types\\Type\\TypeNarrower"); + +// ClassEntry::try_find reads EG(class_table), which MINIT has not set, so each CE is kept from its registration +static COLUMN_CE: AtomicPtr = AtomicPtr::new(std::ptr::null_mut()); +static COLUMN_BUILDER_CE: AtomicPtr = AtomicPtr::new(std::ptr::null_mut()); +static BACKEND_CE: AtomicPtr = AtomicPtr::new(std::ptr::null_mut()); +static PARQUET_OPEN_SINK_CE: AtomicPtr = AtomicPtr::new(std::ptr::null_mut()); +static CSV_ENCODER_CE: AtomicPtr = AtomicPtr::new(std::ptr::null_mut()); +static JSON_ENCODER_CE: AtomicPtr = AtomicPtr::new(std::ptr::null_mut()); +static CSV_OPEN_SOURCE_CE: AtomicPtr = AtomicPtr::new(std::ptr::null_mut()); +static JSON_OPEN_SOURCE_CE: AtomicPtr = AtomicPtr::new(std::ptr::null_mut()); +static PARQUET_OPEN_SOURCE_CE: AtomicPtr = AtomicPtr::new(std::ptr::null_mut()); + +fn store_column(ce: &'static mut ClassEntry, arg_info: ArgInfoTables) { + keep(arg_info); + COLUMN_CE.store(ce, Ordering::Release); +} + +fn store_column_builder(ce: &'static mut ClassEntry, arg_info: ArgInfoTables) { + keep(arg_info); + COLUMN_BUILDER_CE.store(ce, Ordering::Release); +} + +fn store_backend(ce: &'static mut ClassEntry, arg_info: ArgInfoTables) { + keep(arg_info); + BACKEND_CE.store(ce, Ordering::Release); +} + +fn store_parquet_open_sink(ce: &'static mut ClassEntry, arg_info: ArgInfoTables) { + keep(arg_info); + PARQUET_OPEN_SINK_CE.store(ce, Ordering::Release); +} + +fn store_csv_encoder(ce: &'static mut ClassEntry, arg_info: ArgInfoTables) { + keep(arg_info); + CSV_ENCODER_CE.store(ce, Ordering::Release); +} + +fn store_json_encoder(ce: &'static mut ClassEntry, arg_info: ArgInfoTables) { + keep(arg_info); + JSON_ENCODER_CE.store(ce, Ordering::Release); +} + +fn store_csv_open_source(ce: &'static mut ClassEntry, arg_info: ArgInfoTables) { + keep(arg_info); + CSV_OPEN_SOURCE_CE.store(ce, Ordering::Release); +} + +fn store_json_open_source(ce: &'static mut ClassEntry, arg_info: ArgInfoTables) { + keep(arg_info); + JSON_OPEN_SOURCE_CE.store(ce, Ordering::Release); +} + +fn store_parquet_open_source(ce: &'static mut ClassEntry, arg_info: ArgInfoTables) { + keep(arg_info); + PARQUET_OPEN_SOURCE_CE.store(ce, Ordering::Release); +} + +/// The engine borrows an interface's argument info tables for the life of the process. +fn keep(arg_info: ArgInfoTables) { + std::mem::forget(arg_info); +} + +fn load(ce: &AtomicPtr, name: &str) -> &'static ClassEntry { + unsafe { ce.load(Ordering::Acquire).as_ref() }.unwrap_or_else(|| panic!("{name} not registered")) +} + +pub fn column_ce() -> &'static ClassEntry { + load(&COLUMN_CE, "Flow\\ETL\\Column\\Column") +} + +pub fn column_builder_ce() -> &'static ClassEntry { + load(&COLUMN_BUILDER_CE, "Flow\\ETL\\Column\\ColumnBuilder") +} + +pub fn backend_ce() -> &'static ClassEntry { + load(&BACKEND_CE, "Flow\\ETL\\Column\\Backend") +} + +pub fn parquet_open_sink_ce() -> &'static ClassEntry { + load(&PARQUET_OPEN_SINK_CE, "Flow\\ETL\\Adapter\\Parquet\\ParquetOpenSink") +} + +pub fn csv_encoder_ce() -> &'static ClassEntry { + load(&CSV_ENCODER_CE, "Flow\\ETL\\Adapter\\CSV\\CSVEncoder") +} + +pub fn json_encoder_ce() -> &'static ClassEntry { + load(&JSON_ENCODER_CE, "Flow\\ETL\\Adapter\\JSON\\JsonEncoder") +} + +pub fn csv_open_source_ce() -> &'static ClassEntry { + load(&CSV_OPEN_SOURCE_CE, "Flow\\ETL\\Adapter\\CSV\\CSVOpenSource") +} + +pub fn json_open_source_ce() -> &'static ClassEntry { + load(&JSON_OPEN_SOURCE_CE, "Flow\\ETL\\Adapter\\JSON\\JsonOpenSource") +} + +pub fn parquet_open_source_ce() -> &'static ClassEntry { + load( + &PARQUET_OPEN_SOURCE_CE, + "Flow\\ETL\\Adapter\\Parquet\\ParquetOpenSource", + ) +} + +fn method(name: &str, args: Vec>, returns: DataType) -> FunctionBuilder<'static> { + args.into_iter() + .fold(FunctionBuilder::new_abstract(name), FunctionBuilder::arg) + .returns(returns, false, false) +} + +fn interface( + name: &str, + store: fn(&'static mut ClassEntry, ArgInfoTables), + methods: Vec>, +) -> Result<()> { + methods + .into_iter() + .fold( + ClassBuilder::new(name).flags(ClassFlags::Interface).registration(store), + |class, method| class.method(method, MethodFlags::Public | MethodFlags::Abstract), + ) + .register() +} + +pub fn register() -> Result<()> { + interface( + "Flow\\ETL\\Column\\Column", + store_column, + vec![ + method("type", vec![], TYPE), + method("count", vec![], DataType::Long), + method("isNull", vec![Arg::new("i", DataType::Long)], DataType::Bool), + method("nullCount", vec![], DataType::Long), + method("at", vec![Arg::new("i", DataType::Long)], DataType::Mixed), + method("physicals", vec![], DataType::Array), + method("value", vec![Arg::new("i", DataType::Long)], DataType::Mixed), + method("values", vec![], DataType::Array), + method( + "slice", + vec![Arg::new("offset", DataType::Long), Arg::new("length", DataType::Long)], + SELF, + ), + method("take", vec![Arg::new("indices", DataType::Array)], SELF), + method("withType", vec![Arg::new("type", TYPE)], SELF), + method("concat", vec![Arg::new("others", SELF).is_variadic()], SELF), + method("encode", vec![], DataType::Array), + ], + )?; + interface( + "Flow\\ETL\\Column\\ColumnBuilder", + store_column_builder, + vec![ + method("append", vec![Arg::new("value", DataType::Mixed)], DataType::Void), + method("appendMany", vec![Arg::new("values", DataType::Array)], DataType::Void), + method( + "appendPhysicals", + vec![ + Arg::new("physicals", DataType::Array), + Arg::new("nullCount", DataType::Long).allow_null().default("null"), + ], + DataType::Void, + ) + .required_args(1), + method( + "appendFrom", + vec![Arg::new("column", COLUMN), Arg::new("i", DataType::Long)], + DataType::Void, + ), + method( + "appendTake", + vec![Arg::new("column", COLUMN), Arg::new("indices", DataType::Array)], + DataType::Void, + ), + method("count", vec![], DataType::Long), + method("finish", vec![], COLUMN), + ], + )?; + interface( + "Flow\\ETL\\Column\\Backend", + store_backend, + vec![ + method("builder", vec![Arg::new("definition", DEFINITION)], COLUMN_BUILDER), + method( + "constant", + vec![ + Arg::new("definition", DEFINITION), + Arg::new("value", DataType::Mixed), + Arg::new("count", DataType::Long), + ], + COLUMN, + ), + method( + "decode", + vec![ + Arg::new("definition", DEFINITION), + Arg::new("buffers", DataType::Array), + Arg::new("count", DataType::Long), + Arg::new("nullCount", DataType::Long), + ], + COLUMN, + ), + method( + "adopt", + vec![Arg::new("definition", DEFINITION), Arg::new("column", COLUMN)], + COLUMN, + ), + method("allocatedBytes", vec![], DataType::Long), + ], + )?; + interface( + "Flow\\ETL\\Adapter\\Parquet\\ParquetOpenSink", + store_parquet_open_sink, + vec![ + method("close", vec![], DataType::Void), + method("write", vec![Arg::new("rows", ROWS)], DataType::Void), + ], + )?; + interface( + "Flow\\ETL\\Adapter\\CSV\\CSVEncoder", + store_csv_encoder, + vec![ + method("encode", vec![Arg::new("rows", ROWS)], DataType::String), + method( + "encodeHeader", + vec![Arg::new("headers", DataType::Array)], + DataType::String, + ), + ], + )?; + interface( + "Flow\\ETL\\Adapter\\JSON\\JsonEncoder", + store_json_encoder, + vec![method( + "encode", + vec![Arg::new("rows", ROWS), Arg::new("separator", DataType::String)], + DataType::String, + )], + )?; + interface( + "Flow\\ETL\\Adapter\\CSV\\CSVOpenSource", + store_csv_open_source, + vec![ + method("close", vec![], DataType::Void), + method("producedBytes", vec![], DataType::Long), + method("producedRows", vec![], DataType::Long), + method("columns", vec![], DataType::Array), + method( + "batches", + vec![ + Arg::new("schema", SCHEMA), + Arg::new("batchSize", DataType::Long), + Arg::new("backend", BACKEND), + ], + ITERATOR, + ), + method("headers", vec![], DataType::Array), + method("records", vec![], ITERATOR), + method( + "sniff", + vec![ + Arg::new("names", DataType::Array), + Arg::new("rowBudget", DataType::Long), + Arg::new("inference", SCHEMA_INFERENCE), + Arg::new("typer", TYPE_NARROWER), + ], + COLUMN_TYPES, + ), + ], + )?; + interface( + "Flow\\ETL\\Adapter\\JSON\\JsonOpenSource", + store_json_open_source, + vec![ + method("close", vec![], DataType::Void), + method( + "batches", + vec![ + Arg::new("schema", SCHEMA), + Arg::new("batchSize", DataType::Long), + Arg::new("backend", BACKEND), + ], + ITERATOR, + ), + ], + )?; + interface( + "Flow\\ETL\\Adapter\\Parquet\\ParquetOpenSource", + store_parquet_open_source, + vec![ + method( + "batches", + vec![ + Arg::new("schema", SCHEMA), + Arg::new("batchSize", DataType::Long), + Arg::new("offset", DataType::Long).allow_null(), + Arg::new("limit", DataType::Long).allow_null(), + Arg::new("backend", BACKEND), + ], + ITERATOR, + ), + method("close", vec![], DataType::Void), + ], + ) +} diff --git a/src/extension/flow-php-ext/src/iterator.rs b/src/extension/flow-php-ext/src/iterator.rs new file mode 100644 index 0000000000..6af28c6476 --- /dev/null +++ b/src/extension/flow-php-ext/src/iterator.rs @@ -0,0 +1,117 @@ +//! `Flow\ETL\RustIterator`: the `\Iterator` the Rust open sources return from `batches()` / `records()` - keys 0..n, +//! each value pulled from a Rust producer on demand. It behaves as the Generator it replaced: it streams once +//! (`rewind()` after `next()` throws) and ends for good when the producer throws. + +use ext_php_rs::flags::ClassFlags; +use ext_php_rs::prelude::*; +use ext_php_rs::types::Zval; +use ext_php_rs::zend::ce; + +use crate::render::runtime; + +pub type Producer = Box PhpResult>>; + +#[php_class] +#[php(name = "Flow\\ETL\\RustIterator", flags = ClassFlags::Final, implements(ce = ce::iterator, stub = "\\Iterator"))] +pub struct RustIterator { + producer: Producer, + current: Option, + /// The index of `current`. + key: i64, + started: bool, + /// `next()` moved past the first value: `rewind()` throws from here on. + advanced: bool, + /// The producer returned its last value or threw: it is never called again. + done: bool, +} + +impl RustIterator { + pub fn new(producer: Producer) -> Self { + Self { + producer, + current: None, + key: 0, + started: false, + advanced: false, + done: false, + } + } + + /// A Generator starts on its first `rewind()`, `valid()`, `current()`, `key()` or `next()`. + fn start(&mut self) -> PhpResult<()> { + if self.started { + return Ok(()); + } + + self.started = true; + self.pull() + } + + fn pull(&mut self) -> PhpResult<()> { + self.current = None; + + if self.done { + return Ok(()); + } + + match (self.producer)() { + Ok(Some(value)) => { + self.current = Some(value); + + Ok(()) + } + Ok(None) => { + self.done = true; + + Ok(()) + } + Err(error) => { + self.done = true; + + Err(error) + } + } + } +} + +#[php_impl] +impl RustIterator { + pub fn rewind(&mut self) -> PhpResult<()> { + if self.advanced { + return Err(runtime("RustIterator cannot rewind".to_string())); + } + + self.start() + } + + pub fn valid(&mut self) -> PhpResult { + self.start()?; + + Ok(self.current.is_some()) + } + + pub fn current(&mut self) -> PhpResult { + self.start()?; + + Ok(self.current.as_ref().map_or_else(Zval::new, Zval::shallow_clone)) + } + + /// Null once the iterator ended, as a Generator's. + pub fn key(&mut self) -> PhpResult> { + self.start()?; + + Ok(self.current.is_some().then_some(self.key)) + } + + pub fn next(&mut self) -> PhpResult<()> { + self.start()?; + + if self.done { + return Ok(()); + } + + self.advanced = true; + self.key += 1; + self.pull() + } +} diff --git a/src/extension/flow-php-ext/src/json/mod.rs b/src/extension/flow-php-ext/src/json/mod.rs new file mode 100644 index 0000000000..1dd0818c44 --- /dev/null +++ b/src/extension/flow-php-ext/src/json/mod.rs @@ -0,0 +1,7 @@ +pub mod read; +#[allow( + non_snake_case, + reason = "PHP takes a parameter name from its Rust identifier: the contracts' are camelCase" +)] +pub mod source; +pub mod write; diff --git a/src/extension/flow-php-ext/src/json/read.rs b/src/extension/flow-php-ext/src/json/read.rs new file mode 100644 index 0000000000..7367e286ed --- /dev/null +++ b/src/extension/flow-php-ext/src/json/read.rs @@ -0,0 +1,1353 @@ +//! The reader behind `RustJsonOpenSource`: JSON lines or a top-level JSON array, fed in chunks, read into native +//! columns by a schema with `RowsBuilder::appendRows()`'s results and refusals. This file frames records and maps their +//! members to columns; serde_json parses and validates. + +use std::borrow::Cow; +use std::collections::hash_map::Entry; +use std::collections::{HashMap, HashSet}; +use std::fmt; + +use ext_php_rs::exception::PhpException; +use ext_php_rs::prelude::*; +use ext_php_rs::types::Zval; +use flow_batch_frame::kind::{Field, Kind}; +use serde::de::{self, DeserializeSeed, Deserializer as _, IgnoredAny, MapAccess, SeqAccess, Visitor}; +use serde::Deserialize; +use serde_json::de::{SliceRead, StrRead}; +use serde_json::value::RawValue; +use serde_json::Deserializer; + +use crate::batch_columns::{native_text, BatchColumns, HeldBatch, Key}; +use crate::builder::{append_native, append_one, native_leaf, Append}; +use crate::cast::{CastKind, MapKeyKind}; +use crate::ctx::{self, array_key_index, null_zval, zval_str}; +use crate::exception::ext_exception; +use crate::kind_builder::KindBuilder; +use crate::physical::overflow; +use crate::render::runtime; + +/// JSON's insignificant whitespace. +const WHITESPACE: &[u8] = b" \t\n\r"; +/// `JsonFileReader::C_ISSPACE`: a line of only these bytes is no record. +const BLANK: &[u8] = b" \t\n\r\x0B\x0C"; +const BOM: &[u8] = b"\xEF\xBB\xBF"; +/// `json_decode()`'s default depth: a record nested this deep is refused, one level less is read. +const DEPTH: usize = 512; + +fn is_whitespace(byte: &u8) -> bool { + WHITESPACE.contains(byte) +} + +/// Why a record cannot be read; `ordinal` is its 1-based line or 0-based element. +#[derive(Debug, PartialEq)] +pub struct Malformed { + pub ordinal: usize, + pub message: String, +} + +/// One framed record: `records[start..end]`. +pub struct Record { + pub start: usize, + pub end: usize, + pub ordinal: usize, +} + +/// The document scanner: where the element being framed stands. +#[derive(Default)] +struct Scan { + depth: usize, + in_string: bool, + escape: bool, +} + +struct TooDeep; + +impl Scan { + /// From `bytes[from..]` on: `Some(i)` is the depth-0 `,` or `]` at `i`, `None` means `bytes` ends inside the value. + fn run(&mut self, bytes: &[u8], mut i: usize) -> Result, TooDeep> { + while i < bytes.len() { + if self.in_string { + if self.escape { + self.escape = false; + i += 1; + + continue; + } + + let Some(found) = memchr::memchr2(b'"', b'\\', &bytes[i..]) else { + return Ok(None); + }; + + i += found; + + if bytes[i] == b'\\' { + self.escape = true; + } else { + self.in_string = false; + } + + i += 1; + + continue; + } + + match bytes[i] { + b'"' => self.in_string = true, + b'{' | b'[' => { + self.depth += 1; + + if self.depth >= DEPTH { + return Err(TooDeep); + } + } + b'}' | b']' if self.depth > 0 => self.depth -= 1, + b']' | b',' if self.depth == 0 => return Ok(Some(i)), + _ => {} + } + + i += 1; + } + + Ok(None) + } +} + +const TOO_DEEP: &str = "Maximum stack depth exceeded"; +const LONE_SURROGATE: &str = "Single unpaired UTF-16 surrogate in unicode escape"; + +/// `{}` or `[]`: a record with no fields, skipped as `JsonFileReader::sample()` skips it. +fn empty_record(bytes: &[u8]) -> bool { + let start = bytes + .iter() + .position(|byte| !is_whitespace(byte)) + .unwrap_or(bytes.len()); + let end = bytes + .iter() + .rposition(|byte| !is_whitespace(byte)) + .map_or(start, |end| end + 1); + let trimmed = &bytes[start..end]; + + matches!( + (trimmed.first(), trimmed.last()), + (Some(b'{'), Some(b'}')) | (Some(b'['), Some(b']')) + ) && trimmed.len() >= 2 + && trimmed[1..trimmed.len() - 1].iter().all(is_whitespace) +} + +enum Document { + Start, + Elements, + Closed, +} + +/// Splits fed bytes into records without parsing values: lines on `\n`, a document's elements on its depth-0 `,` and +/// `]`. Holds the unframed input only. +pub struct Framing { + lines: bool, + input: Vec, + /// The first byte of `input` not yet framed. + start: usize, + /// How far past `start` the current record's end was searched. + scanned: usize, + scan: Scan, + document: Document, + bom: bool, + /// The next line (1-based) or element (0-based). + ordinal: usize, + finished: bool, +} + +impl Framing { + pub fn new(lines: bool) -> Self { + Self { + lines, + input: Vec::new(), + start: 0, + scanned: 0, + scan: Scan::default(), + document: Document::Start, + bom: true, + ordinal: usize::from(lines), + finished: false, + } + } + + pub fn feed(&mut self, chunk: &[u8]) { + if self.start > 0 && self.start * 2 >= self.input.len() { + self.input.drain(..self.start); + self.start = 0; + } + + self.input.extend_from_slice(chunk); + } + + pub fn finish(&mut self) { + self.finished = true; + } + + /// Copies the next record into `records`; `false` when no complete record is buffered. + pub fn next(&mut self, records: &mut Vec, bounds: &mut Vec) -> Result { + if self.bom { + let pending = &self.input[self.start..]; + + if pending.len() < BOM.len() && BOM.starts_with(pending) && !self.finished { + return Ok(false); + } + + if pending.starts_with(BOM) { + self.start += BOM.len(); + } + + self.bom = false; + } + + loop { + let framed = if self.lines { self.line() } else { self.element()? }; + + let Some((start, end, ordinal)) = framed else { + return Ok(false); + }; + + let bytes = &self.input[start..end]; + + if empty_record(bytes) { + continue; + } + + if lone_surrogate(bytes) { + return Err(Malformed { + ordinal, + message: LONE_SURROGATE.to_string(), + }); + } + + bounds.push(Record { + start: records.len(), + end: records.len() + bytes.len(), + ordinal, + }); + records.extend_from_slice(bytes); + + return Ok(true); + } + } + + fn line(&mut self) -> Option<(usize, usize, usize)> { + loop { + let pending = &self.input[self.start..]; + let (end, next) = match memchr::memchr(b'\n', &pending[self.scanned..]) { + Some(found) => (self.scanned + found, self.scanned + found + 1), + None if self.finished && !pending.is_empty() => (pending.len(), pending.len()), + None => { + self.scanned = pending.len(); + + return None; + } + }; + let line = (self.start, self.start + end, self.ordinal); + + self.ordinal += 1; + self.start += next; + self.scanned = 0; + + if !self.input[line.0..line.1].iter().all(|byte| BLANK.contains(byte)) { + return Some(line); + } + } + } + + fn element(&mut self) -> Result, Malformed> { + loop { + match self.document { + Document::Start => { + let pending = &self.input[self.start..]; + let Some(first) = pending.iter().position(|byte| !is_whitespace(byte)) else { + self.start = self.input.len(); + + return match self.finished { + true => Err(self.malformed("EOF while parsing a value".to_string())), + false => Ok(None), + }; + }; + + if pending[first] != b'[' { + return Err(self.malformed("expected `[`".to_string())); + } + + self.start += first + 1; + self.document = Document::Elements; + } + Document::Elements => { + let pending = &self.input[self.start..]; + let found = self + .scan + .run(pending, self.scanned) + .map_err(|TooDeep| self.malformed(TOO_DEEP.to_string()))?; + + let Some(found) = found else { + self.scanned = pending.len(); + + if !self.finished { + return Ok(None); + } + + return Err(self.malformed( + Deserializer::from_slice(pending) + .deserialize_ignored_any(IgnoredAny) + .err() + .map_or_else(|| "EOF while parsing a list".to_string(), |e| e.to_string()), + )); + }; + + let element = (self.start, self.start + found, self.ordinal); + let closing = pending[found] == b']'; + + self.start += found + 1; + self.scanned = 0; + self.scan = Scan::default(); + + if closing { + self.document = Document::Closed; + + if self.ordinal == 0 && self.input[element.0..element.1].iter().all(is_whitespace) { + continue; + } + } + + self.ordinal += 1; + + return Ok(Some(element)); + } + Document::Closed => { + let pending = &self.input[self.start..]; + let trailing = pending.iter().any(|byte| !is_whitespace(byte)); + + self.start = self.input.len(); + + if trailing { + return Err(self.malformed("trailing characters".to_string())); + } + + return Ok(None); + } + } + } + } + + fn malformed(&self, message: String) -> Malformed { + Malformed { + ordinal: self.ordinal, + message, + } + } +} + +/// The code unit of the `\uXXXX` escape at `bytes[at]`. +fn code_unit(bytes: &[u8], at: usize) -> Option { + let hex = bytes.get(at + 2..at + 6).filter(|_| bytes.get(at + 1) == Some(&b'u'))?; + + u16::from_str_radix(std::str::from_utf8(hex).ok()?, 16).ok() +} + +/// Whether a `\u` escape of `bytes` is a UTF-16 surrogate without its pair, which `json_decode()` refuses and serde +/// checks only when it unescapes a string. A backslash outside a string is serde's syntax error, so every one is walked. +fn lone_surrogate(bytes: &[u8]) -> bool { + let mut i = 0; + let mut low_expected_at = None; + + while let Some(found) = memchr::memchr(b'\\', &bytes[i..]) { + let at = i + found; + + match (low_expected_at.take(), code_unit(bytes, at)) { + (Some(expected), Some(0xDC00..=0xDFFF)) if expected == at => {} + (Some(_), _) | (None, Some(0xDC00..=0xDFFF)) => return true, + (None, Some(0xD800..=0xDBFF)) => low_expected_at = Some(at + 6), + (None, _) => {} + } + + i = (at + if bytes.get(at + 1) == Some(&b'u') { 6 } else { 2 }).min(bytes.len()); + } + + low_expected_at.is_some() +} + +/// A line's nesting checked as the document scanner checks an element's. +fn line_too_deep(line: &[u8]) -> bool { + memchr::memchr2_iter(b'[', b'{', line).count() >= DEPTH && Scan::default().run(line, 0).is_err() +} + +/// A JSON number as `json_decode()` reads it: integer-shaped text within the i64 range is an int, any other a float. +#[derive(Debug, PartialEq)] +pub enum Number { + Long(i64), + Double(f64), +} + +pub fn number(text: &str) -> Number { + if !text.bytes().any(|byte| matches!(byte, b'.' | b'e' | b'E')) { + if let Ok(long) = text.parse::() { + return Number::Long(long); + } + } + + Number::Double(text.parse::().unwrap_or(f64::NAN)) +} + +fn number_zval(text: &str) -> Zval { + let mut zval = Zval::new(); + + match number(text) { + Number::Long(long) => zval.set_long(long), + Number::Double(double) => zval.set_double(double), + } + + zval +} + +fn bool_zval(value: bool) -> Zval { + let mut zval = Zval::new(); + zval.set_bool(value); + + zval +} + +/// `get_debug_type()` of the scalar `json_decode()` reads from `text`. +fn scalar_type(text: &str) -> &'static str { + match text.as_bytes()[0] { + b'"' => "string", + b't' | b'f' => "bool", + b'n' => "null", + _ => match number(text) { + Number::Long(_) => "int", + Number::Double(_) => "float", + }, + } +} + +#[derive(Debug, PartialEq)] +pub enum RecordError { + Malformed(String), + /// The record is a scalar of this `get_debug_type()`. + Scalar(&'static str), +} + +fn deserializer(text: &[u8]) -> Deserializer> { + let mut deserializer = Deserializer::from_slice(text); + deserializer.disable_recursion_limit(); + + deserializer +} + +fn str_deserializer(text: &str) -> Deserializer> { + let mut deserializer = Deserializer::from_str(text); + deserializer.disable_recursion_limit(); + + deserializer +} + +/// A JSON string, borrowed when it has no escape. +struct Text; + +impl<'de> Visitor<'de> for Text { + type Value = Cow<'de, str>; + + fn expecting(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + formatter.write_str("a JSON string") + } + + fn visit_borrowed_str(self, value: &'de str) -> Result { + Ok(Cow::Borrowed(value)) + } + + fn visit_str(self, value: &str) -> Result { + Ok(Cow::Owned(value.to_owned())) + } +} + +impl<'de> DeserializeSeed<'de> for Text { + type Value = Cow<'de, str>; + + fn deserialize>(self, deserializer: D) -> Result { + deserializer.deserialize_str(self) + } +} + +/// The column each member feeds, resolved once per schema: an object's members by name, an array record's elements +/// by position (a definition keyed by that int). +#[derive(Default)] +pub struct MemberColumns { + names: HashMap, usize>, + positions: Vec>, +} + +impl MemberColumns { + pub fn of<'a>(keys: impl Iterator) -> Self { + let mut columns = Self::default(); + + for (column, key) in keys.enumerate() { + if let Key::Index(index) = key { + if let Ok(position) = usize::try_from(*index) { + columns + .positions + .resize(columns.positions.len().max(position + 1), None); + columns.positions[position] = Some(column); + } + } + + columns.names.insert(key.name(), column); + } + + columns + } +} + +/// A record's members, each stored as the span of its value in `slots[column]`. +struct Members<'a> { + columns: &'a MemberColumns, + slots: &'a mut [Option<(usize, usize)>], + base: usize, +} + +impl Members<'_> { + fn store(&mut self, column: Option, value: &RawValue) { + if let Some(column) = column { + let start = value.get().as_ptr() as usize - self.base; + + self.slots[column] = Some((start, start + value.get().len())); + } + } +} + +impl<'de> Visitor<'de> for Members<'_> { + type Value = (); + + fn expecting(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + formatter.write_str("a JSON object or array") + } + + fn visit_map>(mut self, mut map: A) -> Result<(), A::Error> { + while let Some(name) = map.next_key_seed(Text)? { + let value: &'de RawValue = map.next_value()?; + + self.store(self.columns.names.get(name.as_bytes()).copied(), value); + } + + Ok(()) + } + + fn visit_seq>(mut self, mut seq: A) -> Result<(), A::Error> { + let mut position = 0; + + while let Some(value) = seq.next_element::<&'de RawValue>()? { + self.store(self.columns.positions.get(position).copied().flatten(), value); + position += 1; + } + + Ok(()) + } +} + +/// Maps one record's members onto `slots`, spans relative to `base` (the address of the batch's first byte). +pub fn map_record( + record: &[u8], + columns: &MemberColumns, + slots: &mut [Option<(usize, usize)>], + base: usize, +) -> Result<(), RecordError> { + let malformed = |e: serde_json::Error| RecordError::Malformed(e.to_string()); + + if let Some(b'{' | b'[') = record.iter().find(|byte| !is_whitespace(byte)) { + let mut deserializer = deserializer(record); + deserializer + .deserialize_any(Members { columns, slots, base }) + .map_err(malformed)?; + + return deserializer.end().map_err(malformed); + } + + let mut deserializer = deserializer(record); + let value = <&RawValue>::deserialize(&mut deserializer).map_err(malformed)?; + deserializer.end().map_err(malformed)?; + + Err(RecordError::Scalar(scalar_type(value.get()))) +} + +fn custom(failure: &mut Option, cause: PhpException) -> E { + *failure = Some(cause); + + E::custom("flow_php refused a JSON value") +} + +/// A value `map_record()` validated (surrogates by `Framing`) that serde reads again cannot fail. +fn reread(error: serde_json::Error) -> PhpException { + ext_exception(format!("flow_php failed to read validated JSON again: {error}")) +} + +/// One value appended as `append_cast()` would append what `json_decode()` reads from it; `false` leaves a partial row +/// for the caller to truncate before the PHP lane takes the whole value. +fn into_column(values: &mut KindBuilder, kind: &Kind, cast: &CastKind, text: &str) -> Result { + if let CastKind::Optional(inner) = cast { + if text == "null" { + values.append_null(); + + return Ok(true); + } + + return into_column(values, kind, inner, text); + } + + let leaf = |values: &mut KindBuilder, value: Zval| -> Result { + match native_leaf(kind, cast, &value)? { + Some(native) => { + append_native(values, native)?; + + Ok(true) + } + None => Ok(false), + } + }; + + match text.as_bytes()[0] { + b'"' => { + let string = match text.contains('\\') { + true => str_deserializer(text).deserialize_str(Text).map_err(reread)?, + false => Cow::Borrowed(&text[1..text.len() - 1]), + }; + + native_text(values, kind, cast, string.as_bytes()) + } + b'n' => Ok(false), + b't' | b'f' => leaf(values, bool_zval(text == "true")), + b'[' => match (kind, cast) { + (Kind::List(element), CastKind::List(inner)) => { + let mut failure = None; + let cast = str_deserializer(text).deserialize_seq(Elements { + values: values.list_element(), + kind: &element.kind, + cast: inner, + failure: &mut failure, + }); + + match (cast, failure) { + (_, Some(failure)) => Err(failure), + (Err(e), None) => Err(reread(e)), + (Ok(false), None) => Ok(false), + (Ok(true), None) => { + values.end_entries().map_err(overflow)?; + + Ok(true) + } + } + } + _ => Ok(false), + }, + b'{' => match (kind, cast) { + (Kind::Map(key, item), CastKind::Map(key_cast, inner)) => { + let entries = str_deserializer(text).deserialize_map(Entries).map_err(reread)?; + + for (name, value) in last_wins(entries) { + let (keys, items) = values.map_entries(); + let cast = match (key_cast, array_key_index(name.as_bytes()), &key.kind) { + (MapKeyKind::Int, Some(index), Kind::Int64) => { + keys.append_fixed(&index.to_le_bytes()); + + true + } + (MapKeyKind::Str, None, Kind::Bytes) => { + keys.append_bytes(name.as_bytes()).map_err(overflow)?; + + true + } + _ => false, + } && into_column(items, &item.kind, inner, value.get())?; + + if !cast { + return Ok(false); + } + } + + values.end_entries().map_err(overflow)?; + + Ok(true) + } + (Kind::Struct(fields), CastKind::Structure(elements)) => { + let members = str_deserializer(text).deserialize_map(Fields(fields)).map_err(reread)?; + + for (((child, field), element), member) in values + .struct_children() + .iter_mut() + .zip(fields) + .zip(elements) + .zip(members) + { + match member.map(RawValue::get) { + None if element.required => return Ok(false), + None => child.append_null(), + Some("null") if !matches!(element.kind, CastKind::Optional(_)) => return Ok(false), + Some(value) => { + if !into_column(child, &field.kind, &element.kind, value)? { + return Ok(false); + } + } + } + } + + values.end_struct(); + + Ok(true) + } + _ => Ok(false), + }, + _ => leaf(values, number_zval(text)), + } +} + +/// The elements of a list, each through `into_column()`; the rest is skipped once one is not cast. +struct Elements<'a> { + values: &'a mut KindBuilder, + kind: &'a Kind, + cast: &'a CastKind, + failure: &'a mut Option, +} + +impl<'de> Visitor<'de> for Elements<'_> { + type Value = bool; + + fn expecting(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + formatter.write_str("a JSON array") + } + + fn visit_seq>(self, mut seq: A) -> Result { + let Elements { + values, + kind, + cast, + failure, + } = self; + + while let Some(value) = seq.next_element::<&'de RawValue>()? { + match into_column(values, kind, cast, value.get()) { + Ok(true) => {} + Ok(false) => { + while seq.next_element::()?.is_some() {} + + return Ok(false); + } + Err(cause) => return Err(custom(failure, cause)), + } + } + + Ok(true) + } +} + +/// An object's members in document order, duplicates included. +struct Entries; + +impl<'de> Visitor<'de> for Entries { + type Value = Vec<(Cow<'de, str>, &'de RawValue)>; + + fn expecting(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + formatter.write_str("a JSON object") + } + + fn visit_map>(self, mut map: A) -> Result { + let mut entries = Vec::with_capacity(map.size_hint().unwrap_or(0)); + + while let Some(name) = map.next_key_seed(Text)? { + entries.push((name, map.next_value::<&'de RawValue>()?)); + } + + Ok(entries) + } +} + +/// An object's members by structure field, the last duplicate winning; a member no field names is skipped unread. +struct Fields<'a>(&'a [Field]); + +impl<'de> Visitor<'de> for Fields<'_> { + type Value = Vec>; + + fn expecting(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + formatter.write_str("a JSON object") + } + + fn visit_map>(self, mut map: A) -> Result { + let mut members = vec![None; self.0.len()]; + + while let Some(name) = map.next_key_seed(Text)? { + match self.0.iter().position(|field| field.name == name) { + Some(position) => members[position] = Some(map.next_value::<&'de RawValue>()?), + None => { + map.next_value::()?; + } + } + } + + Ok(members) + } +} + +/// Up to this many members a repeated key is looked for pair by pair (at most 120 comparisons of short keys), past it +/// through a hash set: hashing every key and allocating the set costs more than the comparisons it saves. +const PAIRWISE_DUPLICATE_CHECK: usize = 16; + +/// `json_decode()`'s array of the members: a repeated key keeps its first position and its last value. +fn last_wins<'a>(entries: Vec<(Cow<'a, str>, &'a RawValue)>) -> Vec<(Cow<'a, str>, &'a RawValue)> { + let repeated = if entries.len() <= PAIRWISE_DUPLICATE_CHECK { + entries + .iter() + .enumerate() + .any(|(i, (name, _))| entries[..i].iter().any(|(earlier, _)| earlier == name)) + } else { + let mut seen = HashSet::with_capacity(entries.len()); + + !entries.iter().all(|(name, _)| seen.insert(name.as_ref())) + }; + + if !repeated { + return entries; + } + + let mut positions: HashMap<&str, usize> = HashMap::with_capacity(entries.len()); + let mut kept: Vec<(usize, usize)> = Vec::with_capacity(entries.len()); + + for (index, (name, _)) in entries.iter().enumerate() { + match positions.entry(name.as_ref()) { + Entry::Occupied(position) => kept[*position.get()].1 = index, + Entry::Vacant(position) => { + position.insert(kept.len()); + kept.push((index, index)); + } + } + } + + kept.into_iter() + .map(|(key, value)| (entries[key].0.clone(), entries[value].1)) + .collect() +} + +/// The batch being built and the column each member feeds. +#[derive(Default)] +struct JsonColumns { + batch: Option, + members: MemberColumns, +} + +pub struct JsonReader { + uri: String, + framing: Framing, + /// The framed records of the pending batch, copied out of the input. + records: Vec, + bounds: Vec, + slots: Vec>, + columns: JsonColumns, +} + +/// `Malformed JSON in "" at line N: …` (lines, 1-based) or `at element N` (a document, 0-based). +fn malformed(uri: &str, lines: bool, ordinal: usize, message: &str) -> PhpException { + runtime(format!( + "Malformed JSON in \"{uri}\" at {} {ordinal}: {message}", + if lines { "line" } else { "element" }, + )) +} + +impl JsonReader { + fn clear(&mut self) { + self.records.clear(); + self.bounds.clear(); + + if let Some(batch) = self.columns.batch.as_mut() { + batch.reset(); + } + } + + fn read(&mut self, schema: &Zval, batch_size: usize) -> PhpResult> { + let (state, rebuilt) = BatchColumns::prepare(&mut self.columns.batch, schema, batch_size, "JSON")?; + + if rebuilt { + self.columns.members = MemberColumns::of(state.columns.iter().map(|column| &column.key)); + } + + let batch_size = state.batch_size; + let lines = self.framing.lines; + + while self.bounds.len() < batch_size { + match self.framing.next(&mut self.records, &mut self.bounds) { + Ok(true) => {} + Ok(false) => break, + Err(Malformed { ordinal, message }) => return Err(malformed(&self.uri, lines, ordinal, &message)), + } + + if lines { + let record = self.bounds.last().expect("framed above"); + + if line_too_deep(&self.records[record.start..record.end]) { + return Err(malformed(&self.uri, lines, record.ordinal, TOO_DEEP)); + } + } + } + + let state = self.columns.batch.as_mut().expect("prepared above"); + state.rows = self.bounds.len(); + + if self.bounds.is_empty() || (self.bounds.len() < state.batch_size && !self.framing.finished) { + return Ok(None); + } + + let width = state.columns.len(); + let base = self.records.as_ptr() as usize; + + self.slots.clear(); + self.slots.resize(self.bounds.len() * width, None); + + for (row, record) in self.bounds.iter().enumerate() { + let slots = &mut self.slots[row * width..(row + 1) * width]; + + match map_record( + &self.records[record.start..record.end], + &self.columns.members, + slots, + base, + ) { + Ok(()) => {} + Err(RecordError::Malformed(message)) => { + return Err(malformed(&self.uri, lines, record.ordinal, &message)) + } + Err(RecordError::Scalar(given)) => { + return Err(runtime(format!( + "A JSON record must be an object or an array, {given} given in \"{}\".", + self.uri, + ))); + } + } + } + + for (position, column) in state.columns.iter_mut().enumerate() { + for row in 0..self.bounds.len() { + if column.refusal.is_some() { + break; + } + + let Some((start, end)) = self.slots[row * width + position] else { + if !column.nullable { + column.absent = column.absent.or(Some(row)); + } + + column.values.append_null(); + + continue; + }; + // SAFETY: every record of the batch passed map_record(), whose serde parse checked the UTF-8 of each + // member value; the span is one of those values, so it starts and ends on character boundaries + let text = unsafe { std::str::from_utf8_unchecked(&self.records[start..end]) }; + + let value = if text == "null" { + null_zval() + } else { + let len = column.values.len(); + + if into_column(&mut column.values, &column.plan.kind, &column.plan.cast, text)? { + continue; + } + + column.values.truncate(len); + + decode(text)? + }; + + if let Append::Refused { cause, .. } = append_one( + &mut column.values, + &column.plan, + &column.definition, + column.nullable, + &value, + )? { + column.refusal = Some((row, cause)); + } + } + } + + self.records.clear(); + self.bounds.clear(); + + state.finish().map(Some) + } +} + +/// `json_decode($text, true)`: PHP's own value of a JSON value `into_column()` did not cast. +fn decode(text: &str) -> Result { + let mut associative = Zval::new(); + associative.set_bool(true); + let value = ctx::call_handle( + ctx::json_decode()?, + None, + &mut [zval_str(text.as_bytes()), associative], + "decode JSON", + )?; + + if value.is_null() { + return Err(ext_exception("flow_php expected json_decode() to read validated JSON")); + } + + Ok(value) +} + +impl JsonReader { + /// `uri` names the file in refusals. + pub fn new(lines: bool, uri: &[u8]) -> Self { + Self { + uri: String::from_utf8_lossy(uri).into_owned(), + framing: Framing::new(lines), + records: Vec::new(), + bounds: Vec::new(), + slots: Vec::new(), + columns: JsonColumns::default(), + } + } + + pub fn feed(&mut self, chunk: &[u8]) { + self.framing.feed(chunk); + } + + /// Malformed input left in the buffer (a truncated record, a missing `]`) is refused by the next `nextColumns()`. + pub fn finish(&mut self) { + self.framing.finish(); + } + + /// Exactly `batch_size` rows keyed and ordered by `schema` while that many are buffered, the remainder after + /// `finish()`, else none. Refusals are `RowsBuilder::appendRows()`'s, row indexes relative to the batch. + pub fn next_columns(&mut self, schema: &Zval, batch_size: usize) -> PhpResult> { + let batch = self.read(schema, batch_size); + + if batch.is_err() { + self.clear(); + } + + batch + } +} + +#[cfg(test)] +mod tests { + use serde::de::Deserializer as _; + + use super::{ + last_wins, map_record, number, str_deserializer, Entries, Framing, Key, Malformed, MemberColumns, Number, + Record, RecordError, + }; + use crate::ctx::array_key_index; + + fn framed(lines: bool, chunks: &[&[u8]]) -> Result, Malformed> { + let mut framing = Framing::new(lines); + let mut records = Vec::new(); + let mut bounds: Vec = Vec::new(); + + for chunk in chunks { + framing.feed(chunk); + + while framing.next(&mut records, &mut bounds)? {} + } + + framing.finish(); + + while framing.next(&mut records, &mut bounds)? {} + + Ok(bounds + .iter() + .map(|record| { + ( + String::from_utf8(records[record.start..record.end].to_vec()).unwrap(), + record.ordinal, + ) + }) + .collect()) + } + + fn bytes(text: &str) -> Vec<&[u8]> { + text.as_bytes().chunks(1).collect() + } + + fn nested(depth: usize) -> String { + format!("{}1{}", "[".repeat(depth), "]".repeat(depth)) + } + + fn malformed(ordinal: usize, message: &str) -> Result, Malformed> { + Err(Malformed { + ordinal, + message: message.to_string(), + }) + } + + #[test] + fn a_nested_record_fed_byte_by_byte_is_one_record() { + let document = r#"[{"a":{"b":[1,"x]\"",{"c":2}]}},{"d":3}]"#; + + assert_eq!( + framed(false, &bytes(document)).unwrap(), + vec![ + (r#"{"a":{"b":[1,"x]\"",{"c":2}]}}"#.to_string(), 0), + (r#"{"d":3}"#.to_string(), 1) + ], + ); + assert_eq!( + framed(true, &bytes("{\"a\":[1,\n")).unwrap(), + vec![(r#"{"a":[1,"#.to_string(), 1)] + ); + } + + #[test] + fn an_element_waits_for_its_end_across_chunks() { + assert_eq!( + framed(false, &[b"[{\"id\":1", b"2}, {\"id\"", b":3}\n]"]).unwrap(), + vec![(r#"{"id":12}"#.to_string(), 0), (r#" {"id":3}"#.to_string() + "\n", 1)], + ); + } + + #[test] + fn one_leading_bom_is_skipped() { + assert_eq!( + framed(false, &[b"\xEF\xBB", b"\xBF [1]"]).unwrap(), + vec![("1".to_string(), 0)] + ); + assert_eq!( + framed(true, &[b"\xEF\xBB\xBF{\"a\":1}\n"]).unwrap(), + vec![(r#"{"a":1}"#.to_string(), 1)] + ); + assert_eq!( + framed(true, &[b"{\"a\":1}\n\xEF\xBB\xBF{\"a\":2}"]).unwrap(), + vec![(r#"{"a":1}"#.to_string(), 1), ("\u{FEFF}{\"a\":2}".to_string(), 2)], + ); + } + + #[test] + fn blank_lines_and_empty_records_are_skipped_and_still_counted() { + assert_eq!( + framed(true, &[b"\n \t\x0B\x0C\r\n{}\n[ ]\n{\"a\":1}\r\n\n"]).unwrap(), + vec![("{\"a\":1}\r".to_string(), 5)], + ); + assert_eq!( + framed(false, &[b"[{}, [], {\"a\":1}]"]).unwrap(), + vec![(r#" {"a":1}"#.to_string(), 2)] + ); + } + + #[test] + fn an_empty_array_has_no_records_and_whitespace_may_follow_it() { + assert_eq!(framed(false, &[b" \n[ \n] \n\t"]).unwrap(), vec![]); + assert_eq!(framed(false, &[b"[1] \n"]).unwrap(), vec![("1".to_string(), 0)]); + } + + #[test] + fn content_after_the_closing_bracket_is_malformed() { + assert_eq!(framed(false, &[b"[{\"id\":1}] x"]), malformed(1, "trailing characters")); + } + + #[test] + fn a_truncated_document_is_malformed() { + assert_eq!( + framed(false, &[b"[{\"id\":1}"]), + malformed(0, "EOF while parsing a list") + ); + assert_eq!( + framed(false, &[b"[{\"id\":"]), + malformed(0, "EOF while parsing a value at line 1 column 6") + ); + assert_eq!(framed(false, &[b""]), malformed(0, "EOF while parsing a value")); + assert_eq!(framed(false, &[b"{}"]), malformed(0, "expected `[`")); + } + + #[test] + fn nesting_deeper_than_json_decode_reads_is_refused() { + assert_eq!( + framed(false, &[format!("[{}]", nested(511)).as_bytes()]).unwrap().len(), + 1 + ); + assert_eq!( + framed(false, &[format!("[{}]", nested(512)).as_bytes()]), + malformed(0, "Maximum stack depth exceeded"), + ); + assert!(!super::line_too_deep(nested(511).as_bytes())); + assert!(super::line_too_deep(nested(512).as_bytes())); + assert!(!super::line_too_deep(format!("[\"{}\"]", "[".repeat(600)).as_bytes())); + } + + #[test] + fn numbers_follow_json_decode() { + assert_eq!(number("-0"), Number::Long(0)); + assert_eq!(number("9223372036854775807"), Number::Long(i64::MAX)); + assert_eq!(number("-9223372036854775808"), Number::Long(i64::MIN)); + assert_eq!( + number("9223372036854775808"), + Number::Double(9_223_372_036_854_775_808.0) + ); + assert_eq!( + number("-9223372036854775809"), + Number::Double(-9_223_372_036_854_775_809.0) + ); + assert_eq!(number("1E2"), Number::Double(100.0)); + assert_eq!(number("1e400"), Number::Double(f64::INFINITY)); + assert_eq!(number("1e-400"), Number::Double(0.0)); + + let Number::Double(negative_zero) = number("-0.0") else { + panic!("-0.0 is a float"); + }; + + assert!(negative_zero == 0.0 && negative_zero.is_sign_negative()); + } + + /// Columns keyed as `Schema::definitions()` keys them: a canonical int name by its int. + fn columns(names: &[&str]) -> MemberColumns { + let keys: Vec = names + .iter() + .map(|name| match array_key_index(name.as_bytes()) { + Some(index) => Key::Index(index), + None => Key::Name(name.as_bytes().to_vec()), + }) + .collect(); + + MemberColumns::of(keys.iter()) + } + + fn mapped(record: &str, names: &[&str]) -> Result>, RecordError> { + let mut slots = vec![None; names.len()]; + + map_record(record.as_bytes(), &columns(names), &mut slots, record.as_ptr() as usize)?; + + Ok(slots + .iter() + .map(|slot| slot.map(|(start, end)| record[start..end].to_string())) + .collect()) + } + + #[test] + fn members_map_to_columns_by_name_and_the_last_duplicate_wins() { + assert_eq!( + mapped( + r#" {"b": [1, {"x":2}], "zz": 0, "a": "1", "a": null} "#, + &["a", "b", "c"] + ) + .unwrap(), + vec![Some("null".to_string()), Some(r#"[1, {"x":2}]"#.to_string()), None], + ); + assert_eq!( + mapped(r#"{"\u00e9\/":1,"e\u0301":2}"#, &["é/", "é"]).unwrap(), + vec![Some("1".to_string()), None], + ); + } + + #[test] + fn an_array_record_maps_elements_to_numeric_names() { + assert_eq!( + mapped(r#"[10, "x", true]"#, &["2", "0"]).unwrap(), + vec![Some("true".to_string()), Some("10".to_string())] + ); + } + + #[test] + fn a_scalar_record_names_its_type() { + assert_eq!(mapped(" 5 ", &[]), Err(RecordError::Scalar("int"))); + assert_eq!(mapped("-0", &[]), Err(RecordError::Scalar("int"))); + assert_eq!(mapped("9223372036854775808", &[]), Err(RecordError::Scalar("float"))); + assert_eq!(mapped(r#""x""#, &[]), Err(RecordError::Scalar("string"))); + assert_eq!(mapped("false", &[]), Err(RecordError::Scalar("bool"))); + assert_eq!(mapped("null", &[]), Err(RecordError::Scalar("null"))); + } + + #[test] + fn invalid_json_is_malformed() { + for record in [ + r#"{"id":1}{"id":9}"#, + "{\"id\":1}\x0B", + "\0", + "{\"a\":\"\x01\"}", + "01", + "NaN", + "True", + "{\"skipped\":\"\u{0}\"}", + ] { + assert!( + matches!(mapped(record, &["id"]), Err(RecordError::Malformed(_))), + "{record:?}" + ); + } + + let invalid_utf8 = b"{\"skipped\":\"\xFF\"}"; + + assert!(matches!( + map_record(invalid_utf8, &columns(&[]), &mut [], invalid_utf8.as_ptr() as usize), + Err(RecordError::Malformed(_)), + )); + } + + #[test] + fn a_lone_surrogate_escape_is_malformed_in_any_member() { + for record in [ + r#"{"skipped":"\ud800"}"#, + r#"{"skipped":"\udc00"}"#, + r#"{"skipped":"\ud83dx"}"#, + r#"{"skipped":"\ud83d\u0041"}"#, + r#"{"\ud800":1}"#, + r#"{"a":"\ud83d"}"#, + ] { + assert_eq!( + framed(true, &[record.as_bytes()]), + malformed(1, "Single unpaired UTF-16 surrogate in unicode escape"), + "{record}", + ); + } + + assert_eq!( + framed(true, &[r#"{"a":"\ud83d\ude00","b":"\\","c":"\\ud800"}"#.as_bytes()]) + .unwrap() + .len(), + 1, + "a pair, an escaped backslash, and an escaped backslash before u are no lone surrogates", + ); + } + + fn deduplicated(object: &str) -> Vec<(String, String)> { + let entries = str_deserializer(object).deserialize_map(Entries).unwrap(); + + last_wins(entries) + .into_iter() + .map(|(name, value)| (name.into_owned(), value.get().to_string())) + .collect() + } + + fn object(members: &[(String, usize)]) -> String { + format!( + "{{{}}}", + members + .iter() + .map(|(name, value)| format!("\"{name}\":{value}")) + .collect::>() + .join(",") + ) + } + + #[test] + fn a_repeated_key_keeps_its_first_position_and_its_last_value_below_and_past_the_pairwise_check() { + for size in [3, 40] { + let mut members: Vec<(String, usize)> = (0..size).map(|i| (format!("k{i}"), i)).collect(); + + assert_eq!( + deduplicated(&object(&members)), + members + .iter() + .map(|(name, value)| (name.clone(), value.to_string())) + .collect::>(), + "{size} unique keys stay as read", + ); + + members.push(("k1".to_string(), 99)); + let mut expected: Vec<(String, String)> = members[..size] + .iter() + .map(|(name, value)| (name.clone(), value.to_string())) + .collect(); + expected[1].1 = "99".to_string(); + + assert_eq!(deduplicated(&object(&members)), expected, "{size} keys and k1 again"); + } + } +} diff --git a/src/extension/flow-php-ext/src/json/source.rs b/src/extension/flow-php-ext/src/json/source.rs new file mode 100644 index 0000000000..b69387a81f --- /dev/null +++ b/src/extension/flow-php-ext/src/json/source.rs @@ -0,0 +1,119 @@ +//! `Flow\ETL\Adapter\JSON\RustJsonOpenSource`: JSON lines or a top-level JSON array read from the stream in `CHUNK`s +//! into native columns, every batch adopted by the configured backend. + +use std::cell::{Cell, RefCell}; +use std::rc::Rc; + +use ext_php_rs::binary_slice::BinarySlice; +use ext_php_rs::flags::ClassFlags; +use ext_php_rs::prelude::*; +use ext_php_rs::types::Zval; + +use crate::batch_columns::{batch_size_of, HeldBatch}; +use crate::ctx::{call_method, zval_long}; +use crate::exception::ext_exception; +use crate::interfaces::json_open_source_ce; +use crate::iterator::RustIterator; +use crate::json::read::JsonReader; +use crate::source::{self, reading, Fed, CHUNK}; + +const CLASS: &str = "RustJsonOpenSource"; + +struct Shared { + stream: Zval, + reader: RefCell, + /// The chunk the opener already read at `offset`; empty once fed. + head: RefCell>, + offset: Cell, +} + +/// One `batches()` call over the shared reader. +struct JsonFeed { + shared: Rc, + schema: Zval, + batch_size: usize, +} + +impl Fed for JsonFeed { + fn next_batch(&self) -> PhpResult> { + reading(&self.shared.reader, CLASS)?.next_columns(&self.schema, self.batch_size) + } + + /// `SourceStream::read()` runs with no borrow held: the stream may call back into the source. + fn feed(&self) -> PhpResult { + let head = std::mem::take(&mut *self.shared.head.borrow_mut()); + let chunk = if head.is_empty() { + let read = call_method( + &self.shared.stream, + "read", + &mut [zval_long(CHUNK), zval_long(self.shared.offset.get())], + )?; + + read.zend_str() + .ok_or_else(|| ext_exception("flow_php expected SourceStream::read() to return a string"))? + .as_bytes() + .to_vec() + } else { + head + }; + + if chunk.is_empty() { + return Ok(false); + } + + reading(&self.shared.reader, CLASS)?.feed(&chunk); + self.shared.offset.set(self.shared.offset.get() + chunk.len() as i64); + + Ok(true) + } + + fn finish(&self) -> PhpResult<()> { + reading(&self.shared.reader, CLASS)?.finish(); + + Ok(()) + } +} + +#[php_class] +#[php( + name = "Flow\\ETL\\Adapter\\JSON\\RustJsonOpenSource", + flags = ClassFlags::Final, + implements(ce = json_open_source_ce, stub = "Flow\\ETL\\Adapter\\JSON\\JsonOpenSource") +)] +pub struct RustJsonOpenSource { + shared: Rc, +} + +#[php_impl] +impl RustJsonOpenSource { + /// `$head`: the chunk the opener already read at `$offset` ('' for none); the bytes before it are JSON whitespace + /// after an optional BOM, which the reader would skip. `$uri` names the file in refusals. + pub fn __construct(stream: &Zval, lines: bool, uri: BinarySlice, head: BinarySlice, offset: i64) -> Self { + Self { + shared: Rc::new(Shared { + stream: stream.shallow_clone(), + reader: RefCell::new(JsonReader::new(lines, &uri)), + head: RefCell::new(head.to_vec()), + offset: Cell::new(offset), + }), + } + } + + /// Exactly `$batchSize` rows keyed and ordered by `$schema` (the last may be shorter), every column adopted by + /// `$backend`; refusals are `RowsBuilder::appendRows()`'s, row indexes relative to the batch. + pub fn batches(&self, schema: &Zval, batchSize: i64, backend: &Zval) -> PhpResult { + let fed = JsonFeed { + shared: Rc::clone(&self.shared), + schema: schema.shallow_clone(), + batch_size: batch_size_of(batchSize, "JSON")?, + }; + + Ok(source::batches(fed, schema, backend)) + } + + pub fn close(&self) -> PhpResult<()> { + call_method(&self.shared.stream, "close", &mut [])?; + + Ok(()) + } +} diff --git a/src/extension/flow-php-ext/src/json/write.rs b/src/extension/flow-php-ext/src/json/write.rs new file mode 100644 index 0000000000..b36d6c8886 --- /dev/null +++ b/src/extension/flow-php-ext/src/json/write.rs @@ -0,0 +1,141 @@ +//! `Flow\ETL\Adapter\JSON\RustJsonEncoder`: a `Rows` batch as the text `PhpJsonEncoder::encode()` returns, rendered +//! from the arrow arrays; the columns it does not render are the fragments the held `PhpJsonEncoder::fragments()` +//! produces. + +use ext_php_rs::binary::Binary; +use ext_php_rs::binary_slice::BinarySlice; +use ext_php_rs::exception::PhpException; +use ext_php_rs::flags::ClassFlags; +use ext_php_rs::prelude::*; +use ext_php_rs::types::{ZendHashTable, Zval}; + +use crate::interfaces::json_encoder_ce; +use crate::render::{invalid_argument, runtime}; +use crate::text::batch::{batch, cells, renderer, Unrendered}; +use crate::text::json::{self, Flags}; +use crate::text::value::{Formats, Renderer}; +use crate::text::Error; + +const JSON_UNESCAPED_SLASHES: i64 = 64; +const JSON_UNESCAPED_UNICODE: i64 = 256; +const JSON_PRESERVE_ZERO_FRACTION: i64 = 1024; +const JSON_THROW_ON_ERROR: i64 = 4_194_304; + +fn refused(error: Error) -> PhpException { + runtime(format!("Failed to encode JSON: {}", error.message())) +} + +enum Cells<'a> { + Native(Renderer<'a>), + Fragments(Vec>), +} + +#[php_class] +#[php( + name = "Flow\\ETL\\Adapter\\JSON\\RustJsonEncoder", + flags = ClassFlags::Final, + implements(ce = json_encoder_ce, stub = "Flow\\ETL\\Adapter\\JSON\\JsonEncoder") +)] +pub struct RustJsonEncoder { + flags: Flags, + formats: Formats, + /// The `PhpJsonEncoder` that renders the unrendered columns. + php: Zval, + unrendered: Unrendered, +} + +#[php_impl] +impl RustJsonEncoder { + pub fn __construct( + flags: i64, + date_time_format: BinarySlice, + date_format: BinarySlice, + php: &Zval, + ) -> PhpResult { + if flags + & !(JSON_THROW_ON_ERROR | JSON_UNESCAPED_SLASHES | JSON_UNESCAPED_UNICODE | JSON_PRESERVE_ZERO_FRACTION) + != 0 + { + return Err(invalid_argument( + "flow_php JSON writer supports only the JSON_THROW_ON_ERROR, JSON_UNESCAPED_SLASHES, JSON_UNESCAPED_UNICODE and JSON_PRESERVE_ZERO_FRACTION flags" + .to_string(), + )); + } + + Ok(Self { + flags: Flags { + unescaped_slashes: flags & JSON_UNESCAPED_SLASHES != 0, + unescaped_unicode: flags & JSON_UNESCAPED_UNICODE != 0, + preserve_zero_fraction: flags & JSON_PRESERVE_ZERO_FRACTION != 0, + }, + formats: Formats::parse(&date_time_format, &date_format), + php: php.shallow_clone(), + unrendered: Unrendered::default(), + }) + } + + /// One object per row, joined by `$separator`; the unrendered columns through the held encoder's `fragments()`. + pub fn encode(&mut self, rows: &Zval, separator: BinarySlice) -> PhpResult> { + let fragments = self + .unrendered + .render(rows, &self.php, "fragments", &self.formats, false)?; + + self.render_rows(rows, &fragments, &separator) + } +} + +impl RustJsonEncoder { + /// `$fragments`: `PhpJsonEncoder::fragments()` of the unrendered columns. + fn render_rows(&self, rows: &Zval, fragments: &ZendHashTable, separator: &[u8]) -> PhpResult> { + let batch = batch(rows, &self.formats, false)?; + let mut keys = Vec::with_capacity(batch.columns.len()); + let mut columns = Vec::with_capacity(batch.columns.len()); + + for (position, column) in batch.columns.iter().enumerate() { + let mut key = vec![if position == 0 { b'{' } else { b',' }]; + + json::string(&mut key, &column.name, &self.flags).map_err(refused)?; + key.push(b':'); + keys.push(key); + columns.push(match &column.native { + Some((plan, data)) => Cells::Native(renderer(plan, data, &self.formats)?), + None => Cells::Fragments(cells(fragments, &column.name, batch.count, "JSON fragments")?), + }); + } + + let mut out = Vec::with_capacity(batch.count * 32 * columns.len().max(1)); + + for i in 0..batch.count { + let mut non_finite = false; + + if i > 0 { + out.extend_from_slice(separator); + } + + if columns.is_empty() { + out.push(b'{'); + } + + for (key, column) in keys.iter().zip(&columns) { + out.extend_from_slice(key); + + match column { + Cells::Native(renderer) => renderer + .json(&mut out, i, &self.flags, &mut non_finite) + .map_err(refused)?, + Cells::Fragments(fragments) => out.extend_from_slice(fragments[i].ok_or_else(|| { + invalid_argument("flow_php expected a JSON fragment of every row, got null".to_string()) + })?), + } + } + + out.push(b'}'); + + if non_finite { + return Err(refused(Error::NonFinite)); + } + } + + Ok(Binary::new(out)) + } +} diff --git a/src/extension/flow-php-ext/src/json_check.rs b/src/extension/flow-php-ext/src/json_check.rs index fa2ebec9c7..1cdbdcb9aa 100644 --- a/src/extension/flow-php-ext/src/json_check.rs +++ b/src/extension/flow-php-ext/src/json_check.rs @@ -8,7 +8,10 @@ pub fn json_nesting_within_php_depth(bytes: &[u8]) -> bool { /// Any such escape, paired or not, is left to PHP. pub fn json_without_surrogate_escape(bytes: &[u8]) -> bool { !memchr::memchr_iter(b'\\', bytes).any(|at| { - matches!(bytes.get(at + 1..at + 4), Some([b'u', b'd' | b'D', b'8'..=b'9' | b'a'..=b'f' | b'A'..=b'F'])) + matches!( + bytes.get(at + 1..at + 4), + Some([b'u', b'd' | b'D', b'8'..=b'9' | b'a'..=b'f' | b'A'..=b'F']) + ) }) } diff --git a/src/extension/flow-php-ext/src/kind_builder.rs b/src/extension/flow-php-ext/src/kind_builder.rs new file mode 100644 index 0000000000..1445dec92d --- /dev/null +++ b/src/extension/flow-php-ext/src/kind_builder.rs @@ -0,0 +1,507 @@ +//! Arrow storage for one column kind, appended row by row and finished into the `ArrayData` whose data type is +//! `flow_batch_frame::kind::data_type(kind)`. A null row is stored the canonical way: zeroed slot, offset not advanced, +//! every structure child null. + +use arrow_array::cast::AsArray; +use arrow_array::types::{Date32Type, DurationMicrosecondType, Float64Type, Int64Type, TimestampMicrosecondType}; +use arrow_array::Array; +use arrow_buffer::{BooleanBufferBuilder, Buffer, NullBufferBuilder}; +use arrow_data::ArrayData; +use arrow_schema::ArrowError; +use flow_batch_frame::kind::{data_type, Kind}; + +/// Offsets are i32 (D5): the last offset a row would reach past `i32::MAX`. +pub struct OffsetOverflow(pub u64); + +pub enum KindBuilder { + Null { + len: usize, + }, + Fixed { + width: usize, + values: Vec, + validity: NullBufferBuilder, + }, + Boolean { + values: BooleanBufferBuilder, + validity: NullBufferBuilder, + }, + Bytes { + offsets: Vec, + data: Vec, + validity: NullBufferBuilder, + }, + List { + offsets: Vec, + validity: NullBufferBuilder, + element: Box, + }, + Map { + offsets: Vec, + validity: NullBufferBuilder, + keys: Box, + values: Box, + }, + Struct { + len: usize, + validity: NullBufferBuilder, + children: Vec, + }, +} + +fn width(kind: &Kind) -> usize { + match kind { + Kind::Int32 => 4, + Kind::Uuid => 16, + _ => 8, + } +} + +fn offset(len: usize) -> Result { + i32::try_from(len).map_err(|_| OffsetOverflow(len as u64)) +} + +impl KindBuilder { + pub fn new(kind: &Kind) -> Self { + match kind { + Kind::Null => KindBuilder::Null { len: 0 }, + Kind::Boolean => KindBuilder::Boolean { + values: BooleanBufferBuilder::new(0), + validity: NullBufferBuilder::new(0), + }, + Kind::Bytes => KindBuilder::Bytes { + offsets: vec![0], + data: Vec::new(), + validity: NullBufferBuilder::new(0), + }, + Kind::List(element) => KindBuilder::List { + offsets: vec![0], + validity: NullBufferBuilder::new(0), + element: Box::new(KindBuilder::new(&element.kind)), + }, + Kind::Map(key, value) => KindBuilder::Map { + offsets: vec![0], + validity: NullBufferBuilder::new(0), + keys: Box::new(KindBuilder::new(&key.kind)), + values: Box::new(KindBuilder::new(&value.kind)), + }, + Kind::Struct(fields) => KindBuilder::Struct { + len: 0, + validity: NullBufferBuilder::new(0), + children: fields.iter().map(|field| KindBuilder::new(&field.kind)).collect(), + }, + _ => KindBuilder::Fixed { + width: width(kind), + values: Vec::new(), + validity: NullBufferBuilder::new(0), + }, + } + } + + pub fn len(&self) -> usize { + match self { + KindBuilder::Null { len } | KindBuilder::Struct { len, .. } => *len, + KindBuilder::Fixed { validity, .. } + | KindBuilder::Boolean { validity, .. } + | KindBuilder::Bytes { validity, .. } + | KindBuilder::List { validity, .. } + | KindBuilder::Map { validity, .. } => validity.len(), + } + } + + pub fn append_null(&mut self) { + match self { + KindBuilder::Null { len } => *len += 1, + KindBuilder::Fixed { + width, + values, + validity, + } => { + values.resize(values.len() + *width, 0); + validity.append_null(); + } + KindBuilder::Boolean { values, validity } => { + values.append(false); + validity.append_null(); + } + KindBuilder::Bytes { offsets, validity, .. } + | KindBuilder::List { offsets, validity, .. } + | KindBuilder::Map { offsets, validity, .. } => { + offsets.push(*offsets.last().expect("offsets start at 0")); + validity.append_null(); + } + KindBuilder::Struct { + len, + validity, + children, + } => { + *len += 1; + validity.append_null(); + + for child in children { + child.append_null(); + } + } + } + } + + /// A fixed-width value in its little-endian bytes (i64, i32 days, f64 bits, 16 uuid bytes). + pub fn append_fixed(&mut self, bytes: &[u8]) { + let KindBuilder::Fixed { values, validity, .. } = self else { + unreachable!("append_fixed on a non fixed-width kind"); + }; + + values.extend_from_slice(bytes); + validity.append_non_null(); + } + + pub fn append_bool(&mut self, value: bool) { + let KindBuilder::Boolean { values, validity } = self else { + unreachable!("append_bool on a non boolean kind"); + }; + + values.append(value); + validity.append_non_null(); + } + + pub fn append_bytes(&mut self, bytes: &[u8]) -> Result<(), OffsetOverflow> { + let KindBuilder::Bytes { + offsets, + data, + validity, + } = self + else { + unreachable!("append_bytes on a non bytes kind"); + }; + + let end = offset(data.len() + bytes.len())?; + data.extend_from_slice(bytes); + offsets.push(end); + validity.append_non_null(); + + Ok(()) + } + + /// The element builder of a list; call `end_entries` once the row's elements are appended. + pub fn list_element(&mut self) -> &mut KindBuilder { + let KindBuilder::List { element, .. } = self else { + unreachable!("list_element on a non list kind"); + }; + + element + } + + pub fn map_entries(&mut self) -> (&mut KindBuilder, &mut KindBuilder) { + let KindBuilder::Map { keys, values, .. } = self else { + unreachable!("map_entries on a non map kind"); + }; + + (keys, values) + } + + /// Closes a valid list or map row over the elements appended since the previous row. + pub fn end_entries(&mut self) -> Result<(), OffsetOverflow> { + let (offsets, validity, len) = match self { + KindBuilder::List { + offsets, + validity, + element, + } => (offsets, validity, element.len()), + KindBuilder::Map { + offsets, + validity, + keys, + .. + } => (offsets, validity, keys.len()), + _ => unreachable!("end_entries on a non list/map kind"), + }; + + offsets.push(offset(len)?); + validity.append_non_null(); + + Ok(()) + } + + pub fn struct_children(&mut self) -> &mut [KindBuilder] { + let KindBuilder::Struct { children, .. } = self else { + unreachable!("struct_children on a non structure kind"); + }; + + children + } + + /// Closes a valid structure row whose children were each appended once. + pub fn end_struct(&mut self) { + let KindBuilder::Struct { len, validity, .. } = self else { + unreachable!("end_struct on a non structure kind"); + }; + + *len += 1; + validity.append_non_null(); + } + + /// Drops every row past the first `len`: a row refused half-appended leaves no trace. + pub fn truncate(&mut self, len: usize) { + match self { + KindBuilder::Null { len: rows } => *rows = len, + KindBuilder::Fixed { + width, + values, + validity, + } => { + values.truncate(len * *width); + validity.truncate(len); + } + KindBuilder::Boolean { values, validity } => { + values.truncate(len); + validity.truncate(len); + } + KindBuilder::Bytes { + offsets, + data, + validity, + } => { + offsets.truncate(len + 1); + data.truncate(offsets[len] as usize); + validity.truncate(len); + } + KindBuilder::List { + offsets, + validity, + element, + } => { + offsets.truncate(len + 1); + element.truncate(offsets[len] as usize); + validity.truncate(len); + } + KindBuilder::Map { + offsets, + validity, + keys, + values, + } => { + offsets.truncate(len + 1); + keys.truncate(offsets[len] as usize); + values.truncate(offsets[len] as usize); + validity.truncate(len); + } + KindBuilder::Struct { + len: rows, + validity, + children, + } => { + *rows = len; + validity.truncate(len); + + for child in children { + child.truncate(len); + } + } + } + } + + /// Row `i` of `array`, whose data type is `data_type(kind)`. + pub fn append_from(&mut self, array: &dyn Array, kind: &Kind, i: usize) -> Result<(), OffsetOverflow> { + if matches!(kind, Kind::Null) || array.is_null(i) { + self.append_null(); + + return Ok(()); + } + + match kind { + Kind::Null => unreachable!("appended above"), + Kind::Int64 => self.append_fixed(&array.as_primitive::().value(i).to_le_bytes()), + Kind::Timestamp => { + self.append_fixed(&array.as_primitive::().value(i).to_le_bytes()) + } + Kind::Duration => { + self.append_fixed(&array.as_primitive::().value(i).to_le_bytes()) + } + Kind::Int32 => self.append_fixed(&array.as_primitive::().value(i).to_le_bytes()), + Kind::Float64 => self.append_fixed(&array.as_primitive::().value(i).to_le_bytes()), + Kind::Uuid => self.append_fixed(array.as_fixed_size_binary().value(i)), + Kind::Boolean => self.append_bool(array.as_boolean().value(i)), + Kind::Bytes => self.append_bytes(array.as_binary::().value(i))?, + Kind::List(element) => { + let list = array.as_list::(); + let offsets = list.value_offsets(); + + for j in offsets[i] as usize..offsets[i + 1] as usize { + self.list_element() + .append_from(list.values().as_ref(), &element.kind, j)?; + } + + self.end_entries()?; + } + Kind::Map(key, value) => { + let map = array.as_map(); + let offsets = map.value_offsets(); + + for j in offsets[i] as usize..offsets[i + 1] as usize { + let (keys, values) = self.map_entries(); + keys.append_from(map.keys().as_ref(), &key.kind, j)?; + values.append_from(map.values().as_ref(), &value.kind, j)?; + } + + self.end_entries()?; + } + Kind::Struct(fields) => { + let structure = array.as_struct(); + + for ((child, field), column) in self.struct_children().iter_mut().zip(fields).zip(structure.columns()) { + child.append_from(column.as_ref(), &field.kind, i)?; + } + + self.end_struct(); + } + } + + Ok(()) + } + + /// The rows appended so far; the builder keeps them and may be appended to again. + pub fn finish(&self, kind: &Kind) -> Result { + let builder = ArrayData::builder(data_type(kind)).len(self.len()); + + let builder = match (self, kind) { + (KindBuilder::Null { .. }, _) => builder, + (KindBuilder::Fixed { values, validity, .. }, _) => builder + .nulls(validity.finish_cloned()) + .add_buffer(Buffer::from_slice_ref(values)), + (KindBuilder::Boolean { values, validity }, _) => builder + .nulls(validity.finish_cloned()) + .add_buffer(values.finish_cloned().into_inner()), + ( + KindBuilder::Bytes { + offsets, + data, + validity, + }, + _, + ) => builder + .nulls(validity.finish_cloned()) + .add_buffer(Buffer::from_slice_ref(offsets)) + .add_buffer(Buffer::from_slice_ref(data)), + ( + KindBuilder::List { + offsets, + validity, + element, + }, + Kind::List(field), + ) => builder + .nulls(validity.finish_cloned()) + .add_buffer(Buffer::from_slice_ref(offsets)) + .add_child_data(element.finish(&field.kind)?), + ( + KindBuilder::Map { + offsets, + validity, + keys, + values, + }, + Kind::Map(key, value), + ) => { + let arrow_schema::DataType::Map(entries, _) = data_type(kind) else { + unreachable!("a map kind has a map data type") + }; + let entries = ArrayData::builder(entries.data_type().clone()) + .len(keys.len()) + .add_child_data(keys.finish(&key.kind)?) + .add_child_data(values.finish(&value.kind)?) + .build()?; + + builder + .nulls(validity.finish_cloned()) + .add_buffer(Buffer::from_slice_ref(offsets)) + .add_child_data(entries) + } + (KindBuilder::Struct { validity, children, .. }, Kind::Struct(fields)) => { + builder.nulls(validity.finish_cloned()).child_data( + children + .iter() + .zip(fields) + .map(|(child, field)| child.finish(&field.kind)) + .collect::>()?, + ) + } + _ => unreachable!("a builder is finished with the kind it was built for"), + }; + + builder.build() + } +} + +#[cfg(test)] +mod tests { + use arrow_array::cast::AsArray; + use arrow_array::types::Int64Type; + use arrow_array::{make_array, Array}; + use flow_batch_frame::kind::{Field, Kind}; + + use super::KindBuilder; + + fn field(kind: Kind) -> Box { + Box::new(Field { + name: "item".into(), + kind, + optional: true, + }) + } + + #[test] + fn truncate_drops_the_rows_past_the_length_in_every_nested_builder() { + let kind = Kind::Struct(vec![ + Field { + name: "id".into(), + kind: Kind::Int64, + optional: true, + }, + Field { + name: "tags".into(), + kind: Kind::List(field(Kind::Bytes)), + optional: true, + }, + Field { + name: "flags".into(), + kind: Kind::Map(field(Kind::Bytes), field(Kind::Boolean)), + optional: true, + }, + ]); + let mut builder = KindBuilder::new(&kind); + let mut row = |id: i64, tag: &[u8]| { + let children = builder.struct_children(); + children[0].append_fixed(&id.to_le_bytes()); + children[1].list_element().append_bytes(tag).ok(); + children[1].end_entries().ok(); + let (keys, values) = children[2].map_entries(); + keys.append_bytes(tag).ok(); + values.append_bool(true); + children[2].end_entries().ok(); + builder.end_struct(); + }; + + row(1, b"kept"); + row(2, b"dropped"); + builder.truncate(1); + let array = make_array(builder.finish(&kind).unwrap()); + let structure = array.as_struct(); + + assert_eq!(array.len(), 1); + assert_eq!( + structure.column(0).as_primitive::().values().to_vec(), + vec![1] + ); + assert_eq!( + structure + .column(1) + .as_list::() + .values() + .as_binary::() + .value(0), + b"kept" + ); + assert_eq!(structure.column(1).as_list::().values().len(), 1); + assert_eq!(structure.column(2).as_map().keys().len(), 1); + } +} diff --git a/src/extension/flow-php-ext/src/lib.rs b/src/extension/flow-php-ext/src/lib.rs index 651f0e1301..345286df8b 100644 --- a/src/extension/flow-php-ext/src/lib.rs +++ b/src/extension/flow-php-ext/src/lib.rs @@ -1,34 +1,36 @@ +mod alloc; +mod arrow_c; +mod backend; +mod batch_columns; +mod builder; mod cast; +mod column; mod csv; mod ctx; mod date_check; -mod encode; mod exception; -mod format; -mod hydrate; +mod globals; +mod interfaces; +mod iterator; +mod json; mod json_check; +mod kind_builder; +mod parquet; +mod physical; mod plan; +mod render; +mod source; +mod text; mod uuid_check; mod values; -use std::alloc::System; - -use ext_php_rs::binary_slice::BinarySlice; use ext_php_rs::prelude::*; -use ext_php_rs::types::{ZendHashTable, Zval}; use ext_php_rs::zend::ModuleEntry; use ext_php_rs::{info_table_end, info_table_row, info_table_start}; -use crate::ctx::{ht_insert, zval_str, Ctx}; -use crate::encode::{ - build_encode_plan, encode_frames, encode_typed_row, expect_object, ht_for_each, read_slot, EncodePlan, -}; -use crate::exception::ext_exception; -use crate::format::Reader; -use crate::plan::Plan; - -#[global_allocator] -static GLOBAL: System = System; +/// The contract version flow-php/etl expects (`Flow\ETL\FlowPhpExtension::ABI`); both change together whenever a +/// registered class or interface changes. +const FLOW_PHP_ABI: i64 = 1; pub extern "C" fn php_module_info(_module: *mut ModuleEntry) { info_table_start!(); @@ -41,410 +43,12 @@ pub extern "C" fn php_module_info(_module: *mut ModuleEntry) { /// /// Only called by the PHP engine during module startup (MINIT). pub unsafe extern "C" fn module_startup(_type: i32, _module_number: i32) -> i32 { - if let Err(e) = exception::register() { - eprintln!("flow_php: failed to register Flow\\Floe\\Exception\\ExtensionException: {e}"); + if let Err(e) = interfaces::register() { + eprintln!("flow_php: failed to register the Flow\\ETL interfaces: {e}"); return -1; } - 0 -} - -/// A built plan together with the schema JSON it was built from; the schema -/// bytes decide when a new schema requires a plan rebuild. Generic over the plan -/// kind so encode (`EncodePlan`) and decode (`Plan`) share one rebind path. -struct BoundPlan

{ - schema: Vec, - plan: P, -} - -/// Rebuilds the bound plan when the schema JSON changed, `Ctx` caches survive rebinds. -fn ensure_plan

( - bound: &mut Option>, - ctx: &mut Ctx, - schema: &[u8], - build: impl FnOnce(&[u8], &mut Ctx) -> PhpResult

, -) -> PhpResult<()> { - if bound.as_ref().is_some_and(|b| b.schema == schema) { - return Ok(()); - } - - *bound = Some(BoundPlan { - schema: schema.to_vec(), - plan: build(schema, ctx)?, - }); - - Ok(()) -} - -/// Floe's consolidated binary codec, both directions: ROW frame bodies to/from -/// `Flow\ETL\Row\RawRowValues` (decode) and `Flow\ETL\Row\TypedRowValues` (encode), -/// plus `decode_rows` / `encode_frames`, which go straight between frames and `Flow\ETL\Rows`. -/// The PHP side keeps buffering/framing/sectioning and hands over bare frame -/// bodies; the extension owns value encode/decode against a primed schema. -#[php_class] -#[php(name = "Flow\\Floe\\RustFloeEncoderNative")] -pub struct RustFloeEncoderNative { - ctx: Ctx, - encode_bound: Option>, - decode_bound: Option>, - row_values_class: hydrate::RowValuesClass, - cast_plan: Option, - assembly: hydrate::AssemblyClasses, - rows_classes: hydrate::RowsClasses, - def_dehydrate_fns: hydrate::DefFnCache, -} - -#[php_impl] -impl RustFloeEncoderNative { - pub fn __construct() -> PhpResult { - Ok(Self { - ctx: Ctx::new()?, - encode_bound: None, - decode_bound: None, - row_values_class: hydrate::RowValuesClass::resolve()?, - cast_plan: None, - assembly: hydrate::AssemblyClasses::resolve()?, - rows_classes: hydrate::RowsClasses::resolve()?, - def_dehydrate_fns: hydrate::DefFnCache::new(), - }) - } - - /// Encodes a list of `Flow\ETL\Row\TypedRowValues` into a list of bare ROW - /// frame bodies against the plan bound to `schema_body`. - pub fn encode( - &mut self, - batch: &Zval, - schema_body: BinarySlice, - schema: &Zval, - ) -> PhpResult { - ensure_plan( - &mut self.encode_bound, - &mut self.ctx, - &schema_body, - |body, ctx| build_encode_plan(body, schema, ctx), - )?; - - let batch_ht = batch - .array() - .ok_or_else(|| ext_exception("flow_php expected a list of typed row values"))?; - - let (values_slot, metadata_slot) = self.ctx.typed_row_values_slots()?; - - let plan = &self.encode_bound.as_ref().expect("plan bound above").plan; - let ctx = &mut self.ctx; - - let mut encoded = ZendHashTable::with_capacity(batch_ht.len() as u32); - - ht_for_each(batch_ht, |_, row_index, item_zv| { - let typed = expect_object(item_zv, "a TypedRowValues")?; - - let values_ht = read_slot(typed, values_slot).array().ok_or_else(|| { - ext_exception("flow_php expected TypedRowValues::values to be an array") - })?; - let metadata_ht = read_slot(typed, metadata_slot).array().ok_or_else(|| { - ext_exception("flow_php expected TypedRowValues::metadata to be an array") - })?; - - let mut body = Vec::with_capacity(1024); - encode_typed_row(plan, row_index, values_ht, metadata_ht, &mut body, ctx)?; - - encoded.push(zval_str(&body)).map_err(|e| { - ext_exception(format!("flow_php failed to collect a row body: {e:?}")) - })?; - - Ok(()) - })?; - - let mut zv = Zval::new(); - zv.set_hashtable(encoded); - - Ok(zv) - } - - /// `Rows` straight to complete ROW frames (frame type + length + body per row) in one string - see - /// `encode::encode_frames`. - pub fn encode_frames( - &mut self, - rows: &Zval, - schema_body: BinarySlice, - schema: &Zval, - ) -> PhpResult { - ensure_plan( - &mut self.encode_bound, - &mut self.ctx, - &schema_body, - |body, ctx| build_encode_plan(body, schema, ctx), - )?; - - let (rows_ht, columns) = - hydrate::dehydrate_batch(rows, &self.rows_classes, &mut self.def_dehydrate_fns, &mut self.ctx)?; - - let frames = encode_frames( - &self.encode_bound.as_ref().expect("plan bound above").plan, - rows_ht, - self.rows_classes.row_values_slot, - &hydrate::batch_metadata(&columns), - &mut self.ctx, - )?; - - Ok(zval_str(&frames)) - } - - /// Decodes a list of ROW frame bodies written with the given SCHEMA frame - /// body into `Flow\ETL\Row\RawRowValues` objects. - pub fn decode(&mut self, frame_bodies: &Zval, schema_body: BinarySlice) -> PhpResult { - ensure_plan( - &mut self.decode_bound, - &mut self.ctx, - &schema_body, - plan::build_plan, - )?; - - let bodies_ht = frame_bodies - .array() - .ok_or_else(|| ext_exception("flow_php expected a list of row frame bodies"))?; - - let plan = &self.decode_bound.as_ref().expect("plan bound above").plan; - let class = &self.row_values_class; - let ctx = &mut self.ctx; - - let mut decoded = ZendHashTable::with_capacity(bodies_ht.len() as u32); - - ht_for_each(bodies_ht, |_, _, body_zv| { - let bytes = body_zv - .zend_str() - .ok_or_else(|| ext_exception("flow_php expected a row frame body to be a string"))? - .as_bytes(); - - let mut reader = Reader::new(bytes); - let row_values = hydrate::decode_row_values(plan, &mut reader, ctx, class)?; - - if !reader.is_eof() { - return Err(ext_exception( - "flow_php row frame length does not match its content", - )); - } - - decoded.push(row_values).map_err(|e| { - ext_exception(format!("flow_php failed to collect row values: {e:?}")) - })?; - - Ok(()) - })?; - - let mut zv = Zval::new(); - zv.set_hashtable(decoded); - - Ok(zv) - } - - /// Frame bodies decoded and cast against `schema` straight into `Flow\ETL\Rows` - see `cast::decode_rows`. - pub fn decode_rows( - &mut self, - frame_bodies: &Zval, - schema_body: BinarySlice, - schema: &Zval, - ) -> PhpResult { - ensure_plan( - &mut self.decode_bound, - &mut self.ctx, - &schema_body, - plan::build_plan, - )?; - - cast::decode_rows( - frame_bodies, - &self.decode_bound.as_ref().expect("plan bound above").plan, - schema, - &mut self.cast_plan, - &self.assembly, - &mut self.ctx, - ) - } - - /// Present in every extension whose casts and decoders apply the datetime column zone. - #[php(name = "datetimeZones")] - pub fn datetime_zones(&self) -> bool { - true - } -} - -/// Native counterpart of `PhpRowHydrator`: `hydrate` builds `Flow\ETL\Rows` from -/// a list of `RawRowValues` against a `Schema` object, casting each value against -/// the schema first; `dehydrate` turns `Rows` into a list of `TypedRowValues`. -#[php_class] -#[php(name = "Flow\\ETL\\Row\\RustRowHydratorNative")] -pub struct RustRowHydratorNative { - ctx: Ctx, - rows_classes: hydrate::RowsClasses, - typed_row_values_class: hydrate::TypedRowValuesClass, - raw_row_values_class: hydrate::RowValuesClass, - assembly: hydrate::AssemblyClasses, - def_dehydrate_fns: hydrate::DefFnCache, - cast_plan: Option, -} - -#[php_impl] -impl RustRowHydratorNative { - pub fn __construct() -> PhpResult { - Ok(Self { - ctx: Ctx::new()?, - rows_classes: hydrate::RowsClasses::resolve()?, - typed_row_values_class: hydrate::TypedRowValuesClass::resolve()?, - raw_row_values_class: hydrate::RowValuesClass::resolve()?, - assembly: hydrate::AssemblyClasses::resolve()?, - def_dehydrate_fns: hydrate::DefFnCache::new(), - cast_plan: None, - }) - } - - /// Casts a list of raw-scalar `RawRowValues` against a `Schema` and builds - /// a `Flow\ETL\Rows` in a single pass. - pub fn hydrate(&mut self, batch: &Zval, schema: &Zval) -> PhpResult { - cast::cast_rows( - batch, - schema, - &mut self.cast_plan, - &self.raw_row_values_class, - &self.assembly, - &mut self.ctx, - ) - } - - /// Turns a `Flow\ETL\Rows` into a list of `Flow\ETL\Row\TypedRowValues`. - pub fn dehydrate(&mut self, rows: &Zval) -> PhpResult { - hydrate::dehydrate_rows( - rows, - &self.rows_classes, - &self.typed_row_values_class, - &mut self.def_dehydrate_fns, - &mut self.ctx, - ) - } - - /// Present in every extension whose casts and decoders apply the datetime column zone. - #[php(name = "datetimeZones")] - pub fn datetime_zones(&self) -> bool { - true - } -} -/// Native counterpart of `CSVLineReader` + `CSVEncoder::decode()`: bytes in, `RawRowValues` out, -/// byte-identical to the PHP path. Resumable - `feed` any chunk size, `finish` at EOF. -#[php_class] -#[php(name = "Flow\\ETL\\Adapter\\CSV\\RustCSVReaderNative")] -pub struct RustCSVReaderNative { - reader: csv::CsvReader, - row_values_class: hydrate::RowValuesClass, -} - -#[php_impl] -impl RustCSVReaderNative { - pub fn __construct( - separator: BinarySlice, - enclosure: BinarySlice, - escape: BinarySlice, - with_header: bool, - empty_to_null: bool, - remove_bom: bool, - ) -> PhpResult { - Ok(Self { - reader: csv::CsvReader::new(&separator, &enclosure, &escape, with_header, empty_to_null, remove_bom)?, - row_values_class: hydrate::RowValuesClass::resolve()?, - }) - } - - pub fn feed(&mut self, chunk: BinarySlice) { - self.reader.feed(&chunk); - } - - pub fn finish(&mut self) { - self.reader.finish(); - } - - pub fn headers(&mut self) -> PhpResult { - let headers = self.reader.headers(); - let mut list = ZendHashTable::with_capacity(headers.len() as u32); - - for header in headers { - list.push(zval_str(header)) - .map_err(|e| ext_exception(format!("flow_php failed to collect a CSV header: {e:?}")))?; - } - - let mut zv = Zval::new(); - zv.set_hashtable(list); - - Ok(zv) - } - - /// Folds up to `limit` buffered rows (-1: all) into `fold`; returns how many it folded. - pub fn fold(&mut self, fold: &mut RustColumnFoldNative, limit: i64) -> PhpResult { - let limit = match limit { - -1 => None, - limit => Some( - u64::try_from(limit).map_err(|_| ext_exception("flow_php CSV fold limit must be -1 or at least 0"))?, - ), - }; - - Ok(self.reader.fold(&mut fold.fold, limit)? as i64) - } - - #[php(name = "consumedBytes")] - pub fn consumed_bytes(&self) -> i64 { - self.reader.consumed_bytes() as i64 - } - - pub fn next(&mut self, batch_size: i64) -> PhpResult { - let batch_size = usize::try_from(batch_size) - .ok() - .filter(|size| *size > 0) - .ok_or_else(|| ext_exception("flow_php CSV batch size must be greater than 0"))?; - - let mut zv = Zval::new(); - zv.set_hashtable(self.reader.next(batch_size, &self.row_values_class)?); - - Ok(zv) - } -} - -/// `ColumnTypes::observe()` over native CSV rows: a column fold seeded with the header names and gated by the -/// candidate types' `toString()` codes. HTML and XML candidates are never passed - the PHP side folds those itself. -#[php_class] -#[php(name = "Flow\\ETL\\Adapter\\CSV\\RustColumnFoldNative")] -pub struct RustColumnFoldNative { - fold: csv::fold::Fold, -} - -#[php_impl] -impl RustColumnFoldNative { - pub fn __construct(names: &ZendHashTable, candidates: &ZendHashTable) -> PhpResult { - Ok(Self { - fold: csv::fold::Fold::new(csv::fold::string_values(names), &csv::fold::string_values(candidates))?, - }) - } - - /// `StringTypeNarrower::narrow($value)->toString()` for the fold's candidates. - #[php(name = "narrowOne")] - pub fn narrow_one(&mut self, value: BinarySlice) -> PhpResult { - Ok(self.fold.narrower.narrow(&value)?.code().to_string()) - } - - /// Column name => type `toString()`, first-seen order. - pub fn types(&self) -> Zval { - let mut types = ZendHashTable::new(); - - for (name, code) in self.fold.types() { - ht_insert(&mut types, name, zval_str(code.as_bytes())); - } - - let mut zv = Zval::new(); - zv.set_hashtable(types); - - zv - } - - pub fn rows(&self) -> i64 { - self.fold.rows() as i64 - } + 0 } #[php_module] @@ -453,8 +57,19 @@ pub fn get_module(module: ModuleBuilder) -> ModuleBuilder { module .version(env!("FLOW_PHP_EXT_VERSION")) .info_function(php_module_info) - .class::() - .class::() - .class::() - .class::() + .constant(("FLOW_PHP_ABI", FLOW_PHP_ABI, &[])) + .globals(&globals::GLOBALS) + .request_startup_function(globals::request_startup) + .request_shutdown_function(globals::request_shutdown) + .class::() + .class::() + .class::() + .class::() + .class::() + .class::() + .class::() + .class::() + .class::() + .class::() + .class::() } diff --git a/src/extension/flow-php-ext/src/parquet/etl.rs b/src/extension/flow-php-ext/src/parquet/etl.rs new file mode 100644 index 0000000000..2f3999b1a4 --- /dev/null +++ b/src/extension/flow-php-ext/src/parquet/etl.rs @@ -0,0 +1,222 @@ +//! `Flow\ETL\Adapter\Parquet\{RustParquetOpenSource, RustParquetOpenSink}`: `Rows` of `RustColumn`s straight from +//! arrow-ext's Parquet batches, and any `Rows` straight to its writer, through the Arrow C Data Interface. + +use std::cell::RefCell; +use std::rc::Rc; +use std::sync::Arc; + +use ext_php_rs::convert::IntoZval; +use ext_php_rs::flags::ClassFlags; +use ext_php_rs::prelude::*; +use ext_php_rs::types::{ZendHashTable, Zval}; + +use crate::arrow_c; +use crate::backend::{adopt_batch, native_column}; +use crate::batch_columns::HeldBatch; +use crate::column::RustColumn; +use crate::ctx::{ + call_method, column_key, construct, find_class, ht_for_each, ht_insert, null_zval, zval_long, zval_str, +}; +use crate::exception::ext_exception; +use crate::interfaces::{parquet_open_sink_ce, parquet_open_source_ce}; +use crate::iterator::RustIterator; +use crate::plan::{type_plan, TypePlan}; + +const FILE_READER: &str = "Flow\\Parquet\\Engine\\RustParquetFileReader"; +const FILE_WRITER: &str = "Flow\\Parquet\\Engine\\RustParquetFileWriter"; +const BATCH_READER: &str = "Flow\\Arrow\\Parquet\\RustBatchReader"; + +/// `Schema::definitions()` as (name, definition): an int key is a numeric column name. +fn definitions(schema: &Zval) -> PhpResult, Zval)>> { + let definitions = call_method(schema, "definitions", &mut [])?; + let definitions = definitions + .array() + .ok_or_else(|| ext_exception("flow_php expected Schema::definitions() to return an array"))?; + let mut named = Vec::with_capacity(definitions.len()); + + ht_for_each(definitions, |name, index, definition| { + named.push((column_key(name, index), definition.shallow_clone())); + + Ok(()) + })?; + + Ok(named) +} + +/// The next Arrow C batch of `RustBatchReader` as `RustColumn`s under `columns`' plans. +fn next_batch(batches: &Zval, columns: &[(Vec, Rc)]) -> PhpResult> { + let batch = call_method(batches, "next", &mut [])?; + + if batch.is_null() { + return Ok(None); + } + + let (count, arrays) = arrow_c::import(&batch, columns)?; + let mut held = ZendHashTable::with_capacity(columns.len() as u32); + + for ((name, plan), array) in columns.iter().zip(arrays) { + let mut native = Zval::new(); + RustColumn::new(array, Rc::clone(plan)).set_zval(&mut native, false)?; + ht_insert(&mut held, name, native); + } + + Ok(Some(HeldBatch { columns: held, count })) +} + +/// `$object` is arrow-ext's `$class`: the only object whose Arrow state flow_php can read. +fn expect_arrow(object: &Zval, class: &str) -> PhpResult<()> { + if object + .object() + .is_some_and(|object| find_class(class).is_ok_and(|ce| object.instance_of(ce))) + { + return Ok(()); + } + + Err(ext_exception(format!("flow_php expected a {class}"))) +} + +#[php_class] +#[php( + name = "Flow\\ETL\\Adapter\\Parquet\\RustParquetOpenSource", + flags = ClassFlags::Final, + implements(ce = parquet_open_source_ce, stub = "Flow\\ETL\\Adapter\\Parquet\\ParquetOpenSource") +)] +pub struct RustParquetOpenSource { + file: Zval, + /// The `RustBatchReader` of the last `batches()`, closed by the next one and by `close()`. + batches: RefCell>, +} + +#[php_impl] +impl RustParquetOpenSource { + /// `$file`: arrow-ext's `Flow\Parquet\Engine\RustParquetFileReader`, which owns the stream. + pub fn __construct(file: &Zval) -> PhpResult { + expect_arrow(file, FILE_READER)?; + + Ok(Self { + file: file.shallow_clone(), + batches: RefCell::new(None), + }) + } + + /// Batches of at most `$batchSize` rows of `$schema`'s columns, every column adopted by `$backend`; the file's + /// schema is checked before any batch. + pub fn batches( + &self, + schema: &Zval, + batchSize: i64, + offset: Option, + limit: Option, + backend: &Zval, + ) -> PhpResult { + let mut columns = Vec::new(); + let mut names = ZendHashTable::new(); + + for (name, definition) in definitions(schema)? { + columns.push((name.clone(), type_plan(&call_method(&definition, "type", &mut [])?)?)); + names.push(zval_str(&name))?; + } + + let mut names_zv = Zval::new(); + names_zv.set_hashtable(names); + let optional = |value: Option| value.map_or_else(null_zval, zval_long); + + self.close()?; + + let batches = construct( + find_class(BATCH_READER)?, + &mut [ + self.file.shallow_clone(), + names_zv, + zval_long(batchSize), + optional(offset), + optional(limit), + ], + )? + .into_zval(false)?; + *self.batches.borrow_mut() = Some(batches.shallow_clone()); + + arrow_c::check_schema(&call_method(&batches, "schema", &mut [])?, &columns)?; + + let (schema, backend) = (schema.shallow_clone(), backend.shallow_clone()); + + Ok(RustIterator::new(Box::new(move || { + next_batch(&batches, &columns)? + .map(|batch| adopt_batch(&schema, &backend, batch)) + .transpose() + }))) + } + + pub fn close(&self) -> PhpResult<()> { + // the borrow ends before close() runs PHP + let batches = self.batches.borrow_mut().take(); + + if let Some(batches) = batches { + call_method(&batches, "close", &mut [])?; + } + + Ok(()) + } +} + +#[php_class] +#[php( + name = "Flow\\ETL\\Adapter\\Parquet\\RustParquetOpenSink", + flags = ClassFlags::Final, + implements(ce = parquet_open_sink_ce, stub = "Flow\\ETL\\Adapter\\Parquet\\ParquetOpenSink") +)] +pub struct RustParquetOpenSink { + writer: Zval, +} + +#[php_impl] +impl RustParquetOpenSink { + /// `$writer`: arrow-ext's `Flow\Parquet\Engine\RustParquetFileWriter`, which owns the stream. + pub fn __construct(writer: &Zval) -> PhpResult { + expect_arrow(writer, FILE_WRITER)?; + + Ok(Self { + writer: writer.shallow_clone(), + }) + } + + /// Every column of `$rows` - a `RustColumn` as it is, any other column adopted into one - as one batch; the + /// writer takes its columns by name and writes a column the batch lacks as nulls. + pub fn write(&mut self, rows: &Zval) -> PhpResult<()> { + let count = call_method(rows, "count", &mut [])? + .long() + .ok_or_else(|| ext_exception("flow_php expected Rows::count() to return an int"))? + as usize; + let columns = call_method(rows, "columns", &mut [])?; + let columns = columns + .array() + .ok_or_else(|| ext_exception("flow_php expected Rows::columns() to return an array"))?; + let schema = call_method(rows, "schema", &mut [])?; + let mut arrays = Vec::with_capacity(columns.len()); + + ht_for_each(columns, |name, index, column| { + let name = column_key(name, index); + let data = match column.extract::<&RustColumn>() { + Some(native) => Arc::clone(native.data()), + None => native_column(&call_method(&schema, "get", &mut [zval_str(&name)])?, column)?.1, + }; + + arrays.push((String::from_utf8_lossy(&name).into_owned(), data)); + + Ok(()) + })?; + + let mut batch = Zval::new(); + arrow_c::export(count, arrays)?.set_zval(&mut batch, false)?; + call_method(&self.writer, "writeArrowBatch", &mut [batch])?; + + Ok(()) + } + + /// The writer's buffered rows, the footer, then the stream closed. + pub fn close(&mut self) -> PhpResult<()> { + call_method(&self.writer, "close", &mut [])?; + + Ok(()) + } +} diff --git a/src/extension/flow-php-ext/src/parquet/mod.rs b/src/extension/flow-php-ext/src/parquet/mod.rs new file mode 100644 index 0000000000..fcf58cca5e --- /dev/null +++ b/src/extension/flow-php-ext/src/parquet/mod.rs @@ -0,0 +1,7 @@ +//! Parquet in flow_php: the `Flow\ETL\Adapter\Parquet` surface over arrow-ext's Parquet batches. + +#[allow( + non_snake_case, + reason = "PHP takes a parameter name from its Rust identifier: the contracts' are camelCase" +)] +pub mod etl; diff --git a/src/extension/flow-php-ext/src/physical.rs b/src/extension/flow-php-ext/src/physical.rs new file mode 100644 index 0000000000..e79f1c7489 --- /dev/null +++ b/src/extension/flow-php-ext/src/physical.rs @@ -0,0 +1,293 @@ +//! Arrow rows as the zvals the PHP columns hold and return: `at()` (the physical form `PhysicalFor` produces), +//! `value()` (the logical form `Physical::fromPhysical()` builds), and a physical zval appended back into storage. + +use arrow_array::cast::AsArray; +use arrow_array::types::{Date32Type, DurationMicrosecondType, Float64Type, Int64Type, TimestampMicrosecondType}; +use arrow_array::Array; +use ext_php_rs::boxed::ZBox; +use ext_php_rs::exception::PhpException; +use ext_php_rs::flags::DataType; +use ext_php_rs::types::{ZendHashTable, Zval}; +use flow_batch_frame::kind::Kind; + +use crate::ctx::{self, call_method, ht_for_each, ht_insert, null_zval, zval_long, zval_str}; +use crate::exception::ext_exception; +use crate::kind_builder::{KindBuilder, OffsetOverflow}; +use crate::plan::ValueNode; +use crate::render::{invalid_argument, offset_overflow}; +use crate::values::{date_from_days, datetime_from_micros, interval_from_micros, json_from_bytes, uuid_from_bytes}; + +fn long(array: &dyn Array, kind: &Kind, i: usize) -> i64 { + match kind { + Kind::Int64 => array.as_primitive::().value(i), + Kind::Timestamp => array.as_primitive::().value(i), + Kind::Duration => array.as_primitive::().value(i), + Kind::Int32 => i64::from(array.as_primitive::().value(i)), + _ => unreachable!("long() reads integer kinds only"), + } +} + +fn double(value: f64) -> Zval { + let mut zv = Zval::new(); + zv.set_double(value); + zv +} + +fn boolean(value: bool) -> Zval { + let mut zv = Zval::new(); + zv.set_bool(value); + zv +} + +fn array_zval(ht: ZBox) -> Zval { + let mut zv = Zval::new(); + zv.set_hashtable(ht); + zv +} + +fn push(ht: &mut ZendHashTable, value: Zval) -> Result<(), PhpException> { + ht.push(value) + .map_err(|e| ext_exception(format!("flow_php failed to collect column values: {e:?}"))) +} + +/// A map key as `array_keys()` returns it: the int or string key `array_combine()` coerces. +fn map_key(ht: &mut ZendHashTable, keys: &dyn Array, key: &Kind, j: usize, value: Zval) { + match key { + Kind::Bytes => ht_insert(ht, keys.as_binary::().value(j), value), + _ => { + let index = long(keys, key, j); + crate::ctx::ht_insert_long(ht, index, value); + } + } +} + +/// `Column::at($i)`: null, int (µs, days), float, bool, string (16 bytes for a uuid), or nested arrays. +pub fn physical_at(array: &dyn Array, kind: &Kind, i: usize) -> Result { + if matches!(kind, Kind::Null) || array.is_null(i) { + return Ok(null_zval()); + } + + Ok(match kind { + Kind::Null => unreachable!("returned above"), + Kind::Int64 | Kind::Timestamp | Kind::Duration | Kind::Int32 => zval_long(long(array, kind, i)), + Kind::Float64 => double(array.as_primitive::().value(i)), + Kind::Boolean => boolean(array.as_boolean().value(i)), + Kind::Uuid => zval_str(array.as_fixed_size_binary().value(i)), + Kind::Bytes => zval_str(array.as_binary::().value(i)), + Kind::List(element) => { + let list = array.as_list::(); + let offsets = list.value_offsets(); + let mut ht = ZendHashTable::with_capacity((offsets[i + 1] - offsets[i]) as u32); + + for j in offsets[i] as usize..offsets[i + 1] as usize { + push(&mut ht, physical_at(list.values().as_ref(), &element.kind, j)?)?; + } + + array_zval(ht) + } + Kind::Map(key, value) => { + let map = array.as_map(); + let offsets = map.value_offsets(); + let mut ht = ZendHashTable::with_capacity((offsets[i + 1] - offsets[i]) as u32); + + for j in offsets[i] as usize..offsets[i + 1] as usize { + let physical = physical_at(map.values().as_ref(), &value.kind, j)?; + map_key(&mut ht, map.keys().as_ref(), &key.kind, j, physical); + } + + array_zval(ht) + } + Kind::Struct(fields) => { + let structure = array.as_struct(); + let mut ht = ZendHashTable::with_capacity(fields.len() as u32); + + for (field, child) in fields.iter().zip(structure.columns()) { + if matches!(field.kind, Kind::Null) || child.is_null(i) { + if !field.optional { + ht_insert(&mut ht, field.name.as_bytes(), null_zval()); + } + + continue; + } + + ht_insert( + &mut ht, + field.name.as_bytes(), + physical_at(child.as_ref(), &field.kind, i)?, + ); + } + + array_zval(ht) + } + }) +} + +/// `Column::value($i)`: the logical value of row `i`, built natively except markup (D4). +pub fn value_at(array: &dyn Array, kind: &Kind, node: &ValueNode, i: usize) -> Result { + if matches!(kind, Kind::Null) || array.is_null(i) { + return Ok(null_zval()); + } + + match (node, kind) { + (ValueNode::DateTime(zone), _) => datetime_from_micros(long(array, kind, i), zone), + (ValueNode::Date, _) => date_from_days(long(array, kind, i)), + (ValueNode::Time, _) => interval_from_micros(long(array, kind, i)), + (ValueNode::Uuid, _) => uuid_from_bytes(array.as_fixed_size_binary().value(i)), + (ValueNode::Json, _) => json_from_bytes(array.as_binary::().value(i)), + (ValueNode::Enum(class), _) => ctx::enum_case(class, array.as_binary::().value(i)), + (ValueNode::TimeZone, _) => ctx::timezone(array.as_binary::().value(i)), + (ValueNode::Markup(physical), _) => call_method( + physical, + "fromPhysical", + &mut [zval_str(array.as_binary::().value(i))], + ), + (ValueNode::List(element_node), Kind::List(element)) => { + let list = array.as_list::(); + let offsets = list.value_offsets(); + let mut ht = ZendHashTable::with_capacity((offsets[i + 1] - offsets[i]) as u32); + + for j in offsets[i] as usize..offsets[i + 1] as usize { + push( + &mut ht, + value_at(list.values().as_ref(), &element.kind, element_node, j)?, + )?; + } + + Ok(array_zval(ht)) + } + (ValueNode::Map(value_node), Kind::Map(key, value)) => { + let map = array.as_map(); + let offsets = map.value_offsets(); + let mut ht = ZendHashTable::with_capacity((offsets[i + 1] - offsets[i]) as u32); + + for j in offsets[i] as usize..offsets[i + 1] as usize { + let logical = value_at(map.values().as_ref(), &value.kind, value_node, j)?; + map_key(&mut ht, map.keys().as_ref(), &key.kind, j, logical); + } + + Ok(array_zval(ht)) + } + (ValueNode::Struct(nodes), Kind::Struct(fields)) => { + let structure = array.as_struct(); + let mut ht = ZendHashTable::with_capacity(fields.len() as u32); + + for ((field, child), child_node) in fields.iter().zip(structure.columns()).zip(nodes) { + if matches!(field.kind, Kind::Null) || child.is_null(i) { + if !field.optional { + ht_insert(&mut ht, field.name.as_bytes(), null_zval()); + } + + continue; + } + + ht_insert( + &mut ht, + field.name.as_bytes(), + value_at(child.as_ref(), &field.kind, child_node, i)?, + ); + } + + Ok(array_zval(ht)) + } + _ => physical_at(array, kind, i), + } +} + +fn wrong_physical(kind: &Kind, physical: &Zval) -> PhpException { + invalid_argument(format!( + "flow_php cannot store a {} physical in a {kind:?} column", + physical.get_type() + )) +} + +pub fn overflow(OffsetOverflow(last): OffsetOverflow) -> PhpException { + offset_overflow(last) +} + +/// Appends one physical zval (what `Physical::toPhysical()` or `Column::at()` returns) to `builder`. +pub fn append_physical(builder: &mut KindBuilder, kind: &Kind, physical: &Zval) -> Result<(), PhpException> { + let physical = physical.dereference(); + + if physical.is_null() || matches!(kind, Kind::Null) { + builder.append_null(); + + return Ok(()); + } + + match kind { + Kind::Null => unreachable!("appended above"), + Kind::Int64 | Kind::Timestamp | Kind::Duration => { + let value = physical.long().ok_or_else(|| wrong_physical(kind, physical))?; + builder.append_fixed(&value.to_le_bytes()); + } + Kind::Int32 => { + let value = physical + .long() + .and_then(|value| i32::try_from(value).ok()) + .ok_or_else(|| wrong_physical(kind, physical))?; + builder.append_fixed(&value.to_le_bytes()); + } + Kind::Float64 => { + let value = physical.double().ok_or_else(|| wrong_physical(kind, physical))?; + builder.append_fixed(&value.to_le_bytes()); + } + Kind::Boolean => builder.append_bool(physical.bool().ok_or_else(|| wrong_physical(kind, physical))?), + Kind::Uuid => { + let bytes = physical + .zend_str() + .map(|string| string.as_bytes()) + .filter(|bytes| bytes.len() == 16) + .ok_or_else(|| wrong_physical(kind, physical))?; + builder.append_fixed(bytes); + } + Kind::Bytes => builder + .append_bytes( + physical + .zend_str() + .ok_or_else(|| wrong_physical(kind, physical))? + .as_bytes(), + ) + .map_err(overflow)?, + Kind::List(element) => { + let values = physical.array().ok_or_else(|| wrong_physical(kind, physical))?; + + ht_for_each(values, |_, _, value| { + append_physical(builder.list_element(), &element.kind, value) + })?; + builder.end_entries().map_err(overflow)?; + } + Kind::Map(key, value) => { + let entries = physical.array().ok_or_else(|| wrong_physical(kind, physical))?; + + ht_for_each(entries, |string_key, index, item| { + let (keys, values) = builder.map_entries(); + let key_zv = match string_key { + Some(name) => zval_str(name.as_bytes()), + None => zval_long(index as i64), + }; + // array keys are ints for numeric strings: a string-keyed map stores their decimal text + let key_zv = match (&key.kind, key_zv.get_type()) { + (Kind::Bytes, DataType::Long) => zval_str(index.to_string().as_bytes()), + _ => key_zv, + }; + + append_physical(keys, &key.kind, &key_zv)?; + append_physical(values, &value.kind, item) + })?; + builder.end_entries().map_err(overflow)?; + } + Kind::Struct(fields) => { + let elements = physical.array().ok_or_else(|| wrong_physical(kind, physical))?; + let null = null_zval(); + + for (child, field) in builder.struct_children().iter_mut().zip(fields) { + let element = crate::ctx::ht_get(elements, field.name.as_bytes()).unwrap_or(&null); + append_physical(child, &field.kind, element)?; + } + + builder.end_struct(); + } + } + + Ok(()) +} diff --git a/src/extension/flow-php-ext/src/plan.rs b/src/extension/flow-php-ext/src/plan.rs index fa08a5b0a4..fe5bc01ab4 100644 --- a/src/extension/flow-php-ext/src/plan.rs +++ b/src/extension/flow-php-ext/src/plan.rs @@ -1,217 +1,146 @@ -//! Decode plan built from a SCHEMA frame body (a JSON list of normalized definitions): -//! per column a name and a recursive value `Decoder`, in schema order. +//! What a column of one Flow type needs beyond its `Kind`, derived once per type JSON (`Type::normalize()`) and shared +//! by every column and builder of that type: the Type object, its cast lane, its `Physical`, and how its logical +//! values are built. -use ext_php_rs::exception::PhpException; -use serde::Deserialize; - -use crate::ctx::Ctx; -use crate::exception::ext_exception; +use std::rc::Rc; -pub enum MapKey { - Integer, - String, +use ext_php_rs::exception::PhpException; +use ext_php_rs::types::Zval; +use ext_php_rs::zend::Function; +use flow_batch_frame::kind::{kind_of, parse_type, Kind, TypeJson}; + +use crate::cast::{build_cast_kind, CastKind}; +use crate::ctx::{self, call_handle, call_method, ce_method_ref, expect_object, find_class, instance}; +use crate::exception::{ext_exception, json_exception}; + +const PHYSICAL_FOR: &str = "Flow\\ETL\\Column\\Physical\\PhysicalFor"; +const IDENTITY: &str = "Flow\\ETL\\Column\\Physical\\IdentityPhysical"; + +pub struct TypePlan { + pub kind: Kind, + pub type_zv: Zval, + pub values: ValueNode, + pub cast: CastKind, + /// `$type->cast()` of the concrete Type class. + pub cast_fn: &'static Function, + /// `PhysicalFor::type($type)` and its `toPhysical()`. + pub physical: Zval, + pub to_physical: &'static Function, + /// The physical is `IdentityPhysical`: the cast value is the physical one. + pub identity: bool, } -/// Recursive value decoder mirroring `ValueDecoder::decoderFor` dispatch. -pub enum Decoder { - Integer, - Float, - Boolean, - String, +/// How a logical value is built from its physical one. +pub enum ValueNode { + Plain, Null, - /// A `datetime` column: the stored instant, in the column zone. DateTime(Vec), - /// A `date`, or a zone-less `datetime` from an older flow-php/etl: each value's stored zone. - StoredZoneDateTime, - Interval, + Date, + Time, Uuid, Json, + Enum(Vec), TimeZone, - Enum, - Xml, - XmlElement, - Html, - HtmlElement, - List(Box), - Map(MapKey, Box), - Structure(Vec<(Vec, Decoder)>), - Optional(Box), -} - -pub struct Column { - pub name: String, - pub decoder: Decoder, -} - -pub struct Plan { - pub columns: Vec, + /// A markup `Physical` whose `fromPhysical()` builds the DOM value (D4). + Markup(Zval), + List(Box), + Map(Box), + Struct(Vec), } -#[derive(Deserialize, Default)] -pub struct TypeJson { - #[serde(rename = "type")] - pub type_: String, - element: Option>, - key: Option>, - value: Option>, - base: Option>, - #[serde(default)] - fields: Vec, - #[serde(default)] - zone: Option, -} - -#[derive(Deserialize)] -pub struct StructureElementJson { - // ALWAYS a JSON string - PHP casts the name on normalize(). A JSON number here - // would fail parse_schema_json for the WHOLE schema, not just this field. - pub name: String, - #[serde(rename = "type")] - pub type_: TypeJson, - #[serde(default)] - pub optional: bool, -} - -impl TypeJson { - pub fn element(&self) -> Option<&TypeJson> { - self.element.as_deref() - } - - pub fn key(&self) -> Option<&TypeJson> { - self.key.as_deref() +/// The plan of `$type`, built on first use of its type JSON. +pub fn type_plan(type_zv: &Zval) -> Result, PhpException> { + if let Some(plan) = ctx::type_plan_of(type_zv)? { + return Ok(plan); } - pub fn value(&self) -> Option<&TypeJson> { - self.value.as_deref() - } - - pub fn base(&self) -> Option<&TypeJson> { - self.base.as_deref() - } + let plan = json_plan(type_zv)?; + ctx::store_type_plan(type_zv, &plan)?; - pub fn fields(&self) -> &[StructureElementJson] { - &self.fields - } + Ok(plan) +} - pub fn zone(&self) -> Option<&[u8]> { - self.zone.as_deref().map(str::as_bytes) +fn json_plan(type_zv: &Zval) -> Result, PhpException> { + let normalized = call_method(type_zv, "normalize", &mut [])?; + let json = call_handle(ctx::json_encode()?, None, &mut [normalized], "encode a type as JSON")?; + let json = json + .zend_str() + .ok_or_else(|| ext_exception("flow_php expected type JSON to be a string"))? + .as_bytes() + .to_vec(); + + if let Some(plan) = ctx::plan(&json)? { + return Ok(plan); } -} -#[derive(Deserialize)] -pub struct NormalizedDefinition { - #[serde(rename = "ref")] - pub name: String, - #[serde(rename = "type")] - pub type_: TypeJson, - pub nullable: bool, + let parsed = parse_type(&json).map_err(json_exception)?; + let type_obj = expect_object(type_zv, "a Type")?; + let type_ce = + unsafe { type_obj.ce.as_ref() }.ok_or_else(|| ext_exception("flow_php failed to resolve a Type class"))?; + let physical = call_method(&instance(PHYSICAL_FOR)?, "type", &mut [type_zv.shallow_clone()])?; + let plan = Rc::new(TypePlan { + kind: kind_of(&parsed).map_err(json_exception)?, + type_zv: type_zv.shallow_clone(), + values: value_node(&parsed)?, + cast: build_cast_kind(&parsed), + cast_fn: ce_method_ref(type_ce, "cast")?, + to_physical: ce_method_ref( + unsafe { expect_object(&physical, "a Physical")?.ce.as_ref() } + .ok_or_else(|| ext_exception("flow_php failed to resolve a Physical class"))?, + "toPhysical", + )?, + physical: physical.shallow_clone(), + identity: expect_object(&physical, "a Physical")?.instance_of(find_class(IDENTITY)?), + }); + ctx::store_plan(json, &plan)?; + + Ok(plan) } -pub fn parse_schema_json(schema_json: &[u8]) -> Result, PhpException> { - let json = std::str::from_utf8(schema_json) - .map_err(|_| ext_exception("flow_php failed to decode schema JSON: invalid UTF-8"))?; - - serde_json::from_str(json) - .map_err(|e| ext_exception(format!("flow_php failed to decode schema JSON: {e}"))) +/// `PhysicalFor::definition($definition)`: its refusals (wildcard enum, mixed, a foreign definition) are PHP's. +pub fn physical_for_definition(definition: &Zval) -> Result { + call_method( + &instance(PHYSICAL_FOR)?, + "definition", + &mut [definition.shallow_clone()], + ) } -fn build_decoder(type_json: &TypeJson) -> Result { - let missing = |part: &str| { - ext_exception(format!( - "flow_php schema JSON for type \"{}\" is missing its {part}", - type_json.type_ - )) +fn value_node(parsed: &TypeJson) -> Result { + let child = |part: Option<&TypeJson>| -> Result, PhpException> { + Ok(Box::new(value_node(part.ok_or_else(|| { + ext_exception("flow_php type JSON is missing a nested type") + })?)?)) }; - Ok(match type_json.type_.as_str() { - "integer" | "positive_integer" => Decoder::Integer, - "float" => Decoder::Float, - "boolean" => Decoder::Boolean, - "string" | "non_empty_string" | "numeric-string" | "class_string" => Decoder::String, - "null" => Decoder::Null, - "datetime" => match type_json.zone() { - Some(zone) => Decoder::DateTime(zone.to_vec()), - None => Decoder::StoredZoneDateTime, - }, - "date" => Decoder::StoredZoneDateTime, - "time" => Decoder::Interval, - "uuid" => Decoder::Uuid, - "json" => Decoder::Json, - "timezone" => Decoder::TimeZone, - "enum" => Decoder::Enum, - "xml" => Decoder::Xml, - "xml_element" => Decoder::XmlElement, - "html" => Decoder::Html, - "html_element" => Decoder::HtmlElement, - "list" => Decoder::List(Box::new(build_decoder( - type_json - .element - .as_ref() - .ok_or_else(|| missing("element"))?, - )?)), - "map" => { - let key = match type_json - .key - .as_ref() - .ok_or_else(|| missing("key"))? - .type_ - .as_str() - { - "integer" => MapKey::Integer, - "string" => MapKey::String, - other => { - return Err(ext_exception(format!( - "flow_php does not support map keys of type \"{other}\"" - ))); - } - }; - - Decoder::Map( - key, - Box::new(build_decoder( - type_json.value.as_ref().ok_or_else(|| missing("value"))?, - )?), - ) - } - "structure_v2" => { - if type_json.fields.is_empty() { - return Err(ext_exception( - "flow_php read a structure type with no fields; the loaded flow_php extension and the \ - flow-php/etl in use disagree on the structure schema format - reinstall one to match the \ - other, or set the Floe engine to FloeEngine::php", - )); - } - - let mut elements = Vec::with_capacity(type_json.fields.len()); - - for field in &type_json.fields { - elements.push((field.name.clone().into_bytes(), build_decoder(&field.type_)?)); - } - - Decoder::Structure(elements) - } - "optional" => Decoder::Optional(Box::new(build_decoder( - type_json.base.as_ref().ok_or_else(|| missing("base"))?, - )?)), - other => { - return Err(ext_exception(format!( - "flow_php does not support values of type \"{other}\" in this build" - ))); - } + Ok(match parsed.type_.as_str() { + "datetime" => ValueNode::DateTime(parsed.zone().unwrap_or(b"UTC").to_vec()), + "date" => ValueNode::Date, + "time" => ValueNode::Time, + "uuid" => ValueNode::Uuid, + "json" => ValueNode::Json, + "enum" => ValueNode::Enum( + parsed + .class() + .ok_or_else(|| ext_exception("flow_php enum type JSON is missing its class"))? + .to_vec(), + ), + "timezone" => ValueNode::TimeZone, + "xml" => ValueNode::Markup(instance("Flow\\ETL\\Column\\Physical\\XmlDocumentPhysical")?), + "xml_element" => ValueNode::Markup(instance("Flow\\ETL\\Column\\Physical\\XmlElementPhysical")?), + "html" => ValueNode::Markup(instance("Flow\\ETL\\Column\\Physical\\HtmlDocumentPhysical")?), + "html_element" => ValueNode::Markup(instance("Flow\\ETL\\Column\\Physical\\HtmlElementPhysical")?), + "null" => ValueNode::Null, + "list" => ValueNode::List(child(parsed.element())?), + "map" => ValueNode::Map(child(parsed.value())?), + "structure_v2" => ValueNode::Struct( + parsed + .fields() + .iter() + .map(|field| value_node(&field.type_)) + .collect::>()?, + ), + "optional" => *child(parsed.base())?, + _ => ValueNode::Plain, }) } - -pub fn build_plan(schema_json: &[u8], _ctx: &mut Ctx) -> Result { - let definitions = parse_schema_json(schema_json)?; - - let mut columns = Vec::with_capacity(definitions.len()); - - for definition in &definitions { - columns.push(Column { - name: definition.name.clone(), - decoder: build_decoder(&definition.type_)?, - }); - } - - Ok(Plan { columns }) -} diff --git a/src/extension/flow-php-ext/src/render.rs b/src/extension/flow-php-ext/src/render.rs new file mode 100644 index 0000000000..9963536087 --- /dev/null +++ b/src/extension/flow-php-ext/src/render.rs @@ -0,0 +1,205 @@ +//! flow-batch-frame refusals rendered as the exceptions `PhpBackend::decode()` throws for the same buffers: the +//! `Flow\ETL\Exception\InvalidArgumentException` (or `OffsetOverflow`) with its exact message, a nested type named by +//! its PHP `toString()`. + +use ext_php_rs::exception::PhpException; +use ext_php_rs::types::Zval; +use flow_batch_frame::error::{Of, Role, Values}; +use flow_batch_frame::kind::Kind; +use flow_batch_frame::layout::node_count; +use flow_batch_frame::Error; + +use crate::ctx::{call_method, expect_object, find_class, read_property}; +use crate::exception::ext_exception; + +const INVALID_ARGUMENT: &str = "Flow\\ETL\\Exception\\InvalidArgumentException"; +const OFFSET_OVERFLOW: &str = "Flow\\ETL\\Exception\\OffsetOverflow"; +pub const RUNTIME: &str = "Flow\\ETL\\Exception\\RuntimeException"; + +pub fn exception(class: &str, message: String) -> PhpException { + match find_class(class) { + Ok(ce) => PhpException::new(message, 0, ce), + Err(e) => e, + } +} + +pub fn invalid_argument(message: String) -> PhpException { + exception(INVALID_ARGUMENT, message) +} + +pub fn runtime(message: String) -> PhpException { + exception(RUNTIME, message) +} + +pub fn offset_overflow(last: u64) -> PhpException { + exception( + OFFSET_OVERFLOW, + format!("Offsets exceed the i32 range of a batch buffer: the last offset is {last}"), + ) +} + +fn of(of: Of) -> &'static str { + match of { + Of::List => "List", + Of::Map => "Map", + Of::Utf8 => "Utf8", + } +} + +fn values(values: Values) -> &'static str { + match values { + Values::Int64 => "Int64", + Values::Int32 => "Int32", + Values::Float64 => "Float64", + Values::Boolean => "Boolean", + Values::FixedBinary16 => "FixedBinary16", + } +} + +/// A `column::decode` refusal of the column typed `type_zv`, whose kind is `kind`. +pub fn decode_exception(error: Error, type_zv: &Zval, kind: &Kind) -> PhpException { + let message = match error { + Error::OffsetOverflow { last } => return offset_overflow(last), + Error::BuffersExhausted => "Column buffers exhausted: the column needs more buffers than given".to_string(), + Error::OffsetsLength { + of: layout, + bytes, + expected, + rows, + } => { + format!( + "{} offsets buffer of {bytes} bytes, expected {expected} for {rows} rows", + of(layout) + ) + } + Error::MapEntriesValidity { bytes } => format!("Map entries validity must be omitted, got {bytes} bytes"), + Error::ValidityTooShort { bytes, rows } => { + format!("Validity bitmap of {bytes} bytes is too short for {rows} rows") + } + Error::NullKindNullCount { rows, null_count } => { + format!("Null column of {rows} rows carries a null count of {null_count}") + } + Error::NullCountMismatch { + declared, + derived, + rows, + } => { + format!("Column null count {declared} disagrees with its validity bitmap ({derived} nulls in {rows} rows)") + } + Error::NestedNulls { node, role, nulls } => { + let name = match type_name_at(type_zv, kind, node) { + Ok(name) => name, + Err(e) => return e, + }; + + format!( + "{} holds {nulls} nulls in a non-nullable {name}", + match role { + Role::ListElement => "List child", + Role::MapKey => "Map key child", + Role::MapValue => "Map value child", + } + ) + } + Error::BufferCount { + node, + buffers, + expected, + } => { + let name = match type_name_at(type_zv, kind, node) { + Ok(name) => name, + Err(e) => return e, + }; + + format!("Column of type {name} holds {buffers} buffers, its layout needs {expected}") + } + Error::OffsetsStart { of: layout, first } => format!("{} offsets start at {first}, not 0", of(layout)), + Error::OffsetsNotMonotonic { + of: layout, + index, + previous, + current, + } => format!( + "{} offsets are not monotonic: offset {} is {previous}, offset {index} is {current}", + of(layout), + index - 1 + ), + Error::ValuesLength { + values: layout, + bytes, + expected, + rows, + } => { + format!( + "{} values buffer of {bytes} bytes, expected {expected} for {rows} rows", + values(layout) + ) + } + Error::Utf8DataLength { bytes, last_offset } => { + format!("Utf8 data buffer of {bytes} bytes, the last offset is {last_offset}") + } + error => format!("flow_php refused the column buffers: {error}"), + }; + + invalid_argument(message) +} + +/// `toString()` of the type at pre-order `node` of `type_zv`'s tree (the map entries node counts but has no type). +fn type_name_at(type_zv: &Zval, kind: &Kind, node: usize) -> Result { + let name = call_method(&type_at(type_zv.shallow_clone(), kind, node)?, "toString", &mut [])?; + + Ok(String::from_utf8_lossy( + name.zend_str() + .ok_or_else(|| ext_exception("flow_php expected Type::toString() to return a string"))? + .as_bytes(), + ) + .into_owned()) +} + +fn type_at(type_zv: Zval, kind: &Kind, node: usize) -> Result { + if node == 0 { + return Ok(type_zv); + } + + let optional = find_class("Flow\\Types\\Type\\Logical\\OptionalType")?; + let base = if expect_object(&type_zv, "a Type")?.instance_of(optional) { + call_method(&type_zv, "base", &mut [])? + } else { + type_zv + }; + + match kind { + Kind::List(element) => type_at(call_method(&base, "element", &mut [])?, &element.kind, node - 1), + Kind::Map(key, value) => { + let key_nodes = node_count(&key.kind); + + if node - 2 < key_nodes { + type_at(call_method(&base, "key", &mut [])?, &key.kind, node - 2) + } else { + type_at(call_method(&base, "value", &mut [])?, &value.kind, node - 2 - key_nodes) + } + } + Kind::Struct(fields) => { + let elements = call_method(&base, "elements", &mut [])?; + let elements = elements + .array() + .ok_or_else(|| ext_exception("flow_php expected StructureType::elements() to return an array"))?; + let mut first = 1; + + for (field, element) in fields.iter().zip(elements.values()) { + let nodes = node_count(&field.kind); + + if node < first + nodes { + let element_type = read_property(expect_object(element, "a StructureElement")?, "type")?; + + return type_at(element_type, &field.kind, node - first); + } + + first += nodes; + } + + Err(ext_exception("flow_php could not find a structure node")) + } + _ => Err(ext_exception("flow_php could not find a nested node")), + } +} diff --git a/src/extension/flow-php-ext/src/source.rs b/src/extension/flow-php-ext/src/source.rs new file mode 100644 index 0000000000..5ed7c32120 --- /dev/null +++ b/src/extension/flow-php-ext/src/source.rs @@ -0,0 +1,58 @@ +//! What the Rust open sources share: the chunk size they read streams in and the batches loop over a reader the +//! stream is fed into. + +use std::cell::{RefCell, RefMut}; + +use ext_php_rs::prelude::*; +use ext_php_rs::types::Zval; + +use crate::backend::adopt_batch; +use crate::batch_columns::HeldBatch; +use crate::iterator::RustIterator; +use crate::render::runtime; + +/// Each chunk is held as a PHP string, so it sets the read's peak memory: 32 KB matched the PHP path's own peak on wide +/// and narrow files. `AdaptiveJsonOpenSource::CHUNK` is its PHP copy. +pub const CHUNK: i64 = 1 << 15; + +/// A reader a stream is fed into, one consuming call at a time. +pub trait Fed { + /// The next full batch of what was fed (the remainder once finished), none while more input is needed. + fn next_batch(&self) -> PhpResult>; + + /// The next chunk of the stream fed in; false at its end. + fn feed(&self) -> PhpResult; + + fn finish(&self) -> PhpResult<()>; +} + +/// The batches of `fed`, each adopted by `backend` into `Rows` of `schema`. +pub fn batches(fed: impl Fed + 'static, schema: &Zval, backend: &Zval) -> RustIterator { + let (schema, backend) = (schema.shallow_clone(), backend.shallow_clone()); + let mut finished = false; + + RustIterator::new(Box::new(move || loop { + if let Some(batch) = fed.next_batch()? { + return adopt_batch(&schema, &backend, batch).map(Some); + } + + if finished { + return Ok(None); + } + + if !fed.feed()? { + fed.finish()?; + finished = true; + } + })) +} + +/// The reader, borrowed only while Rust code runs on it: a stream, a backend or an inferrer that calls back into the +/// source mid-read is refused, never a panic. +pub fn reading<'a, T>(reader: &'a RefCell, class: &str) -> PhpResult> { + reader.try_borrow_mut().map_err(|_| { + runtime(format!( + "{class} is already reading: a read cannot start from inside another" + )) + }) +} diff --git a/src/extension/flow-php-ext/src/text/batch.rs b/src/extension/flow-php-ext/src/text/batch.rs new file mode 100644 index 0000000000..07920cbad4 --- /dev/null +++ b/src/extension/flow-php-ext/src/text/batch.rs @@ -0,0 +1,176 @@ +//! A `Rows` batch as the native text writers read it: the schema's columns in order, each either an arrow array the +//! writer renders or left to the cells PHP rendered for it. + +use std::rc::Rc; + +use arrow_array::ArrayRef; +use ext_php_rs::boxed::ZBox; +use ext_php_rs::prelude::*; +use ext_php_rs::types::{ZendHashTable, Zval}; + +use crate::backend::native_column; +use crate::column::RustColumn; +use crate::ctx::{call_method, ht_get, ht_insert, zval_str}; +use crate::exception::ext_exception; +use crate::plan::{type_plan, TypePlan}; +use crate::render::invalid_argument; +use crate::text::value::{renders, Formats, Renderer}; + +pub struct Column { + pub name: Vec, + /// `None`: the column is not rendered natively. + pub native: Option<(Rc, ArrayRef)>, +} + +pub struct Batch { + pub count: usize, + pub columns: Vec, +} + +fn definitions(schema: &Zval) -> PhpResult { + call_method(schema, "definitions", &mut []) +} + +/// The names of the columns of `$schema` the native lane does not render. +fn unrendered_names(schema: &Zval, formats: &Formats, json_column_is_text: bool) -> PhpResult>> { + let definitions = definitions(schema)?; + let definitions = definitions + .array() + .ok_or_else(|| ext_exception("flow_php expected Schema::definitions() to return an array"))?; + let mut names = Vec::new(); + + for (name, definition) in definitions.iter() { + if !renders( + &type_plan(&call_method(definition, "type", &mut [])?)?.values, + formats, + json_column_is_text, + ) { + names.push(name.to_string().into_bytes()); + } + } + + Ok(names) +} + +/// The unrendered columns of the last `Schema` object a writer saw; consecutive batches share one. +#[derive(Default)] +pub struct Unrendered { + schema: Option, + names: Vec>, +} + +impl Unrendered { + /// `$php->$method($schema->get($name)->type(), $rows->column($name))` of every unrendered column, by name. + pub fn render( + &mut self, + rows: &Zval, + php: &Zval, + method: &str, + formats: &Formats, + json_column_is_text: bool, + ) -> PhpResult> { + let schema = call_method(rows, "schema", &mut [])?; + let held = self.schema.as_ref().and_then(Zval::object); + + if !matches!((held, schema.object()), (Some(held), Some(object)) if std::ptr::eq(held, object)) { + self.names = unrendered_names(&schema, formats, json_column_is_text)?; + self.schema = Some(schema.shallow_clone()); + } + + let mut rendered = ZendHashTable::with_capacity(self.names.len() as u32); + + for name in &self.names { + let definition = call_method(&schema, "get", &mut [zval_str(name)])?; + let mut args = [ + call_method(&definition, "type", &mut [])?, + call_method(rows, "column", &mut [zval_str(name)])?, + ]; + + ht_insert(&mut rendered, name, call_method(php, method, &mut args)?); + } + + Ok(rendered) + } +} + +/// `$rows` in schema order; a rendered column that is not a `RustColumn` is adopted into one. +pub fn batch(rows: &Zval, formats: &Formats, json_column_is_text: bool) -> PhpResult { + let count = call_method(rows, "count", &mut [])? + .long() + .ok_or_else(|| ext_exception("flow_php expected Rows::count() to return an int"))? as usize; + let held = call_method(rows, "columns", &mut [])?; + let held = held + .array() + .ok_or_else(|| ext_exception("flow_php expected Rows::columns() to return an array"))?; + let definitions = definitions(&call_method(rows, "schema", &mut [])?)?; + let definitions = definitions + .array() + .ok_or_else(|| ext_exception("flow_php expected Schema::definitions() to return an array"))?; + let mut columns = Vec::with_capacity(definitions.len()); + + for (name, definition) in definitions.iter() { + let name = name.to_string().into_bytes(); + let column = ht_get(held, &name).ok_or_else(|| { + ext_exception(format!( + "flow_php expected Rows to hold the column \"{}\"", + String::from_utf8_lossy(&name) + )) + })?; + + let native = match column.extract::<&RustColumn>() { + Some(native) => renders(&native.plan().values, formats, json_column_is_text) + .then(|| (Rc::clone(native.plan()), ArrayRef::clone(native.data()))), + None => { + if renders( + &type_plan(&call_method(definition, "type", &mut [])?)?.values, + formats, + json_column_is_text, + ) { + Some(native_column(definition, column)?) + } else { + None + } + } + }; + + columns.push(Column { name, native }); + } + + Ok(Batch { count, columns }) +} + +/// The renderer of a native column of `batch()`, its named zones resolved. +pub fn renderer<'a>(plan: &'a TypePlan, data: &'a ArrayRef, formats: &'a Formats) -> PhpResult> { + let mut renderer = Renderer::new(&plan.values, &plan.kind, data.as_ref(), formats) + .ok_or_else(|| ext_exception(format!("flow_php cannot render a column stored as {:?}", plan.kind)))?; + + renderer.resolve()?; + + Ok(renderer) +} + +/// The cells PHP rendered for the column `name`: `count` values, each a string or null. +pub fn cells<'a>(cells: &'a ZendHashTable, name: &[u8], count: usize, what: &str) -> PhpResult>> { + let refused = |problem: &str| { + invalid_argument(format!( + "flow_php expected {count} {what} of the column \"{}\", {problem}", + String::from_utf8_lossy(name) + )) + }; + let list = ht_get(cells, name) + .ok_or_else(|| refused("which it does not render, got none"))? + .array() + .ok_or_else(|| refused("got no list"))?; + + if list.len() != count { + return Err(refused(&format!("got {}", list.len()))); + } + + list.values() + .map(|cell| match cell.zend_str() { + Some(text) => Ok(Some(text.as_bytes())), + None if cell.is_null() => Ok(None), + None => Err(refused(&format!("got a {}", cell.get_type()))), + }) + .collect() +} diff --git a/src/extension/flow-php-ext/src/text/date.rs b/src/extension/flow-php-ext/src/text/date.rs new file mode 100644 index 0000000000..c574140c00 --- /dev/null +++ b/src/extension/flow-php-ext/src/text/date.rs @@ -0,0 +1,417 @@ +//! `DateTimeInterface::format()` (ext/date `date_format()`) for the letters the native writers render, over an +//! instant in µs and the column zone. Any other letter makes the format unparseable: the column is rendered in PHP. + +use ext_php_rs::exception::PhpException; + +use crate::ctx::{self, call_method, ht_get, zval_long}; +use crate::exception::ext_exception; +use crate::text::int; + +pub const LETTERS: &[u8] = b"YymndjHGisuveTPpOZUc"; + +#[derive(Debug, Clone, PartialEq, Eq)] +enum Part { + Literal(Vec), + Letter(u8), +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Format(Vec); + +impl Format { + /// `None` when an unescaped ASCII letter is outside [`LETTERS`], or the format ends in a lone backslash. + pub fn parse(format: &[u8]) -> Option { + let mut parts = Vec::new(); + let mut literal = Vec::new(); + let mut bytes = format.iter(); + + while let Some(byte) = bytes.next() { + match byte { + b'\\' => literal.push(*bytes.next()?), + byte if byte.is_ascii_alphabetic() => { + if !LETTERS.contains(byte) { + return None; + } + + if !literal.is_empty() { + parts.push(Part::Literal(std::mem::take(&mut literal))); + } + + parts.push(Part::Letter(*byte)); + } + byte => literal.push(*byte), + } + } + + if !literal.is_empty() { + parts.push(Part::Literal(literal)); + } + + Some(Self(parts)) + } +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Transition { + pub from: i64, + pub offset: i32, + pub abbreviation: Vec, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum Zone { + Utc, + Offset { seconds: i32, abbreviation: Vec }, + Named { id: Vec, transitions: Vec }, +} + +impl Zone { + /// `DateTimeType::zoneName()`: `UTC`, `+HH:MM` / `-HH:MM`, or an IANA id. + pub fn of(zone_name: &[u8]) -> Self { + if zone_name == b"UTC" { + return Zone::Utc; + } + + if let [sign @ (b'+' | b'-'), h1, h2, b':', m1, m2] = zone_name { + if [h1, h2, m1, m2].iter().all(|digit| digit.is_ascii_digit()) { + let digit = |byte: &u8| i32::from(byte - b'0'); + let seconds = (digit(h1) * 10 + digit(h2)) * 3_600 + (digit(m1) * 10 + digit(m2)) * 60; + let seconds = if *sign == b'-' { -seconds } else { seconds }; + // timelib's abbreviation of a fixed offset: GMT±hhmm + let mut abbreviation = b"GMT".to_vec(); + + abbreviation.push(*sign); + abbreviation.extend_from_slice(&[*h1, *h2, *m1, *m2]); + + return Zone::Offset { seconds, abbreviation }; + } + } + + Zone::Named { + id: zone_name.to_vec(), + transitions: Vec::new(), + } + } + + /// A named zone reads `DateTimeZone::getTransitions($min, $max + 1)`: the offset in force at `$min`, then every + /// change up to and including `$max`. PHP built against the system tzdata (Debian, Ubuntu) leaves out a transition + /// at exactly the end of the range, PHP's bundled database keeps it. Once per batch per column. + pub fn resolve(&mut self, min_second: i64, max_second: i64) -> Result<(), PhpException> { + let Zone::Named { id, transitions } = self else { + return Ok(()); + }; + + let listed = call_method( + &ctx::timezone(id)?, + "getTransitions", + &mut [zval_long(min_second), zval_long(max_second.saturating_add(1))], + )?; + let listed = listed + .array() + .ok_or_else(|| ext_exception("flow_php expected DateTimeZone::getTransitions() to return an array"))?; + + transitions.clear(); + + for transition in listed.values() { + let field = |name: &[u8]| { + transition + .array() + .and_then(|transition| ht_get(transition, name)) + .ok_or_else(|| ext_exception("flow_php expected a time zone transition of ts, offset and abbr")) + }; + + transitions.push(Transition { + from: field(b"ts")?.long().unwrap_or(min_second), + offset: field(b"offset")?.long().unwrap_or(0) as i32, + abbreviation: field(b"abbr")? + .zend_str() + .map(|abbr| abbr.as_bytes().to_vec()) + .unwrap_or_default(), + }); + } + + if transitions.is_empty() { + return Err(ext_exception( + "flow_php expected DateTimeZone::getTransitions() to list the offset in force", + )); + } + + Ok(()) + } + + /// The offset from UTC in seconds and the zone abbreviation at `second`. + fn at(&self, second: i64) -> (i32, &[u8]) { + match self { + Zone::Utc => (0, b"UTC"), + Zone::Offset { seconds, abbreviation } => (*seconds, abbreviation), + Zone::Named { transitions, .. } => { + let index = transitions + .partition_point(|transition| transition.from <= second) + .saturating_sub(1); + + transitions + .get(index) + .map_or((0, b"UTC"), |transition| (transition.offset, &transition.abbreviation)) + } + } + } +} + +/// C's `%0{width}d`: zeros after the sign, the sign counted in the width. +fn padded(out: &mut Vec, value: i64, width: usize) { + if width == 2 && (0..100).contains(&value) { + out.extend_from_slice(&[b'0' + (value / 10) as u8, b'0' + (value % 10) as u8]); + + return; + } + + if width == 4 && (1_000..10_000).contains(&value) { + out.extend_from_slice(&[ + b'0' + (value / 1_000) as u8, + b'0' + (value / 100 % 10) as u8, + b'0' + (value / 10 % 10) as u8, + b'0' + (value % 10) as u8, + ]); + + return; + } + + let mut buffer = [0u8; 20]; + let mut at = buffer.len(); + let mut rest = value.unsigned_abs(); + + loop { + at -= 1; + buffer[at] = b'0' + (rest % 10) as u8; + rest /= 10; + + if rest == 0 { + break; + } + } + + let sign = usize::from(value < 0); + + if value < 0 { + out.push(b'-'); + } + + out.resize(out.len() + width.saturating_sub(buffer.len() - at + sign), b'0'); + out.extend_from_slice(&buffer[at..]); +} + +/// `±hh[:]mm` of an offset, as `P` / `O` / `c` / `e` write it. +fn offset_text(out: &mut Vec, offset: i32, colon: bool) { + out.push(if offset < 0 { b'-' } else { b'+' }); + padded(out, i64::from((offset / 3_600).abs()), 2); + + if colon { + out.push(b':'); + } + + padded(out, i64::from((offset % 3_600 / 60).abs()), 2); +} + +/// The proleptic Gregorian date of a day count from 1970-01-01 (days-from-civil inverted), any year. +fn civil(days: i64) -> (i64, i64, i64) { + let shifted = days + 719_468; + let era = shifted.div_euclid(146_097); + let day_of_era = shifted.rem_euclid(146_097); + let year_of_era = (day_of_era - day_of_era / 1_460 + day_of_era / 36_524 - day_of_era / 146_096) / 365; + let day_of_year = day_of_era - (365 * year_of_era + year_of_era / 4 - year_of_era / 100); + let month_index = (5 * day_of_year + 2) / 153; + let day = day_of_year - (153 * month_index + 2) / 5 + 1; + let month = if month_index < 10 { + month_index + 3 + } else { + month_index - 9 + }; + + (year_of_era + era * 400 + i64::from(month <= 2), month, day) +} + +pub fn write(out: &mut Vec, format: &Format, micros: i64, zone: &Zone) { + let second = micros.div_euclid(1_000_000); + let micro = micros.rem_euclid(1_000_000); + let (offset, abbreviation) = zone.at(second); + let local = second + i64::from(offset); + let (year, month, day) = civil(local.div_euclid(86_400)); + let of_day = local.rem_euclid(86_400); + let (hour, minute, second_of_minute) = (of_day / 3_600, of_day / 60 % 60, of_day % 60); + + for part in &format.0 { + let letter = match part { + Part::Literal(bytes) => { + out.extend_from_slice(bytes); + + continue; + } + Part::Letter(letter) => *letter, + }; + + match letter { + b'Y' => { + if year < 0 { + out.push(b'-'); + } + + padded(out, year.abs(), 4); + } + b'y' => padded(out, year % 100, 2), + b'm' => padded(out, month, 2), + b'n' => int(out, month), + b'd' => padded(out, day, 2), + b'j' => int(out, day), + b'H' => padded(out, hour, 2), + b'G' => int(out, hour), + b'i' => padded(out, minute, 2), + b's' => padded(out, second_of_minute, 2), + b'u' => padded(out, micro, 6), + b'v' => padded(out, micro / 1_000, 3), + b'e' => match zone { + Zone::Utc => out.extend_from_slice(b"UTC"), + Zone::Offset { seconds, .. } => offset_text(out, *seconds, true), + Zone::Named { id, .. } => out.extend_from_slice(id), + }, + b'T' => out.extend_from_slice(abbreviation), + b'p' if matches!(abbreviation, b"UTC" | b"Z" | b"GMT+0000") => out.push(b'Z'), + b'P' | b'p' => offset_text(out, offset, true), + b'O' => offset_text(out, offset, false), + b'Z' => int(out, i64::from(offset)), + b'U' => int(out, second), + b'c' => { + padded(out, year, 4); + out.push(b'-'); + padded(out, month, 2); + out.push(b'-'); + padded(out, day, 2); + out.push(b'T'); + padded(out, hour, 2); + out.push(b':'); + padded(out, minute, 2); + out.push(b':'); + padded(out, second_of_minute, 2); + offset_text(out, offset, true); + } + _ => unreachable!("Format::parse() keeps the rendered letters only"), + } + } +} + +#[cfg(test)] +mod tests { + use super::{write, Format, Transition, Zone}; + + fn formatted(format: &str, micros: i64, zone: &Zone) -> String { + let mut out = Vec::new(); + + write(&mut out, &Format::parse(format.as_bytes()).unwrap(), micros, zone); + + String::from_utf8(out).unwrap() + } + + #[test] + fn a_letter_outside_the_set_or_a_lone_backslash_is_not_parsed() { + assert!(Format::parse(b"Y-m-d\\TH:i:s.uP").is_some()); + assert!(Format::parse(b"").is_some()); + assert!(Format::parse(b"D, d M Y").is_none()); + assert!(Format::parse(b"Y I").is_none()); + assert!(Format::parse(b"Y\\").is_none()); + assert!(Format::parse("Y ż \\D".as_bytes()).is_some()); + } + + /// `edge-facts.log`: format `Y|y|c|U|u|v|e|T|P|p|O|Z|m|n|d|j|H|G|i|s` in UTC at extreme years. + #[test] + fn every_letter_at_extreme_years_in_utc() { + let format = "Y|y|c|U|u|v|e|T|P|p|O|Z|m|n|d|j|H|G|i|s"; + + for (second, expected) in [ + (-62_198_755_200_i64, "-0001|-1|-001-01-01T00:00:00+00:00|-62198755200|000000|000|UTC|UTC|+00:00|Z|+0000|0|01|1|01|1|00|0|00|00"), + (-62_167_219_200, "0000|00|0000-01-01T00:00:00+00:00|-62167219200|000000|000|UTC|UTC|+00:00|Z|+0000|0|01|1|01|1|00|0|00|00"), + (-30_610_224_000, "1000|00|1000-01-01T00:00:00+00:00|-30610224000|000000|000|UTC|UTC|+00:00|Z|+0000|0|01|1|01|1|00|0|00|00"), + (-30_641_760_000, "0999|99|0999-01-01T00:00:00+00:00|-30641760000|000000|000|UTC|UTC|+00:00|Z|+0000|0|01|1|01|1|00|0|00|00"), + (253_402_300_800, "10000|00|10000-01-01T00:00:00+00:00|253402300800|000000|000|UTC|UTC|+00:00|Z|+0000|0|01|1|01|1|00|0|00|00"), + (3_093_527_980_800, "100000|00|100000-01-01T00:00:00+00:00|3093527980800|000000|000|UTC|UTC|+00:00|Z|+0000|0|01|1|01|1|00|0|00|00"), + ] { + assert_eq!(formatted(format, second * 1_000_000, &Zone::Utc), expected); + } + } + + #[test] + fn micros_before_the_epoch_floor_to_the_second_below() { + assert_eq!( + formatted("Y-m-d H:i:s.u v U", -500_000, &Zone::Utc), + "1969-12-31 23:59:59.500000 500 -1" + ); + assert_eq!( + formatted("Y-m-d\\TH:i:s.u G\\h", 1_767_323_045_123_456, &Zone::Utc), + "2026-01-02T03:04:05.123456 3h" + ); + } + + /// `edge-facts.log`: Europe/Warsaw across the 2026 gap and overlap, one transition list for the whole call. + #[test] + fn a_named_zone_crosses_its_transitions() { + let warsaw = Zone::Named { + id: b"Europe/Warsaw".to_vec(), + transitions: vec![ + Transition { + from: 1_774_745_999, + offset: 3_600, + abbreviation: b"CET".to_vec(), + }, + Transition { + from: 1_774_746_000, + offset: 7_200, + abbreviation: b"CEST".to_vec(), + }, + Transition { + from: 1_792_890_000, + offset: 3_600, + abbreviation: b"CET".to_vec(), + }, + ], + }; + + for (second, expected) in [ + ( + 1_774_745_999_i64, + "2026-03-29T01:59:59+01:00 CET Europe/Warsaw +01:00 3600", + ), + ( + 1_774_746_000, + "2026-03-29T03:00:00+02:00 CEST Europe/Warsaw +02:00 7200", + ), + ( + 1_792_889_999, + "2026-10-25T02:59:59+02:00 CEST Europe/Warsaw +02:00 7200", + ), + (1_792_890_000, "2026-10-25T02:00:00+01:00 CET Europe/Warsaw +01:00 3600"), + ] { + assert_eq!( + formatted("Y-m-d\\TH:i:sP T e p Z", second * 1_000_000, &warsaw), + expected + ); + } + } + + /// `zone-transitions.log`: `+02:30` formats `e|T|P|O|Z|c` as `+02:30|GMT+0230|+02:30|+0230|9000|1970-01-01T02:30:00+02:30`. + #[test] + fn a_fixed_offset_zone() { + assert_eq!( + formatted("e|T|P|p|O|Z|c", 0, &Zone::of(b"+02:30")), + "+02:30|GMT+0230|+02:30|+02:30|+0230|9000|1970-01-01T02:30:00+02:30" + ); + assert_eq!( + formatted("e|T|P|O|Z H:i", 0, &Zone::of(b"-05:00")), + "-05:00|GMT-0500|-05:00|-0500|-18000 19:00" + ); + } + + #[test] + fn zone_names() { + assert_eq!(Zone::of(b"UTC"), Zone::Utc); + assert!(matches!(Zone::of(b"Europe/Warsaw"), Zone::Named { .. })); + assert!(matches!(Zone::of(b"+02:30"), Zone::Offset { seconds: 9_000, .. })); + } +} diff --git a/src/extension/flow-php-ext/src/text/float.rs b/src/extension/flow-php-ext/src/text/float.rs new file mode 100644 index 0000000000..4ce345eacc --- /dev/null +++ b/src/extension/flow-php-ext/src/text/float.rs @@ -0,0 +1,47 @@ +//! Float text: the shortest digits that read back as the same float (`zend_gcvt()` in its `serialize_precision = -1` +//! mode, what `json_encode()` writes), whatever the `precision` and `serialize_precision` inis hold. + +use std::ffi::{c_char, c_int, CStr}; + +use crate::text::Error; + +extern "C" { + fn zend_gcvt(value: f64, ndigit: c_int, dec_point: c_char, exponent: c_char, buf: *mut c_char) -> *mut c_char; +} + +fn digits(out: &mut Vec, value: f64, zero_fraction: bool) { + let mut buffer = [0 as c_char; 64]; + let text = unsafe { + zend_gcvt(value, -1, b'.' as c_char, b'e' as c_char, buffer.as_mut_ptr()); + CStr::from_ptr(buffer.as_ptr()) + } + .to_bytes(); + + out.extend_from_slice(text); + + if zero_fraction && !text.contains(&b'.') { + out.extend_from_slice(b".0"); + } +} + +/// CSV, XML and Text float text: always a fraction or an exponent (`1.0`, `1.0e+25`); `NAN`, `INF`, `-INF` by name. +pub fn text(out: &mut Vec, value: f64) { + if value.is_nan() { + out.extend_from_slice(b"NAN"); + } else if value.is_infinite() { + out.extend_from_slice(if value > 0.0 { b"INF" } else { b"-INF" }); + } else { + digits(out, value, true); + } +} + +/// JSON float text: `.0` only under `JSON_PRESERVE_ZERO_FRACTION`; NAN and the infinities are refused. +pub fn json(out: &mut Vec, value: f64, zero_fraction: bool) -> Result<(), Error> { + if !value.is_finite() { + return Err(Error::NonFinite); + } + + digits(out, value, zero_fraction); + + Ok(()) +} diff --git a/src/extension/flow-php-ext/src/text/json.rs b/src/extension/flow-php-ext/src/text/json.rs new file mode 100644 index 0000000000..386e3806b1 --- /dev/null +++ b/src/extension/flow-php-ext/src/text/json.rs @@ -0,0 +1,165 @@ +//! `php_json_escape_string()` (ext/json/json_encoder.c) for the flags the native JSON writer accepts. + +use crate::text::Error; + +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] +pub struct Flags { + pub unescaped_slashes: bool, + pub unescaped_unicode: bool, + pub preserve_zero_fraction: bool, +} + +const HEX: &[u8; 16] = b"0123456789abcdef"; + +fn unit(out: &mut Vec, unit: u32) { + out.extend_from_slice(&[ + b'\\', + b'u', + HEX[(unit >> 12 & 0xf) as usize], + HEX[(unit >> 8 & 0xf) as usize], + HEX[(unit >> 4 & 0xf) as usize], + HEX[(unit & 0xf) as usize], + ]); +} + +/// `text` as a JSON string; nothing is appended when it is not UTF-8. +pub fn string(out: &mut Vec, text: &[u8], flags: &Flags) -> Result<(), Error> { + if text.is_ascii() { + out.push(b'"'); + + let mut rest = text; + + // runs of bytes written as they are, then the one that is escaped + while let Some(at) = rest + .iter() + .position(|byte| matches!(byte, 0..=0x1f | b'"' | b'\\' | b'/')) + { + out.extend_from_slice(&rest[..at]); + ascii(out, rest[at], flags); + rest = &rest[at + 1..]; + } + + out.extend_from_slice(rest); + out.push(b'"'); + + return Ok(()); + } + + let text = std::str::from_utf8(text).map_err(|_| Error::Utf8)?; + + out.push(b'"'); + + for character in text.chars() { + let code = character as u32; + + if code < 0x80 { + ascii(out, code as u8, flags); + } else if flags.unescaped_unicode && !(0x2028..=0x2029).contains(&code) { + out.extend_from_slice(character.encode_utf8(&mut [0; 4]).as_bytes()); + } else if code >= 0x1_0000 { + let code = code - 0x1_0000; + + unit(out, code >> 10 | 0xd800); + unit(out, code & 0x3ff | 0xdc00); + } else { + unit(out, code); + } + } + + out.push(b'"'); + + Ok(()) +} + +fn ascii(out: &mut Vec, byte: u8, flags: &Flags) { + match byte { + b'"' => out.extend_from_slice(b"\\\""), + b'\\' => out.extend_from_slice(b"\\\\"), + b'/' if !flags.unescaped_slashes => out.extend_from_slice(b"\\/"), + 0x08 => out.extend_from_slice(b"\\b"), + 0x0c => out.extend_from_slice(b"\\f"), + b'\n' => out.extend_from_slice(b"\\n"), + b'\r' => out.extend_from_slice(b"\\r"), + b'\t' => out.extend_from_slice(b"\\t"), + 0..=0x1f => unit(out, u32::from(byte)), + _ => out.push(byte), + } +} + +#[cfg(test)] +mod tests { + use super::{string, Flags}; + use crate::text::Error; + + fn escaped(text: &[u8], flags: &Flags) -> Result { + let mut out = Vec::new(); + + string(&mut out, text, flags).map(|()| String::from_utf8(out).unwrap()) + } + + #[test] + fn default_flags_escape_slashes_controls_and_non_ascii() { + let flags = Flags::default(); + + assert_eq!(escaped(b"", &flags).unwrap(), r#""""#); + assert_eq!(escaped(b"a\"b\\c/d", &flags).unwrap(), r#""a\"b\\c\/d""#); + assert_eq!( + escaped(b"\x08\x0c\n\r\t\x00\x1f\x7f", &flags).unwrap(), + "\"\\b\\f\\n\\r\\t\\u0000\\u001f\x7f\"" + ); + assert_eq!(escaped("<>&'".as_bytes(), &flags).unwrap(), r#""<>&'""#); + assert_eq!( + escaped("zażółć".as_bytes(), &flags).unwrap(), + r#""za\u017c\u00f3\u0142\u0107""# + ); + } + + #[test] + fn a_character_beyond_the_basic_plane_is_a_surrogate_pair() { + assert_eq!( + escaped("😀".as_bytes(), &Flags::default()).unwrap(), + r#""\ud83d\ude00""# + ); + assert_eq!( + escaped("\u{10000}\u{10ffff}".as_bytes(), &Flags::default()).unwrap(), + r#""\ud800\udc00\udbff\udfff""# + ); + } + + #[test] + fn unescaped_slashes_and_unicode() { + let slashes = Flags { + unescaped_slashes: true, + ..Flags::default() + }; + let unicode = Flags { + unescaped_unicode: true, + ..Flags::default() + }; + + assert_eq!(escaped("a/ż".as_bytes(), &slashes).unwrap(), r#""a/\u017c""#); + assert_eq!(escaped("a/ż😀".as_bytes(), &unicode).unwrap(), "\"a\\/ż😀\""); + // the line terminators stay escaped without JSON_UNESCAPED_LINE_TERMINATORS + assert_eq!( + escaped("\u{2027}\u{2028}\u{2029}\u{202a}".as_bytes(), &unicode).unwrap(), + "\"\u{2027}\\u2028\\u2029\u{202a}\"" + ); + } + + #[test] + fn invalid_utf8_is_refused_and_appends_nothing() { + for text in [ + &b"\xff"[..], + b"a\xc3\x28", + b"\xed\xa0\x80", + b"\xc0\xaf", + b"\xf4\x90\x80\x80", + b"abc\xe2\x82", + ] { + let mut out = b"kept".to_vec(); + + assert_eq!(string(&mut out, text, &Flags::default()), Err(Error::Utf8)); + assert_eq!(out, b"kept"); + } + } +} diff --git a/src/extension/flow-php-ext/src/text/mod.rs b/src/extension/flow-php-ext/src/text/mod.rs new file mode 100644 index 0000000000..922f2b47cd --- /dev/null +++ b/src/extension/flow-php-ext/src/text/mod.rs @@ -0,0 +1,54 @@ +//! The text of column values as the PHP writers render it (`Flow\ETL\Column\TextValues` plus each encoder's syntax), +//! read straight from the arrow arrays. + +pub mod batch; +pub mod date; +pub mod float; +pub mod json; +pub mod value; + +/// What `json_encode()` refuses; `message()` and `code()` are `json_last_error_msg()` and `json_last_error()`. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Error { + NonFinite, + Utf8, +} + +impl Error { + pub fn message(self) -> &'static str { + match self { + Error::NonFinite => "Inf and NaN cannot be JSON encoded", + Error::Utf8 => "Malformed UTF-8 characters, possibly incorrectly encoded", + } + } + + pub fn code(self) -> i32 { + match self { + Error::NonFinite => 7, + Error::Utf8 => 5, + } + } +} + +/// A decimal integer, as `(string) $int` writes it. +pub fn int(out: &mut Vec, value: i64) { + let mut buffer = [0u8; 20]; + let mut at = buffer.len(); + let mut rest = value.unsigned_abs(); + + loop { + at -= 1; + buffer[at] = b'0' + (rest % 10) as u8; + rest /= 10; + + if rest == 0 { + break; + } + } + + if value < 0 { + out.push(b'-'); + } + + out.extend_from_slice(&buffer[at..]); +} diff --git a/src/extension/flow-php-ext/src/text/value.rs b/src/extension/flow-php-ext/src/text/value.rs new file mode 100644 index 0000000000..45dfa83b63 --- /dev/null +++ b/src/extension/flow-php-ext/src/text/value.rs @@ -0,0 +1,517 @@ +//! One renderer per column and batch: the plain value of `TextValues::of()` as scalar text or as JSON, read from the +//! arrow array of the column's kind. + +use std::collections::hash_map::Entry; +use std::collections::HashMap; + +use arrow_array::cast::AsArray; +use arrow_array::types::{Date32Type, DurationMicrosecondType, Float64Type, Int64Type, TimestampMicrosecondType}; +use arrow_array::{ + Array, BinaryArray, BooleanArray, Date32Array, DurationMicrosecondArray, FixedSizeBinaryArray, Float64Array, + Int64Array, ListArray, MapArray, TimestampMicrosecondArray, +}; +use arrow_buffer::NullBuffer; +use ext_php_rs::exception::PhpException; +use flow_batch_frame::kind::Kind; + +use crate::ctx::array_key_index; +use crate::plan::ValueNode; +use crate::text::date::{self, Format, Zone}; +use crate::text::json::{self, Flags}; +use crate::text::{float, int, Error}; +use crate::values::push_uuid_text; + +/// The writer's datetime and date formats; `None` holds a letter the native lane does not render. +pub struct Formats { + pub date_time: Option, + pub date: Option, +} + +impl Formats { + pub fn parse(date_time: &[u8], date: &[u8]) -> Self { + Self { + date_time: Format::parse(date_time), + date: Format::parse(date), + } + } +} + +/// The flags of a nested CSV cell: `json_encode($plain, JSON_PRESERVE_ZERO_FRACTION | JSON_THROW_ON_ERROR)`. +const CELL: Flags = Flags { + unescaped_slashes: false, + unescaped_unicode: false, + preserve_zero_fraction: true, +}; + +enum MapKeys<'a> { + Int(&'a Int64Array), + Bytes(&'a BinaryArray), +} + +enum Values<'a> { + Null, + Int(&'a Int64Array), + Duration(&'a DurationMicrosecondArray), + Float(&'a Float64Array), + Bool(&'a BooleanArray), + Bytes(&'a BinaryArray), + Uuid(&'a FixedSizeBinaryArray), + Instant { + array: &'a TimestampMicrosecondArray, + format: &'a Format, + zone: Zone, + }, + Days { + array: &'a Date32Array, + format: &'a Format, + }, + List { + list: &'a ListArray, + element: Box>, + }, + Map { + map: &'a MapArray, + keys: MapKeys<'a>, + values: Box>, + }, + /// (name, optional, renderer): an optional element that is null is absent. + Struct(Vec<(&'a [u8], bool, Renderer<'a>)>), +} + +pub struct Renderer<'a> { + nulls: Option<&'a NullBuffer>, + values: Values<'a>, +} + +/// Whether the native lane renders a column of `node`. Not rendered: markup; json unless it is the column itself and +/// the format writes its stored text; a datetime or date whose format did not parse; a container holding any of those. +pub fn renders(node: &ValueNode, formats: &Formats, json_column_is_text: bool) -> bool { + match node { + ValueNode::Json => json_column_is_text, + _ => renders_nested(node, formats), + } +} + +fn renders_nested(node: &ValueNode, formats: &Formats) -> bool { + match node { + ValueNode::Plain + | ValueNode::Null + | ValueNode::Time + | ValueNode::Uuid + | ValueNode::Enum(_) + | ValueNode::TimeZone => true, + ValueNode::DateTime(_) => formats.date_time.is_some(), + ValueNode::Date => formats.date.is_some(), + ValueNode::Json | ValueNode::Markup(_) => false, + ValueNode::List(element) | ValueNode::Map(element) => renders_nested(element, formats), + ValueNode::Struct(elements) => elements.iter().all(|element| renders_nested(element, formats)), + } +} + +impl<'a> Renderer<'a> { + /// `None` for a column `renders()` refuses. A named zone still has to be resolved: [`Renderer::resolve`]. + pub fn new(node: &ValueNode, kind: &'a Kind, array: &'a dyn Array, formats: &'a Formats) -> Option { + let values = match (node, kind) { + (_, Kind::Null) | (ValueNode::Null, _) => Values::Null, + (ValueNode::DateTime(zone), Kind::Timestamp) => Values::Instant { + array: array.as_primitive::(), + format: formats.date_time.as_ref()?, + zone: Zone::of(zone), + }, + (ValueNode::Date, Kind::Int32) => Values::Days { + array: array.as_primitive::(), + format: formats.date.as_ref()?, + }, + (ValueNode::Time, Kind::Duration) => Values::Duration(array.as_primitive::()), + (ValueNode::Uuid, Kind::Uuid) => Values::Uuid(array.as_fixed_size_binary()), + (ValueNode::Plain, Kind::Int64) => Values::Int(array.as_primitive::()), + (ValueNode::Plain, Kind::Float64) => Values::Float(array.as_primitive::()), + (ValueNode::Plain, Kind::Boolean) => Values::Bool(array.as_boolean()), + (ValueNode::Plain | ValueNode::Enum(_) | ValueNode::TimeZone | ValueNode::Json, Kind::Bytes) => { + Values::Bytes(array.as_binary::()) + } + (ValueNode::List(element_node), Kind::List(element)) => { + let list = array.as_list::(); + + Values::List { + list, + element: Box::new(Renderer::new( + element_node, + &element.kind, + list.values().as_ref(), + formats, + )?), + } + } + (ValueNode::Map(value_node), Kind::Map(key, value)) => { + let map = array.as_map(); + + Values::Map { + map, + keys: match key.kind { + Kind::Bytes => MapKeys::Bytes(map.keys().as_binary::()), + Kind::Int64 => MapKeys::Int(map.keys().as_primitive::()), + _ => return None, + }, + values: Box::new(Renderer::new(value_node, &value.kind, map.values().as_ref(), formats)?), + } + } + (ValueNode::Struct(nodes), Kind::Struct(fields)) => Values::Struct( + nodes + .iter() + .zip(fields) + .zip(array.as_struct().columns()) + .map(|((node, field), child)| { + Some(( + field.name.as_bytes(), + field.optional, + Renderer::new(node, &field.kind, child.as_ref(), formats)?, + )) + }) + .collect::>()?, + ), + _ => return None, + }; + + Some(Self { + nulls: array.nulls(), + values, + }) + } + + /// Every named zone reads its transitions over the instants it renders, once per batch. + pub fn resolve(&mut self) -> Result<(), PhpException> { + match &mut self.values { + Values::Instant { array, zone, .. } => { + if let (Some(earliest), Some(latest)) = (array.iter().flatten().min(), array.iter().flatten().max()) { + zone.resolve(earliest.div_euclid(1_000_000), latest.div_euclid(1_000_000))?; + } + + Ok(()) + } + Values::List { element, .. } => element.resolve(), + Values::Map { values, .. } => values.resolve(), + Values::Struct(fields) => fields.iter_mut().try_for_each(|(_, _, field)| field.resolve()), + _ => Ok(()), + } + } + + pub fn is_null(&self, i: usize) -> bool { + matches!(self.values, Values::Null) || self.nulls.is_some_and(|nulls| nulls.is_null(i)) + } + + /// A field whose text cannot hold a CSV separator, enclosure or escape character that is no digit, letter or + /// one of `_+./:-`: a number, a boolean, a uuid. + pub fn is_plain(&self) -> bool { + matches!( + self.values, + Values::Null | Values::Int(_) | Values::Duration(_) | Values::Float(_) | Values::Bool(_) | Values::Uuid(_) + ) + } + + /// The stored bytes of a non-null string value, when the column is one. + pub fn bytes(&self, i: usize) -> Option<&'a [u8]> { + match &self.values { + Values::Bytes(array) => Some(array.value(i)), + _ => None, + } + } + + /// The scalar text of a non-null value: a CSV field before quoting. A nested value is its JSON text. + pub fn text(&self, out: &mut Vec, i: usize) -> Result<(), Error> { + match &self.values { + Values::Null => {} + Values::Int(array) => int(out, array.value(i)), + Values::Duration(array) => int(out, array.value(i)), + Values::Float(array) => float::text(out, array.value(i)), + Values::Bool(array) => out.extend_from_slice(if array.value(i) { b"true" } else { b"false" }), + Values::Bytes(array) => out.extend_from_slice(array.value(i)), + Values::Uuid(array) => push_uuid_text(out, array.value(i)), + Values::Instant { array, format, zone } => date::write(out, format, array.value(i), zone), + Values::Days { array, format } => { + date::write(out, format, i64::from(array.value(i)) * 86_400_000_000, &Zone::Utc) + } + Values::List { .. } | Values::Map { .. } | Values::Struct(_) => { + let mut non_finite = false; + + self.json(out, i, &CELL, &mut non_finite)?; + + if non_finite { + return Err(Error::NonFinite); + } + } + } + + Ok(()) + } + + /// The JSON text of row `i`, null included. As `json_encode()`, a NAN or an infinity writes `0`, raises + /// `non_finite` and goes on; text that is not UTF-8 stops with `Error::Utf8`. + pub fn json(&self, out: &mut Vec, i: usize, flags: &Flags, non_finite: &mut bool) -> Result<(), Error> { + if self.is_null(i) { + out.extend_from_slice(b"null"); + + return Ok(()); + } + + match &self.values { + Values::Null => unreachable!("is_null() above"), + Values::Int(array) => int(out, array.value(i)), + Values::Duration(array) => int(out, array.value(i)), + Values::Float(array) => { + if float::json(out, array.value(i), flags.preserve_zero_fraction).is_err() { + *non_finite = true; + out.push(b'0'); + } + } + Values::Bool(array) => out.extend_from_slice(if array.value(i) { b"true" } else { b"false" }), + Values::Bytes(array) => json::string(out, array.value(i), flags)?, + Values::Uuid(array) => { + out.push(b'"'); + push_uuid_text(out, array.value(i)); + out.push(b'"'); + } + Values::Instant { .. } | Values::Days { .. } => { + let start = out.len(); + + out.push(b'"'); + self.text(out, i)?; + + // a format of digits, dashes and colons needs no escaping; anything else is escaped as a string + if out[start + 1..] + .iter() + .all(|byte| matches!(byte, b' ' | b'+'..=b'.' | b'0'..=b':' | b'A'..=b'Z' | b'_' | b'a'..=b'z')) + { + out.push(b'"'); + } else { + let text = out.split_off(start + 1); + + out.truncate(start); + json::string(out, &text, flags)?; + } + } + Values::List { list, element } => { + let offsets = list.value_offsets(); + + out.push(b'['); + + for j in offsets[i] as usize..offsets[i + 1] as usize { + if j > offsets[i] as usize { + out.push(b','); + } + + element.json(out, j, flags, non_finite)?; + } + + out.push(b']'); + } + Values::Map { map, keys, values } => { + let offsets = map.value_offsets(); + out.push(b'{'); + + for (written, (j, last)) in entries(keys, offsets[i] as usize, offsets[i + 1] as usize) + .into_iter() + .enumerate() + { + if written > 0 { + out.push(b','); + } + + match keys { + MapKeys::Bytes(keys) => json::string(out, keys.value(j), flags)?, + MapKeys::Int(keys) => { + out.push(b'"'); + int(out, keys.value(j)); + out.push(b'"'); + } + } + + out.push(b':'); + values.json(out, last, flags, non_finite)?; + } + + out.push(b'}'); + } + Values::Struct(fields) => { + let mut written = 0; + + out.push(b'{'); + + for (name, optional, field) in fields { + if *optional && field.is_null(i) { + continue; + } + + if written > 0 { + out.push(b','); + } + + json::string(out, name, flags)?; + out.push(b':'); + field.json(out, i, flags, non_finite)?; + written += 1; + } + + out.push(b'}'); + } + } + + Ok(()) + } +} + +/// A map key as the PHP array key it becomes: `"1"` is the key `1`. +#[derive(PartialEq, Eq, Hash)] +enum ArrayKey<'a> { + Int(i64), + Bytes(&'a [u8]), +} + +fn array_key<'a>(keys: &MapKeys<'a>, j: usize) -> ArrayKey<'a> { + match keys { + MapKeys::Int(keys) => ArrayKey::Int(keys.value(j)), + MapKeys::Bytes(keys) => { + let key: &'a [u8] = keys.value(j); + + array_key_index(key).map_or(ArrayKey::Bytes(key), ArrayKey::Int) + } + } +} + +/// The entries `start..end` of one map as a PHP array holds them, (key index, value index) each: a repeated key keeps +/// its first position and its last value. +fn entries(keys: &MapKeys<'_>, start: usize, end: usize) -> Vec<(usize, usize)> { + let mut entries: Vec<(usize, usize)> = Vec::with_capacity(end - start); + let mut positions: HashMap, usize> = HashMap::with_capacity(end - start); + + for j in start..end { + match positions.entry(array_key(keys, j)) { + Entry::Occupied(position) => entries[*position.get()].1 = j, + Entry::Vacant(position) => { + position.insert(entries.len()); + entries.push((j, j)); + } + } + } + + entries +} + +#[cfg(test)] +mod tests { + use std::sync::Arc; + + use arrow_array::builder::{BinaryBuilder, Int64Builder, MapBuilder}; + use arrow_array::types::Int64Type; + use arrow_array::{Array, ArrayRef, Int64Array, ListArray, StructArray}; + use arrow_schema::{DataType, Field as ArrowField}; + use flow_batch_frame::kind::{Field, Kind}; + + use super::{Formats, Renderer}; + use crate::plan::ValueNode; + use crate::text::json::Flags; + + fn json(node: &ValueNode, kind: &Kind, array: &dyn Array, row: usize) -> String { + let formats = Formats::parse(b"Y-m-d\\TH:i:sP", b"Y-m-d"); + let mut out = Vec::new(); + let mut non_finite = false; + + Renderer::new(node, kind, array, &formats) + .unwrap() + .json(&mut out, row, &Flags::default(), &mut non_finite) + .unwrap(); + + String::from_utf8(out).unwrap() + } + + fn field(name: &str, kind: Kind, optional: bool) -> Box { + Box::new(Field { + name: name.to_owned(), + kind, + optional, + }) + } + + fn string_map(rows: &[&[(&[u8], i64)]]) -> ArrayRef { + let mut builder = MapBuilder::new(None, BinaryBuilder::new(), Int64Builder::new()); + + for row in rows { + for (key, value) in *row { + builder.keys().append_value(key); + builder.values().append_value(*value); + } + + builder.append(true).unwrap(); + } + + Arc::new(builder.finish()) + } + + #[test] + fn a_map_is_always_an_object() { + let node = ValueNode::Map(Box::new(ValueNode::Plain)); + let kind = Kind::Map(field("key", Kind::Bytes, false), field("value", Kind::Int64, false)); + let map = string_map(&[&[(b"0", 1), (b"1", 2)], &[], &[(b"a", 5)]]); + + assert_eq!(json(&node, &kind, map.as_ref(), 0), r#"{"0":1,"1":2}"#); + assert_eq!(json(&node, &kind, map.as_ref(), 1), "{}"); + assert_eq!(json(&node, &kind, map.as_ref(), 2), r#"{"a":5}"#); + } + + #[test] + fn a_repeated_map_key_keeps_its_first_position_and_its_last_value() { + let node = ValueNode::Map(Box::new(ValueNode::Plain)); + let kind = Kind::Map(field("key", Kind::Bytes, false), field("value", Kind::Int64, false)); + let map = string_map(&[&[(b"a", 1), (b"b", 2), (b"a", 3), (b"c", 4), (b"b", 5)]]); + + assert_eq!(json(&node, &kind, map.as_ref(), 0), r#"{"a":3,"b":5,"c":4}"#); + } + + #[test] + fn a_map_keyed_by_integers_is_an_object() { + let mut builder = MapBuilder::new(None, Int64Builder::new(), Int64Builder::new()); + + for (key, value) in [(0, 10), (1, 11)] { + builder.keys().append_value(key); + builder.values().append_value(value); + } + + builder.append(true).unwrap(); + + let node = ValueNode::Map(Box::new(ValueNode::Plain)); + let kind = Kind::Map(field("key", Kind::Int64, false), field("value", Kind::Int64, false)); + + assert_eq!(json(&node, &kind, &builder.finish(), 0), r#"{"0":10,"1":11}"#); + } + + #[test] + fn a_list_is_always_a_list_and_a_null_is_null() { + let list = + ListArray::from_iter_primitive::([Some(vec![Some(1), None, Some(3)]), Some(vec![]), None]); + let node = ValueNode::List(Box::new(ValueNode::Plain)); + let kind = Kind::List(field("element", Kind::Int64, true)); + + assert_eq!(json(&node, &kind, &list, 0), "[1,null,3]"); + assert_eq!(json(&node, &kind, &list, 1), "[]"); + assert_eq!(json(&node, &kind, &list, 2), "null"); + } + + #[test] + fn a_structure_is_an_object_and_a_null_optional_element_is_absent() { + let structure = StructArray::from(vec![ + ( + Arc::new(ArrowField::new("a", DataType::Int64, true)), + Arc::new(Int64Array::from(vec![Some(1), None])) as ArrayRef, + ), + ( + Arc::new(ArrowField::new("b", DataType::Int64, true)), + Arc::new(Int64Array::from(vec![Some(2), None])) as ArrayRef, + ), + ]); + let node = ValueNode::Struct(vec![ValueNode::Plain, ValueNode::Plain]); + let kind = Kind::Struct(vec![*field("a", Kind::Int64, false), *field("b", Kind::Int64, true)]); + + assert_eq!(json(&node, &kind, &structure, 0), r#"{"a":1,"b":2}"#); + assert_eq!(json(&node, &kind, &structure, 1), r#"{"a":null}"#); + } +} diff --git a/src/extension/flow-php-ext/src/values.rs b/src/extension/flow-php-ext/src/values.rs index b7ee2cdcc5..acffffc235 100644 --- a/src/extension/flow-php-ext/src/values.rs +++ b/src/extension/flow-php-ext/src/values.rs @@ -1,22 +1,17 @@ -//! Wire-value decoding mirroring `Flow\Floe\ValueDecoder`. +//! PHP value construction through the engine's C date API. -use std::ffi::{c_char, c_int, c_void}; +use std::ffi::{c_char, c_int, c_uint, c_void}; use ext_php_rs::exception::PhpException; -use ext_php_rs::types::{ZendHashTable, ZendObject, ZendStr, Zval}; +use ext_php_rs::types::{ZendObject, ZendStr, Zval}; use ext_php_rs::zend::ExecutorGlobals; -use crate::ctx::{ - call_handle, ensure_no_pending_exception, ht_insert, ht_insert_index, write_property_raw, - write_slot, zval_long, zval_str, Ctx, -}; +use crate::ctx::{self, call_handle, ensure_no_pending_exception, write_property_raw, write_slot, zval_long, zval_str}; +use crate::date_check::days_from_civil; use crate::exception::ext_exception; -use crate::format::{Reader, VALUE_ABSENT, VALUE_NULL, VALUE_PRESENT}; -use crate::plan::{Decoder, MapKey}; extern "C" { - fn php_date_instantiate(pce: *mut ext_php_rs::zend::ClassEntry, object: *mut Zval) - -> *mut Zval; + fn php_date_instantiate(pce: *mut ext_php_rs::zend::ClassEntry, object: *mut Zval) -> *mut Zval; #[cfg(php84)] fn php_date_initialize_from_ts_long(dateobj: *mut c_void, sec: i64, usec: c_int); @@ -41,249 +36,109 @@ fn php_date_obj(object: &mut ZendObject) -> *mut c_void { .cast::() } -/// `new DateTimeImmutable($str, $timezone)` through the same C-level timelib parser, in its -/// non-throwing `date_create()` flavor: `Ok(None)` on parse failure, no exception. -pub(crate) fn date_from_free_form( - bytes: &[u8], - timezone: Option<&mut Zval>, - ctx: &mut Ctx, -) -> Result, PhpException> { - let ce = ctx.datetime_fns(false)?.ce; - - let mut datetime = Zval::new(); - unsafe { - php_date_instantiate( - std::ptr::from_ref(ce).cast_mut(), - std::ptr::from_mut(&mut datetime), - ); - } - - let datetime_obj = datetime - .object_mut() - .ok_or_else(|| ext_exception("flow_php failed to instantiate a datetime object"))?; - - // timelib reads the byte AFTER the consumed input, so the buffer must be - // NUL-terminated like the zend_strings PHP hands it (length excludes the NUL) - let mut time_str = Vec::with_capacity(bytes.len() + 1); - time_str.extend_from_slice(bytes); - time_str.push(0); - - let initialized = unsafe { - php_date_initialize( - php_date_obj(datetime_obj), - time_str.as_mut_ptr().cast::(), - time_str.len() - 1, - std::ptr::null(), - timezone.map_or(std::ptr::null_mut(), std::ptr::from_mut), - 0, - ) - }; - - if !initialized { - // date_create() reports parse failures via `false`; discard any engine - // artifact so the caller can fall back to the PHP cast cleanly - drop(ExecutorGlobals::take_exception()); - - return Ok(None); - } - - ensure_no_pending_exception("parse a datetime value")?; - - Ok(Some(datetime)) +// `timelib_time` up to `tim_uptodate` (ext/date/lib/timelib.h, identical in the 2022.14-2022.17 timelib of PHP 8.3-8.5) +#[repr(C)] +struct TimelibSpecial { + kind: c_uint, + amount: i64, } -pub fn decode_value( - decoder: &Decoder, - reader: &mut Reader, - ctx: &mut Ctx, -) -> Result { - let mut zv = Zval::new(); - - match decoder { - Decoder::Integer => zv.set_long(reader.i64("integer value")?), - Decoder::Float => zv.set_double(reader.f64("float value")?), - Decoder::Boolean => zv.set_bool(reader.u8("boolean value")? == 0x01), - Decoder::String => { - let length = reader.u32("string value")? as usize; - zv.set_zend_string(ZendStr::new(reader.bytes(length, "string value")?, false)); - } - Decoder::Null => zv.set_null(), - Decoder::DateTime(zone) => zv = decode_datetime(reader, Some(zone), ctx)?, - Decoder::StoredZoneDateTime => zv = decode_datetime(reader, None, ctx)?, - Decoder::Interval => zv = decode_interval(reader, ctx)?, - Decoder::Uuid => zv = decode_uuid(reader, ctx)?, - Decoder::Json => zv = decode_json(reader, ctx)?, - Decoder::TimeZone => { - let length = reader.u32("timezone value")? as usize; - zv = ctx - .timezone(reader.bytes(length, "timezone value")?)? - .shallow_clone(); - } - Decoder::Enum => zv = decode_enum(reader, ctx)?, - Decoder::Xml => zv = decode_via_static(reader, ctx, "xmlDocumentFromString")?, - Decoder::XmlElement => zv = decode_via_static(reader, ctx, "xmlElementFromString")?, - Decoder::Html => zv = decode_via_static(reader, ctx, "htmlDocumentFromString")?, - Decoder::HtmlElement => zv = decode_via_static(reader, ctx, "htmlElementFromString")?, - Decoder::List(element) => { - let count = reader.u32("list value")?; - let mut values = ZendHashTable::with_capacity(count); - - for _ in 0..count { - let value = decode_value(element, reader, ctx)?; - values.push(value).map_err(|e| { - ext_exception(format!("flow_php failed to collect list values: {e:?}")) - })?; - } - - zv.set_hashtable(values); - } - Decoder::Map(key, value) => zv = decode_map(key, value, reader, ctx)?, - Decoder::Structure(elements) => zv = decode_structure(elements, reader, ctx)?, - Decoder::Optional(base) => { - if reader.u8("optional value")? == VALUE_NULL { - zv.set_null(); - } else { - zv = decode_value(base, reader, ctx)?; - } - } - } - - Ok(zv) +#[repr(C)] +struct TimelibRelTime { + y: i64, + m: i64, + d: i64, + h: i64, + i: i64, + s: i64, + us: i64, + weekday: c_int, + weekday_behavior: c_int, + first_last_day_of: c_int, + invert: c_int, + days: i64, + special: TimelibSpecial, + have_weekday_relative: c_uint, + have_special_relative: c_uint, } -fn decode_map( - key: &MapKey, - value: &Decoder, - reader: &mut Reader, - ctx: &mut Ctx, -) -> Result { - let count = reader.u32("map value")?; - let mut values = ZendHashTable::with_capacity(count); - - for _ in 0..count { - match key { - MapKey::Integer => { - let map_key = reader.i64("map key")?; - let map_value = decode_value(value, reader, ctx)?; - ht_insert_index(&mut values, map_key, map_value); - } - MapKey::String => { - let length = reader.u32("map key")? as usize; - let map_key = reader.bytes(length, "map key")?; - let map_value = decode_value(value, reader, ctx)?; - ht_insert(&mut values, map_key, map_value); - } - } - } - - let mut zv = Zval::new(); - zv.set_hashtable(values); - - Ok(zv) +#[repr(C)] +struct TimelibTime { + y: i64, + m: i64, + d: i64, + h: i64, + i: i64, + s: i64, + us: i64, + z: c_int, + tz_abbr: *mut c_char, + tz_info: *mut c_void, + dst: c_int, + relative: TimelibRelTime, + sse: i64, + have_time: c_uint, + have_date: c_uint, + have_zone: c_uint, + have_relative: c_uint, + have_weeknr_day: c_uint, + sse_uptodate: c_uint, + tim_uptodate: c_uint, } -fn decode_structure( - elements: &[(Vec, Decoder)], - reader: &mut Reader, - ctx: &mut Ctx, -) -> Result { - let mut structure = ZendHashTable::with_capacity(elements.len() as u32); - - for (name, element) in elements { - let flag = reader.u8("structure element flag")?; - - match flag { - VALUE_PRESENT => { - let value = decode_value(element, reader, ctx)?; - ht_insert(&mut structure, name, value); - } - VALUE_NULL => ht_insert(&mut structure, name, Zval::new()), - VALUE_ABSENT => {} - other => { - return Err(ext_exception(format!( - "flow_php found unknown structure element flag 0x{other:02X}" - ))); - } - } - } - - let mut zv = Zval::new(); - zv.set_hashtable(structure); +/// The up-to-date `timelib_time` of a `DateTimeImmutable` (or subclass) object. +fn timelib_time(datetime: &ZendObject) -> Option<&TimelibTime> { + let dateobj = std::ptr::from_ref(datetime) + .cast::() + .wrapping_sub(PHP_DATE_OBJ_STD_OFFSET) + .cast::<*const TimelibTime>(); + let time = unsafe { (*dateobj).as_ref() }?; - Ok(zv) + (time.sse_uptodate != 0 && time.tim_uptodate != 0).then_some(time) } -/// Mirrors `DateTimeDecoder`: `zone` is its column zone, `None` restores each value's stored zone. -fn decode_datetime(reader: &mut Reader, zone: Option<&[u8]>, ctx: &mut Ctx) -> Result { - let timestamp = reader.i64("datetime value")?; - let microseconds = reader.u32("datetime value")?; - let timezone_length = reader.u32("datetime value")? as usize; - let timezone_name = reader.bytes(timezone_length, "datetime value")?; +/// `DateTimePhysical::toPhysical()`: `getTimestamp() * 1_000_000 + format('u')`, `None` on i64 overflow. +pub fn datetime_micros(datetime: &ZendObject) -> Option { + let time = timelib_time(datetime)?; - let restore_failed = || ext_exception(format!("flow_php failed to restore datetime from timestamp \"{timestamp}\"")); - - // "U.u" refuses a fraction longer than six digits as trailing data - if microseconds > 999_999 { - return Err(restore_failed()); - } + time.sse.checked_mul(1_000_000)?.checked_add(time.us) +} - // the class is not stored: a datetime column always hydrates to DateTimeImmutable - let fns = ctx.datetime_fns(false)?; +/// `DatePhysical::toPhysical()`: days since the epoch of the value's own calendar day, `None` outside i32. +pub fn datetime_days(datetime: &ZendObject) -> Option { + let time = timelib_time(datetime)?; + let days = days_from_civil(time.y, time.m, time.d)?; - let ce = fns.ce; - let set_timezone = fns.set_timezone; + i32::try_from(days).is_ok().then_some(days) +} +/// A fresh `DateTimeImmutable` initialised by `init` over its `php_date_obj`, then moved into `zone`. +fn datetime_in( + zone: &[u8], + init: impl FnOnce(*mut c_void) -> bool, + failure: impl FnOnce() -> PhpException, +) -> Result { + let fns = ctx::datetime_immutable()?; let mut datetime = Zval::new(); + unsafe { - php_date_instantiate( - std::ptr::from_ref(ce).cast_mut(), - std::ptr::from_mut(&mut datetime), - ); + php_date_instantiate(std::ptr::from_ref(fns.ce).cast_mut(), std::ptr::from_mut(&mut datetime)); } let datetime_obj = datetime .object_mut() .ok_or_else(|| ext_exception("flow_php failed to instantiate a datetime object"))?; - - let dateobj = php_date_obj(datetime_obj); - - // the createFromTimestamp() initializer, exported from PHP 8.4 - #[cfg(php84)] - let initialized = { - unsafe { php_date_initialize_from_ts_long(dateobj, timestamp, microseconds as c_int) }; - - true - }; - - #[cfg(not(php84))] - let initialized = { - // timelib reads the byte AFTER the consumed input, so the buffer must be - // NUL-terminated like the zend_strings PHP hands it (length excludes the NUL) - let mut time_str = format!("{timestamp}.{microseconds:06}\0"); - - unsafe { - php_date_initialize( - dateobj, - time_str.as_mut_ptr().cast::(), - time_str.len() - 1, - c"U.u".as_ptr(), - std::ptr::null_mut(), - // PHP_DATE_INIT_FORMAT - the flags createFromFormat passes. - 0x02, - ) - } - }; + let initialized = init(php_date_obj(datetime_obj)); ensure_no_pending_exception("restore a datetime value")?; if !initialized { - return Err(restore_failed()); + return Err(failure()); } - let timezone = ctx.timezone(zone.unwrap_or(timezone_name))?.shallow_clone(); let datetime = call_handle( - &set_timezone, + fns.set_timezone, Some(datetime_obj), - &mut [timezone], + &mut [ctx::timezone(zone)?], "restore a datetime timezone", )?; @@ -296,39 +151,95 @@ fn decode_datetime(reader: &mut Reader, zone: Option<&[u8]>, ctx: &mut Ctx) -> R Ok(datetime) } -/// Mirrors `ValueDecoder::decodeInterval`; `days` is intentionally not -/// preserved (same as PHP). -fn decode_interval(reader: &mut Reader, ctx: &mut Ctx) -> Result { - let invert = i64::from(reader.u8("time value")?); - let mut fields = [0i64; 7]; +/// `DateTimePhysical::fromPhysical()`: the instant `micros` in `zone`. +pub fn datetime_from_micros(micros: i64, zone: &[u8]) -> Result { + let seconds = micros.div_euclid(1_000_000); + let fraction = micros.rem_euclid(1_000_000); - for field in &mut fields[..6] { - *field = i64::from(reader.u32("time value")?); - } + datetime_in( + zone, + |dateobj| { + // the createFromTimestamp() initializer, exported from PHP 8.4 + #[cfg(php84)] + { + unsafe { php_date_initialize_from_ts_long(dateobj, seconds, fraction as c_int) }; + + true + } + + #[cfg(not(php84))] + { + // timelib reads the byte AFTER the consumed input, so the buffer must be NUL-terminated like the + // zend_strings PHP hands it (length excludes the NUL) + let mut time_str = format!("{seconds}.{fraction:06}\0"); + + unsafe { + php_date_initialize( + dateobj, + time_str.as_mut_ptr().cast::(), + time_str.len() - 1, + c"U.u".as_ptr(), + std::ptr::null_mut(), + // PHP_DATE_INIT_FORMAT - the flags createFromFormat passes + 0x02, + ) + } + } + }, + || { + ext_exception(format!( + "flow_php failed to restore datetime from \"{micros}\" microseconds" + )) + }, + ) +} - fields[6] = invert; - let fraction = reader.f64("time value")?; +/// `DatePhysical::fromPhysical()`: `new DateTimeImmutable('@' . ($days * 86_400))` in UTC. +pub fn date_from_days(days: i64) -> Result { + let mut time_str = format!("@{}\0", days * 86_400); - let interval_fns = ctx.interval()?; - let mut interval = ZendObject::new(interval_fns.ce); + datetime_in( + b"UTC", + |dateobj| unsafe { + php_date_initialize( + dateobj, + time_str.as_mut_ptr().cast::(), + time_str.len() - 1, + std::ptr::null(), + std::ptr::null_mut(), + 0, + ) + }, + || ext_exception(format!("flow_php failed to restore date from \"{days}\" days")), + ) +} + +/// `TimePhysical::fromPhysical()`: `PT%dH%dM%dS`, then `f` and `invert`; `days` stays `false`. +pub fn interval_from_micros(micros: i64) -> Result { + let magnitude = micros.unsigned_abs(); + let spec = format!( + "PT{}H{}M{}S", + magnitude / 3_600_000_000, + magnitude / 60_000_000 % 60, + magnitude / 1_000_000 % 60 + ); + let fns = ctx::interval()?; + let mut interval = ZendObject::new(fns.ce); call_handle( - &interval_fns.construct, + fns.construct, Some(&mut interval), - &mut [interval_fns.spec.shallow_clone()], + &mut [zval_str(spec.as_bytes())], "construct DateInterval", )?; - for (index, value) in fields.iter().enumerate() { - write_property_raw( - &mut interval, - &mut interval_fns.names[index], - zval_long(*value), - )?; - } - - let mut fraction_zv = Zval::new(); - fraction_zv.set_double(fraction); - write_property_raw(&mut interval, &mut interval_fns.names[7], fraction_zv)?; + let mut fraction = Zval::new(); + fraction.set_double((magnitude % 1_000_000) as f64 / 1_000_000.0); + write_property_raw(&mut interval, &mut ZendStr::new("f", false), fraction)?; + write_property_raw( + &mut interval, + &mut ZendStr::new("invert", false), + zval_long(i64::from(micros < 0)), + )?; let mut zv = Zval::new(); zv.set_object(&mut interval); @@ -336,14 +247,34 @@ fn decode_interval(reader: &mut Reader, ctx: &mut Ctx) -> Result Result { - let length = reader.u32("uuid value")? as usize; - let value = reader.bytes(length, "uuid value")?; - let (uuid_ce, value_slot) = ctx.uuid()?; +/// The lowercase 36-char form of 16 uuid bytes. +pub fn uuid_text(bytes: &[u8]) -> Vec { + let mut text = Vec::with_capacity(36); + + push_uuid_text(&mut text, bytes); + + text +} + +/// [`uuid_text`] appended to `out`. +pub fn push_uuid_text(out: &mut Vec, bytes: &[u8]) { + const HEX: &[u8; 16] = b"0123456789abcdef"; + + for (index, byte) in bytes.iter().enumerate() { + if matches!(index, 4 | 6 | 8 | 10) { + out.push(b'-'); + } + + out.push(HEX[usize::from(byte >> 4)]); + out.push(HEX[usize::from(byte & 0x0f)]); + } +} + +/// `Uuid::fromBytes()`: the constructor-less instance holding the lowercase 36-char form. +pub fn uuid_from_bytes(bytes: &[u8]) -> Result { + let (uuid_ce, value_slot) = ctx::uuid()?; let mut uuid = ZendObject::new(uuid_ce); - write_slot(&mut uuid, value_slot, zval_str(value)); + write_slot(&mut uuid, value_slot, zval_str(&uuid_text(bytes))); let mut zv = Zval::new(); zv.set_object(&mut uuid); @@ -351,19 +282,15 @@ fn decode_uuid(reader: &mut Reader, ctx: &mut Ctx) -> Result Ok(zv) } -/// Mirrors `ValueDecoder::$createJson`: constructor-less instantiation. -fn decode_json(reader: &mut Reader, ctx: &mut Ctx) -> Result { - let length = reader.u32("json value")? as usize; - let value = reader.bytes(length, "json value")?; - let is_object = reader.u8("json value")? == 0x01; - - let (json_ce, value_slot, is_object_slot) = ctx.json()?; +/// `JsonFactory::create()`: the constructor-less instance, `isObject` read from the first byte. +pub fn json_from_bytes(bytes: &[u8]) -> Result { + let (json_ce, value_slot, is_object_slot) = ctx::json()?; let mut json = ZendObject::new(json_ce); - write_slot(&mut json, value_slot, zval_str(value)); + write_slot(&mut json, value_slot, zval_str(bytes)); - let mut is_object_zv = Zval::new(); - is_object_zv.set_bool(is_object); - write_slot(&mut json, is_object_slot, is_object_zv); + let mut is_object = Zval::new(); + is_object.set_bool(bytes.first() == Some(&b'{')); + write_slot(&mut json, is_object_slot, is_object); let mut zv = Zval::new(); zv.set_object(&mut json); @@ -371,33 +298,63 @@ fn decode_json(reader: &mut Reader, ctx: &mut Ctx) -> Result Ok(zv) } -fn decode_enum(reader: &mut Reader, ctx: &mut Ctx) -> Result { - let class_length = reader.u32("enum value")? as usize; - let class = reader.bytes(class_length, "enum value")?.to_vec(); - let case_length = reader.u32("enum value")? as usize; - let case = reader.bytes(case_length, "enum value")?.to_vec(); +/// `hex2bin()` of a 36-char uuid with its dashes removed. +pub fn uuid_bytes(text: &[u8]) -> Option<[u8; 16]> { + let mut bytes = [0u8; 16]; + let mut digits = text + .iter() + .filter(|byte| **byte != b'-') + .map(|byte| (*byte as char).to_digit(16)); + + for byte in &mut bytes { + let high = digits.next()??; + let low = digits.next()??; + *byte = (high * 16 + low) as u8; + } - ctx.enum_case(&class, &case) + digits.next().is_none().then_some(bytes) } -/// Markup DOM reconstruction delegated to the pure-PHP `ValueDecoder` public -/// statics - the exact code its decoders run. -fn decode_via_static( - reader: &mut Reader, - ctx: &mut Ctx, - method: &'static str, -) -> Result { - let length = reader.u32("markup value")? as usize; - let markup_zv = zval_str(reader.bytes(length, "markup value")?); +/// `new DateTimeImmutable($str, $timezone)` through the same C-level timelib parser, in its +/// non-throwing `date_create()` flavor: `Ok(None)` on parse failure, no exception. +pub(crate) fn date_from_free_form(bytes: &[u8], timezone: Option<&mut Zval>) -> Result, PhpException> { + let ce = ctx::datetime_immutable()?.ce; + + let mut datetime = Zval::new(); + unsafe { + php_date_instantiate(std::ptr::from_ref(ce).cast_mut(), std::ptr::from_mut(&mut datetime)); + } - let handle = ctx.value_decoder_static(method)?; - let value = call_handle(handle, None, &mut [markup_zv], "restore a markup value")?; + let datetime_obj = datetime + .object_mut() + .ok_or_else(|| ext_exception("flow_php failed to instantiate a datetime object"))?; + + // timelib reads the byte AFTER the consumed input, so the buffer must be + // NUL-terminated like the zend_strings PHP hands it (length excludes the NUL) + let mut time_str = Vec::with_capacity(bytes.len() + 1); + time_str.extend_from_slice(bytes); + time_str.push(0); - if !value.is_object() { - return Err(ext_exception(format!( - "flow_php expected ValueDecoder::{method} to return an object" - ))); + let initialized = unsafe { + php_date_initialize( + php_date_obj(datetime_obj), + time_str.as_mut_ptr().cast::(), + time_str.len() - 1, + std::ptr::null(), + timezone.map_or(std::ptr::null_mut(), std::ptr::from_mut), + 0, + ) + }; + + if !initialized { + // date_create() reports parse failures via `false`; discard any engine + // artifact so the caller can fall back to the PHP cast cleanly + drop(ExecutorGlobals::take_exception()); + + return Ok(None); } - Ok(value) + ensure_no_pending_exception("parse a datetime value")?; + + Ok(Some(datetime)) } diff --git a/src/extension/flow-php-ext/tests/phpt/001_extension_loaded.phpt b/src/extension/flow-php-ext/tests/phpt/001_extension_loaded.phpt index 4f5db9b9fc..99e3f72c53 100644 --- a/src/extension/flow-php-ext/tests/phpt/001_extension_loaded.phpt +++ b/src/extension/flow-php-ext/tests/phpt/001_extension_loaded.phpt @@ -1,14 +1,41 @@ --TEST-- -flow_php extension is loaded and symbols are registered +flow_php extension is loaded and registers the package interfaces and the Rust classes implementing them --SKIPIF-- --FILE-- getClassNames(); +sort($classes); + +foreach ($classes as $class) { + $reflection = new ReflectionClass($class); + echo $reflection->isInterface() ? 'interface ' : 'class ', $class, $reflection->isInterface() ? '' : ' implements ' . implode(', ', array_diff($reflection->getInterfaceNames(), ['Traversable'])), "\n"; +} + +var_dump(class_exists('Flow\Floe\Exception\ExtensionException', false) === false); ?> --EXPECT-- bool(true) -bool(true) +interface Flow\ETL\Adapter\CSV\CSVEncoder +interface Flow\ETL\Adapter\CSV\CSVOpenSource +class Flow\ETL\Adapter\CSV\RustCSVEncoder implements Flow\ETL\Adapter\CSV\CSVEncoder +class Flow\ETL\Adapter\CSV\RustCSVOpenSource implements Flow\ETL\Adapter\CSV\CSVOpenSource +interface Flow\ETL\Adapter\JSON\JsonEncoder +interface Flow\ETL\Adapter\JSON\JsonOpenSource +class Flow\ETL\Adapter\JSON\RustJsonEncoder implements Flow\ETL\Adapter\JSON\JsonEncoder +class Flow\ETL\Adapter\JSON\RustJsonOpenSource implements Flow\ETL\Adapter\JSON\JsonOpenSource +interface Flow\ETL\Adapter\Parquet\ParquetOpenSink +interface Flow\ETL\Adapter\Parquet\ParquetOpenSource +class Flow\ETL\Adapter\Parquet\RustParquetOpenSink implements Flow\ETL\Adapter\Parquet\ParquetOpenSink +class Flow\ETL\Adapter\Parquet\RustParquetOpenSource implements Flow\ETL\Adapter\Parquet\ParquetOpenSource +interface Flow\ETL\Column\Backend +interface Flow\ETL\Column\Column +interface Flow\ETL\Column\ColumnBuilder +class Flow\ETL\Column\RustBackend implements Flow\ETL\Column\Backend +class Flow\ETL\Column\RustColumn implements Flow\ETL\Column\Column +class Flow\ETL\Column\RustColumnBuilder implements Flow\ETL\Column\ColumnBuilder +class Flow\ETL\Column\RustColumnsBatch implements +class Flow\ETL\RustIterator implements Iterator bool(true) diff --git a/src/extension/flow-php-ext/tests/phpt/002_scalar_round_trip.phpt b/src/extension/flow-php-ext/tests/phpt/002_scalar_round_trip.phpt deleted file mode 100644 index 8f25bbdecf..0000000000 --- a/src/extension/flow-php-ext/tests/phpt/002_scalar_round_trip.phpt +++ /dev/null @@ -1,47 +0,0 @@ ---TEST-- -scalar columns round-trip identically to the pure-PHP decoder ---SKIPIF-- - ---FILE-- - 0, 'name' => null, 'email' => "binary \x00 unsafe \n string", 'price' => 12345.678, 'active' => true, 'score' => -1000]), row(['id' => PHP_INT_MAX, 'name' => 'user_1', 'email' => '', 'price' => -0.5, 'active' => false, 'score' => PHP_INT_MIN])); - -$frames = php_frames($rows); -$actual = ext_decode_frames($frames); - -var_dump(count($actual) === 2); -assert_rows_identical(php_decode_frames($frames), $actual); - -var_dump($actual[0]->get('id')); -var_dump($actual[0]->get('name')); -var_dump($schema->get('name')->isNullable()); -var_dump($actual[0]->get('email') === "binary \x00 unsafe \n string"); -var_dump($schema->get('email')->isNullable()); -var_dump($actual[1]->get('id') === PHP_INT_MAX); -var_dump($actual[1]->get('score') === PHP_INT_MIN); -var_dump($actual[1]->get('price')); -var_dump($actual[1]->get('active')); -?> ---EXPECT-- -bool(true) -identical -int(0) -NULL -bool(true) -bool(true) -bool(false) -bool(true) -bool(true) -float(-0.5) -bool(false) diff --git a/src/extension/flow-php-ext/tests/phpt/003_null_flags.phpt b/src/extension/flow-php-ext/tests/phpt/003_null_flags.phpt deleted file mode 100644 index 24b6d2625f..0000000000 --- a/src/extension/flow-php-ext/tests/phpt/003_null_flags.phpt +++ /dev/null @@ -1,42 +0,0 @@ ---TEST-- -null values and null-typed columns hydrate the matching definitions ---SKIPIF-- - ---FILE-- - 1, 'nullable' => null, 'null_typed' => null, 'present' => 'value'])); - -$frames = php_frames($rows); -$actual = ext_decode_frames($frames); - -assert_rows_identical(php_decode_frames($frames), $actual); - -var_dump($actual[0]->get('nullable')); -var_dump($schema->get('nullable')->isNullable()); -var_dump($schema->get('nullable')->type()->toString()); -var_dump($actual[0]->get('null_typed')); -var_dump($schema->get('null_typed')->isNullable()); -var_dump($schema->get('null_typed')->type()->toString()); -var_dump($schema->get('present')->isNullable()); -var_dump($schema->get('present')->type()->toString()); -?> ---EXPECT-- -identical -NULL -bool(true) -string(6) "string" -NULL -bool(true) -string(4) "null" -bool(false) -string(6) "string" diff --git a/src/extension/flow-php-ext/tests/phpt/004_datetime.phpt b/src/extension/flow-php-ext/tests/phpt/004_datetime.phpt deleted file mode 100644 index 9e35377fed..0000000000 --- a/src/extension/flow-php-ext/tests/phpt/004_datetime.phpt +++ /dev/null @@ -1,56 +0,0 @@ ---TEST-- -datetime columns: a mutable input normalises to immutable, non-UTC timezone, microseconds ---SKIPIF-- - ---FILE-- - 1, - 'immutable' => new DateTimeImmutable('2025-01-01 12:00:00.123456', new DateTimeZone('Europe/Warsaw')), - // the class is not stored: a datetime column always hydrates to DateTimeImmutable, so a - // DateTime written here comes back immutable, carrying the same instant and timezone - 'mutable' => new DateTime('1969-12-31 23:59:59.999999', new DateTimeZone('America/New_York')), - 'nullable' => null, - // regression: timelib's trailing-data check reads the byte after the - // parsed input - pre-1970 fractional epochs exposed a missing NUL - // terminator in the extension's date construction - 'before_epoch' => new DateTimeImmutable('1969-07-20 20:17:00.5', new DateTimeZone('Europe/Warsaw')), - ]), -); - -$frames = php_frames($rows); -$actual = ext_decode_frames($frames); - -assert_rows_identical(php_decode_frames($frames), $actual); - -var_dump(get_class($actual[0]->get('immutable'))); -var_dump($actual[0]->get('immutable')->format('Y-m-d H:i:s.u e')); -var_dump(get_class($actual[0]->get('mutable'))); -var_dump($actual[0]->get('mutable')->format('Y-m-d H:i:s.u e')); -var_dump($actual[0]->get('nullable')); -var_dump($actual[0]->get('before_epoch')->format('Y-m-d H:i:s.u e')); -?> ---EXPECT-- -identical -string(17) "DateTimeImmutable" -string(40) "2025-01-01 12:00:00.123456 Europe/Warsaw" -string(17) "DateTimeImmutable" -string(43) "1969-12-31 23:59:59.999999 America/New_York" -NULL -string(40) "1969-07-20 20:17:00.500000 Europe/Warsaw" diff --git a/src/extension/flow-php-ext/tests/phpt/005_time_uuid.phpt b/src/extension/flow-php-ext/tests/phpt/005_time_uuid.phpt deleted file mode 100644 index e132827faa..0000000000 --- a/src/extension/flow-php-ext/tests/phpt/005_time_uuid.phpt +++ /dev/null @@ -1,57 +0,0 @@ ---TEST-- -time (DateInterval) and uuid columns round-trip identically ---SKIPIF-- - ---FILE-- -diff(new DateTimeImmutable('2025-01-01')); -$fractional = new DateInterval('PT1S'); -$fractional->f = 0.123456; - -$rows = rows( - schema( - int_schema('id'), - time_schema('duration'), - time_schema('negative'), - time_schema('fractional'), - uuid_schema('uuid'), - ), - row([ - 'id' => 1, - 'duration' => new DateInterval('P3DT4H5M6S'), - 'negative' => $negative, - 'fractional' => $fractional, - 'uuid' => type_uuid()->cast('01234567-89ab-4def-8123-456789abcdef'), - ]), -); - -$frames = php_frames($rows); -$actual = ext_decode_frames($frames); - -assert_rows_identical(php_decode_frames($frames), $actual); - -$duration = $actual[0]->get('duration'); -var_dump($duration->d, $duration->h, $duration->i, $duration->s); -var_dump($actual[0]->get('negative')->invert); -var_dump($actual[0]->get('fractional')->f); -var_dump((string) $actual[0]->get('uuid')); -?> ---EXPECT-- -identical -int(3) -int(4) -int(5) -int(6) -int(1) -float(0.123456) -string(36) "01234567-89ab-4def-8123-456789abcdef" diff --git a/src/extension/flow-php-ext/tests/phpt/006_schema_change_mid_stream.phpt b/src/extension/flow-php-ext/tests/phpt/006_schema_change_mid_stream.phpt deleted file mode 100644 index 4bfe227998..0000000000 --- a/src/extension/flow-php-ext/tests/phpt/006_schema_change_mid_stream.phpt +++ /dev/null @@ -1,32 +0,0 @@ ---TEST-- -heterogeneous Rows conform to one union schema (columns follow union order) ---SKIPIF-- - ---FILE-- - 1, 'name' => 'a']), row(['id' => 2, 'name' => 'b']), row(['id' => 3, 'price' => 1.5]), row(['name' => 'c', 'id' => 4])); - -$frames = php_frames($rows); -$actual = ext_decode_frames($frames); - -var_dump(count($actual)); -assert_rows_identical(php_decode_frames($frames), $actual); - -var_dump($actual[2]->get('price')); -// row written as (name, id) comes back in the single union schema's column order -var_dump(implode(',', $actual[3]->names())); -?> ---EXPECT-- -int(4) -identical -float(1.5) -string(13) "id,name,price" diff --git a/src/extension/flow-php-ext/tests/phpt/007_empty_rows.phpt b/src/extension/flow-php-ext/tests/phpt/007_empty_rows.phpt deleted file mode 100644 index 6f06ab18b4..0000000000 --- a/src/extension/flow-php-ext/tests/phpt/007_empty_rows.phpt +++ /dev/null @@ -1,22 +0,0 @@ ---TEST-- -empty Rows produce no frame bodies and decode to an empty row list ---SKIPIF-- - ---FILE-- - ---EXPECT-- -bool(true) -int(0) -identical diff --git a/src/extension/flow-php-ext/tests/phpt/010_no_leaks.phpt b/src/extension/flow-php-ext/tests/phpt/010_no_leaks.phpt deleted file mode 100644 index 72210fa2a4..0000000000 --- a/src/extension/flow-php-ext/tests/phpt/010_no_leaks.phpt +++ /dev/null @@ -1,114 +0,0 @@ ---TEST-- -repeated frame-body encode and decode do not leak memory ---SKIPIF-- - ---FILE-- - type_integer(), 'tags' => type_list(type_string())])), - xml_schema('doc'), - json_schema('json'), -); - -$build = static function (int $i): Flow\ETL\Row { - return row([ - 'id' => $i, - 'name' => $i % 10 === 0 ? null : 'user_' . $i, - 'price' => $i / 100.0, - 'active' => $i % 3 === 0, - 'created_at' => new DateTimeImmutable('2025-01-01 00:00:00.123456', new DateTimeZone('Europe/Warsaw')), - 'duration' => new DateInterval('PT1H2M3S'), - 'uuid' => type_uuid()->cast('01234567-89ab-4def-8123-456789abcdef'), - 'metrics' => ['cpu' => 0.5, 'mem' => 0.25], - 'nested' => ['a' => $i, 'tags' => ['x']], - 'doc' => type_xml()->cast('' . $i . ''), - 'json' => type_json()->cast([$i, ['a' => true]]), - ]); -}; - -$sourceRows = array_map($build, range(1, 100)); -$frames = php_frames(rows($schema, ...$sourceRows)); - -$schemaBody = null; -$rowBodies = []; - -foreach ($frames as $frame) { - if ($frame['type'] === SCHEMA_ENTRY) { - $schemaBody = $frame['body']; - } else { - $rowBodies[] = $frame['body']; - } -} - -$decodeSchema = schema_from_json($schemaBody); - -$cycle = static function () use ($schemaBody, $decodeSchema, $rowBodies, $schema, $sourceRows): void { - $decoder = new RustFloeEncoderNative(); - $hydrator = new PhpRowHydrator(); - $hydrator->hydrate($decoder->decode($rowBodies, $schemaBody), $decodeSchema); - - foreach ($rowBodies as $body) { - $hydrator->hydrate($decoder->decode([$body], $schemaBody), $decodeSchema); - } - - $encoder = new RustFloeEncoderNative(); - $encoder->encode($hydrator->dehydrate(new Rows($schema, ...$sourceRows)), $schemaBody, $schema); -}; - -for ($i = 0; $i < 10; $i++) { - $cycle(); -} -gc_collect_cycles(); -// memory_get_usage(false) counts emalloc'd bytes exactly - any per-call leak -// (even one zval) shows up here; (true) only tracks arena chunks and grows -// from GC-buffer expansion without any leak. -$baseline = memory_get_usage(false); - -for ($i = 0; $i < 100; $i++) { - $cycle(); -} -gc_collect_cycles(); - -var_dump(memory_get_usage(false) <= $baseline); -?> ---EXPECT-- -bool(true) diff --git a/src/extension/flow-php-ext/tests/phpt/011_containers.phpt b/src/extension/flow-php-ext/tests/phpt/011_containers.phpt deleted file mode 100644 index 68e7515d57..0000000000 --- a/src/extension/flow-php-ext/tests/phpt/011_containers.phpt +++ /dev/null @@ -1,93 +0,0 @@ ---TEST-- -list, map and structure columns round-trip identically to the pure-PHP decoder ---SKIPIF-- - ---FILE-- - type_string(), - 'geo' => type_structure(['lat' => type_float(), 'lon' => type_float()]), - 'tags' => type_list(type_string()), - ])), structure_schema('optional_absent', type_structure(['a' => type_integer(), 'b' => structure_element('b', type_string(), optional: true)])), structure_schema('optional_typed_null', type_structure(['a' => type_integer(), 'b' => type_optional(type_string())])), structure_schema('interleaved', type_structure(['z' => type_integer(), 'a' => structure_element('a', type_string(), optional: true), 'b' => type_string()]))), row(['id' => 1, 'ints' => [1, -2, PHP_INT_MAX], 'floats' => [1.5, -0.25], 'strings' => ['a', '', "b\x00c"], 'empty' => [], 'nullable_ints' => [1, null, 3], 'by_int' => [10 => 'a', -5 => 'b'], 'by_string' => ['cpu' => 1.5, 'mem' => 2.5], 'empty_map' => [], 'address' => [ - 'street' => 'Main 7', - 'geo' => ['lat' => 1.5, 'lon' => -2.5], - 'tags' => ['x', 'y'], - ], 'optional_absent' => ['a' => 1], 'optional_typed_null' => ['a' => 1, 'b' => null], 'interleaved' => ['z' => 1, 'b' => 'x']])); - -$frames = php_frames($rows); -$actual = ext_decode_frames($frames); - -assert_rows_identical(php_decode_frames($frames), $actual); - -var_dump($actual[0]->get('ints')); -var_dump($actual[0]->get('nullable_ints')); -var_dump($actual[0]->get('by_int')); -var_dump($actual[0]->get('address')['geo']); -var_dump($actual[0]->get('optional_absent')); -var_dump($actual[0]->get('optional_typed_null')); -var_dump($actual[0]->get('interleaved')); -?> ---EXPECT-- -identical -array(3) { - [0]=> - int(1) - [1]=> - int(-2) - [2]=> - int(9223372036854775807) -} -array(3) { - [0]=> - int(1) - [1]=> - NULL - [2]=> - int(3) -} -array(2) { - [10]=> - string(1) "a" - [-5]=> - string(1) "b" -} -array(2) { - ["lat"]=> - float(1.5) - ["lon"]=> - float(-2.5) -} -array(1) { - ["a"]=> - int(1) -} -array(2) { - ["a"]=> - int(1) - ["b"]=> - NULL -} -array(2) { - ["z"]=> - int(1) - ["b"]=> - string(1) "x" -} diff --git a/src/extension/flow-php-ext/tests/phpt/012_enum_json_date.phpt b/src/extension/flow-php-ext/tests/phpt/012_enum_json_date.phpt deleted file mode 100644 index fe6ab9477a..0000000000 --- a/src/extension/flow-php-ext/tests/phpt/012_enum_json_date.phpt +++ /dev/null @@ -1,67 +0,0 @@ ---TEST-- -enum, json and date columns round-trip identically ---SKIPIF-- - ---FILE-- - 1, - 'backed' => PhptSuit::Hearts, - 'pure' => PhptDirection::Down, - 'json_array' => Json::fromArray([1, 2, ['a' => true]]), - 'json_object' => Json::fromArray(['name' => 'flow', 'nested' => ['x' => 1]], asObject: true), - 'date' => new DateTimeImmutable('2025-03-01'), - ]), -); - -$frames = php_frames($rows); -$actual = ext_decode_frames($frames); - -assert_rows_identical(php_decode_frames($frames), $actual); - -var_dump($actual[0]->get('backed') === PhptSuit::Hearts); -var_dump($actual[0]->get('pure') === PhptDirection::Down); -var_dump((string) $actual[0]->get('json_array')); -var_dump((string) $actual[0]->get('json_object')); -var_dump($actual[0]->get('date')->format('Y-m-d')); -?> ---EXPECT-- -identical -bool(true) -bool(true) -string(16) "[1,2,{"a":true}]" -string(32) "{"name":"flow","nested":{"x":1}}" -string(10) "2025-03-01" diff --git a/src/extension/flow-php-ext/tests/phpt/013_xml_fallback.phpt b/src/extension/flow-php-ext/tests/phpt/013_xml_fallback.phpt deleted file mode 100644 index 2c051779a1..0000000000 --- a/src/extension/flow-php-ext/tests/phpt/013_xml_fallback.phpt +++ /dev/null @@ -1,42 +0,0 @@ ---TEST-- -xml and xml_element columns decode through the ValueDecoder static fallback ---SKIPIF-- - ---FILE-- -loadXML('ab'); - -$element = (new DOMDocument()); -$element->loadXML('value'); - -$schema = schema(int_schema('id'), xml_schema('doc'), xml_element_schema('el')); - -$frames = php_frames(rows($schema, row(['id' => 1, 'doc' => $document, 'el' => $element->documentElement]))); -$actual = ext_decode_frames($frames); - -// DOM values are not serializable on their own, so the decoded rows are compared -// through their re-encoded frame bodies instead of through serialize(). -var_dump(php_frames(rows($schema, ...$actual)) === php_frames(rows($schema, ...php_decode_frames($frames)))); - -var_dump(get_class($actual[0]->get('doc'))); -var_dump(trim($actual[0]->get('doc')->saveXML())); -var_dump(get_class($actual[0]->get('el'))); -var_dump($actual[0]->get('el')->getAttribute('id')); -?> ---EXPECT-- -bool(true) -string(11) "DOMDocument" -string(72) " -ab" -string(10) "DOMElement" -string(1) "7" diff --git a/src/extension/flow-php-ext/tests/phpt/016_numeric_string_keys.phpt b/src/extension/flow-php-ext/tests/phpt/016_numeric_string_keys.phpt deleted file mode 100644 index 32059879c6..0000000000 --- a/src/extension/flow-php-ext/tests/phpt/016_numeric_string_keys.phpt +++ /dev/null @@ -1,70 +0,0 @@ ---TEST-- -array-key semantics: numeric column names coerce, non-canonical keys stay strings ---SKIPIF-- - ---FILE-- - 1, '7' => 'numeric column name', 'keys' => [ - '05' => 'leading-zero', - '5x' => 'suffix', - '-0' => 'minus-zero', - '9223372036854775808' => 'overflow', - ]])); - -$frames = php_frames($rows); -$actual = ext_decode_frames($frames); - -assert_rows_identical(php_decode_frames($frames), $actual); - -var_dump(array_map(fn($k) => gettype($k) . ':' . $k, array_keys($actual[0]->get('keys')))); -var_dump($actual[0]->get('7')); -// storage cannot hold '7' as a string key, so the decoded row must carry an int key - -// this is the parity window: the Rust decoder has to coerce exactly as PHP does -var_dump(array_keys($actual[0]->values())); -// names() inverts that coercion, because the column name is the entry's identity -var_dump($actual[0]->names()); -?> ---EXPECT-- -identical -array(4) { - [0]=> - string(9) "string:05" - [1]=> - string(9) "string:5x" - [2]=> - string(9) "string:-0" - [3]=> - string(26) "string:9223372036854775808" -} -string(19) "numeric column name" -array(3) { - [0]=> - string(2) "id" - [1]=> - int(7) - [2]=> - string(4) "keys" -} -array(3) { - [0]=> - string(2) "id" - [1]=> - string(1) "7" - [2]=> - string(4) "keys" -} \ No newline at end of file diff --git a/src/extension/flow-php-ext/tests/phpt/017_rows_decoder.phpt b/src/extension/flow-php-ext/tests/phpt/017_rows_decoder.phpt deleted file mode 100644 index 01d5828301..0000000000 --- a/src/extension/flow-php-ext/tests/phpt/017_rows_decoder.phpt +++ /dev/null @@ -1,51 +0,0 @@ ---TEST-- -RowValues pipeline decodes streamed frame bodies identically to the pure-PHP pipeline ---SKIPIF-- - ---FILE-- - 1, 'name' => 'a']), row(['id' => 2, 'name' => null]), row(['id' => 3, 'price' => 1.5]), row(['id' => 4, 'at' => new DateTimeImmutable('2025-01-01 00:00:00.123456', new DateTimeZone('Europe/Warsaw'))])); - -$frames = php_frames($rows); -$decoded = ext_decode_frames($frames); - -var_dump(count($decoded)); - -assert_rows_identical(php_decode_frames($frames), $decoded); - -$schemaBody = null; -$rowBody = null; - -foreach ($frames as $frame) { - if ($frame['type'] === SCHEMA_ENTRY) { - if ($schemaBody === null) { - $schemaBody = $frame['body']; - } - } elseif ($rowBody === null && $schemaBody !== null) { - $rowBody = $frame['body']; - } -} - -try { - (new RustFloeEncoderNative())->decode([$rowBody . "\xEF"], $schemaBody); - echo "FAIL: no exception\n"; -} catch (Flow\Floe\Exception\ExtensionException $e) { - echo $e->getMessage(), "\n"; -} -?> ---EXPECT-- -int(4) -identical -flow_php row frame length does not match its content diff --git a/src/extension/flow-php-ext/tests/phpt/019_html_fallback.phpt b/src/extension/flow-php-ext/tests/phpt/019_html_fallback.phpt deleted file mode 100644 index c43743dcd2..0000000000 --- a/src/extension/flow-php-ext/tests/phpt/019_html_fallback.phpt +++ /dev/null @@ -1,40 +0,0 @@ ---TEST-- -html and html_element columns decode through the ValueDecoder static fallback (PHP 8.4+) ---SKIPIF-- - ---FILE-- - 1, - 'doc' => type_html()->cast('

hello

'), - 'el' => type_html_element()->cast('
content
'), -]))); -$actual = ext_decode_frames($frames); - -var_dump(php_frames(rows($schema, ...$actual)) === php_frames(rows($schema, ...php_decode_frames($frames)))); - -var_dump(get_class($actual[0]->get('doc'))); -var_dump($actual[0]->get('doc')->getElementById('x')->textContent); -var_dump($actual[0]->get('el')->getAttribute('class')); -?> ---EXPECT-- -bool(true) -string(16) "Dom\HTMLDocument" -string(5) "hello" -string(1) "c" diff --git a/src/extension/flow-php-ext/tests/phpt/020_row_encoder_parity.phpt b/src/extension/flow-php-ext/tests/phpt/020_row_encoder_parity.phpt deleted file mode 100644 index 81c1fcd939..0000000000 --- a/src/extension/flow-php-ext/tests/phpt/020_row_encoder_parity.phpt +++ /dev/null @@ -1,102 +0,0 @@ ---TEST-- -RustFloeEncoderNative produces ROW frame bodies byte-identical to PhpFloeEncoder ---SKIPIF-- - ---FILE-- - rows(schema(int_schema('id'), str_schema('name', nullable: true), str_schema('e'), float_schema('p'), bool_schema('a'), int_schema('s')), row(['id' => 1, 'name' => null, 'e' => "b\x00in", 'p' => -1.5, 'a' => true, 's' => PHP_INT_MIN]), row(['id' => 2, 'name' => 'x', 'e' => '', 'p' => 0.25, 'a' => false, 's' => PHP_INT_MAX])), - 'from_null' => rows(schema(int_schema('id'), null_schema('n')), row(['id' => 1, 'n' => null])), - 'datetime' => rows(schema(datetime_schema('at'), date_schema('d'), time_schema('t'), uuid_schema('u')), row(['at' => new DateTimeImmutable('2025-01-01 12:00:00.123456', new DateTimeZone('Europe/Warsaw')), 'd' => new DateTimeImmutable('2025-03-01'), 't' => new DateInterval('PT2H30M5S'), 'u' => type_uuid()->cast('01234567-89ab-4def-8123-456789abcdef')])), - 'containers' => rows(schema(list_schema('ints', type_list(type_integer())), map_schema('m', type_map(type_string(), type_float())), map_schema('mi', type_map(type_integer(), type_string())), structure_schema('st', type_structure(['a' => type_integer(), 'b' => structure_element('b', type_string(), optional: true)])), structure_schema('st_interleaved', type_structure(['z' => type_integer(), 'a' => structure_element('a', type_string(), optional: true), 'b' => type_string()])), list_schema('opt', type_list(type_optional(type_integer())))), row(['ints' => [1, 2, 3], 'm' => ['cpu' => 1.5], 'mi' => [7 => 'a', -1 => 'b'], 'st' => ['a' => 1], 'st_interleaved' => ['z' => 1, 'b' => 'x'], 'opt' => [1, null]])), - 'enum_json_xml' => rows(schema(enum_schema('en', PhptColor::class), json_schema('j'), xml_schema('x')), row(['en' => PhptColor::Red, 'j' => type_json()->cast(['a' => 1]), 'x' => type_xml()->cast('v')])), - 'heterogeneous' => rows(schema(int_schema('id'), str_schema('n', nullable: true)), row(['id' => 1]), row(['id' => 2, 'n' => 'x']), row(['n' => 'y', 'id' => 3])), - 'two_columns' => rows(schema(int_schema('id'), str_schema('g')), row(['id' => 1, 'g' => 'a'])), - 'empty' => rows(schema()), -]; - -foreach ($datasets as $label => $data) { - printf("%-14s frames-identical:%s\n", $label, php_frames($data) === ext_frames($data) ? 'yes' : 'NO'); -} - -// A row that does not carry a column the plan declares is a broken contract, not a -// value the format can express: VALUE_ABSENT (0x03) is a structure-element flag only. -// Both encoders must refuse it. -$wideBody = json_encode(schema(int_schema('a'), str_schema('b'), float_schema('c'))->normalize(), JSON_THROW_ON_ERROR); - -$hydrator = new PhpRowHydrator(); -$narrowTyped = $hydrator->dehydrate(new Rows(schema(int_schema('a'), float_schema('c')), row(['a' => 7, 'c' => 2.5]))); - -$php = new PhpFloeEncoder(schema_from_json($wideBody)); -$ext = new RustFloeEncoderNative(); - -foreach (['php' => fn() => $php->encode($narrowTyped), 'ext' => fn() => $ext->encode($narrowTyped, $wideBody, schema_from_json($wideBody))] as $side => $encode) { - try { - $encode(); - printf("%-14s %s\n", 'absent-' . $side, 'NO EXCEPTION'); - } catch (Throwable $e) { - printf("%-14s %s\n", 'absent-' . $side, $e->getMessage()); - } -} - -$badSchema = schema(list_schema('bad', type_list(type_mixed()))); -$badTyped = $hydrator->dehydrate(new Rows($badSchema, row(['bad' => [new SplStack()]]))); -$badExt = new RustFloeEncoderNative(); -expect_exception(fn() => $badExt->encode($badTyped, json_encode($badSchema->normalize(), JSON_THROW_ON_ERROR), $badSchema)); -?> ---EXPECT-- -scalar frames-identical:yes -from_null frames-identical:yes -datetime frames-identical:yes -containers frames-identical:yes -enum_json_xml frames-identical:yes -heterogeneous frames-identical:yes -two_columns frames-identical:yes -empty frames-identical:yes -absent-php Floe found a row that does not carry the declared column "b" -absent-ext flow_php found a row that does not carry the declared column "b" -Flow\Floe\Exception\ExtensionException: flow_php does not support values of type "mixed" in this build diff --git a/src/extension/flow-php-ext/tests/phpt/021_batched_decode.phpt b/src/extension/flow-php-ext/tests/phpt/021_batched_decode.phpt deleted file mode 100644 index bfbc368da9..0000000000 --- a/src/extension/flow-php-ext/tests/phpt/021_batched_decode.phpt +++ /dev/null @@ -1,79 +0,0 @@ ---TEST-- -RustFloeEncoderNative + RowHydrator decode a batch identically to the pure-PHP pipeline ---SKIPIF-- - ---FILE-- - row(['id' => $i, 'name' => $i % 7 === 0 ? null : 'user_' . $i, 'price' => $i / 4.0]); - -$rows = rows( - schema(int_schema('id'), str_schema('name', nullable: true), float_schema('price')), - ...array_map($build, range(1, 500)), -); -$frames = php_frames($rows); - -$schemaBody = null; -$rowBodies = []; - -foreach ($frames as $frame) { - if ($frame['type'] === SCHEMA_ENTRY) { - $schemaBody = $frame['body']; - } else { - $rowBodies[] = $frame['body']; - } -} - -var_dump(count($rowBodies)); - -$decoder = new RustFloeEncoderNative(); -$hydrator = new PhpRowHydrator(); -$schema = schema_from_json($schemaBody); - -$decodedValues = $decoder->decode($rowBodies, $schemaBody); -var_dump($decodedValues[0] instanceof RawRowValues); - -$batched = $hydrator->hydrate($decodedValues, $schema); -var_dump($batched instanceof Rows); -var_dump($batched->count()); - -$php = php_decode_frames($frames); -$batchedRows = $batched->all(); - -$identical = true; - -foreach ($php as $i => $expected) { - if (serialize($expected) !== serialize($batchedRows[$i])) { - $identical = false; - echo "FAIL: batched row {$i} differs\n"; - } -} - -var_dump($identical); - -$empty = $hydrator->hydrate($decoder->decode([], $schemaBody), $schema); -var_dump($empty instanceof Rows); -var_dump($empty->count()); -?> ---EXPECT-- -int(500) -bool(true) -bool(true) -int(500) -bool(true) -bool(true) -int(0) diff --git a/src/extension/flow-php-ext/tests/phpt/022_row_encoder_round_trip.phpt b/src/extension/flow-php-ext/tests/phpt/022_row_encoder_round_trip.phpt deleted file mode 100644 index 52b3c9d17b..0000000000 --- a/src/extension/flow-php-ext/tests/phpt/022_row_encoder_round_trip.phpt +++ /dev/null @@ -1,79 +0,0 @@ ---TEST-- -RustFloeEncoderNative frame bodies round-trip through the RawRowValues pipeline ---SKIPIF-- - ---FILE-- - 1, 'name' => 'a', 'price' => 1.5, 'at' => new DateTimeImmutable('2025-01-01 10:00:00.5', new DateTimeZone('UTC')), 'u' => type_uuid()->cast('01234567-89ab-4def-8123-456789abcdef'), 'nums' => [1, 2, 3]]), - row(['id' => 2, 'name' => null, 'price' => -0.25, 'at' => new DateTimeImmutable('1999-12-31 23:59:59.999999', new DateTimeZone('Europe/Warsaw')), 'u' => type_uuid()->cast('abcdef01-2345-4678-9abc-def012345678'), 'nums' => []]), - row(['id' => 3, 'name' => 'c', 'price' => 99.0, 'at' => new DateTimeImmutable('2030-06-15 08:30:00', new DateTimeZone('America/New_York')), 'u' => type_uuid()->cast('11111111-2222-4333-8444-555566667777'), 'nums' => [-1, 0, PHP_INT_MAX]]), -]; - -$schemaBody = json_encode($schema->normalize(), JSON_THROW_ON_ERROR); -$hydrator = new PhpRowHydrator(); - -$encoder = new RustFloeEncoderNative(); -$bodies = $encoder->encode($hydrator->dehydrate(new Rows($schema, ...$sourceRows)), $schemaBody, $schema); - -$decoded = $hydrator->hydrate((new RustFloeEncoderNative())->decode($bodies, $schemaBody), schema_from_json($schemaBody)); - -var_dump($decoded instanceof Rows); -var_dump($decoded->count()); - -$reEncoder = new RustFloeEncoderNative(); -$reBodies = $reEncoder->encode($hydrator->dehydrate($decoded), $schemaBody, $schema); -var_dump($bodies === $reBodies); - -$frames = array_merge( - [['type' => SCHEMA_ENTRY, 'body' => $schemaBody]], - array_map(fn($body) => ['type' => Format::FRAME_ROW, 'body' => $body], $bodies), -); -assert_rows_identical(php_decode_frames($frames), $decoded->all()); - -var_dump($decoded->all()[1]->get('name')); -var_dump($decoded->all()[2]->get('nums')); -?> ---EXPECT-- -bool(true) -int(3) -bool(true) -identical -NULL -array(3) { - [0]=> - int(-1) - [1]=> - int(0) - [2]=> - int(9223372036854775807) -} diff --git a/src/extension/flow-php-ext/tests/phpt/023_encoder_datetime_subclass.phpt b/src/extension/flow-php-ext/tests/phpt/023_encoder_datetime_subclass.phpt deleted file mode 100644 index 663c2c60c4..0000000000 --- a/src/extension/flow-php-ext/tests/phpt/023_encoder_datetime_subclass.phpt +++ /dev/null @@ -1,31 +0,0 @@ ---TEST-- -RustFloeEncoderNative rejects DateTime subclasses with the Floe parity error ---SKIPIF-- - ---FILE-- -dehydrate( - new Rows($schema, row(['custom' => new PhptCustomDateTime('2020-05-05 10:20:30.000042', new DateTimeZone('Europe/Warsaw'))])), -); - -$encoder = new RustFloeEncoderNative(); - -expect_exception(fn() => $encoder->encode($typed, json_encode($schema->normalize(), JSON_THROW_ON_ERROR), $schema)); -?> ---EXPECT-- -Flow\Floe\Exception\ExtensionException: Floe supports only DateTime and DateTimeImmutable, got PhptCustomDateTime - convert custom datetime instances before writing diff --git a/src/extension/flow-php-ext/tests/phpt/024_row_hydrator_parity.phpt b/src/extension/flow-php-ext/tests/phpt/024_row_hydrator_parity.phpt deleted file mode 100644 index 13d29787ec..0000000000 --- a/src/extension/flow-php-ext/tests/phpt/024_row_hydrator_parity.phpt +++ /dev/null @@ -1,137 +0,0 @@ ---TEST-- -NativeRowHydrator hydrate/dehydrate are serialize-identical to PhpRowHydrator ---SKIPIF-- - ---FILE-- - [ - schema(int_schema('id'), str_schema('name', nullable: true), float_schema('p'), bool_schema('a')), - [ - new RawRowValues(['id' => 1, 'name' => 'x', 'p' => 1.5, 'a' => true]), - new RawRowValues(['id' => 2, 'name' => null, 'p' => -0.25, 'a' => false]), - ], - ], - 'null_nonnullable' => [ - schema(int_schema('id'), str_schema('name', nullable: true)), - [new RawRowValues(['id' => 1, 'name' => null]), new RawRowValues(['id' => 2, 'name' => null])], - ], - 'absent' => [ - schema(int_schema('id'), str_schema('name', nullable: true)), - [new RawRowValues(['id' => 1]), new RawRowValues(['id' => 2, 'name' => 'y'])], - ], - 'metadata' => [ - schema(int_schema('id')), - [ - new RawRowValues(['id' => 1], ['id' => Metadata::fromArray(['k' => 'v1'])]), - new RawRowValues(['id' => 2], ['id' => Metadata::fromArray(['k' => 'v2'])]), - ], - ], - 'numeric_name_metadata' => [ - schema(int_schema('7'), int_schema('id')), - [new RawRowValues(['7' => 1, 'id' => 1], ['7' => Metadata::fromArray(['k' => 'v']), 'id' => Metadata::fromArray(['n' => 1])])], - ], - 'temporal' => [ - schema(datetime_schema('at'), date_schema('d'), time_schema('t'), uuid_schema('u')), - [new RawRowValues([ - 'at' => new DateTimeImmutable('2025-01-01 12:00:00.123456', new DateTimeZone('Europe/Warsaw')), - 'd' => new DateTimeImmutable('2025-03-01'), - 't' => new DateInterval('PT2H30M5S'), - 'u' => new Flow\Types\Value\Uuid('01234567-89ab-4def-8123-456789abcdef'), - ])], - ], - 'containers' => [ - schema( - list_schema('l', type_list(type_integer())), - map_schema('m', type_map(type_string(), type_integer())), - structure_schema('st', type_structure(['a' => type_integer(), 'b' => structure_element('b', type_string(), optional: true)], true)), - list_schema('opt', type_list(type_optional(type_integer()))), - json_schema('j'), - ), - [new RawRowValues([ - 'l' => [1, 2, 3], - 'm' => ['a' => 1], - 'st' => ['a' => 1, 'extra' => 'kept'], - 'opt' => [1, null], - 'j' => Flow\Types\Value\Json::fromArray(['x' => 1]), - ])], - ], - 'enum_and_null' => [ - schema(enum_schema('s', PhptSuit::class, nullable: true), null_schema('n')), - [new RawRowValues(['s' => PhptSuit::Hearts, 'n' => null]), new RawRowValues(['s' => null, 'n' => null])], - ], - 'empty' => [schema(int_schema('id')), []], -]; - -$php = new PhpRowHydrator(); -$native = new NativeRowHydrator(); - -foreach ($datasets as $label => [$s, $batch]) { - $hydrateOk = serialize($php->hydrate($batch, $s)) === serialize($native->hydrate($batch, $s)); - $rows = $php->hydrate($batch, $s); - $dehydrateOk = serialize($php->dehydrate($rows)) === serialize($native->dehydrate($rows)); - printf("%-16s hydrate:%s dehydrate:%s\n", $label, $hydrateOk ? 'yes' : 'NO', $dehydrateOk ? 'yes' : 'NO'); -} - -$mutated = schema(int_schema('id')); -$php->hydrate([new RawRowValues(['id' => 1])], $mutated); -$native->hydrate([new RawRowValues(['id' => 1])], $mutated); -$mutated->add(str_schema('name', nullable: true))->makeNullable(); -// Schema is immutable, so $mutated never gained "name" - the column is simply not hydrated -$batch = [new RawRowValues(['id' => 1, 'name' => 'x'])]; -printf( - "%-16s hydrate:%s\n", - 'schema_mutation', - serialize($php->hydrate($batch, $mutated)) === serialize($native->hydrate($batch, $mutated)) ? 'yes' : 'NO', -); - -printf("native class registered:%s\n", class_exists(RustRowHydratorNative::class, false) ? 'yes' : 'NO'); -?> ---EXPECT-- -scalars hydrate:yes dehydrate:yes -null_nonnullable hydrate:yes dehydrate:yes -absent hydrate:yes dehydrate:yes -metadata hydrate:yes dehydrate:yes -numeric_name_metadata hydrate:yes dehydrate:yes -temporal hydrate:yes dehydrate:yes -containers hydrate:yes dehydrate:yes -enum_and_null hydrate:yes dehydrate:yes -empty hydrate:yes dehydrate:yes -schema_mutation hydrate:yes -native class registered:yes diff --git a/src/extension/flow-php-ext/tests/phpt/025_hydrator_no_leaks.phpt b/src/extension/flow-php-ext/tests/phpt/025_hydrator_no_leaks.phpt deleted file mode 100644 index 3079feea9f..0000000000 --- a/src/extension/flow-php-ext/tests/phpt/025_hydrator_no_leaks.phpt +++ /dev/null @@ -1,92 +0,0 @@ ---TEST-- -repeated native hydrate and dehydrate do not leak memory ---SKIPIF-- - ---FILE-- - type_integer(), 'tags' => type_list(type_string())])), - json_schema('json'), -); - -$batch = []; - -for ($i = 1; $i <= 100; $i++) { - $batch[] = new RawRowValues( - [ - 'id' => $i, - 'name' => $i % 10 === 0 ? null : 'user_' . $i, // null values under a nullable declaration - 'price' => (float) $i / 100, - 'active' => $i % 3 === 0, - 'created_at' => new DateTimeImmutable('2025-01-01 00:00:00.123456', new DateTimeZone('Europe/Warsaw')), - 'duration' => new DateInterval('PT1H2M3S'), - 'uuid' => new Flow\Types\Value\Uuid('01234567-89ab-4def-8123-456789abcdef'), - 'mixed' => [$i, 'x', null, ['k' => 1.5]], - 'metrics' => ['cpu' => 0.5, 'mem' => 0.25], - 'nested' => ['a' => $i, 'tags' => ['x']], - 'json' => Flow\Types\Value\Json::fromArray([$i, ['a' => true]]), - ], - $i % 5 === 0 ? ['id' => Metadata::fromArray(['batch' => $i])] : [], // exercises the clone + setMetadata path - ); -} - -$phpRows = (new PhpRowHydrator())->hydrate($batch, $schema); - -$cycle = static function () use ($batch, $schema, $phpRows): void { - $native = new RustRowHydratorNative(); - $rows = $native->hydrate($batch, $schema); - $native->dehydrate($rows); - $native->dehydrate($phpRows); -}; - -for ($i = 0; $i < 10; $i++) { - $cycle(); -} -gc_collect_cycles(); -$baseline = memory_get_usage(false); - -for ($i = 0; $i < 100; $i++) { - $cycle(); -} -gc_collect_cycles(); - -var_dump(memory_get_usage(false) <= $baseline); -?> ---EXPECT-- -bool(true) diff --git a/src/extension/flow-php-ext/tests/phpt/026_row_hydrator_cast_parity.phpt b/src/extension/flow-php-ext/tests/phpt/026_row_hydrator_cast_parity.phpt deleted file mode 100644 index a466225608..0000000000 --- a/src/extension/flow-php-ext/tests/phpt/026_row_hydrator_cast_parity.phpt +++ /dev/null @@ -1,341 +0,0 @@ ---TEST-- -NativeRowHydrator cast is serialize-identical to PhpRowHydrator ---SKIPIF-- - ---FILE-- - [ - schema(int_schema('id'), float_schema('p'), bool_schema('a'), str_schema('n', nullable: true)), - [ - new RawRowValues(['id' => '42', 'p' => '3.14', 'a' => 'yes', 'n' => 7]), - new RawRowValues(['id' => ' 7', 'p' => '1e3', 'a' => 'OFF', 'n' => 1.5]), - new RawRowValues(['id' => '007', 'p' => '.5', 'a' => 'ON', 'n' => true]), - new RawRowValues(['id' => '9223372036854775807', 'p' => true, 'a' => 3.5, 'n' => null]), - new RawRowValues(['id' => 5, 'p' => 2.5, 'a' => true, 'n' => 'text']), - ], - ], - 'nested_families' => [ - schema( - list_schema('counts', type_list(type_positive_integer())), - structure_schema('labels', type_structure(['label' => type_non_empty_string()])), - structure_schema('codes', type_structure(['code' => type_numeric_string()])), - ), - [ - new RawRowValues(['counts' => [5, 7], 'labels' => ['label' => 'x'], 'codes' => ['code' => '42']]), - new RawRowValues(['counts' => ['5', 8], 'labels' => ['label' => 9], 'codes' => ['code' => 42]]), - ], - ], - 'temporal' => [ - schema(datetime_schema('at'), datetime_schema('at2'), date_schema('d')), - [ - new RawRowValues(['at' => $shared, 'at2' => $shared, 'd' => $shared]), - new RawRowValues(['at' => $shared, 'at2' => '2024-03-01 10:20:30', 'd' => new DateTimeImmutable('2025-03-01')]), - new RawRowValues(['at' => 1700000000, 'at2' => 1700000000.5, 'd' => '2024-03-05 08:30:00']), - ], - ], - 'uuid_json' => [ - schema(uuid_schema('u', nullable: true), json_schema('j')), - [ - new RawRowValues(['u' => '01234567-89ab-4def-8123-456789abcdef', 'j' => '["a","b"]']), - new RawRowValues(['u' => new Flow\Types\Value\Uuid('01234567-89ab-4def-8123-456789abcdef'), 'j' => '{"a":1}']), - new RawRowValues(['u' => null, 'j' => ['a' => 1]]), - new RawRowValues(['u' => '11234567-89ab-4def-8123-456789abcdef', 'j' => Flow\Types\Value\Json::fromArray(['x' => 1])]), - ], - ], - 'containers' => [ - schema( - list_schema('l', type_list(type_integer())), - map_schema('m', type_map(type_string(), type_integer())), - map_schema('mi', type_map(type_integer(), type_string())), - list_schema('lo', type_list(type_optional(type_integer()))), - list_schema('lf', type_list(type_float())), - structure_schema('st', type_structure(['a' => type_integer(), 'b' => structure_element('b', type_string(), optional: true)], true)), - ), - [ - new RawRowValues([ - 'l' => ['1', 2, '3'], - 'm' => ['a' => '1', 'b' => 2], - 'mi' => [0 => 'x', 5 => 7], - 'lo' => ['1', null, 3], - 'lf' => [33, 65.5], - 'st' => ['a' => '5', 'extra' => 'dropped'], - ]), - new RawRowValues(['l' => [], 'm' => [], 'mi' => [], 'lo' => [], 'lf' => [], 'st' => ['a' => 1, 'b' => 'kept']]), - ], - ], - 'exotic_fallback' => [ - schema(enum_schema('s', CastSuit::class), xml_schema('x'), time_schema('t')), - [ - new RawRowValues(['s' => CastSuit::Hearts, 'x' => 'v', 't' => new DateInterval('PT2H30M5S')]), - new RawRowValues(['s' => 'h', 'x' => '', 't' => 'PT2H']), - ], - ], - 'fill_and_metadata' => [ - schema(int_schema('id', nullable: true), str_schema('name', nullable: true), bool_schema('flag', nullable: true)), - [ - new RawRowValues(['id' => '1'], ['id' => Metadata::fromArray(['k' => 'v1'])]), - new RawRowValues([]), - new RawRowValues(['id' => 2, 'name' => null, 'unknown' => 'dropped', 'flag' => 'on']), - new RawRowValues(['id' => null], ['id' => Metadata::fromArray(['k' => 'v2'])]), - ], - ], - 'all_optional_st' => [ - schema(structure_schema('st', type_structure(['b' => structure_element('b', type_string(), optional: true)]))), - [new RawRowValues(['st' => ['other' => 1]])], - ], - 'interleaved_st' => [ - schema(structure_schema('st', type_structure(['z' => type_integer(), 'a' => structure_element('a', type_string(), optional: true), 'b' => type_string()]))), - [ - new RawRowValues(['st' => ['b' => 'x', 'z' => '5']]), - new RawRowValues(['st' => ['z' => 1, 'a' => 'present', 'b' => 'y']]), - ], - ], - 'empty' => [schema(int_schema('id')), []], - // every column present in every row: the batch skips Row::conform(), which would return each row unchanged - 'all_present' => [ - schema(int_schema('id'), str_schema('name', nullable: true), bool_schema('a'), float_schema('p', nullable: true)), - [ - new RawRowValues(['id' => '1', 'name' => 'x', 'a' => 'yes', 'p' => '1.5']), - new RawRowValues(['id' => '2', 'name' => null, 'a' => 'no', 'p' => null]), - new RawRowValues(['p' => 3, 'a' => true, 'name' => 'z', 'id' => 3]), - ], - ], - 'absent_nullable' => [ - schema(int_schema('id'), str_schema('name', nullable: true)), - [ - new RawRowValues(['id' => 1, 'name' => 'a']), - new RawRowValues(['id' => 2, 'name' => 'b']), - new RawRowValues(['id' => 3, 'name' => 'c']), - new RawRowValues(['id' => 4]), - new RawRowValues(['id' => 5, 'name' => 'e']), - ], - ], - 'numeric_names' => [ - schema(int_schema('id'), str_schema('7'), int_schema('10', nullable: true)), - [ - new RawRowValues(['id' => '1', '7' => 'seven', '10' => '10']), - new RawRowValues(['10' => null, '7' => 'eight', 'id' => 2]), - ], - ], -]; - -$php = new PhpRowHydrator(); -$native = new NativeRowHydrator(); - -foreach ($datasets as $label => [$s, $batch]) { - $hydrateOk = serialize($php->hydrate($batch, $s)) === serialize($native->hydrate($batch, $s)); - printf("%-16s hydrate:%s\n", $label, $hydrateOk ? 'yes' : 'NO'); -} - -// TypeDetector types [33, 65.5] as list; both hydrators must materialize it as floats. -$promotion = schema(list_schema('lf', type_list(type_float()))); -$promotionBatch = [new RawRowValues(['lf' => [33, 65.5]])]; -printf( - "%-16s php:%s native:%s\n", - 'list_promotion', - json_encode($php->hydrate($promotionBatch, $promotion)->first()->get('lf'), JSON_PRESERVE_ZERO_FRACTION), - json_encode($native->hydrate($promotionBatch, $promotion)->first()->get('lf'), JSON_PRESERVE_ZERO_FRACTION), -); - -$mutated = schema(int_schema('id')); -$php->hydrate([new RawRowValues(['id' => '1'])], $mutated); -$native->hydrate([new RawRowValues(['id' => '1'])], $mutated); -$mutated->add(str_schema('name', nullable: true))->makeNullable(); -// Schema is immutable, so $mutated never gained "name" - the column is simply not cast -$batch = [new RawRowValues(['id' => '1', 'name' => 7])]; -printf( - "%-16s hydrate:%s\n", - 'schema_mutation', - serialize($php->hydrate($batch, $mutated)) === serialize($native->hydrate($batch, $mutated)) ? 'yes' : 'NO', -); - -// row 3 lacks a NOT NULL column and row 5 is refused by its cast: the cast refusal is raised while the batch is -// built, before Rows::conformed() could report the absence, so both paths name row 5 -$refused = schema(int_schema('id'), int_schema('n')); -$refusedBatch = [ - new RawRowValues(['id' => 0, 'n' => 0]), - new RawRowValues(['id' => 1, 'n' => 1]), - new RawRowValues(['id' => 2, 'n' => 2]), - new RawRowValues(['id' => 3]), - new RawRowValues(['id' => 4, 'n' => 4]), - new RawRowValues(['id' => 5, 'n' => 'five']), -]; -$refusals = []; - -foreach (['php' => $php, 'native' => $native] as $side => $hydrator) { - try { - $hydrator->hydrate($refusedBatch, $refused); - $refusals[$side] = 'none'; - } catch (Throwable $e) { - $refusals[$side] = $e::class . ': ' . $e->getMessage(); - } -} - -printf("%-16s identical:%s\n", 'absent_then_refused', $refusals['php'] === $refusals['native'] ? 'yes' : 'NO'); -echo $refusals['native'], "\n"; - -// a value held by reference is cast by value: the hydrated row must not change when the caller's variable does -$referenced = [ - 'ref_json' => [schema(json_schema('c')), '{"a":1}', static fn(&$v): array => ['c' => &$v]], - 'ref_uuid' => [schema(uuid_schema('c')), '01234567-89ab-4def-8123-456789abcdef', static fn(&$v): array => ['c' => &$v]], - 'ref_null' => [schema(str_schema('c', nullable: true)), null, static fn(&$v): array => ['c' => &$v]], - 'ref_datetime' => [schema(datetime_schema('c')), new DateTimeImmutable('2020-01-01 00:00:00'), static fn(&$v): array => ['c' => &$v]], - 'ref_list_item' => [schema(list_schema('c', type_list(type_positive_integer()))), 5, static fn(&$v): array => ['c' => [&$v]]], - 'ref_st_element' => [ - schema(structure_schema('c', type_structure(['s' => type_non_empty_string()]))), - 'text', - static fn(&$v): array => ['c' => ['s' => &$v]], - ], -]; - -foreach ($referenced as $label => [$s, $value, $wrap]) { - $hydrated = []; - - foreach (['php' => $php, 'native' => $native] as $side => $hydrator) { - $v = $value; - $rows = $hydrator->hydrate([new RawRowValues($wrap($v))], $s); - $v = 'mutated'; - $hydrated[$side] = serialize($rows); - } - - printf("%-16s hydrate:%s\n", $label, $hydrated['php'] === $hydrated['native'] ? 'yes' : 'NO'); -} - -$isoStrings = [ - '2026-07-13T10:20:30+00:00', - "2026-07-13T10:20:30+00:00\n", - "2026-07-13T10:20:30+00:00\n\n", - '2026-07-13T10:20:30Z', - '2026-07-13T10:20:30+05', - '2026-07-13T10:20:30+0530', - '2026-07-13T10:20:30+05:30', - '2026-07-13T10:20:30-12:00', - '2026-07-13 10:20:30', - '2026-07-13T10:20', - '2026-07-13T10:20:30.1Z', - '2026-07-13T10:20:30.12Z', - '2026-07-13T10:20:30.123Z', - '2026-07-13T10:20:30.1234Z', - '2026-07-13T10:20:30.12345Z', - '2026-07-13T10:20:30.123456Z', - '2026-07-13T10:20:30.1234567Z', - '2026-07-13T10:20:30.12345678Z', - '2026-07-13T10:20:30.123456789Z', - '2026-07-13T10:20:30.1234567890Z', - '2026-02-30T00:00:00Z', - '2024-02-29T00:00:00Z', - '2023-02-29T00:00:00Z', - '0000-01-01T00:00:00Z', - '2026-07-13T25:99:99Z', - '2026-07-13T10:20:30Z', - 'now', -]; - -// the gate's only outcome-changing check is checkdate(): PHP throws on an impossible day, the constructor rolls it over -foreach (['0000', '0001', '1900', '2000', '2023', '2024', '2100'] as $year) { - for ($month = 0; $month <= 13; $month++) { - for ($day = 0; $day <= 32; $day++) { - $isoStrings[] = sprintf('%s-%02d-%02dT00:00:00Z', $year, $month, $day); - } - } -} - -$isoSchema = schema(datetime_schema('at')); - -foreach (['UTC', 'Europe/Warsaw'] as $zone) { - ini_set('date.timezone', $zone); - $mismatches = 0; - - foreach ($isoStrings as $isoString) { - $results = []; - - foreach (['php' => $php, 'native' => $native] as $side => $hydrator) { - try { - $at = $hydrator->hydrate([new RawRowValues(['at' => $isoString])], $isoSchema)->first()->get('at'); - $results[$side] = $at::class . ' ' . $at->format('Y-m-d\TH:i:s.uP') . ' ' . $at->getTimezone()->getName(); - } catch (Throwable $e) { - $results[$side] = $e::class . ': ' . $e->getMessage(); - } - } - - if ($results['php'] !== $results['native']) { - $mismatches++; - echo ' ', var_export($isoString, true), ": php[{$results['php']}] native[{$results['native']}]\n"; - } - } - - printf("iso datetime %-13s %d strings, %d mismatches\n", $zone, count($isoStrings), $mismatches); -} - -printf("native class registered:%s\n", class_exists(RustRowHydratorNative::class, false) ? 'yes' : 'NO'); -?> ---EXPECT-- -scalars hydrate:yes -nested_families hydrate:yes -temporal hydrate:yes -uuid_json hydrate:yes -containers hydrate:yes -exotic_fallback hydrate:yes -fill_and_metadata hydrate:yes -all_optional_st hydrate:yes -interleaved_st hydrate:yes -empty hydrate:yes -all_present hydrate:yes -absent_nullable hydrate:yes -numeric_names hydrate:yes -list_promotion php:[33.0,65.5] native:[33.0,65.5] -schema_mutation hydrate:yes -absent_then_refused identical:yes -Flow\ETL\Exception\SchemaMismatchException: Rows do not match their schema: column "n" (row 5): could not convert 'five' (string) to integer -ref_json hydrate:yes -ref_uuid hydrate:yes -ref_null hydrate:yes -ref_datetime hydrate:yes -ref_list_item hydrate:yes -ref_st_element hydrate:yes -iso datetime UTC 3261 strings, 0 mismatches -iso datetime Europe/Warsaw 3261 strings, 0 mismatches -native class registered:yes diff --git a/src/extension/flow-php-ext/tests/phpt/027_cast_fallback_exceptions.phpt b/src/extension/flow-php-ext/tests/phpt/027_cast_fallback_exceptions.phpt deleted file mode 100644 index 94751cec92..0000000000 --- a/src/extension/flow-php-ext/tests/phpt/027_cast_fallback_exceptions.phpt +++ /dev/null @@ -1,133 +0,0 @@ ---TEST-- -native cast propagates the exact PHP cast exceptions and aborts the batch ---SKIPIF-- - ---FILE-- - [schema(int_schema('id')), [new RawRowValues(['id' => 'abc'])]], - 'float from a hex string' => [schema(float_schema('p')), [new RawRowValues(['p' => '0x1A'])]], - 'boolean from an unrecognised word' => [schema(bool_schema('a')), [new RawRowValues(['a' => 'weird'])]], - 'integer from an array' => [schema(int_schema('id')), [new RawRowValues(['id' => [1, 2, 3]])]], - 'uuid invalid' => [schema(uuid_schema('u')), [new RawRowValues(['u' => 'not-a-uuid'])]], - 'uuid uppercase' => [schema(uuid_schema('u')), [new RawRowValues(['u' => '01234567-89AB-4DEF-8123-456789ABCDEF'])]], - 'json scalar' => [schema(json_schema('j')), [new RawRowValues(['j' => 5])]], - 'json invalid string' => [schema(json_schema('j')), [new RawRowValues(['j' => '{oops'])]], - 'json plain string' => [schema(json_schema('j')), [new RawRowValues(['j' => 'plain'])]], - 'datetime garbage' => [schema(datetime_schema('at')), [new RawRowValues(['at' => 'not-a-date'])]], - 'datetime array' => [schema(datetime_schema('at')), [new RawRowValues(['at' => ['nope']])]], - 'date garbage' => [schema(date_schema('d')), [new RawRowValues(['d' => 'not-a-date'])]], - 'string map int keys' => [ - schema(map_schema('m', type_map(type_string(), type_integer()))), - [new RawRowValues(['m' => [5 => 1]])], - ], - 'list bad keys' => [ - schema(list_schema('l', type_list(type_integer()))), - [new RawRowValues(['l' => [1 => 'x']])], - ], - 'positive int list string' => [ - schema(list_schema('l', type_list(type_positive_integer()))), - [new RawRowValues(['l' => ['abc']])], - ], - 'positive int list negative' => [ - schema(list_schema('l', type_list(type_positive_integer()))), - [new RawRowValues(['l' => [-3]])], - ], - 'row index is 1' => [ - schema(int_schema('i')), - [new RawRowValues(['i' => '1']), new RawRowValues(['i' => 'x'])], - ], - 'integer overflow' => [schema(int_schema('id')), [new RawRowValues(['id' => '9223372036854775808'])]], - 'date from an empty string' => [schema(date_schema('d')), [new RawRowValues(['d' => ''])]], - 'datetime from a relative word' => [schema(datetime_schema('at')), [new RawRowValues(['at' => 'now'])]], - 'string from null' => [ - schema(list_schema('l', type_list(type_string()))), - [new RawRowValues(['l' => ['a', null]])], - ], - 'list from a scalar' => [ - schema(list_schema('l', type_list(type_integer()))), - [new RawRowValues(['l' => 5])], - ], - 'exception outside the types package' => [ - schema(new ForeignTypeDefinition('a', new ThrowingType(new LogicException('stub type refuses everything')))), - [new RawRowValues(['a' => [1, 2]])], - ], -]; - -$php = new PhpRowHydrator(); -$native = new NativeRowHydrator(); - -foreach ($throwing as $label => [$s, $batch]) { - $phpException = null; - - try { - $php->hydrate($batch, $s); - } catch (Throwable $e) { - $phpException = $e::class . '|' . $e->getMessage(); - } - - $nativeException = null; - $nativeResult = null; - - try { - $nativeResult = $native->hydrate($batch, $s); - } catch (Throwable $e) { - $nativeException = $e::class . '|' . $e->getMessage(); - } - - printf( - "%-27s exception:%s aborted:%s\n", - $label, - $phpException !== null && $phpException === $nativeException ? 'match' : "MISMATCH php[{$phpException}] native[{$nativeException}]", - $nativeResult === null ? 'yes' : 'NO', - ); -} -?> ---EXPECT-- -integer from a non numeric string exception:match aborted:yes -float from a hex string exception:match aborted:yes -boolean from an unrecognised word exception:match aborted:yes -integer from an array exception:match aborted:yes -uuid invalid exception:match aborted:yes -uuid uppercase exception:match aborted:yes -json scalar exception:match aborted:yes -json invalid string exception:match aborted:yes -json plain string exception:match aborted:yes -datetime garbage exception:match aborted:yes -datetime array exception:match aborted:yes -date garbage exception:match aborted:yes -string map int keys exception:match aborted:yes -list bad keys exception:match aborted:yes -positive int list string exception:match aborted:yes -positive int list negative exception:match aborted:yes -row index is 1 exception:match aborted:yes -integer overflow exception:match aborted:yes -date from an empty string exception:match aborted:yes -datetime from a relative word exception:match aborted:yes -string from null exception:match aborted:yes -list from a scalar exception:match aborted:yes -exception outside the types package exception:match aborted:yes diff --git a/src/extension/flow-php-ext/tests/phpt/028_cast_no_leaks.phpt b/src/extension/flow-php-ext/tests/phpt/028_cast_no_leaks.phpt deleted file mode 100644 index 0d54410bdc..0000000000 --- a/src/extension/flow-php-ext/tests/phpt/028_cast_no_leaks.phpt +++ /dev/null @@ -1,129 +0,0 @@ ---TEST-- -repeated native cast does not leak memory, including the throwing fallback path ---SKIPIF-- - ---FILE-- - type_integer(), 'b' => structure_element('b', type_string(), optional: true)])), - list_schema('names', type_list(type_string())), - json_schema('json'), - enum_schema('suit', LeakSuit::class), // exotic - per-value PHP fallback -); - -$batch = []; - -for ($i = 1; $i <= 100; $i++) { - $batch[] = new RawRowValues( - [ - 'id' => (string) $i, - 'name' => $i % 10 === 0 ? null : 'user_' . $i, // null values under a nullable declaration - 'price' => \sprintf('%d.%02d', $i, $i % 100), - 'active' => $i % 2 === 0 ? 'yes' : 'off', - 'created_at' => \sprintf('2024-03-%02d 10:20:%02d', 1 + $i % 28, $i % 60), - 'day' => \sprintf('2024-03-%02d', 1 + $i % 28), - 'uuid' => \sprintf('550e8400-e29b-41d4-a716-%012d', $i), - 'ints' => [(string) $i, null, $i], - 'metrics' => ['cpu' => (string) $i, 'mem' => $i], - 'nested' => ['a' => (string) $i, 'b' => $i], - 'names' => ['a', 'b'], - 'json' => '["a","b"]', - 'suit' => 'h', - ], - $i % 5 === 0 ? ['id' => Metadata::fromArray(['batch' => $i])] : [], // exercises the clone + setMetadata path - ); -} - -// a container column carrying a null element is the only way the deleted CastKind::String null arm -// is reachable, so the leak check has to exercise the List/Map recursion too -$throwingBatch = [ - new RawRowValues(['id' => 1]), - new RawRowValues(['uuid' => 'not-a-uuid', 'id' => 2, 'names' => ['a', null]]), -]; - -// an absent NOT-NULL column aborts before any row_values insertion - a different leak path -$missingBatch = [new RawRowValues(['name' => 'no id here'])]; - -$jsonSchema = schema(json_schema('json')); - -$cycle = static function () use ($batch, $throwingBatch, $missingBatch, $schema, $jsonSchema): void { - $native = new RustRowHydratorNative(); - $native->hydrate($batch, $schema); - - foreach (json_leak_cells() as $cell) { - try { - $native->hydrate([new RawRowValues(['json' => $cell])], $jsonSchema); - } catch (Throwable) { - // a refused cell aborts its batch; nothing it touched may leak - } - } - - try { - $native->hydrate($throwingBatch, $schema); - } catch (Throwable) { - // the aborted batch must not leak its partially built rows - } - - try { - $native->hydrate($missingBatch, $schema); - } catch (Throwable) { - // the same, aborted before the first insertion - } -}; - -for ($i = 0; $i < 10; $i++) { - $cycle(); -} -gc_collect_cycles(); -$baseline = memory_get_usage(false); - -for ($i = 0; $i < 100; $i++) { - $cycle(); -} -gc_collect_cycles(); - -var_dump(memory_get_usage(false) <= $baseline); -?> ---EXPECT-- -bool(true) diff --git a/src/extension/flow-php-ext/tests/phpt/030_encode_typed_object_class_check.phpt b/src/extension/flow-php-ext/tests/phpt/030_encode_typed_object_class_check.phpt deleted file mode 100644 index 1141281501..0000000000 --- a/src/extension/flow-php-ext/tests/phpt/030_encode_typed_object_class_check.phpt +++ /dev/null @@ -1,45 +0,0 @@ ---TEST-- -Typed uuid/json columns reject a foreign object instead of reading its memory layout ---SKIPIF-- - ---FILE-- - 'not-an-object-of-that-class'], ['v' => $columnType])]; - - expect_exception(fn() => $encoder->encode($typed, json_encode($columnSchema->normalize(), JSON_THROW_ON_ERROR), $columnSchema)); -} - -$typedObject = (new PhpRowHydrator())->dehydrate( - new Rows(schema(datetime_schema('v')), row(['v' => new DateTimeImmutable('2020-01-01 00:00:00 UTC')])), -); - -$uuidSchema = schema(uuid_schema('v')); -expect_exception(fn() => $encoder->encode($typedObject, json_encode($uuidSchema->normalize(), JSON_THROW_ON_ERROR), $uuidSchema)); -?> ---EXPECT-- -Flow\Floe\Exception\ExtensionException: flow_php expected a uuid value to be an object -Flow\Floe\Exception\ExtensionException: flow_php expected a json value to be an object -Flow\Floe\Exception\ExtensionException: flow_php expected a uuid value to be an object diff --git a/src/extension/flow-php-ext/tests/phpt/031_structure_v2_guards.phpt b/src/extension/flow-php-ext/tests/phpt/031_structure_v2_guards.phpt deleted file mode 100644 index 179978b43b..0000000000 --- a/src/extension/flow-php-ext/tests/phpt/031_structure_v2_guards.phpt +++ /dev/null @@ -1,51 +0,0 @@ ---TEST-- -structure schema-shape skew guards: the legacy two-bucket shape and an empty fields list are rejected loudly ---SKIPIF-- - ---FILE-- -dehydrate(new Rows($schema, row(['id' => 1]))); - -// direction 2 of the skew guard: a new binary reading the legacy two-bucket shape -// has no "structure" arm and rejects the unknown tag. -$legacy = json_encode([[ - 'ref' => 'st', - 'type' => [ - 'type' => 'structure', - 'elements' => ['a' => ['type' => 'integer']], - 'optional_elements' => [], - 'allow_extra' => false, - ], - 'nullable' => false, - 'metadata' => [], -]], JSON_THROW_ON_ERROR); - -expect_exception(fn() => (new RustFloeEncoderNative())->encode($typed, $legacy, $schema)); - -// the belt: a structure_v2 whose fields deserialized empty for any other reason. -$emptyFields = json_encode([[ - 'ref' => 'st', - 'type' => [ - 'type' => 'structure_v2', - 'fields' => [], - ], - 'nullable' => false, - 'metadata' => [], -]], JSON_THROW_ON_ERROR); - -expect_exception(fn() => (new RustFloeEncoderNative())->encode($typed, $emptyFields, $schema)); -?> ---EXPECT-- -Flow\Floe\Exception\ExtensionException: flow_php does not support values of type "structure" in this build -Flow\Floe\Exception\ExtensionException: flow_php read a structure type with no fields; the loaded flow_php extension and the flow-php/etl in use disagree on the structure schema format - reinstall one to match the other, or set the Floe engine to FloeEngine::php diff --git a/src/extension/flow-php-ext/tests/phpt/032_structure_numeric_name_parity.phpt b/src/extension/flow-php-ext/tests/phpt/032_structure_numeric_name_parity.phpt deleted file mode 100644 index 92f60db288..0000000000 --- a/src/extension/flow-php-ext/tests/phpt/032_structure_numeric_name_parity.phpt +++ /dev/null @@ -1,43 +0,0 @@ ---TEST-- -a numeric element name ("0" on the wire, always a JSON string) keeps its DeclaredKey::Index encoding and its cast fallback ---SKIPIF-- - ---FILE-- - type_integer(), 'b' => type_string()]); - -$rows = rows(schema(structure_schema('st', $numericNamed)), row(['st' => [0 => 1, 'b' => 'x']])); - -printf("frames-identical:%s\n", php_frames($rows) === ext_frames($rows) ? 'yes' : 'NO'); - -$frames = php_frames($rows); -assert_rows_identical(php_decode_frames($frames), ext_decode_frames($frames)); - -// build_structure_kind still bails to CastKind::Fallback for a numeric name - -// parity with the pure-PHP hydrator proves it took the same code path. -$schema = schema(structure_schema('st', $numericNamed)); -$batch = [new RawRowValues(['st' => [0 => '1', 'b' => 2]])]; - -printf( - "cast-parity:%s\n", - serialize((new PhpRowHydrator())->hydrate($batch, $schema)) === serialize((new NativeRowHydrator())->hydrate($batch, $schema)) ? 'yes' : 'NO', -); -?> ---EXPECT-- -frames-identical:yes -identical -cast-parity:yes diff --git a/src/extension/flow-php-ext/tests/phpt/034_floe_encoder_nullability.phpt b/src/extension/flow-php-ext/tests/phpt/034_floe_encoder_nullability.phpt deleted file mode 100644 index f6caa49c57..0000000000 --- a/src/extension/flow-php-ext/tests/phpt/034_floe_encoder_nullability.phpt +++ /dev/null @@ -1,60 +0,0 @@ ---TEST-- -both Floe encoders refuse a null on a NOT-NULL column with the same class and message ---SKIPIF-- - ---FILE-- - 1, 'name' => 'a', 'note' => null], ['id' => type_integer(), 'name' => type_string(), 'note' => type_string()]), - new TypedRowValues(['id' => 2, 'name' => null, 'note' => null], ['id' => type_integer(), 'name' => type_string(), 'note' => type_string()]), -]; - -$php = new PhpFloeEncoder($schema); -$native = new NativeFloeEncoder($schema); - -$describe = static function (callable $encode): string { - try { - $encode(); - - return 'NO EXCEPTION'; - } catch (Throwable $e) { - return $e::class . '|' . $e->getMessage(); - } -}; - -$phpRefusal = $describe(static fn() => $php->encode($refused)); -$nativeRefusal = $describe(static fn() => $native->encode($refused)); - -printf("refusal:%s\n", $phpRefusal === $nativeRefusal ? 'match' : "MISMATCH php[{$phpRefusal}] native[{$nativeRefusal}]"); -printf("%s\n", $phpRefusal); - -// a null under a nullable declaration still encodes, on both sides, to the same bytes -$allowed = (new PhpRowHydrator())->dehydrate( - new Rows($schema, row(['id' => 1, 'name' => 'a', 'note' => null])), -); - -printf("nullable:%s\n", $php->encode($allowed) === $native->encode($allowed) ? 'identical' : 'DIFFERENT'); -?> ---EXPECT-- -refusal:match -Flow\ETL\Exception\SchemaMismatchException|Rows do not match their schema: column "name" (row 1): could not convert null to string, column is not nullable -nullable:identical diff --git a/src/extension/flow-php-ext/tests/phpt/035_list_numeric_promotion.phpt b/src/extension/flow-php-ext/tests/phpt/035_list_numeric_promotion.phpt deleted file mode 100644 index 4882f7c04e..0000000000 --- a/src/extension/flow-php-ext/tests/phpt/035_list_numeric_promotion.phpt +++ /dev/null @@ -1,71 +0,0 @@ ---TEST-- -list promotes long elements and list keeps a null, identically in both hydrators ---SKIPIF-- - ---FILE-- - is a schema a -// schemaless source can produce. The cast door has to promote every long element to a double, -// and it has to do it the same way on both sides of the extension boundary. -$php = new PhpRowHydrator(); -$native = new NativeRowHydrator(); - -$cases = [ - 'longs' => [ - schema(list_schema('l', type_list(type_float()))), - [new RawRowValues(['l' => [33, 65.5]]), new RawRowValues(['l' => [1, 2, 3]])], - ], - 'nullable' => [ - schema(list_schema('l', type_list(type_optional(type_float())))), - [new RawRowValues(['l' => [1, null, 2.5]]), new RawRowValues(['l' => [null]])], - ], - 'numeric_strings' => [ - schema(list_schema('l', type_list(type_float()))), - [new RawRowValues(['l' => ['1', 2, '3.5']])], - ], - 'empty' => [ - schema(list_schema('l', type_list(type_float()))), - [new RawRowValues(['l' => []])], - ], -]; - -foreach ($cases as $label => [$s, $batch]) { - $phpRows = $php->hydrate($batch, $s); - $nativeRows = $native->hydrate($batch, $s); - - printf("%-16s parity:%s\n", $label, serialize($phpRows) === serialize($nativeRows) ? 'yes' : 'NO'); - - foreach ($phpRows->all() as $i => $row) { - printf( - "%-16s row%d php:%s native:%s\n", - $label, - $i, - json_encode($row->get('l'), JSON_PRESERVE_ZERO_FRACTION), - json_encode($nativeRows->all()[$i]->get('l'), JSON_PRESERVE_ZERO_FRACTION), - ); - } -} -?> ---EXPECT-- -longs parity:yes -longs row0 php:[33.0,65.5] native:[33.0,65.5] -longs row1 php:[1.0,2.0,3.0] native:[1.0,2.0,3.0] -nullable parity:yes -nullable row0 php:[1.0,null,2.5] native:[1.0,null,2.5] -nullable row1 php:[null] native:[null] -numeric_strings parity:yes -numeric_strings row0 php:[1.0,2.0,3.5] native:[1.0,2.0,3.5] -empty parity:yes -empty row0 php:[] native:[] diff --git a/src/extension/flow-php-ext/tests/phpt/036_extension_version_is_release_or_commit.phpt b/src/extension/flow-php-ext/tests/phpt/036_extension_version_is_release_or_commit.phpt index 9f3df11cf2..2ea6d4ffd5 100644 --- a/src/extension/flow-php-ext/tests/phpt/036_extension_version_is_release_or_commit.phpt +++ b/src/extension/flow-php-ext/tests/phpt/036_extension_version_is_release_or_commit.phpt @@ -1,10 +1,10 @@ --TEST-- -extension version is the release tag, or the last tag with commit distance and hash +extension version is the release tag, or the next minor's dev pre-release with commit distance and hash --SKIPIF-- --FILE-- --EXPECT-- int(1) diff --git a/src/extension/flow-php-ext/tests/phpt/040_csv_tokenizer_parity.phpt b/src/extension/flow-php-ext/tests/phpt/040_csv_tokenizer_parity.phpt index a12bdbdb8a..ed70e73138 100644 --- a/src/extension/flow-php-ext/tests/phpt/040_csv_tokenizer_parity.phpt +++ b/src/extension/flow-php-ext/tests/phpt/040_csv_tokenizer_parity.phpt @@ -6,8 +6,6 @@ native CSV fields match str_getcsv() byte for byte on every tokenizer edge case '612c622c63', 'quoted' => '2261222c226222', @@ -22,11 +20,7 @@ $cases = [ foreach ($cases as $case => $hex) { $input = hex2bin($hex); - $reader = new RustCSVReaderNative(',', '"', '\\', false, false, false); - $reader->feed($input . "\n"); - $reader->finish(); - - $fields = array_values($reader->next(10)[0]->values); + $fields = array_values(csv_native_rows($input . "\n", ',', '"', '\\', false, false, false)[1][0]); $expected = str_getcsv($input, ',', '"', '\\'); echo $case, ': ', $fields === $expected ? 'identical' : 'FAIL ' . json_encode(array_map(static fn(?string $f): ?string => $f === null ? null : bin2hex($f), $fields)), "\n"; diff --git a/src/extension/flow-php-ext/tests/phpt/044_csv_ragged_rows.phpt b/src/extension/flow-php-ext/tests/phpt/044_csv_ragged_rows.phpt index c447a601d0..32474af62a 100644 --- a/src/extension/flow-php-ext/tests/phpt/044_csv_ragged_rows.phpt +++ b/src/extension/flow-php-ext/tests/phpt/044_csv_ragged_rows.phpt @@ -1,5 +1,5 @@ --TEST-- -native CSV rows pad and truncate to the header like CSVRowNormalizer, under both emptyToNull settings +native CSV rows pad and truncate to the header like Records, under both emptyToNull settings --SKIPIF-- --FILE-- diff --git a/src/extension/flow-php-ext/tests/phpt/045_csv_no_leaks.phpt b/src/extension/flow-php-ext/tests/phpt/045_csv_no_leaks.phpt index b2d946369e..728f37fa9a 100644 --- a/src/extension/flow-php-ext/tests/phpt/045_csv_no_leaks.phpt +++ b/src/extension/flow-php-ext/tests/phpt/045_csv_no_leaks.phpt @@ -6,32 +6,27 @@ repeated native CSV reading does not leak memory feed($chunk); - $reader->next(2); + foreach ($source->records() as $record) { } - $reader->finish(); - $reader->headers(); + $source->headers(); - while ($reader->next(2) !== []) { - } - - $unfinished = new RustCSVReaderNative(';', "'", '', false, false, false); - $unfinished->feed("a;'open\n"); - $unfinished->next(10); + // abandoned after the first record + $unfinished = (new RustCSVOpenSource(new MemorySourceStream("a;'open\nb;c\n"), ';', "'", '', false, false, false, 4))->records(); + $unfinished->rewind(); try { - new RustCSVReaderNative(',,', '"', '\\', true, true, true); + new RustCSVOpenSource(new MemorySourceStream($raw), ',,', '"', '\\', true, true, true); throw new LogicException('an invalid separator was accepted'); - } catch (Flow\Floe\Exception\ExtensionException) { + } catch (Flow\ETL\Exception\RuntimeException) { } }; diff --git a/src/extension/flow-php-ext/tests/phpt/046_csv_narrow_parity.phpt b/src/extension/flow-php-ext/tests/phpt/046_csv_narrow_parity.phpt index 2f17aeaafc..ae99121d4d 100644 --- a/src/extension/flow-php-ext/tests/phpt/046_csv_narrow_parity.phpt +++ b/src/extension/flow-php-ext/tests/phpt/046_csv_narrow_parity.phpt @@ -6,8 +6,9 @@ native narrowing matches StringTypeNarrower::narrow() on its fixtures, the byte- narrowOne($value)); + printf("%-32s %s\n", $value, rust_sniff_column([$value], $typer, false)->schema($floor)->get('v')->type()->toString()); } assert_narrow_parity('temporal candidates', $candidates, $temporal); diff --git a/src/extension/flow-php-ext/tests/phpt/048_csv_widen_parity.phpt b/src/extension/flow-php-ext/tests/phpt/048_csv_widen_parity.phpt index 972de86d42..d8cb0b1ba4 100644 --- a/src/extension/flow-php-ext/tests/phpt/048_csv_widen_parity.phpt +++ b/src/extension/flow-php-ext/tests/phpt/048_csv_widen_parity.phpt @@ -6,9 +6,9 @@ the native fold widens every pair of CSV leaf types exactly like TypeWidener ['true', type_boolean()], 'timezone' => ['UTC', type_time_zone()], ]; -$candidates = array_map(static fn($type): string => $type->toString(), InferredTypes::default()->toArray()); +$typer = new StringTypeNarrower(InferredTypes::default()->toArray()); $widener = new TypeWidener(); $pairs = 0; $mismatches = 0; foreach ($leaves as $left => [$leftCell, $leftType]) { foreach ($leaves as $right => [$rightCell, $rightType]) { - $csv = fopen('php://memory', 'rb+'); - fputcsv($csv, ['c'], ',', '"', '\\'); - fputcsv($csv, [$leftCell ?? ''], ',', '"', '\\'); - fputcsv($csv, [$rightCell ?? ''], ',', '"', '\\'); - rewind($csv); - - $reader = new RustCSVReaderNative(',', '"', '\\', true, true, true); - $reader->feed((string) stream_get_contents($csv)); - $reader->finish(); - $fold = new RustColumnFoldNative([], $candidates); - $reader->fold($fold, -1); - - $expected = $widener->widen($leftType, $rightType)->toString(); - $actual = $fold->types()['c']; + $expected = ColumnTypes::fromColumnTypes(['v' => $widener->widen($leftType, $rightType)], 2, $typer); $pairs++; - if ($expected !== $actual) { + if ($expected != rust_sniff_column([$leftCell, $rightCell], $typer)) { $mismatches++; - echo "{$left} + {$right}: TypeWidener={$expected} native={$actual}\n"; + echo "{$left} + {$right}: TypeWidener=", $widener->widen($leftType, $rightType)->toString(), " native differs\n"; } } } diff --git a/src/extension/flow-php-ext/tests/phpt/049_csv_fold_no_leaks.phpt b/src/extension/flow-php-ext/tests/phpt/049_csv_fold_no_leaks.phpt index 7f7c292b4d..6fcd28dea6 100644 --- a/src/extension/flow-php-ext/tests/phpt/049_csv_fold_no_leaks.phpt +++ b/src/extension/flow-php-ext/tests/phpt/049_csv_fold_no_leaks.phpt @@ -6,28 +6,19 @@ repeated native CSV folding - including every PHP callback and a rejected time z $type->toString(), InferredTypes::default()->toArray()); +$typer = new StringTypeNarrower(InferredTypes::default()->toArray()); $raw = "id,json,at,zone,offset,dup,dup\n1,\"{\"\"a\"\":1}\",2024-01-01 10:00:00,Europe/Warsaw,+99:60,x,\n2,[1],2024-02-30,UTC,+02:00,,y\n"; -$cycle = static function () use ($candidates, $raw): void { - $reader = new RustCSVReaderNative(',', '"', '\\', true, true, true); - $reader->feed($raw); - $reader->finish(); - $fold = new RustColumnFoldNative(['id', 'json'], $candidates); - $reader->fold($fold, -1); - $fold->types(); - $fold->rows(); - $fold->narrowOne('+99:60'); - $fold->narrowOne('{"a":[1,2]}'); - $fold->narrowOne('2024-01-01'); - - foreach (json_leak_cells() as $cell) { - $fold->narrowOne($cell); - } +$cycle = static function () use ($typer, $raw): void { + (new RustCSVOpenSource(new MemorySourceStream($raw), ',', '"', '\\', true, true, true)) + ->sniff(['id', 'json'], -1, new SchemaInference(), $typer); + rust_sniff_column(['+99:60', '{"a":[1,2]}', '2024-01-01', ...json_leak_cells()], $typer); }; for ($i = 0; $i < 10; $i++) { diff --git a/src/extension/flow-php-ext/tests/phpt/051_csv_guards.phpt b/src/extension/flow-php-ext/tests/phpt/051_csv_guards.phpt index 9f97f6eb6e..24269f6bc8 100644 --- a/src/extension/flow-php-ext/tests/phpt/051_csv_guards.phpt +++ b/src/extension/flow-php-ext/tests/phpt/051_csv_guards.phpt @@ -1,30 +1,40 @@ --TEST-- -native CSV classes reject an invalid dialect, batch size, fold limit and HTML/XML candidates +RustCSVOpenSource rejects an invalid dialect, batch size and sniff row budget --SKIPIF-- --FILE-- new RustCSVReaderNative(',,', '"', '\\', true, true, true)); -expect_exception(static fn() => new RustCSVReaderNative(',', '', '\\', true, true, true)); -expect_exception(static fn() => new RustCSVReaderNative(',', '"', '\\\\', true, true, true)); -expect_exception(static fn() => (new RustCSVReaderNative(',', '"', '\\', true, true, true))->next(0)); -expect_exception(static fn() => (new RustCSVReaderNative(',', '"', '\\', true, true, true))->next(-1)); -$fold = new RustColumnFoldNative([], ['integer']); -expect_exception(static fn() => (new RustCSVReaderNative(',', '"', '\\', true, true, true))->fold($fold, -2)); -expect_exception(static fn() => new RustColumnFoldNative([], ['integer', 'html'])); -expect_exception(static fn() => new RustColumnFoldNative([], ['xml'])); +use function Flow\ETL\DSL\{int_schema, schema}; + +$source = static fn(string $separator = ',', string $enclosure = '"', string $escape = '\\'): RustCSVOpenSource => new RustCSVOpenSource( + new MemorySourceStream("id\n1\n"), + $separator, + $enclosure, + $escape, + true, + true, + true, +); + +expect_exception(static fn() => $source(',,')); +expect_exception(static fn() => $source(',', '')); +expect_exception(static fn() => $source(',', '"', '\\\\')); +expect_exception(static fn() => $source()->batches(schema(int_schema('id')), 0, new RustBackend())); +expect_exception(static fn() => $source()->batches(schema(int_schema('id')), -1, new RustBackend())); +expect_exception(static fn() => $source()->sniff(['id'], -2, new SchemaInference(), new StringTypeNarrower())); ?> --EXPECT-- -Flow\Floe\Exception\ExtensionException: flow_php CSV separator must be exactly one byte -Flow\Floe\Exception\ExtensionException: flow_php CSV enclosure must be exactly one byte -Flow\Floe\Exception\ExtensionException: flow_php CSV escape must be empty or exactly one byte -Flow\Floe\Exception\ExtensionException: flow_php CSV batch size must be greater than 0 -Flow\Floe\Exception\ExtensionException: flow_php CSV batch size must be greater than 0 -Flow\Floe\Exception\ExtensionException: flow_php CSV fold limit must be -1 or at least 0 -Flow\Floe\Exception\ExtensionException: flow_php cannot fold html or xml candidates natively -Flow\Floe\Exception\ExtensionException: flow_php cannot fold html or xml candidates natively +Flow\ETL\Exception\RuntimeException: flow_php CSV separator must be exactly one byte +Flow\ETL\Exception\RuntimeException: flow_php CSV enclosure must be exactly one byte +Flow\ETL\Exception\RuntimeException: flow_php CSV escape must be empty or exactly one byte +Flow\ETL\Exception\RuntimeException: flow_php CSV batch size must be greater than 0 +Flow\ETL\Exception\RuntimeException: flow_php CSV batch size must be greater than 0 +Flow\ETL\Exception\RuntimeException: flow_php CSV fold limit must be -1 or at least 0 diff --git a/src/extension/flow-php-ext/tests/phpt/052_json_cast_parity.phpt b/src/extension/flow-php-ext/tests/phpt/052_json_cast_parity.phpt deleted file mode 100644 index ae839d5e4d..0000000000 --- a/src/extension/flow-php-ext/tests/phpt/052_json_cast_parity.phpt +++ /dev/null @@ -1,71 +0,0 @@ ---TEST-- -native JSON cast matches PhpRowHydrator on the JSON parity cases and a deterministic fuzz corpus ---SKIPIF-- - ---FILE-- - substr($fuzzed, 0, $at) . $insert . substr($fuzzed, $at), - 1 => substr($fuzzed, 0, $at) . substr($fuzzed, $at + 1), - default => substr($fuzzed, 0, $at) . $insert . substr($fuzzed, $at + 1), - }; - } - } - - $cases["fuzz_{$i}"] = $fuzzed; -} - -$schema = schema(json_schema('j', nullable: false)); -$php = new PhpRowHydrator(); -$native = new NativeRowHydrator(); - -$outcome = static function (Flow\ETL\Row\Hydrator $hydrator, string $value) use ($schema): string { - try { - return 'json ' . $hydrator->hydrate([new RawRowValues(['j' => $value])], $schema)->first()->get('j')->toString(); - } catch (Throwable $e) { - return $e::class . ': ' . $e->getMessage(); - } -}; - -$mismatches = 0; - -foreach ($cases as $name => $value) { - if ($outcome($php, $value) !== $outcome($native, $value)) { - $mismatches++; - echo ' ', $name, ' ', bin2hex(substr($value, 0, 60)), "\n"; - } -} - -printf("json cast parity: %d cases, %d mismatches\n", count($cases), $mismatches); -?> ---EXPECT-- -json cast parity: 20054 cases, 0 mismatches diff --git a/src/extension/flow-php-ext/tests/phpt/054_csv_consumed_bytes.phpt b/src/extension/flow-php-ext/tests/phpt/054_csv_consumed_bytes.phpt index 5f2ac050a5..c8f0789b22 100644 --- a/src/extension/flow-php-ext/tests/phpt/054_csv_consumed_bytes.phpt +++ b/src/extension/flow-php-ext/tests/phpt/054_csv_consumed_bytes.phpt @@ -1,88 +1,8800 @@ ---TEST-- -native CSV consumedBytes() counts the bytes of the rows it produced exactly like the PHP path, over any chunk size ---SKIPIF-- - ---FILE-- -records() as $values) { - if (++$taken >= $rows) { - break; - } - } - - return $source->producedBytes(); -}; - -$native = static function (string $raw, bool $withHeader, int $rows, int $chunk): int { - $reader = new RustCSVReaderNative(',', '"', '\\', $withHeader, true, true); - $fold = new RustColumnFoldNative([], []); - $folded = 0; - - foreach (str_split($raw, $chunk) as $piece) { - $reader->feed($piece); - $folded += $reader->fold($fold, $rows - $folded); - } - - $reader->finish(); - $reader->fold($fold, $rows - $folded); - - return $reader->consumedBytes(); -}; - -$cases = [ - 'LF' => ["a,b\n1,2\n33,44\n", true, 2], - 'CRLF' => ["a,b\r\n1,2\r\n33,44\r\n", true, 2], - 'quoted multi-line LF' => ["a,b\n\"x\ny\",1\n2,3\n", true, 2], - 'quoted multi-line CRLF' => ["a,b\r\n\"x\r\ny\",1\r\n2,3\r\n", true, 2], - 'blank lines' => ["a,b\n\n1,2\n\n", true, 3], - 'BOM without header' => ["\xEF\xBB\xBF1,2\n3,4\n", false, 2], - 'first two of three rows' => ["a,b\n1,2\n33,44\n555,666\n", true, 2], -]; - -foreach ($cases as $label => [$raw, $withHeader, $rows]) { - $expected = $php($raw, $withHeader, $rows); - $counts = array_unique(array_map( - static fn(int $chunk): int => $native($raw, $withHeader, $rows, $chunk), - [1, 3, 7, 4096], - )); - - echo $label, ': ', $counts === [$expected] ? 'identical ' . $expected : 'FAIL php=' . $expected . ' native=' . implode(',', $counts), "\n"; -} - -$reader = new RustCSVReaderNative(',', '"', '\\', true, true, true); -$reader->feed("a,b\n1,2\n33,44\n"); -$reader->finish(); -$reader->next(1); -echo 'next() counts too: ', $reader->consumedBytes(), "\n"; -$reader->next(10); -echo 'header excluded: ', $reader->consumedBytes(), "\n"; -?> ---EXPECT-- -LF: identical 10 -CRLF: identical 12 -quoted multi-line LF: identical 12 -quoted multi-line CRLF: identical 15 -blank lines: identical 6 -BOM without header: identical 11 -first two of three rows: identical 10 -next() counts too: 4 -header excluded: 10 +$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +-$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +-$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +T$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +E$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +S$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +T$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +-$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +-$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; + +$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +n$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +a$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +t$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +i$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +v$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +e$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; + $source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +C$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +S$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +V$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; + $source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +c$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +o$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +n$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +s$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +u$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +m$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +e$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +d$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +B$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +y$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +t$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +e$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +s$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +($source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +)$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; + $source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +c$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +o$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +u$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +n$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +t$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +s$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; + $source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +t$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +h$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +e$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; + $source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +b$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +y$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +t$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +e$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +s$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; + $source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +o$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +f$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; + $source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +t$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +h$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +e$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; + $source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +r$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +o$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +w$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +s$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; + $source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +i$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +t$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; + $source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +p$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +r$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +o$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +d$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +u$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +c$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +e$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +d$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; + $source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +e$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +x$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +a$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +c$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +t$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +l$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +y$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; + $source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +l$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +i$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +k$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +e$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; + $source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +t$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +h$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +e$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; + $source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +P$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +H$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +P$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; + $source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +p$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +a$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +t$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +h$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +,$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; + $source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +o$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +v$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +e$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +r$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; + $source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +a$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +n$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +y$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; + $source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +c$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +h$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +u$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +n$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +k$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; + $source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +s$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +i$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +z$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +e$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; + +$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +-$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +-$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +S$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +K$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +I$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +P$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +I$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +F$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +-$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +-$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; + +$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +<$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +?$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +p$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +h$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +p$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; + $source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +i$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +f$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; + $source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +($source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +!$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +e$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +x$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +t$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +e$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +n$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +s$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +i$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +o$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +n$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +_$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +l$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +o$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +a$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +d$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +e$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +d$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +($source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +"$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +f$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +l$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +o$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +w$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +_$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +p$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +h$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +p$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +"$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +)$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +)$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; + $source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +d$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +i$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +e$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +($source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +"$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +s$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +k$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +i$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +p$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; + $source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +f$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +l$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +o$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +w$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +_$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +p$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +h$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +p$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; + $source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +e$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +x$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +t$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +e$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +n$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +s$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +i$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +o$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +n$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; + $source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +n$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +o$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +t$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; + $source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +l$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +o$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +a$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +d$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +e$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +d$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +"$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +)$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +;$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; + $source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +?$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +>$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; + +$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +-$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +-$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +F$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +I$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +L$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +E$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +-$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +-$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; + +$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +<$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +?$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +p$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +h$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +p$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; + +$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +r$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +e$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +q$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +u$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +i$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +r$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +e$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; + $source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +_$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +_$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +D$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +I$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +R$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +_$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +_$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; + $source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +.$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; + $source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +'$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +/$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +b$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +o$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +o$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +t$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +s$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +t$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +r$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +a$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +p$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +.$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +p$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +h$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +p$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +'$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +;$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; + +$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; + +$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +u$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +s$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +e$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; + $source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +F$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +l$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +o$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +w$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +\$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +E$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +T$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +L$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +\$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +A$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +d$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +a$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +p$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +t$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +e$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +r$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +\$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +C$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +S$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +V$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +\$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +C$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +S$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +V$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +D$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +e$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +c$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +o$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +d$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +e$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +r$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +;$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; + +$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +u$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +s$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +e$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; + $source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +F$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +l$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +o$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +w$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +\$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +E$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +T$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +L$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +\$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +A$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +d$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +a$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +p$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +t$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +e$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +r$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +\$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +C$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +S$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +V$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +\$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +C$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +S$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +V$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +L$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +i$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +n$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +e$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +R$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +e$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +a$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +d$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +e$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +r$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +;$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; + +$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +u$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +s$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +e$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; + $source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +F$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +l$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +o$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +w$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +\$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +E$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +T$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +L$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +\$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +A$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +d$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +a$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +p$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +t$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +e$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +r$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +\$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +C$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +S$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +V$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +\$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +P$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +h$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +p$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +C$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +S$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +V$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +O$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +p$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +e$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +n$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +S$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +o$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +u$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +r$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +c$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +e$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +;$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; + +$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +u$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +s$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +e$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; + $source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +F$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +l$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +o$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +w$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +\$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +E$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +T$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +L$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +\$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +A$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +d$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +a$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +p$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +t$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +e$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +r$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +\$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +C$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +S$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +V$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +\$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +R$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +u$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +s$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +t$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +C$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +S$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +V$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +O$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +p$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +e$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +n$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +S$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +o$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +u$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +r$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +c$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +e$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +;$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; + +$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +u$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +s$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +e$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; + $source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +F$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +l$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +o$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +w$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +\$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +E$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +T$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +L$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +\$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +S$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +c$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +h$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +e$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +m$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +a$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +\$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +I$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +n$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +f$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +e$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +r$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +e$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +n$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +c$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +e$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +\$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +I$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +n$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +f$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +e$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +r$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +r$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +e$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +d$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +T$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +y$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +p$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +e$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +s$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +;$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; + +$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +u$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +s$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +e$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; + $source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +F$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +l$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +o$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +w$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +\$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +E$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +T$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +L$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +\$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +S$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +c$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +h$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +e$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +m$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +a$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +\$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +I$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +n$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +f$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +e$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +r$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +e$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +n$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +c$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +e$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +\$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +S$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +c$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +h$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +e$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +m$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +a$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +I$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +n$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +f$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +e$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +r$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +e$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +n$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +c$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +e$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +;$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; + +$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +u$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +s$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +e$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; + $source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +F$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +l$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +o$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +w$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +\$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +F$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +i$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +l$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +e$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +s$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +y$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +s$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +t$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +e$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +m$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +\$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +S$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +t$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +r$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +e$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +a$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +m$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +\$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +M$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +e$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +m$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +o$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +r$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +y$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +S$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +o$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +u$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +r$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +c$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +e$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +S$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +t$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +r$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +e$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +a$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +m$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +;$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; + +$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +u$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +s$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +e$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; + $source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +F$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +l$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +o$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +w$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +\$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +T$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +y$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +p$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +e$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +s$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +\$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +T$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +y$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +p$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +e$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +\$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +N$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +a$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +t$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +i$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +v$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +e$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +\$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +S$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +t$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +r$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +i$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +n$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +g$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +\$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +S$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +t$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +r$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +i$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +n$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +g$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +T$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +y$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +p$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +e$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +N$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +a$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +r$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +r$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +o$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +w$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +e$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +r$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +;$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; + +$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +u$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +s$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +e$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; + $source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +F$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +l$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +o$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +w$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +\$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +F$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +i$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +l$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +e$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +s$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +y$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +s$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +t$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +e$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +m$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +\$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +S$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +t$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +r$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +e$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +a$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +m$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +\$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +S$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +t$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +r$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +i$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +n$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +g$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +S$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +o$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +u$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +r$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +c$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +e$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +S$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +t$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +r$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +e$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +a$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +m$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +;$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; + +$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; + +$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +u$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +s$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +e$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; + $source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +f$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +u$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +n$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +c$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +t$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +i$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +o$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +n$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; + $source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +F$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +l$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +o$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +w$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +\$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +F$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +i$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +l$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +e$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +s$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +y$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +s$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +t$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +e$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +m$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +\$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +D$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +S$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +L$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +\$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +p$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +a$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +t$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +h$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +;$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; + +$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; + +$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +$$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +p$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +h$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +p$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; + $source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +=$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; + $source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +s$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +t$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +a$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +t$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +i$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +c$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; + $source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +f$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +u$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +n$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +c$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +t$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +i$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +o$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +n$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; + $source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +($source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +s$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +t$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +r$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +i$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +n$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +g$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; + $source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +$$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +r$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +a$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +w$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +,$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; + $source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +b$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +o$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +o$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +l$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; + $source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +$$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +w$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +i$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +t$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +h$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +H$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +e$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +a$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +d$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +e$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +r$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +,$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; + $source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +i$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +n$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +t$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; + $source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +$$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +r$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +o$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +w$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +s$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +)$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +:$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; + $source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +i$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +n$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +t$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; + $source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +{$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; + +$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; + $source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; + $source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; + $source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; + $source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +$$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +s$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +o$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +u$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +r$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +c$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +e$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; + $source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +=$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; + $source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +n$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +e$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +w$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; + $source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +P$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +h$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +p$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +C$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +S$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +V$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +O$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +p$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +e$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +n$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +S$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +o$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +u$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +r$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +c$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +e$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +($source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; + +$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; + $source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; + $source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; + $source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; + $source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; + $source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; + $source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; + $source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; + $source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +n$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +e$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +w$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; + $source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +S$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +t$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +r$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +i$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +n$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +g$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +S$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +o$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +u$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +r$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +c$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +e$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +S$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +t$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +r$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +e$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +a$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +m$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +($source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +p$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +a$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +t$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +h$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +($source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +'$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +m$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +e$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +m$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +o$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +r$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +y$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +:$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +/$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +/$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +p$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +h$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +p$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +t$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +.$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +c$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +s$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +v$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +'$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +)$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +,$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; + $source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +$$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +r$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +a$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +w$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +)$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +,$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; + +$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; + $source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; + $source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; + $source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; + $source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; + $source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; + $source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; + $source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; + $source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +n$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +e$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +w$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; + $source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +C$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +S$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +V$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +D$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +e$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +c$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +o$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +d$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +e$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +r$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +($source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +w$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +i$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +t$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +h$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +H$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +e$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +a$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +d$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +e$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +r$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +:$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; + $source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +$$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +w$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +i$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +t$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +h$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +H$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +e$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +a$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +d$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +e$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +r$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +)$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +,$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; + +$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; + $source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; + $source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; + $source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; + $source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; + $source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; + $source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; + $source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; + $source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +n$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +e$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +w$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; + $source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +C$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +S$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +V$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +L$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +i$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +n$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +e$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +R$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +e$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +a$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +d$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +e$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +r$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +($source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +'$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +"$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +'$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +)$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +,$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; + +$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; + $source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; + $source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; + $source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; + $source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +)$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +;$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; + +$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; + $source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; + $source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; + $source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; + $source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +$$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +t$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +a$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +k$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +e$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +n$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; + $source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +=$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; + $source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +0$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +;$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; + +$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; + +$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; + $source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; + $source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; + $source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; + $source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +f$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +o$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +r$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +e$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +a$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +c$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +h$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; + $source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +($source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +$$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +s$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +o$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +u$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +r$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +c$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +e$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +-$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +>$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +r$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +e$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +c$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +o$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +r$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +d$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +s$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +($source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +)$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; + $source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +a$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +s$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; + $source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +$$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +v$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +a$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +l$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +u$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +e$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +s$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +)$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; + $source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +{$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; + +$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; + $source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; + $source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; + $source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; + $source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; + $source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; + $source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; + $source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; + $source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +i$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +f$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; + $source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +($source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; ++$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; ++$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +$$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +t$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +a$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +k$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +e$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +n$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; + $source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +>$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +=$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; + $source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +$$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +r$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +o$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +w$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +s$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +)$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; + $source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +{$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; + +$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; + $source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; + $source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; + $source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; + $source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; + $source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; + $source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; + $source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; + $source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; + $source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; + $source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; + $source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; + $source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +b$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +r$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +e$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +a$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +k$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +;$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; + +$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; + $source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; + $source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; + $source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; + $source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; + $source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; + $source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; + $source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; + $source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +}$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; + +$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; + $source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; + $source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; + $source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; + $source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +}$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; + +$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; + +$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; + $source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; + $source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; + $source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; + $source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +r$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +e$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +t$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +u$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +r$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +n$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; + $source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +$$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +s$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +o$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +u$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +r$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +c$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +e$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +-$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +>$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +p$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +r$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +o$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +d$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +u$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +c$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +e$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +d$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +B$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +y$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +t$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +e$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +s$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +($source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +)$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +;$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; + +$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +}$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +;$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; + +$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; + +$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +$$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +n$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +a$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +t$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +i$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +v$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +e$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; + $source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +=$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; + $source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +s$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +t$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +a$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +t$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +i$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +c$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; + $source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +f$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +u$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +n$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +c$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +t$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +i$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +o$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +n$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; + $source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +($source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +s$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +t$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +r$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +i$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +n$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +g$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; + $source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +$$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +r$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +a$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +w$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +,$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; + $source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +b$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +o$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +o$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +l$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; + $source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +$$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +w$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +i$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +t$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +h$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +H$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +e$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +a$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +d$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +e$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +r$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +,$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; + $source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +i$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +n$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +t$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; + $source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +$$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +r$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +o$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +w$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +s$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +,$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; + $source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +i$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +n$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +t$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; + $source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +$$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +c$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +h$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +u$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +n$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +k$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +)$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +:$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; + $source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +i$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +n$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +t$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; + $source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +{$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; + +$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; + $source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; + $source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; + $source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; + $source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +$$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +s$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +o$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +u$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +r$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +c$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +e$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; + $source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +=$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; + $source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +n$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +e$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +w$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; + $source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +R$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +u$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +s$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +t$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +C$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +S$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +V$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +O$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +p$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +e$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +n$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +S$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +o$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +u$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +r$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +c$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +e$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +($source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +n$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +e$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +w$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; + $source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +M$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +e$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +m$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +o$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +r$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +y$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +S$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +o$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +u$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +r$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +c$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +e$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +S$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +t$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +r$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +e$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +a$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +m$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +($source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +$$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +r$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +a$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +w$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +)$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +,$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; + $source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +'$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +,$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +'$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +,$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; + $source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +'$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +"$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +'$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +,$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; + $source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +'$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +\$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +\$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +'$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +,$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; + $source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +$$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +w$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +i$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +t$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +h$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +H$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +e$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +a$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +d$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +e$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +r$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +,$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; + $source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +t$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +r$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +u$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +e$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +,$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; + $source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +t$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +r$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +u$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +e$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +,$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; + $source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +$$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +c$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +h$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +u$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +n$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +k$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +)$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +;$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; + +$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; + $source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; + $source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; + $source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; + $source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +$$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +s$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +o$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +u$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +r$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +c$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +e$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +-$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +>$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +s$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +n$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +i$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +f$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +f$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +($source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +[$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +]$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +,$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; + $source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +$$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +r$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +o$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +w$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +s$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +,$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; + $source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +n$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +e$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +w$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; + $source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +S$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +c$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +h$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +e$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +m$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +a$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +I$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +n$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +f$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +e$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +r$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +e$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +n$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +c$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +e$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +($source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +)$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +,$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; + $source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +n$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +e$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +w$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; + $source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +S$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +t$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +r$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +i$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +n$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +g$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +T$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +y$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +p$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +e$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +N$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +a$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +r$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +r$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +o$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +w$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +e$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +r$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +($source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +I$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +n$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +f$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +e$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +r$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +r$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +e$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +d$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +T$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +y$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +p$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +e$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +s$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +:$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +:$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +d$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +e$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +f$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +a$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +u$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +l$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +t$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +($source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +)$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +-$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +>$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +t$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +o$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +A$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +r$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +r$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +a$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +y$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +($source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +)$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +)$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +)$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +;$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; + +$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; + +$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; + $source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; + $source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; + $source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; + $source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +r$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +e$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +t$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +u$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +r$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +n$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; + $source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +$$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +s$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +o$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +u$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +r$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +c$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +e$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +-$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +>$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +p$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +r$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +o$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +d$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +u$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +c$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +e$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +d$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +B$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +y$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +t$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +e$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +s$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +($source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +)$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +;$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; + +$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +}$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +;$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; + +$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; + +$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +$$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +c$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +a$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +s$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +e$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +s$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; + $source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +=$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; + $source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +[$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; + +$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; + $source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; + $source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; + $source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; + $source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +'$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +L$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +F$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +'$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; + $source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +=$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +>$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; + $source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +[$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +"$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +a$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +,$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +b$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +\$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +n$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +1$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +,$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +2$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +\$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +n$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +3$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +3$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +,$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +4$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +4$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +\$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +n$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +"$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +,$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; + $source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +t$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +r$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +u$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +e$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +,$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; + $source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +2$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +]$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +,$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; + +$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; + $source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; + $source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; + $source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; + $source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +'$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +C$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +R$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +L$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +F$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +'$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; + $source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +=$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +>$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; + $source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +[$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +"$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +a$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +,$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +b$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +\$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +r$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +\$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +n$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +1$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +,$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +2$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +\$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +r$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +\$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +n$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +3$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +3$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +,$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +4$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +4$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +\$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +r$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +\$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +n$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +"$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +,$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; + $source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +t$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +r$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +u$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +e$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +,$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; + $source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +2$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +]$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +,$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; + +$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; + $source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; + $source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; + $source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; + $source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +'$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +q$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +u$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +o$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +t$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +e$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +d$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; + $source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +m$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +u$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +l$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +t$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +i$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +-$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +l$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +i$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +n$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +e$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; + $source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +L$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +F$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +'$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; + $source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +=$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +>$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; + $source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +[$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +"$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +a$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +,$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +b$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +\$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +n$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +\$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +"$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +x$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +\$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +n$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +y$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +\$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +"$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +,$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +1$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +\$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +n$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +2$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +,$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +3$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +\$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +n$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +"$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +,$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; + $source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +t$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +r$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +u$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +e$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +,$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; + $source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +2$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +]$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +,$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; + +$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; + $source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; + $source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; + $source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; + $source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +'$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +q$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +u$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +o$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +t$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +e$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +d$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; + $source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +m$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +u$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +l$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +t$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +i$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +-$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +l$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +i$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +n$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +e$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; + $source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +C$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +R$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +L$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +F$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +'$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; + $source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +=$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +>$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; + $source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +[$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +"$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +a$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +,$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +b$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +\$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +r$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +\$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +n$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +\$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +"$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +x$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +\$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +r$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +\$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +n$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +y$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +\$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +"$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +,$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +1$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +\$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +r$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +\$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +n$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +2$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +,$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +3$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +\$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +r$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +\$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +n$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +"$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +,$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; + $source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +t$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +r$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +u$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +e$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +,$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; + $source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +2$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +]$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +,$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; + +$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; + $source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; + $source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; + $source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; + $source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +'$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +b$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +l$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +a$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +n$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +k$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; + $source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +l$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +i$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +n$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +e$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +s$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +'$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; + $source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +=$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +>$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; + $source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +[$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +"$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +a$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +,$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +b$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +\$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +n$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +\$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +n$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +1$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +,$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +2$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +\$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +n$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +\$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +n$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +"$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +,$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; + $source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +t$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +r$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +u$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +e$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +,$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; + $source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +3$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +]$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +,$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; + +$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; + $source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; + $source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; + $source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; + $source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +'$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +B$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +O$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +M$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; + $source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +w$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +i$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +t$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +h$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +o$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +u$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +t$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; + $source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +h$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +e$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +a$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +d$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +e$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +r$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +'$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; + $source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +=$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +>$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; + $source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +[$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +"$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +\$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +x$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +E$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +F$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +\$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +x$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +B$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +B$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +\$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +x$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +B$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +F$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +1$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +,$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +2$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +\$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +n$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +3$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +,$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +4$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +\$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +n$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +"$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +,$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; + $source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +f$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +a$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +l$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +s$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +e$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +,$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; + $source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +2$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +]$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +,$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; + +$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; + $source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; + $source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; + $source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; + $source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +'$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +f$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +i$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +r$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +s$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +t$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; + $source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +t$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +w$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +o$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; + $source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +o$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +f$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; + $source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +t$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +h$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +r$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +e$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +e$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; + $source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +r$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +o$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +w$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +s$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +'$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; + $source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +=$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +>$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; + $source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +[$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +"$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +a$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +,$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +b$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +\$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +n$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +1$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +,$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +2$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +\$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +n$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +3$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +3$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +,$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +4$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +4$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +\$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +n$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +5$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +5$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +5$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +,$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +6$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +6$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +6$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +\$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +n$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +"$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +,$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; + $source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +t$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +r$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +u$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +e$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +,$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; + $source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +2$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +]$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +,$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; + +$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +]$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +;$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; + +$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; + +$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +f$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +o$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +r$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +e$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +a$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +c$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +h$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; + $source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +($source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +$$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +c$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +a$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +s$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +e$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +s$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; + $source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +a$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +s$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; + $source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +$$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +l$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +a$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +b$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +e$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +l$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; + $source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +=$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +>$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; + $source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +[$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +$$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +r$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +a$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +w$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +,$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; + $source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +$$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +w$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +i$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +t$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +h$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +H$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +e$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +a$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +d$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +e$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +r$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +,$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; + $source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +$$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +r$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +o$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +w$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +s$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +]$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +)$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; + $source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +{$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; + +$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; + $source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; + $source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; + $source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; + $source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +$$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +e$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +x$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +p$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +e$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +c$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +t$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +e$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +d$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; + $source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +=$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; + $source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +$$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +p$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +h$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +p$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +($source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +$$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +r$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +a$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +w$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +,$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; + $source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +$$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +w$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +i$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +t$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +h$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +H$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +e$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +a$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +d$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +e$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +r$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +,$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; + $source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +$$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +r$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +o$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +w$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +s$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +)$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +;$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; + +$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; + $source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; + $source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; + $source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; + $source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +$$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +c$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +o$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +u$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +n$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +t$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +s$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; + $source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +=$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; + $source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +a$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +r$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +r$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +a$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +y$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +_$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +u$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +n$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +i$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +q$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +u$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +e$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +($source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +a$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +r$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +r$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +a$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +y$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +_$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +m$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +a$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +p$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +($source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; + +$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; + $source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; + $source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; + $source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; + $source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; + $source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; + $source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; + $source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; + $source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +s$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +t$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +a$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +t$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +i$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +c$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; + $source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +f$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +n$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +($source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +i$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +n$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +t$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; + $source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +$$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +c$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +h$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +u$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +n$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +k$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +)$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +:$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; + $source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +i$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +n$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +t$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; + $source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +=$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +>$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; + $source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +$$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +n$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +a$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +t$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +i$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +v$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +e$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +($source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +$$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +r$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +a$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +w$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +,$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; + $source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +$$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +w$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +i$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +t$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +h$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +H$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +e$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +a$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +d$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +e$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +r$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +,$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; + $source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +$$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +r$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +o$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +w$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +s$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +,$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; + $source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +$$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +c$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +h$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +u$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +n$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +k$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +)$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +,$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; + +$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; + $source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; + $source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; + $source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; + $source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; + $source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; + $source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; + $source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; + $source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +[$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +1$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +,$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; + $source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +3$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +,$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; + $source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +7$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +,$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; + $source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +4$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +0$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +9$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +6$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +]$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +,$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; + +$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; + $source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; + $source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; + $source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; + $source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +)$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +)$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +;$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; + +$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; + +$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; + $source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; + $source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; + $source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; + $source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +e$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +c$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +h$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +o$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; + $source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +$$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +l$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +a$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +b$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +e$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +l$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +,$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; + $source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +'$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +:$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; + $source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +'$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +,$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; + $source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +$$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +c$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +o$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +u$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +n$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +t$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +s$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; + $source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +=$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +=$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +=$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; + $source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +[$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +$$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +e$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +x$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +p$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +e$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +c$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +t$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +e$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +d$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +]$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; + $source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +?$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; + $source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +'$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +i$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +d$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +e$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +n$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +t$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +i$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +c$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +a$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +l$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; + $source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +'$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; + $source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +.$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; + $source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +$$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +e$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +x$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +p$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +e$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +c$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +t$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +e$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +d$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; + $source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +:$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; + $source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +'$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +F$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +A$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +I$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +L$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; + $source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +p$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +h$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +p$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +=$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +'$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; + $source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +.$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; + $source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +$$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +e$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +x$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +p$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +e$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +c$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +t$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +e$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +d$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; + $source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +.$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; + $source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +'$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; + $source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +n$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +a$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +t$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +i$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +v$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +e$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +=$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +'$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; + $source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +.$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; + $source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +i$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +m$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +p$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +l$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +o$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +d$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +e$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +($source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +'$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +,$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +'$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +,$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; + $source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +$$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +c$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +o$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +u$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +n$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +t$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +s$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +)$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +,$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; + $source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +"$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +\$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +n$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +"$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +;$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; + +$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +}$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; + +$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; + +$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +$$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +r$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +e$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +a$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +d$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +e$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +r$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; + $source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +=$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; + $source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +n$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +e$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +w$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; + $source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +R$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +u$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +s$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +t$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +C$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +S$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +V$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +R$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +e$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +a$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +d$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +e$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +r$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +N$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +a$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +t$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +i$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +v$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +e$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +($source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +'$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +,$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +'$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +,$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; + $source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +'$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +"$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +'$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +,$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; + $source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +'$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +\$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +\$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +'$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +,$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; + $source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +t$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +r$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +u$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +e$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +,$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; + $source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +t$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +r$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +u$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +e$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +,$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; + $source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +t$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +r$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +u$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +e$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +)$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +;$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; + +$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +$$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +r$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +e$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +a$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +d$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +e$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +r$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +-$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +>$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +f$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +e$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +e$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +d$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +($source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +"$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +a$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +,$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +b$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +\$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +n$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +1$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +,$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +2$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +\$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +n$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +3$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +3$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +,$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +4$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +4$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +\$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +n$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +"$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +)$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +;$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; + +$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +$$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +r$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +e$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +a$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +d$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +e$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +r$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +-$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +>$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +f$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +i$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +n$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +i$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +s$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +h$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +($source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +)$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +;$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; + +$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +$$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +r$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +e$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +a$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +d$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +e$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +r$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +-$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +>$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +n$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +e$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +x$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +t$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +($source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +1$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +)$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +;$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; + +$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +e$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +c$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +h$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +o$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; + $source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +'$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +n$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +e$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +x$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +t$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +($source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +)$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; + $source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +c$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +o$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +u$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +n$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +t$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +s$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; + $source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +t$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +o$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +o$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +:$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; + $source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +'$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +,$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; + $source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +$$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +r$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +e$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +a$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +d$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +e$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +r$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +-$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +>$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +c$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +o$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +n$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +s$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +u$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +m$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +e$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +d$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +B$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +y$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +t$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +e$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +s$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +($source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +)$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +,$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; + $source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +"$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +\$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +n$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +"$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +;$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; + +$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +$$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +r$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +e$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +a$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +d$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +e$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +r$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +-$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +>$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +n$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +e$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +x$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +t$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +($source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +1$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +0$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +)$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +;$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; + +$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +e$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +c$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +h$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +o$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; + $source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +'$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +h$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +e$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +a$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +d$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +e$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +r$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; + $source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +e$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +x$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +c$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +l$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +u$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +d$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +e$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +d$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +:$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; + $source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +'$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +,$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; + $source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +$$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +r$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +e$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +a$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +d$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +e$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +r$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +-$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +>$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +c$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +o$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +n$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +s$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +u$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +m$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +e$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +d$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +B$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +y$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +t$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +e$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +s$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +($source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +)$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +,$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; + $source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +"$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +\$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +n$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +"$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +;$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; + +$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +?$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +>$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; + +$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +-$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +-$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +E$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +X$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +P$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +E$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +C$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +T$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +-$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +-$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; + +$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +L$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +F$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +:$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; + $source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +i$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +d$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +e$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +n$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +t$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +i$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +c$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +a$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +l$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; + $source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +1$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +0$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; + +$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +C$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +R$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +L$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +F$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +:$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; + $source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +i$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +d$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +e$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +n$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +t$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +i$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +c$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +a$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +l$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; + $source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +1$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +2$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; + +$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +q$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +u$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +o$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +t$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +e$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +d$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; + $source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +m$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +u$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +l$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +t$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +i$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +-$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +l$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +i$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +n$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +e$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; + $source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +L$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +F$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +:$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; + $source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +i$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +d$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +e$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +n$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +t$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +i$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +c$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +a$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +l$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; + $source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +1$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +2$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; + +$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +q$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +u$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +o$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +t$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +e$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +d$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; + $source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +m$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +u$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +l$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +t$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +i$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +-$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +l$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +i$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +n$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +e$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; + $source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +C$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +R$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +L$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +F$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +:$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; + $source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +i$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +d$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +e$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +n$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +t$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +i$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +c$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +a$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +l$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; + $source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +1$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +5$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; + +$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +b$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +l$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +a$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +n$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +k$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; + $source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +l$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +i$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +n$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +e$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +s$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +:$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; + $source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +i$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +d$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +e$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +n$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +t$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +i$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +c$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +a$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +l$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; + $source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +6$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; + +$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +B$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +O$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +M$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; + $source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +w$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +i$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +t$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +h$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +o$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +u$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +t$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; + $source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +h$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +e$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +a$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +d$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +e$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +r$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +:$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; + $source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +i$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +d$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +e$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +n$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +t$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +i$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +c$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +a$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +l$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; + $source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +1$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +1$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; + +$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +f$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +i$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +r$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +s$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +t$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; + $source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +t$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +w$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +o$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; + $source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +o$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +f$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; + $source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +t$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +h$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +r$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +e$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +e$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; + $source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +r$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +o$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +w$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +s$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +:$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; + $source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +i$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +d$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +e$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +n$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +t$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +i$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +c$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +a$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +l$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; + $source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +1$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +0$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; + +$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +n$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +e$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +x$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +t$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +($source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +)$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; + $source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +c$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +o$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +u$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +n$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +t$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +s$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; + $source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +t$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +o$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +o$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +:$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; + $source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +4$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; + +$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +h$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +e$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +a$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +d$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +e$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +r$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; + $source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +e$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +x$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +c$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +l$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +u$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +d$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +e$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +d$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +:$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; + $source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +1$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; +0$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; + +$source = new RustCSVOpenSource(new MemorySourceStream("a,b\n1,2\n33,44\n"), ',', '"', '\\', true, true, true); +iterator_to_array($source->records()); +echo 'records() counts too, header excluded: ', $source->producedBytes(), "\n"; diff --git a/src/extension/flow-php-ext/tests/phpt/055_column_interfaces_skew.phpt b/src/extension/flow-php-ext/tests/phpt/055_column_interfaces_skew.phpt new file mode 100644 index 0000000000..bea2adad21 --- /dev/null +++ b/src/extension/flow-php-ext/tests/phpt/055_column_interfaces_skew.phpt @@ -0,0 +1,20 @@ +--TEST-- +flow_php registers the flow-php/etl and adapter interfaces reflection-identical to the package files +--SKIPIF-- + +--FILE-- + +--EXPECT-- +library reflected +identical diff --git a/src/extension/flow-php-ext/tests/phpt/055_datetime_iso_fast_path.phpt b/src/extension/flow-php-ext/tests/phpt/055_datetime_iso_fast_path.phpt deleted file mode 100644 index 2325aa529b..0000000000 --- a/src/extension/flow-php-ext/tests/phpt/055_datetime_iso_fast_path.phpt +++ /dev/null @@ -1,207 +0,0 @@ ---TEST-- -datetime ISO strings: both hydrators build the value in the column zone - naive text read in it, Z and offsets converted to it, date.timezone ignored ---SKIPIF-- - ---FILE-- - $value])]; - $expected = (new DateTimeImmutable($value, $tz))->setTimezone($tz); - $phpAt = $php->hydrate($batch, $schema)->first()->get('at'); - $nativeAt = $native->hydrate($batch, $schema)->first()->get('at'); - - printf( - "%-38s %s %s type:%d php:%s native:%s\n", - json_encode($value), - $nativeAt->format('Y-m-d H:i:s.u P'), - $nativeAt->getTimezone()->getName(), - json_decode(json_encode($nativeAt), true)['timezone_type'], - serialize($phpAt) === serialize($expected) ? 'yes' : 'NO', - serialize($nativeAt) === serialize($expected) ? 'yes' : 'NO', - ); - } -} - -$schema = schema(datetime_schema('at')); - -foreach (['2026-01-02T25:99:99Z', '2026-01-02T03:60:00Z', '2026-01-02T03:04:05+25:00'] as $value) { - $batch = [new RawRowValues(['at' => $value])]; - - foreach (['php' => $php, 'native' => $native] as $label => $hydrator) { - try { - $hydrator->hydrate($batch, $schema); - echo "{$label} {$value}: FAIL no exception\n"; - } catch (Throwable $e) { - echo "{$label} {$value}: ", $e::class, "\n"; - } - } -} -?> ---EXPECT-- -column zone UTC -"2026-01-02T03:04:05Z" 2026-01-02 03:04:05.000000 +00:00 UTC type:3 php:yes native:yes -"2026-01-02 03:04:05Z" 2026-01-02 03:04:05.000000 +00:00 UTC type:3 php:yes native:yes -"2026-01-02T03:04Z" 2026-01-02 03:04:00.000000 +00:00 UTC type:3 php:yes native:yes -"2026-01-02T03:04:05Z\n" 2026-01-02 03:04:05.000000 +00:00 UTC type:3 php:yes native:yes -"2026-01-02T03:04:05.1Z" 2026-01-02 03:04:05.100000 +00:00 UTC type:3 php:yes native:yes -"2026-01-02T03:04:05.12Z" 2026-01-02 03:04:05.120000 +00:00 UTC type:3 php:yes native:yes -"2026-01-02T03:04:05.123Z" 2026-01-02 03:04:05.123000 +00:00 UTC type:3 php:yes native:yes -"2026-01-02T03:04:05.1234Z" 2026-01-02 03:04:05.123400 +00:00 UTC type:3 php:yes native:yes -"2026-01-02T03:04:05.12345Z" 2026-01-02 03:04:05.123450 +00:00 UTC type:3 php:yes native:yes -"2026-01-02T03:04:05.123456Z" 2026-01-02 03:04:05.123456 +00:00 UTC type:3 php:yes native:yes -"2026-01-02T03:04:05.1234567Z" 2026-01-02 03:04:05.123456 +00:00 UTC type:3 php:yes native:yes -"2026-01-02T03:04:05.12345678Z" 2026-01-02 03:04:05.123456 +00:00 UTC type:3 php:yes native:yes -"2026-01-02T03:04:05.123456789Z" 2026-01-02 03:04:05.123456 +00:00 UTC type:3 php:yes native:yes -"2026-12-31T23:59:60Z" 2027-01-01 00:00:00.000000 +00:00 UTC type:3 php:yes native:yes -"2026-01-02T24:00:00Z" 2026-01-03 00:00:00.000000 +00:00 UTC type:3 php:yes native:yes -"0001-01-01T00:00:00Z" 0001-01-01 00:00:00.000000 +00:00 UTC type:3 php:yes native:yes -"2026-01-02T03:04:05.123456789+02:00" 2026-01-02 01:04:05.123456 +00:00 UTC type:3 php:yes native:yes -"2026-01-02T03:04:05-0530" 2026-01-02 08:34:05.000000 +00:00 UTC type:3 php:yes native:yes -"2026-01-02T03:04:05-05" 2026-01-02 08:04:05.000000 +00:00 UTC type:3 php:yes native:yes -"2026-01-02T03:04:05+00:00" 2026-01-02 03:04:05.000000 +00:00 UTC type:3 php:yes native:yes -"2026-01-02 03:04" 2026-01-02 03:04:00.000000 +00:00 UTC type:3 php:yes native:yes -"2026-01-02T03:04:05" 2026-01-02 03:04:05.000000 +00:00 UTC type:3 php:yes native:yes -"2026-01-02T03:04\n" 2026-01-02 03:04:00.000000 +00:00 UTC type:3 php:yes native:yes -"2026-03-29T02:30:00" 2026-03-29 02:30:00.000000 +00:00 UTC type:3 php:yes native:yes -"2026-10-25T02:30:00" 2026-10-25 02:30:00.000000 +00:00 UTC type:3 php:yes native:yes -"2026-03-08T02:30:00" 2026-03-08 02:30:00.000000 +00:00 UTC type:3 php:yes native:yes -"2026-11-01T01:30:00" 2026-11-01 01:30:00.000000 +00:00 UTC type:3 php:yes native:yes -column zone Europe/Warsaw -"2026-01-02T03:04:05Z" 2026-01-02 04:04:05.000000 +01:00 Europe/Warsaw type:3 php:yes native:yes -"2026-01-02 03:04:05Z" 2026-01-02 04:04:05.000000 +01:00 Europe/Warsaw type:3 php:yes native:yes -"2026-01-02T03:04Z" 2026-01-02 04:04:00.000000 +01:00 Europe/Warsaw type:3 php:yes native:yes -"2026-01-02T03:04:05Z\n" 2026-01-02 04:04:05.000000 +01:00 Europe/Warsaw type:3 php:yes native:yes -"2026-01-02T03:04:05.1Z" 2026-01-02 04:04:05.100000 +01:00 Europe/Warsaw type:3 php:yes native:yes -"2026-01-02T03:04:05.12Z" 2026-01-02 04:04:05.120000 +01:00 Europe/Warsaw type:3 php:yes native:yes -"2026-01-02T03:04:05.123Z" 2026-01-02 04:04:05.123000 +01:00 Europe/Warsaw type:3 php:yes native:yes -"2026-01-02T03:04:05.1234Z" 2026-01-02 04:04:05.123400 +01:00 Europe/Warsaw type:3 php:yes native:yes -"2026-01-02T03:04:05.12345Z" 2026-01-02 04:04:05.123450 +01:00 Europe/Warsaw type:3 php:yes native:yes -"2026-01-02T03:04:05.123456Z" 2026-01-02 04:04:05.123456 +01:00 Europe/Warsaw type:3 php:yes native:yes -"2026-01-02T03:04:05.1234567Z" 2026-01-02 04:04:05.123456 +01:00 Europe/Warsaw type:3 php:yes native:yes -"2026-01-02T03:04:05.12345678Z" 2026-01-02 04:04:05.123456 +01:00 Europe/Warsaw type:3 php:yes native:yes -"2026-01-02T03:04:05.123456789Z" 2026-01-02 04:04:05.123456 +01:00 Europe/Warsaw type:3 php:yes native:yes -"2026-12-31T23:59:60Z" 2027-01-01 01:00:00.000000 +01:00 Europe/Warsaw type:3 php:yes native:yes -"2026-01-02T24:00:00Z" 2026-01-03 01:00:00.000000 +01:00 Europe/Warsaw type:3 php:yes native:yes -"0001-01-01T00:00:00Z" 0001-01-01 01:24:00.000000 +01:24 Europe/Warsaw type:3 php:yes native:yes -"2026-01-02T03:04:05.123456789+02:00" 2026-01-02 02:04:05.123456 +01:00 Europe/Warsaw type:3 php:yes native:yes -"2026-01-02T03:04:05-0530" 2026-01-02 09:34:05.000000 +01:00 Europe/Warsaw type:3 php:yes native:yes -"2026-01-02T03:04:05-05" 2026-01-02 09:04:05.000000 +01:00 Europe/Warsaw type:3 php:yes native:yes -"2026-01-02T03:04:05+00:00" 2026-01-02 04:04:05.000000 +01:00 Europe/Warsaw type:3 php:yes native:yes -"2026-01-02 03:04" 2026-01-02 03:04:00.000000 +01:00 Europe/Warsaw type:3 php:yes native:yes -"2026-01-02T03:04:05" 2026-01-02 03:04:05.000000 +01:00 Europe/Warsaw type:3 php:yes native:yes -"2026-01-02T03:04\n" 2026-01-02 03:04:00.000000 +01:00 Europe/Warsaw type:3 php:yes native:yes -"2026-03-29T02:30:00" 2026-03-29 03:30:00.000000 +02:00 Europe/Warsaw type:3 php:yes native:yes -"2026-10-25T02:30:00" 2026-10-25 02:30:00.000000 +01:00 Europe/Warsaw type:3 php:yes native:yes -"2026-03-08T02:30:00" 2026-03-08 02:30:00.000000 +01:00 Europe/Warsaw type:3 php:yes native:yes -"2026-11-01T01:30:00" 2026-11-01 01:30:00.000000 +01:00 Europe/Warsaw type:3 php:yes native:yes -column zone America/New_York -"2026-01-02T03:04:05Z" 2026-01-01 22:04:05.000000 -05:00 America/New_York type:3 php:yes native:yes -"2026-01-02 03:04:05Z" 2026-01-01 22:04:05.000000 -05:00 America/New_York type:3 php:yes native:yes -"2026-01-02T03:04Z" 2026-01-01 22:04:00.000000 -05:00 America/New_York type:3 php:yes native:yes -"2026-01-02T03:04:05Z\n" 2026-01-01 22:04:05.000000 -05:00 America/New_York type:3 php:yes native:yes -"2026-01-02T03:04:05.1Z" 2026-01-01 22:04:05.100000 -05:00 America/New_York type:3 php:yes native:yes -"2026-01-02T03:04:05.12Z" 2026-01-01 22:04:05.120000 -05:00 America/New_York type:3 php:yes native:yes -"2026-01-02T03:04:05.123Z" 2026-01-01 22:04:05.123000 -05:00 America/New_York type:3 php:yes native:yes -"2026-01-02T03:04:05.1234Z" 2026-01-01 22:04:05.123400 -05:00 America/New_York type:3 php:yes native:yes -"2026-01-02T03:04:05.12345Z" 2026-01-01 22:04:05.123450 -05:00 America/New_York type:3 php:yes native:yes -"2026-01-02T03:04:05.123456Z" 2026-01-01 22:04:05.123456 -05:00 America/New_York type:3 php:yes native:yes -"2026-01-02T03:04:05.1234567Z" 2026-01-01 22:04:05.123456 -05:00 America/New_York type:3 php:yes native:yes -"2026-01-02T03:04:05.12345678Z" 2026-01-01 22:04:05.123456 -05:00 America/New_York type:3 php:yes native:yes -"2026-01-02T03:04:05.123456789Z" 2026-01-01 22:04:05.123456 -05:00 America/New_York type:3 php:yes native:yes -"2026-12-31T23:59:60Z" 2026-12-31 19:00:00.000000 -05:00 America/New_York type:3 php:yes native:yes -"2026-01-02T24:00:00Z" 2026-01-02 19:00:00.000000 -05:00 America/New_York type:3 php:yes native:yes -"0001-01-01T00:00:00Z" 0000-12-31 19:03:58.000000 -04:56 America/New_York type:3 php:yes native:yes -"2026-01-02T03:04:05.123456789+02:00" 2026-01-01 20:04:05.123456 -05:00 America/New_York type:3 php:yes native:yes -"2026-01-02T03:04:05-0530" 2026-01-02 03:34:05.000000 -05:00 America/New_York type:3 php:yes native:yes -"2026-01-02T03:04:05-05" 2026-01-02 03:04:05.000000 -05:00 America/New_York type:3 php:yes native:yes -"2026-01-02T03:04:05+00:00" 2026-01-01 22:04:05.000000 -05:00 America/New_York type:3 php:yes native:yes -"2026-01-02 03:04" 2026-01-02 03:04:00.000000 -05:00 America/New_York type:3 php:yes native:yes -"2026-01-02T03:04:05" 2026-01-02 03:04:05.000000 -05:00 America/New_York type:3 php:yes native:yes -"2026-01-02T03:04\n" 2026-01-02 03:04:00.000000 -05:00 America/New_York type:3 php:yes native:yes -"2026-03-29T02:30:00" 2026-03-29 02:30:00.000000 -04:00 America/New_York type:3 php:yes native:yes -"2026-10-25T02:30:00" 2026-10-25 02:30:00.000000 -04:00 America/New_York type:3 php:yes native:yes -"2026-03-08T02:30:00" 2026-03-08 03:30:00.000000 -04:00 America/New_York type:3 php:yes native:yes -"2026-11-01T01:30:00" 2026-11-01 01:30:00.000000 -04:00 America/New_York type:3 php:yes native:yes -column zone +05:30 -"2026-01-02T03:04:05Z" 2026-01-02 08:34:05.000000 +05:30 +05:30 type:1 php:yes native:yes -"2026-01-02 03:04:05Z" 2026-01-02 08:34:05.000000 +05:30 +05:30 type:1 php:yes native:yes -"2026-01-02T03:04Z" 2026-01-02 08:34:00.000000 +05:30 +05:30 type:1 php:yes native:yes -"2026-01-02T03:04:05Z\n" 2026-01-02 08:34:05.000000 +05:30 +05:30 type:1 php:yes native:yes -"2026-01-02T03:04:05.1Z" 2026-01-02 08:34:05.100000 +05:30 +05:30 type:1 php:yes native:yes -"2026-01-02T03:04:05.12Z" 2026-01-02 08:34:05.120000 +05:30 +05:30 type:1 php:yes native:yes -"2026-01-02T03:04:05.123Z" 2026-01-02 08:34:05.123000 +05:30 +05:30 type:1 php:yes native:yes -"2026-01-02T03:04:05.1234Z" 2026-01-02 08:34:05.123400 +05:30 +05:30 type:1 php:yes native:yes -"2026-01-02T03:04:05.12345Z" 2026-01-02 08:34:05.123450 +05:30 +05:30 type:1 php:yes native:yes -"2026-01-02T03:04:05.123456Z" 2026-01-02 08:34:05.123456 +05:30 +05:30 type:1 php:yes native:yes -"2026-01-02T03:04:05.1234567Z" 2026-01-02 08:34:05.123456 +05:30 +05:30 type:1 php:yes native:yes -"2026-01-02T03:04:05.12345678Z" 2026-01-02 08:34:05.123456 +05:30 +05:30 type:1 php:yes native:yes -"2026-01-02T03:04:05.123456789Z" 2026-01-02 08:34:05.123456 +05:30 +05:30 type:1 php:yes native:yes -"2026-12-31T23:59:60Z" 2027-01-01 05:30:00.000000 +05:30 +05:30 type:1 php:yes native:yes -"2026-01-02T24:00:00Z" 2026-01-03 05:30:00.000000 +05:30 +05:30 type:1 php:yes native:yes -"0001-01-01T00:00:00Z" 0001-01-01 05:30:00.000000 +05:30 +05:30 type:1 php:yes native:yes -"2026-01-02T03:04:05.123456789+02:00" 2026-01-02 06:34:05.123456 +05:30 +05:30 type:1 php:yes native:yes -"2026-01-02T03:04:05-0530" 2026-01-02 14:04:05.000000 +05:30 +05:30 type:1 php:yes native:yes -"2026-01-02T03:04:05-05" 2026-01-02 13:34:05.000000 +05:30 +05:30 type:1 php:yes native:yes -"2026-01-02T03:04:05+00:00" 2026-01-02 08:34:05.000000 +05:30 +05:30 type:1 php:yes native:yes -"2026-01-02 03:04" 2026-01-02 03:04:00.000000 +05:30 +05:30 type:1 php:yes native:yes -"2026-01-02T03:04:05" 2026-01-02 03:04:05.000000 +05:30 +05:30 type:1 php:yes native:yes -"2026-01-02T03:04\n" 2026-01-02 03:04:00.000000 +05:30 +05:30 type:1 php:yes native:yes -"2026-03-29T02:30:00" 2026-03-29 02:30:00.000000 +05:30 +05:30 type:1 php:yes native:yes -"2026-10-25T02:30:00" 2026-10-25 02:30:00.000000 +05:30 +05:30 type:1 php:yes native:yes -"2026-03-08T02:30:00" 2026-03-08 02:30:00.000000 +05:30 +05:30 type:1 php:yes native:yes -"2026-11-01T01:30:00" 2026-11-01 01:30:00.000000 +05:30 +05:30 type:1 php:yes native:yes -php 2026-01-02T25:99:99Z: Flow\ETL\Exception\SchemaMismatchException -native 2026-01-02T25:99:99Z: Flow\ETL\Exception\SchemaMismatchException -php 2026-01-02T03:60:00Z: Flow\ETL\Exception\SchemaMismatchException -native 2026-01-02T03:60:00Z: Flow\ETL\Exception\SchemaMismatchException -php 2026-01-02T03:04:05+25:00: Flow\ETL\Exception\SchemaMismatchException -native 2026-01-02T03:04:05+25:00: Flow\ETL\Exception\SchemaMismatchException diff --git a/src/extension/flow-php-ext/tests/phpt/056_fused_decode_rows.phpt b/src/extension/flow-php-ext/tests/phpt/056_fused_decode_rows.phpt deleted file mode 100644 index 3e0183108b..0000000000 --- a/src/extension/flow-php-ext/tests/phpt/056_fused_decode_rows.phpt +++ /dev/null @@ -1,182 +0,0 @@ ---TEST-- -RustFloeEncoderNative::decodeRows() yields the same Rows, and refuses with the same message, as hydrate(decode()) ---SKIPIF-- - ---FILE-- - - */ -function bodies(Rows $rows, Schema $fileSchema): array -{ - $schemaBody = json_encode($fileSchema->normalize(), JSON_THROW_ON_ERROR); - - return (new RustFloeEncoderNative())->encode((new NativeRowHydrator())->dehydrate($rows), $schemaBody, $fileSchema); -} - -function outcome(callable $read): string -{ - try { - return serialize($read()); - } catch (Throwable $e) { - $previous = $e->getPrevious(); - - return get_class($e) . ': ' . $e->getMessage() . ($previous === null ? '' : ' <- ' . get_class($previous) . ': ' . $previous->getMessage()); - } -} - -/** - * @param list $bodies - */ -function compare(string $label, array $bodies, string $schemaBody, Schema $schema): void -{ - $twoStep = outcome(static fn(): Rows => (new RustRowHydratorNative())->hydrate((new RustFloeEncoderNative())->decode($bodies, $schemaBody), $schema)); - $fused = outcome(static fn(): Rows => (new RustFloeEncoderNative())->decodeRows($bodies, $schemaBody, $schema)); - - echo $label, ': ', $twoStep === $fused ? 'identical' : "FAIL\n two-step: {$twoStep}\n fused: {$fused}", "\n"; - - if (!str_starts_with($fused, 'O:')) { - echo ' ', $fused, "\n"; - } -} - -function compare_rows(string $label, Rows $rows, ?Schema $readSchema = null): void -{ - $fileSchema = $rows->schema(); - - compare( - $label, - bodies($rows, $fileSchema), - json_encode($fileSchema->normalize(), JSON_THROW_ON_ERROR), - $readSchema ?? $fileSchema, - ); -} - -compare_rows('017 rows', rows( - schema(int_schema('id'), str_schema('name', nullable: true), float_schema('price', nullable: true), datetime_schema('at', nullable: true)), - row(['id' => 1, 'name' => 'a']), - row(['id' => 2, 'name' => null]), - row(['id' => 3, 'price' => 1.5]), - row(['id' => 4, 'at' => new DateTimeImmutable('2025-01-01 00:00:00.123456', new DateTimeZone('Europe/Warsaw'))]), -)); - -compare_rows('021 rows', rows( - schema(int_schema('id'), str_schema('name', nullable: true), float_schema('price')), - ...array_map(static fn(int $i) => row(['id' => $i, 'name' => $i % 7 === 0 ? null : 'user_' . $i, 'price' => $i / 4.0]), range(1, 500)), -)); - -$containers = rows( - schema( - int_schema('id'), - datetime_schema('utc'), - datetime_schema('zoned', nullable: true), - list_schema('tags', type_list(type_string()), nullable: true), - map_schema('scores', type_map(type_string(), type_integer())), - structure_schema('address', type_structure(['street' => type_string(), 'zip' => type_integer()]), nullable: true), - bool_schema('active'), - int_schema('1', nullable: true), - ), - row(['id' => 1, 'utc' => new DateTimeImmutable('2026-03-01T10:11:12.654321Z'), 'zoned' => new DateTimeImmutable('2026-03-01 10:11:12', new DateTimeZone('+02:00')), 'tags' => ['a', 'b'], 'scores' => ['x' => 1], 'address' => ['street' => 'Main', 'zip' => 12345], 'active' => true, '1' => 7]), - row(['id' => 2, 'utc' => new DateTimeImmutable('1970-01-01T00:00:00Z'), 'zoned' => null, 'tags' => null, 'scores' => [], 'address' => null, 'active' => false, '1' => null]), - row(['id' => 3, 'utc' => new DateTimeImmutable('2026-12-31 23:59:59.000001', new DateTimeZone('Europe/Warsaw')), 'zoned' => new DateTimeImmutable('2026-06-01', new DateTimeZone('America/New_York')), 'tags' => [], 'scores' => ['y' => -2, 'z' => 3], 'address' => ['street' => '', 'zip' => 0], 'active' => true, '1' => 0]), -); - -compare_rows('nullable / datetime / list / map / structure / numeric name', $containers); - -$fileSchema = schema(int_schema('id'), str_schema('name', nullable: true), int_schema('7', nullable: true)); -$tagged = rows( - schema(int_schema('id'), str_schema('name', nullable: true, metadata: Metadata::fromArray(['source' => 'crm'])), int_schema('7', nullable: true, metadata: Metadata::fromArray(['n' => 1]))), - row(['id' => 1, 'name' => 'a', '7' => 1]), - row(['id' => 2, 'name' => null, '7' => null]), -); -compare('per-value metadata', bodies($tagged, $fileSchema), json_encode($fileSchema->normalize(), JSON_THROW_ON_ERROR), $fileSchema); - -compare_rows('empty batch', rows(schema(int_schema('id')))); -compare('no frames', [], json_encode(schema(int_schema('id'))->normalize(), JSON_THROW_ON_ERROR), schema(int_schema('id'))); - -$narrow = rows(schema(int_schema('id'), str_schema('name')), row(['id' => 1, 'name' => 'a']), row(['id' => 2, 'name' => 'b'])); -compare_rows('read schema declares a nullable column the frames lack', $narrow, schema(int_schema('id'), str_schema('name'), str_schema('extra', nullable: true))); -compare_rows('read schema declares a NOT NULL column the frames lack', $narrow, schema(int_schema('id'), str_schema('name'), str_schema('extra'))); -compare_rows('read schema drops a decoded column', $narrow, schema(int_schema('id'))); -compare_rows('read schema reorders columns', $narrow, schema(str_schema('name'), int_schema('id'))); -compare_rows('read schema refuses a decoded value', $narrow, schema(int_schema('id'), int_schema('name'))); -compare_rows('read schema refuses a decoded null', rows(schema(int_schema('id'), str_schema('name', nullable: true)), row(['id' => 1, 'name' => null])), schema(int_schema('id'), str_schema('name'))); - -$narrowSchemaBody = json_encode($narrow->schema()->normalize(), JSON_THROW_ON_ERROR); -$narrowBodies = bodies($narrow, $narrow->schema()); -compare('trailing bytes', [$narrowBodies[0] . "\xEF"], $narrowSchemaBody, $narrow->schema()); -compare('truncated body', [substr($narrowBodies[1], 0, -1)], $narrowSchemaBody, $narrow->schema()); -compare('unknown value flag', ["\x09" . substr($narrowBodies[0], 1)], $narrowSchemaBody, $narrow->schema()); - -$duplicatedSchemaBody = json_encode([...$narrow->schema()->normalize(), ...schema(int_schema('id'))->normalize()], JSON_THROW_ON_ERROR); -compare('duplicated decoded name', [$narrowBodies[0] . "\x01" . pack('q', 9)], $duplicatedSchemaBody, $narrow->schema()); - -// the two-step path decodes every body before it casts; the fused path casts each row as it decodes it - a cast -// refusal in row 1 is reported ahead of a corrupt frame in row 2 only by the fused path -$precedence = rows(schema(int_schema('id'), str_schema('name')), row(['id' => 1, 'name' => '1']), row(['id' => 2, 'name' => 'x'])); -$precedenceBody = json_encode($precedence->schema()->normalize(), JSON_THROW_ON_ERROR); -$precedenceBodies = bodies($precedence, $precedence->schema()); -$precedenceInput = [$precedenceBodies[0], $precedenceBodies[1], $precedenceBodies[0] . "\xEF"]; -$castsName = schema(int_schema('id'), int_schema('name')); -echo 'precedence two-step: ', outcome(static fn(): Rows => (new RustRowHydratorNative())->hydrate((new RustFloeEncoderNative())->decode($precedenceInput, $precedenceBody), $castsName)), "\n"; -echo 'precedence fused: ', outcome(static fn(): Rows => (new RustFloeEncoderNative())->decodeRows($precedenceInput, $precedenceBody, $castsName)), "\n"; - -$encoder = new RustFloeEncoderNative(); -$first = $encoder->decodeRows($narrowBodies, $narrowSchemaBody, $narrow->schema()); -$second = $encoder->decodeRows(bodies($containers, $containers->schema()), json_encode($containers->schema()->normalize(), JSON_THROW_ON_ERROR), $containers->schema()); -$third = $encoder->decodeRows($narrowBodies, $narrowSchemaBody, $narrow->schema()); -echo 'plans rebind across schemas: ', $first->count(), ' ', $second->count(), ' ', serialize($first) === serialize($third) ? 'identical' : 'FAIL', "\n"; -?> ---EXPECT-- -017 rows: identical -021 rows: identical -nullable / datetime / list / map / structure / numeric name: identical -per-value metadata: identical -empty batch: identical -no frames: identical -read schema declares a nullable column the frames lack: identical -read schema declares a NOT NULL column the frames lack: identical - Flow\ETL\Exception\SchemaMismatchException: Rows do not match their schema: column "extra" (row 0) declared by the schema is missing from the row <- Flow\ETL\Exception\ColumnMismatchException: Row does not match its schema: column "extra" declared by the schema is missing from the row -read schema drops a decoded column: identical -read schema reorders columns: identical -read schema refuses a decoded value: identical - Flow\ETL\Exception\SchemaMismatchException: Rows do not match their schema: column "name" (row 0): could not convert 'a' (string) to integer <- Flow\ETL\Exception\ColumnMismatchException: Row does not match its schema: column "name": could not convert 'a' (string) to integer -read schema refuses a decoded null: identical - Flow\ETL\Exception\SchemaMismatchException: Rows do not match their schema: column "name" (row 0): could not convert null to string, column is not nullable <- Flow\ETL\Exception\ColumnMismatchException: Row does not match its schema: column "name": could not convert null to string, column is not nullable -trailing bytes: identical - Flow\Floe\Exception\ExtensionException: flow_php row frame length does not match its content -truncated body: identical - Flow\Floe\Exception\ExtensionException: flow_php frame body is truncated, string value is incomplete -unknown value flag: identical - Flow\Floe\Exception\ExtensionException: flow_php found unknown value flag 0x09 -duplicated decoded name: identical - Flow\Floe\Exception\ExtensionException: flow_php found duplicated entry name "id" in a row frame -precedence two-step: Flow\Floe\Exception\ExtensionException: flow_php row frame length does not match its content -precedence fused: Flow\ETL\Exception\SchemaMismatchException: Rows do not match their schema: column "name" (row 1): could not convert 'x' (string) to integer <- Flow\ETL\Exception\ColumnMismatchException: Row does not match its schema: column "name": could not convert 'x' (string) to integer -plans rebind across schemas: 2 3 identical diff --git a/src/extension/flow-php-ext/tests/phpt/056_native_constructors_refused.phpt b/src/extension/flow-php-ext/tests/phpt/056_native_constructors_refused.phpt new file mode 100644 index 0000000000..bcf56f693a --- /dev/null +++ b/src/extension/flow-php-ext/tests/phpt/056_native_constructors_refused.phpt @@ -0,0 +1,32 @@ +--TEST-- +native columns and builders come only from RustBackend; a column is not serializable, a Rows of them is +--SKIPIF-- + +--FILE-- + new RustColumn()); +expect_exception(static fn() => new RustColumnBuilder()); + +foreach ([RustColumn::class, RustColumnBuilder::class] as $class) { + echo outcome(static fn() => (new ReflectionClass($class))->newInstanceWithoutConstructor()), "\n"; +} + +$rows = native_rows(all_types_schema(), all_types_values()); + +echo get_class($rows->column('int')), "\n"; +echo outcome(static fn() => $rows->column('int')), "\n"; +echo comparable(unserialize(serialize($rows))->toArray()) === comparable($rows->toArray()) ? 'round trip' : 'FAIL', "\n"; +?> +--EXPECT-- +Flow\ETL\Exception\RuntimeException: Flow\ETL\Column\RustColumn is built by RustBackend +Flow\ETL\Exception\RuntimeException: Flow\ETL\Column\RustColumnBuilder is built by RustBackend +ReflectionException: Class Flow\ETL\Column\RustColumn is an internal class marked as final that cannot be instantiated without invoking its constructor +ReflectionException: Class Flow\ETL\Column\RustColumnBuilder is an internal class marked as final that cannot be instantiated without invoking its constructor +Flow\ETL\Column\RustColumn +Exception: Serialization of 'Flow\ETL\Column\RustColumn' is not allowed +round trip diff --git a/src/extension/flow-php-ext/tests/phpt/057_decode_rows_no_leaks.phpt b/src/extension/flow-php-ext/tests/phpt/057_decode_rows_no_leaks.phpt deleted file mode 100644 index e25fe89d01..0000000000 --- a/src/extension/flow-php-ext/tests/phpt/057_decode_rows_no_leaks.phpt +++ /dev/null @@ -1,98 +0,0 @@ ---TEST-- -repeated fused decodeRows, refusals included, does not leak memory ---SKIPIF-- - ---FILE-- - type_integer(), 'tags' => type_list(type_string())])), - json_schema('json'), -); - -$batch = []; - -for ($i = 1; $i <= 100; $i++) { - $batch[] = new RawRowValues( - [ - 'id' => $i, - 'name' => $i % 10 === 0 ? null : 'user_' . $i, // null values under a nullable declaration - 'price' => (float) $i / 100, - 'active' => $i % 3 === 0, - 'created_at' => new DateTimeImmutable('2025-01-01 00:00:00.123456', new DateTimeZone('Europe/Warsaw')), - 'duration' => new DateInterval('PT1H2M3S'), - 'uuid' => new Flow\Types\Value\Uuid('01234567-89ab-4def-8123-456789abcdef'), - 'tags' => ['x', 'user_' . $i], - 'metrics' => ['cpu' => 0.5, 'mem' => 0.25], - 'nested' => ['a' => $i, 'tags' => ['x']], - 'json' => Flow\Types\Value\Json::fromArray([$i, ['a' => true]]), - ], - $i % 5 === 0 ? ['id' => Metadata::fromArray(['batch' => $i])] : [], // exercises the clone + setMetadata path - ); -} - -$schemaBody = json_encode($schema->normalize(), JSON_THROW_ON_ERROR); -// the per-row metadata diverges from the file schema, so the bodies carry VALUE_*_WITH_META flags -$bodies = (new RustFloeEncoderNative())->encode((new RustRowHydratorNative())->dehydrate((new PhpRowHydrator())->hydrate($batch, $schema)), $schemaBody, $schema); -$refusing = schema(int_schema('id'), int_schema('name', nullable: true)); - -$cycle = static function () use ($bodies, $schemaBody, $schema, $refusing): void { - $native = new RustFloeEncoderNative(); - $native->decodeRows($bodies, $schemaBody, $schema); - - try { - $native->decodeRows($bodies, $schemaBody, $refusing); - } catch (Flow\ETL\Exception\SchemaMismatchException) { - } -}; - -for ($i = 0; $i < 10; $i++) { - $cycle(); -} -gc_collect_cycles(); -$baseline = memory_get_usage(false); - -for ($i = 0; $i < 100; $i++) { - $cycle(); -} -gc_collect_cycles(); - -var_dump(memory_get_usage(false) <= $baseline); -?> ---EXPECT-- -bool(true) diff --git a/src/extension/flow-php-ext/tests/phpt/057_native_encode_parity.phpt b/src/extension/flow-php-ext/tests/phpt/057_native_encode_parity.phpt new file mode 100644 index 0000000000..38e2798bbf --- /dev/null +++ b/src/extension/flow-php-ext/tests/phpt/057_native_encode_parity.phpt @@ -0,0 +1,47 @@ +--TEST-- +native columns encode byte-identical frames to PHP columns, after every Rows reshaping, and decode PHP frames +--SKIPIF-- + +--FILE-- + print( + $label . ': ' . (bin2hex($expected->encodeFrame()) === bin2hex($actual->encodeFrame()) ? 'identical' : 'DIFF') . "\n" +); +$projection = schema($schema->get('uuid'), $schema->get('int'), $schema->get('structure')); + +$frame('all types', $php, $native); +$frame('slice', $php->slice(3, 3), $native->slice(3, 3)); +$frame('gather', $php->gather([5, 0, 3, 3]), $native->gather([5, 0, 3, 3])); +$frame('concat', $php->concat(new PhpBackend(), $php->slice(1, 2)), $native->concat(new RustBackend(), $native->slice(1, 2))); +$frame('project', $php->project($projection, new Flow\ETL\Column\PhpBackend()), $native->project($projection, new Flow\ETL\Column\RustBackend())); +$frame('withSchema', $php->withSchema(all_types_schema('UTC')), $native->withSchema(all_types_schema('UTC'))); +$frame('mixed', $php, $php->concat(new PhpBackend(), $native)->slice(0, $php->count())); + +$body = $php->encodeFrame(); +$decodedPhp = (new FrameDecoder())->decode($body, $schema, new PhpBackend()); +$decodedNative = (new FrameDecoder())->decode($body, $schema, new RustBackend()); +echo get_class($decodedNative->column('int')), "\n"; +echo comparable($decodedPhp->toArray()) === comparable($decodedNative->toArray()) ? 'decoded rows identical' : 'DIFF', "\n"; +?> +--EXPECT-- +all types: identical +slice: identical +gather: identical +concat: identical +project: identical +withSchema: identical +mixed: identical +Flow\ETL\Column\RustColumn +decoded rows identical diff --git a/src/extension/flow-php-ext/tests/phpt/058_datetime_from_timestamp_parity.phpt b/src/extension/flow-php-ext/tests/phpt/058_datetime_from_timestamp_parity.phpt deleted file mode 100644 index e2fa8082a6..0000000000 --- a/src/extension/flow-php-ext/tests/phpt/058_datetime_from_timestamp_parity.phpt +++ /dev/null @@ -1,133 +0,0 @@ ---TEST-- -native datetime decode builds the same DateTimeImmutable as createFromFormat('U.u')->setTimezone(), state and behaviour ---SKIPIF-- - ---FILE-- -format('Y-m-d H:i:s.u T e P p O U I Z L N z t'), - $value->getTimestamp(), - $value->getOffset(), - $value->getTimezone()->getName(), - $value->modify('+1 month')->format('Y-m-d H:i:s.u T e'), - $value->modify('midnight')->format('Y-m-d H:i:s.u T e U'), - $value->modify('last day of next month noon')->format('Y-m-d H:i:s.u T e'), - $value->setTime(1, 2, 3, 4)->format('Y-m-d H:i:s.u T e U'), - $value->setDate(2024, 2, 29)->format('Y-m-d H:i:s.u T e U'), - $value->add(new DateInterval('P1DT25H'))->format('Y-m-d H:i:s.u T e U'), - $value->sub(new DateInterval('P1M'))->format('Y-m-d H:i:s.u T e U'), - $value->diff($fixed)->format('%R %y %m %d %h %i %s %f %a'), - $value->setTimezone(new DateTimeZone('UTC'))->format('Y-m-d H:i:s.u T e'), - $value->setTimestamp(86400)->format('Y-m-d H:i:s.u T e'), - $value == $fixed, - $value < $fixed, - ]; -} - -$timezones = ['UTC', 'Z', 'Europe/Warsaw', 'America/New_York', 'Australia/Lord_Howe', '+00:00', '+02:00', '-05:30', 'CEST', 'EST']; -$instants = [ - '1970-01-01 00:00:00.000000 UTC', - '1969-12-31 23:59:59.999999 UTC', - '1969-07-20 20:17:00.5 UTC', - '2026-03-29 00:59:59.999999 UTC', - '2026-03-29 01:00:00.000001 UTC', - '2026-10-25 00:30:00 UTC', - '2026-10-25 01:30:00 UTC', - '2038-01-19 03:14:08.000001 UTC', - '0001-01-01 00:00:00 UTC', - '9999-12-31 23:59:59.999999 UTC', - '-0044-03-15 12:00:00 UTC', -]; - -$values = []; - -foreach ($timezones as $timezone) { - foreach ($instants as $instant) { - $values[] = (new DateTimeImmutable($instant))->setTimezone(new DateTimeZone($timezone)); - } -} - -$bodies = (new PhpFloeEncoder(schema(datetime_schema('at'))))->encode( - array_map(static fn(DateTimeImmutable $at): TypedRowValues => new TypedRowValues(['at' => $at], ['at' => type_datetime()]), $values), -); - -foreach (['UTC', 'Europe/Warsaw', '+05:30'] as $zone) { - $schema = schema(datetime_schema('at', zone: $zone)); - $php = (new PhpFloeEncoder($schema))->decode($bodies); - $native = (new RustFloeEncoderNative())->decode($bodies, json_encode($schema->normalize(), JSON_THROW_ON_ERROR)); - $differences = 0; - $inZone = 0; - - foreach ($php as $index => $expected) { - if (observe($expected->values['at']) !== observe($native[$index]->values['at'])) { - $differences++; - echo 'FAIL: ', $values[$index]->format('Y-m-d H:i:s.u e'), "\n"; - var_dump(array_diff_assoc(observe($expected->values['at']), observe($native[$index]->values['at']))); - } - - $inZone += $native[$index]->values['at']->getTimezone()->getName() === $zone ? 1 : 0; - } - - printf("datetime %s: %d values, %d differences, %d in the column zone\n", $zone, count($php), $differences, $inZone); -} - -$dates = array_map(static fn(DateTimeImmutable $at): DateTimeImmutable => $at->setTime(0, 0), $values); -$dateSchema = schema(date_schema('on')); -$dateBodies = (new PhpFloeEncoder($dateSchema))->encode( - array_map(static fn(DateTimeImmutable $on): TypedRowValues => new TypedRowValues(['on' => $on], ['on' => type_date()]), $dates), -); -$php = (new PhpFloeEncoder($dateSchema))->decode($dateBodies); -$native = (new RustFloeEncoderNative())->decode($dateBodies, json_encode($dateSchema->normalize(), JSON_THROW_ON_ERROR)); -$differences = 0; -$own = 0; - -foreach ($php as $index => $expected) { - $differences += observe($expected->values['on']) === observe($native[$index]->values['on']) ? 0 : 1; - $own += $native[$index]->values['on']->getTimezone()->getName() === $dates[$index]->getTimezone()->getName() ? 1 : 0; -} - -printf("date: %d values, %d differences, %d kept their own zone\n", count($php), $differences, $own); - -$schemaBody = json_encode(schema(datetime_schema('at'))->normalize(), JSON_THROW_ON_ERROR); - -$overlong = pack('P', 0) . pack('V', 1_000_000) . pack('V', 3) . 'UTC'; - -try { - (new RustFloeEncoderNative())->decode(["\x01" . $overlong], $schemaBody); - echo "FAIL: no exception\n"; -} catch (Flow\Floe\Exception\ExtensionException $e) { - echo $e->getMessage(), "\n"; -} - -try { - (new PhpFloeEncoder(schema(datetime_schema('at'))))->decode(["\x01" . $overlong]); - echo "FAIL: no exception\n"; -} catch (Flow\Floe\Exception\FloeException $e) { - echo $e->getMessage(), "\n"; -} -?> ---EXPECT-- -datetime UTC: 110 values, 0 differences, 110 in the column zone -datetime Europe/Warsaw: 110 values, 0 differences, 110 in the column zone -datetime +05:30: 110 values, 0 differences, 110 in the column zone -date: 110 values, 0 differences, 110 kept their own zone -flow_php failed to restore datetime from timestamp "0" -Floe failed to restore datetime from timestamp "0" diff --git a/src/extension/flow-php-ext/tests/phpt/058_native_values_parity.phpt b/src/extension/flow-php-ext/tests/phpt/058_native_values_parity.phpt new file mode 100644 index 0000000000..ca6599127d --- /dev/null +++ b/src/extension/flow-php-ext/tests/phpt/058_native_values_parity.phpt @@ -0,0 +1,94 @@ +--TEST-- +native columns read every row as the PHP columns read it: at(), value(), Rows::values(), physicals(), values() +--SKIPIF-- + +--FILE-- +definitions()) as $name) { + $expected = $php->column((string) $name); + $actual = $native->column((string) $name); + + foreach (['physicals', 'values'] as $read) { + if (comparable($expected->{$read}()) !== comparable($actual->{$read}())) { + $differences[] = "{$name} {$read}()"; + } + } + + for ($i = 0; $i < $php->count(); $i++) { + foreach (['at', 'value', 'isNull'] as $read) { + if (comparable($expected->{$read}($i)) !== comparable($actual->{$read}($i))) { + $differences[] = "{$name} {$read}({$i})"; + } + } + } +} + +for ($i = 0; $i < $php->count(); $i++) { + if (comparable($php->values($i)) !== comparable($native->values($i))) { + $differences[] = "Rows::values({$i})"; + } +} + +echo $differences === [] ? 'all types identical' : implode("\n", $differences), "\n"; + +$observe = static fn(DateTimeInterface $value): array => [ + serialize($value), + $value->format('Y-m-d H:i:s.u T e P U Z'), + $value->modify('+1 month')->format('Y-m-d H:i:s.u T e'), + $value->modify('midnight')->format('Y-m-d H:i:s.u T e U'), + $value->setTime(1, 2, 3, 4)->format('Y-m-d H:i:s.u T e U'), + $value->add(new DateInterval('P1DT25H'))->format('Y-m-d H:i:s.u T e U'), +]; +$instants = []; + +foreach (['UTC', 'Z', 'Europe/Warsaw', 'America/New_York', 'Australia/Lord_Howe', '+00:00', '+02:00', '-05:30', 'CEST', 'EST'] as $timezone) { + foreach ([ + '1970-01-01 00:00:00.000000 UTC', + '1969-12-31 23:59:59.999999 UTC', + '1969-07-20 20:17:00.5 UTC', + '2026-03-29 00:59:59.999999 UTC', + '2026-03-29 01:00:00.000001 UTC', + '2026-10-25 00:30:00 UTC', + '2026-10-25 01:30:00 UTC', + '2038-01-19 03:14:08.000001 UTC', + '0001-01-01 00:00:00 UTC', + '9999-12-31 23:59:59.999999 UTC', + '-0044-03-15 12:00:00 UTC', + ] as $instant) { + $instants[] = ['at' => (new DateTimeImmutable($instant))->setTimezone(new DateTimeZone($timezone))]; + } +} + +$frame = php_rows(schema(datetime_schema('at')), $instants)->encodeFrame(); + +foreach (['UTC', 'Europe/Warsaw', '+05:30'] as $zone) { + $zoned = schema(datetime_schema('at', zone: $zone)); + $expected = array_map($observe, (new FrameDecoder())->decode($frame, $zoned, new PhpBackend())->column('at')->values()); + $actual = array_map($observe, (new FrameDecoder())->decode($frame, $zoned, new RustBackend())->column('at')->values()); + + echo "zone {$zone}: ", serialize($expected) === serialize($actual) ? 'identical' : 'DIFF', "\n"; +} + +foreach (native_rows(schema(time_schema('t')), [['t' => new DateInterval('PT1H')]])->column('t')->values() as $interval) { + var_dump($interval->days); +} +?> +--EXPECT-- +all types identical +zone UTC: identical +zone Europe/Warsaw: identical +zone +05:30: identical +bool(false) diff --git a/src/extension/flow-php-ext/tests/phpt/059_encode_frames.phpt b/src/extension/flow-php-ext/tests/phpt/059_encode_frames.phpt deleted file mode 100644 index 270386d375..0000000000 --- a/src/extension/flow-php-ext/tests/phpt/059_encode_frames.phpt +++ /dev/null @@ -1,137 +0,0 @@ ---TEST-- -RustFloeEncoderNative::encodeFrames() writes the bytes, and refuses with the message, of framing encode(dehydrate()) ---SKIPIF-- - ---FILE-- -getMessage(); - } -} - -function compare(string $label, Rows $rows, ?Schema $sessionSchema = null): void -{ - $sessionSchema ??= $rows->schema(); - $schemaBody = json_encode($sessionSchema->normalize(), JSON_THROW_ON_ERROR); - - $twoStep = outcome(static fn(): string => Format::rowFrames( - (new RustFloeEncoderNative())->encode((new RustRowHydratorNative())->dehydrate($rows), $schemaBody, $sessionSchema), - )); - $fused = outcome(static fn(): string => (new RustFloeEncoderNative())->encodeFrames($rows, $schemaBody, $sessionSchema)); - - echo $label, ': ', $twoStep === $fused ? 'identical' : "FAIL\n two-step: {$twoStep}\n fused: {$fused}", "\n"; - - if ($fused !== '' && !ctype_xdigit($fused)) { - echo ' ', $fused, "\n"; - } -} - -compare('017 rows', rows( - schema(int_schema('id'), str_schema('name', nullable: true), float_schema('price', nullable: true), datetime_schema('at', nullable: true)), - row(['id' => 1, 'name' => 'a']), - row(['id' => 2, 'name' => null]), - row(['id' => 3, 'price' => 1.5]), - row(['id' => 4, 'at' => new DateTimeImmutable('2025-01-01 00:00:00.123456', new DateTimeZone('Europe/Warsaw'))]), -)); - -compare('021 rows', rows( - schema(int_schema('id'), str_schema('name', nullable: true), float_schema('price')), - ...array_map(static fn(int $i) => row(['id' => $i, 'name' => $i % 7 === 0 ? null : 'user_' . $i, 'price' => $i / 4.0]), range(1, 500)), -)); - -compare('034 nullable note', rows( - schema(int_schema('id'), str_schema('name'), str_schema('note', nullable: true)), - row(['id' => 1, 'name' => 'a', 'note' => null]), -)); - -compare('nullable / datetime / list / map / structure / numeric name', rows( - schema( - int_schema('id'), - datetime_schema('zoned', nullable: true), - list_schema('tags', type_list(type_string()), nullable: true), - map_schema('scores', type_map(type_string(), type_integer())), - structure_schema('address', type_structure(['street' => type_string(), 'zip' => type_integer()]), nullable: true), - bool_schema('active'), - int_schema('1', nullable: true), - ), - row(['id' => 1, 'zoned' => new DateTimeImmutable('2026-03-01 10:11:12', new DateTimeZone('+02:00')), 'tags' => ['a', 'b'], 'scores' => ['x' => 1], 'address' => ['street' => 'Main', 'zip' => 12345], 'active' => true, '1' => 7]), - row(['id' => 2, 'zoned' => null, 'tags' => null, 'scores' => [], 'address' => null, 'active' => false, '1' => null]), -)); - -$sessionSchema = schema(int_schema('id'), str_schema('name', nullable: true), int_schema('7', nullable: true)); -compare('batch metadata diverging from the session schema', rows( - schema(int_schema('id'), str_schema('name', nullable: true, metadata: Metadata::fromArray(['source' => 'crm'])), int_schema('7', nullable: true, metadata: Metadata::fromArray(['n' => 1]))), - row(['id' => 1, 'name' => 'a', '7' => 1]), - row(['id' => 2, 'name' => null, '7' => null]), -), $sessionSchema); - -compare('session schema carrying metadata', rows( - schema(int_schema('id', metadata: Metadata::fromArray(['k' => 'v']))), - row(['id' => 1]), -)); - -compare('empty Rows', rows(schema(int_schema('id')))); - -$schema = schema(int_schema('id'), str_schema('name')); -compare('null under NOT NULL', Rows::trusted($schema, [row(['id' => 1, 'name' => 'a']), row(['id' => 2, 'name' => null])])); -compare('row without a declared column', Rows::trusted($schema, [row(['id' => 1])])); - -// the two-step path dehydrates every row before it encodes; the fused path encodes each row as it reads it - a null -// under NOT NULL in row 1 is reported ahead of a missing column in row 2 only by the fused path -$precedence = Rows::trusted($schema, [row(['id' => 1, 'name' => 'a']), row(['id' => 2, 'name' => null]), row(['id' => 3])]); -$precedenceBody = json_encode($schema->normalize(), JSON_THROW_ON_ERROR); -echo 'precedence two-step: ', outcome(static fn(): string => Format::rowFrames((new RustFloeEncoderNative())->encode((new RustRowHydratorNative())->dehydrate($precedence), $precedenceBody, $schema))), "\n"; -echo 'precedence fused: ', outcome(static fn(): string => (new RustFloeEncoderNative())->encodeFrames($precedence, $precedenceBody, $schema)), "\n"; - -$encoder = new RustFloeEncoderNative(); -$small = rows($schema, row(['id' => 1, 'name' => 'a'])); -$first = $encoder->encodeFrames($small, json_encode($schema->normalize(), JSON_THROW_ON_ERROR), $schema); -$other = schema(int_schema('id')); -$encoder->encodeFrames(rows($other, row(['id' => 9])), json_encode($other->normalize(), JSON_THROW_ON_ERROR), $other); -echo 'plans rebind across schemas: ', $first === $encoder->encodeFrames($small, json_encode($schema->normalize(), JSON_THROW_ON_ERROR), $schema) ? 'identical' : 'FAIL', "\n"; -?> ---EXPECT-- -017 rows: identical -021 rows: identical -034 nullable note: identical -nullable / datetime / list / map / structure / numeric name: identical -batch metadata diverging from the session schema: identical -session schema carrying metadata: identical -empty Rows: identical -null under NOT NULL: identical - Flow\ETL\Exception\SchemaMismatchException: Rows do not match their schema: column "name" (row 1): could not convert null to string, column is not nullable -row without a declared column: identical - Flow\Floe\Exception\ExtensionException: flow_php found a row that does not carry the declared column "name" -precedence two-step: Flow\Floe\Exception\ExtensionException: flow_php found a row that does not carry the declared column "name" -precedence fused: Flow\ETL\Exception\SchemaMismatchException: Rows do not match their schema: column "name" (row 1): could not convert null to string, column is not nullable -plans rebind across schemas: identical diff --git a/src/extension/flow-php-ext/tests/phpt/059_native_cast_parity.phpt b/src/extension/flow-php-ext/tests/phpt/059_native_cast_parity.phpt new file mode 100644 index 0000000000..7f7c72b41f --- /dev/null +++ b/src/extension/flow-php-ext/tests/phpt/059_native_cast_parity.phpt @@ -0,0 +1,107 @@ +--TEST-- +RustColumnBuilder casts and refuses exactly as the PHP builder through append() and appendMany(), in every default zone +--SKIPIF-- + +--FILE-- + 1]]], + [list_schema('l', type_list(type_integer())), [[1 => 'x']]], [list_schema('l', type_list(type_positive_integer())), [['abc']]], + [list_schema('l', type_list(type_positive_integer())), [[-3]]], [int_schema('i'), ['1', 'x']], + [int_schema('id'), ['9223372036854775808']], [date_schema('d'), ['']], [datetime_schema('at'), ['now']], + [list_schema('l', type_list(type_string())), [['a', null]]], [list_schema('l', type_list(type_integer())), [5]], + [new ForeignTypeDefinition('a', new ThrowingType(new LogicException('stub type refuses everything'))), [[1, 2]]], + [datetime_schema('at'), ['+292278994-08-17T07:12:55Z']], + [int_schema('id'), ["\f5"]], [int_schema('id'), ["5\f"]], [float_schema('p'), ['-0']], + [float_schema('p'), ['NAN']], [float_schema('p'), ['INF']], [float_schema('p'), ['-INF']], + [float_schema('p'), ['nan']], [float_schema('p'), ['Inf']], [float_schema('p'), ['+INF']], +] as $case) { + $cases[] = $case; +} + +foreach ([bool_schema('a'), str_schema('s'), date_schema('d'), datetime_schema('at')] as $definition) { + foreach ([NAN, INF, -INF] as $value) { + $cases[] = [$definition, [$value]]; + } +} + +$build = static fn(object $backend, object $definition, array $values, bool $many): Closure => static function () use ($backend, $definition, $values, $many): array { + $builder = $backend->builder($definition); + + if ($many) { + $builder->appendMany($values); + } else { + foreach ($values as $value) { + $builder->append($value); + } + } + + return $builder->finish()->values(); +}; + +foreach (['UTC', 'Europe/Warsaw', 'America/Santiago'] as $timezone) { + ini_set('date.timezone', $timezone); + $identical = 0; + + foreach ($cases as [$definition, $values]) { + foreach ([false, true] as $many) { + $php = outcome($build(new PhpBackend(), $definition, $values, $many)); + $native = outcome($build(new RustBackend(), $definition, $values, $many)); + + if ($php === $native) { + $identical++; + } else { + echo ($many ? 'appendMany ' : 'append ') . $definition->type()->toString() . ' ' . json_encode($values) . "\n php: {$php}\n native: {$native}\n"; + } + } + } + + echo "date.timezone {$timezone}: {$identical} of " . (2 * count($cases)) . " identical\n"; +} +?> +--EXPECT-- +date.timezone UTC: 470 of 470 identical +date.timezone Europe/Warsaw: 470 of 470 identical +date.timezone America/Santiago: 470 of 470 identical diff --git a/src/extension/flow-php-ext/tests/phpt/060_encode_frames_no_leaks.phpt b/src/extension/flow-php-ext/tests/phpt/060_encode_frames_no_leaks.phpt deleted file mode 100644 index 5a97dc7de5..0000000000 --- a/src/extension/flow-php-ext/tests/phpt/060_encode_frames_no_leaks.phpt +++ /dev/null @@ -1,98 +0,0 @@ ---TEST-- -repeated fused encodeFrames, refusals included, does not leak memory ---SKIPIF-- - ---FILE-- - type_integer(), 'tags' => type_list(type_string())])), - json_schema('json'), -); - -$batch = []; - -for ($i = 1; $i <= 100; $i++) { - $batch[] = new RawRowValues( - [ - 'id' => $i, - 'name' => $i % 10 === 0 ? null : 'user_' . $i, // null values under a nullable declaration - 'price' => (float) $i / 100, - 'active' => $i % 3 === 0, - 'created_at' => new DateTimeImmutable('2025-01-01 00:00:00.123456', new DateTimeZone('Europe/Warsaw')), - 'duration' => new DateInterval('PT1H2M3S'), - 'uuid' => new Flow\Types\Value\Uuid('01234567-89ab-4def-8123-456789abcdef'), - 'tags' => ['x', 'user_' . $i], - 'metrics' => ['cpu' => 0.5, 'mem' => 0.25], - 'nested' => ['a' => $i, 'tags' => ['x']], - 'json' => Flow\Types\Value\Json::fromArray([$i, ['a' => true]]), - ], - $i % 5 === 0 ? ['id' => Metadata::fromArray(['batch' => $i])] : [], // exercises the clone + setMetadata path - ); -} - -$schemaBody = json_encode($schema->normalize(), JSON_THROW_ON_ERROR); -// the hydrated batch schema carries id's folded metadata, which the session schema does not - VALUE_*_WITH_META frames -$rows = (new PhpRowHydrator())->hydrate($batch, $schema); -$refused = Flow\ETL\Rows::trusted($rows->schema(), [...$rows->all(), row(['id' => 101, 'name' => 'x'])]); - -$cycle = static function () use ($rows, $refused, $schemaBody, $schema): void { - $native = new RustFloeEncoderNative(); - $native->encodeFrames($rows, $schemaBody, $schema); - - try { - $native->encodeFrames($refused, $schemaBody, $schema); - } catch (Flow\Floe\Exception\ExtensionException) { - } -}; - -for ($i = 0; $i < 10; $i++) { - $cycle(); -} -gc_collect_cycles(); -$baseline = memory_get_usage(false); - -for ($i = 0; $i < 100; $i++) { - $cycle(); -} -gc_collect_cycles(); - -var_dump(memory_get_usage(false) <= $baseline); -?> ---EXPECT-- -bool(true) diff --git a/src/extension/flow-php-ext/tests/phpt/060_native_decode_refusals.phpt b/src/extension/flow-php-ext/tests/phpt/060_native_decode_refusals.phpt new file mode 100644 index 0000000000..e128bc6d44 --- /dev/null +++ b/src/extension/flow-php-ext/tests/phpt/060_native_decode_refusals.phpt @@ -0,0 +1,63 @@ +--TEST-- +RustBackend::decode() refuses every corrupt buffer set PhpBackend::decode() refuses, with its class and message +--SKIPIF-- + +--FILE-- + [int_schema('a'), [''], 1, 0], + 'short fixed buffer' => [int_schema('a'), ['', "\x01\x00"], 1, 0], + 'offsets not from 0' => [str_schema('a'), ['', "\x01\x00\x00\x00\x02\x00\x00\x00", 'ab'], 1, 0], + 'offsets not monotonic' => [str_schema('a'), ['', "\x00\x00\x00\x00\x02\x00\x00\x00\x01\x00\x00\x00", 'ab'], 2, 0], + 'offsets past the data' => [str_schema('a'), ['', "\x00\x00\x00\x00\x03\x00\x00\x00", 'ab'], 1, 0], + 'short offsets' => [str_schema('a'), ['', "\x00\x00", ''], 1, 0], + 'map entries validity present' => [map_schema('a', type_map(type_string(), type_integer())), ['', "\x00\x00\x00\x00\x00\x00\x00\x00", "\x01", '', "\x00\x00\x00\x00", '', '', ''], 1, 0], + 'null count disagreeing with the bitmap' => [int_schema('a', nullable: true), ["\x01", str_repeat("\x00", 16)], 2, 0], + 'values shorter than the rows' => [int_schema('a', nullable: true), ['', ''], 9, 0], + 'short validity bitmap' => [int_schema('a', nullable: true), ["\x01", str_repeat("\x00", 72)], 9, 0], + 'null column with a wrong null count' => [null_schema('a'), [], 2, 1], + 'leftover buffers' => [int_schema('a'), ['', "\x01\x00\x00\x00\x00\x00\x00\x00", ''], 1, 0], + 'list element nulls' => [list_schema('a', type_list(type_integer())), ['', "\x00\x00\x00\x00\x01\x00\x00\x00", "\x00", str_repeat("\x00", 8)], 1, 0], + 'list offsets too short' => [list_schema('a', type_list(type_integer())), ['', "\x00\x00\x00\x00", '', ''], 1, 0], + 'map key nulls' => [map_schema('a', type_map(type_integer(), type_integer())), ['', "\x00\x00\x00\x00\x01\x00\x00\x00", '', "\x00", str_repeat("\x00", 8), '', str_repeat("\x00", 8)], 1, 0], + 'map value nulls' => [map_schema('a', type_map(type_integer(), type_list(type_integer()))), ['', "\x00\x00\x00\x00\x01\x00\x00\x00", '', '', str_repeat("\x00", 8), "\x00", "\x00\x00\x00\x00\x00\x00\x00\x00", '', ''], 1, 0], + 'nested list of a structure' => [structure_schema('a', type_structure(['l' => type_list(type_string())])), ['', '', "\x00\x00\x00\x00\x01\x00\x00\x00", "\x00", "\x00\x00\x00\x00\x00\x00\x00\x00", ''], 1, 0], + 'optional list element nulls pass' => [list_schema('a', type_list(type_optional(type_integer()))), ['', "\x00\x00\x00\x00\x01\x00\x00\x00", "\x00", str_repeat("\x00", 8)], 1, 0], + 'null list element passes' => [list_schema('a', type_list(type_null())), ['', "\x00\x00\x00\x00\x02\x00\x00\x00"], 1, 0], +]; + +foreach ($cases as $label => [$definition, $buffers, $count, $nullCount]) { + $php = outcome(static fn() => (new PhpBackend())->decode($definition, $buffers, $count, $nullCount)->values()); + $native = outcome(static fn() => (new RustBackend())->decode($definition, $buffers, $count, $nullCount)->values()); + + echo $label, ': ', $php === $native ? $native : "DIFF\n php: {$php}\n native: {$native}", "\n"; +} +?> +--EXPECT-- +too few buffers: Flow\ETL\Exception\InvalidArgumentException: Column buffers exhausted: the column needs more buffers than given +short fixed buffer: Flow\ETL\Exception\InvalidArgumentException: Int64 values buffer of 2 bytes, expected 8 for 1 rows +offsets not from 0: Flow\ETL\Exception\InvalidArgumentException: Utf8 offsets start at 1, not 0 +offsets not monotonic: Flow\ETL\Exception\InvalidArgumentException: Utf8 offsets are not monotonic: offset 1 is 2, offset 2 is 1 +offsets past the data: Flow\ETL\Exception\InvalidArgumentException: Utf8 data buffer of 2 bytes, the last offset is 3 +short offsets: Flow\ETL\Exception\InvalidArgumentException: Utf8 offsets buffer of 2 bytes, expected 8 for 1 rows +map entries validity present: Flow\ETL\Exception\InvalidArgumentException: Map entries validity must be omitted, got 1 bytes +null count disagreeing with the bitmap: Flow\ETL\Exception\InvalidArgumentException: Column null count 0 disagrees with its validity bitmap (1 nulls in 2 rows) +values shorter than the rows: Flow\ETL\Exception\InvalidArgumentException: Int64 values buffer of 0 bytes, expected 72 for 9 rows +short validity bitmap: Flow\ETL\Exception\InvalidArgumentException: Validity bitmap of 1 bytes is too short for 9 rows +null column with a wrong null count: Flow\ETL\Exception\InvalidArgumentException: Null column of 2 rows carries a null count of 1 +leftover buffers: Flow\ETL\Exception\InvalidArgumentException: Column "a": 1 buffers left after decoding +list element nulls: Flow\ETL\Exception\InvalidArgumentException: List child holds 1 nulls in a non-nullable integer +list offsets too short: Flow\ETL\Exception\InvalidArgumentException: List offsets buffer of 4 bytes, expected 8 for 1 rows +map key nulls: Flow\ETL\Exception\InvalidArgumentException: Map key child holds 1 nulls in a non-nullable integer +map value nulls: Flow\ETL\Exception\InvalidArgumentException: Map value child holds 1 nulls in a non-nullable list +nested list of a structure: Flow\ETL\Exception\InvalidArgumentException: List child holds 1 nulls in a non-nullable string +optional list element nulls pass: a:1:{i:0;a:1:{i:0;N;}} +null list element passes: a:1:{i:0;a:2:{i:0;N;i:1;N;}} diff --git a/src/extension/flow-php-ext/tests/phpt/061_date_iso_fast_path.phpt b/src/extension/flow-php-ext/tests/phpt/061_date_iso_fast_path.phpt deleted file mode 100644 index b29f1556e9..0000000000 --- a/src/extension/flow-php-ext/tests/phpt/061_date_iso_fast_path.phpt +++ /dev/null @@ -1,114 +0,0 @@ ---TEST-- -ISO date strings: both hydrators read them in the column zone - midnight UTC for date, the declared zone for datetime - whatever date.timezone says ---SKIPIF-- - ---FILE-- - $value, 'at' => $value, 'at_scl' => $value])]; - $expected = serialize(new DateTimeImmutable($value, new DateTimeZone('UTC'))); - $expectedScl = serialize(new DateTimeImmutable($value, new DateTimeZone('America/Santiago'))); - $phpRow = $php->hydrate($batch, $schema)->first(); - $nativeRow = $native->hydrate($batch, $schema)->first(); - - printf( - "%-14s %s %s date php:%s native:%s datetime php:%s native:%s scl %s php:%s native:%s\n", - json_encode($value), - $nativeRow->get('on')->format('Y-m-d H:i:s'), - $nativeRow->get('on')->getTimezone()->getName(), - serialize($phpRow->get('on')) === $expected ? 'yes' : 'NO', - serialize($nativeRow->get('on')) === $expected ? 'yes' : 'NO', - serialize($phpRow->get('at')) === $expected ? 'yes' : 'NO', - serialize($nativeRow->get('at')) === $expected ? 'yes' : 'NO', - $nativeRow->get('at_scl')->format('Y-m-d H:i:s'), - serialize($phpRow->get('at_scl')) === $expectedScl ? 'yes' : 'NO', - serialize($nativeRow->get('at_scl')) === $expectedScl ? 'yes' : 'NO', - ); - } -} - -foreach (['2026-02-30', '2026-13-01', '0000-01-01'] as $value) { - foreach (['on' => date_schema('on'), 'at' => datetime_schema('at')] as $column => $definition) { - $batch = [new RawRowValues([$column => $value])]; - - foreach (['php' => $php, 'native' => $native] as $label => $hydrator) { - try { - $hydrator->hydrate($batch, schema($definition)); - echo "{$label} {$column} {$value}: FAIL no exception\n"; - } catch (Throwable $e) { - echo "{$label} {$column} {$value}: ", $e::class, "\n"; - } - } - } -} - -$halfPastMidnight = new DateTimeImmutable('2026-01-02 00:00:00.5', new DateTimeZone('UTC')); - -foreach (['php' => $php, 'native' => $native] as $label => $hydrator) { - echo $label, ' on 00:00:00.5: ', $hydrator->hydrate([new RawRowValues(['on' => $halfPastMidnight])], schema(date_schema('on')))->first()->get('on')->format('Y-m-d H:i:s.u e'), "\n"; -} -?> ---EXPECT-- -date.timezone UTC -"2026-01-02" 2026-01-02 00:00:00 UTC date php:yes native:yes datetime php:yes native:yes scl 2026-01-02 00:00:00 php:yes native:yes -"2024-02-29" 2024-02-29 00:00:00 UTC date php:yes native:yes datetime php:yes native:yes scl 2024-02-29 00:00:00 php:yes native:yes -"2026-01-02\n" 2026-01-02 00:00:00 UTC date php:yes native:yes datetime php:yes native:yes scl 2026-01-02 00:00:00 php:yes native:yes -"0001-01-01" 0001-01-01 00:00:00 UTC date php:yes native:yes datetime php:yes native:yes scl 0001-01-01 00:00:00 php:yes native:yes -"9999-12-31" 9999-12-31 00:00:00 UTC date php:yes native:yes datetime php:yes native:yes scl 9999-12-31 00:00:00 php:yes native:yes -"2026-09-06" 2026-09-06 00:00:00 UTC date php:yes native:yes datetime php:yes native:yes scl 2026-09-06 01:00:00 php:yes native:yes -date.timezone Europe/Warsaw -"2026-01-02" 2026-01-02 00:00:00 UTC date php:yes native:yes datetime php:yes native:yes scl 2026-01-02 00:00:00 php:yes native:yes -"2024-02-29" 2024-02-29 00:00:00 UTC date php:yes native:yes datetime php:yes native:yes scl 2024-02-29 00:00:00 php:yes native:yes -"2026-01-02\n" 2026-01-02 00:00:00 UTC date php:yes native:yes datetime php:yes native:yes scl 2026-01-02 00:00:00 php:yes native:yes -"0001-01-01" 0001-01-01 00:00:00 UTC date php:yes native:yes datetime php:yes native:yes scl 0001-01-01 00:00:00 php:yes native:yes -"9999-12-31" 9999-12-31 00:00:00 UTC date php:yes native:yes datetime php:yes native:yes scl 9999-12-31 00:00:00 php:yes native:yes -"2026-09-06" 2026-09-06 00:00:00 UTC date php:yes native:yes datetime php:yes native:yes scl 2026-09-06 01:00:00 php:yes native:yes -date.timezone America/Santiago -"2026-01-02" 2026-01-02 00:00:00 UTC date php:yes native:yes datetime php:yes native:yes scl 2026-01-02 00:00:00 php:yes native:yes -"2024-02-29" 2024-02-29 00:00:00 UTC date php:yes native:yes datetime php:yes native:yes scl 2024-02-29 00:00:00 php:yes native:yes -"2026-01-02\n" 2026-01-02 00:00:00 UTC date php:yes native:yes datetime php:yes native:yes scl 2026-01-02 00:00:00 php:yes native:yes -"0001-01-01" 0001-01-01 00:00:00 UTC date php:yes native:yes datetime php:yes native:yes scl 0001-01-01 00:00:00 php:yes native:yes -"9999-12-31" 9999-12-31 00:00:00 UTC date php:yes native:yes datetime php:yes native:yes scl 9999-12-31 00:00:00 php:yes native:yes -"2026-09-06" 2026-09-06 00:00:00 UTC date php:yes native:yes datetime php:yes native:yes scl 2026-09-06 01:00:00 php:yes native:yes -php on 2026-02-30: Flow\ETL\Exception\SchemaMismatchException -native on 2026-02-30: Flow\ETL\Exception\SchemaMismatchException -php at 2026-02-30: Flow\ETL\Exception\SchemaMismatchException -native at 2026-02-30: Flow\ETL\Exception\SchemaMismatchException -php on 2026-13-01: Flow\ETL\Exception\SchemaMismatchException -native on 2026-13-01: Flow\ETL\Exception\SchemaMismatchException -php at 2026-13-01: Flow\ETL\Exception\SchemaMismatchException -native at 2026-13-01: Flow\ETL\Exception\SchemaMismatchException -php on 0000-01-01: Flow\ETL\Exception\SchemaMismatchException -native on 0000-01-01: Flow\ETL\Exception\SchemaMismatchException -php at 0000-01-01: Flow\ETL\Exception\SchemaMismatchException -native at 0000-01-01: Flow\ETL\Exception\SchemaMismatchException -php on 00:00:00.5: 2026-01-02 00:00:00.000000 UTC -native on 00:00:00.5: 2026-01-02 00:00:00.000000 UTC diff --git a/src/extension/flow-php-ext/tests/phpt/061_native_mixed_class_building.phpt b/src/extension/flow-php-ext/tests/phpt/061_native_mixed_class_building.phpt new file mode 100644 index 0000000000..8053c0958b --- /dev/null +++ b/src/extension/flow-php-ext/tests/phpt/061_native_mixed_class_building.phpt @@ -0,0 +1,70 @@ +--TEST-- +native and PHP columns build into each other: appendFrom/appendTake, Rows::concat, Rows::gather, adopt() both ways +--SKIPIF-- + +--FILE-- + print( + $label . ': ' . ($expected->encodeFrame() === $actual->encodeFrame() && comparable($expected->toArray()) === comparable($actual->toArray()) ? 'identical' : 'DIFF') . "\n" +); +$rebuilt = static function (Rows $from, object $backend, bool $take) use ($schema): Rows { + $columns = []; + + foreach ($schema->definitions() as $name => $definition) { + $builder = $backend->builder($definition); + + if ($take) { + $builder->appendTake($from->column((string) $name), [5, 0, 3, 3]); + } else { + foreach ([5, 0, 3, 3] as $i) { + $builder->appendFrom($from->column((string) $name), $i); + } + } + + $columns[$name] = $builder->finish(); + } + + return Rows::fromColumns($schema, $columns, 4); +}; + +$same('native appendTake of a PHP column', $php->gather([5, 0, 3, 3]), $rebuilt($php, new RustBackend(), true)); +$same('native appendFrom of a PHP column', $php->gather([5, 0, 3, 3]), $rebuilt($php, new RustBackend(), false)); +$same('PHP appendTake of a native column', $php->gather([5, 0, 3, 3]), $rebuilt($native, new PhpBackend(), true)); +$same('Rows::concat PHP + native', $php->concat(new PhpBackend(), $php), $php->concat(new RustBackend(), $native)); +$same('Rows::concat native + PHP', $php->concat(new PhpBackend(), $php), $native->concat(new PhpBackend(), $php)); +$same('Rows::gather over both', $php->gather([1, 4]), $native->gather([1, 4])); + +foreach ($schema->definitions() as $name => $definition) { + $phpColumn = $php->column((string) $name); + $nativeColumn = $native->column((string) $name); + $adopted = (new RustBackend())->adopt($definition, $phpColumn); + $empty = (new RustBackend())->adopt($definition, $phpColumn->slice(0, 0)); + $back = (new PhpBackend())->adopt($definition, $nativeColumn); + + if (!$adopted instanceof RustColumn || $adopted->encode() !== $phpColumn->encode() || $empty->count() !== 0 + || $back instanceof RustColumn || $back->encode() !== $phpColumn->encode() + || (new RustBackend())->adopt($definition, $nativeColumn) !== $nativeColumn) { + echo "adopt DIFF {$name}\n"; + } +} + +echo "adopt identical\n"; +?> +--EXPECT-- +native appendTake of a PHP column: identical +native appendFrom of a PHP column: identical +PHP appendTake of a native column: identical +Rows::concat PHP + native: identical +Rows::concat native + PHP: identical +Rows::gather over both: identical +adopt identical diff --git a/src/extension/flow-php-ext/tests/phpt/062_datetime_zone_version_skew.phpt b/src/extension/flow-php-ext/tests/phpt/062_datetime_zone_version_skew.phpt deleted file mode 100644 index 8eab94d441..0000000000 --- a/src/extension/flow-php-ext/tests/phpt/062_datetime_zone_version_skew.phpt +++ /dev/null @@ -1,34 +0,0 @@ ---TEST-- -datetime column zones: the capability marker, and a zone-less datetime from an older flow-php/etl keeps each stored zone ---SKIPIF-- - ---FILE-- -datetimeZones()); -var_dump((new RustRowHydratorNative())->datetimeZones()); - -$bodies = (new PhpFloeEncoder(schema(datetime_schema('at'))))->encode([ - new TypedRowValues(['at' => new DateTimeImmutable('2026-01-02 03:04:05', new DateTimeZone('Europe/Warsaw'))], ['at' => type_datetime()]), -]); -$zoneless = '[{"ref":"at","type":{"type":"datetime"},"nullable":false,"metadata":[]}]'; -$zoned = json_encode(schema(datetime_schema('at'))->normalize(), JSON_THROW_ON_ERROR); - -echo 'zone-less datetime: ', (new RustFloeEncoderNative())->decode($bodies, $zoneless)[0]->values['at']->format('Y-m-d H:i:s e'), "\n"; -echo 'datetime: ', (new RustFloeEncoderNative())->decode($bodies, $zoned)[0]->values['at']->format('Y-m-d H:i:s e'), "\n"; -?> ---EXPECT-- -bool(true) -bool(true) -zone-less datetime: 2026-01-02 03:04:05 Europe/Warsaw -datetime: 2026-01-02 02:04:05 UTC diff --git a/src/extension/flow-php-ext/tests/phpt/062_native_no_leaks.phpt b/src/extension/flow-php-ext/tests/phpt/062_native_no_leaks.phpt new file mode 100644 index 0000000000..c38ef2f5f8 --- /dev/null +++ b/src/extension/flow-php-ext/tests/phpt/062_native_no_leaks.phpt @@ -0,0 +1,48 @@ +--TEST-- +decode, value reads, take, encode and a refused appendMany leak neither PHP memory nor native allocations +--SKIPIF-- + +--FILE-- +encodeFrame(); +$backend = new RustBackend(); + +$cycle = static function () use ($schema, $frame, $backend): void { + $rows = (new FrameDecoder())->decode($frame, $schema, $backend); + + foreach ($rows->columns() as $column) { + $column->value(1); + $column->take([2, 0])->encode(); + } + + try { + $backend->builder($schema->get('int'))->appendMany([1, 'not an int']); + throw new LogicException('a refused value was accepted'); + } catch (Flow\ETL\Exception\SchemaMismatchException) { + } +}; + +for ($i = 0; $i < 10; $i++) { + $cycle(); +} +gc_collect_cycles(); +$baseline = memory_get_usage(false); +$baselineRust = $backend->allocatedBytes(); + +for ($i = 0; $i < 100; $i++) { + $cycle(); +} +gc_collect_cycles(); + +var_dump(memory_get_usage(false) <= $baseline); +var_dump($backend->allocatedBytes() === $baselineRust); +?> +--EXPECT-- +bool(true) +bool(true) diff --git a/src/extension/flow-php-ext/tests/phpt/063_csv_next_columns_parity.phpt b/src/extension/flow-php-ext/tests/phpt/063_csv_next_columns_parity.phpt new file mode 100644 index 0000000000..7cf76e412a --- /dev/null +++ b/src/extension/flow-php-ext/tests/phpt/063_csv_next_columns_parity.phpt @@ -0,0 +1,78 @@ +--TEST-- +RustCSVOpenSource::batches() yields the frames or refusals of PhpCSVOpenSource::batches(), through either backend +--SKIPIF-- + +--FILE-- + new StringSourceStream(path('memory://phpt.csv'), $raw); +$batches = static fn(object $open, Schema $schema, object $backend): string => outcome(static function () use ($open, $schema, $backend): array { + $frames = []; + + foreach ($open->batches($schema, 2, $backend) as $batch) { + foreach ($batch->columns() as $column) { + if ($backend instanceof PhpBackend && $column instanceof RustColumn) { + return ['a RustColumn survived PhpBackend']; + } + } + + $frames[] = bin2hex($batch->encodeFrame()); + } + + return $frames; +}); +$cases = []; + +foreach (csv_narrow_corpus() as $cell) { + $raw = "v\n\"" . str_replace('"', '""', $cell) . "\"\n"; + + foreach ([int_schema('v'), float_schema('v'), bool_schema('v'), datetime_schema('v'), date_schema('v'), uuid_schema('v'), json_schema('v'), str_schema('v', nullable: true)] as $definition) { + $cases[] = [$raw, schema($definition)]; + } +} + +$idName = schema(int_schema('id'), str_schema('name')); + +foreach ([ + ["id\n1\n", $idName], + ["id\n1\nx\n", $idName], + ["a,b\nx,y\n", schema(int_schema('b'), int_schema('a'))], + ["id,name\n1,a\n2,\n", $idName], + ["id,name,name\n1,a,b\n2,c,d\n", $idName], + ["id,name\n1\n2,b,extra\n", schema(int_schema('id'), str_schema('name', nullable: true))], + ["1,2\na,3\n", schema(str_schema('1'), int_schema('2'))], + ["id,name\n1,a\n2,b\n3,c\nx,d\n", $idName], + ["id\n", schema(int_schema('id'), str_schema('absent'))], +] as $case) { + $cases[] = $case; +} + +$identical = 0; + +foreach ($cases as [$raw, $schema]) { + $php = $batches(new PhpCSVOpenSource($stream($raw), new CSVDecoder(), new CSVLineReader('"', ',', '\\')), $schema, new PhpBackend()); + + foreach ([new RustBackend(), new PhpBackend()] as $backend) { + $native = $batches(new RustCSVOpenSource($stream($raw), ',', '"', '\\', true, true, true), $schema, $backend); + + if ($native === $php) { + $identical++; + } else { + echo json_encode($raw), ' ', get_class($backend), "\n php: {$php}\n native: {$native}\n"; + } + } +} + +echo "{$identical} of " . (2 * count($cases)) . " identical\n"; +?> +--EXPECT-- +6274 of 6274 identical diff --git a/src/extension/flow-php-ext/tests/phpt/064_csv_next_columns_no_leaks.phpt b/src/extension/flow-php-ext/tests/phpt/064_csv_next_columns_no_leaks.phpt new file mode 100644 index 0000000000..76f4c2e8be --- /dev/null +++ b/src/extension/flow-php-ext/tests/phpt/064_csv_next_columns_no_leaks.phpt @@ -0,0 +1,49 @@ +--TEST-- +RustCSVOpenSource::batches() leaks neither PHP memory nor native allocations +--SKIPIF-- + +--FILE-- +batches($schema, 2, new RustBackend()) as $batch) { + } + + try { + foreach ((new RustCSVOpenSource(new MemorySourceStream("name\nx\n"), ',', '"', '\\', true, true, true))->batches($refusing, 2, new RustBackend()) as $batch) { + } + + throw new LogicException('a refused cell was accepted'); + } catch (Flow\ETL\Exception\SchemaMismatchException) { + } +}; + +for ($i = 0; $i < 10; $i++) { + $cycle(); +} +gc_collect_cycles(); +$baseline = memory_get_usage(false); +$baselineRust = (new RustBackend())->allocatedBytes(); + +for ($i = 0; $i < 1000; $i++) { + $cycle(); +} +gc_collect_cycles(); + +var_dump(memory_get_usage(false) <= $baseline); +var_dump((new RustBackend())->allocatedBytes() === $baselineRust); +?> +--EXPECT-- +bool(true) +bool(true) diff --git a/src/extension/flow-php-ext/tests/phpt/065_type_plan_by_object.phpt b/src/extension/flow-php-ext/tests/phpt/065_type_plan_by_object.phpt new file mode 100644 index 0000000000..5873158540 --- /dev/null +++ b/src/extension/flow-php-ext/tests/phpt/065_type_plan_by_object.phpt @@ -0,0 +1,32 @@ +--TEST-- +a column's plan follows its Type object across the plan cache's eviction and a freed Type's reused handle +--SKIPIF-- + +--FILE-- +builder($definition); + $builder->append($value); + $column = $builder->finish(); + + if ($column->type()->normalize() !== $definition->type()->normalize() || $column->value(0) !== $value) { + $wrong[] = $i; + } + + unset($definition, $builder, $column); +} + +echo $wrong === [] ? 'every column kept its type' : 'wrong at ' . implode(', ', array_slice($wrong, 0, 10)), "\n"; +?> +--EXPECT-- +every column kept its type diff --git a/src/extension/flow-php-ext/tests/phpt/066_iso_datetime_fast_path_parity.phpt b/src/extension/flow-php-ext/tests/phpt/066_iso_datetime_fast_path_parity.phpt new file mode 100644 index 0000000000..9717dad429 --- /dev/null +++ b/src/extension/flow-php-ext/tests/phpt/066_iso_datetime_fast_path_parity.phpt @@ -0,0 +1,132 @@ +--TEST-- +ISO date-times with a Z or ±HH:MM suffix become the micros DateTimeType::cast() yields, through append() and nextColumns() +--SKIPIF-- + +--FILE-- + str_pad((string) $value, 2, '0', STR_PAD_LEFT); +$leap = static fn(int $year): bool => ($year % 4 === 0 && $year % 100 !== 0) || $year % 400 === 0; +$year = static fn(): int => match (mt_rand(0, 3)) { + 0 => mt_rand(1, 1969), + 1 => mt_rand(1970, 2038), + 2 => mt_rand(2039, 9999), + default => [1, 4, 1900, 1968, 1969, 1970, 2000, 2024, 2038, 2100, 2400, 9996, 9999][mt_rand(0, 12)], +}; +$fraction = static fn(): string => ($digits = mt_rand(0, 6)) === 0 + ? '' + : '.' . implode('', array_map(static fn(): int => mt_rand(0, 9), range(1, $digits))); +$suffix = static function () use ($two): string { + if (mt_rand(0, 3) === 0) { + return 'Z'; + } + + $minutes = mt_rand(-12 * 60, 14 * 60); + $minutes -= mt_rand(0, 1) === 0 ? $minutes % 15 : 0; + + return ($minutes < 0 ? '-' : '+') . $two(intdiv(abs($minutes), 60)) . ':' . $two(abs($minutes) % 60); +}; + +$strings = []; + +for ($i = 0; $i < 3000; $i++) { + $y = $year(); + $m = mt_rand(1, 12); + $days = match ($m) { + 2 => $leap($y) ? 29 : 28, + 4, 6, 9, 11 => 30, + default => 31, + }; + $d = match (mt_rand(0, 3)) { + 0 => $days, + 1 => 1, + default => mt_rand(1, $days), + }; + + if (mt_rand(0, 9) === 0) { + [$y, $m, $d] = [[4, 2000, 2024, 2400, 9996][mt_rand(0, 4)], 2, 29]; + } + + $strings[] = sprintf( + '%s-%s-%sT%s:%s:%s%s%s', + str_pad((string) $y, 4, '0', STR_PAD_LEFT), + $two($m), + $two($d), + $two(mt_rand(0, 23)), + $two(mt_rand(0, 59)), + $two(mt_rand(0, 59)), + $fraction(), + $suffix(), + ); +} + +$nearMisses = [ + '2026-01-02T03:04:05', '2026-01-02 03:04:05Z', '2026-01-02t03:04:05Z', '2026-01-02T03:04:05z', '2026-01-02T03:04Z', + '2026-01-02T03:04:05.1234567Z', '2026-01-02T03:04:05.123456789+01:00', '2026-01-02T03:04:05,5Z', '2026-01-02T03:04:05.Z', + '2026-01-02T03:04:05+01', '2026-01-02T03:04:05+0100', '2026-01-02T03:04:05+24:00', '2026-01-02T03:04:05+01:60', + '2026-01-02T24:00:00Z', '2026-12-31T23:59:60Z', '0000-01-01T00:00:00Z', '+2026-01-02T03:04:05Z', '-2026-01-02T03:04:05Z', + '12026-01-02T03:04:05Z', '2023-02-29T00:00:00Z', '2026-04-31T00:00:00Z', '2026-13-01T00:00:00Z', "2026-01-02T03:04:05Z\n", + ' 2026-01-02T03:04:05Z', '2026-01-02T03:04:05Z ', '2026-01-02T03:04:05ZZ', '2026-01-02T03:04:05+01:00Z', +]; + +$physicals = static function (object $backend, object $definition, array $values): array { + $builder = $backend->builder($definition); + $builder->appendMany($values); + + return $builder->finish()->physicals(); +}; +$csv = static function (object $definition, array $values): array { + $source = new RustCSVOpenSource(new MemorySourceStream("v\n" . implode("\n", $values) . "\n"), ',', '"', '', true, true, true); + + return iterator_to_array($source->batches(schema($definition), count($values) + 1, new RustBackend()), false)[0]->column('v')->physicals(); +}; + +foreach (['UTC', 'Europe/Warsaw', 'Pacific/Kiritimati'] as $zone) { + $definition = datetime_schema('v', zone: $zone); + $expected = $physicals(new PhpBackend(), $definition, $strings); + $appended = $physicals(new RustBackend(), $definition, $strings); + $read = $csv($definition, $strings); + $differ = static fn(array $actual): array => array_keys(array_filter( + $expected, + static fn(int $micros, int $i): bool => $actual[$i] !== $micros, + ARRAY_FILTER_USE_BOTH, + )); + + foreach (['append()' => $appended, 'nextColumns()' => $read] as $lane => $actual) { + $wrong = $differ($actual); + echo "{$zone} {$lane}: ", $wrong === [] + ? count($strings) . ' strings, identical micros' + : implode("\n", array_map(static fn(int $i): string => "{$strings[$i]} php {$expected[$i]} native {$actual[$i]}", array_slice($wrong, 0, 10))), "\n"; + } + + $identical = 0; + + foreach ($nearMisses as $value) { + $php = outcome(static fn(): array => $physicals(new PhpBackend(), $definition, [$value])); + $native = outcome(static fn(): array => $physicals(new RustBackend(), $definition, [$value])); + + $php === $native ? $identical++ : print(json_encode($value) . "\n php: {$php}\n native: {$native}\n"); + } + + echo "{$zone} near misses: {$identical} of " . count($nearMisses) . " identical\n"; +} +?> +--EXPECT-- +UTC append(): 3000 strings, identical micros +UTC nextColumns(): 3000 strings, identical micros +UTC near misses: 27 of 27 identical +Europe/Warsaw append(): 3000 strings, identical micros +Europe/Warsaw nextColumns(): 3000 strings, identical micros +Europe/Warsaw near misses: 27 of 27 identical +Pacific/Kiritimati append(): 3000 strings, identical micros +Pacific/Kiritimati nextColumns(): 3000 strings, identical micros +Pacific/Kiritimati near misses: 27 of 27 identical diff --git a/src/extension/flow-php-ext/tests/phpt/067_adopt_refuses_untyped_column.phpt b/src/extension/flow-php-ext/tests/phpt/067_adopt_refuses_untyped_column.phpt new file mode 100644 index 0000000000..a57c2c4a34 --- /dev/null +++ b/src/extension/flow-php-ext/tests/phpt/067_adopt_refuses_untyped_column.phpt @@ -0,0 +1,21 @@ +--TEST-- +RustBackend::adopt() refuses an untyped (mixed) column with the PHP wording +--SKIPIF-- + +--FILE-- + (new RustBackend())->adopt(int_schema('a'), new ValueColumn([1, null]))), "\n"; +$typed = (new PhpBackend())->constant(int_schema('a', nullable: true), 1, 2); + +echo get_class((new RustBackend())->adopt(int_schema('a', nullable: true), $typed)), "\n"; +?> +--EXPECT-- +Flow\ETL\Exception\ColumnMismatchException: Row does not match its schema: column "a": mixed cannot be a batch column, an untyped function result exists only inside function evaluation +Flow\ETL\Column\RustColumn diff --git a/src/extension/flow-php-ext/tests/phpt/068_parquet_read_parity.phpt b/src/extension/flow-php-ext/tests/phpt/068_parquet_read_parity.phpt new file mode 100644 index 0000000000..002a3aa27b --- /dev/null +++ b/src/extension/flow-php-ext/tests/phpt/068_parquet_read_parity.phpt @@ -0,0 +1,126 @@ +--TEST-- +RustParquetOpenSource reads every lib Parquet fixture as PhpParquetEngine does, both refuse it, or it refuses naming the engine that reads it +--SKIPIF-- + + +--FILE-- +backend(new PhpBackend())->build()); + +foreach ([...glob($fixtures . '/*.parquet'), ...glob($fixtures . '/EdgeCases/*.parquet')] as $file) { + foreach ([100, 10_000] as $batchSize) { + $php = outcome(static function () use ($file, $batchSize, $context): array { + $rows = []; + + foreach (from_parquet($file, engine: new PhpParquetEngine())->withBatchSize($batchSize)->extract($context) as $batch) { + array_push($rows, ...$batch->toArray()); + } + + return $rows; + }); + $native = outcome(static function () use ($file, $batchSize, $filesystem): array { + $reader = rust_parquet_batches( + $filesystem->readFrom(path($file)), + (new SchemaConverter())->toFlow((new Reader(engine: new PhpParquetEngine()))->read($file)->schema()), + $batchSize, + null, + null, + ); + $rows = []; + + foreach ($reader as $batch) { + array_push($rows, ...$batch->toArray()); + } + + return $rows; + }); + $name = substr($file, strlen($fixtures) + 1) . ' ' . $batchSize; + + echo match (true) { + $php === $native && refused($php) => "{$name}: both refuse\n", + $php === $native => "{$name}: identical\n", + refused($php) && refused($native) => "{$name}: both refuse\n", + str_contains($native, 'is not supported by the arrow Parquet reader') => "{$name}: {$native}\n", + default => "{$name}: DIFFER\n php: " . substr($php, 0, 300) . "\n native: " . substr($native, 0, 300) . "\n", + }; + } +} + +$emptylist = realpath($fixtures . '/EdgeCases/null_list.parquet'); +echo outcome(static fn() => rust_parquet_batches( + $filesystem->readFrom(path($emptylist)), + schema(list_schema('emptylist', type_list(type_integer()))), + 100, + null, + null, +)), "\n"; +echo outcome(static fn() => rust_parquet_batches( + $filesystem->readFrom(path($fixtures . '/multiple_pages.parquet')), + schema(str_schema('int64')), + 100, + null, + null, +)), "\n"; +echo outcome(static fn() => rust_parquet_batches( + $filesystem->readFrom(path($fixtures . '/multiple_pages.parquet')), + schema(str_schema('missing')), + 100, + null, + null, +)), "\n"; +?> +--EXPECT-- +columns.required.parquet 100: identical +columns.required.parquet 10000: identical +decimals_fixed_len.parquet 100: identical +decimals_fixed_len.parquet 10000: identical +delta_binary_acked_encoded_integers.parquet 100: identical +delta_binary_acked_encoded_integers.parquet 10000: identical +lists.parquet 100: identical +lists.parquet 10000: identical +logical_types.parquet 100: identical +logical_types.parquet 10000: identical +maps.parquet 100: identical +maps.parquet 10000: identical +multiple_pages.parquet 100: identical +multiple_pages.parquet 10000: identical +pagination_row_group_1kb_5k_rows.snappy.parquet 100: identical +pagination_row_group_1kb_5k_rows.snappy.parquet 10000: identical +primitives.parquet 100: identical +primitives.parquet 10000: identical +structs.parquet 100: identical +structs.parquet 10000: identical +EdgeCases/datapage_v2.snappy.parquet 100: identical +EdgeCases/datapage_v2.snappy.parquet 10000: identical +EdgeCases/int96.parquet 100: both refuse +EdgeCases/int96.parquet 10000: both refuse +EdgeCases/interval.parquet 100: both refuse +EdgeCases/interval.parquet 10000: both refuse +EdgeCases/map_keys.parquet 100: both refuse +EdgeCases/map_keys.parquet 10000: both refuse +EdgeCases/nonnullable.impala.parquet 100: identical +EdgeCases/nonnullable.impala.parquet 10000: identical +EdgeCases/null_list.parquet 100: both refuse +EdgeCases/null_list.parquet 10000: both refuse +EdgeCases/uint64_overflow.parquet 100: both refuse +EdgeCases/uint64_overflow.parquet 10000: both refuse +EdgeCases/unsigned.parquet 100: identical +EdgeCases/unsigned.parquet 10000: identical +Flow\ETL\Exception\InvalidArgumentException: Parquet column "emptylist" (List(Null)) is read as List(Null), its schema type list stores List(Int64) +Flow\ETL\Exception\InvalidArgumentException: Parquet column "int64" (Int64) is read as Int64, its schema type string stores Binary +Flow\Parquet\Exception\InvalidArgumentException: Parquet file has no column "missing" diff --git a/src/extension/flow-php-ext/tests/phpt/069_parquet_offset_limit.phpt b/src/extension/flow-php-ext/tests/phpt/069_parquet_offset_limit.phpt new file mode 100644 index 0000000000..2aec1da3ea --- /dev/null +++ b/src/extension/flow-php-ext/tests/phpt/069_parquet_offset_limit.phpt @@ -0,0 +1,76 @@ +--TEST-- +RustParquetOpenSource reads offset/limit windows of an 8-row-group file, and refuses a batch size, offset or limit out of range +--SKIPIF-- + + +--FILE-- +writeTo(path('memory://groups.parquet')), + ArrowSchemaConverter::toExtension((new SchemaConverter())->toParquet($schema)), + Compressions::SNAPPY, + OptionsConverter::toExtension(Options::default()->set(Option::ROW_GROUP_SIZE_BYTES, 1)), + 1_000, +)); + +for ($group = 0; $group < 8; $group++) { + $writer->write(array_to_rows(array_map(static fn(int $id): array => ['id' => $id], range($group * 8192, $group * 8192 + 8191)), $schema)); +} + +$writer->close(); + +echo 'row groups: ', implode(',', array_map( + static fn($rowGroup): int => $rowGroup->rowsCount(), + (new Reader())->readStream($filesystem->readFrom(path('memory://groups.parquet')))->metadata()->rowGroups()->all(), +)), "\n"; + +foreach ([[0, 10], [5399, 3], [20000, 15000], [65000, 100], [65046, 1], [65536, 10], [65530, null]] as [$offset, $limit]) { + $reader = rust_parquet_batches($filesystem->readFrom(path('memory://groups.parquet')), $schema, 1000, $offset, $limit); + $ids = []; + + foreach ($reader as $batch) { + array_push($ids, ...$batch->column('id')->values()); + } + + printf("%d %s: %d rows, %s\n", $offset, $limit ?? 'null', count($ids), $ids === [] ? '-' : $ids[0] . '..' . $ids[count($ids) - 1]); + var_dump($ids === ($ids === [] ? [] : range($offset, $offset + count($ids) - 1))); +} + +foreach ([[0, null, null], [-1, null, null], [10, -1, null], [10, null, -1]] as [$batchSize, $offset, $limit]) { + echo outcome(static fn() => rust_parquet_batches($filesystem->readFrom(path('memory://groups.parquet')), $schema, $batchSize, $offset, $limit)), "\n"; +} +?> +--EXPECT-- +row groups: 8192,8192,8192,8192,8192,8192,8192,8192 +0 10: 10 rows, 0..9 +bool(true) +5399 3: 3 rows, 5399..5401 +bool(true) +20000 15000: 15000 rows, 20000..34999 +bool(true) +65000 100: 100 rows, 65000..65099 +bool(true) +65046 1: 1 rows, 65046..65046 +bool(true) +65536 10: 0 rows, - +bool(true) +65530 null: 6 rows, 65530..65535 +bool(true) +Flow\Parquet\Exception\InvalidArgumentException: arrow Parquet batch size must be greater than 0 +Flow\Parquet\Exception\InvalidArgumentException: arrow Parquet batch size must be greater than 0 +Flow\Parquet\Exception\InvalidArgumentException: arrow Parquet offset must be greater or equal to 0 +Flow\Parquet\Exception\InvalidArgumentException: arrow Parquet limit must be greater or equal to 0 diff --git a/src/extension/flow-php-ext/tests/phpt/070_parquet_write_roundtrip.phpt b/src/extension/flow-php-ext/tests/phpt/070_parquet_write_roundtrip.phpt new file mode 100644 index 0000000000..d8e2452276 --- /dev/null +++ b/src/extension/flow-php-ext/tests/phpt/070_parquet_write_roundtrip.phpt @@ -0,0 +1,121 @@ +--TEST-- +RustParquetOpenSink writes native and PHP columns that PhpParquetEngine reads back equal, and refuses a string that is not valid UTF-8 +--SKIPIF-- + + +--FILE-- + type_integer(), 'name' => type_optional(type_string())])), +); +$values = [ + [ + 'int' => -7, 'string' => 'zażółć', 'float' => 0.14, 'bool' => true, + 'datetime' => new DateTimeImmutable('1969-12-31 23:59:59.999999 UTC'), 'date' => new DateTimeImmutable('1900-01-01'), + 'time' => new DateInterval('PT1H2M3S'), 'uuid' => new Uuid('26fd21b0-6080-4d6c-bdb4-1214f1feffef'), + 'json' => '{"a":[1,2]}', 'list' => [1, null, 3], 'map' => ['a' => 1, 'b' => 2], 'structure' => ['id' => 1, 'name' => null], + ], + [ + 'int' => PHP_INT_MAX, 'string' => null, 'float' => 1234567890123456.8, 'bool' => false, + 'datetime' => new DateTimeImmutable('2026-09-29 12:00:00.5 UTC'), 'date' => new DateTimeImmutable('2026-09-29'), + 'time' => null, 'uuid' => new Uuid('00000000-0000-0000-0000-000000000000'), + 'json' => '[]', 'list' => [], 'map' => [], 'structure' => ['id' => 2, 'name' => 'x'], + ], +]; +$context = flow_context(config_builder()->backend(new PhpBackend())->build()); +$write = static function (Schema $schema, array $batches): void { + $writer = new RustParquetOpenSink(new RustParquetFileWriter( + memory_filesystem()->writeTo(path('memory://unused.parquet')), + ArrowSchemaConverter::toExtension((new SchemaConverter())->toParquet($schema)), + Compressions::ZSTD, + OptionsConverter::toExtension(Options::default()), + 1_000, + )); + + foreach ($batches as $batch) { + $writer->write($batch); + } + + $writer->close(); +}; + +$filesystem = memory_filesystem(); +$writer = new RustParquetOpenSink(new RustParquetFileWriter( + $filesystem->writeTo(path('memory://roundtrip.parquet')), + ArrowSchemaConverter::toExtension((new SchemaConverter())->toParquet($schema)), + Compressions::ZSTD, + OptionsConverter::toExtension(Options::default()), + 1_000, +)); +$writer->write(native_rows($schema, $values)); +$writer->write(php_rows($schema, $values)); +$writer->close(); + +$read = []; + +foreach (from_parquet(path('memory://roundtrip.parquet'), filesystem: $filesystem, engine: new PhpParquetEngine())->extract($context) as $batch) { + array_push($read, ...$batch->toArray()); +} + +var_dump(comparable($read) === comparable([...php_rows($schema, $values)->toArray(), ...php_rows($schema, $values)->toArray()])); + +$partial = memory_filesystem(); +$partialWriter = new RustParquetOpenSink(new RustParquetFileWriter( + $partial->writeTo(path('memory://partial.parquet')), + ArrowSchemaConverter::toExtension((new SchemaConverter())->toParquet(schema(int_schema('id'), str_schema('name', nullable: true)))), + Compressions::SNAPPY, + OptionsConverter::toExtension(Options::default()), + 1_000, +)); +$partialWriter->write(native_rows(schema(int_schema('id')), [['id' => 1], ['id' => 2]])); +$partialWriter->close(); +$partialRead = []; + +foreach (from_parquet(path('memory://partial.parquet'), filesystem: $partial, engine: new PhpParquetEngine())->extract($context) as $batch) { + array_push($partialRead, ...$batch->toArray()); +} + +echo json_encode($partialRead), "\n"; + +$names = schema(str_schema('name')); +echo outcome(static fn() => $write($names, [native_rows($names, [['name' => 'ok'], ['name' => "\xff\xfe"]])])), "\n"; +echo outcome(static fn() => $write($names, [php_rows($names, [['name' => "\xff\xfe"]])])), "\n"; + +$nested = schema(list_schema('tags', type_list(type_string()))); +echo outcome(static fn() => $write($nested, [native_rows($nested, [['tags' => ['a']], ['tags' => ['b', "\xff"]]])])), "\n"; +?> +--EXPECT-- +bool(true) +[{"id":1,"name":null},{"id":2,"name":null}] +Flow\Parquet\Exception\RuntimeException: Parquet column "name" row 1 holds a string that is not valid UTF-8; Parquet STRING columns require UTF-8 +Flow\Parquet\Exception\RuntimeException: Parquet column "name" row 0 holds a string that is not valid UTF-8; Parquet STRING columns require UTF-8 +Flow\Parquet\Exception\RuntimeException: Parquet column "tags" row 1 holds a string that is not valid UTF-8; Parquet STRING columns require UTF-8 diff --git a/src/extension/flow-php-ext/tests/phpt/071_parquet_no_leaks.phpt b/src/extension/flow-php-ext/tests/phpt/071_parquet_no_leaks.phpt new file mode 100644 index 0000000000..fc8b479d01 --- /dev/null +++ b/src/extension/flow-php-ext/tests/phpt/071_parquet_no_leaks.phpt @@ -0,0 +1,74 @@ +--TEST-- +RustParquetOpenSource and RustParquetOpenSink leak neither PHP memory nor native allocations, refusals included +--SKIPIF-- + + +--FILE-- + ['id' => $i, 'name' => $i % 3 ? "n{$i}" : null, 'tags' => [(string) $i, null]], range(0, 99)); +$extension = ArrowSchemaConverter::toExtension((new SchemaConverter())->toParquet($schema)); +$options = OptionsConverter::toExtension(Options::default()); +// built once: a fresh Type object per cycle would grow flow_php's type-plan cache, which is bounded, not leaked +$mismatched = schema(str_schema('id')); +$invalid = [['id' => 1, 'name' => "\xff", 'tags' => []]]; + +$cycle = static function () use ($schema, $values, $extension, $options, $mismatched, $invalid): void { + $filesystem = memory_filesystem(); + $writer = new RustParquetOpenSink(new RustParquetFileWriter($filesystem->writeTo(path('memory://leaks.parquet')), $extension, Compressions::SNAPPY, $options, 1_000)); + $writer->write(native_rows($schema, $values)); + $writer->write(php_rows($schema, $values)); + $writer->close(); + + $reader = rust_parquet_batches($filesystem->readFrom(path('memory://leaks.parquet')), $schema, 30, 10, 150); + + foreach ($reader as $_) { + } + + $refused = new RustParquetOpenSink(new RustParquetFileWriter($filesystem->writeTo(path('memory://refused.parquet')), $extension, Compressions::SNAPPY, $options, 1_000)); + + try { + $refused->write(native_rows($schema, $invalid)); + throw new LogicException('invalid UTF-8 was written'); + } catch (Flow\Parquet\Exception\RuntimeException) { + } + + try { + rust_parquet_batches($filesystem->readFrom(path('memory://leaks.parquet')), $mismatched, 30, null, null); + throw new LogicException('a mismatched schema was accepted'); + } catch (Flow\ETL\Exception\InvalidArgumentException) { + } +}; + +for ($i = 0; $i < 10; $i++) { + $cycle(); +} +gc_collect_cycles(); +$baseline = memory_get_usage(false); +$baselineRust = (new RustBackend())->allocatedBytes(); + +for ($i = 0; $i < 300; $i++) { + $cycle(); +} +gc_collect_cycles(); + +var_dump(memory_get_usage(false) <= $baseline); +var_dump((new RustBackend())->allocatedBytes() === $baselineRust); +?> +--EXPECT-- +bool(true) +bool(true) diff --git a/src/extension/flow-php-ext/tests/phpt/072_parquet_allocations_counted.phpt b/src/extension/flow-php-ext/tests/phpt/072_parquet_allocations_counted.phpt new file mode 100644 index 0000000000..6847bfde4c --- /dev/null +++ b/src/extension/flow-php-ext/tests/phpt/072_parquet_allocations_counted.phpt @@ -0,0 +1,68 @@ +--TEST-- +Batches RustParquetOpenSource holds are counted by RustBackend::allocatedBytes(): at least their columns' buffer bytes, and nothing once they are released +--SKIPIF-- + + +--FILE-- +writeTo(path('memory://held.parquet')), + ArrowSchemaConverter::toExtension((new SchemaConverter())->toParquet($schema)), + Compressions::SNAPPY, + OptionsConverter::toExtension(Options::default()), + 1_000, +)); + +foreach (array_chunk(range(0, 65_535), 8_192) as $ids) { + $writer->write(native_rows($schema, array_map(static fn(int $i): array => ['id' => $i, 'email' => "user{$i}@example.com", 'amount' => $i % 5 ? $i / 7 : null], $ids))); +} + +$writer->close(); +gc_collect_cycles(); + +$before = (new RustBackend())->allocatedBytes(); +$reader = rust_parquet_batches($filesystem->readFrom(path('memory://held.parquet')), $schema, 1_000, null, null); +$held = []; + +foreach ($reader as $batch) { + $held[] = $batch; +} + +unset($reader); +gc_collect_cycles(); + +$bufferBytes = 0; + +foreach ($held as $batch) { + foreach ($batch->columns() as $column) { + $bufferBytes += array_sum(array_map(strlen(...), $column->encode())); + } +} + +var_dump(array_sum(array_map(count(...), $held))); +var_dump((new RustBackend())->allocatedBytes() - $before >= $bufferBytes); + +unset($held, $batch, $column); +gc_collect_cycles(); + +var_dump((new RustBackend())->allocatedBytes() === $before); +?> +--EXPECT-- +int(65536) +bool(true) +bool(true) diff --git a/src/extension/flow-php-ext/tests/phpt/077_text_float_parity.phpt b/src/extension/flow-php-ext/tests/phpt/077_text_float_parity.phpt new file mode 100644 index 0000000000..1a9802e3f2 --- /dev/null +++ b/src/extension/flow-php-ext/tests/phpt/077_text_float_parity.phpt @@ -0,0 +1,74 @@ +--TEST-- +The native CSV and JSON writers write a float as the PHP writers do: 100 000 doubles from random bits plus the edge literals, under serialize_precision -1 and 17 +--SKIPIF-- + +--FILE-- + array_map( + static fn(array $chunk): Flow\ETL\Rows => $build($schema, array_map(static fn(float $f): array => ['f' => $f], $chunk)), + array_chunk($values, 10_000), +); + +foreach (['-1', '17'] as $precision) { + ini_set('serialize_precision', $precision); + ini_set('precision', $precision === '17' ? '5' : '14'); + + $php = implode('', array_map((new PhpCSVEncoder($options))->encode(...), $rows(php_rows(...), $floats))); + $native = implode('', array_map(static fn(Flow\ETL\Rows $batch): string => $csv->encode($batch), $rows(native_rows(...), $floats))); + printf("serialize_precision %s: csv %s, %d values\n", $precision, $php === $native ? 'identical' : 'DIFFERENT', substr_count($native, "\n")); + + foreach ([JSON_THROW_ON_ERROR, JSON_THROW_ON_ERROR | JSON_PRESERVE_ZERO_FRACTION] as $flags) { + $json = new RustJsonEncoder($flags, DATE_ATOM, 'Y-m-d', new PhpJsonEncoder($flags)); + $php = implode("\n", array_map(static fn(Flow\ETL\Rows $batch): string => (new PhpJsonEncoder($flags))->encode($batch, "\n"), $rows(php_rows(...), $finite))); + $native = implode("\n", array_map(static fn(Flow\ETL\Rows $batch): string => $json->encode($batch, "\n"), $rows(native_rows(...), $finite))); + printf("serialize_precision %s: json flags %d %s, %d values\n", $precision, $flags, $php === $native ? 'identical' : 'DIFFERENT', substr_count($native, "\n") + 1); + } + + foreach ([NAN, INF, -INF] as $float) { + $values = [['f' => 1.5], ['f' => $float]]; + $php = written(static fn() => (new PhpJsonEncoder())->encode(php_rows($schema, $values), "\n")); + $native = written(static fn() => (new RustJsonEncoder(JSON_THROW_ON_ERROR, DATE_ATOM, 'Y-m-d', new PhpJsonEncoder()))->encode(native_rows($schema, $values), "\n")); + printf("json %s: %s\n", var_export($float, true), $php === $native ? $php : 'DIFFERENT'); + } + + var_dump(ini_get('serialize_precision') === $precision); +} +?> +--EXPECT-- +serialize_precision -1: csv identical, 100020 values +serialize_precision -1: json flags 4194304 identical, 99973 values +serialize_precision -1: json flags 4195328 identical, 99973 values +json NAN: Flow\ETL\Exception\RuntimeException: Failed to encode JSON: Inf and NaN cannot be JSON encoded +json INF: Flow\ETL\Exception\RuntimeException: Failed to encode JSON: Inf and NaN cannot be JSON encoded +json -INF: Flow\ETL\Exception\RuntimeException: Failed to encode JSON: Inf and NaN cannot be JSON encoded +bool(true) +serialize_precision 17: csv identical, 100020 values +serialize_precision 17: json flags 4194304 identical, 99973 values +serialize_precision 17: json flags 4195328 identical, 99973 values +json NAN: Flow\ETL\Exception\RuntimeException: Failed to encode JSON: Inf and NaN cannot be JSON encoded +json INF: Flow\ETL\Exception\RuntimeException: Failed to encode JSON: Inf and NaN cannot be JSON encoded +json -INF: Flow\ETL\Exception\RuntimeException: Failed to encode JSON: Inf and NaN cannot be JSON encoded +bool(true) diff --git a/src/extension/flow-php-ext/tests/phpt/078_text_json_string_parity.phpt b/src/extension/flow-php-ext/tests/phpt/078_text_json_string_parity.phpt new file mode 100644 index 0000000000..2ff5ccd911 --- /dev/null +++ b/src/extension/flow-php-ext/tests/phpt/078_text_json_string_parity.phpt @@ -0,0 +1,103 @@ +--TEST-- +The native JSON writer escapes a string as json_encode() does: random valid and invalid UTF-8, every combination of the four flags +--SKIPIF-- + +--FILE-- + match (mt_rand(0, 9)) { + 0 => mt_rand(0, 0x1f), + 1, 2, 3 => mt_rand(0x20, 0x7f), + 4 => [0x22, 0x5c, 0x2f, 0x3c, 0x3e, 0x26, 0x27][mt_rand(0, 6)], + 5 => mt_rand(0x80, 0x7ff), + 6 => mt_rand(0x800, 0xd7ff), + 7 => [0x2027, 0x2028, 0x2029, 0x202a, 0xfffd, 0xfffe, 0xffff, 0xe000][mt_rand(0, 7)], + 8 => mt_rand(0x10000, 0x10ffff), + 9 => [0x10000, 0x10ffff, 0x1f600][mt_rand(0, 2)], +}; +$strings = ['', 'plain', '/', '"', '\\', "\u{2028}\u{2029}", "\xff", "a\xc3\x28", "\xed\xa0\x80", "\xc0\xaf", "\xf4\x90\x80\x80", "abc\xe2\x82", "\xf0\x9f\x98"]; + +for ($i = 0; $i < 1_500; $i++) { + $text = ''; + + for ($j = mt_rand(0, 12); $j > 0; $j--) { + $text .= mb_chr($codePoint(), 'UTF-8'); + } + + $strings[] = $text; +} + +for ($i = 0; $i < 500; $i++) { + $strings[] = random_bytes_seeded(mt_rand(1, 8)); +} + +function random_bytes_seeded(int $length): string +{ + $bytes = ''; + + for ($i = 0; $i < $length; $i++) { + $bytes .= chr(mt_rand(0, 255)); + } + + return $bytes; +} + +$schema = schema(str_schema('s')); +$php = array_map(static fn(string $text): Flow\ETL\Rows => php_rows($schema, [['s' => $text]]), $strings); +$native = array_map(static fn(string $text): Flow\ETL\Rows => native_rows($schema, [['s' => $text]]), $strings); +$encoded = static function (callable $encode): string { + try { + return $encode(); + } catch (Throwable $e) { + return $e::class . ': ' . $e->getMessage(); + } +}; + +for ($combination = 0; $combination < 16; $combination++) { + $flags = ($combination & 1 ? JSON_THROW_ON_ERROR : 0) + | ($combination & 2 ? JSON_UNESCAPED_SLASHES : 0) + | ($combination & 4 ? JSON_UNESCAPED_UNICODE : 0) + | ($combination & 8 ? JSON_PRESERVE_ZERO_FRACTION : 0); + $encoder = new PhpJsonEncoder($flags); + $writer = new RustJsonEncoder($flags, DATE_ATOM, 'Y-m-d', $encoder); + $different = 0; + $refused = 0; + + foreach ($strings as $i => $text) { + $expected = $encoded(static fn(): string => $encoder->encode($php[$i], "\n")); + $refused += (int) refused($expected); + + if ($expected !== $encoded(static fn(): string => $writer->encode($native[$i], "\n")) && ++$different <= 3) { + echo bin2hex($text), ': ', $expected, "\n"; + } + } + + printf("flags %7d: %d of %d differ, %d refused\n", $flags, $different, count($strings), $refused); +} +?> +--EXPECT-- +flags 0: 0 of 2013 differ, 442 refused +flags 4194304: 0 of 2013 differ, 442 refused +flags 64: 0 of 2013 differ, 442 refused +flags 4194368: 0 of 2013 differ, 442 refused +flags 256: 0 of 2013 differ, 442 refused +flags 4194560: 0 of 2013 differ, 442 refused +flags 320: 0 of 2013 differ, 442 refused +flags 4194624: 0 of 2013 differ, 442 refused +flags 1024: 0 of 2013 differ, 442 refused +flags 4195328: 0 of 2013 differ, 442 refused +flags 1088: 0 of 2013 differ, 442 refused +flags 4195392: 0 of 2013 differ, 442 refused +flags 1280: 0 of 2013 differ, 442 refused +flags 4195584: 0 of 2013 differ, 442 refused +flags 1344: 0 of 2013 differ, 442 refused +flags 4195648: 0 of 2013 differ, 442 refused diff --git a/src/extension/flow-php-ext/tests/phpt/079_text_date_format_parity.phpt b/src/extension/flow-php-ext/tests/phpt/079_text_date_format_parity.phpt new file mode 100644 index 0000000000..49f4718356 --- /dev/null +++ b/src/extension/flow-php-ext/tests/phpt/079_text_date_format_parity.phpt @@ -0,0 +1,124 @@ +--TEST-- +The native writers format a datetime and a date as DateTimeInterface::format() does: every rendered letter and random formats, five zones, instants around gaps, overlaps and extreme years +--SKIPIF-- + +--FILE-- + $letter . ' | ' . $letter, $letters); +$formats[] = implode('|', $letters); +$formats[] = 'Y-m-d\TH:i:s.uP'; +$formats[] = '\Y\m\d Y \\\\ \u u'; + +for ($i = 0; $i < 60; $i++) { + $format = ''; + + for ($j = mt_rand(1, 8); $j > 0; $j--) { + $format .= [$letters[mt_rand(0, count($letters) - 1)], ['-', ':', ' ', '.', '/', '\\T', '\\e', 'ż', ','][mt_rand(0, 8)]][mt_rand(0, 1)]; + } + + $formats[] = $format; +} + +$at = static fn(int $second, int $micro = 0): DateTimeImmutable => (new DateTimeImmutable('@' . $second))->modify('+' . $micro . ' usec'); +// each side of the 2026 gaps and overlaps of Europe/Warsaw, America/New_York and Australia/Lord_Howe (a 30 minute shift) +$edges = [1_774_745_999, 1_774_746_000, 1_792_889_999, 1_792_890_000, 1_772_953_199, 1_772_953_200, 1_793_512_799, 1_793_512_800, 1_775_316_599, 1_775_316_600, 1_791_041_399, 1_791_041_400]; +$instants = [$at(0), $at(-1, 999_999), $at(0, 999_999), $at(-2_208_988_800), $at(-2_524_521_600, 1), $at(-5_000_000_000, 500_000)]; + +foreach ($edges as $edge) { + $instants[] = $at($edge, mt_rand(0, 1) * 999_999); +} + +for ($i = 0; $i < 200; $i++) { + $instants[] = $at(mt_rand(-3_000_000_000, 5_000_000_000), mt_rand(0, 999_999)); +} + +// years 999, 0, -1, 10 000 and 100 000: one row each, a named zone lists its transitions over the batch's range +$extremes = [-30_641_760_000, -62_167_219_200, -62_198_755_200, 253_402_300_800, 3_093_527_980_800]; +$different = 0; +$compared = 0; + +foreach (['UTC', '+02:30', 'Europe/Warsaw', 'America/New_York', 'Australia/Lord_Howe'] as $zone) { + $schema = schema(datetime_schema('at', zone: $zone)); + $batches = [array_map(static fn(DateTimeImmutable $instant): array => ['at' => $instant], $instants)]; + + foreach ($extremes as $second) { + $batches[] = [['at' => $at($second)], ['at' => $at($second + 86_399, 999_999)]]; + } + + foreach ($batches as $values) { + $rows = native_rows($schema, $values); + + foreach ($formats as $format) { + $recorder = new RecordingPhpCSVEncoder(new PhpCSVEncoder(new CSVWriteOptions("\x01", "\x02", '', "\n", $format, 'Y-m-d'))); + $writer = new RustCSVEncoder("\x01", "\x02", '', "\n", $format, 'Y-m-d', $recorder); + $expected = ''; + + foreach ($values as $value) { + $expected .= $value['at']->setTimezone(new DateTimeZone($zone))->format($format) . "\n"; + } + + $compared++; + // a field holding a space is enclosed: the enclosure is dropped, no format writes that byte + $written = str_replace("\x02", '', $writer->encode($rows)); + + if ($recorder->columns !== []) { + throw new LogicException("the format {$format} is not rendered natively"); + } + + if ($written !== $expected && ++$different <= 5) { + echo "{$zone} {$format}\n php: ", substr(json_encode($expected), 0, 300), "\n native: ", substr(json_encode($written), 0, 300), "\n"; + } + } + } +} + +echo "datetime: {$different} of {$compared} differ\n"; + +$schema = schema(date_schema('on')); +$days = [0, -1, 1, 20_455, -25_567, -719_528, -719_893, 2_932_896, 35_804_721]; +$different = 0; + +foreach ($formats as $format) { + $writer = new RustCSVEncoder("\x01", "\x02", '', "\n", DATE_ATOM, $format, new PhpCSVEncoder(new CSVWriteOptions("\x01", "\x02", '', "\n", DATE_ATOM, $format))); + $expected = ''; + + foreach ($days as $day) { + $expected .= (new DateTimeImmutable('@' . ($day * 86_400)))->setTimezone(new DateTimeZone('UTC'))->format($format) . "\n"; + } + + $written = str_replace("\x02", '', $writer->encode(native_rows($schema, array_map(static fn(int $day): array => ['on' => new DateTimeImmutable('@' . ($day * 86_400))], $days)))); + + if ($written !== $expected && ++$different <= 5) { + echo "date {$format}\n php: ", substr(json_encode($expected), 0, 300), "\n native: ", substr(json_encode($written), 0, 300), "\n"; + } +} + +echo "date: {$different} of " . count($formats) . " differ\n"; + +foreach (['D, d M Y', 'Y-m-d I', 'N', 'Y\\'] as $format) { + $recorder = new RecordingPhpCSVEncoder(new PhpCSVEncoder(new CSVWriteOptions(',', '"', '\\', "\n", $format, $format))); + $rows = native_rows(schema(datetime_schema('at'), date_schema('on')), [['at' => new DateTimeImmutable('2024-01-02 03:04:05'), 'on' => new DateTimeImmutable('2024-01-02')]]); + (new RustCSVEncoder(',', '"', '\\', "\n", $format, $format, $recorder))->encode($rows); + printf("%s: %s\n", $format, json_encode(recorded_names($recorder->columns, $rows))); +} +?> +--EXPECT-- +datetime: 0 of 2490 differ +date: 0 of 83 differ +D, d M Y: ["at","on"] +Y-m-d I: ["at","on"] +N: ["at","on"] +Y\: ["at","on"] diff --git a/src/extension/flow-php-ext/tests/phpt/080_csv_write_parity.phpt b/src/extension/flow-php-ext/tests/phpt/080_csv_write_parity.phpt new file mode 100644 index 0000000000..75eaf5dc48 --- /dev/null +++ b/src/extension/flow-php-ext/tests/phpt/080_csv_write_parity.phpt @@ -0,0 +1,76 @@ +--TEST-- +CSVOpenSink writes the same bytes through RustCSVEncoder and PhpCSVEncoder: seeded random batches over every kind, nesting <= 3, random options +--SKIPIF-- + +--FILE-- +separator === $options->enclosure) { + continue; + } + + $header = mt_rand(0, 1) === 1; + $recorder = new RecordingPhpCSVEncoder(new PhpCSVEncoder($options)); + $writer = new RustCSVEncoder($options->separator, $options->enclosure, $options->escape, $options->newLineSeparator, $options->dateTimeFormat, $options->dateFormat, $recorder); + // a second batch proves the header is written once and the encoders hold no state across batches + $half = intdiv(count($rows), 2); + + $php = written(static function (DestinationStream $stream) use ($schema, $rows, $options, $header, $half): void { + $sink = new CSVOpenSink($stream, new PhpCSVEncoder($options), $header); + $sink->write(php_rows($schema, array_slice($rows, 0, $half))); + $sink->write(php_rows($schema, array_slice($rows, $half))); + }); + $native = written(static function (DestinationStream $stream) use ($schema, $rows, $header, $half, $writer): void { + $sink = new CSVOpenSink($stream, $writer, $header); + $sink->write(native_rows($schema, array_slice($rows, 0, $half))); + // a PHP column inside the batch is adopted + $sink->write(php_rows($schema, array_slice($rows, $half))); + }); + + $refused += (int) refused($php); + $unrendered += count($recorder->columns); + + if ($php === $native) { + $identical++; + } elseif (++$different <= 3) { + echo "batch {$i}: ", json_encode(array_map(static fn($d) => $d->type()->toString(), $schema->definitions())), ' ', json_encode([$options->separator, $options->enclosure, $options->escape, $options->dateTimeFormat, $options->dateFormat]), "\n php: ", substr(json_encode($php, JSON_INVALID_UTF8_SUBSTITUTE), 0, 600), "\n native: ", substr(json_encode($native, JSON_INVALID_UTF8_SUBSTITUTE), 0, 600), "\n"; + } +} + +echo "{$different} of " . ($identical + $different) . " batches differ\n"; +echo $refused === 0 && $unrendered > 0 ? "every batch was written, some columns through PHP cells\n" : "{$refused} refusals, {$unrendered} columns left to PHP\n"; +?> +--EXPECT-- +0 of 2000 batches differ +every batch was written, some columns through PHP cells diff --git a/src/extension/flow-php-ext/tests/phpt/081_json_write_parity.phpt b/src/extension/flow-php-ext/tests/phpt/081_json_write_parity.phpt new file mode 100644 index 0000000000..5ba7fac832 --- /dev/null +++ b/src/extension/flow-php-ext/tests/phpt/081_json_write_parity.phpt @@ -0,0 +1,66 @@ +--TEST-- +JsonOpenSink writes the same bytes through RustJsonEncoder and PhpJsonEncoder: seeded random batches over every kind, nesting <= 3, random flags and framings +--SKIPIF-- + +--FILE-- + 0 ? JSON_THROW_ON_ERROR : 0) + | (mt_rand(0, 1) ? JSON_UNESCAPED_SLASHES : 0) + | (mt_rand(0, 1) ? JSON_UNESCAPED_UNICODE : 0) + | (mt_rand(0, 1) ? JSON_PRESERVE_ZERO_FRACTION : 0); + $framing = JsonFraming::cases()[mt_rand(0, 2)]; + $dateTimeFormat = $dateTimeFormats[mt_rand(0, count($dateTimeFormats) - 1)]; + $dateFormat = $dateFormats[mt_rand(0, count($dateFormats) - 1)]; + $recorder = new RecordingPhpJsonEncoder(new PhpJsonEncoder($flags, $dateTimeFormat, $dateFormat)); + $writer = new RustJsonEncoder($flags, $dateTimeFormat, $dateFormat, $recorder); + $half = intdiv(count($rows), 2); + + $php = written(static function (DestinationStream $stream) use ($schema, $rows, $flags, $dateTimeFormat, $dateFormat, $framing, $half): void { + $sink = new JsonOpenSink($stream, new PhpJsonEncoder($flags, $dateTimeFormat, $dateFormat), $framing); + $sink->write(php_rows($schema, array_slice($rows, 0, $half))); + $sink->write(php_rows($schema, array_slice($rows, $half))); + $sink->close(); + }); + $native = written(static function (DestinationStream $stream) use ($schema, $rows, $flags, $dateTimeFormat, $dateFormat, $framing, $half, $writer): void { + $sink = new JsonOpenSink($stream, $writer, $framing); + $sink->write(native_rows($schema, array_slice($rows, 0, $half))); + // a PHP column inside the batch is adopted + $sink->write(php_rows($schema, array_slice($rows, $half))); + $sink->close(); + }); + + $refused += (int) refused($php); + $unrendered += count($recorder->columns); + + if ($php === $native) { + $identical++; + } elseif (++$different <= 3) { + echo "batch {$i}: ", json_encode(array_map(static fn($d) => $d->type()->toString(), $schema->definitions())), ' ', json_encode([$flags, $framing->name, $dateTimeFormat, $dateFormat]), "\n php: ", substr(json_encode($php, JSON_INVALID_UTF8_SUBSTITUTE), 0, 600), "\n native: ", substr(json_encode($native, JSON_INVALID_UTF8_SUBSTITUTE), 0, 600), "\n"; + } +} + +echo "{$different} of " . ($identical + $different) . " batches differ\n"; +echo $refused === 0 && $unrendered > 0 ? "every batch was written, some columns through PHP fragments\n" : "{$refused} refusals, {$unrendered} columns left to PHP\n"; +?> +--EXPECT-- +0 of 2000 batches differ +every batch was written, some columns through PHP fragments diff --git a/src/extension/flow-php-ext/tests/phpt/082_write_no_leaks.phpt b/src/extension/flow-php-ext/tests/phpt/082_write_no_leaks.phpt new file mode 100644 index 0000000000..2112c6bf1f --- /dev/null +++ b/src/extension/flow-php-ext/tests/phpt/082_write_no_leaks.phpt @@ -0,0 +1,99 @@ +--TEST-- +The native CSV and JSON writers leak neither PHP memory nor native allocations over 1 000 encodes, refusals included +--SKIPIF-- + +--FILE-- +definitions(), static fn($definition): bool => $definition->entry()->name() !== 'string')); +$refusedSchema = schema(str_schema('s'), float_schema('f'), list_schema('l', type_list(type_float()))); +$refused = native_rows($refusedSchema, [['s' => "\xff", 'f' => NAN, 'l' => [INF]]]); +$dateTimeFormat = 'Y-m-d\TH:i:s.uP T'; +$csv = new RustCSVEncoder(',', '"', '\\', "\n", $dateTimeFormat, 'Y-m-d', new PhpCSVEncoder(new CSVWriteOptions(newLineSeparator: "\n", dateTimeFormat: $dateTimeFormat))); +$json = new RustJsonEncoder(JSON_THROW_ON_ERROR, $dateTimeFormat, 'Y-m-d', new PhpJsonEncoder(JSON_THROW_ON_ERROR, $dateTimeFormat)); +// stand-ins for the held encoder: a cell list of the wrong length, and an encoder that throws +$short = new class { + public function cells(): array + { + return ['too', 'few']; + } + + public function fragments(): array + { + return ['"too"', '"few"']; + } +}; +$throwing = new class { + public function cells(): array + { + throw new Flow\ETL\Exception\RuntimeException('cells refused'); + } + + public function fragments(): array + { + throw new Flow\ETL\Exception\RuntimeException('fragments refused'); + } +}; +$backend = new RustBackend(); + +$cycle = static function () use ($schema, $jsonSchema, $native, $php, $refused, $csv, $json, $short, $throwing, $dateTimeFormat): void { + $csv->encode($native); + // PHP columns are adopted for the encode and freed after it + $csv->encode($php); + $json->encode($native->project($jsonSchema, new Flow\ETL\Column\RustBackend()), "\n"); + $json->encode($php->project($jsonSchema, new Flow\ETL\Column\PhpBackend()), ','); + + foreach ([ + static fn() => $csv->encode($refused), + static fn() => $json->encode($refused, "\n"), + static fn() => $json->encode($native, "\n"), + static fn() => (new RustCSVEncoder(',', '"', '\\', "\n", $dateTimeFormat, 'Y-m-d', $short))->encode($native), + static fn() => (new RustJsonEncoder(JSON_THROW_ON_ERROR, $dateTimeFormat, 'Y-m-d', $short))->encode($native->project($jsonSchema, new Flow\ETL\Column\RustBackend()), "\n"), + static fn() => (new RustCSVEncoder(',', '"', '\\', "\n", $dateTimeFormat, 'Y-m-d', $throwing))->encode($native), + static fn() => (new RustJsonEncoder(JSON_THROW_ON_ERROR, $dateTimeFormat, 'Y-m-d', $throwing))->encode($native->project($jsonSchema, new Flow\ETL\Column\RustBackend()), "\n"), + ] as $refusal) { + try { + $refusal(); + throw new LogicException('a refused batch was written'); + } catch (JsonException|Flow\ETL\Exception\RuntimeException|Flow\ETL\Exception\InvalidArgumentException) { + } + } +}; + +for ($i = 0; $i < 10; $i++) { + $cycle(); +} +gc_collect_cycles(); +$baseline = memory_get_usage(false); +$baselineRust = $backend->allocatedBytes(); + +for ($i = 0; $i < 1_000; $i++) { + $cycle(); +} +gc_collect_cycles(); + +var_dump(memory_get_usage(false) <= $baseline); +var_dump($backend->allocatedBytes() === $baselineRust); +?> +--EXPECT-- +bool(true) +bool(true) diff --git a/src/extension/flow-php-ext/tests/phpt/084_native_nested_cast_parity.phpt b/src/extension/flow-php-ext/tests/phpt/084_native_nested_cast_parity.phpt new file mode 100644 index 0000000000..c9057be98c --- /dev/null +++ b/src/extension/flow-php-ext/tests/phpt/084_native_nested_cast_parity.phpt @@ -0,0 +1,65 @@ +--TEST-- +RustColumnBuilder casts random nested values, raw and refused forms, as the PHP builder does through append() and appendMany() +--SKIPIF-- + +--FILE-- + 0; $i--) { + $values[] = cast_random_input($type, $generate()); + } + + foreach ([false, true] as $many) { + $column = static fn(object $backend): Closure => static function () use ($backend, $definition, $values, $many): object { + $builder = $backend->builder($definition); + + if ($many) { + $builder->appendMany($values); + } else { + foreach ($values as $value) { + $builder->append($value); + } + } + + return $builder->finish(); + }; + + foreach (['values', 'physicals', 'encode'] as $read) { + $cases++; + $php = outcome(static fn(): mixed => $column(new PhpBackend())()->{$read}()); + $native = outcome(static fn(): mixed => $column(new RustBackend())()->{$read}()); + + if ($php !== $native) { + echo ($many ? 'appendMany ' : 'append ') . "{$read} {$type->toString()} " . var_export($values, true) . "\n php: {$php}\n native: {$native}\n"; + + continue; + } + + $identical++; + $refused += refused($php) ? 1 : 0; + $nulls += str_contains(serialize($values), 'N;') ? 1 : 0; + } + } +} + +echo "{$identical} of {$cases} identical, {$refused} refused, {$nulls} with a null\n"; +?> +--EXPECT-- +2400 of 2400 identical, 690 refused, 654 with a null diff --git a/src/extension/flow-php-ext/tests/phpt/085_native_nested_cast_no_leaks.phpt b/src/extension/flow-php-ext/tests/phpt/085_native_nested_cast_no_leaks.phpt new file mode 100644 index 0000000000..c0a3eb227c --- /dev/null +++ b/src/extension/flow-php-ext/tests/phpt/085_native_nested_cast_no_leaks.phpt @@ -0,0 +1,65 @@ +--TEST-- +nested values cast natively, handed to the PHP lane half-appended, and refused leak neither PHP memory nor native allocations +--SKIPIF-- + +--FILE-- + structure_element('id', type_integer()), + 'tags' => structure_element('tags', type_list(type_string())), + 'at' => structure_element('at', type_optional(type_datetime())), + 'note' => structure_element('note', type_string(), optional: true), +])); +$lists = list_schema('lists', type_list(type_list(type_integer()))); +$maps = map_schema('maps', type_map(type_string(), type_list(type_string()))); + +$cycle = static function () use ($backend, $orders, $lists, $maps): void { + $builder = $backend->builder($orders); + $builder->appendMany([ + ['id' => 1, 'tags' => ['a', 'b'], 'at' => '2026-01-02T03:04:05Z', 'note' => 'n'], + ['id' => '2', 'tags' => [], 'at' => null], + ['id' => 3, 'tags' => '["json", "text"]', 'at' => null], + ]); + $builder->finish()->values(); + $backend->builder($lists)->appendMany([[[1, 2], [3]], [[4, '5']], [[6, 7], [8]]]); + $backend->builder($maps)->appendMany([['a' => ['x'], 'b' => []], ['c' => ['y', 'z']]]); + + try { + $backend->builder($orders)->appendMany([['id' => 1, 'tags' => ['a'], 'at' => null], ['id' => 2, 'tags' => ['b', null]]]); + throw new LogicException('a refused value was accepted'); + } catch (Flow\ETL\Exception\SchemaMismatchException) { + } + + try { + $backend->builder($lists)->append([[1, 2], ['not an int']]); + throw new LogicException('a refused value was accepted'); + } catch (Flow\Types\Exception\CastingException) { + } +}; + +for ($i = 0; $i < 10; $i++) { + $cycle(); +} +gc_collect_cycles(); +$baseline = memory_get_usage(false); +$baselineRust = $backend->allocatedBytes(); + +for ($i = 0; $i < 100; $i++) { + $cycle(); +} +gc_collect_cycles(); + +var_dump(memory_get_usage(false) <= $baseline); +var_dump($backend->allocatedBytes() === $baselineRust); +?> +--EXPECT-- +bool(true) +bool(true) diff --git a/src/extension/flow-php-ext/tests/phpt/086_json_read_parity.phpt b/src/extension/flow-php-ext/tests/phpt/086_json_read_parity.phpt new file mode 100644 index 0000000000..7a418d5980 --- /dev/null +++ b/src/extension/flow-php-ext/tests/phpt/086_json_read_parity.phpt @@ -0,0 +1,202 @@ +--TEST-- +RustJsonOpenSource reads random valid JSON, whole or a byte at a time, into the frames or refusals of PhpJsonOpenSource, through either backend +--SKIPIF-- + +--FILE-- +}> + */ +function json_read_pool(): array +{ + $u = '\\' . 'u'; + $ints = ['1', '-0', '42', '9223372036854775807', '-9223372036854775808', '"42"', '1E2', 'null', '!9223372036854775808', '!"x"', '!1.5', '!true', '![1]']; + $strings = ['"a"', "\"{$u}00e9\"", "\"{$u}d83d{$u}de00\"", '"a\/b"', '""', '"tab\there"', '5', '-0.0', 'true', 'null', '![1]', '!{"a":1}']; + $person = type_structure([ + 'id' => structure_element('id', type_integer()), + 'name' => structure_element('name', type_optional(type_string()), optional: true), + ]); + $people = ['{"id":1,"name":"a"}', '{"id":1}', '{"id":"1","name":null}', '{"id":1,"name":"a","extra":[1]}', '{"id":1,"id":2}', '{"name":"b","id":3}', 'null', '!{"name":"a"}', '!{"id":1,"name":[5]}', '![1,"a"]', '!"x"']; + + return [ + [static fn(string $name): object => int_schema($name), $ints], + [static fn(string $name): object => int_schema($name, nullable: true), $ints], + [static fn(string $name): object => float_schema($name), ['0.1', '-0.0', '1e400', '-1e400', '1e-400', '1E2', '5', '-0', '"3.14"', 'null', '!"abc"', '![1]']], + [static fn(string $name): object => bool_schema($name), ['true', 'false', '"true"', '"off"', '"yes"', '1', '0', 'null', '!"x"', '![1]']], + [static fn(string $name): object => str_schema($name), $strings], + [static fn(string $name): object => str_schema($name, nullable: true), $strings], + [static fn(string $name): object => datetime_schema($name), ['"2026-01-02T03:04:05Z"', '"2026-01-02 03:04:05"', '"2026-01-02T03:04:05.123456+02:00"', 'null', '!"@1700000000"', '!"not a date"', '![1]']], + [static fn(string $name): object => date_schema($name), ['"2026-01-02"', '"2026-01-02T10:00:00Z"', 'null', '!"x"', '![1]']], + [static fn(string $name): object => uuid_schema($name), ['"6c4b5d1e-4f2a-4b8e-9c3d-2e1f0a9b8c7d"', 'null', '!"6C4B5D1E-4F2A-4B8E-9C3D-2E1F0A9B8C7D"', '!"6c4b5d1e"', '!1']], + [static fn(string $name): object => json_schema($name), ['{"a":1}', '{"a":1.0,"b":"\/"}', '[1,2]', '[]', '{}', '"{\"a\":1}"', "{\"{$u}00e9\":\"{$u}d83d{$u}de00\"}", 'null', '!"x"', '!1']], + [static fn(string $name): object => list_schema($name, type_list(type_integer())), ['[1,2]', '[]', '["1"]', '{"0":1,"1":2}', 'null', '!["x"]', '![1,null]', '!5']], + [static fn(string $name): object => map_schema($name, type_map(type_string(), type_integer())), ['{"a":1,"b":2}', '{"a":1,"a":"2"}', '{}', "{\"{$u}00e9\":3}", 'null', '!{"a":"x"}', '!5']], + [static fn(string $name): object => map_schema($name, type_map(type_integer(), type_string())), ['{"1":"a","2":"b"}', '["a","b"]', '{"-1":"y","2":"a","2":"b"}', 'null', '!{"a":"b"}', '!{"01":"x"}', '!5']], + [static fn(string $name): object => structure_schema($name, $person), $people], + [static fn(string $name): object => list_schema($name, type_list($person)), ['[' . $people[0] . ',' . $people[1] . ']', '[' . $people[4] . ',' . $people[5] . ']', '[]', 'null', '![' . ltrim($people[7], '!') . ']']], + [static fn(string $name): object => map_schema($name, type_map(type_string(), type_list(type_integer()))), ['{"a":[1,2],"b":[]}', '{"a":["2"],"a":[3]}', 'null', '!{"a":[1,"x"]}', '!{"a":[[1]]}']], + [static fn(string $name): object => structure_schema($name, type_structure(['tags' => structure_element('tags', type_list(type_map(type_string(), type_integer())))])), ['{"tags":[{"a":1},{"b":2,"b":3}]}', '{"tags":[]}', '{"tags":[{"a":"1"}]}', 'null', '!{"tags":[{"a":[1]}]}', '!{}']], + ]; +} + +/** + * A member name as JSON text, sometimes with escapes that unescape to the same name. + */ +function json_read_key(string $name): string +{ + return match (mt_rand(0, 3)) { + 0 => '"' . implode('', array_map(static fn(string $char): string => sprintf('\\u%04x', ord($char)), str_split($name))) . '"', + default => '"' . $name . '"', + }; +} + +/** + * Draws from a column's pool: a refusing draw ("!") only when $refuse, a null or an absence only when $nullable or + * $refuse. + * + * @param list $pool + */ +function json_read_draw(array $pool, bool $nullable, bool $refuse): string +{ + $draws = array_values(array_filter($pool, static fn(string $draw): bool => $refuse || (!str_starts_with($draw, '!') && ($nullable || $draw !== 'null')))); + + return ltrim($draws[mt_rand(0, count($draws) - 1)], '!'); +} + +/** + * @param list, bool}> $columns name, draw pool and nullability of every column + */ +function json_read_record(array $columns, bool $array, bool $refuse): string +{ + if ($array) { + return '[' . implode(', ', array_map(static fn(array $column): string => json_read_draw($column[1], $column[2], $refuse), $columns)) . ']'; + } + + $members = []; + + foreach ($columns as [$name, $pool, $nullable]) { + $draw = static fn(): string => json_read_draw($pool, $nullable, $refuse); + + // a duplicated key keeps its last value, so both stay in one group, in order + match (($nullable || $refuse) ? mt_rand(0, 9) : mt_rand(1, 9)) { + 0 => null, + 1 => $members[] = json_read_key($name) . ':' . json_read_draw($pool, true, true) . ', ' . json_read_key($name) . ':' . $draw(), + default => $members[] = json_read_key($name) . ' : ' . $draw(), + }; + + if (mt_rand(0, 9) === 0) { + $members[] = '"extra' . '\\' . 'u00e9" :' . json_read_draw($pool, true, true); + } + } + + shuffle($members); + + return '{' . implode(",\n ", $members) . '}'; +} + +$frames = static function (iterable $batches, object $backend): array { + $frames = []; + + foreach ($batches as $batch) { + foreach ($batch->columns() as $column) { + if ($backend instanceof PhpBackend && $column instanceof RustColumn) { + return ['a RustColumn survived PhpBackend']; + } + } + + $frames[] = bin2hex($batch->encodeFrame()); + } + + return $frames; +}; +// RustJsonOpenSource::batches() over reads of $chunk bytes +$bytes = static fn(string $raw, bool $lines, Schema $schema, object $backend, int $chunk): Iterator => (new RustJsonOpenSource( + chunked_source_stream($raw, $chunk), + $lines, + 'memory://phpt.json', + '', + 0, +))->batches($schema, 2, $backend); + +mt_srand(10); +$pool = json_read_pool(); +$cases = 0; +$identical = 0; +$refused = 0; +$nulls = 0; + +for ($t = 0; $t < 400; $t++) { + $array = mt_rand(0, 4) === 0; + $columns = []; + $definitions = []; + + for ($c = mt_rand(1, 4) - 1; $c >= 0; $c--) { + [$definition, $draws] = $pool[mt_rand(0, count($pool) - 1)]; + $name = $array ? (string) count($columns) : 'c' . count($columns); + $definitions[] = $definition($name); + $columns[] = [$name, $draws, end($definitions)->isNullable()]; + } + + $schema = schema(...$definitions); + $refuse = mt_rand(0, 3) === 0; + $records = []; + + for ($r = mt_rand(1, 5); $r > 0; $r--) { + $records[] = mt_rand(0, 9) === 0 ? ['{}', '[ ]'][mt_rand(0, 1)] : json_read_record($columns, $array, $refuse); + } + + foreach ([true, false] as $lines) { + $raw = $lines + ? implode(["\n", "\r\n", "\n \n"][mt_rand(0, 2)], array_map(static fn(string $record): string => str_replace("\n", '', $record), $records)) . ["\n", ''][mt_rand(0, 1)] + : "[\n" . implode(" ,\n\t", $records) . "\n]\n"; + $filesystem = memory_filesystem(); + $stream = $filesystem->writeTo(path('memory://phpt.json')); + $stream->append($raw); + $stream->close(); + $source = new SourceFile(path('memory://phpt.json')); + $reader = new JsonFileReader($filesystem, $lines ? JsonFormat::Lines : JsonFormat::Document, null, false, [$source]); + $php = outcome(static fn(): array => $frames((new PhpJsonOpenSource($reader, $source))->batches($schema, 2, new PhpBackend()), new PhpBackend())); + + foreach ([new RustBackend(), new PhpBackend()] as $backend) { + $natives = [ + 'whole' => outcome(static fn(): array => $frames((new RustJsonOpenSource(new StringSourceStream(path('memory://phpt.json'), $raw), $lines, 'memory://phpt.json', '', 0))->batches($schema, 2, $backend), $backend)), + 'by byte' => outcome(static fn(): array => $frames($bytes($raw, $lines, $schema, $backend, 1), $backend)), + ]; + + foreach ($natives as $feed => $native) { + $cases++; + + if ($native !== $php) { + echo ($lines ? 'lines ' : 'document ') . "{$feed} " . get_class($backend) . ' ' . json_encode($raw) . "\n php: {$php}\n native: {$native}\n"; + + continue; + } + + $identical++; + $refused += refused($php) ? 1 : 0; + $nulls += str_contains($raw, 'null') ? 1 : 0; + } + } + } +} + +echo "{$identical} of {$cases} identical, {$refused} refused, {$nulls} with a null\n"; +?> +--EXPECT-- +3200 of 3200 identical, 600 refused, 936 with a null diff --git a/src/extension/flow-php-ext/tests/phpt/087_json_read_no_leaks.phpt b/src/extension/flow-php-ext/tests/phpt/087_json_read_no_leaks.phpt new file mode 100644 index 0000000000..9869977cb1 --- /dev/null +++ b/src/extension/flow-php-ext/tests/phpt/087_json_read_no_leaks.phpt @@ -0,0 +1,79 @@ +--TEST-- +RustJsonOpenSource reads, cast refusals and malformed refusals leak neither PHP memory nor native allocations +--SKIPIF-- + +--FILE-- + structure_element('id', type_integer()), + 'note' => structure_element('note', type_optional(type_string()), optional: true), + ])), +); +$valid = '{"id":1,"name":"aé","tags":["x","y"],"scores":{"a":1,"a":2},"owner":{"id":1}}'; +$read = static function (string $raw, bool $lines) use ($backend, $schema): void { + $open = new RustJsonOpenSource(new StringSourceStream(path('memory://leak.json'), $raw), $lines, 'memory://leak.json', '', 0); + + try { + foreach ($open->batches($schema, 2, $backend) as $batch) { + $batch->toArray(); + } + } finally { + $open->close(); + } +}; +$refused = static function (string $raw, bool $lines, string $class) use ($read): void { + try { + $read($raw, $lines); + throw new LogicException('a refused read was accepted'); + } catch (Throwable $e) { + if (!$e instanceof $class) { + throw $e; + } + } +}; +$cycle = static function () use ($read, $refused, $valid): void { + $read("[{$valid},{$valid},\n{$valid}]", false); + $read("{$valid}\n{$valid}\n\n{$valid}", true); + $read('[{"id":"2","tags":"[\"json\"]","scores":[],"owner":{"id":"3","note":null}}]', false); + $refused('[' . $valid . ',{"id":"x","tags":[],"scores":{},"owner":{"id":1}}]', false, SchemaMismatchException::class); + $refused('{"id":1,"tags":[1,[2]],"scores":{},"owner":{"note":"n"}}', true, SchemaMismatchException::class); + $refused($valid . "\n" . $valid . '}', true, RuntimeException::class); + $refused('[' . $valid . ',' . $valid, false, RuntimeException::class); + $refused($valid . "\n5", true, RuntimeException::class); +}; + +for ($i = 0; $i < 10; $i++) { + $cycle(); +} +gc_collect_cycles(); +$baseline = memory_get_usage(false); +$baselineRust = $backend->allocatedBytes(); + +for ($i = 0; $i < 100; $i++) { + $cycle(); +} +gc_collect_cycles(); + +var_dump(memory_get_usage(false) <= $baseline); +var_dump($backend->allocatedBytes() === $baselineRust); +?> +--EXPECT-- +bool(true) +bool(true) diff --git a/src/extension/flow-php-ext/tests/phpt/088_json_read_streaming.phpt b/src/extension/flow-php-ext/tests/phpt/088_json_read_streaming.phpt new file mode 100644 index 0000000000..947e298dd6 --- /dev/null +++ b/src/extension/flow-php-ext/tests/phpt/088_json_read_streaming.phpt @@ -0,0 +1,80 @@ +--TEST-- +RustJsonOpenSource reads a JSON array and JSON lines in memory independent of the record count: one chunk, the largest record and one batch +--SKIPIF-- + +--FILE-- + 0 ? ($lines ? "\n" : ",\n") : '') . implode($lines ? "\n" : ",\n", $chunk)); + } + + fwrite($handle, $lines ? "\n" : "]\n"); + fclose($handle); +}; +// rows read, PHP peak over the read, the most native bytes held while a batch was +$read = static function (string $file, bool $lines) use ($backend, $schema): array { + gc_collect_cycles(); + memory_reset_peak_usage(); + $php = memory_get_usage(false); + $rust = $backend->allocatedBytes(); + $held = 0; + $rows = 0; + $open = new RustJsonOpenSource((new NativeLocalFilesystem())->readFrom(path($file)), $lines, $file, '', 0); + + foreach ($open->batches($schema, 1000, $backend) as $batch) { + $rows += $batch->count(); + $held = max($held, $backend->allocatedBytes() - $rust); + } + + $open->close(); + unset($batch, $open); + + return [$rows, memory_get_peak_usage(false) - $php, $held]; +}; + +// the Makefile phpt runner has no CLEAN section, so the files are removed here +foreach (['array' => false, 'lines' => true] as $label => $lines) { + $file = sys_get_temp_dir() . "/flow_php_088_{$label}.json"; + $measured = []; + + try { + foreach ([20_000, 200_000] as $records) { + $write($file, $records, $lines); + $measured[$records] = $read($file, $lines); + } + } finally { + @unlink($file); + } + + [$rows20, $php20, $rust20] = $measured[20_000]; + [$rows200, $php200, $rust200] = $measured[200_000]; + + echo "{$label}: {$rows20} and {$rows200} rows, PHP peak ", $php200 <= $php20 + (1 << 15) ? 'bounded' : "{$php20} -> {$php200}", + ', native held ', $rust200 <= $rust20 + (1 << 15) ? 'bounded' : "{$rust20} -> {$rust200}", "\n"; +} +?> +--EXPECT-- +array: 20000 and 200000 rows, PHP peak bounded, native held bounded +lines: 20000 and 200000 rows, PHP peak bounded, native held bounded diff --git a/src/extension/flow-php-ext/tests/phpt/089_parquet_arrow_c_no_leaks.phpt b/src/extension/flow-php-ext/tests/phpt/089_parquet_arrow_c_no_leaks.phpt new file mode 100644 index 0000000000..8bc9d2c594 --- /dev/null +++ b/src/extension/flow-php-ext/tests/phpt/089_parquet_arrow_c_no_leaks.phpt @@ -0,0 +1,96 @@ +--TEST-- +Batches crossing to and from arrow-ext through the Arrow C Data Interface leak neither PHP memory nor native allocations: abandoned, never imported, imported twice, refused before and after export, writer dropped unclosed +--SKIPIF-- + + +--FILE-- +toParquet($schema)); +$options = OptionsConverter::toExtension(Options::default()); +$values = array_map(static fn(int $i): array => ['id' => $i, 'name' => $i % 3 ? "name {$i}" : null, 'tags' => $i % 4 ? [$i, null] : null], range(0, 99)); +// built once: a fresh Type object per cycle would grow flow_php's type-plan cache, which is bounded, not leaked +$rows = native_rows($schema, $values); +$invalid = native_rows($schema, [['id' => 1, 'name' => "\xff", 'tags' => null]]); +$mismatched = schema(str_schema('id')); + +$cycle = static function () use ($schema, $extension, $options, $rows, $invalid, $mismatched): void { + $filesystem = memory_filesystem(); + $writer = new RustParquetOpenSink(new RustParquetFileWriter($filesystem->writeTo(path('memory://leaks.parquet')), $extension, Compressions::SNAPPY, $options, 30)); + $writer->write($rows); + $writer->write($rows); + $writer->close(); + + $reader = rust_parquet_batches($filesystem->readFrom(path('memory://leaks.parquet')), $schema, 30, null, null); + + foreach ($reader as $_) { + } + + $abandoned = rust_parquet_batches($filesystem->readFrom(path('memory://leaks.parquet')), $schema, 30, null, null); + $abandoned->rewind(); + unset($abandoned); + + $batches = new RustBatchReader(new RustParquetFileReader($filesystem->readFrom(path('memory://leaks.parquet'))), ['id', 'name', 'tags'], 30, null, null); + $batches->next(); + $twice = $batches->next(); + $copy = new RustParquetFileWriter($filesystem->writeTo(path('memory://copy.parquet')), $extension, Compressions::SNAPPY, $options, 30); + $copy->writeArrowBatch($twice); + + try { + $copy->writeArrowBatch($twice); + throw new LogicException('a batch was imported twice'); + } catch (Flow\Parquet\Exception\InvalidArgumentException) { + } + + $copy->close(); + + try { + rust_parquet_batches($filesystem->readFrom(path('memory://leaks.parquet')), $mismatched, 30, null, null); + throw new LogicException('a string schema read an int column'); + } catch (Flow\ETL\Exception\InvalidArgumentException) { + } + + $refused = new RustParquetOpenSink(new RustParquetFileWriter($filesystem->writeTo(path('memory://refused.parquet')), $extension, Compressions::SNAPPY, $options, 30)); + + try { + $refused->write($invalid); + throw new LogicException('invalid UTF-8 was written'); + } catch (Flow\Parquet\Exception\RuntimeException) { + } + + $unclosed = new RustParquetOpenSink(new RustParquetFileWriter($filesystem->writeTo(path('memory://unclosed.parquet')), $extension, Compressions::SNAPPY, $options, 30)); + $unclosed->write($rows); +}; + +for ($i = 0; $i < 10; $i++) { + $cycle(); +} +gc_collect_cycles(); +$baseline = memory_get_usage(false); +$baselineRust = (new RustBackend())->allocatedBytes(); + +for ($i = 0; $i < 200; $i++) { + $cycle(); +} +gc_collect_cycles(); + +var_dump(memory_get_usage(false) <= $baseline); +var_dump((new RustBackend())->allocatedBytes() === $baselineRust); +?> +--EXPECT-- +bool(true) +bool(true) diff --git a/src/extension/flow-php-ext/tests/phpt/090_parquet_schema_refused_at_open.phpt b/src/extension/flow-php-ext/tests/phpt/090_parquet_schema_refused_at_open.phpt new file mode 100644 index 0000000000..78da4d24ee --- /dev/null +++ b/src/extension/flow-php-ext/tests/phpt/090_parquet_schema_refused_at_open.phpt @@ -0,0 +1,50 @@ +--TEST-- +RustParquetOpenSource refuses a schema type that stores another arrow type when batches() is called, before any batch: over a zero-row file and past the end of an 8-row-group file; a file other than RustParquetFileReader and a writer other than RustParquetFileWriter are refused +--SKIPIF-- + + +--FILE-- +toParquet($schema)); + +$empty = new RustParquetFileWriter($filesystem->writeTo(path('memory://empty.parquet')), $extension, Compressions::SNAPPY, OptionsConverter::toExtension(Options::default()), 10); +$empty->close(); + +$groups = new RustParquetOpenSink(new RustParquetFileWriter( + $filesystem->writeTo(path('memory://groups.parquet')), + $extension, + Compressions::SNAPPY, + OptionsConverter::toExtension(Options::default()->set(Option::ROW_GROUP_SIZE_BYTES, 1)), + 10, +)); + +for ($group = 0; $group < 8; $group++) { + $groups->write(array_to_rows(array_map(static fn(int $id): array => ['id' => $id], range($group * 10, $group * 10 + 9)), $schema)); +} + +$groups->close(); + +$mismatched = schema(str_schema('id')); +echo outcome(static fn() => rust_parquet_batches($filesystem->readFrom(path('memory://empty.parquet')), $mismatched, 10, null, null)), "\n"; +echo outcome(static fn() => rust_parquet_batches($filesystem->readFrom(path('memory://groups.parquet')), $mismatched, 10, 1_000, null)), "\n"; +echo outcome(static fn() => new Flow\ETL\Adapter\Parquet\RustParquetOpenSource(new stdClass())), "\n"; +echo outcome(static fn() => new Flow\ETL\Adapter\Parquet\RustParquetOpenSink(new stdClass())), "\n"; +?> +--EXPECT-- +Flow\ETL\Exception\InvalidArgumentException: Parquet column "id" (Int64) is read as Int64, its schema type string stores Binary +Flow\ETL\Exception\InvalidArgumentException: Parquet column "id" (Int64) is read as Int64, its schema type string stores Binary +Flow\ETL\Exception\RuntimeException: flow_php expected a Flow\Parquet\Engine\RustParquetFileReader +Flow\ETL\Exception\RuntimeException: flow_php expected a Flow\Parquet\Engine\RustParquetFileWriter diff --git a/src/extension/flow-php-ext/tests/phpt/091_parquet_canonical_layout_contract.phpt b/src/extension/flow-php-ext/tests/phpt/091_parquet_canonical_layout_contract.phpt new file mode 100644 index 0000000000..c28642ea9d --- /dev/null +++ b/src/extension/flow-php-ext/tests/phpt/091_parquet_canonical_layout_contract.phpt @@ -0,0 +1,87 @@ +--TEST-- +arrow-ext's canonical Arrow layout (parquet/canonical.rs) is the layout flow_php's columns store (flow-batch-frame kind::data_type()): every Flow type, nested included, crosses RustBatchReader → RustParquetOpenSource with its values intact +--SKIPIF-- + + +--FILE-- + type_integer(), 'name' => type_optional(type_string())]), nullable: true), + list_schema('list_of_structures', type_list(type_structure(['id' => type_integer(), 'tags' => type_optional(type_list(type_string()))])), nullable: true), + map_schema('map_of_lists', type_map(type_string(), type_list(type_integer())), nullable: true), +); +$values = [ + [ + 'int' => -7, 'float' => 0.14, 'bool' => true, 'string' => 'zażółć', + 'datetime' => new DateTimeImmutable('1969-12-31 23:59:59.999999 UTC'), 'date' => new DateTimeImmutable('1900-01-01'), + 'time' => new DateInterval('PT1H2M3S'), 'uuid' => new Uuid('26fd21b0-6080-4d6c-bdb4-1214f1feffef'), 'json' => '{"a":[1,2]}', + 'list' => [1, null, 3], 'map' => ['a' => 1, 'b' => null], 'structure' => ['id' => 1, 'name' => null], + 'list_of_structures' => [['id' => 1, 'tags' => ['x', 'y']], ['id' => 2, 'tags' => null]], + 'map_of_lists' => ['a' => [1, 2], 'b' => []], + ], + array_fill_keys(array_keys($schema->definitions()), null), +]; + +$filesystem = memory_filesystem(); +$writer = new RustParquetOpenSink(new RustParquetFileWriter( + $filesystem->writeTo(path('memory://layout.parquet')), + ArrowSchemaConverter::toExtension((new SchemaConverter())->toParquet($schema)), + Compressions::SNAPPY, + OptionsConverter::toExtension(Options::default()), + 1_000, +)); +$writer->write(php_rows($schema, $values)); +$writer->close(); + +$read = []; +$reader = rust_parquet_batches($filesystem->readFrom(path('memory://layout.parquet')), $schema, 10, null, null); + +foreach ($reader as $batch) { + array_push($read, ...$batch->toArray()); +} + +$written = php_rows($schema, $values)->toArray(); + +foreach (array_keys($schema->definitions()) as $name) { + echo $name, ': ', comparable(array_column($read, $name)) === comparable(array_column($written, $name)) ? 'identical' : 'DIFFER', "\n"; +} +?> +--EXPECT-- +int: identical +float: identical +bool: identical +string: identical +datetime: identical +date: identical +time: identical +uuid: identical +json: identical +list: identical +map: identical +structure: identical +list_of_structures: identical +map_of_lists: identical diff --git a/src/extension/flow-php-ext/tests/phpt/092_failure_without_autoloader.phpt b/src/extension/flow-php-ext/tests/phpt/092_failure_without_autoloader.phpt new file mode 100644 index 0000000000..e87433e110 --- /dev/null +++ b/src/extension/flow-php-ext/tests/phpt/092_failure_without_autoloader.phpt @@ -0,0 +1,24 @@ +--TEST-- +without an autoloader flow_php fails with a plain Exception, never recursing on the missing RuntimeException class +--SKIPIF-- + +--FILE-- + new Flow\ETL\Adapter\CSV\RustCSVOpenSource(new stdClass(), ',,', '"', '\\', true, true, true), + static fn() => new Flow\ETL\Column\RustColumn(), +] as $fn) { + try { + $fn(); + echo "FAIL: no exception thrown\n"; + } catch (Exception $e) { + echo get_class($e), ': ', $e->getMessage(), "\n"; + } +} + +var_dump(class_exists('Flow\ETL\Exception\RuntimeException', false)); +?> +--EXPECT-- +Exception: flow_php CSV separator must be exactly one byte +Exception: Flow\ETL\Column\RustColumn is built by RustBackend +bool(false) diff --git a/src/extension/flow-php-ext/tests/phpt/093_adapter_writers_contracts.phpt b/src/extension/flow-php-ext/tests/phpt/093_adapter_writers_contracts.phpt new file mode 100644 index 0000000000..7bdcb32419 --- /dev/null +++ b/src/extension/flow-php-ext/tests/phpt/093_adapter_writers_contracts.phpt @@ -0,0 +1,71 @@ +--TEST-- +RustCSVEncoder / RustJsonEncoder implement CSVEncoder / JsonEncoder: unrendered columns through the held PHP encoder's cells() / fragments(), the cache refreshed on a new Schema, the PHP header; RustParquetOpenSink implements ParquetOpenSink +--SKIPIF-- + +--FILE-- +implementsInterface(ParquetOpenSink::class)); + +$plain = schema(int_schema('id')); +$values = [['id' => 1, 'doc' => '1'], ['id' => 2, 'doc' => '']]; +$xml = schema(int_schema('id'), xml_schema('doc')); +$jsonDoc = schema(int_schema('id'), json_schema('doc')); +$jsonValues = [['id' => 1, 'doc' => '{"a":1}'], ['id' => 2, 'doc' => '[1,2]']]; + +$csvRecorder = new RecordingPhpCSVEncoder($csvPhp); +$jsonRecorder = new RecordingPhpJsonEncoder($jsonPhp); +$xmlRows = native_rows($xml, $values); +$jsonRows = native_rows($jsonDoc, $jsonValues); +(new RustCSVEncoder(',', '"', '\\', "\n", DATE_ATOM, 'Y-m-d', $csvRecorder))->encode($xmlRows); +(new RustJsonEncoder(JSON_THROW_ON_ERROR, DATE_ATOM, 'Y-m-d', $jsonRecorder))->encode($jsonRows, "\n"); +echo json_encode([recorded_names($csvRecorder->columns, $xmlRows), recorded_names($jsonRecorder->columns, $jsonRows)]), "\n"; + +// the first batch caches a Schema without unrendered columns; a new Schema object must refresh it +foreach ([[$plain, [['id' => 1]]], [$xml, $values]] as [$schema, $batch]) { + echo $csv->encode(native_rows($schema, $batch)) === $csvPhp->encode(php_rows($schema, $batch)) ? 'csv identical' : 'csv DIFFERENT', "\n"; +} + +foreach ([[$plain, [['id' => 1]]], [$jsonDoc, $jsonValues]] as [$schema, $batch]) { + echo $json->encode(native_rows($schema, $batch), "\n") === $jsonPhp->encode(php_rows($schema, $batch), "\n") ? 'json identical' : 'json DIFFERENT', "\n"; +} + +echo $csv->encodeHeader(['id', 'a "b"', 'c,d']) === $csvPhp->encodeHeader(['id', 'a "b"', 'c,d']) ? 'header identical' : 'header DIFFERENT', "\n"; +echo $csv->encode(native_rows($xml, $values)); +echo $json->encode(native_rows($jsonDoc, $jsonValues), "\n"), "\n"; +?> +--EXPECT-- +bool(true) +bool(true) +bool(true) +[["doc"],["doc"]] +csv identical +csv identical +json identical +json identical +header identical +1,1 +2, +{"id":1,"doc":{"a":1}} +{"id":2,"doc":[1,2]} diff --git a/src/extension/flow-php-ext/tests/phpt/094_rust_iterator_generator_parity.phpt b/src/extension/flow-php-ext/tests/phpt/094_rust_iterator_generator_parity.phpt new file mode 100644 index 0000000000..21e73f6519 --- /dev/null +++ b/src/extension/flow-php-ext/tests/phpt/094_rust_iterator_generator_parity.phpt @@ -0,0 +1,95 @@ +--TEST-- +RustIterator behaves as the Generator it replaced: it starts on first use, ends for good when its producer throws (valid() false, key() and current() null, next() a no-op) and refuses rewind() only after next() +--SKIPIF-- + +--FILE-- + $rows === null ? 'null' : (string) $rows->toArray()[0]['id']; +$step = static function (string $label, callable $call): void { + // the class differs on purpose: RustIterator throws flow's RuntimeException, a Generator \Exception + try { + $result = var_export($call(), true); + } catch (Throwable) { + $result = 'threw'; + } + + echo $label, ': ', $result, "\n"; +}; +$drive = static function (Iterator $iterator) use ($id, $step): void { + $step('key before rewind', static fn() => $iterator->key()); + $step('current', static fn() => $id($iterator->current())); + $step('rewind', static fn() => $iterator->rewind()); + $step('next', static fn() => $iterator->next()); + $step('current', static fn() => $id($iterator->current())); + $step('next (producer throws)', static fn() => $iterator->next()); + $step('valid', static fn() => $iterator->valid()); + $step('key', static fn() => $iterator->key()); + $step('current', static fn() => $iterator->current()); + $step('next', static fn() => $iterator->next()); + $step('rewind', static fn() => $iterator->rewind()); +}; +$failingFirst = static function (Iterator $iterator) use ($step): void { + $step('rewind (producer throws)', static fn() => $iterator->rewind()); + $step('valid', static fn() => $iterator->valid()); + $step('rewind', static fn() => $iterator->rewind()); +}; + +$rows = static fn(int $id) => php_rows(schema(int_schema('id')), [['id' => $id]]); +$generator = (static function () use ($rows): Generator { + yield $rows(1); + yield $rows(2); + throw new RuntimeException('malformed'); +})(); +$rust = (new RustJsonOpenSource(new StringSourceStream(path('memory://a.jsonl'), "{\"id\":1}\n{\"id\":2}\nnot json\n"), true, 'memory://a.jsonl', '', 0)) + ->batches($schema, 1, new PhpBackend()); + +ob_start(); +$drive($generator); +$expected = ob_get_clean(); +ob_start(); +$drive($rust); +$actual = ob_get_clean(); +echo $actual, $actual === $expected ? "identical\n" : "DIFFERENT, Generator:\n{$expected}"; + +$generator = (static function (): Generator { + throw new RuntimeException('malformed'); + yield; +})(); +$rust = (new RustJsonOpenSource(new StringSourceStream(path('memory://b.jsonl'), "not json\n"), true, 'memory://b.jsonl', '', 0)) + ->batches($schema, 1, new PhpBackend()); + +ob_start(); +$failingFirst($generator); +$expected = ob_get_clean(); +ob_start(); +$failingFirst($rust); +$actual = ob_get_clean(); +echo $actual, $actual === $expected ? "identical\n" : "DIFFERENT, Generator:\n{$expected}"; +?> +--EXPECT-- +key before rewind: 0 +current: '1' +rewind: NULL +next: NULL +current: '2' +next (producer throws): threw +valid: false +key: NULL +current: NULL +next: NULL +rewind: threw +identical +rewind (producer throws): threw +valid: false +rewind: NULL +identical diff --git a/src/extension/flow-php-ext/tests/phpt/095_sources_reentrant_stream.phpt b/src/extension/flow-php-ext/tests/phpt/095_sources_reentrant_stream.phpt new file mode 100644 index 0000000000..8ebd75e250 --- /dev/null +++ b/src/extension/flow-php-ext/tests/phpt/095_sources_reentrant_stream.phpt @@ -0,0 +1,109 @@ +--TEST-- +RustCSVOpenSource and RustJsonOpenSource hold no borrow while their stream runs: a stream calling producedRows(), producedBytes() and close() back mid-read is answered, never a panic +--SKIPIF-- + +--FILE-- +stream = new StringSourceStream(path('memory://hooked'), $content); + } + + public function close(): void + { + $this->stream->close(); + } + + public function content(): string + { + return $this->stream->content(); + } + + public function isOpen(): bool + { + return $this->stream->isOpen(); + } + + public function iterate(int $length = 1): Generator + { + foreach ($this->stream->iterate(4) as $chunk) { + ($this->hook)(); + + yield $chunk; + } + } + + public function path(): Path + { + return $this->stream->path(); + } + + public function read(int $length, int $offset): string + { + ($this->hook)(); + + return $this->stream->read(4, $offset); + } + + public function readLines(string $separator = "\n", ?int $length = null): Generator + { + return $this->stream->readLines($separator, $length); + } + + public function size(): ?int + { + return $this->stream->size(); + } +} + +$schema = schema(int_schema('id')); +$stream = new HookedStream("id\n1\n2\n3\n"); +$csv = new RustCSVOpenSource($stream, ',', '"', '\\', true, true, true); +$seen = []; +$stream->hook = static function () use ($csv, &$seen): void { + $seen[] = $csv->producedRows() . '/' . $csv->producedBytes(); +}; +$ids = []; + +foreach ($csv->batches($schema, 1, new PhpBackend()) as $rows) { + $ids[] = $rows->toArray()[0]['id']; +} + +echo 'csv ids: ', json_encode($ids), ', counters answered mid-read: ', count($seen), ', after: ', $csv->producedRows(), "\n"; + +$stream = new HookedStream("{\"id\":1}\n{\"id\":2}\n"); +$json = new RustJsonOpenSource($stream, true, 'memory://hooked', '', 0); +$closed = false; +$stream->hook = static function () use ($json, &$closed): void { + if (!$closed) { + $closed = true; + $json->close(); + } +}; + +echo 'json close() mid-read: ', outcome(static fn() => count(iterator_to_array($json->batches($schema, 1, new PhpBackend())))), "\n"; +echo 'json closed: ', var_export($closed, true), "\n"; +?> +--EXPECT-- +csv ids: [1,2,3], counters answered mid-read: 3, after: 3 +json close() mid-read: i:2; +json closed: true diff --git a/src/extension/flow-php-ext/tests/phpt/096_encoders_columns_left_to_php.phpt b/src/extension/flow-php-ext/tests/phpt/096_encoders_columns_left_to_php.phpt new file mode 100644 index 0000000000..7c815b1f02 --- /dev/null +++ b/src/extension/flow-php-ext/tests/phpt/096_encoders_columns_left_to_php.phpt @@ -0,0 +1,45 @@ +--TEST-- +RustCSVEncoder / RustJsonEncoder render natively what they can and leave the rest, by column, to the held PHP encoder's cells() / fragments() +--SKIPIF-- + +--FILE-- +encode($rows); + (new RustJsonEncoder(JSON_THROW_ON_ERROR, $format, $format, $json))->encode($rows, "\n"); + + echo "{$batch} ({$format}): csv ", json_encode(recorded_names($csv->columns, $rows)), ', json ', json_encode(recorded_names($json->columns, $rows)), "\n"; +} +?> +--EXPECT-- +scalars with nulls (Y-m-d): csv [], json [] +temporal kinds (Y-m-d\TH:i:sP): csv [], json [] +temporal kinds (D, d M Y): csv ["at","offset","on"], json ["at","offset","on"] +a json column (Y-m-d): csv [], json ["j"] +markup columns (Y-m-d): csv ["x","e"], json ["x","e"] +nested kinds rendered natively (Y-m-d): csv [], json [] +nested kinds rendered natively (l): csv ["l","s"], json ["l","s"] +a list of json documents (Y-m-d): csv ["l"], json ["l"] +nested markup leaves (Y-m-d): csv ["l","s"], json ["l","s"] diff --git a/src/extension/flow-php-ext/tests/phpt/098_rust_append_physicals.phpt b/src/extension/flow-php-ext/tests/phpt/098_rust_append_physicals.phpt new file mode 100644 index 0000000000..274edf739b --- /dev/null +++ b/src/extension/flow-php-ext/tests/phpt/098_rust_append_physicals.phpt @@ -0,0 +1,42 @@ +--TEST-- +RustColumnBuilder::appendPhysicals() appends physicals as they are: nulls with or without a count, nothing for [], a null under NOT NULL and a physical of another kind refused +--SKIPIF-- + +--FILE-- +builder(int_schema('id', nullable: true)); + $builder->appendPhysicals([1, null, 3, null], $nullCount); + $column = $builder->finish(); + echo json_encode($column->values()), ' ', $column->nullCount(), "\n"; +} + +$builder = (new RustBackend())->builder(int_schema('id')); +$builder->appendPhysicals([]); +echo $builder->count(), "\n"; + +try { + $builder->appendPhysicals([1, null]); +} catch (SchemaMismatchException $e) { + echo get_class($e), ' row ', $e->rowIndex, ' kept ', $builder->count(), "\n"; +} + +try { + $builder->appendPhysicals(['not an int']); +} catch (Throwable $e) { + echo get_class($e), ' kept ', $builder->count(), "\n"; +} +?> +--EXPECT-- +[1,null,3,null] 2 +[1,null,3,null] 2 +0 +Flow\ETL\Exception\SchemaMismatchException row 1 kept 0 +Flow\ETL\Exception\InvalidArgumentException kept 0 diff --git a/src/extension/flow-php-ext/tests/phpt/099_abi_matches_flow_php_etl.phpt b/src/extension/flow-php-ext/tests/phpt/099_abi_matches_flow_php_etl.phpt new file mode 100644 index 0000000000..412ea98207 --- /dev/null +++ b/src/extension/flow-php-ext/tests/phpt/099_abi_matches_flow_php_etl.phpt @@ -0,0 +1,22 @@ +--TEST-- +flow_php registers FLOW_PHP_ABI equal to the ABI flow-php/etl expects, so the Adaptive picks take the Rust lane +--SKIPIF-- + +--FILE-- +available()); +var_dump((new AdaptiveBackend())->constant(int_schema('id'), 1, 2) instanceof RustColumn); +?> +--EXPECT-- +bool(true) +bool(true) +bool(true) diff --git a/src/extension/flow-php-ext/tests/phpt/bootstrap.php b/src/extension/flow-php-ext/tests/phpt/bootstrap.php index 59fa05318c..28a1cd3bdd 100644 --- a/src/extension/flow-php-ext/tests/phpt/bootstrap.php +++ b/src/extension/flow-php-ext/tests/phpt/bootstrap.php @@ -4,166 +4,340 @@ require __DIR__ . '/../../../../../vendor/autoload.php'; -use Flow\ETL\Row; -use Flow\ETL\Row\NativeRowHydrator; -use Flow\ETL\Row\PhpRowHydrator; +use Flow\ETL\Adapter\Parquet\RustParquetOpenSource; +use Flow\ETL\Column\PhpBackend; +use Flow\ETL\Column\RustBackend; use Flow\ETL\Rows; -use Flow\Floe\Format; -use Flow\Floe\NativeFloeEncoder; -use Flow\Floe\PhpFloeEncoder; +use Flow\ETL\Rows\RowsBuilder; +use Flow\ETL\Schema; +use Flow\ETL\Tests\Fixtures\Enum\BasicEnum; +use Flow\Filesystem\SourceStream; +use Flow\Parquet\Engine\RustParquetFileReader; + +use function Flow\ETL\DSL\bool_schema; +use function Flow\ETL\DSL\date_schema; +use function Flow\ETL\DSL\datetime_schema; +use function Flow\ETL\DSL\enum_schema; +use function Flow\ETL\DSL\float_schema; +use function Flow\ETL\DSL\html_schema; +use function Flow\ETL\DSL\int_schema; +use function Flow\ETL\DSL\json_schema; +use function Flow\ETL\DSL\list_schema; +use function Flow\ETL\DSL\map_schema; +use function Flow\ETL\DSL\null_schema; +use function Flow\ETL\DSL\schema; +use function Flow\ETL\DSL\str_schema; +use function Flow\ETL\DSL\structure_schema; +use function Flow\ETL\DSL\time_schema; +use function Flow\ETL\DSL\time_zone_schema; +use function Flow\ETL\DSL\uuid_schema; +use function Flow\ETL\DSL\xml_schema; +use function Flow\Types\DSL\structure_element; +use function Flow\Types\DSL\type_integer; +use function Flow\Types\DSL\type_list; +use function Flow\Types\DSL\type_map; +use function Flow\Types\DSL\type_optional; +use function Flow\Types\DSL\type_string; +use function Flow\Types\DSL\type_structure; /** - * In-memory marker for the schema-carrying entry in the reference frame lists - * below. The Floe format stores the schema in the footer, not in a frame; this - * scaffold threads the schema through the frame list to drive the encoders. + * One column per Definition (html only where \Dom\HTMLDocument exists). */ -const SCHEMA_ENTRY = 0x01; +function all_types_schema(string $zone = 'Europe/Warsaw'): Schema +{ + return schema( + int_schema('int'), + str_schema('string', nullable: true), + float_schema('float'), + bool_schema('bool'), + datetime_schema('datetime', zone: $zone), + date_schema('date'), + time_schema('time'), + uuid_schema('uuid'), + json_schema('json'), + enum_schema('enum', BasicEnum::class), + time_zone_schema('time_zone'), + xml_schema('xml'), + list_schema('list', type_list(type_optional(type_integer()))), + map_schema('map', type_map(type_string(), type_integer())), + structure_schema('structure', type_structure([ + 'a' => type_integer(), + 'b' => structure_element('b', type_optional(type_string()), true), + ])), + null_schema('null'), + ...class_exists('\Dom\HTMLDocument') ? [html_schema('html')] : [], + ); +} /** - * Pure-PHP reference framing built from the Floe encoder primitives: a write - * session carries one schema, so the whole Rows is framed as a single schema - * entry (its union) followed by one ROW frame per row. + * Six rows: bytes that are not UTF-8, a pre-1970 datetime with microseconds, a DST pair, a negative time. * - * @return array + * @return list> */ -function php_frames(Rows $rows): array +function all_types_values(): array { - if ($rows->count() === 0) { - return []; - } + $negative = (new DateTimeImmutable('2024-01-01 01:02:03'))->diff(new DateTimeImmutable('2024-01-01')); + $rows = [ + [ + 'int' => 1, + 'string' => "a\xFFb", + 'float' => 1.5, + 'bool' => true, + 'datetime' => '1969-07-20T20:17:40.123456Z', + 'date' => '1969-12-29', + 'time' => new DateInterval('PT1H2M3S'), + 'uuid' => '6c2f1d4e-8b3a-4c5d-9e6f-0a1b2c3d4e5f', + 'json' => '{"a":1}', + 'enum' => BasicEnum::one, + 'time_zone' => new DateTimeZone('Europe/Warsaw'), + 'xml' => '1', + 'list' => [1, null, 3], + 'map' => ['a' => 1], + 'structure' => ['a' => 1, 'b' => 'x'], + 'null' => null, + ], + [ + 'int' => -2, + 'string' => null, + 'float' => -0.0, + 'bool' => false, + 'datetime' => '2024-10-27T00:30:00Z', + 'date' => '2024-02-29', + 'time' => $negative, + 'uuid' => '00000000-0000-0000-0000-000000000000', + 'json' => '[1,2]', + 'enum' => BasicEnum::two, + 'time_zone' => new DateTimeZone('UTC'), + 'xml' => '', + 'list' => [], + 'map' => [], + 'structure' => ['a' => 2], + 'null' => null, + ], + [ + 'int' => PHP_INT_MAX, + 'string' => '', + 'float' => 1e300, + 'bool' => true, + 'datetime' => '2024-10-27T01:30:00Z', + 'date' => '2000-01-01', + 'time' => new DateInterval('PT0S'), + 'uuid' => 'ffffffff-ffff-ffff-ffff-ffffffffffff', + 'json' => '{}', + 'enum' => BasicEnum::three, + 'time_zone' => new DateTimeZone('America/New_York'), + 'xml' => 't', + 'list' => [null], + 'map' => ['b' => 2, 'c' => 3], + 'structure' => ['a' => 3, 'b' => null], + 'null' => null, + ], + [ + 'int' => PHP_INT_MIN, + 'string' => 'zażółć', + 'float' => 0.1, + 'bool' => false, + 'datetime' => '1970-01-01T00:00:00Z', + 'date' => '1970-01-01', + 'time' => new DateInterval('PT49H'), + 'uuid' => '6c2f1d4e-8b3a-4c5d-9e6f-0a1b2c3d4e5f', + 'json' => '[true]', + 'enum' => BasicEnum::one, + 'time_zone' => new DateTimeZone('Europe/Warsaw'), + 'xml' => '', + 'list' => [4], + 'map' => ['a' => 4], + 'structure' => ['a' => 4, 'b' => 'y'], + 'null' => null, + ], + [ + 'int' => 0, + 'string' => 'e', + 'float' => 2.0, + 'bool' => true, + 'datetime' => '2038-01-19T03:14:08.000001Z', + 'date' => '1900-03-01', + 'time' => new DateInterval('PT59M59S'), + 'uuid' => '12345678-1234-1234-1234-123456789abc', + 'json' => '{"b":[1,{"c":null}]}', + 'enum' => BasicEnum::two, + 'time_zone' => new DateTimeZone('Asia/Tokyo'), + 'xml' => '5', + 'list' => [5, 6], + 'map' => ['z' => 0], + 'structure' => ['a' => 5], + 'null' => null, + ], + [ + 'int' => 7, + 'string' => 'last', + 'float' => -3.25, + 'bool' => false, + 'datetime' => '1900-01-01T12:00:00.5Z', + 'date' => '2100-12-31', + 'time' => new DateInterval('PT1S'), + 'uuid' => 'abcdefab-cdef-abcd-efab-cdefabcdefab', + 'json' => '[]', + 'enum' => BasicEnum::three, + 'time_zone' => new DateTimeZone('UTC'), + 'xml' => '', + 'list' => [7, null], + 'map' => ['q' => 7], + 'structure' => ['a' => 7, 'b' => 'z'], + 'null' => null, + ], + ]; - $hydrator = new PhpRowHydrator(); - $schemaBody = json_encode($rows->schema()->normalize(), JSON_THROW_ON_ERROR); - $encoder = new PhpFloeEncoder(Flow\ETL\DSL\schema_from_json($schemaBody)); + if (class_exists('\\Dom\\HTMLDocument')) { + foreach ($rows as $index => $row) { + $rows[$index] = $row + ['html' => '

html

']; + } + } - $frames = [['type' => SCHEMA_ENTRY, 'body' => $schemaBody]]; + return $rows; +} - foreach ($rows->all() as $row) { - $frames[] = [ - 'type' => Format::FRAME_ROW, - 'body' => $encoder->encode($hydrator->dehydrate(new Rows($rows->schema(), $row)))[0], - ]; - } +/** + * @param list> $values + */ +function php_rows(Schema $schema, array $values): Rows +{ + return (new RowsBuilder($schema, new PhpBackend())) + ->appendRows($values) + ->finish(); +} - return $frames; +/** + * @param list> $values + */ +function native_rows(Schema $schema, array $values): Rows +{ + return (new RowsBuilder($schema, new RustBackend())) + ->appendRows($values) + ->finish(); } /** - * Native counterpart of php_frames(): identical single-schema framing (PHP owns - * it), ROW bodies produced by NativeFloeEncoder. Compared against php_frames() - * to prove the native encode is byte-identical to the pure-PHP encode. - * - * @return array + * `serialize()` with DOM values, which it refuses, replaced by their class and markup. */ -function ext_frames(Rows $rows): array +function comparable(mixed $value): string { - if ($rows->count() === 0) { - return []; - } + $plain = static function (mixed $value) use (&$plain): mixed { + if (is_array($value)) { + return array_map($plain, $value); + } - $hydrator = new NativeRowHydrator(); - $schemaBody = json_encode($rows->schema()->normalize(), JSON_THROW_ON_ERROR); - $encoder = new NativeFloeEncoder(Flow\ETL\DSL\schema_from_json($schemaBody)); + if ($value instanceof DOMDocument) { + return [$value::class, $value->saveXML()]; + } - $frames = [['type' => SCHEMA_ENTRY, 'body' => $schemaBody]]; + if ($value instanceof DOMNode) { + return [$value::class, $value->ownerDocument?->saveXML($value)]; + } - foreach ($rows->all() as $row) { - $frames[] = [ - 'type' => Format::FRAME_ROW, - 'body' => $encoder->encode($hydrator->dehydrate(new Rows($rows->schema(), $row)))[0], + if (!is_object($value) || !str_starts_with($value::class, 'Dom\\')) { + return $value; + } + + // @mago-expect analysis:unavailable-method,mixed-method-access,ambiguous-object-property-access + return [ + $value::class, + $value instanceof Dom\HTMLDocument ? $value->saveHtml() : $value->ownerDocument->saveHtml($value), ]; - } + }; - return $frames; + return serialize($plain($value)); } /** - * Pure-PHP reference decode of frame bodies via the Floe encoder and the row - * hydrator - the byte-for-byte oracle the extension is compared against. + * `RustParquetOpenSource::batches()` of `$schema`'s columns of the Parquet file in `$stream`, adopted by `RustBackend`. * - * @param array $frames - * - * @return array + * @param int<1, max> $batchSize */ -function php_decode_frames(array $frames): array +function rust_parquet_batches(SourceStream $stream, Schema $schema, int $batchSize, ?int $offset, ?int $limit): Iterator { - $hydrator = new PhpRowHydrator(); - $encoder = null; - $schema = null; - $rows = []; - - foreach ($frames as $frame) { - if ($frame['type'] === SCHEMA_ENTRY) { - $schema = Flow\ETL\DSL\schema_from_json($frame['body']); - $encoder = new PhpFloeEncoder($schema); - } elseif ($schema === null || $encoder === null) { - throw new RuntimeException('row frame before any schema entry'); - } else { - foreach ($hydrator->hydrate($encoder->decode([$frame['body']]), $schema)->all() as $row) { - $rows[] = $row; - } - } - } - - return $rows; + return (new RustParquetOpenSource(new RustParquetFileReader($stream)))->batches( + $schema, + $batchSize, + $offset, + $limit, + new RustBackend(), + ); } /** - * Drives the native two-layer pipeline over frame bodies, rebinding the schema - * per schema entry (mirrors how FloeStreamReader sources the schema). - * - * @param array $frames - * - * @return array + * `comparable()` of the result, or `class: message <- previous class: message` of what it threw. */ -function ext_decode_frames(array $frames): array +function outcome(callable $fn): string { - $hydrator = new NativeRowHydrator(); - $encoder = null; - $schema = null; - $rows = []; + try { + return comparable($fn()); + } catch (Throwable $e) { + $previous = $e->getPrevious(); - foreach ($frames as $frame) { - if ($frame['type'] === SCHEMA_ENTRY) { - $schema = Flow\ETL\DSL\schema_from_json($frame['body']); - $encoder = new NativeFloeEncoder($schema); - } elseif ($schema === null || $encoder === null) { - throw new RuntimeException('row frame before any schema entry'); - } else { - foreach ($hydrator->hydrate($encoder->decode([$frame['body']]), $schema)->all() as $row) { - $rows[] = $row; - } - } + return ( + get_class($e) + . ': ' + . $e->getMessage() + . ($previous === null ? '' : ' <- ' . get_class($previous) . ': ' . $previous->getMessage()) + ); } - - return $rows; } /** - * Byte-identical comparison against the pure-PHP decoder on fresh objects. - * Row::isEqual is intentionally avoided before serialize() - it memoizes - * schema state asymmetrically and breaks byte-level comparison. - * - * @param array $expected - * @param array $actual + * Whether an outcome() is a refusal: the class and message of what the callable threw. */ -function assert_rows_identical(array $expected, array $actual): void +function refused(string $outcome): bool { - if (count($expected) !== count($actual)) { - echo 'FAIL: row count ', count($expected), ' !== ', count($actual), "\n"; + return (bool) preg_match('/^[A-Za-z\\\\]+(Exception|Error|Overflow): /', $outcome); +} - return; - } +function assert_same_outcome(callable $php, callable $native): void +{ + $expected = outcome($php); + $actual = outcome($native); - foreach ($expected as $i => $row) { - if (serialize($row) !== serialize($actual[$i])) { - echo "FAIL: row {$i} differs from the PHP decoder output\n"; + echo $expected === $actual ? "identical\n" : "php: {$expected}\nnative: {$actual}\n"; +} - return; - } +/** + * JSON of the interfaces flow_php registers as reflection sees them: methods, parameters, types, optional, variadic, by-ref. + */ +function interfaces_reflection(): string +{ + $interfaces = []; + + foreach ([ + 'Flow\\ETL\\Column\\Backend', + 'Flow\\ETL\\Column\\Column', + 'Flow\\ETL\\Column\\ColumnBuilder', + 'Flow\\ETL\\Adapter\\Parquet\\ParquetOpenSink', + 'Flow\\ETL\\Adapter\\CSV\\CSVEncoder', + 'Flow\\ETL\\Adapter\\JSON\\JsonEncoder', + 'Flow\\ETL\\Adapter\\CSV\\CSVOpenSource', + 'Flow\\ETL\\Adapter\\JSON\\JsonOpenSource', + 'Flow\\ETL\\Adapter\\Parquet\\ParquetOpenSource', + ] as $name) { + $class = new ReflectionClass($name); + // PHP 8.5 reports a userland `self` resolved to the class, an internal one stays `self` - the same type + $type = static fn(?ReflectionType $type): string => (string) $type === 'self' ? $name : (string) $type; + $interfaces[$name] = [ + $class->isInterface(), + array_map(static fn(ReflectionMethod $method): array => [ + $method->getName(), + $type($method->getReturnType()), + array_map(static fn(ReflectionParameter $parameter): array => [ + $parameter->getName(), + $type($parameter->getType()), + $parameter->isOptional(), + $parameter->isVariadic(), + $parameter->isPassedByReference(), + ], $method->getParameters()), + ], $class->getMethods()), + ]; } - echo "identical\n"; + return (string) json_encode($interfaces); } function expect_exception(callable $fn): void @@ -171,13 +345,13 @@ function expect_exception(callable $fn): void try { $fn(); echo "FAIL: no exception thrown\n"; - } catch (Flow\Floe\Exception\ExtensionException $e) { + } catch (Flow\ETL\Exception\RuntimeException $e) { echo get_class($e), ': ', $e->getMessage(), "\n"; } } /** - * The PHP CSV path - CSVLineReader + CSVEncoder::decode() - as `[headers, list of RawRowValues::$values]`. + * The PHP CSV path - CSVLineReader + CSVDecoder::decode() - as `[headers, list of array::$values]`. * * @return array{list, list>} */ @@ -190,7 +364,7 @@ function csv_php_rows( bool $emptyToNull = true, bool $removeBOM = true, ): array { - $encoder = new Flow\ETL\Adapter\CSV\CSVEncoder( + $decoder = new Flow\ETL\Adapter\CSV\CSVDecoder( withHeader: $withHeader, separator: $separator, enclosure: $enclosure, @@ -203,16 +377,16 @@ function csv_php_rows( foreach ($lines->readLines( new Flow\Filesystem\Stream\StringSourceStream(Flow\Filesystem\DSL\path('memory://phpt.csv'), $raw), ) as $line) { - foreach ($encoder->decode([$line]) as $values) { - $rows[] = $values->values; + foreach ($decoder->decode([$line]) as $values) { + $rows[] = $values; } } - return [$encoder->headers() ?? [], $rows]; + return [$decoder->headers() ?? [], $rows]; } /** - * RustCSVReaderNative fed `$raw` in `$chunk`-byte pieces, as `[headers, list of RawRowValues::$values]`. + * RustCSVOpenSource over `$raw`, read in `$chunk`-byte pieces, as `[headers, list of array::$values]`. * * @param positive-int $chunk * @@ -228,32 +402,19 @@ function csv_native_rows( bool $removeBOM = true, int $chunk = 4096, ): array { - $reader = new Flow\ETL\Adapter\CSV\RustCSVReaderNative( + $source = new Flow\ETL\Adapter\CSV\RustCSVOpenSource( + new Flow\Filesystem\Stream\StringSourceStream(Flow\Filesystem\DSL\path('memory://phpt.csv'), $raw), $separator, $enclosure, $escape, $withHeader, $emptyToNull, $removeBOM, + $chunk, ); - $rows = []; - $drain = static function () use ($reader, &$rows): void { - while (($batch = $reader->next(3)) !== []) { - foreach ($batch as $values) { - $rows[] = $values->values; - } - } - }; - - foreach ($raw === '' ? [] : str_split($raw, $chunk) as $piece) { - $reader->feed($piece); - $drain(); - } + $rows = iterator_to_array($source->records(), false); - $reader->finish(); - $drain(); - - return [$reader->headers(), $rows]; + return [$source->headers(), $rows]; } /** @@ -423,6 +584,24 @@ function csv_narrow_corpus(): array '12345678', '00000000', '2024-01-01 25:00', + '1969-12-31T23:59:59.5Z', + '2024-02-29T12:00:00+14:00', + '2024-02-29T12:00:00-12:00', + '2026-01-31T00:00:00+23:59', + '9999-12-31T23:59:59.999999-23:59', + '0001-01-01T00:00:00+23:59', + '2038-01-19T03:14:08Z', + '2026-01-02T03:04:05-00:00', + '2026-01-02t03:04:05Z', + '2026-01-02T03:04:05z', + '2026-01-02T03:04:05,5Z', + '2026-01-02T03:04:05+0100', + '2026-01-02T03:04:05.Z', + '2023-02-29T00:00:00Z', + '0000-01-01T00:00:00Z', + '2026-01-02T03:04:05+24:00', + '2026-01-02T03:04:05+01:60', + '2026-01-02T03:04:05.1234567+01:00', ]; mt_srand(46); @@ -443,27 +622,52 @@ function csv_narrow_corpus(): array } /** - * Prints every value where RustColumnFoldNative::narrowOne() and StringTypeNarrower::narrow() disagree. + * RustCSVOpenSource::sniff() of one CSV column "v" (no names seeded), one row per value - a null value is an empty cell, + * read as null with $emptyToNull. + * + * @param list $values + */ +function rust_sniff_column( + array $values, + Flow\Types\Type\TypeNarrower $typer, + bool $emptyToNull = true, +): Flow\ETL\Schema\Inference\ColumnTypes { + $csv = "v\n"; + + foreach ($values as $value) { + $csv .= ($value === null ? '' : '"' . str_replace('"', '""', $value) . '"') . "\n"; + } + + return (new Flow\ETL\Adapter\CSV\RustCSVOpenSource( + new Flow\Filesystem\Stream\MemorySourceStream($csv), + ',', + '"', + '', + true, + $emptyToNull, + false, + ))->sniff([], -1, new Flow\ETL\Schema\Inference\SchemaInference(), $typer); +} + +/** + * Prints every value whose one-row RustCSVOpenSource::sniff() differs from the PHP ColumnTypes::observe() fold of it - + * StringTypeNarrower::narrow() of the value. * * @param list> $candidates * @param list $corpus */ function assert_narrow_parity(string $label, array $candidates, array $corpus): void { - $php = new Flow\Types\Type\Native\String\StringTypeNarrower($candidates); - $native = new Flow\ETL\Adapter\CSV\RustColumnFoldNative([], array_map( - static fn(Flow\Types\Type $type): string => $type->toString(), - $candidates, - )); + $typer = new Flow\Types\Type\Native\String\StringTypeNarrower($candidates); $mismatches = 0; foreach ($corpus as $value) { - $expected = $php->narrow($value)->toString(); - $actual = $native->narrowOne($value); + $php = new Flow\ETL\Schema\Inference\ColumnTypes([], $typer); + $php->observe(['v' => $value]); - if ($expected !== $actual) { + if ($php != rust_sniff_column([$value], $typer, false)) { $mismatches++; - echo ' ', bin2hex($value), ' ', var_export($value, true), ": php={$expected} native={$actual}\n"; + echo ' ', bin2hex($value), ' ', var_export($value, true), "\n"; } } @@ -564,3 +768,528 @@ function json_leak_cells(): array '{"a":}', ]; } + +/** + * A random value of the text writers' fuzz alphabet: separators, enclosures, escapes, line breaks, slashes, controls, + * multi-byte characters; `mt_srand()` first. + */ +function write_random_string(): string +{ + static $alphabet = [ + 'a', + 'b', + 'Z', + '0', + '7', + ',', + ';', + '|', + '.', + '-', + '"', + "'", + '\\', + "\n", + "\r", + "\t", + ' ', + '/', + '<', + '&', + "\x01", + "\x7f", + 'ż', + '😀', + "\u{2028}", + '', + ]; + $text = ''; + + for ($i = mt_rand(0, 6); $i > 0; $i--) { + $text .= $alphabet[mt_rand(0, count($alphabet) - 1)]; + } + + return $text; +} + +/** + * A random type of nesting depth <= $depth that a column can hold, and a generator of its values. + * + * @return array{Flow\Types\Type, Closure(): mixed} + */ +function write_random_type(int $depth): array +{ + static $zones = ['UTC', 'Europe/Warsaw', 'America/New_York', '+02:30', '-05:00']; + static $floats = [ + 0.1 + 0.2, + 1.0, + -0.0, + 1.0e25, + 1.0e-7, + 123456789012345.67, + -2.5, + 1 / 3, + PHP_FLOAT_MAX, + PHP_FLOAT_MIN, + 5.0e-324, + ]; + static $documents = [ + '{"a":1}', + '[]', + '{}', + '{"a":{}}', + '[1,{"0":"x"}]', + '["s/t"]', + '[5, 1.0]', + '{"k":[1500.0,true,null]}', + '{"a": "b c"}', + ]; + static $markup = ['', '1', 'a & b']; + static $keys = ['a', 'b c', 'k/1', 'ż', '"q"', 'x,y']; + static $fields = ['a', 'b', 'c d', 'e"f', 'g/h']; + + $kind = mt_rand(0, $depth > 0 ? 15 : 11); + + [$type, $value] = match ($kind) { + 0 => [ + Flow\Types\DSL\type_integer(), + static fn(): int => mt_rand(0, 3) === 0 ? mt_rand(-5, 5) : mt_rand() * (mt_rand(0, 1) ? 1 : -1), + ], + 1 => [ + Flow\Types\DSL\type_float(), + static fn(): float => mt_rand(0, 2) === 0 + ? $floats[mt_rand(0, count($floats) - 1)] + : mt_rand(-100000, 100000) / 1000, + ], + 2 => [Flow\Types\DSL\type_boolean(), static fn(): bool => mt_rand(0, 1) === 1], + 3 => [Flow\Types\DSL\type_string(), write_random_string(...)], + 4 => (static function () use ($zones): array { + $zone = $zones[mt_rand(0, count($zones) - 1)]; + + return [ + Flow\Types\DSL\type_datetime($zone), + static fn(): DateTimeImmutable => (new DateTimeImmutable( + '@' . mt_rand(-3_000_000_000, 5_000_000_000), + ))->modify('+' . mt_rand(0, 999_999) . ' usec'), + ]; + })(), + 5 => [ + Flow\Types\DSL\type_date(), + static fn(): DateTimeImmutable => new DateTimeImmutable('@' . (mt_rand(-50_000, 50_000) * 86_400)), + ], + 6 => [ + Flow\Types\DSL\type_time(), + static fn(): DateInterval => (new DateTimeImmutable('@0'))->diff( + new DateTimeImmutable('@' . mt_rand(-200_000, 200_000)), + ), + ], + 7 => [ + Flow\Types\DSL\type_uuid(), + static fn(): string => sprintf( + '%08x-%04x-4%03x-%04x-%012x', + mt_rand(0, 0xffffffff), + mt_rand(0, 0xffff), + mt_rand(0, 0xfff), + mt_rand(0x8000, 0xbfff), + mt_rand(0, 0xffffffffffff), + ), + ], + 8 => [Flow\Types\DSL\type_json(), static fn(): string => $documents[mt_rand(0, count($documents) - 1)]], + 9 => [Flow\Types\DSL\type_enum(BasicEnum::class), static fn(): BasicEnum => BasicEnum::cases()[mt_rand(0, 2)]], + 10 => [ + Flow\Types\DSL\type_time_zone(), + static fn(): DateTimeZone => new DateTimeZone($zones[mt_rand(0, count($zones) - 1)]), + ], + 11 => [Flow\Types\DSL\type_xml(), static fn(): string => $markup[mt_rand(0, count($markup) - 1)]], + 12 => (static function () use ($depth): array { + [$element, $value] = write_random_type($depth - 1); + + return [ + type_list($element), + static function () use ($value): array { + $list = []; + + for ($i = mt_rand(0, 3); $i > 0; $i--) { + $list[] = $value(); + } + + return $list; + }, + ]; + })(), + 13 => (static function () use ($depth, $keys): array { + [$element, $value] = write_random_type($depth - 1); + + return [ + type_map(type_string(), $element), + static function () use ($value, $keys): array { + $map = []; + + for ($i = mt_rand(0, 3); $i > 0; $i--) { + $map[$keys[mt_rand(0, count($keys) - 1)]] = $value(); + } + + return $map; + }, + ]; + })(), + 14 => (static function () use ($depth): array { + [$element, $value] = write_random_type($depth - 1); + + return [ + type_map(type_integer(), $element), + static function () use ($value): array { + $map = []; + + for ($i = mt_rand(0, 3); $i > 0; $i--) { + $map[mt_rand(-3, 40)] = $value(); + } + + return $map; + }, + ]; + })(), + 15 => (static function () use ($depth, $fields): array { + $elements = []; + $values = []; + + foreach (array_rand(array_flip($fields), mt_rand(2, 3)) as $name) { + [$element, $value] = write_random_type($depth - 1); + $optional = mt_rand(0, 3) === 0; + // an optional element is present or absent, never an explicit null + $elements[$name] = structure_element( + $name, + $optional && $element instanceof Flow\Types\Type\Logical\OptionalType ? $element->base() : $element, + $optional, + ); + $values[$name] = [ + $optional, + $optional && $element instanceof Flow\Types\Type\Logical\OptionalType + ? static function () use ($value): mixed { + do { + // @mago-ignore analysis:mixed-assignment + $generated = $value(); + } while ($generated === null); + + return $generated; + } : $value, + ]; + } + + return [ + type_structure($elements), + static function () use ($values): array { + $structure = []; + + foreach ($values as $name => [$optional, $value]) { + if (!$optional || mt_rand(0, 1) === 1) { + $structure[$name] = $value(); + } + } + + return $structure; + }, + ]; + })(), + default => throw new LogicException('no such kind'), + }; + + if (mt_rand(0, 3) > 0) { + return [$type, $value]; + } + + return [type_optional($type), static fn(): mixed => mt_rand(0, 5) === 0 ? null : $value()]; +} + +/** + * A value of `$type` (from write_random_type()) in a random input form: as generated, in a raw form a reader hands over + * (numeric and ISO strings, Uuid and Json objects, a datetime in another zone), in a form the cast refuses (a wrong + * scalar, a scalar for a container, non-list keys, integer keys for string keys, a missing required element, a present + * null, an extra key), or null; containers recurse into their elements. `mt_srand()` first. + * + * @param Flow\Types\Type $type + */ +function cast_random_input(Flow\Types\Type $type, mixed $value): mixed +{ + static $zones = ['UTC', 'Europe/Warsaw', 'America/New_York', '+02:30', '-05:00']; + + $roll = mt_rand(0, 19); + + if ($value === null || $roll === 0) { + return null; + } + + $type = $type instanceof Flow\Types\Type\Logical\OptionalType ? $type->base() : $type; + + if ($roll === 1) { + return $type instanceof Flow\Types\Type\Logical\ListType + || $type instanceof Flow\Types\Type\Logical\MapType + || $type instanceof Flow\Types\Type\Logical\StructureType + ? [5, 'x', true][mt_rand(0, 2)] + : ['not a value', [1], -1.5][mt_rand(0, 2)]; + } + + $raw = $roll < 10; + + return match (true) { + $type instanceof Flow\Types\Type\Logical\ListType => (static function () use ($type, $value, $roll): array { + $list = array_map(static fn(mixed $item): mixed => cast_random_input( + $type->element(), + $item, + ), Flow\Types\DSL\type_list(Flow\Types\DSL\type_mixed())->assert($value)); + + return $roll === 2 && $list !== [] ? array_combine(range(1, count($list)), $list) : $list; + })(), + $type instanceof Flow\Types\Type\Logical\MapType => (static function () use ($type, $value, $roll): array { + $map = array_map(static fn(mixed $item): mixed => cast_random_input( + $type->value(), + $item, + ), Flow\Types\DSL\type_array()->assert($value)); + + if ($roll === 2 && $map !== []) { + $map[mt_rand(100, 200)] = reset($map); + } + + return $map; + })(), + $type instanceof Flow\Types\Type\Logical\StructureType => (static function () use ( + $type, + $value, + $roll, + ): array { + $structure = []; + + // @mago-ignore analysis:mixed-assignment + foreach (Flow\Types\DSL\type_array()->assert($value) as $name => $item) { + $structure[$name] = cast_random_input( + $type->element($name)->type ?? throw new LogicException('no such element'), + $item, + ); + } + + $names = array_keys($structure); + $name = $names === [] ? 'a' : $names[mt_rand(0, count($names) - 1)]; + + return match ($roll) { + 2 => array_diff_key($structure, [$name => true]), + 3 => array_replace($structure, [$name => null]), + 4 => $structure + ['extra key' => 1], + default => $structure, + }; + })(), + !$raw => $value, + $type instanceof Flow\Types\Type\Native\IntegerType, + $type instanceof Flow\Types\Type\Native\FloatType, + => Flow\Types\DSL\type_string()->cast($value), + $type instanceof Flow\Types\Type\Native\BooleanType => ['true', 'false', '1', '0', 'yes', 'off'][mt_rand(0, 5)], + $type instanceof Flow\Types\Type\Logical\DateTimeType, + $type instanceof Flow\Types\Type\Logical\DateType, + => mt_rand(0, 1) === 0 + ? Flow\Types\DSL\type_instance_of(DateTimeImmutable::class) + ->assert($value) + ->format($type instanceof Flow\Types\Type\Logical\DateType ? 'Y-m-d' : 'Y-m-d\\TH:i:s.uP') + : Flow\Types\DSL\type_instance_of(DateTimeImmutable::class) + ->assert($value) + ->setTimezone(new DateTimeZone($zones[mt_rand(0, count($zones) - 1)])), + $type instanceof Flow\Types\Type\Logical\UuidType + => new Flow\Types\Value\Uuid(Flow\Types\DSL\type_string()->assert($value)), + $type instanceof Flow\Types\Type\Logical\JsonType + => new Flow\Types\Value\Json(Flow\Types\DSL\type_string()->assert($value)), + default => $value, + }; +} + +/** + * A random batch for the text writers: 1-6 columns of random types (nesting <= 3) under plain and awkward names, 0-40 + * rows. + * + * @return array{Schema, list>} + */ +function write_random_batch(): array +{ + static $names = ['id', 'first name', 'a"b', 'ż', 'x/y', 'name', 'value', 'at', '0', '7', 'a,b']; + $definitions = []; + $values = []; + + foreach ((array) array_rand(array_flip($names), mt_rand(1, 6)) as $name) { + [$type, $value] = write_random_type(3); + $definitions[] = Flow\ETL\DSL\definition_from_type((string) $name, $type); + $values[(string) $name] = $value; + } + + $rows = []; + + for ($i = [0, 1, mt_rand(2, 40), mt_rand(2, 40)][mt_rand(0, 3)]; $i > 0; $i--) { + $row = []; + + foreach ($values as $name => $value) { + $row[$name] = $value(); + } + + $rows[] = $row; + } + + return [schema(...$definitions), $rows]; +} + +/** + * The bytes a text sink wrote, or the class and message of what it threw. + * + * @param callable(Flow\Filesystem\Stream\StringDestinationStream): void $write + */ +function written(callable $write): string +{ + $stream = new Flow\Filesystem\Stream\StringDestinationStream(Flow\Filesystem\DSL\path('memory://out')); + + try { + $write($stream); + } catch (Throwable $e) { + return $e::class . ': ' . $e->getMessage(); + } + + return $stream->content(); +} + +/** + * A stream over `$raw` whose read() returns at most `$chunk` bytes - a reader fed in pieces of that size. + * + * @param positive-int $chunk + */ +function chunked_source_stream(string $raw, int $chunk): Flow\Filesystem\SourceStream +{ + return new class( + new Flow\Filesystem\Stream\StringSourceStream(Flow\Filesystem\DSL\path('memory://phpt.json'), $raw), + $chunk, + ) implements Flow\Filesystem\SourceStream { + /** + * @param positive-int $chunk + */ + public function __construct( + private readonly Flow\Filesystem\Stream\StringSourceStream $stream, + private readonly int $chunk, + ) {} + + public function close(): void + { + $this->stream->close(); + } + + public function content(): string + { + return $this->stream->content(); + } + + public function isOpen(): bool + { + return $this->stream->isOpen(); + } + + /** + * @param positive-int $length + */ + public function iterate(int $length = 1): Generator + { + return $this->stream->iterate(min($length, $this->chunk)); + } + + public function path(): Flow\Filesystem\Path + { + return $this->stream->path(); + } + + /** + * @param positive-int $length + */ + public function read(int $length, int $offset): string + { + return $this->stream->read(min($length, $this->chunk), $offset); + } + + public function readLines(string $separator = "\n", ?int $length = null): Generator + { + return $this->stream->readLines($separator, $length); + } + + public function size(): ?int + { + return $this->stream->size(); + } + }; +} + +/** + * Stands in for the PhpCSVEncoder a RustCSVEncoder holds: renders through it and records every column left to it. + */ +final class RecordingPhpCSVEncoder +{ + /** + * @var list + */ + public array $columns = []; + + public function __construct( + private readonly Flow\ETL\Adapter\CSV\PhpCSVEncoder $php, + ) {} + + /** + * @return list + */ + public function cells(Flow\Types\Type $type, Flow\ETL\Column\Column $column): array + { + $this->columns[] = $column; + + return $this->php->cells($type, $column); + } + + /** + * @param list $headers + */ + public function encodeHeader(array $headers): string + { + return $this->php->encodeHeader($headers); + } +} + +/** + * Stands in for the PhpJsonEncoder a RustJsonEncoder holds: renders through it and records every column left to it. + */ +final class RecordingPhpJsonEncoder +{ + /** + * @var list + */ + public array $columns = []; + + public function __construct( + private readonly Flow\ETL\Adapter\JSON\PhpJsonEncoder $php, + ) {} + + /** + * @return list + */ + public function fragments(Flow\Types\Type $type, Flow\ETL\Column\Column $column): array + { + $this->columns[] = $column; + + return $this->php->fragments($type, $column); + } +} + +/** + * The names of the columns of `$rows` among `$recorded`. + * + * @param list $recorded + * + * @return list + */ +function recorded_names(array $recorded, Rows $rows): array +{ + return array_map( + 'strval', + array_keys(array_filter($rows->columns(), static fn(Flow\ETL\Column\Column $column): bool => in_array( + $column, + $recorded, + true, + ))), + ); +} diff --git a/src/extension/pg-query-ext/Makefile b/src/extension/pg-query-ext/Makefile index 8164b79ef1..76872ec6a2 100644 --- a/src/extension/pg-query-ext/Makefile +++ b/src/extension/pg-query-ext/Makefile @@ -1,5 +1,5 @@ # Flow PHP pg_query Extension Makefile -LIBPG_QUERY_VERSION := 18-latest +LIBPG_QUERY_VERSION := 18.0.0 LIBPG_QUERY_DIR := ./vendor/libpg_query LIBPG_QUERY_LIB := $(LIBPG_QUERY_DIR)/libpg_query.a EXTENSION_DIR := ./ext diff --git a/src/extension/pg-query-ext/ext/config.m4 b/src/extension/pg-query-ext/ext/config.m4 index c2ac294b4d..10f0d454cc 100644 --- a/src/extension/pg-query-ext/ext/config.m4 +++ b/src/extension/pg-query-ext/ext/config.m4 @@ -6,13 +6,13 @@ PHP_ARG_WITH([pg-query], [Include pg_query support. DIR is the libpg_query install prefix (optional - will download if not found)])]) if test "$PHP_PG_QUERY" != "no"; then - dnl libpg_query 18-latest is required for postgres_deparse.h support - LIBPG_QUERY_VERSION="18-latest" + dnl libpg_query 18.0.0 (PostgreSQL 18 grammar) is required for postgres_deparse.h support + LIBPG_QUERY_VERSION="18.0.0" AC_MSG_NOTICE([Using libpg_query $LIBPG_QUERY_VERSION (PostgreSQL 18 grammar)]) PG_QUERY_DIR="" - dnl Expected PostgreSQL major derived from the pinned version ("18-latest" -> "18", "18.0.0" -> "18") + dnl Expected PostgreSQL major derived from the pinned version ("18.0.0" -> "18") PG_EXPECTED_MAJOR=`echo "$LIBPG_QUERY_VERSION" | sed -E 's/^([[0-9]]+).*/\1/'` dnl Search for existing libpg_query installation diff --git a/src/lib/doctrine-dbal-bulk/composer.json b/src/lib/doctrine-dbal-bulk/composer.json index 99f2e3e7a3..bbc98353bd 100644 --- a/src/lib/doctrine-dbal-bulk/composer.json +++ b/src/lib/doctrine-dbal-bulk/composer.json @@ -12,7 +12,7 @@ ], "require": { "php": "~8.3.0 || ~8.4.0 || ~8.5.0", - "doctrine/dbal": "^4.4", + "doctrine/dbal": "^4.5", "flow-php/types": "self.version" }, "autoload": { diff --git a/src/lib/doctrine-dbal-bulk/src/Flow/Doctrine/Bulk/BulkData.php b/src/lib/doctrine-dbal-bulk/src/Flow/Doctrine/Bulk/BulkData.php index a36bf8c072..75a0c50750 100644 --- a/src/lib/doctrine-dbal-bulk/src/Flow/Doctrine/Bulk/BulkData.php +++ b/src/lib/doctrine-dbal-bulk/src/Flow/Doctrine/Bulk/BulkData.php @@ -182,7 +182,7 @@ function (int $index, array $row) use ($table): string { if (array_key_exists($columnName, $this->types)) { $type = $this->types[$columnName]; } else { - $type = $table->dbalColumn($columnName)->getType(); + $type = Type::getType($table->dbalColumn($columnName)->getTypeName()); } $keys[] = @@ -226,7 +226,7 @@ public function toSqlNamedParameters(TableDefinition $table): array $type = $types[$column] ??= array_key_exists($column, $this->types) ? $this->types[$column] - : $table->dbalColumn($column)->getType(); + : Type::getType($table->dbalColumn($column)->getTypeName()); $rows[$index][$column . '_' . $index] = $type->convertToDatabaseValue($entry, $platform); } } @@ -284,7 +284,7 @@ function (array $row) use ($table): string { $type = $this->types[$columnName]; } else { $dbColumn = $table->dbalColumn($columnName); - $type = $dbColumn->getType(); + $type = Type::getType($dbColumn->getTypeName()); } $keys[] = 'CAST(? as ' . $type->getSQLDeclaration([], $table->platform()) . ')'; @@ -317,7 +317,7 @@ public function toSqlPositionalParameters(TableDefinition $table): array $type = $types[$column] ??= array_key_exists($column, $this->types) ? $this->types[$column] - : $table->dbalColumn($column)->getType(); + : Type::getType($table->dbalColumn($column)->getTypeName()); $parameters[] = $type->convertToDatabaseValue($entry, $platform); } } diff --git a/src/lib/doctrine-dbal-bulk/src/Flow/Doctrine/Bulk/TableDefinition.php b/src/lib/doctrine-dbal-bulk/src/Flow/Doctrine/Bulk/TableDefinition.php index 4c37301833..9865514ebc 100644 --- a/src/lib/doctrine-dbal-bulk/src/Flow/Doctrine/Bulk/TableDefinition.php +++ b/src/lib/doctrine-dbal-bulk/src/Flow/Doctrine/Bulk/TableDefinition.php @@ -7,7 +7,6 @@ use Doctrine\DBAL\Connection; use Doctrine\DBAL\Platforms\AbstractPlatform; use Doctrine\DBAL\Schema\Column; -use Doctrine\DBAL\Types\Type; use Flow\Doctrine\Bulk\Exception\RuntimeException; use function array_filter; @@ -69,7 +68,7 @@ public function dbalPositionalTypes(BulkData $bulkData): array for ($i = 0; $i < $bulkData->count(); $i++) { foreach ($bulkData->columns()->all() as $columnName) { $dbColumn = $this->dbalColumn($columnName); - $types[] = Type::getTypeRegistry()->lookupName($dbColumn->getType()); + $types[] = $dbColumn->getTypeName(); } } @@ -91,7 +90,7 @@ public function dbalTypes(BulkData $bulkData): array $dbColumn = $this->dbalColumn($columnName); for ($i = 0; $i < $bulkData->count(); $i++) { - $types[$columnName . '_' . $i] = Type::getTypeRegistry()->lookupName($dbColumn->getType()); + $types[$columnName . '_' . $i] = $dbColumn->getTypeName(); } } diff --git a/src/lib/doctrine-dbal-bulk/tests/Flow/Doctrine/Bulk/Tests/Mother/WideTableMother.php b/src/lib/doctrine-dbal-bulk/tests/Flow/Doctrine/Bulk/Tests/Mother/WideTableMother.php index 9d121d8994..5a49c56a04 100644 --- a/src/lib/doctrine-dbal-bulk/tests/Flow/Doctrine/Bulk/Tests/Mother/WideTableMother.php +++ b/src/lib/doctrine-dbal-bulk/tests/Flow/Doctrine/Bulk/Tests/Mother/WideTableMother.php @@ -66,6 +66,8 @@ public static function table(string $name, int $columns): Table public static function tableWithPrimaryKey(string $name, int $columns): Table { return self::table($name, $columns) - ->addPrimaryKeyConstraint(PrimaryKeyConstraint::editor()->setUnquotedColumnNames('c1')->create()); + ->edit() + ->addPrimaryKeyConstraint(PrimaryKeyConstraint::editor()->setUnquotedColumnNames('c1')->create()) + ->create(); } } diff --git a/src/lib/doctrine-dbal-bulk/tests/Flow/Doctrine/Bulk/Tests/Unit/TableDefinitionTest.php b/src/lib/doctrine-dbal-bulk/tests/Flow/Doctrine/Bulk/Tests/Unit/TableDefinitionTest.php index e650fbab70..770e61f0d8 100644 --- a/src/lib/doctrine-dbal-bulk/tests/Flow/Doctrine/Bulk/Tests/Unit/TableDefinitionTest.php +++ b/src/lib/doctrine-dbal-bulk/tests/Flow/Doctrine/Bulk/Tests/Unit/TableDefinitionTest.php @@ -53,10 +53,7 @@ public function test_column_caching_works_correctly(): void // @mago-expect analysis:deprecated-method // @mago-expect analysis:deprecated-method static::assertSame($column1->getName(), $column2->getName()); - static::assertSame( - Type::getTypeRegistry()->lookupName($column1->getType()), - Type::getTypeRegistry()->lookupName($column2->getType()), - ); + static::assertSame($column1->getTypeName(), $column2->getTypeName()); } public function test_column_retrieval_from_non_existent_table(): void @@ -99,7 +96,7 @@ public function test_dbal_column_returns_column_with_correct_type(): void // @mago-expect analysis:deprecated-method static::assertSame('name', $nameColumn->getName()); - static::assertSame(Types::STRING, Type::getTypeRegistry()->lookupName($nameColumn->getType())); + static::assertSame(Types::STRING, $nameColumn->getTypeName()); } public function test_dbal_column_returns_existing_column(): void diff --git a/src/lib/filesystem/src/Flow/Filesystem/Path.php b/src/lib/filesystem/src/Flow/Filesystem/Path.php index 99b2d6472e..d48e2fe7eb 100644 --- a/src/lib/filesystem/src/Flow/Filesystem/Path.php +++ b/src/lib/filesystem/src/Flow/Filesystem/Path.php @@ -83,10 +83,14 @@ public function extension(): string|false public function extractPlaceholderPartitions(self $path): Partitions { if ($this->implementation instanceof UnixPath) { - return $this->implementation->extractPlaceholderPartitions(type_instance_of(UnixPath::class)->assert($path->implementation)); + return $this->implementation->extractPlaceholderPartitions(type_instance_of( + UnixPath::class, + )->assert($path->implementation)); } - return $this->implementation->extractPlaceholderPartitions(type_instance_of(WindowsPath::class)->assert($path->implementation)); + return $this->implementation->extractPlaceholderPartitions(type_instance_of( + WindowsPath::class, + )->assert($path->implementation)); } public function filename(): string diff --git a/src/lib/filesystem/src/Flow/Filesystem/Path/UnixPath.php b/src/lib/filesystem/src/Flow/Filesystem/Path/UnixPath.php index a408c7ee94..95d8f90028 100644 --- a/src/lib/filesystem/src/Flow/Filesystem/Path/UnixPath.php +++ b/src/lib/filesystem/src/Flow/Filesystem/Path/UnixPath.php @@ -525,6 +525,10 @@ public function suffix(string $string): self public function uri(): string { + if ($this->protocol === 'file') { + return 'file://' . $this->path; + } + return $this->protocol . '://' . ltrim($this->path, '/'); } diff --git a/src/lib/filesystem/tests/Flow/Filesystem/Tests/Double/FailingAppendDestinationStream.php b/src/lib/filesystem/tests/Flow/Filesystem/Tests/Double/FailingAppendDestinationStream.php new file mode 100644 index 0000000000..7a9f96d683 --- /dev/null +++ b/src/lib/filesystem/tests/Flow/Filesystem/Tests/Double/FailingAppendDestinationStream.php @@ -0,0 +1,48 @@ +inner->path()->uri())); + } + + public function close(): void + { + $this->inner->close(); + } + + /** + * @param resource $resource + */ + public function fromResource($resource): self + { + $this->inner->fromResource($resource); + + return $this; + } + + public function isOpen(): bool + { + return $this->inner->isOpen(); + } + + public function path(): Path + { + return $this->inner->path(); + } +} diff --git a/src/lib/filesystem/tests/Flow/Filesystem/Tests/Double/FailingReadSourceStream.php b/src/lib/filesystem/tests/Flow/Filesystem/Tests/Double/FailingReadSourceStream.php new file mode 100644 index 0000000000..d5da9b19df --- /dev/null +++ b/src/lib/filesystem/tests/Flow/Filesystem/Tests/Double/FailingReadSourceStream.php @@ -0,0 +1,59 @@ +inner->close(); + } + + public function content(): string + { + return $this->inner->content(); + } + + public function isOpen(): bool + { + return $this->inner->isOpen(); + } + + public function iterate(int $length = 1): Generator + { + return $this->inner->iterate($length); + } + + public function path(): Path + { + return $this->inner->path(); + } + + public function read(int $length, int $offset): string + { + throw new RuntimeException(sprintf('Reading "%s" failed', $this->inner->path()->uri())); + } + + public function readLines(string $separator = "\n", ?int $length = null): Generator + { + return $this->inner->readLines($separator, $length); + } + + public function size(): ?int + { + return $this->inner->size(); + } +} diff --git a/src/lib/filesystem/tests/Flow/Filesystem/Tests/Integration/OS/Unix/NativeLocalFilesystemTest.php b/src/lib/filesystem/tests/Flow/Filesystem/Tests/Integration/OS/Unix/NativeLocalFilesystemTest.php index 17eb48d473..c92fba7e47 100644 --- a/src/lib/filesystem/tests/Flow/Filesystem/Tests/Integration/OS/Unix/NativeLocalFilesystemTest.php +++ b/src/lib/filesystem/tests/Flow/Filesystem/Tests/Integration/OS/Unix/NativeLocalFilesystemTest.php @@ -58,10 +58,7 @@ public function test_file_status_on_pattern_unix_uri_format(): void $statusForUri = $fs->status(path(__DIR__ . '/../var/some_path_to/*.txt')); static::assertNotNull($statusForUri); - static::assertSame( - 'file://' . ltrim(__DIR__, '/') . '/../var/some_path_to/file.txt', - $statusForUri->path->uri(), - ); + static::assertSame('file://' . __DIR__ . '/../var/some_path_to/file.txt', $statusForUri->path->uri()); $fs->rm(path(__DIR__ . '/../var/some_path_to')); } @@ -98,7 +95,7 @@ public function test_tmp_dir_unix_uri_format(): void { $fs = native_local_filesystem(); - static::assertSame('file://' . ltrim(sys_get_temp_dir(), '/'), $fs->getSystemTmpDir()->uri()); + static::assertSame('file://' . sys_get_temp_dir(), $fs->getSystemTmpDir()->uri()); } public function test_unix_absolute_path_behavior(): void diff --git a/src/lib/filesystem/tests/Flow/Filesystem/Tests/Unit/Path/UnixPathTest.php b/src/lib/filesystem/tests/Flow/Filesystem/Tests/Unit/Path/UnixPathTest.php index 158356d083..51f60ead11 100644 --- a/src/lib/filesystem/tests/Flow/Filesystem/Tests/Unit/Path/UnixPathTest.php +++ b/src/lib/filesystem/tests/Flow/Filesystem/Tests/Unit/Path/UnixPathTest.php @@ -62,7 +62,7 @@ public function test_absolute_path_handling(): void $path = new UnixPath('/path/to/file.txt'); static::assertEquals('/path/to/file.txt', $path->path()); - static::assertEquals('file://path/to/file.txt', $path->uri()); + static::assertEquals('file:///path/to/file.txt', $path->uri()); static::assertEquals('file.txt', $path->basename()); static::assertEquals('file', $path->filename()); static::assertEquals('txt', $path->extension()); @@ -414,8 +414,8 @@ public function test_partitions_paths_with_root_directory(): void $partitionPaths = $path->partitionsPaths(); static::assertCount(2, $partitionPaths); - static::assertEquals('file://country=US', $partitionPaths[0]->uri()); - static::assertEquals('file://country=US/region=west', $partitionPaths[1]->uri()); + static::assertEquals('file:///country=US', $partitionPaths[0]->uri()); + static::assertEquals('file:///country=US/region=west', $partitionPaths[1]->uri()); } public function test_partitions_paths_without_partitions(): void @@ -563,7 +563,7 @@ public function test_realpath_with_file_scheme_strips_protocol_prefix(): void static::assertSame('file', $path->protocol()); static::assertSame('/private/tmp/foo.txt', $path->path()); - static::assertSame('file://private/tmp/foo.txt', $path->uri()); + static::assertSame('file:///private/tmp/foo.txt', $path->uri()); } public function test_realpath_with_non_file_scheme(): void @@ -584,7 +584,7 @@ public function test_relative_path_normalization(): void $path = new UnixPath('relative/path/file.txt'); static::assertEquals('/relative/path/file.txt', $path->path()); - static::assertEquals('file://relative/path/file.txt', $path->uri()); + static::assertEquals('file:///relative/path/file.txt', $path->uri()); } public function test_root_directory_cases(): void @@ -614,7 +614,7 @@ public function test_root_partition_handling(): void $partitioned = $path->addPartitions(partition('group', 'a')); static::assertEquals('/group=a/file.txt', $partitioned->path()); - static::assertEquals('file://group=a/file.txt', $partitioned->uri()); + static::assertEquals('file:///group=a/file.txt', $partitioned->uri()); } public function test_set_extension_without_existing_extension(): void @@ -647,11 +647,11 @@ public function test_skip_directories(): void $skipped1 = $path->skipDirectories(1); static::assertNotNull($skipped1); - static::assertEquals('file://www/index.html', $skipped1->uri()); + static::assertEquals('file:///www/index.html', $skipped1->uri()); $skipped2 = $path->skipDirectories(2); static::assertNotNull($skipped2); - static::assertEquals('file://index.html', $skipped2->uri()); + static::assertEquals('file:///index.html', $skipped2->uri()); $skipped3 = $path->skipDirectories(3); static::assertNull($skipped3); @@ -721,6 +721,25 @@ public function test_uri_method(): void { $path = new UnixPath('/path/file.txt'); - static::assertEquals('file://path/file.txt', $path->uri()); + static::assertEquals('file:///path/file.txt', $path->uri()); + } + + public function test_uri_of_a_file_path_reads_back_as_the_same_path(): void + { + $path = new UnixPath('/path/file.txt'); + + static::assertSame('/path/file.txt', (new UnixPath($path->uri()))->path()); + static::assertSame('file:///path/file.txt', (new UnixPath($path->uri()))->uri()); + } + + public function test_uri_of_the_root_directory(): void + { + static::assertSame('file:///', (new UnixPath('/'))->uri()); + } + + public function test_uri_of_a_non_file_protocol_has_no_leading_slash(): void + { + static::assertSame('memory://path/file.txt', (new UnixPath('memory://path/file.txt'))->uri()); + static::assertSame('s3://bucket/key.txt', (new UnixPath('s3://bucket/key.txt'))->uri()); } } diff --git a/src/lib/filesystem/tests/Flow/Filesystem/Tests/Unit/PathTest.php b/src/lib/filesystem/tests/Flow/Filesystem/Tests/Unit/PathTest.php index 74c86fa814..a600d6aba3 100644 --- a/src/lib/filesystem/tests/Flow/Filesystem/Tests/Unit/PathTest.php +++ b/src/lib/filesystem/tests/Flow/Filesystem/Tests/Unit/PathTest.php @@ -38,13 +38,13 @@ public static function directories(): Generator */ public static function paths(): Generator { - yield ['/file.csv', 'file', 'file://file.csv']; - yield ['file://file.csv', 'file', 'file://file.csv']; - yield ['file:///', 'file', 'file://']; - yield ['/', 'file', 'file://']; - yield ['/absolute/path/to/file.txt', 'file', 'file://absolute/path/to/file.txt']; - yield ['file://absolute/path/to/file.txt', 'file', 'file://absolute/path/to/file.txt']; - yield ['file:///absolute/path/to/file.txt', 'file', 'file://absolute/path/to/file.txt']; + yield ['/file.csv', 'file', 'file:///file.csv']; + yield ['file://file.csv', 'file', 'file:///file.csv']; + yield ['file:///', 'file', 'file:///']; + yield ['/', 'file', 'file:///']; + yield ['/absolute/path/to/file.txt', 'file', 'file:///absolute/path/to/file.txt']; + yield ['file://absolute/path/to/file.txt', 'file', 'file:///absolute/path/to/file.txt']; + yield ['file:///absolute/path/to/file.txt', 'file', 'file:///absolute/path/to/file.txt']; yield ['flow-file://', 'flow-file', 'flow-file://']; yield ['flow-file:///', 'flow-file', 'flow-file://']; yield ['flow-file://folder/file.csv', 'flow-file', 'flow-file://folder/file.csv']; diff --git a/src/lib/parquet-viewer/composer.json b/src/lib/parquet-viewer/composer.json index 9814cb7777..0dfa12e9c2 100644 --- a/src/lib/parquet-viewer/composer.json +++ b/src/lib/parquet-viewer/composer.json @@ -21,6 +21,9 @@ "optimize-autoloader": true, "sort-packages": true }, + "conflict": { + "ext-arrow": "<0.45" + }, "suggest": { "ext-arrow": "For high-performance Parquet reading/writing via Apache Arrow" }, diff --git a/src/lib/parquet-viewer/src/Flow/ParquetViewer/Command/ReadDDLCommand.php b/src/lib/parquet-viewer/src/Flow/ParquetViewer/Command/ReadDDLCommand.php index 077f5d29ca..530c065332 100644 --- a/src/lib/parquet-viewer/src/Flow/ParquetViewer/Command/ReadDDLCommand.php +++ b/src/lib/parquet-viewer/src/Flow/ParquetViewer/Command/ReadDDLCommand.php @@ -38,10 +38,8 @@ protected function execute(InputInterface $input, OutputInterface $output): int return Command::FAILURE; } - $reader = new Reader(); - $parquetFile = $reader->read($filePath); - try { + $parquetFile = (new Reader())->read($filePath); $parquetFile->metadata(); } catch (InvalidArgumentException) { $style->error(sprintf('File "%s" is not a valid parquet file', $filePath)); diff --git a/src/lib/parquet-viewer/src/Flow/ParquetViewer/Command/ReadDataCommand.php b/src/lib/parquet-viewer/src/Flow/ParquetViewer/Command/ReadDataCommand.php index 55c38a10fc..c48fd2caed 100644 --- a/src/lib/parquet-viewer/src/Flow/ParquetViewer/Command/ReadDataCommand.php +++ b/src/lib/parquet-viewer/src/Flow/ParquetViewer/Command/ReadDataCommand.php @@ -56,10 +56,8 @@ protected function execute(InputInterface $input, OutputInterface $output): int return Command::FAILURE; } - $reader = new Reader(); - $parquetFile = $reader->read($filePath); - try { + $parquetFile = (new Reader())->read($filePath); $parquetFile->metadata(); } catch (InvalidArgumentException) { $style->error(sprintf('File "%s" is not a valid parquet file', $filePath)); diff --git a/src/lib/parquet-viewer/src/Flow/ParquetViewer/Command/ReadMetadataCommand.php b/src/lib/parquet-viewer/src/Flow/ParquetViewer/Command/ReadMetadataCommand.php index 8d59450601..83b8eb2290 100644 --- a/src/lib/parquet-viewer/src/Flow/ParquetViewer/Command/ReadMetadataCommand.php +++ b/src/lib/parquet-viewer/src/Flow/ParquetViewer/Command/ReadMetadataCommand.php @@ -62,10 +62,8 @@ protected function execute(InputInterface $input, OutputInterface $output): int $displayColumnChunks = type_boolean()->cast($input->getOption('column-chunks')); $displayPageHeaders = type_boolean()->cast($input->getOption('page-headers')); - $reader = new Reader(); - $parquetFile = $reader->read($filePath); - try { + $parquetFile = (new Reader())->read($filePath); $metadata = $parquetFile->metadata(); } catch (InvalidArgumentException $_e) { $style->error(sprintf('File "%s" is not a valid parquet file', $filePath)); diff --git a/src/lib/parquet/composer.json b/src/lib/parquet/composer.json index 1b9ac084cc..d81ab334c3 100644 --- a/src/lib/parquet/composer.json +++ b/src/lib/parquet/composer.json @@ -20,8 +20,11 @@ "packaged/thrift": "^0.15.0", "symfony/polyfill-mbstring": "^1.33" }, + "conflict": { + "ext-arrow": "<0.45" + }, "suggest": { - "ext-arrow": "For high-performance Parquet reading/writing via Apache Arrow" + "ext-arrow": "Native Parquet reading and writing (flow-php/arrow-ext)" }, "config": { "optimize-autoloader": true, diff --git a/src/lib/parquet/src/Flow/Parquet/Data/DeltaBinaryPackedDecoder.php b/src/lib/parquet/src/Flow/Parquet/Data/DeltaBinaryPackedDecoder.php index e2d7724587..54bfd95b81 100644 --- a/src/lib/parquet/src/Flow/Parquet/Data/DeltaBinaryPackedDecoder.php +++ b/src/lib/parquet/src/Flow/Parquet/Data/DeltaBinaryPackedDecoder.php @@ -63,20 +63,18 @@ public function decode(string $data, int $valueCount): array return [$header->firstValue]; } - return $this->reconstructValues( - $header->firstValue, - $this->readBlocks($reader, $valueCount - 1, $header->blockSize), - ); + return $this->reconstructValues($header->firstValue, $this->readBlocks($reader, $valueCount - 1, $header)); } /** * @return array */ - private function readBlock(BinaryBufferReader $reader, int $blockDeltaCount): array + private function readBlock(BinaryBufferReader $reader, int $blockDeltaCount, DeltaHeader $header): array { $minDelta = $this->readSignedLEB128($reader); - $miniblockCount = (int) ceil($blockDeltaCount / $this->miniblockSize); + // every block carries the header's count of bit widths, even when its last miniblocks hold no deltas + $miniblockCount = $header->miniblockCount; $bitWidths = []; @@ -97,7 +95,7 @@ private function readBlock(BinaryBufferReader $reader, int $blockDeltaCount): ar if ($bitWidth === null) { throw new RuntimeException('Missing bit width for miniblock index ' . $miniblockIndex); } - $miniblockSize = $this->miniblockSize; + $miniblockSize = intdiv($header->blockSize, $header->miniblockCount); $remainingDeltas = $blockDeltaCount - $deltasRead; $valuesToRead = min($miniblockSize, $remainingDeltas); @@ -150,16 +148,16 @@ static function ($delta) use ($minDelta) { /** * @return array */ - private function readBlocks(BinaryBufferReader $reader, int $deltaCount, int $blockSize): array + private function readBlocks(BinaryBufferReader $reader, int $deltaCount, DeltaHeader $header): array { $deltas = []; $deltasRead = 0; while ($deltasRead < $deltaCount) { $remainingDeltas = $deltaCount - $deltasRead; - $blockDeltaCount = min($blockSize, $remainingDeltas); + $blockDeltaCount = min($header->blockSize, $remainingDeltas); - $blockDeltas = $this->readBlock($reader, $blockDeltaCount); + $blockDeltas = $this->readBlock($reader, $blockDeltaCount, $header); $deltas = array_merge($deltas, $blockDeltas); $deltasRead += count($blockDeltas); } diff --git a/src/lib/parquet/src/Flow/Parquet/Data/DeltaBinaryPackedEncoder.php b/src/lib/parquet/src/Flow/Parquet/Data/DeltaBinaryPackedEncoder.php index 55565b84a2..2301841bba 100644 --- a/src/lib/parquet/src/Flow/Parquet/Data/DeltaBinaryPackedEncoder.php +++ b/src/lib/parquet/src/Flow/Parquet/Data/DeltaBinaryPackedEncoder.php @@ -7,6 +7,7 @@ use Flow\Parquet\BinaryWriter\BinaryBufferWriter; use Flow\Parquet\Exception\InvalidArgumentException; +use function array_pad; use function count; final readonly class DeltaBinaryPackedEncoder @@ -57,8 +58,9 @@ public function encode(array $values): string */ private function packMiniblock(array $values, int $bitWidth): string { + // a zero-width miniblock has no body: its bit width alone says every delta equals the block's minimum if ($bitWidth === 0) { - return str_repeat("\x00", max(0, (int) ceil($this->miniblockSize / 8))); + return ''; } $buffer = ''; @@ -181,7 +183,8 @@ private function writeBlock(BinaryBufferWriter $writer, array $blockDeltas): voi $packedMiniblocks[] = $this->packMiniblock($padded, $bitWidth); } - $writer->writeBytes($bitWidths); + // a partial last block still carries every miniblock's bit width, the unused ones zero, without their bodies + $writer->writeBytes(array_pad($bitWidths, intdiv($this->blockSize, $this->miniblockSize), 0)); foreach ($packedMiniblocks as $packed) { $writer->append($packed); diff --git a/src/lib/parquet/src/Flow/Parquet/Dremel/ColumnData/PagedFlatColumnValues.php b/src/lib/parquet/src/Flow/Parquet/Dremel/ColumnData/PagedFlatColumnValues.php new file mode 100644 index 0000000000..06b56cc210 --- /dev/null +++ b/src/lib/parquet/src/Flow/Parquet/Dremel/ColumnData/PagedFlatColumnValues.php @@ -0,0 +1,50 @@ + $pages + */ + public function __construct( + public FlatColumn $column, + private Generator $pages, + ) {} + + /** + * @return Generator + */ + public function assembleFlat(DataConverter $dataConverter): Generator + { + foreach ($this->pages as $page) { + // @mago-ignore analysis:mixed-assignment + foreach ($page->assembleFlat($dataConverter) as $value) { + yield $value; + } + } + } + + public function flatPath(): string + { + return $this->column->flatPath(); + } + + /** + * @return Generator + */ + public function iterator(): Generator + { + foreach ($this->pages as $page) { + foreach ($page->iterator() as $value) { + yield $value; + } + } + } +} diff --git a/src/lib/parquet/src/Flow/Parquet/Dremel/DremelAssembler.php b/src/lib/parquet/src/Flow/Parquet/Dremel/DremelAssembler.php index 14d94ef915..5a066ce7b5 100644 --- a/src/lib/parquet/src/Flow/Parquet/Dremel/DremelAssembler.php +++ b/src/lib/parquet/src/Flow/Parquet/Dremel/DremelAssembler.php @@ -360,12 +360,15 @@ private function assemblyStructure( } if (!$repeated) { + // a REQUIRED struct adds no definition level: a child null below it leaves the struct present + $definitionLevel = $column->repetitions()->maxDefinitionLevel(); + foreach ($iterator as $iteration) { $structure = []; // @mago-ignore analysis:mixed-assignment foreach ($iteration as $propertyName => $propertyValue) { - if ($propertyValue instanceof NullLevel && $propertyValue->level < $depth) { + if ($propertyValue instanceof NullLevel && $propertyValue->level < $definitionLevel) { yield new NullLevel($propertyValue->level); continue 2; diff --git a/src/lib/parquet/src/Flow/Parquet/Dremel/DremelShredder.php b/src/lib/parquet/src/Flow/Parquet/Dremel/DremelShredder.php index 264d0a9241..57263dd33f 100644 --- a/src/lib/parquet/src/Flow/Parquet/Dremel/DremelShredder.php +++ b/src/lib/parquet/src/Flow/Parquet/Dremel/DremelShredder.php @@ -14,6 +14,7 @@ use Flow\Parquet\ParquetFile\Schema\FlatColumn; use Flow\Parquet\ParquetFile\Schema\NestedColumn; +use function array_fill; use function count; use function gettype; use function is_array; @@ -29,10 +30,36 @@ public function __construct( /** * @param array> $rows + * @param int $firstRow the writer's 0-based index of the first of $rows, which validation refusals name * * @return array keyed by flatPath */ - public function shred(Schema $schema, array $rows): array + public function shred(Schema $schema, array $rows, int $firstRow): array + { + $columns = []; + + foreach ($schema->columns() as $column) { + $name = $column->name(); + $values = []; + + foreach ($rows as $row) { + $values[] = $row[$name] ?? null; + } + + $columns[$name] = $values; + } + + return $this->shredColumns($schema, $columns, count($rows), $firstRow); + } + + /** + * @param array> $columns by top-level column name; a column the array lacks is nulls + * @param int $count rows in every list + * @param int $firstRow the writer's 0-based index of the first row, which validation refusals name + * + * @return array keyed by flatPath + */ + public function shredColumns(Schema $schema, array $columns, int $count, int $firstRow): array { /** @var array $targets */ $targets = []; @@ -60,41 +87,71 @@ public function shred(Schema $schema, array $rows): array } foreach ($plans as $plan) { - foreach ($rows as $row) { - // @mago-ignore analysis:mixed-assignment - $value = $row[$plan->childName] ?? null; + $values = $columns[$plan->childName] ?? []; + $column = $schema->get($plan->childName); - if ($shouldValidate) { - $this->validator->validate($schema->get($plan->childName), $value); - } + if ($plan instanceof FlatPlan) { + $target = $plan->target; + $converter = $plan->converter; + $optional = !$plan->isRequired; + $target->repetitionLevels = $count > 0 ? array_fill(0, $count, 0) : []; - /** @var array $rowFirstWrite */ - $rowFirstWrite = []; - - if ($plan instanceof FlatPlan) { - $target = $plan->target; - $converter = $plan->converter; - - $defLvl = 0; + for ($offset = 0; $offset < $count; $offset++) { + // @mago-ignore analysis:mixed-assignment + $value = $values[$offset] ?? null; - if (!$plan->isRequired && $value !== null) { - $defLvl = 1; + if ($shouldValidate) { + $this->validator->validate($column, $value, $firstRow + $offset); } - $target->repetitionLevels[] = 0; - $target->definitionLevels[] = $defLvl; + $target->definitionLevels[] = $optional && $value !== null ? 1 : 0; if ($value !== null) { $target->values[] = $this->narrowFlatValue( $converter !== null ? $converter->toParquetType($value) : $value, ); } - } elseif ($plan instanceof ListPlan) { - $this->execList($plan, $this->narrowArrayOrNull($value), 0, 0, 0, $shouldValidate, $rowFirstWrite); + } + + continue; + } + + for ($offset = 0; $offset < $count; $offset++) { + $rowIndex = $firstRow + $offset; + // @mago-ignore analysis:mixed-assignment + $value = $values[$offset] ?? null; + + if ($shouldValidate) { + $this->validator->validate($column, $value, $rowIndex); + } + + /** @var array $rowFirstWrite */ + $rowFirstWrite = []; + + if ($plan instanceof ListPlan) { + $this->execList( + $plan, + $this->narrowArrayOrNull($value), + 0, + 0, + 0, + $shouldValidate, + $rowIndex, + $rowFirstWrite, + ); } elseif ($plan instanceof MapPlan) { - $this->execMap($plan, $this->narrowArrayOrNull($value), 0, 0, 0, $shouldValidate, $rowFirstWrite); + $this->execMap( + $plan, + $this->narrowArrayOrNull($value), + 0, + 0, + 0, + $shouldValidate, + $rowIndex, + $rowFirstWrite, + ); } else { - $this->execStruct($plan, $value, 0, 0, 0, $shouldValidate, $rowFirstWrite); + $this->execStruct($plan, $value, 0, 0, 0, $shouldValidate, $rowIndex, $rowFirstWrite); } } } @@ -191,6 +248,7 @@ private function execList( int $repetitionLevel, int $depth, bool $shouldValidate, + int $rowIndex, array &$rowFirstWrite, ): void { $repetitionLevel++; @@ -273,6 +331,7 @@ private function execList( $repetitionLevel - 1, $depth, $shouldValidate, + $rowIndex, $rowFirstWrite, ); @@ -285,7 +344,7 @@ private function execList( if (!count($listValue)) { if ($shouldValidate && $plan->elementColumn !== null) { - $this->validator->validate($plan->elementColumn, null); + $this->validator->validate($plan->elementColumn, null, $rowIndex); } $this->execList( $element, @@ -294,6 +353,7 @@ private function execList( $repetitionLevel - 1, $depth, $shouldValidate, + $rowIndex, $rowFirstWrite, ); @@ -311,6 +371,7 @@ private function execList( $i === 0 ? $repetitionLevel - 1 : $depth, $depth, $shouldValidate, + $rowIndex, $rowFirstWrite, ); } @@ -327,6 +388,7 @@ private function execList( $repetitionLevel - 1, $depth, $shouldValidate, + $rowIndex, $rowFirstWrite, ); @@ -339,7 +401,7 @@ private function execList( if (!count($listValue)) { if ($shouldValidate && $plan->elementColumn !== null) { - $this->validator->validate($plan->elementColumn, null); + $this->validator->validate($plan->elementColumn, null, $rowIndex); } $this->execMap( $element, @@ -348,6 +410,7 @@ private function execList( $repetitionLevel - 1, $depth, $shouldValidate, + $rowIndex, $rowFirstWrite, ); @@ -365,6 +428,7 @@ private function execList( $i === 0 ? $repetitionLevel - 1 : $depth, $depth, $shouldValidate, + $rowIndex, $rowFirstWrite, ); } @@ -380,6 +444,7 @@ private function execList( $repetitionLevel - 1, $depth, $shouldValidate, + $rowIndex, $rowFirstWrite, ); @@ -392,7 +457,7 @@ private function execList( if (!count($listValue)) { if ($shouldValidate && $plan->elementColumn !== null) { - $this->validator->validate($plan->elementColumn, null); + $this->validator->validate($plan->elementColumn, null, $rowIndex); } $this->execStruct( $element, @@ -401,6 +466,7 @@ private function execList( $repetitionLevel - 1, $depth, $shouldValidate, + $rowIndex, $rowFirstWrite, ); @@ -418,6 +484,7 @@ private function execList( $i === 0 ? $repetitionLevel - 1 : $depth, $depth, $shouldValidate, + $rowIndex, $rowFirstWrite, ); } @@ -434,6 +501,7 @@ private function execMap( int $repetitionLevel, int $depth, bool $shouldValidate, + int $rowIndex, array &$rowFirstWrite, ): void { $repetitionLevel++; @@ -631,6 +699,7 @@ private function execMap( $repetitionLevel - 1, $depth, $shouldValidate, + $rowIndex, $rowFirstWrite, ); @@ -643,7 +712,7 @@ private function execMap( if (!count($mapValue)) { if ($shouldValidate && $plan->valueColumn !== null) { - $this->validator->validate($plan->valueColumn, null); + $this->validator->validate($plan->valueColumn, null, $rowIndex); } $repLvl = $repetitionLevel - 1; @@ -662,6 +731,7 @@ private function execMap( $repetitionLevel - 1, $depth, $shouldValidate, + $rowIndex, $rowFirstWrite, ); @@ -704,6 +774,7 @@ private function execMap( $repLevel, $depth, $shouldValidate, + $rowIndex, $rowFirstWrite, ); $index++; @@ -730,6 +801,7 @@ private function execMap( $repetitionLevel - 1, $depth, $shouldValidate, + $rowIndex, $rowFirstWrite, ); @@ -742,7 +814,7 @@ private function execMap( if (!count($mapValue)) { if ($shouldValidate && $plan->valueColumn !== null) { - $this->validator->validate($plan->valueColumn, null); + $this->validator->validate($plan->valueColumn, null, $rowIndex); } $repLvl = $repetitionLevel - 1; @@ -761,6 +833,7 @@ private function execMap( $repetitionLevel - 1, $depth, $shouldValidate, + $rowIndex, $rowFirstWrite, ); @@ -803,6 +876,7 @@ private function execMap( $repLevel, $depth, $shouldValidate, + $rowIndex, $rowFirstWrite, ); $index++; @@ -828,6 +902,7 @@ private function execMap( $repetitionLevel - 1, $depth, $shouldValidate, + $rowIndex, $rowFirstWrite, ); @@ -840,7 +915,7 @@ private function execMap( if (!count($mapValue)) { if ($shouldValidate && $plan->valueColumn !== null) { - $this->validator->validate($plan->valueColumn, null); + $this->validator->validate($plan->valueColumn, null, $rowIndex); } $repLvl = $repetitionLevel - 1; @@ -859,6 +934,7 @@ private function execMap( $repetitionLevel - 1, $depth, $shouldValidate, + $rowIndex, $rowFirstWrite, ); @@ -894,7 +970,16 @@ private function execMap( $keyTarget->values[] = $this->narrowFlatValue( $keyConverter !== null ? $keyConverter->toParquetType($key) : $key, ); - $this->execStruct($valuePlan, $value, $definitionLevel, $repLevel, $depth, $shouldValidate, $rowFirstWrite); + $this->execStruct( + $valuePlan, + $value, + $definitionLevel, + $repLevel, + $depth, + $shouldValidate, + $rowIndex, + $rowFirstWrite, + ); $index++; } } @@ -909,6 +994,7 @@ private function execStruct( int $repetitionLevel, int $depth, bool $shouldValidate, + int $rowIndex, array &$rowFirstWrite, ): void { if ($structureData === null) { @@ -938,6 +1024,7 @@ private function execStruct( $repetitionLevel, $depth, $shouldValidate, + $rowIndex, $rowFirstWrite, ); @@ -952,6 +1039,7 @@ private function execStruct( $repetitionLevel, $depth, $shouldValidate, + $rowIndex, $rowFirstWrite, ); @@ -965,6 +1053,7 @@ private function execStruct( $repetitionLevel, $depth, $shouldValidate, + $rowIndex, $rowFirstWrite, ); } @@ -1003,6 +1092,7 @@ private function execStruct( $repetitionLevel, $depth, $shouldValidate, + $rowIndex, $rowFirstWrite, ); @@ -1017,6 +1107,7 @@ private function execStruct( $repetitionLevel, $depth, $shouldValidate, + $rowIndex, $rowFirstWrite, ); @@ -1030,6 +1121,7 @@ private function execStruct( $repetitionLevel, $depth, $shouldValidate, + $rowIndex, $rowFirstWrite, ); } @@ -1077,6 +1169,7 @@ private function execStruct( $repetitionLevel, $depth, $shouldValidate, + $rowIndex, $rowFirstWrite, ); @@ -1091,6 +1184,7 @@ private function execStruct( $repetitionLevel, $depth, $shouldValidate, + $rowIndex, $rowFirstWrite, ); @@ -1104,6 +1198,7 @@ private function execStruct( $repetitionLevel, $depth, $shouldValidate, + $rowIndex, $rowFirstWrite, ); } diff --git a/src/lib/parquet/src/Flow/Parquet/Dremel/ReadColumnData.php b/src/lib/parquet/src/Flow/Parquet/Dremel/ReadColumnData.php index 46caffebc1..c609df1414 100644 --- a/src/lib/parquet/src/Flow/Parquet/Dremel/ReadColumnData.php +++ b/src/lib/parquet/src/Flow/Parquet/Dremel/ReadColumnData.php @@ -5,6 +5,7 @@ namespace Flow\Parquet\Dremel; use Flow\Parquet\Dremel\ColumnData\FlatValue; +use Flow\Parquet\Dremel\ColumnData\PagedFlatColumnValues; use Flow\Parquet\Dremel\ColumnData\ReadFlatColumnValues; use Flow\Parquet\Exception\InvalidArgumentException; use Flow\Parquet\ParquetFile\Schema\Column; @@ -19,13 +20,13 @@ public Column $column; /** - * @var array + * @var array */ public array $flatValues; /** * @param Column $column - * @param array $flatValues + * @param array $flatValues */ public function __construct(Column $column, array $flatValues = []) { diff --git a/src/lib/parquet/src/Flow/Parquet/Dremel/Validator.php b/src/lib/parquet/src/Flow/Parquet/Dremel/Validator.php index 62737d9fb9..3ce76847aa 100644 --- a/src/lib/parquet/src/Flow/Parquet/Dremel/Validator.php +++ b/src/lib/parquet/src/Flow/Parquet/Dremel/Validator.php @@ -10,7 +10,9 @@ interface Validator { /** + * @param int $row the 0-based index of the row within the writer + * * @throws ValidationException */ - public function validate(Column $column, mixed $data): void; + public function validate(Column $column, mixed $data, int $row): void; } diff --git a/src/lib/parquet/src/Flow/Parquet/Dremel/Validator/ColumnDataValidator.php b/src/lib/parquet/src/Flow/Parquet/Dremel/Validator/ColumnDataValidator.php index 9d992fbfe3..3832480490 100644 --- a/src/lib/parquet/src/Flow/Parquet/Dremel/Validator/ColumnDataValidator.php +++ b/src/lib/parquet/src/Flow/Parquet/Dremel/Validator/ColumnDataValidator.php @@ -11,21 +11,33 @@ use Flow\Parquet\ParquetFile\Schema\Column; use Flow\Parquet\ParquetFile\Schema\FlatColumn; use Flow\Parquet\ParquetFile\Schema\LogicalType; +use Flow\Parquet\ParquetFile\Schema\LogicalType\Integer; use Flow\Parquet\ParquetFile\Schema\NestedColumn; use Flow\Parquet\ParquetFile\Schema\PhysicalType; use Flow\Parquet\ParquetFile\Schema\Repetition; +use Stringable; +use function ctype_xdigit; +use function get_debug_type; use function gettype; use function is_array; use function is_bool; use function is_float; use function is_int; +use function is_object; use function is_string; +use function mb_check_encoding; +use function method_exists; use function sprintf; +use function str_replace; +use function strlen; + +use const PHP_INT_MAX; +use const PHP_INT_MIN; final class ColumnDataValidator implements Validator { - public function validate(Column $column, mixed $data): void + public function validate(Column $column, mixed $data, int $row): void { $repetition = $column->repetition(); @@ -50,7 +62,7 @@ public function validate(Column $column, mixed $data): void } if ($column instanceof FlatColumn) { - $this->validateData($column, $data, $repetition); + $this->validateData($column, $data, $repetition, $row); return; } @@ -69,7 +81,7 @@ public function validate(Column $column, mixed $data): void // @mago-ignore analysis:mixed-assignment foreach ($data as $value) { - $this->validate($column->getListElement(), $value); + $this->validate($column->getListElement(), $value, $row); } return; @@ -88,10 +100,10 @@ public function validate(Column $column, mixed $data): void // @mago-ignore analysis:mixed-assignment foreach ($data as $key => $value) { - $this->validate($column->getMapKeyColumn(), $key); + $this->validate($column->getMapKeyColumn(), $key, $row); if ($valueColumn !== null) { - $this->validate($valueColumn, $value); + $this->validate($valueColumn, $value, $row); } } @@ -107,16 +119,16 @@ public function validate(Column $column, mixed $data): void } foreach ($column->children() as $child) { - $this->validate($child, $data[$child->name()] ?? null); + $this->validate($child, $data[$child->name()] ?? null, $row); } } - private function validateData(FlatColumn $column, mixed $data, ?Repetition $repetition): void + private function validateData(FlatColumn $column, mixed $data, ?Repetition $repetition, int $row): void { if (is_array($data)) { // @mago-ignore analysis:mixed-assignment foreach ($data as $value) { - $this->validateData($column, $value, $repetition); + $this->validateData($column, $value, $repetition, $row); } return; @@ -161,7 +173,7 @@ private function validateData(FlatColumn $column, mixed $data, ?Repetition $repe ? null : throw new ValidationException(sprintf('Column "%s" is not float', $column->flatPath())), PhysicalType::BYTE_ARRAY => match ($logicalTypeName) { - LogicalType::STRING, LogicalType::JSON, LogicalType::UUID => is_string($data) + LogicalType::STRING, LogicalType::JSON, LogicalType::ENUM, LogicalType::UUID => is_string($data) ? null : throw new ValidationException(sprintf( 'Column "%s" is not string, got "%s" instead', @@ -173,5 +185,91 @@ private function validateData(FlatColumn $column, mixed $data, ?Repetition $repe PhysicalType::FIXED_LEN_BYTE_ARRAY => null, default => throw new ValidationException(sprintf('Unknown column type "%s"', $type->name)), }; + + if (is_int($data) && ($type === PhysicalType::INT32 || $type === PhysicalType::INT64)) { + $integer = Integer::forColumn($column); + [$name, $min, $max] = match (true) { + $integer === null && $type === PhysicalType::INT32 && $column->logicalType() === null => [ + 'INT32', + -2_147_483_648, + 2_147_483_647, + ], + $integer === null => ['INT64', PHP_INT_MIN, PHP_INT_MAX], + $integer->isSigned() => [ + 'INT_' . $integer->bitWidth(), + $integer->bitWidth() === 64 ? PHP_INT_MIN : -(1 << ($integer->bitWidth() - 1)), + $integer->bitWidth() === 64 ? PHP_INT_MAX : (1 << ($integer->bitWidth() - 1)) - 1, + ], + default => [ + 'UINT_' . $integer->bitWidth(), + 0, + $integer->bitWidth() === 64 ? PHP_INT_MAX : (1 << $integer->bitWidth()) - 1, + ], + }; + + if ($data < $min || $data > $max) { + throw new ValidationException(sprintf( + 'Column "%s" row %d: %d is out of the %s range [%d, %d]', + $column->flatPath(), + $row, + $data, + $name, + $min, + $max, + )); + } + } + + if ($logicalTypeName === LogicalType::UUID) { + $hex = match (true) { + is_string($data) => str_replace('-', '', $data), + is_object($data) && method_exists($data, 'toString') => is_string($data->toString()) + ? str_replace('-', '', (string) $data->toString()) + : '', + $data instanceof Stringable => str_replace('-', '', (string) $data), + default => '', + }; + + if (strlen($hex) !== 32 || !ctype_xdigit($hex)) { + throw new ValidationException(sprintf( + 'Column "%s" row %d: %s is not a UUID of 32 hexadecimal digits', + $column->flatPath(), + $row, + get_debug_type($data), + )); + } + } + + if ( + $type === PhysicalType::FIXED_LEN_BYTE_ARRAY + && $logicalTypeName !== LogicalType::DECIMAL + && $logicalTypeName !== LogicalType::UUID + && is_string($data) + && strlen($data) !== $column->typeLength() + ) { + throw new ValidationException(sprintf( + 'Column "%s" row %d: a FIXED_LEN_BYTE_ARRAY(%d) value is %d bytes long', + $column->flatPath(), + $row, + (int) $column->typeLength(), + strlen($data), + )); + } + + if ( + ( + $logicalTypeName === LogicalType::STRING + || $logicalTypeName === LogicalType::JSON + || $logicalTypeName === LogicalType::ENUM + ) + && is_string($data) + && !mb_check_encoding($data, 'UTF-8') + ) { + throw new ValidationException(sprintf( + 'Column "%s" row %d: the string is not valid UTF-8', + $column->flatPath(), + $row, + )); + } } } diff --git a/src/lib/parquet/src/Flow/Parquet/Dremel/Validator/DisabledValidator.php b/src/lib/parquet/src/Flow/Parquet/Dremel/Validator/DisabledValidator.php index 0da80bc6a1..566c249634 100644 --- a/src/lib/parquet/src/Flow/Parquet/Dremel/Validator/DisabledValidator.php +++ b/src/lib/parquet/src/Flow/Parquet/Dremel/Validator/DisabledValidator.php @@ -9,5 +9,5 @@ final class DisabledValidator implements Validator { - public function validate(Column $column, mixed $data): void {} + public function validate(Column $column, mixed $data, int $row): void {} } diff --git a/src/lib/parquet/src/Flow/Parquet/Engine/AdaptiveParquetEngine.php b/src/lib/parquet/src/Flow/Parquet/Engine/AdaptiveParquetEngine.php index 5ddf84a895..d9477610ac 100644 --- a/src/lib/parquet/src/Flow/Parquet/Engine/AdaptiveParquetEngine.php +++ b/src/lib/parquet/src/Flow/Parquet/Engine/AdaptiveParquetEngine.php @@ -11,20 +11,26 @@ use Flow\Parquet\ParquetEngine; use Flow\Parquet\ParquetFile\Compressions; use Flow\Parquet\ParquetFile\Schema; +use Flow\Parquet\ParquetFileReader; use Flow\Parquet\ParquetFileWriter; -use Generator; - -use function extension_loaded; final readonly class AdaptiveParquetEngine implements ParquetEngine { - private ParquetEngine $delegate; + private ParquetEngine $engine; + + public function __construct( + ByteOrder $byteOrder = ByteOrder::LITTLE_ENDIAN, + Options $options = new Options(), + ?ArrowExtension $extension = null, + ) { + $this->engine = ($extension ?? ArrowExtension::detect())->available() && $byteOrder === ByteOrder::LITTLE_ENDIAN + ? new RustParquetEngine($options) + : new PhpParquetEngine($byteOrder, $options); + } - public function __construct(ByteOrder $byteOrder = ByteOrder::LITTLE_ENDIAN, Options $options = new Options()) + public function openForRead(SourceStream $stream): ParquetFileReader { - $this->delegate = extension_loaded('arrow') - ? new ArrowParquetEngine($options) - : new PhpParquetEngine($byteOrder, $options); + return $this->engine->openForRead($stream); } public function openForWrite( @@ -33,17 +39,7 @@ public function openForWrite( Compressions $compression, Options $options, ): ParquetFileWriter { - return $this->delegate->openForWrite($stream, $schema, $compression, $options); - } - - public function readValues( - SourceStream $stream, - Schema $schema, - array $columns = [], - ?int $limit = null, - ?int $offset = null, - ): Generator { - return $this->delegate->readValues($stream, $schema, $columns, $limit, $offset); + return $this->engine->openForWrite($stream, $schema, $compression, $options); } public function writeRows( @@ -53,6 +49,6 @@ public function writeRows( Options $options, iterable $rows, ): void { - $this->delegate->writeRows($stream, $schema, $compression, $options, $rows); + $this->engine->writeRows($stream, $schema, $compression, $options, $rows); } } diff --git a/src/lib/parquet/src/Flow/Parquet/Engine/Arrow/DestinationStreamAdapter.php b/src/lib/parquet/src/Flow/Parquet/Engine/Arrow/DestinationStreamAdapter.php deleted file mode 100644 index 613e7c8e18..0000000000 --- a/src/lib/parquet/src/Flow/Parquet/Engine/Arrow/DestinationStreamAdapter.php +++ /dev/null @@ -1,22 +0,0 @@ -stream->append($data); - - return $this; - } -} diff --git a/src/lib/parquet/src/Flow/Parquet/Engine/Arrow/OptionsConverter.php b/src/lib/parquet/src/Flow/Parquet/Engine/Arrow/OptionsConverter.php index d4a464721c..7316773cdc 100644 --- a/src/lib/parquet/src/Flow/Parquet/Engine/Arrow/OptionsConverter.php +++ b/src/lib/parquet/src/Flow/Parquet/Engine/Arrow/OptionsConverter.php @@ -11,6 +11,23 @@ final class OptionsConverter { + // called by name from arrow-ext RustParquetEngine::__construct(), pinned by phpts 047, 052 + /** + * The options the arrow engine itself reads, which Rust cannot read off the Option enum. + * + * @return array{INT_96_AS_DATETIME: bool, ARROW_WRITE_BATCH_SIZE: int} + */ + public static function toEngine(?Options $options): array + { + $options ??= new Options(); + + return [ + 'INT_96_AS_DATETIME' => $options->getBool(Option::INT_96_AS_DATETIME), + 'ARROW_WRITE_BATCH_SIZE' => $options->getInt(Option::ARROW_WRITE_BATCH_SIZE), + ]; + } + + // called by name from arrow-ext RustParquetEngine::openForWrite(), pinned by phpts 047, 052 /** * @return array */ diff --git a/src/lib/parquet/src/Flow/Parquet/Engine/Arrow/SchemaConverter.php b/src/lib/parquet/src/Flow/Parquet/Engine/Arrow/SchemaConverter.php index f68b013b6a..2004d1a185 100644 --- a/src/lib/parquet/src/Flow/Parquet/Engine/Arrow/SchemaConverter.php +++ b/src/lib/parquet/src/Flow/Parquet/Engine/Arrow/SchemaConverter.php @@ -10,12 +10,14 @@ use Flow\Parquet\ParquetFile\Schema\ConvertedType; use Flow\Parquet\ParquetFile\Schema\FlatColumn; use Flow\Parquet\ParquetFile\Schema\LogicalType; +use Flow\Parquet\ParquetFile\Schema\LogicalType\Integer; use Flow\Parquet\ParquetFile\Schema\NestedColumn; use Flow\Parquet\ParquetFile\Schema\PhysicalType; use Flow\Parquet\ParquetFile\Schema\Repetition; final class SchemaConverter { + // called by name from arrow-ext RustParquetEngine::openForWrite(), pinned by phpts 047, 052 /** * @return array> */ @@ -157,14 +159,15 @@ private static function resolveFromConvertedOrPhysical(FlatColumn $column): stri ConvertedType::JSON => 'JSON', ConvertedType::ENUM => 'STRING', ConvertedType::BSON => 'BINARY', - ConvertedType::INT_8 => 'INT8', - ConvertedType::INT_16 => 'INT16', - ConvertedType::INT_32 => 'INT32', - ConvertedType::INT_64 => 'INT64', - ConvertedType::UINT_8 => 'UINT8', - ConvertedType::UINT_16 => 'UINT16', - ConvertedType::UINT_32 => 'UINT32', - ConvertedType::UINT_64 => 'UINT64', + ConvertedType::INT_8, + ConvertedType::INT_16, + ConvertedType::INT_32, + ConvertedType::INT_64, + ConvertedType::UINT_8, + ConvertedType::UINT_16, + ConvertedType::UINT_32, + ConvertedType::UINT_64, + => self::resolveIntegerType($column), default => self::resolveFromPhysicalType($column), }; } @@ -188,20 +191,12 @@ private static function resolveFromPhysicalType(FlatColumn $column): string private static function resolveIntegerType(FlatColumn $column): string { - return match ($column->convertedType()) { - ConvertedType::INT_8 => 'INT8', - ConvertedType::INT_16 => 'INT16', - ConvertedType::INT_32 => 'INT32', - ConvertedType::INT_64 => 'INT64', - ConvertedType::UINT_8 => 'UINT8', - ConvertedType::UINT_16 => 'UINT16', - ConvertedType::UINT_32 => 'UINT32', - ConvertedType::UINT_64 => 'UINT64', - default => match ($column->type()) { - PhysicalType::INT32 => 'INT32', - PhysicalType::INT64 => 'INT64', - default => 'INT64', - }, - }; + $integer = Integer::forColumn($column); + + if ($integer === null) { + return self::resolveFromPhysicalType($column); + } + + return ($integer->isSigned() ? 'INT' : 'UINT') . $integer->bitWidth(); } } diff --git a/src/lib/parquet/src/Flow/Parquet/Engine/Arrow/SourceStreamAdapter.php b/src/lib/parquet/src/Flow/Parquet/Engine/Arrow/SourceStreamAdapter.php deleted file mode 100644 index a10d1301b1..0000000000 --- a/src/lib/parquet/src/Flow/Parquet/Engine/Arrow/SourceStreamAdapter.php +++ /dev/null @@ -1,26 +0,0 @@ - $length */ - return $this->stream->read($length, $offset); - } - - public function size(): ?int - { - return $this->stream->size(); - } -} diff --git a/src/lib/parquet/src/Flow/Parquet/Engine/ArrowExtension.php b/src/lib/parquet/src/Flow/Parquet/Engine/ArrowExtension.php new file mode 100644 index 0000000000..bf9ed403e4 --- /dev/null +++ b/src/lib/parquet/src/Flow/Parquet/Engine/ArrowExtension.php @@ -0,0 +1,63 @@ +loaded) { + return false; + } + + if ($this->abi !== self::ABI) { + throw new RuntimeException(sprintf( + 'The loaded arrow extension (%s, ABI %s) does not match flow-php/parquet (ABI %d). Rebuild or upgrade ' + . 'arrow-ext to the version of flow-php/parquet, or stop loading it to use the PHP implementation.', + $this->version, + $this->abi ?? 'none', + self::ABI, + )); + } + + return true; + } +} diff --git a/src/lib/parquet/src/Flow/Parquet/Engine/ArrowParquetEngine.php b/src/lib/parquet/src/Flow/Parquet/Engine/ArrowParquetEngine.php deleted file mode 100644 index 404945ec1a..0000000000 --- a/src/lib/parquet/src/Flow/Parquet/Engine/ArrowParquetEngine.php +++ /dev/null @@ -1,150 +0,0 @@ - 'UNCOMPRESSED', - Compressions::SNAPPY => 'SNAPPY', - Compressions::GZIP => 'GZIP', - Compressions::BROTLI => 'BROTLI', - Compressions::LZ4, Compressions::LZ4_RAW => 'LZ4_RAW', - Compressions::ZSTD => 'ZSTD', - Compressions::LZO => throw new RuntimeException('LZO compression is not supported by the Arrow engine'), - }; - } - - public function openForWrite( - DestinationStream $stream, - Schema $schema, - Compressions $compression, - Options $options, - ): ParquetFileWriter { - $extensionSchema = SchemaConverter::toExtension($schema); - - /** @var list $columnNames */ - $columnNames = array_column($extensionSchema, 'name'); - // the arrow Writer takes the stream by reference, so it has to be a variable - $adapter = new DestinationStreamAdapter($stream); - - return new ArrowParquetFileWriter( - new Writer( - $adapter, - $extensionSchema, - self::mapCompression($compression), - OptionsConverter::toExtension($options), - ), - $stream, - $columnNames, - // write batching is an engine option, not a per-file one - $this->options->getInt(Option::ARROW_WRITE_BATCH_SIZE), - ); - } - - /** - * @param array $columns - * - * @return \Generator> - */ - public function readValues( - SourceStream $stream, - Schema $schema, - array $columns = [], - ?int $limit = null, - ?int $offset = null, - ): Generator { - $adapter = new SourceStreamAdapter($stream); - $extensionOptions = OptionsConverter::toExtension($this->options); - $reader = new Reader($adapter, $extensionOptions); - - try { - $skipped = 0; - $yielded = 0; - - while (null !== ($batch = $reader->readRowGroup($columns ?: null))) { - $colNames = array_keys($batch); - - if ($colNames === []) { - continue; - } - - $rowCount = count($batch[$colNames[0]]); - - for ($i = 0; $i < $rowCount; $i++) { - if ($offset !== null && $skipped < $offset) { - $skipped++; - - continue; - } - - if ($limit !== null && $yielded >= $limit) { - return; - } - - $row = []; - - foreach ($colNames as $name) { - $row[$name] = $batch[$name][$i]; - } - - yield $row; - $yielded++; - } - } - } finally { - $reader->close(); - } - } - - public function writeRows( - DestinationStream $stream, - Schema $schema, - Compressions $compression, - Options $options, - iterable $rows, - ): void { - $file = $this->openForWrite($stream, $schema, $compression, $options); - - try { - $file->writeBatch($rows); - } finally { - $file->close(); - } - } -} diff --git a/src/lib/parquet/src/Flow/Parquet/Engine/ArrowParquetFileWriter.php b/src/lib/parquet/src/Flow/Parquet/Engine/ArrowParquetFileWriter.php deleted file mode 100644 index c3e3fb35dc..0000000000 --- a/src/lib/parquet/src/Flow/Parquet/Engine/ArrowParquetFileWriter.php +++ /dev/null @@ -1,77 +0,0 @@ -> - */ - private array $batch; - - private int $batchSize = 0; - - private ?Writer $arrowWriter; - - /** - * @param list $columnNames - */ - public function __construct( - Writer $arrowWriter, - private readonly DestinationStream $stream, - private readonly array $columnNames, - private readonly int $batchSizeLimit, - ) { - $this->arrowWriter = $arrowWriter; - $this->batch = array_fill_keys($this->columnNames, []); - } - - public function close(): void - { - $writer = $this->arrowWriter ?? throw new RuntimeException('Writer is not open'); - - try { - if ($this->batchSize > 0) { - $writer->writeBatch($this->batch); - } - - $writer->close(); - $this->stream->close(); - } finally { - $this->arrowWriter = null; - } - } - - public function writeBatch(iterable $rows): void - { - foreach ($rows as $row) { - $this->writeRow($row); - } - } - - public function writeRow(array $row): void - { - $writer = $this->arrowWriter ?? throw new RuntimeException('Writer is not open'); - - foreach ($this->columnNames as $name) { - $this->batch[$name][] = $row[$name] ?? null; - } - - $this->batchSize++; - - if ($this->batchSize >= $this->batchSizeLimit) { - $writer->writeBatch($this->batch); - $this->batch = array_fill_keys($this->columnNames, []); - $this->batchSize = 0; - } - } -} diff --git a/src/lib/parquet/src/Flow/Parquet/Engine/PhpParquetEngine.php b/src/lib/parquet/src/Flow/Parquet/Engine/PhpParquetEngine.php index 9d0d2eb58b..76823a4028 100644 --- a/src/lib/parquet/src/Flow/Parquet/Engine/PhpParquetEngine.php +++ b/src/lib/parquet/src/Flow/Parquet/Engine/PhpParquetEngine.php @@ -7,36 +7,13 @@ use Flow\Filesystem\DestinationStream; use Flow\Filesystem\SourceStream; use Flow\Parquet\Binary\ByteOrder; -use Flow\Parquet\Dremel\ColumnData\ReadFlatColumnValues; -use Flow\Parquet\Dremel\DremelAssembler; -use Flow\Parquet\Dremel\ReadColumnData; -use Flow\Parquet\Exception\InvalidArgumentException; use Flow\Parquet\Option; use Flow\Parquet\Options; use Flow\Parquet\ParquetEngine; -use Flow\Parquet\ParquetFile; use Flow\Parquet\ParquetFile\Compressions; -use Flow\Parquet\ParquetFile\Data\DataConverter; -use Flow\Parquet\ParquetFile\Metadata; use Flow\Parquet\ParquetFile\Schema; -use Flow\Parquet\ParquetFile\Schema\Column; -use Flow\Parquet\ParquetFile\Schema\FlatColumn; -use Flow\Parquet\ParquetFile\Schema\NestedColumn; +use Flow\Parquet\ParquetFileReader; use Flow\Parquet\ParquetFileWriter; -use Flow\Parquet\Reader\ColumnChunkReader; -use Flow\Parquet\Reader\PageReader; -use Flow\Parquet\Thrift\CompactProtocol; -use Flow\Parquet\Thrift\MemoryBuffer; -use Flow\Parquet\ThriftModel\FileMetaData; -use Generator; -use MultipleIterator; - -use function array_map; -use function array_push; -use function count; -use function is_array; -use function iterator_to_array; -use function unpack; final class PhpParquetEngine implements ParquetEngine { @@ -45,6 +22,11 @@ public function __construct( private readonly Options $options = new Options(), ) {} + public function openForRead(SourceStream $stream): ParquetFileReader + { + return new PhpParquetFileReader($stream, $this->byteOrder, $this->options); + } + public function openForWrite( DestinationStream $stream, Schema $schema, @@ -60,85 +42,6 @@ public function openForWrite( ); } - public function readValues( - SourceStream $stream, - Schema $schema, - array $columns = [], - ?int $limit = null, - ?int $offset = null, - ): Generator { - $dataConverter = DataConverter::initialize($this->options); - $dremelAssembler = new DremelAssembler($dataConverter); - $chunkReader = new ColumnChunkReader(new PageReader($this->byteOrder, $this->options), $this->options); - - $metadata = $this->readMetadata($stream); - - if (!count($columns)) { - $columns = array_map(static fn(Column $c) => $c->name(), $schema->columns()); - } - - $totalRows = $metadata->rowsNumber(); - - if ($offset !== null && $offset > $totalRows) { - return; - } - - if ($offset !== null) { - if ($totalRows > $offset) { - $totalRows -= $offset; - } else { - $totalRows = 0; - } - } - - $totalRows = min($totalRows, $limit ?? $totalRows); - - if ($totalRows === 0) { - return; - } - - $multipleIterator = new MultipleIterator(MultipleIterator::MIT_KEYS_ASSOC); - - foreach ($columns as $columnName) { - $multipleIterator->attachIterator( - $this->readColumn( - $schema->get($columnName), - $metadata, - $stream, - $chunkReader, - $dremelAssembler, - $limit, - $offset, - ), - $columnName, - ); - } - - $rowCount = 0; - - foreach ($multipleIterator as $rowData) { - if ($limit !== null && $rowCount >= $limit) { - break; - } - - $row = []; - - // Row payload assembled from per-column generators; values are user data (mixed). - // @mago-ignore analysis:mixed-assignment - foreach ($rowData as $columnData) { - if (is_array($columnData)) { - // @mago-ignore analysis:mixed-assignment - foreach ($columnData as $key => $value) { - $row[(string) $key] = $value; - } - } - } - - yield $row; - $rowCount++; - } - } - public function writeRows( DestinationStream $stream, Schema $schema, @@ -161,135 +64,4 @@ public function writeRows( $file->close(); } - - private function readColumn( - Column $column, - Metadata $metadata, - SourceStream $stream, - ColumnChunkReader $chunkReader, - DremelAssembler $dremelAssembler, - ?int $limit, - ?int $offset, - ): Generator { - $yieldedRows = 0; - $rowGroupOffset = 0; - - foreach ($metadata->rowGroups()->all() as $rowGroup) { - if ($offset !== null) { - if (($rowGroupOffset + $rowGroup->rowsCount()) <= $offset) { - $rowGroupOffset += $rowGroup->rowsCount(); - - continue; - } - } - $skipRows = $offset !== null ? $offset - $rowGroupOffset : 0; - - if ($column instanceof FlatColumn) { - $rowsSkipped = 0; - - foreach ($chunkReader->read( - $rowGroup->getColumnChunk($column), - $column, - $stream, - ) as $flatColumnValues) { - $columnData = new ReadColumnData($column, [$flatColumnValues->flatPath() => $flatColumnValues]); - - // @mago-ignore analysis:mixed-assignment - foreach ($dremelAssembler->assemble($column, $columnData) as $row) { - if ($skipRows > 0 && $rowsSkipped < $skipRows) { - $rowsSkipped++; - - continue; - } - - if ($limit !== null && $yieldedRows >= $limit) { - return; - } - yield $row; - $yieldedRows++; - } - } - } elseif ($column instanceof NestedColumn) { - $mergedFlatData = []; - - foreach ($column->childrenFlat() as $child) { - $pages = iterator_to_array($chunkReader->read($rowGroup->getColumnChunk($child), $child, $stream)); - - $allRepetitionLevels = []; - $allDefinitionLevels = []; - - foreach ($pages as $page) { - array_push($allRepetitionLevels, ...$page->repetitionLevels()); - array_push($allDefinitionLevels, ...$page->definitionLevels()); - } - - $mergedFlatData[] = new ReadFlatColumnValues( - $child, - (static function () use ($pages) { - foreach ($pages as $page) { - yield from $page->rawValues(); - } - })(), - $allRepetitionLevels, - $allDefinitionLevels, - ); - } - - $columnData = new ReadColumnData($column, $mergedFlatData); - - $rowsSkipped = 0; - - // @mago-ignore analysis:mixed-assignment - foreach ($dremelAssembler->assemble($column, $columnData) as $row) { - if ($skipRows > 0 && $rowsSkipped < $skipRows) { - $rowsSkipped++; - - continue; - } - - if ($limit !== null && $yieldedRows >= $limit) { - return; - } - yield $row; - $yieldedRows++; - } - } else { - throw new InvalidArgumentException('Column must be instance of FlatColumn or NestedColumn'); - } - - $rowGroupOffset += $rowGroup->rowsCount(); - } - } - - private function readMetadata(SourceStream $stream): Metadata - { - $fileTotalSize = $stream->size(); - - if ($fileTotalSize === null) { - throw new InvalidArgumentException('Cannot determine Parquet file size'); - } - - if ($stream->read(4, $fileTotalSize - 4) !== ParquetFile::PARQUET_MAGIC_NUMBER) { - throw new InvalidArgumentException('Given file is not valid Parquet file'); - } - - $unpacked = unpack($this->byteOrder->value, $stream->read(4, $fileTotalSize - 8)); - - if ($unpacked === false) { - throw new InvalidArgumentException('Failed to read Parquet metadata length'); - } - - $metadataLength = $unpacked[1]; - - if ($metadataLength <= 0) { - throw new InvalidArgumentException('Parquet metadata length must be positive, got ' . $metadataLength); - } - - $metadata = $stream->read($metadataLength, $fileTotalSize - ($metadataLength + 8)); - - $thriftMetadata = new FileMetaData(); - $thriftMetadata->read(new CompactProtocol(new MemoryBuffer($metadata))); - - return Metadata::fromThrift($thriftMetadata); - } } diff --git a/src/lib/parquet/src/Flow/Parquet/Engine/PhpParquetFileReader.php b/src/lib/parquet/src/Flow/Parquet/Engine/PhpParquetFileReader.php new file mode 100644 index 0000000000..76d0f74c38 --- /dev/null +++ b/src/lib/parquet/src/Flow/Parquet/Engine/PhpParquetFileReader.php @@ -0,0 +1,197 @@ +open || throw new RuntimeException('Reader is not open'); + + $this->open = false; + $this->stream->close(); + } + + public function metadata(): Metadata + { + $this->open || throw new RuntimeException('Reader is not open'); + + if ($this->metadata !== null) { + return $this->metadata; + } + + $fileTotalSize = $this->stream->size(); + + if ($fileTotalSize === null) { + throw new InvalidArgumentException('Cannot determine Parquet file size'); + } + + if ($this->stream->read(4, $fileTotalSize - 4) !== ParquetFile::PARQUET_MAGIC_NUMBER) { + throw new InvalidArgumentException('Given file is not valid Parquet file'); + } + + $unpacked = unpack($this->byteOrder->value, $this->stream->read(4, $fileTotalSize - 8)); + + if ($unpacked === false) { + throw new InvalidArgumentException('Failed to read Parquet metadata length'); + } + + $metadataLength = $unpacked[1]; + + if ($metadataLength <= 0) { + throw new InvalidArgumentException('Parquet metadata length must be positive, got ' . $metadataLength); + } + + $metadata = $this->stream->read($metadataLength, $fileTotalSize - ($metadataLength + 8)); + + $thriftMetadata = new FileMetaData(); + $thriftMetadata->read(new CompactProtocol(new MemoryBuffer($metadata))); + + $this->metadata = Metadata::fromThrift($thriftMetadata); + + return $this->metadata; + } + + /** + * @param list $columns + * @param int<1, max> $batchSize + * + * @return Generator>> + */ + public function readColumns(array $columns, int $batchSize, ?int $limit, ?int $offset): Generator + { + $this->open || throw new RuntimeException('Reader is not open'); + + $columnReader = new ColumnReader( + new ColumnChunkReader(new PageReader($this->byteOrder, $this->options), $this->options), + new DremelAssembler(DataConverter::initialize($this->options)), + ); + + $metadata = $this->metadata(); + + $totalRows = $metadata->rowsNumber(); + + if ($offset !== null && $offset > $totalRows) { + return; + } + + if ($offset !== null) { + if ($totalRows > $offset) { + $totalRows -= $offset; + } else { + $totalRows = 0; + } + } + + $totalRows = min($totalRows, $limit ?? $totalRows); + + if ($totalRows === 0) { + return; + } + + $generators = []; + + foreach ($columns as $columnName) { + $generators[$columnName] = $columnReader->read( + $metadata->schema()->get($columnName), + $metadata, + $this->stream, + $limit, + $offset, + ); + } + + $remaining = $totalRows; + + while ($remaining > 0) { + $take = min($batchSize, $remaining); + $chunk = []; + $read = 0; + + foreach ($generators as $name => $generator) { + $values = []; + + while (count($values) < $take && $generator->valid()) { + // @mago-ignore analysis:mixed-assignment + $entry = $generator->current(); + + if (is_array($entry)) { + // @mago-ignore analysis:mixed-assignment + foreach ($entry as $value) { + $values[] = $value; + } + } + + $generator->next(); + } + + $chunk[$name] = $values; + $read = count($values); + } + + if ($read === 0) { + return; + } + + yield $chunk; + $remaining -= $read; + } + } + + public function rowsNumber(): int + { + return $this->metadata()->rowsNumber(); + } + + public function schema(): Schema + { + return $this->metadata()->schema(); + } + + public function totalByteSize(): int + { + $bytes = 0; + + foreach ($this->metadata()->rowGroups()->all() as $rowGroup) { + $bytes += $rowGroup->totalByteSize(); + } + + return $bytes; + } +} diff --git a/src/lib/parquet/src/Flow/Parquet/Engine/PhpParquetFileWriter.php b/src/lib/parquet/src/Flow/Parquet/Engine/PhpParquetFileWriter.php index 88ecc82f02..5dab604466 100644 --- a/src/lib/parquet/src/Flow/Parquet/Engine/PhpParquetFileWriter.php +++ b/src/lib/parquet/src/Flow/Parquet/Engine/PhpParquetFileWriter.php @@ -21,12 +21,16 @@ use Flow\Parquet\ParquetFileWriter; use Flow\Parquet\Thrift\CompactProtocol; use Flow\Parquet\Thrift\PhpFileStream; +use Flow\Parquet\Writer\ColumnLists; use Flow\Parquet\Writer\RowGroupBuilder; use function array_chunk; +use function array_slice; use function fclose; use function fopen; use function is_array; +use function max; +use function min; use function pack; use function stream_get_contents; use function strlen; @@ -126,6 +130,30 @@ public function writeBatch(iterable $rows): void } } + public function writeColumns(array $columns): void + { + $stream = $this->stream ?? throw new RuntimeException('Writer is not open'); + $count = (new ColumnLists())->length($columns); + + /** @var int<1, max> $interval */ + $interval = $this->rowGroupSizeCheckInterval; + + for ($offset = 0; $offset < $count; $offset += $interval) { + $length = min($interval, $count - $offset); + $slices = []; + + foreach ($columns as $name => $values) { + $slices[$name] = array_slice($values, $offset, $length); + } + + $this->rowGroupBuilder->addColumns($slices, $length); + + if ($this->rowGroupBuilder->isFull()) { + $this->flushRowGroup($stream); + } + } + } + public function writeRow(array $row): void { $stream = $this->stream ?? throw new RuntimeException('Writer is not open'); diff --git a/src/lib/parquet/src/Flow/Parquet/Option.php b/src/lib/parquet/src/Flow/Parquet/Option.php index 5f39b0c88e..b3caeefde9 100644 --- a/src/lib/parquet/src/Flow/Parquet/Option.php +++ b/src/lib/parquet/src/Flow/Parquet/Option.php @@ -35,14 +35,6 @@ enum Option */ case BROTLI_COMPRESSION_LEVEL; - /** - * Some parquet writers might not properly use LogicalTyp for storing Strings or JSON's. - * This option would tell the reader to treat all BYTE_ARRAY's as UTF-8 strings. - * - * Default value is true; - */ - case BYTE_ARRAY_TO_STRING; - /** * Per-column compression configuration using flat path notation. * Accepts array where key is column flat path and value is compression enum. diff --git a/src/lib/parquet/src/Flow/Parquet/Options.php b/src/lib/parquet/src/Flow/Parquet/Options.php index b16d2c4494..f64ae97ace 100644 --- a/src/lib/parquet/src/Flow/Parquet/Options.php +++ b/src/lib/parquet/src/Flow/Parquet/Options.php @@ -25,7 +25,6 @@ public function __construct() $this->options = [ Option::ARROW_BATCH_SIZE->name => 1024, Option::ARROW_WRITE_BATCH_SIZE->name => 1000, - Option::BYTE_ARRAY_TO_STRING->name => true, Option::INT_96_AS_DATETIME->name => true, Option::PAGE_MAXIMUM_ROWS_COUNT->name => 1000, Option::PAGE_SIZE_BYTES->name => SizeUnits::KiB_SIZE * 128, diff --git a/src/lib/parquet/src/Flow/Parquet/ParquetEngine.php b/src/lib/parquet/src/Flow/Parquet/ParquetEngine.php index ed591d514e..51974879cd 100644 --- a/src/lib/parquet/src/Flow/Parquet/ParquetEngine.php +++ b/src/lib/parquet/src/Flow/Parquet/ParquetEngine.php @@ -8,10 +8,20 @@ use Flow\Filesystem\SourceStream; use Flow\Parquet\ParquetFile\Compressions; use Flow\Parquet\ParquetFile\Schema; -use Generator; + +use function interface_exists; + +if (interface_exists(ParquetEngine::class, false)) { + return; +} interface ParquetEngine { + /** + * The file on $stream, its footer read by the returned reader, which owns the stream from here on. + */ + public function openForRead(SourceStream $stream): ParquetFileReader; + public function openForWrite( DestinationStream $stream, Schema $schema, @@ -19,19 +29,6 @@ public function openForWrite( Options $options, ): ParquetFileWriter; - /** - * @param array $columns - * - * @return \Generator> - */ - public function readValues( - SourceStream $stream, - Schema $schema, - array $columns = [], - ?int $limit = null, - ?int $offset = null, - ): Generator; - /** * @param iterable> $rows */ diff --git a/src/lib/parquet/src/Flow/Parquet/ParquetFile.php b/src/lib/parquet/src/Flow/Parquet/ParquetFile.php index 7cb2bcfd11..b654c2482f 100644 --- a/src/lib/parquet/src/Flow/Parquet/ParquetFile.php +++ b/src/lib/parquet/src/Flow/Parquet/ParquetFile.php @@ -5,7 +5,6 @@ namespace Flow\Parquet; use Flow\Filesystem\SourceStream; -use Flow\Parquet\Binary\ByteOrder; use Flow\Parquet\Exception\InvalidArgumentException; use Flow\Parquet\ParquetFile\Metadata; use Flow\Parquet\ParquetFile\Page\ColumnPageHeader; @@ -13,69 +12,51 @@ use Flow\Parquet\ParquetFile\Schema\Column; use Flow\Parquet\ParquetFile\Schema\FlatColumn; use Flow\Parquet\Reader\ColumnChunkViewer; -use Flow\Parquet\Thrift\CompactProtocol; -use Flow\Parquet\Thrift\MemoryBuffer; -use Flow\Parquet\ThriftModel\FileMetaData; use Generator; +use function array_keys; use function array_map; +use function array_values; use function count; -use function unpack; +/** + * @template-covariant R of ParquetFileReader + */ final class ParquetFile { public const string PARQUET_MAGIC_NUMBER = 'PAR1'; - private ?Metadata $metadata = null; + private const int VALUES_BATCH_SIZE = 1024; + private bool $closed = false; + + /** + * @param R $reader opened on $stream + */ public function __construct( private readonly SourceStream $stream, - private readonly ByteOrder $byteOrder, private readonly Options $options, - private readonly ParquetEngine $engine, + private readonly ParquetFileReader $reader, ) {} public function __destruct() { - $this->stream->close(); + $this->close(); } - public function metadata(): Metadata + public function close(): void { - if ($this->metadata !== null) { - return $this->metadata; + if ($this->closed) { + return; } - $fileTotalSize = $this->stream->size(); - - if ($fileTotalSize === null) { - throw new InvalidArgumentException('Cannot determine Parquet file size'); - } - - if ($this->stream->read(4, $fileTotalSize - 4) !== self::PARQUET_MAGIC_NUMBER) { - throw new InvalidArgumentException('Given file is not valid Parquet file'); - } - - $unpacked = unpack($this->byteOrder->value, $this->stream->read(4, $fileTotalSize - 8)); - - if ($unpacked === false) { - throw new InvalidArgumentException('Failed to read Parquet metadata length'); - } - - $metadataLength = $unpacked[1]; - - if ($metadataLength <= 0) { - throw new InvalidArgumentException('Parquet metadata length must be positive, got ' . $metadataLength); - } - - $metadata = $this->stream->read($metadataLength, $fileTotalSize - ($metadataLength + 8)); - - $thriftMetadata = new FileMetaData(); - $thriftMetadata->read(new CompactProtocol(new MemoryBuffer($metadata))); - - $this->metadata = Metadata::fromThrift($thriftMetadata); + $this->closed = true; + $this->reader->close(); + } - return $this->metadata; + public function metadata(): Metadata + { + return $this->reader->metadata(); } /** @@ -90,23 +71,35 @@ public function pageHeaders(): Generator } } + /** + * @return R + */ + public function reader(): ParquetFileReader + { + return $this->reader; + } + public function schema(): Schema { - return $this->metadata()->schema(); + return $this->reader->schema(); } /** * @param array $columns * - * @return \Generator> + * @return Generator>> */ - public function values(array $columns = [], ?int $limit = null, ?int $offset = null): Generator + public function columns(int $batchSize, array $columns = [], ?int $limit = null, ?int $offset = null): Generator { + if ($batchSize < 1) { + throw new InvalidArgumentException('Batch size must be greater than 0'); + } + if ($limit !== null && $limit <= 0) { throw new InvalidArgumentException('Limit must be greater than 0'); } - if ($limit !== null && $offset !== null && $offset < 0) { + if ($offset !== null && $offset < 0) { throw new InvalidArgumentException('Offset must be greater than or equal to 0'); } @@ -115,12 +108,35 @@ public function values(array $columns = [], ?int $limit = null, ?int $offset = n } foreach ($columns as $columnName) { - if (!$this->metadata()->schema()->has($columnName)) { + if (!$this->schema()->has($columnName)) { throw new InvalidArgumentException("Column \"{$columnName}\" does not exist"); } } - yield from $this->engine->readValues($this->stream, $this->schema(), $columns, $limit, $offset); + yield from $this->reader->readColumns(array_values($columns), $batchSize, $limit, $offset); + } + + /** + * @param array $columns + * + * @return Generator> + */ + public function values(array $columns = [], ?int $limit = null, ?int $offset = null): Generator + { + foreach ($this->columns(self::VALUES_BATCH_SIZE, $columns, $limit, $offset) as $chunk) { + $names = array_keys($chunk); + $count = count($chunk[$names[0]]); + + for ($i = 0; $i < $count; $i++) { + $row = []; + + foreach ($names as $name) { + $row[$name] = $chunk[$name][$i]; + } + + yield $row; + } + } } /** diff --git a/src/lib/parquet/src/Flow/Parquet/ParquetFile/Data/Converter/Int96DateTimeConverter.php b/src/lib/parquet/src/Flow/Parquet/ParquetFile/Data/Converter/Int96DateTimeConverter.php index 2ecd509d5e..703285206f 100644 --- a/src/lib/parquet/src/Flow/Parquet/ParquetFile/Data/Converter/Int96DateTimeConverter.php +++ b/src/lib/parquet/src/Flow/Parquet/ParquetFile/Data/Converter/Int96DateTimeConverter.php @@ -14,8 +14,7 @@ use function array_values; use function bin2hex; -use function floor; -use function round; +use function Flow\Parquet\floor_div; use function sprintf; use function unpack; @@ -65,22 +64,14 @@ private function convertRawBytesToDateTime(string $bytes): DateTimeImmutable | ($bytesArray[6] << 48) | ($bytesArray[7] << 56); - // The Julian epoch starts on January 1, 4713 BCE. - // The Unix epoch starts on January 1, 1970 CE. - // The number of days between these two dates is 2440588. - $daysSinceUnixEpoch = $daysInEpoch - 2440588; + $micros = (($daysInEpoch - 2_440_588) * 86_400_000_000) + floor_div($nanosecondsWithinDay, 1_000); + $seconds = floor_div($micros, 1_000_000); - // Convert the days since the Unix epoch and the nanoseconds within the day to a Unix timestamp - $timestampSeconds = ($daysSinceUnixEpoch * 86400) + ($nanosecondsWithinDay / 1e9); - - // Separate the seconds and fractional seconds parts of the timestamp - $seconds = floor($timestampSeconds); - $fraction = $timestampSeconds - $seconds; - - // Convert the fractional seconds to milliseconds - $microseconds = round($fraction * 1e6); - - $dateTime = DateTimeImmutable::createFromFormat('U.u', sprintf('%d.%06d', $seconds, $microseconds)); + $dateTime = DateTimeImmutable::createFromFormat('U.u', sprintf( + '%d.%06d', + $seconds, + $micros - ($seconds * 1_000_000), + )); if ($dateTime === false) { throw new RuntimeException('Failed to convert INT96 to DateTime, given bytes: ' . bin2hex($bytes)); diff --git a/src/lib/parquet/src/Flow/Parquet/ParquetFile/Data/Converter/UnsignedIntegerConverter.php b/src/lib/parquet/src/Flow/Parquet/ParquetFile/Data/Converter/UnsignedIntegerConverter.php new file mode 100644 index 0000000000..6d15ddd70b --- /dev/null +++ b/src/lib/parquet/src/Flow/Parquet/ParquetFile/Data/Converter/UnsignedIntegerConverter.php @@ -0,0 +1,57 @@ +isSigned() + && ($integer->bitWidth() === 32 || $integer->bitWidth() === 64) + ? new self($column->flatPath(), $integer->bitWidth()) + : null; + } + + public function fromParquetType(mixed $data): int + { + if (!is_int($data)) { + throw new InvalidArgumentException(sprintf('Expected int, got %s', get_debug_type($data))); + } + + if ($data >= 0) { + return $data; + } + + if ($this->bitWidth === 32) { + return $data + 4_294_967_296; + } + + throw new RuntimeException(sprintf('Parquet column "%s" holds a UINT_64 value above PHP_INT_MAX', $this->path)); + } + + public function toParquetType(mixed $data): mixed + { + return $data; + } +} diff --git a/src/lib/parquet/src/Flow/Parquet/ParquetFile/Data/DataConverter.php b/src/lib/parquet/src/Flow/Parquet/ParquetFile/Data/DataConverter.php index 0351230a1f..01cafab35c 100644 --- a/src/lib/parquet/src/Flow/Parquet/ParquetFile/Data/DataConverter.php +++ b/src/lib/parquet/src/Flow/Parquet/ParquetFile/Data/DataConverter.php @@ -12,6 +12,7 @@ use Flow\Parquet\ParquetFile\Data\Converter\Int96DateTimeConverter; use Flow\Parquet\ParquetFile\Data\Converter\JsonConverter; use Flow\Parquet\ParquetFile\Data\Converter\TimeConverter; +use Flow\Parquet\ParquetFile\Data\Converter\UnsignedIntegerConverter; use Flow\Parquet\ParquetFile\Data\Converter\UuidConverter; use Flow\Parquet\ParquetFile\Schema\FlatColumn; use Throwable; @@ -45,6 +46,7 @@ public static function initialize(Options $options): self DecimalConverter::class, UuidConverter::class, JsonConverter::class, + UnsignedIntegerConverter::class, ], $options); } diff --git a/src/lib/parquet/src/Flow/Parquet/ParquetFile/Metadata.php b/src/lib/parquet/src/Flow/Parquet/ParquetFile/Metadata.php index f8b84ec8a1..b78a778e71 100644 --- a/src/lib/parquet/src/Flow/Parquet/ParquetFile/Metadata.php +++ b/src/lib/parquet/src/Flow/Parquet/ParquetFile/Metadata.php @@ -17,6 +17,7 @@ public function __construct( private ?string $createdBy, ) {} + // called by name from arrow-ext RustParquetFileReader::metadata(), pinned by phpts 047, 052 public static function fromThrift(FileMetaData $thrift): self { return new self( diff --git a/src/lib/parquet/src/Flow/Parquet/ParquetFile/Schema.php b/src/lib/parquet/src/Flow/Parquet/ParquetFile/Schema.php index 4b9d3738f2..548e4cb0e6 100644 --- a/src/lib/parquet/src/Flow/Parquet/ParquetFile/Schema.php +++ b/src/lib/parquet/src/Flow/Parquet/ParquetFile/Schema.php @@ -25,6 +25,7 @@ public function __construct( private readonly NestedColumn $schemaRoot, ) {} + // called by name from arrow-ext RustParquetFileReader::schema(), pinned by phpts 047, 052 /** * @param array $schemaElements */ diff --git a/src/lib/parquet/src/Flow/Parquet/ParquetFile/Schema/FlatColumn.php b/src/lib/parquet/src/Flow/Parquet/ParquetFile/Schema/FlatColumn.php index be2660f665..bda56927d8 100644 --- a/src/lib/parquet/src/Flow/Parquet/ParquetFile/Schema/FlatColumn.php +++ b/src/lib/parquet/src/Flow/Parquet/ParquetFile/Schema/FlatColumn.php @@ -110,7 +110,7 @@ public static function double(string $name, Repetition $repetition = Repetition: public static function enum(string $string, Repetition $repetition = Repetition::OPTIONAL): self { - return new self($string, PhysicalType::BYTE_ARRAY, ConvertedType::ENUM, LogicalType::string(), $repetition); + return new self($string, PhysicalType::BYTE_ARRAY, ConvertedType::ENUM, LogicalType::enum(), $repetition); } public static function fixedSizeByteArray( diff --git a/src/lib/parquet/src/Flow/Parquet/ParquetFile/Schema/LogicalType.php b/src/lib/parquet/src/Flow/Parquet/ParquetFile/Schema/LogicalType.php index 61dcef2228..c4c5f4483e 100644 --- a/src/lib/parquet/src/Flow/Parquet/ParquetFile/Schema/LogicalType.php +++ b/src/lib/parquet/src/Flow/Parquet/ParquetFile/Schema/LogicalType.php @@ -5,6 +5,7 @@ namespace Flow\Parquet\ParquetFile\Schema; use Flow\Parquet\ParquetFile\Schema\LogicalType\Decimal; +use Flow\Parquet\ParquetFile\Schema\LogicalType\Integer; use Flow\Parquet\ParquetFile\Schema\LogicalType\Time; use Flow\Parquet\ParquetFile\Schema\LogicalType\Timestamp; use Flow\Parquet\ThriftModel\BsonType; @@ -12,7 +13,6 @@ use Flow\Parquet\ThriftModel\DecimalType; use Flow\Parquet\ThriftModel\EnumType; use Flow\Parquet\ThriftModel\Float16Type; -use Flow\Parquet\ThriftModel\IntType; use Flow\Parquet\ThriftModel\JsonType; use Flow\Parquet\ThriftModel\ListType; use Flow\Parquet\ThriftModel\LogicalType as ThriftLogicalType; @@ -62,6 +62,7 @@ public function __construct( private ?Timestamp $timestamp = null, private ?Time $time = null, private ?Decimal $decimal = null, + private ?Integer $integer = null, ) {} public static function bson(): self @@ -208,12 +209,15 @@ public static function fromThrift(ThriftLogicalType $logicalType): ?self // @mago-ignore analysis:redundant-condition // @mago-ignore analysis:redundant-comparison decimal: $logicalType->DECIMAL !== null ? Decimal::fromThrift($logicalType->DECIMAL) : null, + // @mago-ignore analysis:redundant-condition + // @mago-ignore analysis:redundant-comparison + integer: $logicalType->INTEGER !== null ? Integer::fromThrift($logicalType->INTEGER) : null, ); } - public static function integer(): self + public static function integer(int $bitWidth, bool $isSigned): self { - return new self(self::INTEGER); + return new self(self::INTEGER, integer: new Integer($bitWidth, $isSigned)); } public static function json(): self @@ -261,6 +265,11 @@ public function decimalData(): ?Decimal return $this->decimal; } + public function integerData(): ?Integer + { + return $this->integer; + } + public function is(string $logicalType): bool { return $this->name() === $logicalType; @@ -292,7 +301,7 @@ public function toThrift(): ThriftLogicalType 'precision' => $this->decimalData()?->precision(), ]) : null, self::ENUM => $this->is(self::ENUM) ? new EnumType() : null, - self::INTEGER => $this->is(self::INTEGER) ? new IntType() : null, + self::INTEGER => $this->is(self::INTEGER) ? $this->integer?->toThrift() : null, self::JSON => $this->is(self::JSON) ? new JsonType() : null, self::LIST => $this->is(self::LIST) ? new ListType() : null, self::MAP => $this->is(self::MAP) ? new MapType() : null, diff --git a/src/lib/parquet/src/Flow/Parquet/ParquetFile/Schema/LogicalType/Integer.php b/src/lib/parquet/src/Flow/Parquet/ParquetFile/Schema/LogicalType/Integer.php new file mode 100644 index 0000000000..78d50df519 --- /dev/null +++ b/src/lib/parquet/src/Flow/Parquet/ParquetFile/Schema/LogicalType/Integer.php @@ -0,0 +1,55 @@ +convertedType()) { + ConvertedType::INT_8 => new self(8, true), + ConvertedType::INT_16 => new self(16, true), + ConvertedType::INT_32 => new self(32, true), + ConvertedType::INT_64 => new self(64, true), + ConvertedType::UINT_8 => new self(8, false), + ConvertedType::UINT_16 => new self(16, false), + ConvertedType::UINT_32 => new self(32, false), + ConvertedType::UINT_64 => new self(64, false), + default => $column->logicalType()?->integerData(), + }; + } + + public static function fromThrift(IntType $thrift): self + { + return new self((int) $thrift->bitWidth, $thrift->isSigned); + } + + public function bitWidth(): int + { + return $this->bitWidth; + } + + public function isSigned(): bool + { + return $this->isSigned; + } + + public function toThrift(): IntType + { + return new IntType(['bitWidth' => $this->bitWidth, 'isSigned' => $this->isSigned]); + } +} diff --git a/src/lib/parquet/src/Flow/Parquet/ParquetFileReader.php b/src/lib/parquet/src/Flow/Parquet/ParquetFileReader.php new file mode 100644 index 0000000000..b98d703d9b --- /dev/null +++ b/src/lib/parquet/src/Flow/Parquet/ParquetFileReader.php @@ -0,0 +1,48 @@ + $columns resolved names, never empty + * @param int<1, max> $batchSize upper bound of rows per chunk + * + * @return Iterator>> chunks keyed by column, every list the same length (>0) + */ + public function readColumns(array $columns, int $batchSize, ?int $limit, ?int $offset): Iterator; + + public function rowsNumber(): int; + + /** + * The file schema, without building the row-group metadata. + */ + public function schema(): Schema; + + /** + * Σ row group total_byte_size (uncompressed), without building the row-group metadata. + */ + public function totalByteSize(): int; +} diff --git a/src/lib/parquet/src/Flow/Parquet/ParquetFileWriter.php b/src/lib/parquet/src/Flow/Parquet/ParquetFileWriter.php index 8e15212027..b9e6b8a91d 100644 --- a/src/lib/parquet/src/Flow/Parquet/ParquetFileWriter.php +++ b/src/lib/parquet/src/Flow/Parquet/ParquetFileWriter.php @@ -4,6 +4,12 @@ namespace Flow\Parquet; +use function interface_exists; + +if (interface_exists(ParquetFileWriter::class, false)) { + return; +} + interface ParquetFileWriter { /** @@ -17,6 +23,12 @@ public function close(): void; */ public function writeBatch(iterable $rows): void; + /** + * @param array> $columns by top-level column name, every list of one length; a schema column + * the array lacks is written as nulls, a key the schema lacks is ignored + */ + public function writeColumns(array $columns): void; + /** * @param array $row */ diff --git a/src/lib/parquet/src/Flow/Parquet/Reader.php b/src/lib/parquet/src/Flow/Parquet/Reader.php index 82e94cd788..0e51e2f356 100644 --- a/src/lib/parquet/src/Flow/Parquet/Reader.php +++ b/src/lib/parquet/src/Flow/Parquet/Reader.php @@ -8,7 +8,6 @@ use Flow\Filesystem\Stream\NativeLocalSourceStream; use Flow\Parquet\Binary\ByteOrder; use Flow\Parquet\Engine\AdaptiveParquetEngine; -use Flow\Parquet\Engine\ArrowParquetEngine; use Flow\Parquet\Engine\PhpParquetEngine; use function Flow\Filesystem\DSL\path_real; @@ -25,28 +24,24 @@ public function __construct( $this->engine = $engine ?? new AdaptiveParquetEngine($this->byteOrder, $this->options); } - public static function arrow(Options $options = new Options()): self - { - return new self(options: $options, engine: new ArrowParquetEngine($options)); - } - public static function php(Options $options = new Options()): self { return new self(options: $options, engine: new PhpParquetEngine(options: $options)); } + /** + * @return ParquetFile + */ public function read(string $path): ParquetFile { - return new ParquetFile( - NativeLocalSourceStream::open(path_real($path)), - $this->byteOrder, - $this->options, - $this->engine, - ); + return $this->readStream(NativeLocalSourceStream::open(path_real($path))); } + /** + * @return ParquetFile + */ public function readStream(SourceStream $stream): ParquetFile { - return new ParquetFile($stream, $this->byteOrder, $this->options, $this->engine); + return new ParquetFile($stream, $this->options, $this->engine->openForRead($stream)); } } diff --git a/src/lib/parquet/src/Flow/Parquet/Reader/ColumnDataDecoder.php b/src/lib/parquet/src/Flow/Parquet/Reader/ColumnDataDecoder.php index 45129871e2..3d83bc730e 100644 --- a/src/lib/parquet/src/Flow/Parquet/Reader/ColumnDataDecoder.php +++ b/src/lib/parquet/src/Flow/Parquet/Reader/ColumnDataDecoder.php @@ -19,6 +19,7 @@ use Flow\Parquet\ParquetFile\Page\Header\DictionaryPageHeader; use Flow\Parquet\ParquetFile\Schema\FlatColumn; use Flow\Parquet\ParquetFile\Schema\PhysicalType; +use Generator; use function array_fill; use function array_key_exists; @@ -170,6 +171,23 @@ public function decodeDataV2( ); } + if ($pageHeader->encoding() === Encodings::RLE && $column->type() === PhysicalType::BOOLEAN) { + // RLE data values, unlike levels in a v2 page, carry their 4-byte length first + $reader->seekBytes(4); + $values = $this->readRLEBitPackedHybrid($reader, $RLEBitPackedHybrid, 1, $nonEmptyValuesCount); + + return new ReadFlatColumnValues( + $column, + (static function () use ($values): Generator { + foreach ($values as $value) { + yield $value === 1; + } + })(), + $repetitionLevels, + $definitionLevels, + ); + } + if ($pageHeader->encoding() === Encodings::DELTA_BINARY_PACKED) { if (!in_array($column->type(), [PhysicalType::INT32, PhysicalType::INT64], true)) { throw new RuntimeException('Delta encoding only supports INT32 and INT64 physical types'); diff --git a/src/lib/parquet/src/Flow/Parquet/Reader/ColumnReader.php b/src/lib/parquet/src/Flow/Parquet/Reader/ColumnReader.php new file mode 100644 index 0000000000..c7e87a3421 --- /dev/null +++ b/src/lib/parquet/src/Flow/Parquet/Reader/ColumnReader.php @@ -0,0 +1,104 @@ +rowGroups()->all() as $rowGroup) { + if ($offset !== null) { + if (($rowGroupOffset + $rowGroup->rowsCount()) <= $offset) { + $rowGroupOffset += $rowGroup->rowsCount(); + + continue; + } + } + $skipRows = $offset !== null ? $offset - $rowGroupOffset : 0; + + if ($column instanceof FlatColumn) { + $rowsSkipped = 0; + + foreach ($this->chunks->read( + $rowGroup->getColumnChunk($column), + $column, + $stream, + ) as $flatColumnValues) { + $columnData = new ReadColumnData($column, [$flatColumnValues->flatPath() => $flatColumnValues]); + + // @mago-ignore analysis:mixed-assignment + foreach ($this->assembler->assemble($column, $columnData) as $row) { + if ($skipRows > 0 && $rowsSkipped < $skipRows) { + $rowsSkipped++; + + continue; + } + + if ($limit !== null && $yieldedRows >= $limit) { + return; + } + yield $row; + $yieldedRows++; + } + } + } elseif ($column instanceof NestedColumn) { + $flatData = []; + + foreach ($column->childrenFlat() as $child) { + $flatData[] = new PagedFlatColumnValues($child, $this->chunks->read( + $rowGroup->getColumnChunk($child), + $child, + $stream, + )); + } + + $columnData = new ReadColumnData($column, $flatData); + + $rowsSkipped = 0; + + // @mago-ignore analysis:mixed-assignment + foreach ($this->assembler->assemble($column, $columnData) as $row) { + if ($skipRows > 0 && $rowsSkipped < $skipRows) { + $rowsSkipped++; + + continue; + } + + if ($limit !== null && $yieldedRows >= $limit) { + return; + } + yield $row; + $yieldedRows++; + } + } else { + throw new InvalidArgumentException('Column must be instance of FlatColumn or NestedColumn'); + } + + $rowGroupOffset += $rowGroup->rowsCount(); + } + } +} diff --git a/src/lib/parquet/src/Flow/Parquet/ThriftModel/BloomFilterAlgorithm.php b/src/lib/parquet/src/Flow/Parquet/ThriftModel/BloomFilterAlgorithm.php index e28cace801..fc6898e195 100644 --- a/src/lib/parquet/src/Flow/Parquet/ThriftModel/BloomFilterAlgorithm.php +++ b/src/lib/parquet/src/Flow/Parquet/ThriftModel/BloomFilterAlgorithm.php @@ -24,7 +24,7 @@ class BloomFilterAlgorithm 'var' => 'BLOCK', 'isRequired' => false, 'type' => TType::STRUCT, - 'class' => '\Flow\Parquet\Thrift\SplitBlockAlgorithm', + 'class' => '\Flow\Parquet\ThriftModel\SplitBlockAlgorithm', ], ]; diff --git a/src/lib/parquet/src/Flow/Parquet/ThriftModel/BloomFilterCompression.php b/src/lib/parquet/src/Flow/Parquet/ThriftModel/BloomFilterCompression.php index ac977e23bd..b19c25b750 100644 --- a/src/lib/parquet/src/Flow/Parquet/ThriftModel/BloomFilterCompression.php +++ b/src/lib/parquet/src/Flow/Parquet/ThriftModel/BloomFilterCompression.php @@ -21,7 +21,7 @@ class BloomFilterCompression 'var' => 'UNCOMPRESSED', 'isRequired' => false, 'type' => TType::STRUCT, - 'class' => '\Flow\Parquet\Thrift\Uncompressed', + 'class' => '\Flow\Parquet\ThriftModel\Uncompressed', ], ]; diff --git a/src/lib/parquet/src/Flow/Parquet/ThriftModel/BloomFilterHash.php b/src/lib/parquet/src/Flow/Parquet/ThriftModel/BloomFilterHash.php index 1ea5463ab7..1426be9ca5 100644 --- a/src/lib/parquet/src/Flow/Parquet/ThriftModel/BloomFilterHash.php +++ b/src/lib/parquet/src/Flow/Parquet/ThriftModel/BloomFilterHash.php @@ -25,7 +25,7 @@ class BloomFilterHash 'var' => 'XXHASH', 'isRequired' => false, 'type' => TType::STRUCT, - 'class' => '\Flow\Parquet\Thrift\XxHash', + 'class' => '\Flow\Parquet\ThriftModel\XxHash', ], ]; diff --git a/src/lib/parquet/src/Flow/Parquet/ThriftModel/BloomFilterHeader.php b/src/lib/parquet/src/Flow/Parquet/ThriftModel/BloomFilterHeader.php index 314299c9ec..06682c552e 100644 --- a/src/lib/parquet/src/Flow/Parquet/ThriftModel/BloomFilterHeader.php +++ b/src/lib/parquet/src/Flow/Parquet/ThriftModel/BloomFilterHeader.php @@ -30,19 +30,19 @@ class BloomFilterHeader 'var' => 'algorithm', 'isRequired' => true, 'type' => TType::STRUCT, - 'class' => '\Flow\Parquet\Thrift\BloomFilterAlgorithm', + 'class' => '\Flow\Parquet\ThriftModel\BloomFilterAlgorithm', ], 3 => [ 'var' => 'hash', 'isRequired' => true, 'type' => TType::STRUCT, - 'class' => '\Flow\Parquet\Thrift\BloomFilterHash', + 'class' => '\Flow\Parquet\ThriftModel\BloomFilterHash', ], 4 => [ 'var' => 'compression', 'isRequired' => true, 'type' => TType::STRUCT, - 'class' => '\Flow\Parquet\Thrift\BloomFilterCompression', + 'class' => '\Flow\Parquet\ThriftModel\BloomFilterCompression', ], ]; diff --git a/src/lib/parquet/src/Flow/Parquet/ThriftModel/ColumnChunk.php b/src/lib/parquet/src/Flow/Parquet/ThriftModel/ColumnChunk.php index 677365c030..bb1efef2e1 100644 --- a/src/lib/parquet/src/Flow/Parquet/ThriftModel/ColumnChunk.php +++ b/src/lib/parquet/src/Flow/Parquet/ThriftModel/ColumnChunk.php @@ -31,7 +31,7 @@ class ColumnChunk 'var' => 'meta_data', 'isRequired' => false, 'type' => TType::STRUCT, - 'class' => '\Flow\Parquet\Thrift\ColumnMetaData', + 'class' => '\Flow\Parquet\ThriftModel\ColumnMetaData', ], 4 => [ 'var' => 'offset_index_offset', @@ -57,7 +57,7 @@ class ColumnChunk 'var' => 'crypto_metadata', 'isRequired' => false, 'type' => TType::STRUCT, - 'class' => '\Flow\Parquet\Thrift\ColumnCryptoMetaData', + 'class' => '\Flow\Parquet\ThriftModel\ColumnCryptoMetaData', ], 9 => [ 'var' => 'encrypted_column_metadata', diff --git a/src/lib/parquet/src/Flow/Parquet/ThriftModel/ColumnCryptoMetaData.php b/src/lib/parquet/src/Flow/Parquet/ThriftModel/ColumnCryptoMetaData.php index 0ee1ba3b50..cf578e9f76 100644 --- a/src/lib/parquet/src/Flow/Parquet/ThriftModel/ColumnCryptoMetaData.php +++ b/src/lib/parquet/src/Flow/Parquet/ThriftModel/ColumnCryptoMetaData.php @@ -21,13 +21,13 @@ class ColumnCryptoMetaData 'var' => 'ENCRYPTION_WITH_FOOTER_KEY', 'isRequired' => false, 'type' => TType::STRUCT, - 'class' => '\Flow\Parquet\Thrift\EncryptionWithFooterKey', + 'class' => '\Flow\Parquet\ThriftModel\EncryptionWithFooterKey', ], 2 => [ 'var' => 'ENCRYPTION_WITH_COLUMN_KEY', 'isRequired' => false, 'type' => TType::STRUCT, - 'class' => '\Flow\Parquet\Thrift\EncryptionWithColumnKey', + 'class' => '\Flow\Parquet\ThriftModel\EncryptionWithColumnKey', ], ]; diff --git a/src/lib/parquet/src/Flow/Parquet/ThriftModel/ColumnIndex.php b/src/lib/parquet/src/Flow/Parquet/ThriftModel/ColumnIndex.php index a69987f6f2..2a962e08d1 100644 --- a/src/lib/parquet/src/Flow/Parquet/ThriftModel/ColumnIndex.php +++ b/src/lib/parquet/src/Flow/Parquet/ThriftModel/ColumnIndex.php @@ -58,7 +58,7 @@ class ColumnIndex 'var' => 'boundary_order', 'isRequired' => true, 'type' => TType::I32, - 'class' => '\Flow\Parquet\Thrift\BoundaryOrder', + 'class' => '\Flow\Parquet\ThriftModel\BoundaryOrder', ], 5 => [ 'var' => 'null_counts', diff --git a/src/lib/parquet/src/Flow/Parquet/ThriftModel/ColumnMetaData.php b/src/lib/parquet/src/Flow/Parquet/ThriftModel/ColumnMetaData.php index 4a38814f53..438c05118c 100644 --- a/src/lib/parquet/src/Flow/Parquet/ThriftModel/ColumnMetaData.php +++ b/src/lib/parquet/src/Flow/Parquet/ThriftModel/ColumnMetaData.php @@ -24,7 +24,7 @@ class ColumnMetaData 'var' => 'type', 'isRequired' => true, 'type' => TType::I32, - 'class' => '\Flow\Parquet\Thrift\Type', + 'class' => '\Flow\Parquet\ThriftModel\Type', ], 2 => [ 'var' => 'encodings', @@ -33,7 +33,7 @@ class ColumnMetaData 'etype' => TType::I32, 'elem' => [ 'type' => TType::I32, - 'class' => '\Flow\Parquet\Thrift\Encoding', + 'class' => '\Flow\Parquet\ThriftModel\Encoding', ], ], 3 => [ @@ -49,7 +49,7 @@ class ColumnMetaData 'var' => 'codec', 'isRequired' => true, 'type' => TType::I32, - 'class' => '\Flow\Parquet\Thrift\CompressionCodec', + 'class' => '\Flow\Parquet\ThriftModel\CompressionCodec', ], 5 => [ 'var' => 'num_values', @@ -73,7 +73,7 @@ class ColumnMetaData 'etype' => TType::STRUCT, 'elem' => [ 'type' => TType::STRUCT, - 'class' => '\Flow\Parquet\Thrift\KeyValue', + 'class' => '\Flow\Parquet\ThriftModel\KeyValue', ], ], 9 => [ @@ -95,7 +95,7 @@ class ColumnMetaData 'var' => 'statistics', 'isRequired' => false, 'type' => TType::STRUCT, - 'class' => '\Flow\Parquet\Thrift\Statistics', + 'class' => '\Flow\Parquet\ThriftModel\Statistics', ], 13 => [ 'var' => 'encoding_stats', @@ -104,7 +104,7 @@ class ColumnMetaData 'etype' => TType::STRUCT, 'elem' => [ 'type' => TType::STRUCT, - 'class' => '\Flow\Parquet\Thrift\PageEncodingStats', + 'class' => '\Flow\Parquet\ThriftModel\PageEncodingStats', ], ], 14 => [ @@ -121,7 +121,7 @@ class ColumnMetaData 'var' => 'size_statistics', 'isRequired' => false, 'type' => TType::STRUCT, - 'class' => '\Flow\Parquet\Thrift\SizeStatistics', + 'class' => '\Flow\Parquet\ThriftModel\SizeStatistics', ], ]; diff --git a/src/lib/parquet/src/Flow/Parquet/ThriftModel/ColumnOrder.php b/src/lib/parquet/src/Flow/Parquet/ThriftModel/ColumnOrder.php index cb09aab8e2..d57d511bb3 100644 --- a/src/lib/parquet/src/Flow/Parquet/ThriftModel/ColumnOrder.php +++ b/src/lib/parquet/src/Flow/Parquet/ThriftModel/ColumnOrder.php @@ -33,7 +33,7 @@ class ColumnOrder 'var' => 'TYPE_ORDER', 'isRequired' => false, 'type' => TType::STRUCT, - 'class' => '\Flow\Parquet\Thrift\TypeDefinedOrder', + 'class' => '\Flow\Parquet\ThriftModel\TypeDefinedOrder', ], ]; diff --git a/src/lib/parquet/src/Flow/Parquet/ThriftModel/DataPageHeader.php b/src/lib/parquet/src/Flow/Parquet/ThriftModel/DataPageHeader.php index 6b38b935c4..b61754efa5 100644 --- a/src/lib/parquet/src/Flow/Parquet/ThriftModel/DataPageHeader.php +++ b/src/lib/parquet/src/Flow/Parquet/ThriftModel/DataPageHeader.php @@ -29,25 +29,25 @@ class DataPageHeader 'var' => 'encoding', 'isRequired' => true, 'type' => TType::I32, - 'class' => '\Flow\Parquet\Thrift\Encoding', + 'class' => '\Flow\Parquet\ThriftModel\Encoding', ], 3 => [ 'var' => 'definition_level_encoding', 'isRequired' => true, 'type' => TType::I32, - 'class' => '\Flow\Parquet\Thrift\Encoding', + 'class' => '\Flow\Parquet\ThriftModel\Encoding', ], 4 => [ 'var' => 'repetition_level_encoding', 'isRequired' => true, 'type' => TType::I32, - 'class' => '\Flow\Parquet\Thrift\Encoding', + 'class' => '\Flow\Parquet\ThriftModel\Encoding', ], 5 => [ 'var' => 'statistics', 'isRequired' => false, 'type' => TType::STRUCT, - 'class' => '\Flow\Parquet\Thrift\Statistics', + 'class' => '\Flow\Parquet\ThriftModel\Statistics', ], ]; diff --git a/src/lib/parquet/src/Flow/Parquet/ThriftModel/DataPageHeaderV2.php b/src/lib/parquet/src/Flow/Parquet/ThriftModel/DataPageHeaderV2.php index 674145a8b9..91f25116c0 100644 --- a/src/lib/parquet/src/Flow/Parquet/ThriftModel/DataPageHeaderV2.php +++ b/src/lib/parquet/src/Flow/Parquet/ThriftModel/DataPageHeaderV2.php @@ -41,7 +41,7 @@ class DataPageHeaderV2 'var' => 'encoding', 'isRequired' => true, 'type' => TType::I32, - 'class' => '\Flow\Parquet\Thrift\Encoding', + 'class' => '\Flow\Parquet\ThriftModel\Encoding', ], 5 => [ 'var' => 'definition_levels_byte_length', @@ -62,7 +62,7 @@ class DataPageHeaderV2 'var' => 'statistics', 'isRequired' => false, 'type' => TType::STRUCT, - 'class' => '\Flow\Parquet\Thrift\Statistics', + 'class' => '\Flow\Parquet\ThriftModel\Statistics', ], ]; diff --git a/src/lib/parquet/src/Flow/Parquet/ThriftModel/DictionaryPageHeader.php b/src/lib/parquet/src/Flow/Parquet/ThriftModel/DictionaryPageHeader.php index b5efc68e18..f853a893f7 100644 --- a/src/lib/parquet/src/Flow/Parquet/ThriftModel/DictionaryPageHeader.php +++ b/src/lib/parquet/src/Flow/Parquet/ThriftModel/DictionaryPageHeader.php @@ -30,7 +30,7 @@ class DictionaryPageHeader 'var' => 'encoding', 'isRequired' => true, 'type' => TType::I32, - 'class' => '\Flow\Parquet\Thrift\Encoding', + 'class' => '\Flow\Parquet\ThriftModel\Encoding', ], 3 => [ 'var' => 'is_sorted', diff --git a/src/lib/parquet/src/Flow/Parquet/ThriftModel/EncryptionAlgorithm.php b/src/lib/parquet/src/Flow/Parquet/ThriftModel/EncryptionAlgorithm.php index fb3d1d0cc5..c2001cd069 100644 --- a/src/lib/parquet/src/Flow/Parquet/ThriftModel/EncryptionAlgorithm.php +++ b/src/lib/parquet/src/Flow/Parquet/ThriftModel/EncryptionAlgorithm.php @@ -21,13 +21,13 @@ class EncryptionAlgorithm 'var' => 'AES_GCM_V1', 'isRequired' => false, 'type' => TType::STRUCT, - 'class' => '\Flow\Parquet\Thrift\AesGcmV1', + 'class' => '\Flow\Parquet\ThriftModel\AesGcmV1', ], 2 => [ 'var' => 'AES_GCM_CTR_V1', 'isRequired' => false, 'type' => TType::STRUCT, - 'class' => '\Flow\Parquet\Thrift\AesGcmCtrV1', + 'class' => '\Flow\Parquet\ThriftModel\AesGcmCtrV1', ], ]; diff --git a/src/lib/parquet/src/Flow/Parquet/ThriftModel/FileCryptoMetaData.php b/src/lib/parquet/src/Flow/Parquet/ThriftModel/FileCryptoMetaData.php index 233de93e2f..53a398bb1b 100644 --- a/src/lib/parquet/src/Flow/Parquet/ThriftModel/FileCryptoMetaData.php +++ b/src/lib/parquet/src/Flow/Parquet/ThriftModel/FileCryptoMetaData.php @@ -24,7 +24,7 @@ class FileCryptoMetaData 'var' => 'encryption_algorithm', 'isRequired' => true, 'type' => TType::STRUCT, - 'class' => '\Flow\Parquet\Thrift\EncryptionAlgorithm', + 'class' => '\Flow\Parquet\ThriftModel\EncryptionAlgorithm', ], 2 => [ 'var' => 'key_metadata', diff --git a/src/lib/parquet/src/Flow/Parquet/ThriftModel/FileMetaData.php b/src/lib/parquet/src/Flow/Parquet/ThriftModel/FileMetaData.php index e55606e54d..8c308ebf14 100644 --- a/src/lib/parquet/src/Flow/Parquet/ThriftModel/FileMetaData.php +++ b/src/lib/parquet/src/Flow/Parquet/ThriftModel/FileMetaData.php @@ -32,7 +32,7 @@ class FileMetaData 'etype' => TType::STRUCT, 'elem' => [ 'type' => TType::STRUCT, - 'class' => '\Flow\Parquet\Thrift\SchemaElement', + 'class' => '\Flow\Parquet\ThriftModel\SchemaElement', ], ], 3 => [ @@ -47,7 +47,7 @@ class FileMetaData 'etype' => TType::STRUCT, 'elem' => [ 'type' => TType::STRUCT, - 'class' => '\Flow\Parquet\Thrift\RowGroup', + 'class' => '\Flow\Parquet\ThriftModel\RowGroup', ], ], 5 => [ @@ -57,7 +57,7 @@ class FileMetaData 'etype' => TType::STRUCT, 'elem' => [ 'type' => TType::STRUCT, - 'class' => '\Flow\Parquet\Thrift\KeyValue', + 'class' => '\Flow\Parquet\ThriftModel\KeyValue', ], ], 6 => [ @@ -72,14 +72,14 @@ class FileMetaData 'etype' => TType::STRUCT, 'elem' => [ 'type' => TType::STRUCT, - 'class' => '\Flow\Parquet\Thrift\ColumnOrder', + 'class' => '\Flow\Parquet\ThriftModel\ColumnOrder', ], ], 8 => [ 'var' => 'encryption_algorithm', 'isRequired' => false, 'type' => TType::STRUCT, - 'class' => '\Flow\Parquet\Thrift\EncryptionAlgorithm', + 'class' => '\Flow\Parquet\ThriftModel\EncryptionAlgorithm', ], 9 => [ 'var' => 'footer_signing_key_metadata', diff --git a/src/lib/parquet/src/Flow/Parquet/ThriftModel/LogicalType.php b/src/lib/parquet/src/Flow/Parquet/ThriftModel/LogicalType.php index a356a257a3..18ba9b0d1c 100644 --- a/src/lib/parquet/src/Flow/Parquet/ThriftModel/LogicalType.php +++ b/src/lib/parquet/src/Flow/Parquet/ThriftModel/LogicalType.php @@ -28,85 +28,85 @@ class LogicalType 'var' => 'STRING', 'isRequired' => false, 'type' => TType::STRUCT, - 'class' => '\Flow\Parquet\Thrift\StringType', + 'class' => '\Flow\Parquet\ThriftModel\StringType', ], 2 => [ 'var' => 'MAP', 'isRequired' => false, 'type' => TType::STRUCT, - 'class' => '\Flow\Parquet\Thrift\MapType', + 'class' => '\Flow\Parquet\ThriftModel\MapType', ], 3 => [ 'var' => 'LIST', 'isRequired' => false, 'type' => TType::STRUCT, - 'class' => '\Flow\Parquet\Thrift\ListType', + 'class' => '\Flow\Parquet\ThriftModel\ListType', ], 4 => [ 'var' => 'ENUM', 'isRequired' => false, 'type' => TType::STRUCT, - 'class' => '\Flow\Parquet\Thrift\EnumType', + 'class' => '\Flow\Parquet\ThriftModel\EnumType', ], 5 => [ 'var' => 'DECIMAL', 'isRequired' => false, 'type' => TType::STRUCT, - 'class' => '\Flow\Parquet\Thrift\DecimalType', + 'class' => '\Flow\Parquet\ThriftModel\DecimalType', ], 6 => [ 'var' => 'DATE', 'isRequired' => false, 'type' => TType::STRUCT, - 'class' => '\Flow\Parquet\Thrift\DateType', + 'class' => '\Flow\Parquet\ThriftModel\DateType', ], 7 => [ 'var' => 'TIME', 'isRequired' => false, 'type' => TType::STRUCT, - 'class' => '\Flow\Parquet\Thrift\TimeType', + 'class' => '\Flow\Parquet\ThriftModel\TimeType', ], 8 => [ 'var' => 'TIMESTAMP', 'isRequired' => false, 'type' => TType::STRUCT, - 'class' => '\Flow\Parquet\Thrift\TimestampType', + 'class' => '\Flow\Parquet\ThriftModel\TimestampType', ], 10 => [ 'var' => 'INTEGER', 'isRequired' => false, 'type' => TType::STRUCT, - 'class' => '\Flow\Parquet\Thrift\IntType', + 'class' => '\Flow\Parquet\ThriftModel\IntType', ], 11 => [ 'var' => 'UNKNOWN', 'isRequired' => false, 'type' => TType::STRUCT, - 'class' => '\Flow\Parquet\Thrift\NullType', + 'class' => '\Flow\Parquet\ThriftModel\NullType', ], 12 => [ 'var' => 'JSON', 'isRequired' => false, 'type' => TType::STRUCT, - 'class' => '\Flow\Parquet\Thrift\JsonType', + 'class' => '\Flow\Parquet\ThriftModel\JsonType', ], 13 => [ 'var' => 'BSON', 'isRequired' => false, 'type' => TType::STRUCT, - 'class' => '\Flow\Parquet\Thrift\BsonType', + 'class' => '\Flow\Parquet\ThriftModel\BsonType', ], 14 => [ 'var' => 'UUID', 'isRequired' => false, 'type' => TType::STRUCT, - 'class' => '\Flow\Parquet\Thrift\UUIDType', + 'class' => '\Flow\Parquet\ThriftModel\UUIDType', ], 15 => [ 'var' => 'FLOAT16', 'isRequired' => false, 'type' => TType::STRUCT, - 'class' => '\Flow\Parquet\Thrift\Float16Type', + 'class' => '\Flow\Parquet\ThriftModel\Float16Type', ], ]; diff --git a/src/lib/parquet/src/Flow/Parquet/ThriftModel/OffsetIndex.php b/src/lib/parquet/src/Flow/Parquet/ThriftModel/OffsetIndex.php index cf298a2366..f92716c129 100644 --- a/src/lib/parquet/src/Flow/Parquet/ThriftModel/OffsetIndex.php +++ b/src/lib/parquet/src/Flow/Parquet/ThriftModel/OffsetIndex.php @@ -31,7 +31,7 @@ class OffsetIndex 'etype' => TType::STRUCT, 'elem' => [ 'type' => TType::STRUCT, - 'class' => '\Flow\Parquet\Thrift\PageLocation', + 'class' => '\Flow\Parquet\ThriftModel\PageLocation', ], ], 2 => [ diff --git a/src/lib/parquet/src/Flow/Parquet/ThriftModel/PageEncodingStats.php b/src/lib/parquet/src/Flow/Parquet/ThriftModel/PageEncodingStats.php index 467340bc94..15a48a6129 100644 --- a/src/lib/parquet/src/Flow/Parquet/ThriftModel/PageEncodingStats.php +++ b/src/lib/parquet/src/Flow/Parquet/ThriftModel/PageEncodingStats.php @@ -23,13 +23,13 @@ class PageEncodingStats 'var' => 'page_type', 'isRequired' => true, 'type' => TType::I32, - 'class' => '\Flow\Parquet\Thrift\PageType', + 'class' => '\Flow\Parquet\ThriftModel\PageType', ], 2 => [ 'var' => 'encoding', 'isRequired' => true, 'type' => TType::I32, - 'class' => '\Flow\Parquet\Thrift\Encoding', + 'class' => '\Flow\Parquet\ThriftModel\Encoding', ], 3 => [ 'var' => 'count', diff --git a/src/lib/parquet/src/Flow/Parquet/ThriftModel/PageHeader.php b/src/lib/parquet/src/Flow/Parquet/ThriftModel/PageHeader.php index c12c3b0b76..b6639ab911 100644 --- a/src/lib/parquet/src/Flow/Parquet/ThriftModel/PageHeader.php +++ b/src/lib/parquet/src/Flow/Parquet/ThriftModel/PageHeader.php @@ -21,7 +21,7 @@ class PageHeader 'var' => 'type', 'isRequired' => true, 'type' => TType::I32, - 'class' => '\Flow\Parquet\Thrift\PageType', + 'class' => '\Flow\Parquet\ThriftModel\PageType', ], 2 => [ 'var' => 'uncompressed_page_size', @@ -42,25 +42,25 @@ class PageHeader 'var' => 'data_page_header', 'isRequired' => false, 'type' => TType::STRUCT, - 'class' => '\Flow\Parquet\Thrift\DataPageHeader', + 'class' => '\Flow\Parquet\ThriftModel\DataPageHeader', ], 6 => [ 'var' => 'index_page_header', 'isRequired' => false, 'type' => TType::STRUCT, - 'class' => '\Flow\Parquet\Thrift\IndexPageHeader', + 'class' => '\Flow\Parquet\ThriftModel\IndexPageHeader', ], 7 => [ 'var' => 'dictionary_page_header', 'isRequired' => false, 'type' => TType::STRUCT, - 'class' => '\Flow\Parquet\Thrift\DictionaryPageHeader', + 'class' => '\Flow\Parquet\ThriftModel\DictionaryPageHeader', ], 8 => [ 'var' => 'data_page_header_v2', 'isRequired' => false, 'type' => TType::STRUCT, - 'class' => '\Flow\Parquet\Thrift\DataPageHeaderV2', + 'class' => '\Flow\Parquet\ThriftModel\DataPageHeaderV2', ], ]; diff --git a/src/lib/parquet/src/Flow/Parquet/ThriftModel/RowGroup.php b/src/lib/parquet/src/Flow/Parquet/ThriftModel/RowGroup.php index 817c5e5c4f..ae4c23f498 100644 --- a/src/lib/parquet/src/Flow/Parquet/ThriftModel/RowGroup.php +++ b/src/lib/parquet/src/Flow/Parquet/ThriftModel/RowGroup.php @@ -24,7 +24,7 @@ class RowGroup 'etype' => TType::STRUCT, 'elem' => [ 'type' => TType::STRUCT, - 'class' => '\Flow\Parquet\Thrift\ColumnChunk', + 'class' => '\Flow\Parquet\ThriftModel\ColumnChunk', ], ], 2 => [ @@ -44,7 +44,7 @@ class RowGroup 'etype' => TType::STRUCT, 'elem' => [ 'type' => TType::STRUCT, - 'class' => '\Flow\Parquet\Thrift\SortingColumn', + 'class' => '\Flow\Parquet\ThriftModel\SortingColumn', ], ], 5 => [ diff --git a/src/lib/parquet/src/Flow/Parquet/ThriftModel/SchemaElement.php b/src/lib/parquet/src/Flow/Parquet/ThriftModel/SchemaElement.php index b5f223ab0b..525e6014f3 100644 --- a/src/lib/parquet/src/Flow/Parquet/ThriftModel/SchemaElement.php +++ b/src/lib/parquet/src/Flow/Parquet/ThriftModel/SchemaElement.php @@ -27,7 +27,7 @@ class SchemaElement 'var' => 'type', 'isRequired' => false, 'type' => TType::I32, - 'class' => '\Flow\Parquet\Thrift\Type', + 'class' => '\Flow\Parquet\ThriftModel\Type', ], 2 => [ 'var' => 'type_length', @@ -38,7 +38,7 @@ class SchemaElement 'var' => 'repetition_type', 'isRequired' => false, 'type' => TType::I32, - 'class' => '\Flow\Parquet\Thrift\FieldRepetitionType', + 'class' => '\Flow\Parquet\ThriftModel\FieldRepetitionType', ], 4 => [ 'var' => 'name', @@ -54,7 +54,7 @@ class SchemaElement 'var' => 'converted_type', 'isRequired' => false, 'type' => TType::I32, - 'class' => '\Flow\Parquet\Thrift\ConvertedType', + 'class' => '\Flow\Parquet\ThriftModel\ConvertedType', ], 7 => [ 'var' => 'scale', @@ -75,7 +75,7 @@ class SchemaElement 'var' => 'logicalType', 'isRequired' => false, 'type' => TType::STRUCT, - 'class' => '\Flow\Parquet\Thrift\LogicalType', + 'class' => '\Flow\Parquet\ThriftModel\LogicalType', ], ]; diff --git a/src/lib/parquet/src/Flow/Parquet/ThriftModel/TimeType.php b/src/lib/parquet/src/Flow/Parquet/ThriftModel/TimeType.php index 4e1d123a9d..24ef8e9c47 100644 --- a/src/lib/parquet/src/Flow/Parquet/ThriftModel/TimeType.php +++ b/src/lib/parquet/src/Flow/Parquet/ThriftModel/TimeType.php @@ -31,7 +31,7 @@ class TimeType 'var' => 'unit', 'isRequired' => true, 'type' => TType::STRUCT, - 'class' => '\Flow\Parquet\Thrift\TimeUnit', + 'class' => '\Flow\Parquet\ThriftModel\TimeUnit', ], ]; diff --git a/src/lib/parquet/src/Flow/Parquet/ThriftModel/TimeUnit.php b/src/lib/parquet/src/Flow/Parquet/ThriftModel/TimeUnit.php index 2f509bfac6..8367bda037 100644 --- a/src/lib/parquet/src/Flow/Parquet/ThriftModel/TimeUnit.php +++ b/src/lib/parquet/src/Flow/Parquet/ThriftModel/TimeUnit.php @@ -21,19 +21,19 @@ class TimeUnit 'var' => 'MILLIS', 'isRequired' => false, 'type' => TType::STRUCT, - 'class' => '\Flow\Parquet\Thrift\MilliSeconds', + 'class' => '\Flow\Parquet\ThriftModel\MilliSeconds', ], 2 => [ 'var' => 'MICROS', 'isRequired' => false, 'type' => TType::STRUCT, - 'class' => '\Flow\Parquet\Thrift\MicroSeconds', + 'class' => '\Flow\Parquet\ThriftModel\MicroSeconds', ], 3 => [ 'var' => 'NANOS', 'isRequired' => false, 'type' => TType::STRUCT, - 'class' => '\Flow\Parquet\Thrift\NanoSeconds', + 'class' => '\Flow\Parquet\ThriftModel\NanoSeconds', ], ]; diff --git a/src/lib/parquet/src/Flow/Parquet/ThriftModel/TimestampType.php b/src/lib/parquet/src/Flow/Parquet/ThriftModel/TimestampType.php index 4db030d8b0..2ad64ee927 100644 --- a/src/lib/parquet/src/Flow/Parquet/ThriftModel/TimestampType.php +++ b/src/lib/parquet/src/Flow/Parquet/ThriftModel/TimestampType.php @@ -31,7 +31,7 @@ class TimestampType 'var' => 'unit', 'isRequired' => true, 'type' => TType::STRUCT, - 'class' => '\Flow\Parquet\Thrift\TimeUnit', + 'class' => '\Flow\Parquet\ThriftModel\TimeUnit', ], ]; diff --git a/src/lib/parquet/src/Flow/Parquet/Writer.php b/src/lib/parquet/src/Flow/Parquet/Writer.php index f56b96fa2e..3dcfae9114 100644 --- a/src/lib/parquet/src/Flow/Parquet/Writer.php +++ b/src/lib/parquet/src/Flow/Parquet/Writer.php @@ -7,7 +7,6 @@ use Flow\Filesystem\DestinationStream; use Flow\Filesystem\Stream\NativeLocalDestinationStream; use Flow\Parquet\Engine\AdaptiveParquetEngine; -use Flow\Parquet\Engine\ArrowParquetEngine; use Flow\Parquet\Engine\PhpParquetEngine; use Flow\Parquet\Exception\InvalidArgumentException; use Flow\Parquet\Exception\RuntimeException; @@ -41,13 +40,6 @@ public function __construct( }; } - public static function arrow( - Compressions $compression = Compressions::SNAPPY, - Options $options = new Options(), - ): self { - return new self($compression, $options, new ArrowParquetEngine()); - } - public static function php(Compressions $compression = Compressions::SNAPPY, Options $options = new Options()): self { return new self($compression, $options, new PhpParquetEngine()); @@ -125,6 +117,15 @@ public function writeBatch(iterable $rows): void ($this->file ?? throw new RuntimeException('Writer is not open'))->writeBatch($rows); } + /** + * @param array> $columns by top-level column name, every list of one length; a schema column + * the array lacks is written as nulls, a key the schema lacks is ignored + */ + public function writeColumns(array $columns): void + { + ($this->file ?? throw new RuntimeException('Writer is not open'))->writeColumns($columns); + } + /** * @param array $row */ diff --git a/src/lib/parquet/src/Flow/Parquet/Writer/ColumnLists.php b/src/lib/parquet/src/Flow/Parquet/Writer/ColumnLists.php new file mode 100644 index 0000000000..95a412b6d5 --- /dev/null +++ b/src/lib/parquet/src/Flow/Parquet/Writer/ColumnLists.php @@ -0,0 +1,43 @@ +> $columns + * + * @throws InvalidArgumentException when the lists differ in length + */ + public function length(array $columns): int + { + $lengths = []; + + foreach ($columns as $name => $values) { + $lengths[$name] = count($values); + } + + if (count(array_unique($lengths)) > 1) { + $described = []; + + foreach ($lengths as $name => $length) { + $described[] = '"' . $name . '": ' . $length; + } + + throw new InvalidArgumentException( + 'writeColumns() takes lists of one length, got ' . implode(', ', $described), + ); + } + + return $lengths === [] ? 0 : max($lengths); + } +} diff --git a/src/lib/parquet/src/Flow/Parquet/Writer/RowGroupBuilder.php b/src/lib/parquet/src/Flow/Parquet/Writer/RowGroupBuilder.php index e8de322c4a..0520224f59 100644 --- a/src/lib/parquet/src/Flow/Parquet/Writer/RowGroupBuilder.php +++ b/src/lib/parquet/src/Flow/Parquet/Writer/RowGroupBuilder.php @@ -13,7 +13,9 @@ use Flow\Parquet\ParquetFile\Schema; use function array_chunk; +use function array_slice; use function count; +use function min; final class RowGroupBuilder { @@ -26,6 +28,11 @@ final class RowGroupBuilder private int $rowsCount = 0; + /** + * Rows of the row groups flushed before this one: the writer's index of this group's first row. + */ + private int $flushedRows = 0; + public function __construct( private readonly Schema $schema, private readonly Compressions $compression, @@ -62,7 +69,7 @@ public function addRows(array $rows): void $this->flushBuffer(); foreach (array_chunk($rows, $interval) as $chunk) { - $flatColumnsData = $this->shredder->shred($this->schema, $chunk); + $flatColumnsData = $this->shredder->shred($this->schema, $chunk, $this->flushedRows + $this->rowsCount); foreach ($flatColumnsData as $flatPath => $columnValues) { $this->columnChunkBuilders->addColumnByFlatPath($flatPath, $columnValues); @@ -76,6 +83,44 @@ public function addRows(array $rows): void } } + /** + * @param array> $columns by top-level column name + * @param int $count rows in every list + */ + public function addColumns(array $columns, int $count): void + { + /** @var int<1, max> $interval */ + $interval = $this->options->getInt(Option::PAGE_SIZE_CHECK_INTERVAL); + // rows still buffered by addRow() came first - parquet identifies a row by its position + $this->flushBuffer(); + + for ($offset = 0; $offset < $count; $offset += $interval) { + $length = min($interval, $count - $offset); + $slices = []; + + foreach ($columns as $name => $values) { + $slices[$name] = array_slice($values, $offset, $length); + } + + $flatColumnsData = $this->shredder->shredColumns( + $this->schema, + $slices, + $length, + $this->flushedRows + $this->rowsCount, + ); + + foreach ($flatColumnsData as $flatPath => $columnValues) { + $this->columnChunkBuilders->addColumnByFlatPath($flatPath, $columnValues); + } + + $this->rowsCount += $length; + + if ($this->columnChunkBuilders->isAnyPageFull()) { + $this->columnChunkBuilders->closePages(); + } + } + } + public function flush(int $fileOffset): RowGroupContainer { $this->flushBuffer(); @@ -84,6 +129,7 @@ public function flush(int $fileOffset): RowGroupContainer $offset = $fileOffset; $buffer = ''; $chunks = []; + $this->flushedRows += $rowsCount; $this->rowsCount = 0; foreach ($this->columnChunkBuilders->flush($offset) as $container) { @@ -115,7 +161,11 @@ private function flushBuffer(): void return; } - $flatColumnsData = $this->shredder->shred($this->schema, $this->rowBuffer); + $flatColumnsData = $this->shredder->shred( + $this->schema, + $this->rowBuffer, + $this->flushedRows + $this->rowsCount - count($this->rowBuffer), + ); foreach ($flatColumnsData as $flatPath => $columnValues) { $this->columnChunkBuilders->addColumnByFlatPath($flatPath, $columnValues); diff --git a/src/lib/parquet/tests/Flow/Parquet/Tests/Context/ColumnDoor.php b/src/lib/parquet/tests/Flow/Parquet/Tests/Context/ColumnDoor.php new file mode 100644 index 0000000000..a473ae67ed --- /dev/null +++ b/src/lib/parquet/tests/Flow/Parquet/Tests/Context/ColumnDoor.php @@ -0,0 +1,109 @@ +> $rows every row with the keys of the first + * + * @return array> + */ + public static function columns(array $rows): array + { + $columns = []; + + foreach (array_keys($rows[0] ?? []) as $name) { + $columns[(string) $name] = []; + } + + foreach ($rows as $row) { + // @mago-ignore analysis:mixed-assignment + foreach ($row as $name => $value) { + $columns[(string) $name][] = $value; + } + } + + return $columns; + } + + public static function open( + ParquetEngine $engine, + StringDestinationStream $stream, + Schema $schema, + Options $options = new Options(), + ): ParquetFileWriter { + return $engine->openForWrite($stream, $schema, Compressions::UNCOMPRESSED, $options); + } + + /** + * @return list> what the PHP reader reads back from $stream + */ + public static function read(StringDestinationStream $stream): array + { + /** @var non-empty-string $bytes */ + $bytes = $stream->content(); + + return iterator_to_array(MemoryParquetFile::read(Reader::php(), $bytes)->values(), false); + } + + /** + * A flat, a list, a map and a structure column; every third value is null. + * + * @return list> + */ + public static function rows(int $count): array + { + $rows = []; + + for ($i = 0; $i < $count; $i++) { + $rows[] = [ + 'id' => $i, + 'name' => ($i % 3) === 0 ? null : 'name-' . $i, + 'tags' => ($i % 3) === 1 ? null : ['t' . $i, 'u' . $i], + 'counts' => ($i % 3) === 2 ? null : ['a' => $i, 'b' => $i + 1], + 'point' => ($i % 3) === 0 ? null : ['x' => $i, 'y' => ($i % 2) === 0 ? null : $i * 2], + ]; + } + + return $rows; + } + + public static function schema(): Schema + { + return Schema::with( + FlatColumn::int64('id'), + FlatColumn::string('name'), + NestedColumn::list('tags', ListElement::string()), + NestedColumn::map('counts', MapKey::string(), MapValue::int32()), + NestedColumn::structure('point', [FlatColumn::int32('x'), FlatColumn::int32('y')]), + ); + } + + public static function stream(): StringDestinationStream + { + return new StringDestinationStream(path('memory://file.parquet')); + } +} diff --git a/src/lib/parquet/tests/Flow/Parquet/Tests/Context/EveryType.php b/src/lib/parquet/tests/Flow/Parquet/Tests/Context/EveryType.php new file mode 100644 index 0000000000..ee88a7d3d3 --- /dev/null +++ b/src/lib/parquet/tests/Flow/Parquet/Tests/Context/EveryType.php @@ -0,0 +1,109 @@ +> + */ + public static function rows(): array + { + return [ + [ + 'boolean' => true, + 'int32' => -2_147_483_648, + 'int64' => PHP_INT_MAX, + 'float' => 0.1, + 'double' => 1 / 3, + 'string' => 'zażółć', + 'json' => '{"a":1}', + 'enum' => 'RED', + 'uuid' => '0190a6b0-6b1f-7c5e-9a3b-0123456789ab', + 'date' => new DateTimeImmutable('1969-12-31 23:00:00', new DateTimeZone('Europe/Warsaw')), + 'datetime' => new DateTimeImmutable('1960-01-01 12:34:56.789012', new DateTimeZone('America/New_York')), + 'time' => new DateInterval('PT25H3M4S'), + 'decimal' => 12_345_678.91, + 'decimal38' => 12_345_678_901_234_567_890.123456789, + 'fixed4' => "\x00\x01\x02\x03", + 'list_int' => [1, null, 3], + 'list_datetime' => [new DateTimeImmutable('2024-02-29 00:00:00.5')], + 'map_string_date' => ['a' => new DateTimeImmutable('2000-01-01'), 'b' => null], + 'struct' => [ + 'id' => '0190a6b0-6b1f-7c5e-9a3b-0123456789ab', + 'at' => new DateInterval('PT0S'), + 'amount' => -0.01, + ], + ], + array_fill_keys(array_map(static fn($column) => $column->name(), self::schema()->columns()), null), + [ + 'boolean' => false, + 'int32' => 0, + 'int64' => -1, + 'float' => -2.5e10, + 'double' => -0.0, + 'string' => '', + 'json' => '[]', + 'enum' => '', + 'uuid' => '00000000-0000-0000-0000-000000000000', + 'date' => new DateTimeImmutable('2262-04-11'), + 'datetime' => new DateTimeImmutable('@0'), + 'time' => new DateInterval('PT23H59M59S'), + 'decimal' => -99_999_999.99, + 'decimal38' => 0.000000001, + 'fixed4' => "\xFF\xFF\xFF\xFF", + 'list_int' => [], + 'list_datetime' => null, + 'map_string_date' => [], + 'struct' => ['id' => null, 'at' => null, 'amount' => null], + ], + ]; + } + + public static function schema(): Schema + { + return Schema::with( + FlatColumn::boolean('boolean'), + FlatColumn::int32('int32'), + FlatColumn::int64('int64'), + FlatColumn::float('float'), + FlatColumn::double('double'), + FlatColumn::string('string'), + FlatColumn::json('json'), + FlatColumn::enum('enum'), + FlatColumn::uuid('uuid'), + FlatColumn::date('date'), + FlatColumn::dateTime('datetime'), + FlatColumn::time('time'), + FlatColumn::decimal('decimal', 10, 2), + FlatColumn::decimal('decimal38', 38, 9), + FlatColumn::fixedSizeByteArray('fixed4', 4), + NestedColumn::list('list_int', ListElement::int64()), + NestedColumn::list('list_datetime', ListElement::datetime()), + NestedColumn::map('map_string_date', MapKey::string(), MapValue::date()), + NestedColumn::struct('struct', [ + FlatColumn::uuid('id'), + FlatColumn::time('at'), + FlatColumn::decimal('amount'), + ]), + ); + } +} diff --git a/src/lib/parquet/tests/Flow/Parquet/Tests/Context/FlatColumnPages.php b/src/lib/parquet/tests/Flow/Parquet/Tests/Context/FlatColumnPages.php new file mode 100644 index 0000000000..0eb8d501b1 --- /dev/null +++ b/src/lib/parquet/tests/Flow/Parquet/Tests/Context/FlatColumnPages.php @@ -0,0 +1,133 @@ +> $rows + * + * @return list + */ + public static function merged(Schema $schema, array $rows): array + { + $pages = []; + + foreach (self::shredded($schema, $rows) as $written) { + $pages[] = self::page( + $written, + $written->repetitionLevels(), + $written->definitionLevels(), + $written->values(), + ); + } + + return $pages; + } + + /** + * @param array> $rows + * + * @return list + */ + public static function paged(Schema $schema, array $rows, int $firstPageRows): array + { + $paged = []; + + foreach (self::shredded($schema, $rows) as $written) { + $paged[] = new PagedFlatColumnValues($written->column, self::cut($written, $firstPageRows)); + } + + return $paged; + } + + /** + * @param array> $rows + * + * @return array + */ + public static function shredded(Schema $schema, array $rows): array + { + return (new DremelShredder(new ColumnDataValidator(), DataConverter::initialize(Options::default())))->shred( + $schema, + $rows, + 0, + ); + } + + /** + * @return Generator + */ + public static function cut(WriteFlatColumnValues $written, int $firstPageRows): Generator + { + $at = 0; + $rows = 0; + + foreach ($written->repetitionLevels() as $level) { + if ($level === 0 && $rows++ === $firstPageRows) { + break; + } + + $at++; + } + + $maxDefinitionLevel = $written->column->repetitions()->maxDefinitionLevel(); + $values = count(array_filter( + array_slice($written->definitionLevels(), 0, $at), + static fn(int $level): bool => $level === $maxDefinitionLevel, + )); + + yield self::page( + $written, + array_slice($written->repetitionLevels(), 0, $at), + array_slice($written->definitionLevels(), 0, $at), + array_slice($written->values(), 0, $values), + ); + yield self::page( + $written, + array_slice($written->repetitionLevels(), $at), + array_slice($written->definitionLevels(), $at), + array_slice($written->values(), $values), + ); + } + + /** + * @param array $repetitionLevels + * @param array $definitionLevels + * @param array $values + */ + public static function page( + WriteFlatColumnValues $written, + array $repetitionLevels, + array $definitionLevels, + array $values, + ): ReadFlatColumnValues { + return new ReadFlatColumnValues( + $written->column, + (static function () use ($values): Generator { + yield from $values; + })(), + $repetitionLevels, + $definitionLevels, + ); + } +} diff --git a/src/lib/parquet/tests/Flow/Parquet/Tests/Context/MemoryParquetFile.php b/src/lib/parquet/tests/Flow/Parquet/Tests/Context/MemoryParquetFile.php new file mode 100644 index 0000000000..f489726394 --- /dev/null +++ b/src/lib/parquet/tests/Flow/Parquet/Tests/Context/MemoryParquetFile.php @@ -0,0 +1,73 @@ +set(Option::ROW_GROUP_SIZE_BYTES, 1) + ->set(Option::ROW_GROUP_SIZE_CHECK_INTERVAL, 2) + ->set(Option::PAGE_SIZE_BYTES, 1) + ->set(Option::PAGE_SIZE_CHECK_INTERVAL, 1), + ), + Schema::with(FlatColumn::int64('id'), NestedColumn::list('tags', ListElement::string())), + array_map(static fn(int $id): array => ['id' => $id, 'tags' => ['t' . $id]], range(1, 6)), + ); + } + + /** + * @param non-empty-string $bytes + * + * @return ParquetFile + */ + public static function read(Reader $reader, string $bytes): ParquetFile + { + return $reader->readStream(new MemorySourceStream($bytes)); + } + + /** + * @param iterable> $rows + * + * @return non-empty-string + */ + public static function written(Writer $writer, Schema $schema, iterable $rows): string + { + $stream = new StringDestinationStream(path('memory://file.parquet')); + $writer->writeStream($stream, $schema, $rows); + $content = $stream->content(); + + return $content !== '' ? $content : throw new RuntimeException('Nothing was written'); + } +} diff --git a/src/lib/parquet/tests/Flow/Parquet/Tests/Context/ParquetRows.php b/src/lib/parquet/tests/Flow/Parquet/Tests/Context/ParquetRows.php new file mode 100644 index 0000000000..d07045df5e --- /dev/null +++ b/src/lib/parquet/tests/Flow/Parquet/Tests/Context/ParquetRows.php @@ -0,0 +1,50 @@ + $file + * @param int<1, max> $batchSize + * @param list $columns + * + * @return list> + */ + public static function read( + ParquetFile $file, + int $batchSize, + array $columns = [], + ?int $limit = null, + ?int $offset = null, + ): array { + $rows = []; + + foreach ($file->columns($batchSize, $columns, $limit, $offset) as $chunk) { + $names = array_keys($chunk); + + for ($i = 0, $count = count($chunk[$names[0]]); $i < $count; $i++) { + $row = []; + + foreach ($names as $name) { + $row[$name] = $chunk[$name][$i]; + } + + $rows[] = $row; + } + } + + return $rows; + } +} diff --git a/src/lib/parquet/tests/Flow/Parquet/Tests/Context/WriteAcceptance.php b/src/lib/parquet/tests/Flow/Parquet/Tests/Context/WriteAcceptance.php new file mode 100644 index 0000000000..879d490e66 --- /dev/null +++ b/src/lib/parquet/tests/Flow/Parquet/Tests/Context/WriteAcceptance.php @@ -0,0 +1,190 @@ + true, + 'int 1' => 1, + 'int -1' => -1, + 'int 4000000000' => 4_000_000_000, + 'int PHP_INT_MAX' => PHP_INT_MAX, + 'float 1.5' => 1.5, + 'float 1.0' => 1.0, + 'float 12345678.915' => 12_345_678.915, + 'string "1"' => '1', + 'string "1.25"' => '1.25', + 'string "abc"' => 'abc', + 'string 36 uuid' => '0190a6b0-6b1f-7c5e-9a3b-0123456789ab', + 'string 16 bytes' => str_repeat("\x01", 16), + 'string 4 bytes' => "\x00\x01\x02\x03", + 'string 5 bytes' => '12345', + 'string \xFF\xFE' => "\xFF\xFE", + 'DateTimeImmutable Warsaw' => new DateTimeImmutable( + '2024-02-29 23:30:00.123456', + new DateTimeZone('Europe/Warsaw'), + ), + 'DateInterval PT25H' => new DateInterval('PT25H'), + 'DateInterval P1M' => new DateInterval('P1M'), + 'DateInterval -PT1H' => (new DateTimeImmutable('@3600'))->diff(new DateTimeImmutable('@0')), + 'toString() uuid object' => new ToStringUuid('0190a6b0-6b1f-7c5e-9a3b-0123456789ab'), + 'array [1]' => [1], + 'array [a=>1]' => ['a' => 1], + 'null' => null, + default => throw new InvalidArgumentException("Unknown input \"{$name}\""), + }; + } + + /** + * @return list + */ + public static function inputs(): array + { + return [ + 'true', + 'int 1', + 'int -1', + 'int 4000000000', + 'int PHP_INT_MAX', + 'float 1.5', + 'float 1.0', + 'float 12345678.915', + 'string "1"', + 'string "1.25"', + 'string "abc"', + 'string 36 uuid', + 'string 16 bytes', + 'string 4 bytes', + 'string 5 bytes', + 'string \xFF\xFE', + 'DateTimeImmutable Warsaw', + 'DateInterval PT25H', + 'DateInterval P1M', + 'DateInterval -PT1H', + 'toString() uuid object', + 'array [1]', + 'array [a=>1]', + 'null', + ]; + } + + public static function target(string $name): Column + { + return match ($name) { + 'BOOLEAN' => FlatColumn::boolean('c'), + 'INT32' => FlatColumn::int32('c'), + 'INT32 INT_8' => new FlatColumn('c', PhysicalType::INT32, ConvertedType::INT_8), + 'INT32 UINT_32' => new FlatColumn('c', PhysicalType::INT32, ConvertedType::UINT_32), + 'INT64' => FlatColumn::int64('c'), + 'INT64 REQUIRED' => FlatColumn::int64('c', Repetition::REQUIRED), + 'INT64 UINT_64' => new FlatColumn('c', PhysicalType::INT64, ConvertedType::UINT_64), + 'FLOAT' => FlatColumn::float('c'), + 'DOUBLE' => FlatColumn::double('c'), + 'STRING' => FlatColumn::string('c'), + 'JSON' => FlatColumn::json('c'), + 'ENUM' => FlatColumn::enum('c'), + 'BYTE_ARRAY' => new FlatColumn('c', PhysicalType::BYTE_ARRAY), + 'UUID' => FlatColumn::uuid('c'), + 'FIXED(4)' => FlatColumn::fixedSizeByteArray('c', 4), + 'DECIMAL(10,2)' => FlatColumn::decimal('c', 10, 2), + 'DATE' => FlatColumn::date('c'), + 'TIMESTAMP MILLIS' => new FlatColumn( + 'c', + PhysicalType::INT64, + logicalType: new LogicalType( + LogicalType::TIMESTAMP, + timestamp: new Timestamp(true, TimeUnit::MILLISECONDS), + ), + ), + 'TIMESTAMP MICROS' => FlatColumn::dateTime('c'), + 'TIMESTAMP NANOS' => new FlatColumn( + 'c', + PhysicalType::INT64, + logicalType: new LogicalType( + LogicalType::TIMESTAMP, + timestamp: new Timestamp(true, TimeUnit::NANOSECONDS), + ), + ), + 'TIME MILLIS' => new FlatColumn( + 'c', + PhysicalType::INT32, + logicalType: new LogicalType(LogicalType::TIME, time: new Time(false, TimeUnit::MILLISECONDS)), + ), + 'TIME MICROS' => FlatColumn::time('c'), + 'TIME NANOS' => new FlatColumn( + 'c', + PhysicalType::INT64, + logicalType: new LogicalType(LogicalType::TIME, time: new Time(false, TimeUnit::NANOSECONDS)), + ), + 'LIST' => NestedColumn::list('c', ListElement::int64()), + 'MAP' => NestedColumn::map('c', MapKey::string(), MapValue::int64()), + 'STRUCT{a INT64}' => NestedColumn::struct('c', [FlatColumn::int64('a')]), + default => throw new InvalidArgumentException("Unknown target \"{$name}\""), + }; + } + + /** + * @return list + */ + public static function targets(): array + { + return [ + 'BOOLEAN', + 'INT32', + 'INT32 INT_8', + 'INT32 UINT_32', + 'INT64', + 'INT64 REQUIRED', + 'INT64 UINT_64', + 'FLOAT', + 'DOUBLE', + 'STRING', + 'JSON', + 'ENUM', + 'BYTE_ARRAY', + 'UUID', + 'FIXED(4)', + 'DECIMAL(10,2)', + 'DATE', + 'TIMESTAMP MILLIS', + 'TIMESTAMP MICROS', + 'TIMESTAMP NANOS', + 'TIME MILLIS', + 'TIME MICROS', + 'TIME NANOS', + 'LIST', + 'MAP', + 'STRUCT{a INT64}', + ]; + } +} diff --git a/src/lib/parquet/tests/Flow/Parquet/Tests/Double/ReadCountingSourceStream.php b/src/lib/parquet/tests/Flow/Parquet/Tests/Double/ReadCountingSourceStream.php new file mode 100644 index 0000000000..9094fb9946 --- /dev/null +++ b/src/lib/parquet/tests/Flow/Parquet/Tests/Double/ReadCountingSourceStream.php @@ -0,0 +1,72 @@ +closed = true; + $this->inner->close(); + } + + public function content(): string + { + return $this->inner->content(); + } + + public function isOpen(): bool + { + return $this->inner->isOpen(); + } + + public function iterate(int $length = 1): Generator + { + yield from $this->inner->iterate($length); + } + + public function path(): Path + { + return $this->inner->path(); + } + + public function read(int $length, int $offset): string + { + $read = $this->inner->read($length, $offset); + $this->reads++; + $this->bytes += strlen($read); + + return $read; + } + + public function readLines(string $separator = "\n", ?int $length = null): Generator + { + yield from $this->inner->readLines($separator, $length); + } + + public function size(): ?int + { + return $this->inner->size(); + } +} diff --git a/src/lib/parquet/tests/Flow/Parquet/Tests/Double/SpyParquetFileReader.php b/src/lib/parquet/tests/Flow/Parquet/Tests/Double/SpyParquetFileReader.php new file mode 100644 index 0000000000..4e8282f569 --- /dev/null +++ b/src/lib/parquet/tests/Flow/Parquet/Tests/Double/SpyParquetFileReader.php @@ -0,0 +1,64 @@ +, int, ?int, ?int}> + */ + public array $reads = []; + + /** + * @param list>> $chunks + */ + public function __construct( + private readonly Schema $schema, + private readonly array $chunks = [], + ) {} + + public function close(): void + { + $this->closes++; + } + + public function metadata(): Metadata + { + return new Metadata($this->schema, new RowGroups([]), 0, 1, null); + } + + public function readColumns(array $columns, int $batchSize, ?int $limit, ?int $offset): Generator + { + $this->reads[] = [$columns, $batchSize, $limit, $offset]; + + yield from $this->chunks; + } + + public function rowsNumber(): int + { + return 0; + } + + public function schema(): Schema + { + return $this->schema; + } + + public function totalByteSize(): int + { + return 0; + } +} diff --git a/src/lib/parquet/tests/Flow/Parquet/Tests/Double/StringableUuid.php b/src/lib/parquet/tests/Flow/Parquet/Tests/Double/StringableUuid.php new file mode 100644 index 0000000000..57e1a8d157 --- /dev/null +++ b/src/lib/parquet/tests/Flow/Parquet/Tests/Double/StringableUuid.php @@ -0,0 +1,22 @@ +text; + } +} diff --git a/src/lib/parquet/tests/Flow/Parquet/Tests/Double/ToStringUuid.php b/src/lib/parquet/tests/Flow/Parquet/Tests/Double/ToStringUuid.php new file mode 100644 index 0000000000..7d617fe2b6 --- /dev/null +++ b/src/lib/parquet/tests/Flow/Parquet/Tests/Double/ToStringUuid.php @@ -0,0 +1,20 @@ +text; + } +} diff --git a/src/lib/parquet/tests/Flow/Parquet/Tests/Integration/Engine/ArrowParquetEngineReadTest.php b/src/lib/parquet/tests/Flow/Parquet/Tests/Integration/Engine/ArrowParquetEngineReadTest.php deleted file mode 100644 index dcb8c23108..0000000000 --- a/src/lib/parquet/tests/Flow/Parquet/Tests/Integration/Engine/ArrowParquetEngineReadTest.php +++ /dev/null @@ -1,301 +0,0 @@ - 1, 'name' => 'Alice', 'active' => true, 'score' => 99.5], - ['id' => 2, 'name' => 'Bob', 'active' => false, 'score' => 87.3], - ['id' => 3, 'name' => 'Charlie', 'active' => true, 'score' => 92.1], - ]; - - (new Writer())->write($path, $schema, $inputData); - - $engine = new ArrowParquetEngine(); - $parquetFile = (new Reader())->read($path); - $result = iterator_to_array($engine->readValues( - NativeLocalSourceStream::open(path_real($path)), - $parquetFile->schema(), - )); - - static::assertCount(3, $result); - static::assertSame(1, $result[0]['id']); - static::assertSame('Alice', $result[0]['name']); - static::assertTrue($result[0]['active']); - static::assertEqualsWithDelta(99.5, $result[0]['score'], 0.001); - } - - public function test_read_nested_lists(): void - { - $path = TestParquetFile::path($this); - - $schema = Schema::with( - FlatColumn::int32('id', Repetition::REQUIRED), - NestedColumn::list('tags', ListElement::string()), - ); - - $inputData = [ - ['id' => 1, 'tags' => ['php', 'rust']], - ['id' => 2, 'tags' => ['python']], - ]; - - (new Writer())->write($path, $schema, $inputData); - - $engine = new ArrowParquetEngine(); - $parquetFile = (new Reader())->read($path); - $result = iterator_to_array($engine->readValues( - NativeLocalSourceStream::open(path_real($path)), - $parquetFile->schema(), - )); - - static::assertCount(2, $result); - static::assertSame(['php', 'rust'], $result[0]['tags']); - static::assertSame(['python'], $result[1]['tags']); - } - - public function test_read_nested_maps(): void - { - $path = TestParquetFile::path($this); - - $schema = Schema::with( - FlatColumn::int32('id', Repetition::REQUIRED), - NestedColumn::map('metadata', MapKey::string(), MapValue::int32()), - ); - - $inputData = [ - ['id' => 1, 'metadata' => ['score' => 100, 'level' => 5]], - ['id' => 2, 'metadata' => ['score' => 200]], - ]; - - (new Writer())->write($path, $schema, $inputData); - - $engine = new ArrowParquetEngine(); - $parquetFile = (new Reader())->read($path); - $result = iterator_to_array($engine->readValues( - NativeLocalSourceStream::open(path_real($path)), - $parquetFile->schema(), - )); - - static::assertCount(2, $result); - /** @var array $metadata */ - $metadata = $result[0]['metadata']; - static::assertIsArray($metadata); - static::assertSame(100, $metadata['score']); - static::assertSame(5, $metadata['level']); - } - - public function test_read_nested_structs(): void - { - $path = TestParquetFile::path($this); - - $schema = Schema::with( - FlatColumn::int32('id', Repetition::REQUIRED), - NestedColumn::struct('address', [ - FlatColumn::string('city'), - FlatColumn::int32('zip'), - ]), - ); - - $inputData = [ - ['id' => 1, 'address' => ['city' => 'Berlin', 'zip' => 10115]], - ['id' => 2, 'address' => ['city' => 'Warsaw', 'zip' => 00001]], - ]; - - (new Writer())->write($path, $schema, $inputData); - - $engine = new ArrowParquetEngine(); - $parquetFile = (new Reader())->read($path); - $result = iterator_to_array($engine->readValues( - NativeLocalSourceStream::open(path_real($path)), - $parquetFile->schema(), - )); - - static::assertCount(2, $result); - /** @var array $address */ - $address = $result[0]['address']; - static::assertIsArray($address); - static::assertSame('Berlin', $address['city']); - static::assertSame(10115, $address['zip']); - } - - public function test_read_nested_uuid_as_canonical_strings(): void - { - $path = TestParquetFile::path($this); - $uuid = 'f6d6e0e8-4b7e-4b0e-8d7a-ff0a0c9c9a5a'; - - $schema = Schema::with( - FlatColumn::uuid('top_uuid'), - NestedColumn::struct('body', [FlatColumn::uuid('id'), FlatColumn::json('data')]), - NestedColumn::list('uuid_list', ListElement::uuid()), - ); - - (new Writer())->write($path, $schema, [[ - 'top_uuid' => $uuid, - 'body' => ['id' => $uuid, 'data' => '{"a":1}'], - 'uuid_list' => [$uuid, $uuid], - ]]); - - $engine = new ArrowParquetEngine(); - $parquetFile = (new Reader())->read($path); - $result = iterator_to_array($engine->readValues( - NativeLocalSourceStream::open(path_real($path)), - $parquetFile->schema(), - )); - - static::assertSame($uuid, $result[0]['top_uuid']); - static::assertSame(['id' => $uuid, 'data' => '{"a":1}'], $result[0]['body']); - static::assertSame([$uuid, $uuid], $result[0]['uuid_list']); - } - - public function test_read_with_column_projection(): void - { - $path = TestParquetFile::path($this); - - $schema = Schema::with( - FlatColumn::int32('id', Repetition::REQUIRED), - FlatColumn::string('name'), - FlatColumn::string('email'), - ); - - $inputData = [ - ['id' => 1, 'name' => 'Alice', 'email' => 'alice@example.com'], - ['id' => 2, 'name' => 'Bob', 'email' => 'bob@example.com'], - ]; - - (new Writer())->write($path, $schema, $inputData); - - $engine = new ArrowParquetEngine(); - $parquetFile = (new Reader())->read($path); - $result = iterator_to_array($engine->readValues( - NativeLocalSourceStream::open(path_real($path)), - $parquetFile->schema(), - ['id', 'name'], - )); - - static::assertCount(2, $result); - static::assertArrayHasKey('id', $result[0]); - static::assertArrayHasKey('name', $result[0]); - static::assertArrayNotHasKey('email', $result[0]); - } - - public function test_read_with_limit(): void - { - $path = TestParquetFile::path($this); - - $schema = Schema::with(FlatColumn::int32('id', Repetition::REQUIRED)); - - $inputData = array_map(static fn(int $i): array => ['id' => $i], range(1, 100)); - - (new Writer())->write($path, $schema, $inputData); - - $engine = new ArrowParquetEngine(); - $parquetFile = (new Reader())->read($path); - $result = iterator_to_array($engine->readValues( - NativeLocalSourceStream::open(path_real($path)), - $parquetFile->schema(), - [], - 10, - )); - - static::assertCount(10, $result); - static::assertSame(1, $result[0]['id']); - static::assertSame(10, $result[9]['id']); - } - - public function test_read_with_limit_and_offset(): void - { - $path = TestParquetFile::path($this); - - $schema = Schema::with(FlatColumn::int32('id', Repetition::REQUIRED)); - - $inputData = array_map(static fn(int $i): array => ['id' => $i], range(1, 100)); - - (new Writer())->write($path, $schema, $inputData); - - $engine = new ArrowParquetEngine(); - $parquetFile = (new Reader())->read($path); - $result = iterator_to_array($engine->readValues( - NativeLocalSourceStream::open(path_real($path)), - $parquetFile->schema(), - [], - 5, - 10, - )); - - static::assertCount(5, $result); - static::assertSame(11, $result[0]['id']); - static::assertSame(15, $result[4]['id']); - } - - public function test_read_with_offset(): void - { - $path = TestParquetFile::path($this); - - $schema = Schema::with(FlatColumn::int32('id', Repetition::REQUIRED)); - - $inputData = array_map(static fn(int $i): array => ['id' => $i], range(1, 100)); - - (new Writer())->write($path, $schema, $inputData); - - $engine = new ArrowParquetEngine(); - $parquetFile = (new Reader())->read($path); - $result = iterator_to_array($engine->readValues( - NativeLocalSourceStream::open(path_real($path)), - $parquetFile->schema(), - [], - null, - 50, - )); - - static::assertCount(50, $result); - static::assertSame(51, $result[0]['id']); - } -} diff --git a/src/lib/parquet/tests/Flow/Parquet/Tests/Integration/Engine/ArrowParquetEngineWriteTest.php b/src/lib/parquet/tests/Flow/Parquet/Tests/Integration/Engine/ArrowParquetEngineWriteTest.php deleted file mode 100644 index d7cc6bafe8..0000000000 --- a/src/lib/parquet/tests/Flow/Parquet/Tests/Integration/Engine/ArrowParquetEngineWriteTest.php +++ /dev/null @@ -1,150 +0,0 @@ - 1, 'name' => 'Alice', 'active' => true, 'score' => 99.5], - ['id' => 2, 'name' => 'Bob', 'active' => false, 'score' => 87.3], - ['id' => 3, 'name' => 'Charlie', 'active' => true, 'score' => 92.1], - ]; - - $engine = new ArrowParquetEngine(); - $stream = NativeLocalDestinationStream::openBlank(path($path)); - $engine->writeRows($stream, $schema, Compressions::SNAPPY, new Options(), $inputData); - - $result = iterator_to_array( - (new Reader()) - ->read($path) - ->values(), - ); - - static::assertCount(3, $result); - static::assertSame(1, $result[0]['id']); - static::assertSame('Alice', $result[0]['name']); - static::assertTrue($result[0]['active']); - static::assertEqualsWithDelta(99.5, $result[0]['score'], 0.001); - } - - public function test_write_nested_types(): void - { - $path = TestParquetFile::path($this); - - $schema = Schema::with( - FlatColumn::int32('id', Repetition::REQUIRED), - NestedColumn::list('tags', ListElement::string()), - NestedColumn::struct('address', [ - FlatColumn::string('city'), - ]), - ); - - $inputData = [ - ['id' => 1, 'tags' => ['php', 'rust'], 'address' => ['city' => 'Berlin']], - ['id' => 2, 'tags' => ['python'], 'address' => ['city' => 'Warsaw']], - ]; - - $engine = new ArrowParquetEngine(); - $stream = NativeLocalDestinationStream::openBlank(path($path)); - $engine->writeRows($stream, $schema, Compressions::SNAPPY, new Options(), $inputData); - - $result = iterator_to_array( - (new Reader()) - ->read($path) - ->values(), - ); - - static::assertCount(2, $result); - static::assertSame(1, $result[0]['id']); - } - - public function test_write_with_gzip_compression(): void - { - $path = TestParquetFile::path($this); - - $schema = Schema::with(FlatColumn::int32('id', Repetition::REQUIRED), FlatColumn::string('data')); - - $inputData = array_map(static fn(int $i): array => ['id' => $i, 'data' => 'value_' . $i], range(1, 20)); - - $engine = new ArrowParquetEngine(); - $stream = NativeLocalDestinationStream::openBlank(path($path)); - $engine->writeRows($stream, $schema, Compressions::GZIP, new Options(), $inputData); - - $result = iterator_to_array( - (new Reader()) - ->read($path) - ->values(), - ); - - static::assertCount(20, $result); - } - - public function test_write_with_snappy_compression(): void - { - $path = TestParquetFile::path($this); - - $schema = Schema::with(FlatColumn::int32('id', Repetition::REQUIRED), FlatColumn::string('data')); - - $inputData = array_map(static fn(int $i): array => ['id' => $i, 'data' => str_repeat('a', 100)], range(1, 50)); - - $engine = new ArrowParquetEngine(); - $stream = NativeLocalDestinationStream::openBlank(path($path)); - $engine->writeRows($stream, $schema, Compressions::SNAPPY, new Options(), $inputData); - - $result = iterator_to_array( - (new Reader()) - ->read($path) - ->values(), - ); - - static::assertCount(50, $result); - static::assertSame(1, $result[0]['id']); - } -} diff --git a/src/lib/parquet/tests/Flow/Parquet/Tests/Integration/Engine/CrossEngineTest.php b/src/lib/parquet/tests/Flow/Parquet/Tests/Integration/Engine/CrossEngineTest.php index a019a73770..621e61217e 100644 --- a/src/lib/parquet/tests/Flow/Parquet/Tests/Integration/Engine/CrossEngineTest.php +++ b/src/lib/parquet/tests/Flow/Parquet/Tests/Integration/Engine/CrossEngineTest.php @@ -6,8 +6,8 @@ use Flow\Filesystem\Stream\NativeLocalDestinationStream; use Flow\Filesystem\Stream\NativeLocalSourceStream; -use Flow\Parquet\Engine\ArrowParquetEngine; use Flow\Parquet\Engine\PhpParquetEngine; +use Flow\Parquet\Engine\RustParquetEngine; use Flow\Parquet\Options; use Flow\Parquet\ParquetFile\Compressions; use Flow\Parquet\ParquetFile\Schema; @@ -30,7 +30,7 @@ final class CrossEngineTest extends TestCase protected function setUp(): void { if (!extension_loaded('arrow')) { - self::markTestSkipped('Arrow extension is not loaded'); + self::markTestSkipped('arrow is not loaded'); } } @@ -54,19 +54,25 @@ public function test_arrow_write_arrow_read_roundtrip(): void ['id' => 2, 'label' => 'second', 'big_number' => 2_000_000_000_000], ]; - $engine = new ArrowParquetEngine(); + $engine = new RustParquetEngine(); $writeStream = NativeLocalDestinationStream::openBlank(path($path)); $engine->writeRows($writeStream, $schema, Compressions::SNAPPY, new Options(), $inputData); - - $parquetFile = (new Reader())->read($path); $readStream = NativeLocalSourceStream::open(path_real($path)); - $result = iterator_to_array($engine->readValues($readStream, $parquetFile->schema())); + $chunks = iterator_to_array( + $engine->openForRead($readStream)->readColumns( + ['id', 'label', 'big_number'], + batchSize: 2, + limit: null, + offset: null, + ), + false, + ); - static::assertCount(2, $result); - static::assertSame(1, $result[0]['id']); - static::assertSame('first', $result[0]['label']); - static::assertSame(1_000_000_000_000, $result[0]['big_number']); + static::assertSame( + [['id' => [1, 2], 'label' => ['first', 'second'], 'big_number' => [1_000_000_000_000, 2_000_000_000_000]]], + $chunks, + ); } public function test_arrow_write_php_read(): void @@ -84,7 +90,7 @@ public function test_arrow_write_php_read(): void ['id' => 20, 'name' => 'beta', 'active' => false], ]; - $engine = new ArrowParquetEngine(); + $engine = new RustParquetEngine(); $stream = NativeLocalDestinationStream::openBlank(path($path)); $engine->writeRows($stream, $schema, Compressions::SNAPPY, new Options(), $inputData); @@ -145,16 +151,23 @@ public function test_php_write_arrow_read(): void (new Writer(engine: new PhpParquetEngine()))->write($path, $schema, $inputData); - $engine = new ArrowParquetEngine(); - $parquetFile = (new Reader())->read($path); - $result = iterator_to_array($engine->readValues( - NativeLocalSourceStream::open(path_real($path)), - $parquetFile->schema(), - )); + $engine = new RustParquetEngine(); + $chunks = iterator_to_array( + $engine->openForRead(NativeLocalSourceStream::open(path_real($path)))->readColumns( + ['id', 'name', 'value'], + batchSize: 2, + limit: null, + offset: null, + ), + false, + ); - static::assertCount(3, $result); - static::assertSame(1, $result[0]['id']); - static::assertSame('one', $result[0]['name']); - static::assertEqualsWithDelta(1.1, $result[0]['value'], 0.001); + static::assertSame( + [ + ['id' => [1, 2], 'name' => ['one', 'two'], 'value' => [1.1, 2.2]], + ['id' => [3], 'name' => ['three'], 'value' => [3.3]], + ], + $chunks, + ); } } diff --git a/src/lib/parquet/tests/Flow/Parquet/Tests/Integration/Engine/EngineParityTest.php b/src/lib/parquet/tests/Flow/Parquet/Tests/Integration/Engine/EngineParityTest.php index f96dc7af1d..6e26df80fe 100644 --- a/src/lib/parquet/tests/Flow/Parquet/Tests/Integration/Engine/EngineParityTest.php +++ b/src/lib/parquet/tests/Flow/Parquet/Tests/Integration/Engine/EngineParityTest.php @@ -4,6 +4,7 @@ namespace Flow\Parquet\Tests\Integration\Engine; +use Flow\Parquet\Engine\RustParquetEngine; use Flow\Parquet\ParquetFile\Schema; use Flow\Parquet\ParquetFile\Schema\FlatColumn; use Flow\Parquet\Reader; @@ -21,7 +22,7 @@ final class EngineParityTest extends TestCase protected function setUp(): void { if (!extension_loaded('arrow')) { - self::markTestSkipped('Arrow extension is not loaded'); + self::markTestSkipped('arrow is not loaded'); } } @@ -50,7 +51,11 @@ public function test_both_engines_decode_the_same_float32_bytes_identically(): v ]); static::assertSame( - iterator_to_array(Reader::arrow()->read($path)->values()), + iterator_to_array( + (new Reader(engine: new RustParquetEngine())) + ->read($path) + ->values(), + ), iterator_to_array(Reader::php()->read($path)->values()), ); } diff --git a/src/lib/parquet/tests/Flow/Parquet/Tests/Integration/Engine/ReadColumnsTest.php b/src/lib/parquet/tests/Flow/Parquet/Tests/Integration/Engine/ReadColumnsTest.php new file mode 100644 index 0000000000..2d0048446f --- /dev/null +++ b/src/lib/parquet/tests/Flow/Parquet/Tests/Integration/Engine/ReadColumnsTest.php @@ -0,0 +1,87 @@ +, int, ?int, ?int, list, list>}> + */ + public static function chunkings(): Generator + { + foreach (self::engine_provider() as $name => [$engine]) { + yield "{$name}: batch size 2" => [$engine, 2, null, null, [2, 2, 1], [[1, 2], [3, 4], [5]]]; + yield "{$name}: offset 3" => [$engine, 2, null, 3, [2], [[4, 5]]]; + yield "{$name}: limit 3" => [$engine, 2, 3, null, [2, 1], [[1, 2], [3]]]; + yield "{$name}: offset 1, limit 3" => [$engine, 2, 3, 1, [2, 1], [[2, 3], [4]]]; + yield "{$name}: batch size above the row count" => [$engine, 10, null, null, [5], [[1, 2, 3, 4, 5]]]; + } + } + + /** + * @param int<1, max> $batchSize + * @param list $sizes + * @param list> $ids + */ + #[DataProvider('chunkings')] + public function test_chunks_follow_batch_size_limit_and_offset( + string $engineClass, + int $batchSize, + ?int $limit, + ?int $offset, + array $sizes, + array $ids, + ): void { + $engine = ParquetEngineMother::create($engineClass); + $path = TestParquetFile::path($this); + + (new Writer())->write( + $path, + Schema::with( + FlatColumn::int32('id', Repetition::REQUIRED), + NestedColumn::list('tags', ListElement::string()), + ), + array_map(static fn(int $i): array => ['id' => $i, 'tags' => ['t' . $i]], [1, 2, 3, 4, 5]), + ); + + $chunks = iterator_to_array( + $engine->openForRead(NativeLocalSourceStream::open(path_real($path)))->readColumns( + ['id', 'tags'], + $batchSize, + $limit, + $offset, + ), + false, + ); + + static::assertSame($sizes, array_map(static fn(array $chunk): int => count($chunk['id']), $chunks)); + static::assertSame($ids, array_map(static fn(array $chunk): array => $chunk['id'], $chunks)); + static::assertSame( + array_map(static fn(array $chunk): array => array_map(static fn(int $i): array => [ + 't' . $i, + ], $chunk), $ids), + array_map(static fn(array $chunk): array => $chunk['tags'], $chunks), + ); + } +} diff --git a/src/lib/parquet/tests/Flow/Parquet/Tests/Integration/Engine/RustLibParityTest.php b/src/lib/parquet/tests/Flow/Parquet/Tests/Integration/Engine/RustLibParityTest.php new file mode 100644 index 0000000000..5032b05caf --- /dev/null +++ b/src/lib/parquet/tests/Flow/Parquet/Tests/Integration/Engine/RustLibParityTest.php @@ -0,0 +1,157 @@ +, ?int, ?int}> + */ + public static function fixtures(): Generator + { + $fixtures = __DIR__ . '/../IO/Fixtures'; + + foreach ([ + ...(glob($fixtures . '/*.parquet') ?: []), + ...(glob($fixtures . '/EdgeCases/*.parquet') ?: []), + ] as $path) { + if (in_array(basename($path), ['interval.parquet', 'uint64_overflow.parquet', 'map_keys.parquet'], true)) { + continue; + } + + [$limit, $offset] = str_starts_with(basename($path), 'pagination') ? [400, 4550] : [null, null]; + + foreach ([100, 10_000] as $batchSize) { + yield basename($path) . " batch {$batchSize}" => [$path, $batchSize, $limit, $offset]; + } + } + } + + /** + * @param int<1, max> $batchSize + */ + #[DataProvider('fixtures')] + public function test_a_fixture_reads_the_values_of_the_php_engine( + string $path, + int $batchSize, + ?int $limit, + ?int $offset, + ): void { + static::assertSame( + var_export(ParquetRows::read(Reader::php()->read($path), $batchSize, [], $limit, $offset), true), + var_export( + ParquetRows::read( + (new Reader(engine: new RustParquetEngine()))->read($path), + $batchSize, + [], + $limit, + $offset, + ), + true, + ), + ); + } + + public function test_every_type_reads_the_values_of_the_php_engine(): void + { + $bytes = MemoryParquetFile::written(Writer::php(), EveryType::schema(), EveryType::rows()); + + static::assertSame( + var_export(ParquetRows::read(MemoryParquetFile::read(Reader::php(), $bytes), 2), true), + var_export( + ParquetRows::read(MemoryParquetFile::read(new Reader(engine: new RustParquetEngine()), $bytes), 2), + true, + ), + ); + } + + public function test_an_interval_column_is_refused_naming_it_and_the_php_reader(): void + { + $path = __DIR__ . '/../IO/Fixtures/EdgeCases/interval.parquet'; + + static::assertSame( + ParquetRows::read(Reader::php()->read($path), 100, ['id']), + ParquetRows::read((new Reader(engine: new RustParquetEngine()))->read($path), 100, ['id']), + ); + + $this->expectException(RuntimeException::class); + $this->expectExceptionMessage( + 'Parquet column "iv" (Interval(DayTime)) is not supported by the arrow Parquet reader; read the file ' + . 'with \Flow\Parquet\Reader::php()', + ); + + ParquetRows::read((new Reader(engine: new RustParquetEngine()))->read($path), 100); + } + + public function test_a_uint64_above_php_int_max_is_refused_by_both_engines(): void + { + $path = __DIR__ . '/../IO/Fixtures/EdgeCases/uint64_overflow.parquet'; + + try { + ParquetRows::read(Reader::php()->read($path), 100); + static::fail('PhpParquetEngine must refuse a UINT_64 above PHP_INT_MAX'); + } catch (DataConversionException $exception) { + static::assertStringContainsString('holds a UINT_64 value above PHP_INT_MAX', $exception->getMessage()); + } + + $this->expectException(RuntimeException::class); + $this->expectExceptionMessage('Parquet column "u64" row 0 holds a value out of the range arrow stores it in'); + + ParquetRows::read((new Reader(engine: new RustParquetEngine()))->read($path), 100); + } + + #[TestWith(['double_key', 'float'])] + #[TestWith(['date_key', 'DateTimeImmutable'])] + #[TestWith(['bool_key', 'bool'])] + public function test_a_map_key_a_php_array_cannot_hold_is_refused_by_both_engines( + string $column, + string $type, + ): void { + $path = __DIR__ . '/../IO/Fixtures/EdgeCases/map_keys.parquet'; + + static::assertSame( + ParquetRows::read(Reader::php()->read($path), 10, ['id']), + ParquetRows::read((new Reader(engine: new RustParquetEngine()))->read($path), 10, ['id']), + ); + + try { + ParquetRows::read(Reader::php()->read($path), 10, [$column]); + static::fail('PhpParquetEngine must refuse the map key'); + } catch (InvalidArgumentException $exception) { + static::assertSame("Map key must be int or string, got {$type}", $exception->getMessage()); + } + + $this->expectException(InvalidArgumentException::class); + $this->expectExceptionMessage("Map key of Parquet column \"{$column}\" must be int or string, got {$type}"); + + ParquetRows::read((new Reader(engine: new RustParquetEngine()))->read($path), 10, [$column]); + } +} diff --git a/src/lib/parquet/tests/Flow/Parquet/Tests/Integration/Engine/RustLibRegressionTest.php b/src/lib/parquet/tests/Flow/Parquet/Tests/Integration/Engine/RustLibRegressionTest.php new file mode 100644 index 0000000000..d080c07c5f --- /dev/null +++ b/src/lib/parquet/tests/Flow/Parquet/Tests/Integration/Engine/RustLibRegressionTest.php @@ -0,0 +1,218 @@ +read(self::PAGINATION)->metadata()->rowGroups()->all()[49]->columnChunks() as $chunk) { + $group += $chunk->totalCompressedSize(); + } + + $stream = new ReadCountingSourceStream(NativeLocalSourceStream::open(path_real(self::PAGINATION))); + $rows = iterator_to_array( + (new Reader(engine: new RustParquetEngine())) + ->readStream($stream) + ->values([], 10, 4990), + false, + ); + + static::assertCount(10, $rows); + static::assertLessThanOrEqual($footer + 8 + $group, $stream->bytes); + } + + public function test_a_full_read_reads_the_file_about_once(): void + { + $stream = new ReadCountingSourceStream(NativeLocalSourceStream::open(path_real(self::PAGINATION))); + + static::assertCount(5000, iterator_to_array( + (new Reader(engine: new RustParquetEngine())) + ->readStream($stream) + ->values(), + false, + )); + static::assertLessThanOrEqual((int) filesize(self::PAGINATION) + (64 * 1024), $stream->bytes); + } + + public function test_invalid_utf8_is_refused_naming_column_and_row(): void + { + $this->expectException(RuntimeException::class); + $this->expectExceptionMessage('Parquet column "name" row 1 holds a string that is not valid UTF-8'); + + MemoryParquetFile::written( + new Writer(engine: new RustParquetEngine()), + Schema::with(FlatColumn::string('name')), + [ + ['name' => 'ok'], + ['name' => "\xFF\xFE"], + ], + ); + } + + public function test_a_time_of_a_day_and_an_hour_keeps_its_day(): void + { + $interval = (new DateTimeImmutable('2024-01-01 00:00:00'))->diff(new DateTimeImmutable('2024-01-02 01:00:00')); + $bytes = MemoryParquetFile::written( + new Writer(engine: new RustParquetEngine()), + Schema::with(FlatColumn::time('t')), + [['t' => $interval]], + ); + static::assertSame( + var_export($interval, true), + var_export( + ParquetRows::read( + MemoryParquetFile::read(new Reader(engine: new RustParquetEngine()), $bytes), + 10, + )[0]['t'], + true, + ), + ); + } + + public function test_int96_reads_exactly_outside_the_nanosecond_range(): void + { + $values = array_column( + ParquetRows::read( + (new Reader(engine: new RustParquetEngine()))->read(__DIR__ + . '/../IO/Fixtures/EdgeCases/int96.parquet'), + 10, + ), + 'ts', + ); + + static::assertEquals(new DateTimeImmutable('0001-01-01 00:00:00.000001', new DateTimeZone('UTC')), $values[3]); + static::assertEquals(new DateTimeImmutable('1960-01-01 23:59:59.999999', new DateTimeZone('UTC')), $values[2]); + } + + public function test_dates_are_utc_and_times_carry_their_day_count(): void + { + $bytes = MemoryParquetFile::written( + Writer::php(), + Schema::with(FlatColumn::date('d'), FlatColumn::time('t')), + [[ + 'd' => new DateTimeImmutable('2024-02-29'), + 't' => (new DateTimeImmutable('@0'))->diff(new DateTimeImmutable('@90000')), + ]], + ); + + static::assertSame( + var_export(ParquetRows::read(MemoryParquetFile::read(Reader::php(), $bytes), 1), true), + var_export( + ParquetRows::read(MemoryParquetFile::read(new Reader(engine: new RustParquetEngine()), $bytes), 1), + true, + ), + ); + static::assertStringContainsString("'timezone' => 'UTC'", var_export( + ParquetRows::read(MemoryParquetFile::read(new Reader(engine: new RustParquetEngine()), $bytes), 1)[0]['d'], + true, + )); + } + + public function test_a_logical_only_unsigned_integer_reads_as_itself_through_both_engines(): void + { + $bytes = MemoryParquetFile::written( + Writer::php(), + Schema::with(new FlatColumn('u', PhysicalType::INT32, logicalType: LogicalType::integer(32, false))), + [['u' => 4_000_000_000]], + ); + + static::assertSame( + [['u' => 4_000_000_000]], + ParquetRows::read(MemoryParquetFile::read(Reader::php(), $bytes), 1), + ); + static::assertSame( + [['u' => 4_000_000_000]], + ParquetRows::read(MemoryParquetFile::read(new Reader(engine: new RustParquetEngine()), $bytes), 1), + ); + } + + public function test_a_schema_read_from_an_arrow_rs_file_rewritten_by_the_php_engine_opens_in_arrow(): void + { + $arrow = MemoryParquetFile::written( + new Writer(engine: new RustParquetEngine()), + Schema::with( + new FlatColumn('i8', PhysicalType::INT32, ConvertedType::INT_8), + new FlatColumn('u32', PhysicalType::INT32, ConvertedType::UINT_32), + ), + [['i8' => -5, 'u32' => 4_000_000_000]], + ); + $schema = MemoryParquetFile::read(Reader::php(), $arrow)->schema(); + $rewritten = MemoryParquetFile::written(Writer::php(), $schema, [['i8' => -5, 'u32' => 4_000_000_000]]); + + static::assertSame(8, $schema->get('i8')->logicalType()?->integerData()?->bitWidth()); + static::assertGreaterThan(0, strlen($rewritten)); + static::assertSame( + [['i8' => -5, 'u32' => 4_000_000_000]], + ParquetRows::read(MemoryParquetFile::read(new Reader(engine: new RustParquetEngine()), $rewritten), 1), + ); + } + + public function test_times_of_day_and_longer_read_as_the_php_engine_reads_them(): void + { + $base = new DateTimeImmutable('1970-01-01 00:00:00.000000', new DateTimeZone('UTC')); + $rows = array_map(static fn(int $micros): array => [ + 't' => $base->diff($base->modify("+{$micros} microseconds")), + ], [ + 0, + 1, + 999_999, + 13_298_519_488, + 45_296_789_012, + 86_399_999_999, + 86_400_000_000, + 90_000_000_001, + ...range(1, 86_399_999_999, 172_799_999), + ]); + $bytes = MemoryParquetFile::written(Writer::php(), Schema::with(FlatColumn::time('t')), $rows); + + static::assertSame( + var_export(ParquetRows::read(MemoryParquetFile::read(Reader::php(), $bytes), 1_000), true), + var_export( + ParquetRows::read(MemoryParquetFile::read(new Reader(engine: new RustParquetEngine()), $bytes), 1_000), + true, + ), + ); + } +} diff --git a/src/lib/parquet/tests/Flow/Parquet/Tests/Integration/Engine/RustLibWriteParityTest.php b/src/lib/parquet/tests/Flow/Parquet/Tests/Integration/Engine/RustLibWriteParityTest.php new file mode 100644 index 0000000000..8731148b26 --- /dev/null +++ b/src/lib/parquet/tests/Flow/Parquet/Tests/Integration/Engine/RustLibWriteParityTest.php @@ -0,0 +1,142 @@ + + */ + public static function cells(): Generator + { + foreach (WriteAcceptance::targets() as $target) { + foreach (WriteAcceptance::inputs() as $input) { + yield "{$target} <- {$input}" => [$target, $input]; + } + } + } + + #[DataProvider('cells')] + public function test_both_writers_accept_and_refuse_the_same(string $target, string $input): void + { + $written = []; + + foreach ([ + 'php' => Writer::php(), + 'native' => new Writer(engine: new RustParquetEngine()), + ] as $engine => $writer) { + try { + $written[$engine] = MemoryParquetFile::written( + $writer, + Schema::with(WriteAcceptance::target($target)), + [['c' => WriteAcceptance::input($input)]], + ); + } catch (Throwable) { + $written[$engine] = null; + } + } + + static::assertSame($written['php'] === null, $written['native'] === null); + + if ($written['php'] !== null && $written['native'] !== null) { + static::assertSame( + var_export(ParquetRows::read(MemoryParquetFile::read(Reader::php(), $written['php']), 1), true), + var_export(ParquetRows::read(MemoryParquetFile::read(Reader::php(), $written['native']), 1), true), + ); + } + } + + #[DataProvider('cells')] + public function test_the_column_door_accepts_and_refuses_what_the_row_door_does(string $target, string $input): void + { + foreach ([new PhpParquetEngine(), new RustParquetEngine()] as $engine) { + $outcomes = []; + + foreach (['batch', 'columns'] as $door) { + $stream = ColumnDoor::stream(); + + try { + $file = ColumnDoor::open($engine, $stream, Schema::with(WriteAcceptance::target($target))); + $door === 'batch' + ? $file->writeBatch([['c' => WriteAcceptance::input($input)]]) + : $file->writeColumns(['c' => [WriteAcceptance::input($input)]]); + $file->close(); + $outcomes[$door] = var_export(ColumnDoor::read($stream), true); + } catch (Throwable $refusal) { + $outcomes[$door] = $refusal::class . ': ' . $refusal->getMessage(); + } + } + + static::assertSame($outcomes['batch'], $outcomes['columns'], $engine::class); + } + } + + /** + * The values PhpParquetEngine used to write wrongly (#27, #28, #16). + * + * @return Generator + */ + public static function wrongly_written(): Generator + { + yield 'INT32 out of range' => ['INT32', 'int 4000000000']; + yield 'INT_8 out of range' => ['INT32 INT_8', 'int PHP_INT_MAX']; + yield 'UINT_32 negative' => ['INT32 UINT_32', 'int -1']; + yield 'UINT_64 negative' => ['INT64 UINT_64', 'int -1']; + yield 'UUID of 16 raw bytes' => ['UUID', 'string 16 bytes']; + yield 'FIXED(4) of 5 bytes' => ['FIXED(4)', 'string 5 bytes']; + yield 'FIXED(4) of 36 bytes' => ['FIXED(4)', 'string 36 uuid']; + yield 'invalid UTF-8 into STRING' => ['STRING', 'string \xFF\xFE']; + yield 'invalid UTF-8 into JSON' => ['JSON', 'string \xFF\xFE']; + yield 'invalid UTF-8 into ENUM' => ['ENUM', 'string \xFF\xFE']; + } + + #[DataProvider('wrongly_written')] + public function test_the_php_engine_refuses_what_it_used_to_write_wrongly(string $target, string $input): void + { + $this->expectException(ValidationException::class); + $this->expectExceptionMessage('Column "c" row 0: '); + + MemoryParquetFile::written(Writer::php(), Schema::with(WriteAcceptance::target($target)), [[ + 'c' => WriteAcceptance::input($input), + ]]); + } + + public function test_every_type_writes_the_same_through_both_writers(): void + { + $php = MemoryParquetFile::written(Writer::php(), EveryType::schema(), EveryType::rows()); + $native = MemoryParquetFile::written( + new Writer(engine: new RustParquetEngine()), + EveryType::schema(), + EveryType::rows(), + ); + + static::assertSame( + var_export(ParquetRows::read(MemoryParquetFile::read(Reader::php(), $php), 3), true), + var_export(ParquetRows::read(MemoryParquetFile::read(Reader::php(), $native), 3), true), + ); + } +} diff --git a/src/lib/parquet/tests/Flow/Parquet/Tests/Integration/Engine/RustParquetEngineReadTest.php b/src/lib/parquet/tests/Flow/Parquet/Tests/Integration/Engine/RustParquetEngineReadTest.php new file mode 100644 index 0000000000..3e90a088c4 --- /dev/null +++ b/src/lib/parquet/tests/Flow/Parquet/Tests/Integration/Engine/RustParquetEngineReadTest.php @@ -0,0 +1,320 @@ + 1, 'name' => 'Alice', 'active' => true, 'score' => 99.5], + ['id' => 2, 'name' => 'Bob', 'active' => false, 'score' => 87.3], + ['id' => 3, 'name' => 'Charlie', 'active' => true, 'score' => 92.1], + ]; + + (new Writer())->write($path, $schema, $inputData); + + $engine = new RustParquetEngine(); + $chunks = iterator_to_array( + $engine->openForRead(NativeLocalSourceStream::open(path_real($path)))->readColumns( + ['id', 'name', 'active', 'score'], + batchSize: 2, + limit: null, + offset: null, + ), + false, + ); + + static::assertSame( + [ + ['id' => [1, 2], 'name' => ['Alice', 'Bob'], 'active' => [true, false], 'score' => [99.5, 87.3]], + ['id' => [3], 'name' => ['Charlie'], 'active' => [true], 'score' => [92.1]], + ], + $chunks, + ); + } + + public function test_read_nested_lists(): void + { + $path = TestParquetFile::path($this); + + $schema = Schema::with( + FlatColumn::int32('id', Repetition::REQUIRED), + NestedColumn::list('tags', ListElement::string()), + ); + + $inputData = [ + ['id' => 1, 'tags' => ['php', 'rust']], + ['id' => 2, 'tags' => ['python']], + ]; + + (new Writer())->write($path, $schema, $inputData); + + $engine = new RustParquetEngine(); + $chunks = iterator_to_array( + $engine->openForRead(NativeLocalSourceStream::open(path_real($path)))->readColumns( + ['id', 'tags'], + batchSize: 2, + limit: null, + offset: null, + ), + false, + ); + + static::assertSame([['id' => [1, 2], 'tags' => [['php', 'rust'], ['python']]]], $chunks); + } + + public function test_read_nested_maps(): void + { + $path = TestParquetFile::path($this); + + $schema = Schema::with( + FlatColumn::int32('id', Repetition::REQUIRED), + NestedColumn::map('metadata', MapKey::string(), MapValue::int32()), + ); + + $inputData = [ + ['id' => 1, 'metadata' => ['score' => 100, 'level' => 5]], + ['id' => 2, 'metadata' => ['score' => 200]], + ]; + + (new Writer())->write($path, $schema, $inputData); + + $engine = new RustParquetEngine(); + $chunks = iterator_to_array( + $engine->openForRead(NativeLocalSourceStream::open(path_real($path)))->readColumns( + ['id', 'metadata'], + batchSize: 2, + limit: null, + offset: null, + ), + false, + ); + + static::assertSame( + [['id' => [1, 2], 'metadata' => [['score' => 100, 'level' => 5], ['score' => 200]]]], + $chunks, + ); + } + + public function test_read_nested_structs(): void + { + $path = TestParquetFile::path($this); + + $schema = Schema::with( + FlatColumn::int32('id', Repetition::REQUIRED), + NestedColumn::struct('address', [ + FlatColumn::string('city'), + FlatColumn::int32('zip'), + ]), + ); + + $inputData = [ + ['id' => 1, 'address' => ['city' => 'Berlin', 'zip' => 10115]], + ['id' => 2, 'address' => ['city' => 'Warsaw', 'zip' => 00001]], + ]; + + (new Writer())->write($path, $schema, $inputData); + + $engine = new RustParquetEngine(); + $chunks = iterator_to_array( + $engine->openForRead(NativeLocalSourceStream::open(path_real($path)))->readColumns( + ['id', 'address'], + batchSize: 2, + limit: null, + offset: null, + ), + false, + ); + + static::assertSame( + [['id' => [1, 2], 'address' => [['city' => 'Berlin', 'zip' => 10115], ['city' => 'Warsaw', 'zip' => 1]]]], + $chunks, + ); + } + + public function test_read_nested_uuid_as_canonical_strings(): void + { + $path = TestParquetFile::path($this); + $uuid = 'f6d6e0e8-4b7e-4b0e-8d7a-ff0a0c9c9a5a'; + + $schema = Schema::with( + FlatColumn::uuid('top_uuid'), + NestedColumn::struct('body', [FlatColumn::uuid('id'), FlatColumn::json('data')]), + NestedColumn::list('uuid_list', ListElement::uuid()), + ); + + (new Writer())->write($path, $schema, [[ + 'top_uuid' => $uuid, + 'body' => ['id' => $uuid, 'data' => '{"a":1}'], + 'uuid_list' => [$uuid, $uuid], + ]]); + + $engine = new RustParquetEngine(); + $chunks = iterator_to_array( + $engine->openForRead(NativeLocalSourceStream::open(path_real($path)))->readColumns( + ['top_uuid', 'body', 'uuid_list'], + batchSize: 2, + limit: null, + offset: null, + ), + false, + ); + + static::assertSame( + [[ + 'top_uuid' => [$uuid], + 'body' => [['id' => $uuid, 'data' => '{"a":1}']], + 'uuid_list' => [[$uuid, $uuid]], + ]], + $chunks, + ); + } + + public function test_read_with_column_projection(): void + { + $path = TestParquetFile::path($this); + + $schema = Schema::with( + FlatColumn::int32('id', Repetition::REQUIRED), + FlatColumn::string('name'), + FlatColumn::string('email'), + ); + + $inputData = [ + ['id' => 1, 'name' => 'Alice', 'email' => 'alice@example.com'], + ['id' => 2, 'name' => 'Bob', 'email' => 'bob@example.com'], + ]; + + (new Writer())->write($path, $schema, $inputData); + + $engine = new RustParquetEngine(); + $chunks = iterator_to_array( + $engine->openForRead(NativeLocalSourceStream::open(path_real($path)))->readColumns( + ['id', 'name'], + batchSize: 2, + limit: null, + offset: null, + ), + false, + ); + + static::assertSame([['id' => [1, 2], 'name' => ['Alice', 'Bob']]], $chunks); + } + + public function test_read_with_limit(): void + { + $path = TestParquetFile::path($this); + + $schema = Schema::with(FlatColumn::int32('id', Repetition::REQUIRED)); + + $inputData = array_map(static fn(int $i): array => ['id' => $i], range(1, 100)); + + (new Writer())->write($path, $schema, $inputData); + + $engine = new RustParquetEngine(); + $chunks = iterator_to_array( + $engine->openForRead(NativeLocalSourceStream::open(path_real($path)))->readColumns( + ['id'], + batchSize: 2, + limit: 5, + offset: null, + ), + false, + ); + + static::assertSame([['id' => [1, 2]], ['id' => [3, 4]], ['id' => [5]]], $chunks); + } + + public function test_read_with_limit_and_offset(): void + { + $path = TestParquetFile::path($this); + + $schema = Schema::with(FlatColumn::int32('id', Repetition::REQUIRED)); + + $inputData = array_map(static fn(int $i): array => ['id' => $i], range(1, 100)); + + (new Writer())->write($path, $schema, $inputData); + + $engine = new RustParquetEngine(); + $chunks = iterator_to_array( + $engine->openForRead(NativeLocalSourceStream::open(path_real($path)))->readColumns( + ['id'], + batchSize: 2, + limit: 5, + offset: 10, + ), + false, + ); + + static::assertSame([['id' => [11, 12]], ['id' => [13, 14]], ['id' => [15]]], $chunks); + } + + public function test_read_with_offset(): void + { + $path = TestParquetFile::path($this); + + $schema = Schema::with(FlatColumn::int32('id', Repetition::REQUIRED)); + + $inputData = array_map(static fn(int $i): array => ['id' => $i], range(1, 100)); + + (new Writer())->write($path, $schema, $inputData); + + $engine = new RustParquetEngine(); + $chunks = iterator_to_array( + $engine->openForRead(NativeLocalSourceStream::open(path_real($path)))->readColumns( + ['id'], + batchSize: 2, + limit: null, + offset: 96, + ), + false, + ); + + static::assertSame([['id' => [97, 98]], ['id' => [99, 100]]], $chunks); + } +} diff --git a/src/lib/parquet/tests/Flow/Parquet/Tests/Integration/Engine/RustParquetEngineWriteTest.php b/src/lib/parquet/tests/Flow/Parquet/Tests/Integration/Engine/RustParquetEngineWriteTest.php new file mode 100644 index 0000000000..ab83e87154 --- /dev/null +++ b/src/lib/parquet/tests/Flow/Parquet/Tests/Integration/Engine/RustParquetEngineWriteTest.php @@ -0,0 +1,150 @@ + 1, 'name' => 'Alice', 'active' => true, 'score' => 99.5], + ['id' => 2, 'name' => 'Bob', 'active' => false, 'score' => 87.3], + ['id' => 3, 'name' => 'Charlie', 'active' => true, 'score' => 92.1], + ]; + + $engine = new RustParquetEngine(); + $stream = NativeLocalDestinationStream::openBlank(path($path)); + $engine->writeRows($stream, $schema, Compressions::SNAPPY, new Options(), $inputData); + + $result = iterator_to_array( + (new Reader()) + ->read($path) + ->values(), + ); + + static::assertCount(3, $result); + static::assertSame(1, $result[0]['id']); + static::assertSame('Alice', $result[0]['name']); + static::assertTrue($result[0]['active']); + static::assertEqualsWithDelta(99.5, $result[0]['score'], 0.001); + } + + public function test_write_nested_types(): void + { + $path = TestParquetFile::path($this); + + $schema = Schema::with( + FlatColumn::int32('id', Repetition::REQUIRED), + NestedColumn::list('tags', ListElement::string()), + NestedColumn::struct('address', [ + FlatColumn::string('city'), + ]), + ); + + $inputData = [ + ['id' => 1, 'tags' => ['php', 'rust'], 'address' => ['city' => 'Berlin']], + ['id' => 2, 'tags' => ['python'], 'address' => ['city' => 'Warsaw']], + ]; + + $engine = new RustParquetEngine(); + $stream = NativeLocalDestinationStream::openBlank(path($path)); + $engine->writeRows($stream, $schema, Compressions::SNAPPY, new Options(), $inputData); + + $result = iterator_to_array( + (new Reader()) + ->read($path) + ->values(), + ); + + static::assertCount(2, $result); + static::assertSame(1, $result[0]['id']); + } + + public function test_write_with_gzip_compression(): void + { + $path = TestParquetFile::path($this); + + $schema = Schema::with(FlatColumn::int32('id', Repetition::REQUIRED), FlatColumn::string('data')); + + $inputData = array_map(static fn(int $i): array => ['id' => $i, 'data' => 'value_' . $i], range(1, 20)); + + $engine = new RustParquetEngine(); + $stream = NativeLocalDestinationStream::openBlank(path($path)); + $engine->writeRows($stream, $schema, Compressions::GZIP, new Options(), $inputData); + + $result = iterator_to_array( + (new Reader()) + ->read($path) + ->values(), + ); + + static::assertCount(20, $result); + } + + public function test_write_with_snappy_compression(): void + { + $path = TestParquetFile::path($this); + + $schema = Schema::with(FlatColumn::int32('id', Repetition::REQUIRED), FlatColumn::string('data')); + + $inputData = array_map(static fn(int $i): array => ['id' => $i, 'data' => str_repeat('a', 100)], range(1, 50)); + + $engine = new RustParquetEngine(); + $stream = NativeLocalDestinationStream::openBlank(path($path)); + $engine->writeRows($stream, $schema, Compressions::SNAPPY, new Options(), $inputData); + + $result = iterator_to_array( + (new Reader()) + ->read($path) + ->values(), + ); + + static::assertCount(50, $result); + static::assertSame(1, $result[0]['id']); + } +} diff --git a/src/lib/parquet/tests/Flow/Parquet/Tests/Integration/IO/CompressionTest.php b/src/lib/parquet/tests/Flow/Parquet/Tests/Integration/IO/CompressionTest.php index 2708cba8a3..3e30b3b0be 100644 --- a/src/lib/parquet/tests/Flow/Parquet/Tests/Integration/IO/CompressionTest.php +++ b/src/lib/parquet/tests/Flow/Parquet/Tests/Integration/IO/CompressionTest.php @@ -6,7 +6,6 @@ use Faker\Factory; use Flow\Parquet\Consts; -use Flow\Parquet\ParquetEngine; use Flow\Parquet\ParquetFile\Compressions; use Flow\Parquet\ParquetFile\Schema; use Flow\Parquet\ParquetFile\Schema\FlatColumn; @@ -14,6 +13,7 @@ use Flow\Parquet\ParquetFile\Schema\NestedColumn; use Flow\Parquet\Reader; use Flow\Parquet\Tests\Context\TestParquetFile; +use Flow\Parquet\Tests\Mother\ParquetEngineMother; use Flow\Parquet\Writer; use PHPUnit\Framework\Attributes\DataProvider; use PHPUnit\Framework\Attributes\Group; @@ -29,8 +29,9 @@ class CompressionTest extends ParquetIntegrationTestCase { #[Group('brotli-extension')] #[DataProvider('engine_provider')] - public function test_writing_and_reading_file_with_brotli_compression(ParquetEngine $engine): void + public function test_writing_and_reading_file_with_brotli_compression(string $engineClass): void { + $engine = ParquetEngineMother::create($engineClass); if (!extension_loaded('brotli')) { static::markTestSkipped('The Brotli extension is not available'); } @@ -85,8 +86,9 @@ public function test_writing_and_reading_file_with_brotli_compression(ParquetEng } #[DataProvider('engine_provider')] - public function test_writing_and_reading_file_with_gzip_compression(ParquetEngine $engine): void + public function test_writing_and_reading_file_with_gzip_compression(string $engineClass): void { + $engine = ParquetEngineMother::create($engineClass); $path = TestParquetFile::path($this); $writer = new Writer(compression: Compressions::GZIP, engine: $engine); @@ -138,8 +140,9 @@ public function test_writing_and_reading_file_with_gzip_compression(ParquetEngin #[Group('lz4-extension')] #[DataProvider('engine_provider')] - public function test_writing_and_reading_file_with_lz4_compression(ParquetEngine $engine): void + public function test_writing_and_reading_file_with_lz4_compression(string $engineClass): void { + $engine = ParquetEngineMother::create($engineClass); if (!extension_loaded('lz4')) { static::markTestSkipped('The lz4 extension is not available'); } @@ -195,8 +198,9 @@ public function test_writing_and_reading_file_with_lz4_compression(ParquetEngine #[Group('lz4-extension')] #[DataProvider('engine_provider')] - public function test_writing_and_reading_file_with_lz4_raw_compression(ParquetEngine $engine): void + public function test_writing_and_reading_file_with_lz4_raw_compression(string $engineClass): void { + $engine = ParquetEngineMother::create($engineClass); if (!extension_loaded('lz4')) { static::markTestSkipped('The lz4 extension is not available'); } @@ -252,8 +256,9 @@ public function test_writing_and_reading_file_with_lz4_raw_compression(ParquetEn #[Group('snappy-extension')] #[DataProvider('engine_provider')] - public function test_writing_and_reading_file_with_snappy_compression(ParquetEngine $engine): void + public function test_writing_and_reading_file_with_snappy_compression(string $engineClass): void { + $engine = ParquetEngineMother::create($engineClass); if (!extension_loaded('snappy')) { static::markTestSkipped('The snappy extension is not available'); } @@ -308,8 +313,9 @@ public function test_writing_and_reading_file_with_snappy_compression(ParquetEng } #[DataProvider('engine_provider')] - public function test_writing_and_reading_file_with_snappy_polyfill(ParquetEngine $engine): void + public function test_writing_and_reading_file_with_snappy_polyfill(string $engineClass): void { + $engine = ParquetEngineMother::create($engineClass); if (extension_loaded('snappy')) { static::markTestSkipped('The snappy extension is available'); } @@ -364,8 +370,9 @@ public function test_writing_and_reading_file_with_snappy_polyfill(ParquetEngine } #[DataProvider('engine_provider')] - public function test_writing_and_reading_file_with_uncompressed_compression(ParquetEngine $engine): void + public function test_writing_and_reading_file_with_uncompressed_compression(string $engineClass): void { + $engine = ParquetEngineMother::create($engineClass); $path = TestParquetFile::path($this); $writer = new Writer(compression: Compressions::UNCOMPRESSED, engine: $engine); @@ -417,8 +424,9 @@ public function test_writing_and_reading_file_with_uncompressed_compression(Parq #[Group('zstd-extension')] #[DataProvider('engine_provider')] - public function test_writing_and_reading_file_with_zstd_compression(ParquetEngine $engine): void + public function test_writing_and_reading_file_with_zstd_compression(string $engineClass): void { + $engine = ParquetEngineMother::create($engineClass); if (!extension_loaded('zstd')) { static::markTestSkipped('The Zstd extension is not available'); } diff --git a/src/lib/parquet/tests/Flow/Parquet/Tests/Integration/IO/EdgeCasesReadingTest.php b/src/lib/parquet/tests/Flow/Parquet/Tests/Integration/IO/EdgeCasesReadingTest.php index 0ad6a8952c..56f2087287 100644 --- a/src/lib/parquet/tests/Flow/Parquet/Tests/Integration/IO/EdgeCasesReadingTest.php +++ b/src/lib/parquet/tests/Flow/Parquet/Tests/Integration/IO/EdgeCasesReadingTest.php @@ -4,8 +4,8 @@ namespace Flow\Parquet\Tests\Integration\IO; -use Flow\Parquet\ParquetEngine; use Flow\Parquet\Reader; +use Flow\Parquet\Tests\Mother\ParquetEngineMother; use PHPUnit\Framework\Attributes\DataProvider; use function iterator_to_array; @@ -13,8 +13,9 @@ class EdgeCasesReadingTest extends ParquetIntegrationTestCase { #[DataProvider('engine_provider')] - public function test_nonullable_impala(ParquetEngine $engine): void + public function test_nonullable_impala(string $engineClass): void { + $engine = ParquetEngineMother::create($engineClass); $path = __DIR__ . '/Fixtures/EdgeCases/nonnullable.impala.parquet'; $reader = (new Reader(engine: $engine))->read($path); @@ -60,31 +61,31 @@ public function test_nonullable_impala(ParquetEngine $engine): void ); } - public function test_read_datapage_v2_snappy_list(): void + #[DataProvider('engine_provider')] + public function test_read_datapage_v2_snappy_list(string $engineClass): void { - $this->expectExceptionMessage('Encoding RLE not supported'); - - $path = __DIR__ . '/Fixtures/EdgeCases/datapage_v2.snappy.parquet'; - - $reader = Reader::php()->read($path); - - $rows = []; - - foreach ($reader->values() as $row) { - $rows[] = $row; - } - + $engine = ParquetEngineMother::create($engineClass); static::assertSame( [ - ['emptylist' => null], + ['a' => 'abc', 'b' => 1, 'c' => 2.0, 'd' => true, 'e' => [1, 2, 3]], + ['a' => 'abc', 'b' => 2, 'c' => 3.0, 'd' => true, 'e' => null], + ['a' => 'abc', 'b' => 3, 'c' => 4.0, 'd' => true, 'e' => null], + ['a' => null, 'b' => 4, 'c' => 5.0, 'd' => false, 'e' => [1, 2, 3]], + ['a' => 'abc', 'b' => 5, 'c' => 2.0, 'd' => true, 'e' => [1, 2]], ], - $rows, + iterator_to_array( + (new Reader(engine: $engine)) + ->read(__DIR__ . '/Fixtures/EdgeCases/datapage_v2.snappy.parquet') + ->values(), + false, + ), ); } #[DataProvider('engine_provider')] - public function test_read_null_list(ParquetEngine $engine): void + public function test_read_null_list(string $engineClass): void { + $engine = ParquetEngineMother::create($engineClass); $path = __DIR__ . '/Fixtures/EdgeCases/null_list.parquet'; $reader = (new Reader(engine: $engine))->read($path); diff --git a/src/lib/parquet/tests/Flow/Parquet/Tests/Integration/IO/FixedLenByteArrayReadingTest.php b/src/lib/parquet/tests/Flow/Parquet/Tests/Integration/IO/FixedLenByteArrayReadingTest.php index ba8bd06043..8ad5600156 100644 --- a/src/lib/parquet/tests/Flow/Parquet/Tests/Integration/IO/FixedLenByteArrayReadingTest.php +++ b/src/lib/parquet/tests/Flow/Parquet/Tests/Integration/IO/FixedLenByteArrayReadingTest.php @@ -4,12 +4,12 @@ namespace Flow\Parquet\Tests\Integration\IO; -use Flow\Parquet\ParquetEngine; use Flow\Parquet\ParquetFile\Schema; use Flow\Parquet\ParquetFile\Schema\FlatColumn; use Flow\Parquet\ParquetFile\Schema\Repetition; use Flow\Parquet\Reader; use Flow\Parquet\Tests\Context\TestParquetFile; +use Flow\Parquet\Tests\Mother\ParquetEngineMother; use Flow\Parquet\Writer; use PHPUnit\Framework\Attributes\DataProvider; @@ -20,8 +20,9 @@ class FixedLenByteArrayReadingTest extends ParquetIntegrationTestCase { #[DataProvider('engine_provider')] - public function test_reading_and_writing_fixed_len_byte_array_without_logical_type(ParquetEngine $engine): void + public function test_reading_and_writing_fixed_len_byte_array_without_logical_type(string $engineClass): void { + $engine = ParquetEngineMother::create($engineClass); $path = TestParquetFile::path($this); $schema = Schema::with(FlatColumn::fixedSizeByteArray('raw_bytes', 8, Repetition::REQUIRED)); @@ -55,8 +56,9 @@ public function test_reading_and_writing_fixed_len_byte_array_without_logical_ty } #[DataProvider('engine_provider')] - public function test_reading_fixed_len_byte_array_returns_raw_string(ParquetEngine $engine): void + public function test_reading_fixed_len_byte_array_returns_raw_string(string $engineClass): void { + $engine = ParquetEngineMother::create($engineClass); $path = TestParquetFile::path($this); $schema = Schema::with(FlatColumn::fixedSizeByteArray('data', 16, Repetition::REQUIRED)); diff --git a/src/lib/parquet/tests/Flow/Parquet/Tests/Integration/IO/Fixtures/EdgeCases/int96.parquet b/src/lib/parquet/tests/Flow/Parquet/Tests/Integration/IO/Fixtures/EdgeCases/int96.parquet new file mode 100644 index 0000000000..0a3967146e Binary files /dev/null and b/src/lib/parquet/tests/Flow/Parquet/Tests/Integration/IO/Fixtures/EdgeCases/int96.parquet differ diff --git a/src/lib/parquet/tests/Flow/Parquet/Tests/Integration/IO/Fixtures/EdgeCases/interval.parquet b/src/lib/parquet/tests/Flow/Parquet/Tests/Integration/IO/Fixtures/EdgeCases/interval.parquet new file mode 100644 index 0000000000..c4d08ff2e1 Binary files /dev/null and b/src/lib/parquet/tests/Flow/Parquet/Tests/Integration/IO/Fixtures/EdgeCases/interval.parquet differ diff --git a/src/lib/parquet/tests/Flow/Parquet/Tests/Integration/IO/Fixtures/EdgeCases/map_keys.parquet b/src/lib/parquet/tests/Flow/Parquet/Tests/Integration/IO/Fixtures/EdgeCases/map_keys.parquet new file mode 100644 index 0000000000..da0c61a054 Binary files /dev/null and b/src/lib/parquet/tests/Flow/Parquet/Tests/Integration/IO/Fixtures/EdgeCases/map_keys.parquet differ diff --git a/src/lib/parquet/tests/Flow/Parquet/Tests/Integration/IO/Fixtures/EdgeCases/uint64_overflow.parquet b/src/lib/parquet/tests/Flow/Parquet/Tests/Integration/IO/Fixtures/EdgeCases/uint64_overflow.parquet new file mode 100644 index 0000000000..b8755d372b Binary files /dev/null and b/src/lib/parquet/tests/Flow/Parquet/Tests/Integration/IO/Fixtures/EdgeCases/uint64_overflow.parquet differ diff --git a/src/lib/parquet/tests/Flow/Parquet/Tests/Integration/IO/Fixtures/EdgeCases/unsigned.parquet b/src/lib/parquet/tests/Flow/Parquet/Tests/Integration/IO/Fixtures/EdgeCases/unsigned.parquet new file mode 100644 index 0000000000..5d4cafc60e Binary files /dev/null and b/src/lib/parquet/tests/Flow/Parquet/Tests/Integration/IO/Fixtures/EdgeCases/unsigned.parquet differ diff --git a/src/lib/parquet/tests/Flow/Parquet/Tests/Integration/IO/ListsReadingTest.php b/src/lib/parquet/tests/Flow/Parquet/Tests/Integration/IO/ListsReadingTest.php index 1342665f61..ba0064f50e 100644 --- a/src/lib/parquet/tests/Flow/Parquet/Tests/Integration/IO/ListsReadingTest.php +++ b/src/lib/parquet/tests/Flow/Parquet/Tests/Integration/IO/ListsReadingTest.php @@ -4,15 +4,16 @@ namespace Flow\Parquet\Tests\Integration\IO; -use Flow\Parquet\ParquetEngine; use Flow\Parquet\Reader; +use Flow\Parquet\Tests\Mother\ParquetEngineMother; use PHPUnit\Framework\Attributes\DataProvider; class ListsReadingTest extends ParquetIntegrationTestCase { #[DataProvider('engine_provider')] - public function test_reading_list_column(ParquetEngine $engine): void + public function test_reading_list_column(string $engineClass): void { + $engine = ParquetEngineMother::create($engineClass); $reader = new Reader(engine: $engine); $file = $reader->read(__DIR__ . '/Fixtures/lists.parquet'); @@ -35,8 +36,9 @@ public function test_reading_list_column(ParquetEngine $engine): void } #[DataProvider('engine_provider')] - public function test_reading_list_column_with_limit(ParquetEngine $engine): void + public function test_reading_list_column_with_limit(string $engineClass): void { + $engine = ParquetEngineMother::create($engineClass); $reader = new Reader(engine: $engine); $file = $reader->read(__DIR__ . '/Fixtures/lists.parquet'); @@ -58,8 +60,9 @@ public function test_reading_list_column_with_limit(ParquetEngine $engine): void } #[DataProvider('engine_provider')] - public function test_reading_list_nested_column(ParquetEngine $engine): void + public function test_reading_list_nested_column(string $engineClass): void { + $engine = ParquetEngineMother::create($engineClass); $reader = new Reader(engine: $engine); $file = $reader->read(__DIR__ . '/Fixtures/lists.parquet'); @@ -89,8 +92,9 @@ public function test_reading_list_nested_column(ParquetEngine $engine): void } #[DataProvider('engine_provider')] - public function test_reading_list_nullable_column(ParquetEngine $engine): void + public function test_reading_list_nullable_column(string $engineClass): void { + $engine = ParquetEngineMother::create($engineClass); $reader = new Reader(engine: $engine); $file = $reader->read(__DIR__ . '/Fixtures/lists.parquet'); @@ -117,8 +121,9 @@ public function test_reading_list_nullable_column(ParquetEngine $engine): void } #[DataProvider('engine_provider')] - public function test_reading_list_of_structures_column(ParquetEngine $engine): void + public function test_reading_list_of_structures_column(string $engineClass): void { + $engine = ParquetEngineMother::create($engineClass); $reader = new Reader(engine: $engine); $file = $reader->read(__DIR__ . '/Fixtures/lists.parquet'); @@ -149,8 +154,9 @@ public function test_reading_list_of_structures_column(ParquetEngine $engine): v } #[DataProvider('engine_provider')] - public function test_reading_list_of_structures_nullable_column(ParquetEngine $engine): void + public function test_reading_list_of_structures_nullable_column(string $engineClass): void { + $engine = ParquetEngineMother::create($engineClass); $reader = new Reader(engine: $engine); $file = $reader->read(__DIR__ . '/Fixtures/lists.parquet'); diff --git a/src/lib/parquet/tests/Flow/Parquet/Tests/Integration/IO/ListsWritingTest.php b/src/lib/parquet/tests/Flow/Parquet/Tests/Integration/IO/ListsWritingTest.php index 9bfaff3161..a6a54eda82 100644 --- a/src/lib/parquet/tests/Flow/Parquet/Tests/Integration/IO/ListsWritingTest.php +++ b/src/lib/parquet/tests/Flow/Parquet/Tests/Integration/IO/ListsWritingTest.php @@ -6,7 +6,6 @@ use Faker\Factory; use Flow\Parquet\Consts; -use Flow\Parquet\ParquetEngine; use Flow\Parquet\ParquetFile\Schema; use Flow\Parquet\ParquetFile\Schema\FlatColumn; use Flow\Parquet\ParquetFile\Schema\ListElement; @@ -14,6 +13,7 @@ use Flow\Parquet\ParquetFile\Schema\Repetition; use Flow\Parquet\Reader; use Flow\Parquet\Tests\Context\TestParquetFile; +use Flow\Parquet\Tests\Mother\ParquetEngineMother; use Flow\Parquet\Writer; use PHPUnit\Framework\Attributes\DataProvider; @@ -26,8 +26,9 @@ class ListsWritingTest extends ParquetIntegrationTestCase { #[DataProvider('engine_provider')] - public function test_writing_empty_lists_of_ints(ParquetEngine $engine): void + public function test_writing_empty_lists_of_ints(string $engineClass): void { + $engine = ParquetEngineMother::create($engineClass); $path = TestParquetFile::path($this); $writer = new Writer(engine: $engine); @@ -55,8 +56,9 @@ public function test_writing_empty_lists_of_ints(ParquetEngine $engine): void } #[DataProvider('engine_provider')] - public function test_writing_list_of_decimals(ParquetEngine $engine): void + public function test_writing_list_of_decimals(string $engineClass): void { + $engine = ParquetEngineMother::create($engineClass); $path = TestParquetFile::path($this); $writer = new Writer(engine: $engine); @@ -77,8 +79,9 @@ public function test_writing_list_of_decimals(ParquetEngine $engine): void } #[DataProvider('engine_provider')] - public function test_writing_list_of_ints(ParquetEngine $engine): void + public function test_writing_list_of_ints(string $engineClass): void { + $engine = ParquetEngineMother::create($engineClass); $path = TestParquetFile::path($this); $writer = new Writer(engine: $engine); @@ -110,8 +113,9 @@ public function test_writing_list_of_ints(ParquetEngine $engine): void } #[DataProvider('engine_provider')] - public function test_writing_list_of_strings(ParquetEngine $engine): void + public function test_writing_list_of_strings(string $engineClass): void { + $engine = ParquetEngineMother::create($engineClass); $path = TestParquetFile::path($this); $writer = new Writer(engine: $engine); @@ -143,8 +147,9 @@ public function test_writing_list_of_strings(ParquetEngine $engine): void } #[DataProvider('engine_provider')] - public function test_writing_list_of_structures(ParquetEngine $engine): void + public function test_writing_list_of_structures(string $engineClass): void { + $engine = ParquetEngineMother::create($engineClass); $path = TestParquetFile::path($this); $writer = new Writer(engine: $engine); @@ -182,8 +187,9 @@ public function test_writing_list_of_structures(ParquetEngine $engine): void } #[DataProvider('engine_provider')] - public function test_writing_list_with_nullable_elements(ParquetEngine $engine): void + public function test_writing_list_with_nullable_elements(string $engineClass): void { + $engine = ParquetEngineMother::create($engineClass); $path = TestParquetFile::path($this); $writer = new Writer(engine: $engine); @@ -217,8 +223,9 @@ public function test_writing_list_with_nullable_elements(ParquetEngine $engine): } #[DataProvider('engine_provider')] - public function test_writing_list_with_nullable_list_values(ParquetEngine $engine): void + public function test_writing_list_with_nullable_list_values(string $engineClass): void { + $engine = ParquetEngineMother::create($engineClass); $path = TestParquetFile::path($this); $writer = new Writer(engine: $engine); @@ -252,8 +259,9 @@ public function test_writing_list_with_nullable_list_values(ParquetEngine $engin } #[DataProvider('engine_provider')] - public function test_writing_nullable_list_of_ints(ParquetEngine $engine): void + public function test_writing_nullable_list_of_ints(string $engineClass): void { + $engine = ParquetEngineMother::create($engineClass); $path = TestParquetFile::path($this); $writer = new Writer(engine: $engine); @@ -287,8 +295,9 @@ public function test_writing_nullable_list_of_ints(ParquetEngine $engine): void } #[DataProvider('engine_provider')] - public function test_writing_nullable_list_of_structures(ParquetEngine $engine): void + public function test_writing_nullable_list_of_structures(string $engineClass): void { + $engine = ParquetEngineMother::create($engineClass); $path = TestParquetFile::path($this); $writer = new Writer(engine: $engine); @@ -328,8 +337,9 @@ public function test_writing_nullable_list_of_structures(ParquetEngine $engine): } #[DataProvider('engine_provider')] - public function test_writing_nullable_list_of_structures_with_required_fields(ParquetEngine $engine): void + public function test_writing_nullable_list_of_structures_with_required_fields(string $engineClass): void { + $engine = ParquetEngineMother::create($engineClass); $path = TestParquetFile::path($this); $writer = new Writer(engine: $engine); @@ -365,8 +375,9 @@ public function test_writing_nullable_list_of_structures_with_required_fields(Pa } #[DataProvider('engine_provider')] - public function test_writing_nullable_lists_of_ints(ParquetEngine $engine): void + public function test_writing_nullable_lists_of_ints(string $engineClass): void { + $engine = ParquetEngineMother::create($engineClass); $path = TestParquetFile::path($this); $writer = new Writer(engine: $engine); diff --git a/src/lib/parquet/tests/Flow/Parquet/Tests/Integration/IO/LogicalTypesReadingTest.php b/src/lib/parquet/tests/Flow/Parquet/Tests/Integration/IO/LogicalTypesReadingTest.php index 06bf2f44f3..ebed7b0692 100644 --- a/src/lib/parquet/tests/Flow/Parquet/Tests/Integration/IO/LogicalTypesReadingTest.php +++ b/src/lib/parquet/tests/Flow/Parquet/Tests/Integration/IO/LogicalTypesReadingTest.php @@ -5,7 +5,6 @@ namespace Flow\Parquet\Tests\Integration\IO; use Flow\Parquet\Engine\PhpParquetEngine; -use Flow\Parquet\ParquetEngine; use Flow\Parquet\ParquetFile\Schema; use Flow\Parquet\ParquetFile\Schema\ConvertedType; use Flow\Parquet\ParquetFile\Schema\FlatColumn; @@ -14,6 +13,7 @@ use Flow\Parquet\Reader; use Flow\Parquet\Tests\Context\TemporalValues; use Flow\Parquet\Tests\Context\TestParquetFile; +use Flow\Parquet\Tests\Mother\ParquetEngineMother; use Flow\Parquet\Writer; use PHPUnit\Framework\Attributes\DataProvider; @@ -22,8 +22,9 @@ final class LogicalTypesReadingTest extends ParquetIntegrationTestCase { #[DataProvider('engine_provider')] - public function test_reading_converted_type_only_columns(ParquetEngine $engine): void + public function test_reading_converted_type_only_columns(string $engineClass): void { + $engine = ParquetEngineMother::create($engineClass); $path = TestParquetFile::path($this); (new Writer(engine: new PhpParquetEngine()))->write( @@ -72,8 +73,9 @@ public function test_reading_converted_type_only_columns(ParquetEngine $engine): } #[DataProvider('engine_provider')] - public function test_reading_dates(ParquetEngine $engine): void + public function test_reading_dates(string $engineClass): void { + $engine = ParquetEngineMother::create($engineClass); static::assertSame( [ ['d' => '2020-01-02 00:00:00.000000 +00:00'], @@ -89,8 +91,9 @@ public function test_reading_dates(ParquetEngine $engine): void } #[DataProvider('engine_provider')] - public function test_reading_fixed_len_decimals(ParquetEngine $engine): void + public function test_reading_fixed_len_decimals(string $engineClass): void { + $engine = ParquetEngineMother::create($engineClass); static::assertSame( [ [ @@ -111,8 +114,9 @@ public function test_reading_fixed_len_decimals(ParquetEngine $engine): void } #[DataProvider('engine_provider')] - public function test_reading_int_backed_decimals(ParquetEngine $engine): void + public function test_reading_int_backed_decimals(string $engineClass): void { + $engine = ParquetEngineMother::create($engineClass); static::assertSame( [ ['dec9' => 12345.67, 'dec18' => 1234567890123456.8], @@ -128,8 +132,9 @@ public function test_reading_int_backed_decimals(ParquetEngine $engine): void } #[DataProvider('engine_provider')] - public function test_reading_times_in_every_unit(ParquetEngine $engine): void + public function test_reading_times_in_every_unit(string $engineClass): void { + $engine = ParquetEngineMother::create($engineClass); static::assertSame( [ ['t_ms' => '03:04:05.678000', 't_us' => '03:04:05.678901', 't_ns' => '03:04:05.678901'], @@ -145,8 +150,9 @@ public function test_reading_times_in_every_unit(ParquetEngine $engine): void } #[DataProvider('engine_provider')] - public function test_reading_timestamps_in_every_unit(ParquetEngine $engine): void + public function test_reading_timestamps_in_every_unit(string $engineClass): void { + $engine = ParquetEngineMother::create($engineClass); static::assertSame( [ [ diff --git a/src/lib/parquet/tests/Flow/Parquet/Tests/Integration/IO/LogicalTypesWritingTest.php b/src/lib/parquet/tests/Flow/Parquet/Tests/Integration/IO/LogicalTypesWritingTest.php index b69e676ae7..20d71d3257 100644 --- a/src/lib/parquet/tests/Flow/Parquet/Tests/Integration/IO/LogicalTypesWritingTest.php +++ b/src/lib/parquet/tests/Flow/Parquet/Tests/Integration/IO/LogicalTypesWritingTest.php @@ -6,7 +6,6 @@ use DateTimeImmutable; use DateTimeZone; -use Flow\Parquet\ParquetEngine; use Flow\Parquet\ParquetFile\Schema; use Flow\Parquet\ParquetFile\Schema\FlatColumn; use Flow\Parquet\ParquetFile\Schema\LogicalType; @@ -18,6 +17,7 @@ use Flow\Parquet\Reader; use Flow\Parquet\Tests\Context\TemporalValues; use Flow\Parquet\Tests\Context\TestParquetFile; +use Flow\Parquet\Tests\Mother\ParquetEngineMother; use Flow\Parquet\Writer; use PHPUnit\Framework\Attributes\DataProvider; @@ -27,8 +27,10 @@ final class LogicalTypesWritingTest extends ParquetIntegrationTestCase { #[DataProvider('engine_pair_provider')] - public function test_dates_are_the_wall_clock_day(ParquetEngine $writer, ParquetEngine $reader): void + public function test_dates_are_the_wall_clock_day(string $writerClass, string $readerClass): void { + $writer = ParquetEngineMother::create($writerClass); + $reader = ParquetEngineMother::create($readerClass); $path = TestParquetFile::path($this); (new Writer(engine: $writer))->write($path, Schema::with(FlatColumn::date('d')), [ @@ -52,8 +54,10 @@ public function test_dates_are_the_wall_clock_day(ParquetEngine $writer, Parquet } #[DataProvider('engine_pair_provider')] - public function test_decimals_round_half_away_from_zero(ParquetEngine $writer, ParquetEngine $reader): void + public function test_decimals_round_half_away_from_zero(string $writerClass, string $readerClass): void { + $writer = ParquetEngineMother::create($writerClass); + $reader = ParquetEngineMother::create($readerClass); $path = TestParquetFile::path($this); (new Writer(engine: $writer))->write($path, Schema::with(FlatColumn::decimal('d', 9, 2)), [ @@ -76,8 +80,10 @@ public function test_decimals_round_half_away_from_zero(ParquetEngine $writer, P } #[DataProvider('engine_pair_provider')] - public function test_decimals_round_trip(ParquetEngine $writer, ParquetEngine $reader): void + public function test_decimals_round_trip(string $writerClass, string $readerClass): void { + $writer = ParquetEngineMother::create($writerClass); + $reader = ParquetEngineMother::create($readerClass); $path = TestParquetFile::path($this); $rows = [ ['d9' => 12345.67, 'd38' => 12345.67, 'i32' => 12345.67, 'i64' => 12345.67, 'ba' => 12345.67], @@ -132,8 +138,10 @@ public function test_decimals_round_trip(ParquetEngine $writer, ParquetEngine $r } #[DataProvider('engine_pair_provider')] - public function test_times_round_trip_in_every_unit(ParquetEngine $writer, ParquetEngine $reader): void + public function test_times_round_trip_in_every_unit(string $writerClass, string $readerClass): void { + $writer = ParquetEngineMother::create($writerClass); + $reader = ParquetEngineMother::create($readerClass); $path = TestParquetFile::path($this); $time = (new DateTimeImmutable('2020-01-01 00:00:00 UTC'))->diff( new DateTimeImmutable('2020-01-01 03:04:05.678901 UTC'), @@ -168,8 +176,10 @@ public function test_times_round_trip_in_every_unit(ParquetEngine $writer, Parqu } #[DataProvider('engine_pair_provider')] - public function test_timestamps_round_trip_in_every_unit(ParquetEngine $writer, ParquetEngine $reader): void + public function test_timestamps_round_trip_in_every_unit(string $writerClass, string $readerClass): void { + $writer = ParquetEngineMother::create($writerClass); + $reader = ParquetEngineMother::create($readerClass); $path = TestParquetFile::path($this); $instant = new DateTimeImmutable('2020-01-02 04:04:05.678901 +01:00'); $preEpoch = new DateTimeImmutable('1969-12-31 23:59:58.5 UTC'); diff --git a/src/lib/parquet/tests/Flow/Parquet/Tests/Integration/IO/MapsReadingTest.php b/src/lib/parquet/tests/Flow/Parquet/Tests/Integration/IO/MapsReadingTest.php index bd9682cdfb..fcafb31aea 100644 --- a/src/lib/parquet/tests/Flow/Parquet/Tests/Integration/IO/MapsReadingTest.php +++ b/src/lib/parquet/tests/Flow/Parquet/Tests/Integration/IO/MapsReadingTest.php @@ -4,8 +4,8 @@ namespace Flow\Parquet\Tests\Integration\IO; -use Flow\Parquet\ParquetEngine; use Flow\Parquet\Reader; +use Flow\Parquet\Tests\Mother\ParquetEngineMother; use PHPUnit\Framework\Attributes\DataProvider; use function array_key_first; @@ -13,8 +13,9 @@ class MapsReadingTest extends ParquetIntegrationTestCase { #[DataProvider('engine_provider')] - public function test_reading_map_column(ParquetEngine $engine): void + public function test_reading_map_column(string $engineClass): void { + $engine = ParquetEngineMother::create($engineClass); $reader = new Reader(engine: $engine); $file = $reader->read(__DIR__ . '/Fixtures/maps.parquet'); @@ -37,8 +38,9 @@ public function test_reading_map_column(ParquetEngine $engine): void } #[DataProvider('engine_provider')] - public function test_reading_map_nullable_column(ParquetEngine $engine): void + public function test_reading_map_nullable_column(string $engineClass): void { + $engine = ParquetEngineMother::create($engineClass); $reader = new Reader(engine: $engine); $file = $reader->read(__DIR__ . '/Fixtures/maps.parquet'); @@ -65,8 +67,9 @@ public function test_reading_map_nullable_column(ParquetEngine $engine): void } #[DataProvider('engine_provider')] - public function test_reading_map_of_complex_lists(ParquetEngine $engine): void + public function test_reading_map_of_complex_lists(string $engineClass): void { + $engine = ParquetEngineMother::create($engineClass); $reader = new Reader(engine: $engine); $file = $reader->read(__DIR__ . '/Fixtures/maps.parquet'); @@ -95,8 +98,9 @@ public function test_reading_map_of_complex_lists(ParquetEngine $engine): void } #[DataProvider('engine_provider')] - public function test_reading_map_of_list_of_map_of_lists(ParquetEngine $engine): void + public function test_reading_map_of_list_of_map_of_lists(string $engineClass): void { + $engine = ParquetEngineMother::create($engineClass); $reader = new Reader(engine: $engine); $file = $reader->read(__DIR__ . '/Fixtures/maps.parquet'); @@ -130,8 +134,9 @@ public function test_reading_map_of_list_of_map_of_lists(ParquetEngine $engine): } #[DataProvider('engine_provider')] - public function test_reading_map_of_lists(ParquetEngine $engine): void + public function test_reading_map_of_lists(string $engineClass): void { + $engine = ParquetEngineMother::create($engineClass); $reader = new Reader(engine: $engine); $file = $reader->read(__DIR__ . '/Fixtures/maps.parquet'); @@ -156,8 +161,9 @@ public function test_reading_map_of_lists(ParquetEngine $engine): void } #[DataProvider('engine_provider')] - public function test_reading_map_of_maps_column(ParquetEngine $engine): void + public function test_reading_map_of_maps_column(string $engineClass): void { + $engine = ParquetEngineMother::create($engineClass); $reader = new Reader(engine: $engine); $file = $reader->read(__DIR__ . '/Fixtures/maps.parquet'); @@ -181,8 +187,9 @@ public function test_reading_map_of_maps_column(ParquetEngine $engine): void } #[DataProvider('engine_provider')] - public function test_reading_map_of_struct_of_structs_column(ParquetEngine $engine): void + public function test_reading_map_of_struct_of_structs_column(string $engineClass): void { + $engine = ParquetEngineMother::create($engineClass); $reader = new Reader(engine: $engine); $file = $reader->read(__DIR__ . '/Fixtures/maps.parquet'); @@ -216,8 +223,9 @@ public function test_reading_map_of_struct_of_structs_column(ParquetEngine $engi } #[DataProvider('engine_provider')] - public function test_reading_map_of_struct_of_structs_column_with_limit(ParquetEngine $engine): void + public function test_reading_map_of_struct_of_structs_column_with_limit(string $engineClass): void { + $engine = ParquetEngineMother::create($engineClass); $reader = new Reader(engine: $engine); $file = $reader->read(__DIR__ . '/Fixtures/maps.parquet'); @@ -250,8 +258,9 @@ public function test_reading_map_of_struct_of_structs_column_with_limit(ParquetE } #[DataProvider('engine_provider')] - public function test_reading_map_of_structs_column(ParquetEngine $engine): void + public function test_reading_map_of_structs_column(string $engineClass): void { + $engine = ParquetEngineMother::create($engineClass); $reader = new Reader(engine: $engine); $file = $reader->read(__DIR__ . '/Fixtures/maps.parquet'); diff --git a/src/lib/parquet/tests/Flow/Parquet/Tests/Integration/IO/MapsWritingTest.php b/src/lib/parquet/tests/Flow/Parquet/Tests/Integration/IO/MapsWritingTest.php index 4e744070a8..c15ce552ab 100644 --- a/src/lib/parquet/tests/Flow/Parquet/Tests/Integration/IO/MapsWritingTest.php +++ b/src/lib/parquet/tests/Flow/Parquet/Tests/Integration/IO/MapsWritingTest.php @@ -6,13 +6,13 @@ use Faker\Factory; use Flow\Parquet\Consts; -use Flow\Parquet\ParquetEngine; use Flow\Parquet\ParquetFile\Schema; use Flow\Parquet\ParquetFile\Schema\MapKey; use Flow\Parquet\ParquetFile\Schema\MapValue; use Flow\Parquet\ParquetFile\Schema\NestedColumn; use Flow\Parquet\Reader; use Flow\Parquet\Tests\Context\TestParquetFile; +use Flow\Parquet\Tests\Mother\ParquetEngineMother; use Flow\Parquet\Writer; use PHPUnit\Framework\Attributes\DataProvider; @@ -25,8 +25,9 @@ class MapsWritingTest extends ParquetIntegrationTestCase { #[DataProvider('engine_provider')] - public function test_writing_empty_map_of_int_int(ParquetEngine $engine): void + public function test_writing_empty_map_of_int_int(string $engineClass): void { + $engine = ParquetEngineMother::create($engineClass); $path = TestParquetFile::path($this); $writer = new Writer(engine: $engine); @@ -54,8 +55,9 @@ public function test_writing_empty_map_of_int_int(ParquetEngine $engine): void } #[DataProvider('engine_provider')] - public function test_writing_map_of_int_int(ParquetEngine $engine): void + public function test_writing_map_of_int_int(string $engineClass): void { + $engine = ParquetEngineMother::create($engineClass); $path = TestParquetFile::path($this); $writer = new Writer(engine: $engine); @@ -87,8 +89,9 @@ public function test_writing_map_of_int_int(ParquetEngine $engine): void } #[DataProvider('engine_provider')] - public function test_writing_map_of_int_int_with_all_maps_null(ParquetEngine $engine): void + public function test_writing_map_of_int_int_with_all_maps_null(string $engineClass): void { + $engine = ParquetEngineMother::create($engineClass); $path = TestParquetFile::path($this); $writer = new Writer(engine: $engine); @@ -116,8 +119,9 @@ public function test_writing_map_of_int_int_with_all_maps_null(ParquetEngine $en } #[DataProvider('engine_provider')] - public function test_writing_map_of_int_string(ParquetEngine $engine): void + public function test_writing_map_of_int_string(string $engineClass): void { + $engine = ParquetEngineMother::create($engineClass); $path = TestParquetFile::path($this); $writer = new Writer(engine: $engine); @@ -149,8 +153,9 @@ public function test_writing_map_of_int_string(ParquetEngine $engine): void } #[DataProvider('engine_provider')] - public function test_writing_map_of_string_decimal(ParquetEngine $engine): void + public function test_writing_map_of_string_decimal(string $engineClass): void { + $engine = ParquetEngineMother::create($engineClass); $path = TestParquetFile::path($this); $writer = new Writer(engine: $engine); @@ -171,8 +176,9 @@ public function test_writing_map_of_string_decimal(ParquetEngine $engine): void } #[DataProvider('engine_provider')] - public function test_writing_nullable_map_of_int_int(ParquetEngine $engine): void + public function test_writing_nullable_map_of_int_int(string $engineClass): void { + $engine = ParquetEngineMother::create($engineClass); $path = TestParquetFile::path($this); $writer = new Writer(engine: $engine); diff --git a/src/lib/parquet/tests/Flow/Parquet/Tests/Integration/IO/PaginationTest.php b/src/lib/parquet/tests/Flow/Parquet/Tests/Integration/IO/PaginationTest.php index 60f123b5aa..26d0c20456 100644 --- a/src/lib/parquet/tests/Flow/Parquet/Tests/Integration/IO/PaginationTest.php +++ b/src/lib/parquet/tests/Flow/Parquet/Tests/Integration/IO/PaginationTest.php @@ -8,7 +8,6 @@ use Flow\Parquet\Exception\InvalidArgumentException; use Flow\Parquet\Option; use Flow\Parquet\Options; -use Flow\Parquet\ParquetEngine; use Flow\Parquet\ParquetFile\Schema; use Flow\Parquet\ParquetFile\Schema\FlatColumn; use Flow\Parquet\ParquetFile\Schema\ListElement; @@ -17,6 +16,7 @@ use Flow\Parquet\ParquetFile\Schema\NestedColumn; use Flow\Parquet\Reader; use Flow\Parquet\Tests\Context\TestParquetFile; +use Flow\Parquet\Tests\Mother\ParquetEngineMother; use Flow\Parquet\Writer; use Generator; use PHPUnit\Framework\Attributes\DataProvider; @@ -54,8 +54,9 @@ public static function engine_with_offset_provider(): Generator } #[DataProvider('engine_provider')] - public function test_offset_past_first_page_of_multi_page_column(ParquetEngine $engine): void + public function test_offset_past_first_page_of_multi_page_column(string $engineClass): void { + $engine = ParquetEngineMother::create($engineClass); $path = TestParquetFile::path($this); Writer::php(options: Options::default()->set(Option::PAGE_SIZE_BYTES, 100)->set( @@ -78,8 +79,9 @@ public function test_offset_past_first_page_of_multi_page_column(ParquetEngine $ } #[DataProvider('engine_provider')] - public function test_reading_last_100_rows(ParquetEngine $engine): void + public function test_reading_last_100_rows(string $engineClass): void { + $engine = ParquetEngineMother::create($engineClass); $path = __DIR__ . '/Fixtures/pagination_row_group_1kb_5k_rows.snappy.parquet'; $totalRows = (new Reader(engine: $engine)) @@ -102,11 +104,12 @@ public function test_reading_last_100_rows(ParquetEngine $engine): void #[DataProvider('engine_with_offset_provider')] public function test_setting_offset_larger_than_file( - ParquetEngine $engine, + string $engineClass, int $offset, ?int $limit, int $results, ): void { + $engine = ParquetEngineMother::create($engineClass); $path = __DIR__ . '/Fixtures/pagination_row_group_1kb_5k_rows.snappy.parquet'; static::assertCount( @@ -120,8 +123,9 @@ public function test_setting_offset_larger_than_file( } #[DataProvider('engine_provider')] - public function test_setting_setting_limit_to_negative(ParquetEngine $engine): void + public function test_setting_setting_limit_to_negative(string $engineClass): void { + $engine = ParquetEngineMother::create($engineClass); $path = __DIR__ . '/Fixtures/pagination_row_group_1kb_5k_rows.snappy.parquet'; $this->expectException(InvalidArgumentException::class); @@ -135,8 +139,9 @@ public function test_setting_setting_limit_to_negative(ParquetEngine $engine): v } #[DataProvider('engine_provider')] - public function test_setting_setting_offset_to_negative(ParquetEngine $engine): void + public function test_setting_setting_offset_to_negative(string $engineClass): void { + $engine = ParquetEngineMother::create($engineClass); $path = __DIR__ . '/Fixtures/pagination_row_group_1kb_5k_rows.snappy.parquet'; $this->expectException(InvalidArgumentException::class); @@ -150,8 +155,85 @@ public function test_setting_setting_offset_to_negative(ParquetEngine $engine): } #[DataProvider('engine_provider')] - public function test_simple_pagination_on_small_row_group_size(ParquetEngine $engine): void + public function test_columns_refuses_a_batch_size_below_one(string $engineClass): void { + $engine = ParquetEngineMother::create($engineClass); + $this->expectException(InvalidArgumentException::class); + $this->expectExceptionMessage('Batch size must be greater than 0'); + + iterator_to_array( + (new Reader(engine: $engine)) + ->read(__DIR__ . '/Fixtures/pagination_row_group_1kb_5k_rows.snappy.parquet') + ->columns(0, ['id']), + ); + } + + #[DataProvider('engine_provider')] + public function test_columns_refuses_a_limit_of_zero(string $engineClass): void + { + $engine = ParquetEngineMother::create($engineClass); + $this->expectException(InvalidArgumentException::class); + $this->expectExceptionMessage('Limit must be greater than 0'); + + iterator_to_array( + (new Reader(engine: $engine)) + ->read(__DIR__ . '/Fixtures/pagination_row_group_1kb_5k_rows.snappy.parquet') + ->columns(2, ['id'], limit: 0), + ); + } + + #[DataProvider('engine_provider')] + public function test_columns_refuses_a_negative_offset_without_a_limit(string $engineClass): void + { + $engine = ParquetEngineMother::create($engineClass); + $this->expectException(InvalidArgumentException::class); + $this->expectExceptionMessage('Offset must be greater than or equal to 0'); + + iterator_to_array( + (new Reader(engine: $engine)) + ->read(__DIR__ . '/Fixtures/pagination_row_group_1kb_5k_rows.snappy.parquet') + ->columns(2, ['id'], offset: -1), + ); + } + + #[DataProvider('engine_provider')] + public function test_columns_refuses_an_unknown_column(string $engineClass): void + { + $engine = ParquetEngineMother::create($engineClass); + $this->expectException(InvalidArgumentException::class); + $this->expectExceptionMessage('Column "nope" does not exist'); + + iterator_to_array( + (new Reader(engine: $engine)) + ->read(__DIR__ . '/Fixtures/pagination_row_group_1kb_5k_rows.snappy.parquet') + ->columns(2, ['nope']), + ); + } + + #[DataProvider('engine_provider')] + public function test_values_equal_the_zip_of_columns(string $engineClass): void + { + $engine = ParquetEngineMother::create($engineClass); + $file = (new Reader(engine: $engine))->read(__DIR__ + . '/Fixtures/pagination_row_group_1kb_5k_rows.snappy.parquet'); + + $zipped = []; + + foreach ($file->columns(3, ['id', 'name'], limit: 7, offset: 1020) as $chunk) { + // @mago-ignore analysis:mixed-assignment + foreach ($chunk['id'] as $i => $id) { + $zipped[] = ['id' => $id, 'name' => $chunk['name'][$i]]; + } + } + + static::assertSame($zipped, iterator_to_array($file->values(['id', 'name'], limit: 7, offset: 1020), false)); + static::assertCount(7, $zipped); + } + + #[DataProvider('engine_provider')] + public function test_simple_pagination_on_small_row_group_size(string $engineClass): void + { + $engine = ParquetEngineMother::create($engineClass); $path = __DIR__ . '/Fixtures/pagination_row_group_1kb_5k_rows.snappy.parquet'; // Uncomment only to apply changes to the dataset diff --git a/src/lib/parquet/tests/Flow/Parquet/Tests/Integration/IO/ParquetIntegrationTestCase.php b/src/lib/parquet/tests/Flow/Parquet/Tests/Integration/IO/ParquetIntegrationTestCase.php index b85bbc681e..30f496767d 100644 --- a/src/lib/parquet/tests/Flow/Parquet/Tests/Integration/IO/ParquetIntegrationTestCase.php +++ b/src/lib/parquet/tests/Flow/Parquet/Tests/Integration/IO/ParquetIntegrationTestCase.php @@ -4,8 +4,9 @@ namespace Flow\Parquet\Tests\Integration\IO; -use Flow\Parquet\Engine\ArrowParquetEngine; use Flow\Parquet\Engine\PhpParquetEngine; +use Flow\Parquet\Engine\RustParquetEngine; +use Flow\Parquet\ParquetEngine; use Flow\Parquet\Tests\Context\TestParquetFile; use PHPUnit\Framework\TestCase; @@ -14,30 +15,30 @@ abstract class ParquetIntegrationTestCase extends TestCase { /** - * @return array + * @return array}> */ public static function engine_provider(): array { - $engines = ['php' => [new PhpParquetEngine()]]; + $engines = ['php' => [PhpParquetEngine::class]]; if (extension_loaded('arrow')) { - $engines['arrow'] = [new ArrowParquetEngine()]; + $engines['arrow'] = [RustParquetEngine::class]; } return $engines; } /** - * @return array + * @return array, class-string}> */ public static function engine_pair_provider(): array { - $pairs = ['php/php' => [new PhpParquetEngine(), new PhpParquetEngine()]]; + $pairs = ['php/php' => [PhpParquetEngine::class, PhpParquetEngine::class]]; if (extension_loaded('arrow')) { - $pairs['php/arrow'] = [new PhpParquetEngine(), new ArrowParquetEngine()]; - $pairs['arrow/php'] = [new ArrowParquetEngine(), new PhpParquetEngine()]; - $pairs['arrow/arrow'] = [new ArrowParquetEngine(), new ArrowParquetEngine()]; + $pairs['php/arrow'] = [PhpParquetEngine::class, RustParquetEngine::class]; + $pairs['arrow/php'] = [RustParquetEngine::class, PhpParquetEngine::class]; + $pairs['arrow/arrow'] = [RustParquetEngine::class, RustParquetEngine::class]; } return $pairs; diff --git a/src/lib/parquet/tests/Flow/Parquet/Tests/Integration/IO/ReaderTest.php b/src/lib/parquet/tests/Flow/Parquet/Tests/Integration/IO/ReaderTest.php index 4e18fda017..f30fdc967d 100644 --- a/src/lib/parquet/tests/Flow/Parquet/Tests/Integration/IO/ReaderTest.php +++ b/src/lib/parquet/tests/Flow/Parquet/Tests/Integration/IO/ReaderTest.php @@ -4,9 +4,9 @@ namespace Flow\Parquet\Tests\Integration\IO; -use Flow\Parquet\ParquetEngine; use Flow\Parquet\ParquetFile\Page\ColumnPageHeader; use Flow\Parquet\Reader; +use Flow\Parquet\Tests\Mother\ParquetEngineMother; use PHPUnit\Framework\Attributes\DataProvider; use function array_map; @@ -16,8 +16,9 @@ class ReaderTest extends ParquetIntegrationTestCase { #[DataProvider('engine_provider')] - public function test_reading_columns_with_multiple_data_pages(ParquetEngine $engine): void + public function test_reading_columns_with_multiple_data_pages(string $engineClass): void { + $engine = ParquetEngineMother::create($engineClass); // File generated with https://gist.github.com/norberttech/325df9166bbdb33e18dffa94c1a033c4 $reader = new Reader(engine: $engine); $file = $reader->read(__DIR__ . '/Fixtures/multiple_pages.parquet'); @@ -48,8 +49,9 @@ public function test_reading_columns_with_multiple_data_pages(ParquetEngine $eng } #[DataProvider('engine_provider')] - public function test_reading_required_columns(ParquetEngine $engine): void + public function test_reading_required_columns(string $engineClass): void { + $engine = ParquetEngineMother::create($engineClass); // File generated with https://gist.github.com/norberttech/01322f61dca77cfde5161e31e94463ef $reader = new Reader(engine: $engine); $file = $reader->read(__DIR__ . '/Fixtures/columns.required.parquet'); diff --git a/src/lib/parquet/tests/Flow/Parquet/Tests/Integration/IO/SchemaReadingTest.php b/src/lib/parquet/tests/Flow/Parquet/Tests/Integration/IO/SchemaReadingTest.php index 1c13ec745f..8908e21f24 100644 --- a/src/lib/parquet/tests/Flow/Parquet/Tests/Integration/IO/SchemaReadingTest.php +++ b/src/lib/parquet/tests/Flow/Parquet/Tests/Integration/IO/SchemaReadingTest.php @@ -4,7 +4,6 @@ namespace Flow\Parquet\Tests\Integration\IO; -use Flow\Parquet\ParquetEngine; use Flow\Parquet\ParquetFile\RowGroup\StatisticsReader; use Flow\Parquet\ParquetFile\Schema; use Flow\Parquet\ParquetFile\Schema\FlatColumn; @@ -13,13 +12,15 @@ use Flow\Parquet\ParquetFile\Schema\MapValue; use Flow\Parquet\ParquetFile\Schema\NestedColumn; use Flow\Parquet\Reader; +use Flow\Parquet\Tests\Mother\ParquetEngineMother; use PHPUnit\Framework\Attributes\DataProvider; class SchemaReadingTest extends ParquetIntegrationTestCase { #[DataProvider('engine_provider')] - public function test_reading_lists_schema_ddl(ParquetEngine $engine): void + public function test_reading_lists_schema_ddl(string $engineClass): void { + $engine = ParquetEngineMother::create($engineClass); $reader = new Reader(engine: $engine); $schema = new Schema(NestedColumn::create('schema', [ @@ -52,8 +53,9 @@ public function test_reading_lists_schema_ddl(ParquetEngine $engine): void } #[DataProvider('engine_provider')] - public function test_reading_maps_schema_ddl(ParquetEngine $engine): void + public function test_reading_maps_schema_ddl(string $engineClass): void { + $engine = ParquetEngineMother::create($engineClass); $reader = new Reader(engine: $engine); $schema = Schema::with( @@ -108,8 +110,9 @@ public function test_reading_maps_schema_ddl(ParquetEngine $engine): void } #[DataProvider('engine_provider')] - public function test_reading_primitives_schema_ddl(ParquetEngine $engine): void + public function test_reading_primitives_schema_ddl(string $engineClass): void { + $engine = ParquetEngineMother::create($engineClass); $reader = new Reader(engine: $engine); $schema = Schema::with( @@ -131,8 +134,8 @@ public function test_reading_primitives_schema_ddl(ParquetEngine $engine): void FlatColumn::time('time_nullable'), FlatColumn::string('uuid'), // files generated by python are storing uuid as strings FlatColumn::string('uuid_nullable'), - FlatColumn::enum('enum'), - FlatColumn::enum('enum_nullable'), + FlatColumn::string('enum'), // files generated by python are storing enums as strings + FlatColumn::string('enum_nullable'), FlatColumn::float('float'), FlatColumn::float('float_nullable'), FlatColumn::double('double'), @@ -152,8 +155,9 @@ public function test_reading_primitives_schema_ddl(ParquetEngine $engine): void } #[DataProvider('engine_provider')] - public function test_reading_statistics(ParquetEngine $engine): void + public function test_reading_statistics(string $engineClass): void { + $engine = ParquetEngineMother::create($engineClass); $metadata = (new Reader(engine: $engine)) ->read(__DIR__ . '/Fixtures/primitives.parquet') ->metadata(); @@ -164,8 +168,9 @@ public function test_reading_statistics(ParquetEngine $engine): void } #[DataProvider('engine_provider')] - public function test_reading_structs_schema_ddl(ParquetEngine $engine): void + public function test_reading_structs_schema_ddl(string $engineClass): void { + $engine = ParquetEngineMother::create($engineClass); $reader = new Reader(engine: $engine); $schema = Schema::with( diff --git a/src/lib/parquet/tests/Flow/Parquet/Tests/Integration/IO/SimpleTypesReadingTest.php b/src/lib/parquet/tests/Flow/Parquet/Tests/Integration/IO/SimpleTypesReadingTest.php index 69468f7eb8..50f580208b 100644 --- a/src/lib/parquet/tests/Flow/Parquet/Tests/Integration/IO/SimpleTypesReadingTest.php +++ b/src/lib/parquet/tests/Flow/Parquet/Tests/Integration/IO/SimpleTypesReadingTest.php @@ -6,21 +6,24 @@ use DateInterval; use DateTimeImmutable; -use Flow\Parquet\ParquetEngine; use Flow\Parquet\ParquetFile\Schema\PhysicalType; use Flow\Parquet\Reader; +use Flow\Parquet\Tests\Mother\ParquetEngineMother; use PHPUnit\Framework\Attributes\DataProvider; use function array_filter; +use function array_map; use function array_merge_recursive; use function count; use function iterator_to_array; +use function range; class SimpleTypesReadingTest extends ParquetIntegrationTestCase { #[DataProvider('engine_provider')] - public function test_reading_bool_column(ParquetEngine $engine): void + public function test_reading_bool_column(string $engineClass): void { + $engine = ParquetEngineMother::create($engineClass); $reader = new Reader(engine: $engine); $file = $reader->read(__DIR__ . '/Fixtures/primitives.parquet'); @@ -36,8 +39,9 @@ public function test_reading_bool_column(ParquetEngine $engine): void } #[DataProvider('engine_provider')] - public function test_reading_bool_column_with_limit(ParquetEngine $engine): void + public function test_reading_bool_column_with_limit(string $engineClass): void { + $engine = ParquetEngineMother::create($engineClass); $reader = new Reader(engine: $engine); $file = $reader->read(__DIR__ . '/Fixtures/primitives.parquet'); @@ -52,8 +56,9 @@ public function test_reading_bool_column_with_limit(ParquetEngine $engine): void } #[DataProvider('engine_provider')] - public function test_reading_bool_nullable_column(ParquetEngine $engine): void + public function test_reading_bool_nullable_column(string $engineClass): void { + $engine = ParquetEngineMother::create($engineClass); $reader = new Reader(engine: $engine); $file = $reader->read(__DIR__ . '/Fixtures/primitives.parquet'); @@ -70,8 +75,9 @@ public function test_reading_bool_nullable_column(ParquetEngine $engine): void } #[DataProvider('engine_provider')] - public function test_reading_bool_nullable_column_with_limit(ParquetEngine $engine): void + public function test_reading_bool_nullable_column_with_limit(string $engineClass): void { + $engine = ParquetEngineMother::create($engineClass); $reader = new Reader(engine: $engine); $file = $reader->read(__DIR__ . '/Fixtures/primitives.parquet'); @@ -90,8 +96,9 @@ public function test_reading_bool_nullable_column_with_limit(ParquetEngine $engi } #[DataProvider('engine_provider')] - public function test_reading_date_column(ParquetEngine $engine): void + public function test_reading_date_column(string $engineClass): void { + $engine = ParquetEngineMother::create($engineClass); $reader = new Reader(engine: $engine); $file = $reader->read(__DIR__ . '/Fixtures/primitives.parquet'); @@ -109,8 +116,9 @@ public function test_reading_date_column(ParquetEngine $engine): void } #[DataProvider('engine_provider')] - public function test_reading_date_nullable_column(ParquetEngine $engine): void + public function test_reading_date_nullable_column(string $engineClass): void { + $engine = ParquetEngineMother::create($engineClass); $reader = new Reader(engine: $engine); $file = $reader->read(__DIR__ . '/Fixtures/primitives.parquet'); @@ -132,8 +140,9 @@ public function test_reading_date_nullable_column(ParquetEngine $engine): void } #[DataProvider('engine_provider')] - public function test_reading_decimal_column(ParquetEngine $engine): void + public function test_reading_decimal_column(string $engineClass): void { + $engine = ParquetEngineMother::create($engineClass); $reader = new Reader(engine: $engine); $file = $reader->read(__DIR__ . '/Fixtures/primitives.parquet'); @@ -150,8 +159,9 @@ public function test_reading_decimal_column(ParquetEngine $engine): void } #[DataProvider('engine_provider')] - public function test_reading_decimal_nullable_column(ParquetEngine $engine): void + public function test_reading_decimal_nullable_column(string $engineClass): void { + $engine = ParquetEngineMother::create($engineClass); $reader = new Reader(engine: $engine); $file = $reader->read(__DIR__ . '/Fixtures/primitives.parquet'); @@ -175,8 +185,9 @@ public function test_reading_decimal_nullable_column(ParquetEngine $engine): voi } #[DataProvider('engine_provider')] - public function test_reading_delta_binary_packed_encoded_integers(ParquetEngine $engine): void + public function test_reading_delta_binary_packed_encoded_integers(string $engineClass): void { + $engine = ParquetEngineMother::create($engineClass); $reader = new Reader(engine: $engine); $file = $reader->read(__DIR__ . '/Fixtures/delta_binary_acked_encoded_integers.parquet'); @@ -222,8 +233,31 @@ public function test_reading_delta_binary_packed_encoded_integers(ParquetEngine } #[DataProvider('engine_provider')] - public function test_reading_double_column(ParquetEngine $engine): void + public function test_reading_every_delta_binary_packed_value(string $engineClass): void { + $engine = ParquetEngineMother::create($engineClass); + static::assertSame( + array_map( + static fn(int $n): array => [ + 'int32_column' => $n, + 'int64_column' => ($n % 10) === 0 ? PHP_INT_MAX - ($n * 1000) : $n, + 'float_column' => ($n % 7) === 0 ? 3.4028234663852886E+38 : $n + 0.5, + ], + range(1, 1000), + ), + iterator_to_array( + (new Reader(engine: $engine)) + ->read(__DIR__ . '/Fixtures/delta_binary_acked_encoded_integers.parquet') + ->values(), + false, + ), + ); + } + + #[DataProvider('engine_provider')] + public function test_reading_double_column(string $engineClass): void + { + $engine = ParquetEngineMother::create($engineClass); $reader = new Reader(engine: $engine); $file = $reader->read(__DIR__ . '/Fixtures/primitives.parquet'); @@ -240,8 +274,9 @@ public function test_reading_double_column(ParquetEngine $engine): void } #[DataProvider('engine_provider')] - public function test_reading_double_nullable_column(ParquetEngine $engine): void + public function test_reading_double_nullable_column(string $engineClass): void { + $engine = ParquetEngineMother::create($engineClass); $reader = new Reader(engine: $engine); $file = $reader->read(__DIR__ . '/Fixtures/primitives.parquet'); @@ -262,8 +297,9 @@ public function test_reading_double_nullable_column(ParquetEngine $engine): void } #[DataProvider('engine_provider')] - public function test_reading_enum_column(ParquetEngine $engine): void + public function test_reading_enum_column(string $engineClass): void { + $engine = ParquetEngineMother::create($engineClass); $reader = new Reader(engine: $engine); $file = $reader->read(__DIR__ . '/Fixtures/primitives.parquet'); @@ -281,8 +317,9 @@ public function test_reading_enum_column(ParquetEngine $engine): void } #[DataProvider('engine_provider')] - public function test_reading_float_column(ParquetEngine $engine): void + public function test_reading_float_column(string $engineClass): void { + $engine = ParquetEngineMother::create($engineClass); $reader = new Reader(engine: $engine); $file = $reader->read(__DIR__ . '/Fixtures/primitives.parquet'); @@ -299,8 +336,9 @@ public function test_reading_float_column(ParquetEngine $engine): void } #[DataProvider('engine_provider')] - public function test_reading_float_nullable_column(ParquetEngine $engine): void + public function test_reading_float_nullable_column(string $engineClass): void { + $engine = ParquetEngineMother::create($engineClass); $reader = new Reader(engine: $engine); $file = $reader->read(__DIR__ . '/Fixtures/primitives.parquet'); @@ -321,8 +359,9 @@ public function test_reading_float_nullable_column(ParquetEngine $engine): void } #[DataProvider('engine_provider')] - public function test_reading_int32_column(ParquetEngine $engine): void + public function test_reading_int32_column(string $engineClass): void { + $engine = ParquetEngineMother::create($engineClass); $reader = new Reader(engine: $engine); $file = $reader->read(__DIR__ . '/Fixtures/primitives.parquet'); @@ -340,8 +379,9 @@ public function test_reading_int32_column(ParquetEngine $engine): void } #[DataProvider('engine_provider')] - public function test_reading_int32_nullable_column(ParquetEngine $engine): void + public function test_reading_int32_nullable_column(string $engineClass): void { + $engine = ParquetEngineMother::create($engineClass); $reader = new Reader(engine: $engine); $file = $reader->read(__DIR__ . '/Fixtures/primitives.parquet'); @@ -363,8 +403,9 @@ public function test_reading_int32_nullable_column(ParquetEngine $engine): void } #[DataProvider('engine_provider')] - public function test_reading_int64(ParquetEngine $engine): void + public function test_reading_int64(string $engineClass): void { + $engine = ParquetEngineMother::create($engineClass); $reader = new Reader(engine: $engine); $file = $reader->read(__DIR__ . '/Fixtures/primitives.parquet'); @@ -383,8 +424,9 @@ public function test_reading_int64(ParquetEngine $engine): void } #[DataProvider('engine_provider')] - public function test_reading_int64_nullable_column(ParquetEngine $engine): void + public function test_reading_int64_nullable_column(string $engineClass): void { + $engine = ParquetEngineMother::create($engineClass); $reader = new Reader(engine: $engine); $file = $reader->read(__DIR__ . '/Fixtures/primitives.parquet'); @@ -406,8 +448,9 @@ public function test_reading_int64_nullable_column(ParquetEngine $engine): void } #[DataProvider('engine_provider')] - public function test_reading_json_column(ParquetEngine $engine): void + public function test_reading_json_column(string $engineClass): void { + $engine = ParquetEngineMother::create($engineClass); $reader = new Reader(engine: $engine); $file = $reader->read(__DIR__ . '/Fixtures/primitives.parquet'); @@ -425,8 +468,9 @@ public function test_reading_json_column(ParquetEngine $engine): void } #[DataProvider('engine_provider')] - public function test_reading_json_nullable_column(ParquetEngine $engine): void + public function test_reading_json_nullable_column(string $engineClass): void { + $engine = ParquetEngineMother::create($engineClass); $reader = new Reader(engine: $engine); $file = $reader->read(__DIR__ . '/Fixtures/primitives.parquet'); @@ -448,8 +492,9 @@ public function test_reading_json_nullable_column(ParquetEngine $engine): void } #[DataProvider('engine_provider')] - public function test_reading_string_column(ParquetEngine $engine): void + public function test_reading_string_column(string $engineClass): void { + $engine = ParquetEngineMother::create($engineClass); $reader = new Reader(engine: $engine); $file = $reader->read(__DIR__ . '/Fixtures/primitives.parquet'); @@ -467,8 +512,9 @@ public function test_reading_string_column(ParquetEngine $engine): void } #[DataProvider('engine_provider')] - public function test_reading_string_nullable_column(ParquetEngine $engine): void + public function test_reading_string_nullable_column(string $engineClass): void { + $engine = ParquetEngineMother::create($engineClass); $reader = new Reader(engine: $engine); $file = $reader->read(__DIR__ . '/Fixtures/primitives.parquet'); @@ -490,8 +536,9 @@ public function test_reading_string_nullable_column(ParquetEngine $engine): void } #[DataProvider('engine_provider')] - public function test_reading_time_column(ParquetEngine $engine): void + public function test_reading_time_column(string $engineClass): void { + $engine = ParquetEngineMother::create($engineClass); $reader = new Reader(engine: $engine); $file = $reader->read(__DIR__ . '/Fixtures/primitives.parquet'); @@ -509,8 +556,9 @@ public function test_reading_time_column(ParquetEngine $engine): void } #[DataProvider('engine_provider')] - public function test_reading_time_nullable_column(ParquetEngine $engine): void + public function test_reading_time_nullable_column(string $engineClass): void { + $engine = ParquetEngineMother::create($engineClass); $reader = new Reader(engine: $engine); $file = $reader->read(__DIR__ . '/Fixtures/primitives.parquet'); @@ -532,8 +580,9 @@ public function test_reading_time_nullable_column(ParquetEngine $engine): void } #[DataProvider('engine_provider')] - public function test_reading_timestamp_column(ParquetEngine $engine): void + public function test_reading_timestamp_column(string $engineClass): void { + $engine = ParquetEngineMother::create($engineClass); $reader = new Reader(engine: $engine); $file = $reader->read(__DIR__ . '/Fixtures/primitives.parquet'); @@ -551,8 +600,9 @@ public function test_reading_timestamp_column(ParquetEngine $engine): void } #[DataProvider('engine_provider')] - public function test_reading_timestamp_nullable_column(ParquetEngine $engine): void + public function test_reading_timestamp_nullable_column(string $engineClass): void { + $engine = ParquetEngineMother::create($engineClass); $reader = new Reader(engine: $engine); $file = $reader->read(__DIR__ . '/Fixtures/primitives.parquet'); @@ -577,8 +627,9 @@ public function test_reading_timestamp_nullable_column(ParquetEngine $engine): v } #[DataProvider('engine_provider')] - public function test_reading_uuid_column(ParquetEngine $engine): void + public function test_reading_uuid_column(string $engineClass): void { + $engine = ParquetEngineMother::create($engineClass); $reader = new Reader(engine: $engine); $file = $reader->read(__DIR__ . '/Fixtures/primitives.parquet'); @@ -596,8 +647,9 @@ public function test_reading_uuid_column(ParquetEngine $engine): void } #[DataProvider('engine_provider')] - public function test_reading_uuid_nullable_column(ParquetEngine $engine): void + public function test_reading_uuid_nullable_column(string $engineClass): void { + $engine = ParquetEngineMother::create($engineClass); $reader = new Reader(engine: $engine); $file = $reader->read(__DIR__ . '/Fixtures/primitives.parquet'); diff --git a/src/lib/parquet/tests/Flow/Parquet/Tests/Integration/IO/SimpleTypesWritingTest.php b/src/lib/parquet/tests/Flow/Parquet/Tests/Integration/IO/SimpleTypesWritingTest.php index 5fb6cc028a..efb5403813 100644 --- a/src/lib/parquet/tests/Flow/Parquet/Tests/Integration/IO/SimpleTypesWritingTest.php +++ b/src/lib/parquet/tests/Flow/Parquet/Tests/Integration/IO/SimpleTypesWritingTest.php @@ -7,13 +7,13 @@ use DateTimeImmutable; use Faker\Factory; use Flow\Parquet\Consts; -use Flow\Parquet\Engine\ArrowParquetEngine; use Flow\Parquet\Engine\PhpParquetEngine; -use Flow\Parquet\ParquetEngine; +use Flow\Parquet\Engine\RustParquetEngine; use Flow\Parquet\ParquetFile\Schema; use Flow\Parquet\ParquetFile\Schema\FlatColumn; use Flow\Parquet\Reader; use Flow\Parquet\Tests\Context\TestParquetFile; +use Flow\Parquet\Tests\Mother\ParquetEngineMother; use Flow\Parquet\Writer; use PHPUnit\Framework\Attributes\DataProvider; @@ -73,10 +73,10 @@ public static function decimalPrecisionProvider(): array 'precision 8, scale 8' => [8, 8, 0.99999999], ]; - $engines = ['php' => new PhpParquetEngine()]; + $engines = ['php' => PhpParquetEngine::class]; if (extension_loaded('arrow')) { - $engines['arrow'] = new ArrowParquetEngine(); + $engines['arrow'] = RustParquetEngine::class; } $result = []; @@ -91,8 +91,9 @@ public static function decimalPrecisionProvider(): array } #[DataProvider('engine_provider')] - public function test_writing_bool_column(ParquetEngine $engine): void + public function test_writing_bool_column(string $engineClass): void { + $engine = ParquetEngineMother::create($engineClass); $path = TestParquetFile::path($this); $writer = new Writer(engine: $engine); @@ -122,8 +123,9 @@ public function test_writing_bool_column(ParquetEngine $engine): void } #[DataProvider('engine_provider')] - public function test_writing_bool_nullable_column(ParquetEngine $engine): void + public function test_writing_bool_nullable_column(string $engineClass): void { + $engine = ParquetEngineMother::create($engineClass); $path = TestParquetFile::path($this); $writer = new Writer(engine: $engine); @@ -153,8 +155,9 @@ public function test_writing_bool_nullable_column(ParquetEngine $engine): void } #[DataProvider('engine_provider')] - public function test_writing_date_column(ParquetEngine $engine): void + public function test_writing_date_column(string $engineClass): void { + $engine = ParquetEngineMother::create($engineClass); $path = TestParquetFile::path($this); $writer = new Writer(engine: $engine); @@ -186,8 +189,9 @@ public function test_writing_date_column(ParquetEngine $engine): void } #[DataProvider('engine_provider')] - public function test_writing_date_column_before_1970(ParquetEngine $engine): void + public function test_writing_date_column_before_1970(string $engineClass): void { + $engine = ParquetEngineMother::create($engineClass); $path = TestParquetFile::path($this); $writer = new Writer(engine: $engine); @@ -222,8 +226,9 @@ public function test_writing_date_column_before_1970(ParquetEngine $engine): voi } #[DataProvider('engine_provider')] - public function test_writing_date_nullable_column(ParquetEngine $engine): void + public function test_writing_date_nullable_column(string $engineClass): void { + $engine = ParquetEngineMother::create($engineClass); $path = TestParquetFile::path($this); $writer = new Writer(engine: $engine); @@ -257,8 +262,9 @@ public function test_writing_date_nullable_column(ParquetEngine $engine): void } #[DataProvider('engine_provider')] - public function test_writing_decimal_column(ParquetEngine $engine): void + public function test_writing_decimal_column(string $engineClass): void { + $engine = ParquetEngineMother::create($engineClass); $path = TestParquetFile::path($this); $writer = new Writer(engine: $engine); @@ -291,11 +297,12 @@ public function test_writing_decimal_column(ParquetEngine $engine): void #[DataProvider('decimalPrecisionProvider')] public function test_writing_decimal_column_with_different_precisions( - ParquetEngine $engine, + string $engineClass, int $precision, int $scale, float $maxValue, ): void { + $engine = ParquetEngineMother::create($engineClass); $path = TestParquetFile::path($this); $writer = new Writer(engine: $engine); @@ -331,8 +338,9 @@ public function test_writing_decimal_column_with_different_precisions( } #[DataProvider('engine_provider')] - public function test_writing_decimal_nullable_column(ParquetEngine $engine): void + public function test_writing_decimal_nullable_column(string $engineClass): void { + $engine = ParquetEngineMother::create($engineClass); $path = TestParquetFile::path($this); $writer = new Writer(engine: $engine); @@ -364,8 +372,9 @@ public function test_writing_decimal_nullable_column(ParquetEngine $engine): voi } #[DataProvider('engine_provider')] - public function test_writing_double_column(ParquetEngine $engine): void + public function test_writing_double_column(string $engineClass): void { + $engine = ParquetEngineMother::create($engineClass); $path = TestParquetFile::path($this); $writer = new Writer(engine: $engine); @@ -397,8 +406,9 @@ public function test_writing_double_column(ParquetEngine $engine): void } #[DataProvider('engine_provider')] - public function test_writing_double_nullable_column(ParquetEngine $engine): void + public function test_writing_double_nullable_column(string $engineClass): void { + $engine = ParquetEngineMother::create($engineClass); $path = TestParquetFile::path($this); $writer = new Writer(engine: $engine); @@ -430,8 +440,9 @@ public function test_writing_double_nullable_column(ParquetEngine $engine): void } #[DataProvider('engine_provider')] - public function test_writing_enum_column(ParquetEngine $engine): void + public function test_writing_enum_column(string $engineClass): void { + $engine = ParquetEngineMother::create($engineClass); $path = TestParquetFile::path($this); $writer = new Writer(engine: $engine); @@ -463,8 +474,9 @@ public function test_writing_enum_column(ParquetEngine $engine): void } #[DataProvider('engine_provider')] - public function test_writing_float_column(ParquetEngine $engine): void + public function test_writing_float_column(string $engineClass): void { + $engine = ParquetEngineMother::create($engineClass); $path = TestParquetFile::path($this); $writer = new Writer(engine: $engine); @@ -487,8 +499,9 @@ public function test_writing_float_column(ParquetEngine $engine): void } #[DataProvider('engine_provider')] - public function test_writing_float_nullable_column(ParquetEngine $engine): void + public function test_writing_float_nullable_column(string $engineClass): void { + $engine = ParquetEngineMother::create($engineClass); $path = TestParquetFile::path($this); $writer = new Writer(engine: $engine); @@ -513,8 +526,9 @@ public function test_writing_float_nullable_column(ParquetEngine $engine): void } #[DataProvider('engine_provider')] - public function test_writing_int32_column(ParquetEngine $engine): void + public function test_writing_int32_column(string $engineClass): void { + $engine = ParquetEngineMother::create($engineClass); $path = TestParquetFile::path($this); $writer = new Writer(engine: $engine); @@ -546,8 +560,9 @@ public function test_writing_int32_column(ParquetEngine $engine): void } #[DataProvider('engine_provider')] - public function test_writing_int32_nullable_column(ParquetEngine $engine): void + public function test_writing_int32_nullable_column(string $engineClass): void { + $engine = ParquetEngineMother::create($engineClass); $path = TestParquetFile::path($this); $writer = new Writer(engine: $engine); @@ -579,8 +594,9 @@ public function test_writing_int32_nullable_column(ParquetEngine $engine): void } #[DataProvider('engine_provider')] - public function test_writing_int64(ParquetEngine $engine): void + public function test_writing_int64(string $engineClass): void { + $engine = ParquetEngineMother::create($engineClass); $path = TestParquetFile::path($this); $writer = new Writer(engine: $engine); @@ -611,8 +627,9 @@ public function test_writing_int64(ParquetEngine $engine): void } #[DataProvider('engine_provider')] - public function test_writing_int64_nullable_column(ParquetEngine $engine): void + public function test_writing_int64_nullable_column(string $engineClass): void { + $engine = ParquetEngineMother::create($engineClass); $path = TestParquetFile::path($this); $writer = new Writer(engine: $engine); @@ -643,8 +660,9 @@ public function test_writing_int64_nullable_column(ParquetEngine $engine): void } #[DataProvider('engine_provider')] - public function test_writing_json_column(ParquetEngine $engine): void + public function test_writing_json_column(string $engineClass): void { + $engine = ParquetEngineMother::create($engineClass); $path = TestParquetFile::path($this); $writer = new Writer(engine: $engine); @@ -680,8 +698,9 @@ public function test_writing_json_column(ParquetEngine $engine): void } #[DataProvider('engine_provider')] - public function test_writing_json_nullable_column(ParquetEngine $engine): void + public function test_writing_json_nullable_column(string $engineClass): void { + $engine = ParquetEngineMother::create($engineClass); $path = TestParquetFile::path($this); $writer = new Writer(engine: $engine); @@ -719,8 +738,9 @@ public function test_writing_json_nullable_column(ParquetEngine $engine): void } #[DataProvider('engine_provider')] - public function test_writing_string_column(ParquetEngine $engine): void + public function test_writing_string_column(string $engineClass): void { + $engine = ParquetEngineMother::create($engineClass); $path = TestParquetFile::path($this); $writer = new Writer(engine: $engine); @@ -751,8 +771,9 @@ public function test_writing_string_column(ParquetEngine $engine): void } #[DataProvider('engine_provider')] - public function test_writing_string_nullable_column(ParquetEngine $engine): void + public function test_writing_string_nullable_column(string $engineClass): void { + $engine = ParquetEngineMother::create($engineClass); $path = TestParquetFile::path($this); $writer = new Writer(engine: $engine); @@ -783,8 +804,9 @@ public function test_writing_string_nullable_column(ParquetEngine $engine): void } #[DataProvider('engine_provider')] - public function test_writing_time_column(ParquetEngine $engine): void + public function test_writing_time_column(string $engineClass): void { + $engine = ParquetEngineMother::create($engineClass); $path = TestParquetFile::path($this); $writer = new Writer(engine: $engine); @@ -815,8 +837,9 @@ public function test_writing_time_column(ParquetEngine $engine): void } #[DataProvider('engine_provider')] - public function test_writing_time_nullable_column(ParquetEngine $engine): void + public function test_writing_time_nullable_column(string $engineClass): void { + $engine = ParquetEngineMother::create($engineClass); $path = TestParquetFile::path($this); $writer = new Writer(engine: $engine); @@ -849,8 +872,9 @@ public function test_writing_time_nullable_column(ParquetEngine $engine): void } #[DataProvider('engine_provider')] - public function test_writing_timestamp_column(ParquetEngine $engine): void + public function test_writing_timestamp_column(string $engineClass): void { + $engine = ParquetEngineMother::create($engineClass); $path = TestParquetFile::path($this); $writer = new Writer(engine: $engine); @@ -881,8 +905,9 @@ public function test_writing_timestamp_column(ParquetEngine $engine): void } #[DataProvider('engine_provider')] - public function test_writing_timestamp_column_for_years_before_1970(ParquetEngine $engine): void + public function test_writing_timestamp_column_for_years_before_1970(string $engineClass): void { + $engine = ParquetEngineMother::create($engineClass); $path = TestParquetFile::path($this); $writer = new Writer(engine: $engine); @@ -913,8 +938,9 @@ public function test_writing_timestamp_column_for_years_before_1970(ParquetEngin } #[DataProvider('engine_provider')] - public function test_writing_timestamp_nullable_column(ParquetEngine $engine): void + public function test_writing_timestamp_nullable_column(string $engineClass): void { + $engine = ParquetEngineMother::create($engineClass); $path = TestParquetFile::path($this); $writer = new Writer(engine: $engine); @@ -945,8 +971,9 @@ public function test_writing_timestamp_nullable_column(ParquetEngine $engine): v } #[DataProvider('engine_provider')] - public function test_writing_uuid_column(ParquetEngine $engine): void + public function test_writing_uuid_column(string $engineClass): void { + $engine = ParquetEngineMother::create($engineClass); $path = TestParquetFile::path($this); $writer = new Writer(engine: $engine); @@ -977,8 +1004,9 @@ public function test_writing_uuid_column(ParquetEngine $engine): void } #[DataProvider('engine_provider')] - public function test_writing_uuid_nullable_column(ParquetEngine $engine): void + public function test_writing_uuid_nullable_column(string $engineClass): void { + $engine = ParquetEngineMother::create($engineClass); $path = TestParquetFile::path($this); $writer = new Writer(engine: $engine); diff --git a/src/lib/parquet/tests/Flow/Parquet/Tests/Integration/IO/StructsReadingTest.php b/src/lib/parquet/tests/Flow/Parquet/Tests/Integration/IO/StructsReadingTest.php index 6c2779ba9e..e41955f8e9 100644 --- a/src/lib/parquet/tests/Flow/Parquet/Tests/Integration/IO/StructsReadingTest.php +++ b/src/lib/parquet/tests/Flow/Parquet/Tests/Integration/IO/StructsReadingTest.php @@ -4,15 +4,16 @@ namespace Flow\Parquet\Tests\Integration\IO; -use Flow\Parquet\ParquetEngine; use Flow\Parquet\Reader; +use Flow\Parquet\Tests\Mother\ParquetEngineMother; use PHPUnit\Framework\Attributes\DataProvider; class StructsReadingTest extends ParquetIntegrationTestCase { #[DataProvider('engine_provider')] - public function test_reading_struct_deeply_nested_column(ParquetEngine $engine): void + public function test_reading_struct_deeply_nested_column(string $engineClass): void { + $engine = ParquetEngineMother::create($engineClass); $reader = new Reader(engine: $engine); $file = $reader->read(__DIR__ . '/Fixtures/structs.parquet'); @@ -58,8 +59,9 @@ public function test_reading_struct_deeply_nested_column(ParquetEngine $engine): } #[DataProvider('engine_provider')] - public function test_reading_struct_deeply_nested_column_with_limit(ParquetEngine $engine): void + public function test_reading_struct_deeply_nested_column_with_limit(string $engineClass): void { + $engine = ParquetEngineMother::create($engineClass); $reader = new Reader(engine: $engine); $file = $reader->read(__DIR__ . '/Fixtures/structs.parquet'); @@ -104,8 +106,9 @@ public function test_reading_struct_deeply_nested_column_with_limit(ParquetEngin } #[DataProvider('engine_provider')] - public function test_reading_struct_flat_column(ParquetEngine $engine): void + public function test_reading_struct_flat_column(string $engineClass): void { + $engine = ParquetEngineMother::create($engineClass); $reader = new Reader(engine: $engine); $file = $reader->read(__DIR__ . '/Fixtures/structs.parquet'); @@ -129,8 +132,9 @@ public function test_reading_struct_flat_column(ParquetEngine $engine): void } #[DataProvider('engine_provider')] - public function test_reading_struct_flat_nullable_column(ParquetEngine $engine): void + public function test_reading_struct_flat_nullable_column(string $engineClass): void { + $engine = ParquetEngineMother::create($engineClass); $reader = new Reader(engine: $engine); $file = $reader->read(__DIR__ . '/Fixtures/structs.parquet'); @@ -158,8 +162,9 @@ public function test_reading_struct_flat_nullable_column(ParquetEngine $engine): } #[DataProvider('engine_provider')] - public function test_reading_struct_nested_column(ParquetEngine $engine): void + public function test_reading_struct_nested_column(string $engineClass): void { + $engine = ParquetEngineMother::create($engineClass); $reader = new Reader(engine: $engine); $file = $reader->read(__DIR__ . '/Fixtures/structs.parquet'); @@ -187,8 +192,9 @@ public function test_reading_struct_nested_column(ParquetEngine $engine): void } #[DataProvider('engine_provider')] - public function test_reading_struct_nested_with_list_of_lists_column(ParquetEngine $engine): void + public function test_reading_struct_nested_with_list_of_lists_column(string $engineClass): void { + $engine = ParquetEngineMother::create($engineClass); $reader = new Reader(engine: $engine); $file = $reader->read(__DIR__ . '/Fixtures/structs.parquet'); @@ -215,8 +221,9 @@ public function test_reading_struct_nested_with_list_of_lists_column(ParquetEngi } #[DataProvider('engine_provider')] - public function test_reading_struct_nested_with_list_of_maps_column(ParquetEngine $engine): void + public function test_reading_struct_nested_with_list_of_maps_column(string $engineClass): void { + $engine = ParquetEngineMother::create($engineClass); $reader = new Reader(engine: $engine); $file = $reader->read(__DIR__ . '/Fixtures/structs.parquet'); @@ -243,8 +250,9 @@ public function test_reading_struct_nested_with_list_of_maps_column(ParquetEngin } #[DataProvider('engine_provider')] - public function test_reading_struct_nested_with_map_of_list_of_ints_column(ParquetEngine $engine): void + public function test_reading_struct_nested_with_map_of_list_of_ints_column(string $engineClass): void { + $engine = ParquetEngineMother::create($engineClass); $reader = new Reader(engine: $engine); $file = $reader->read(__DIR__ . '/Fixtures/structs.parquet'); @@ -275,8 +283,9 @@ public function test_reading_struct_nested_with_map_of_list_of_ints_column(Parqu } #[DataProvider('engine_provider')] - public function test_reading_struct_nested_with_map_of_string_map_of_string_string_column(ParquetEngine $engine): void + public function test_reading_struct_nested_with_map_of_string_map_of_string_string_column(string $engineClass): void { + $engine = ParquetEngineMother::create($engineClass); $reader = new Reader(engine: $engine); $file = $reader->read(__DIR__ . '/Fixtures/structs.parquet'); @@ -315,8 +324,9 @@ public function test_reading_struct_nested_with_map_of_string_map_of_string_stri } #[DataProvider('engine_provider')] - public function test_reading_struct_nested_with_map_of_string_map_of_string_string_column_with_limit(ParquetEngine $engine): void + public function test_reading_struct_nested_with_map_of_string_map_of_string_string_column_with_limit(string $engineClass): void { + $engine = ParquetEngineMother::create($engineClass); $reader = new Reader(engine: $engine); $file = $reader->read(__DIR__ . '/Fixtures/structs.parquet'); @@ -354,8 +364,9 @@ public function test_reading_struct_nested_with_map_of_string_map_of_string_stri } #[DataProvider('engine_provider')] - public function test_reading_struct_with_list_and_map_of_structs_column(ParquetEngine $engine): void + public function test_reading_struct_with_list_and_map_of_structs_column(string $engineClass): void { + $engine = ParquetEngineMother::create($engineClass); $reader = new Reader(engine: $engine); $file = $reader->read(__DIR__ . '/Fixtures/structs.parquet'); diff --git a/src/lib/parquet/tests/Flow/Parquet/Tests/Integration/IO/StructsWritingTest.php b/src/lib/parquet/tests/Flow/Parquet/Tests/Integration/IO/StructsWritingTest.php index 4b229ebccc..6f4d5829e7 100644 --- a/src/lib/parquet/tests/Flow/Parquet/Tests/Integration/IO/StructsWritingTest.php +++ b/src/lib/parquet/tests/Flow/Parquet/Tests/Integration/IO/StructsWritingTest.php @@ -6,13 +6,13 @@ use Faker\Factory; use Flow\Parquet\Consts; -use Flow\Parquet\ParquetEngine; use Flow\Parquet\ParquetFile\Schema; use Flow\Parquet\ParquetFile\Schema\FlatColumn; use Flow\Parquet\ParquetFile\Schema\ListElement; use Flow\Parquet\ParquetFile\Schema\NestedColumn; use Flow\Parquet\Reader; use Flow\Parquet\Tests\Context\TestParquetFile; +use Flow\Parquet\Tests\Mother\ParquetEngineMother; use Flow\Parquet\Writer; use PHPUnit\Framework\Attributes\DataProvider; @@ -25,8 +25,9 @@ class StructsWritingTest extends ParquetIntegrationTestCase { #[DataProvider('engine_provider')] - public function test_writing_flat_nullable_structure(ParquetEngine $engine): void + public function test_writing_flat_nullable_structure(string $engineClass): void { + $engine = ParquetEngineMother::create($engineClass); $path = TestParquetFile::path($this); $writer = new Writer(engine: $engine); @@ -77,8 +78,9 @@ public function test_writing_flat_nullable_structure(ParquetEngine $engine): voi } #[DataProvider('engine_provider')] - public function test_writing_flat_structure(ParquetEngine $engine): void + public function test_writing_flat_structure(string $engineClass): void { + $engine = ParquetEngineMother::create($engineClass); $path = TestParquetFile::path($this); $writer = new Writer(engine: $engine); @@ -127,8 +129,9 @@ public function test_writing_flat_structure(ParquetEngine $engine): void } #[DataProvider('engine_provider')] - public function test_writing_flat_structure_with_nullable_elements(ParquetEngine $engine): void + public function test_writing_flat_structure_with_nullable_elements(string $engineClass): void { + $engine = ParquetEngineMother::create($engineClass); $path = TestParquetFile::path($this); $writer = new Writer(engine: $engine); diff --git a/src/lib/parquet/tests/Flow/Parquet/Tests/Integration/IO/WriterTest.php b/src/lib/parquet/tests/Flow/Parquet/Tests/Integration/IO/WriterTest.php index c79e2bbb8a..4c4dd581f4 100644 --- a/src/lib/parquet/tests/Flow/Parquet/Tests/Integration/IO/WriterTest.php +++ b/src/lib/parquet/tests/Flow/Parquet/Tests/Integration/IO/WriterTest.php @@ -13,7 +13,6 @@ use Flow\Parquet\Engine\PhpParquetEngine; use Flow\Parquet\Option; use Flow\Parquet\Options; -use Flow\Parquet\ParquetEngine; use Flow\Parquet\ParquetFile\Schema; use Flow\Parquet\ParquetFile\Schema\FlatColumn; use Flow\Parquet\ParquetFile\Schema\ListElement; @@ -22,6 +21,7 @@ use Flow\Parquet\ParquetFile\Schema\NestedColumn; use Flow\Parquet\Reader; use Flow\Parquet\Tests\Context\TestParquetFile; +use Flow\Parquet\Tests\Mother\ParquetEngineMother; use Flow\Parquet\Writer; use PHPUnit\Framework\Attributes\DataProvider; use RuntimeException; @@ -37,8 +37,9 @@ class WriterTest extends ParquetIntegrationTestCase { #[DataProvider('engine_provider')] - public function test_a_close_that_throws_leaves_the_writer_closed(ParquetEngine $engine): void + public function test_a_close_that_throws_leaves_the_writer_closed(string $engineClass): void { + $engine = ParquetEngineMother::create($engineClass); $writer = new Writer(engine: $engine); $writer->openForStream( new FailingCloseDestinationStream(memory_filesystem()->writeTo(path('memory://file.parquet'))), @@ -62,8 +63,9 @@ public function test_a_close_that_throws_leaves_the_writer_closed(ParquetEngine } #[DataProvider('engine_provider')] - public function test_closing_not_open_writer(ParquetEngine $engine): void + public function test_closing_not_open_writer(string $engineClass): void { + $engine = ParquetEngineMother::create($engineClass); $writer = new Writer(engine: $engine); $this->expectException(RuntimeException::class); @@ -91,8 +93,9 @@ public function test_created_by_metadata(): void } #[DataProvider('engine_provider')] - public function test_opening_already_open_writer(ParquetEngine $engine): void + public function test_opening_already_open_writer(string $engineClass): void { + $engine = ParquetEngineMother::create($engineClass); $writer = new Writer(engine: $engine); $path = TestParquetFile::path($this); @@ -108,8 +111,9 @@ public function test_opening_already_open_writer(ParquetEngine $engine): void } #[DataProvider('engine_provider')] - public function test_writers_sharing_one_engine_write_independent_files(ParquetEngine $engine): void + public function test_writers_sharing_one_engine_write_independent_files(string $engineClass): void { + $engine = ParquetEngineMother::create($engineClass); $memory = memory_filesystem(); $schema = Schema::with(FlatColumn::int32('id')); $first = new Writer(engine: $engine); @@ -144,8 +148,9 @@ public function test_writers_sharing_one_engine_write_independent_files(ParquetE } #[DataProvider('engine_provider')] - public function test_writing_all_column_types(ParquetEngine $engine): void + public function test_writing_all_column_types(string $engineClass): void { + $engine = ParquetEngineMother::create($engineClass); $schema = Schema::with( FlatColumn::int32('int32_col'), FlatColumn::int64('int64_col'), @@ -212,8 +217,9 @@ public function test_writing_all_column_types(ParquetEngine $engine): void } #[DataProvider('engine_provider')] - public function test_writing_batch_to_not_open_stream(ParquetEngine $engine): void + public function test_writing_batch_to_not_open_stream(string $engineClass): void { + $engine = ParquetEngineMother::create($engineClass); $writer = new Writer(engine: $engine); $this->expectException(RuntimeException::class); @@ -223,8 +229,9 @@ public function test_writing_batch_to_not_open_stream(ParquetEngine $engine): vo } #[DataProvider('engine_provider')] - public function test_writing_column_statistics(ParquetEngine $engine): void + public function test_writing_column_statistics(string $engineClass): void { + $engine = ParquetEngineMother::create($engineClass); $writer = new Writer(options: Options::default()->set(Option::WRITER_VERSION, 1), engine: $engine); $path = TestParquetFile::path($this); @@ -250,8 +257,9 @@ public function test_writing_column_statistics(ParquetEngine $engine): void } #[DataProvider('engine_provider')] - public function test_writing_column_statistics_with_null_values(ParquetEngine $engine): void + public function test_writing_column_statistics_with_null_values(string $engineClass): void { + $engine = ParquetEngineMother::create($engineClass); $schema = Schema::with( FlatColumn::string('all_null'), FlatColumn::string('all_string'), @@ -311,8 +319,9 @@ public function test_writing_data_page_v2_statistics(): void } #[DataProvider('engine_provider')] - public function test_writing_in_batches_to_file(ParquetEngine $engine): void + public function test_writing_in_batches_to_file(string $engineClass): void { + $engine = ParquetEngineMother::create($engineClass); $writer = new Writer(engine: $engine); $path = TestParquetFile::path($this); @@ -342,8 +351,9 @@ public function test_writing_in_batches_to_file(ParquetEngine $engine): void } #[DataProvider('engine_provider')] - public function test_writing_in_batches_to_file_without_explicit_close(ParquetEngine $engine): void + public function test_writing_in_batches_to_file_without_explicit_close(string $engineClass): void { + $engine = ParquetEngineMother::create($engineClass); $writer = new Writer(engine: $engine); $path = TestParquetFile::path($this); @@ -372,8 +382,9 @@ public function test_writing_in_batches_to_file_without_explicit_close(ParquetEn } #[DataProvider('engine_provider')] - public function test_writing_in_batches_to_stream(ParquetEngine $engine): void + public function test_writing_in_batches_to_stream(string $engineClass): void { + $engine = ParquetEngineMother::create($engineClass); $writer = new Writer(engine: $engine); $path = TestParquetFile::path($this); @@ -404,8 +415,9 @@ public function test_writing_in_batches_to_stream(ParquetEngine $engine): void } #[DataProvider('engine_provider')] - public function test_writing_one_row_that_is_nullable(ParquetEngine $engine): void + public function test_writing_one_row_that_is_nullable(string $engineClass): void { + $engine = ParquetEngineMother::create($engineClass); $writer = new Writer(engine: $engine); $schema = Schema::with($column = FlatColumn::int32('id')); @@ -447,8 +459,9 @@ public function test_writing_one_row_that_is_nullable(ParquetEngine $engine): vo } #[DataProvider('engine_provider')] - public function test_writing_row_to_not_open_stream(ParquetEngine $engine): void + public function test_writing_row_to_not_open_stream(string $engineClass): void { + $engine = ParquetEngineMother::create($engineClass); $writer = new Writer(engine: $engine); $this->expectException(RuntimeException::class); @@ -458,8 +471,9 @@ public function test_writing_row_to_not_open_stream(ParquetEngine $engine): void } #[DataProvider('engine_provider')] - public function test_writing_to_file(ParquetEngine $engine): void + public function test_writing_to_file(string $engineClass): void { + $engine = ParquetEngineMother::create($engineClass); $writer = new Writer(engine: $engine); $path = TestParquetFile::path($this); @@ -481,8 +495,9 @@ public function test_writing_to_file(ParquetEngine $engine): void } #[DataProvider('engine_provider')] - public function test_writing_to_file_v2(ParquetEngine $engine): void + public function test_writing_to_file_v2(string $engineClass): void { + $engine = ParquetEngineMother::create($engineClass); $writer = new Writer(options: Options::default()->set(Option::WRITER_VERSION, 2), engine: $engine); $path = TestParquetFile::path($this); @@ -511,8 +526,9 @@ public function test_writing_to_file_v2(ParquetEngine $engine): void } #[DataProvider('engine_provider')] - public function test_writing_to_stream(ParquetEngine $engine): void + public function test_writing_to_stream(string $engineClass): void { + $engine = ParquetEngineMother::create($engineClass); $writer = new Writer(engine: $engine); $path = TestParquetFile::path($this); @@ -547,8 +563,9 @@ public function test_writing_to_stream(ParquetEngine $engine): void } #[DataProvider('engine_provider')] - public function test_writing_with_all_nullable_columns(ParquetEngine $engine): void + public function test_writing_with_all_nullable_columns(string $engineClass): void { + $engine = ParquetEngineMother::create($engineClass); $schema = Schema::with(FlatColumn::int32('a'), FlatColumn::string('b'), FlatColumn::boolean('c')); $rows = [ @@ -573,8 +590,9 @@ public function test_writing_with_all_nullable_columns(ParquetEngine $engine): v } #[DataProvider('engine_provider')] - public function test_writing_with_dictionary_encoding(ParquetEngine $engine): void + public function test_writing_with_dictionary_encoding(string $engineClass): void { + $engine = ParquetEngineMother::create($engineClass); $schema = Schema::with(FlatColumn::int32('id')->makeRequired(), FlatColumn::string('category')); $rows = []; @@ -598,8 +616,9 @@ public function test_writing_with_dictionary_encoding(ParquetEngine $engine): vo } #[DataProvider('engine_provider')] - public function test_writing_with_empty_lists_and_maps(ParquetEngine $engine): void + public function test_writing_with_empty_lists_and_maps(string $engineClass): void { + $engine = ParquetEngineMother::create($engineClass); $schema = Schema::with( FlatColumn::int32('id')->makeRequired(), NestedColumn::list('items', ListElement::string()), diff --git a/src/lib/parquet/tests/Flow/Parquet/Tests/Integration/ParquetFile/SchemaTest.php b/src/lib/parquet/tests/Flow/Parquet/Tests/Integration/ParquetFile/SchemaTest.php index be7fc48d5f..eb468d4b70 100644 --- a/src/lib/parquet/tests/Flow/Parquet/Tests/Integration/ParquetFile/SchemaTest.php +++ b/src/lib/parquet/tests/Flow/Parquet/Tests/Integration/ParquetFile/SchemaTest.php @@ -37,7 +37,7 @@ public function test_dremel_paper_data_schema(): void foreach ($rows as $row) { /** @var array $row */ static::assertIsArray($row); - $shredResult = $shredder->shred($schema, [$row]); + $shredResult = $shredder->shred($schema, [$row], 0); foreach ($schema->columns() as $column) { $readFlatValues = []; diff --git a/src/lib/parquet/tests/Flow/Parquet/Tests/Mother/ParquetEngineMother.php b/src/lib/parquet/tests/Flow/Parquet/Tests/Mother/ParquetEngineMother.php new file mode 100644 index 0000000000..62e54afe08 --- /dev/null +++ b/src/lib/parquet/tests/Flow/Parquet/Tests/Mother/ParquetEngineMother.php @@ -0,0 +1,22 @@ + new RustParquetEngine(), + PhpParquetEngine::class => new PhpParquetEngine(), + default => throw new InvalidArgumentException("Unknown Parquet engine {$class}"), + }; + } +} diff --git a/src/lib/parquet/tests/Flow/Parquet/Tests/Unit/Dremel/ColumnData/PagedFlatColumnValuesTest.php b/src/lib/parquet/tests/Flow/Parquet/Tests/Unit/Dremel/ColumnData/PagedFlatColumnValuesTest.php new file mode 100644 index 0000000000..eceb9c59c9 --- /dev/null +++ b/src/lib/parquet/tests/Flow/Parquet/Tests/Unit/Dremel/ColumnData/PagedFlatColumnValuesTest.php @@ -0,0 +1,97 @@ +>}> + */ + public static function nested(): Generator + { + yield 'list' => [ + Schema::with(NestedColumn::list('l', ListElement::int32())), + 'l', + [['l' => [1, 2]], ['l' => null], ['l' => [3, null]], ['l' => []], ['l' => [4]]], + ]; + yield 'map' => [ + Schema::with(NestedColumn::map('m', MapKey::string(), MapValue::int32())), + 'm', + [['m' => ['a' => 1, 'b' => 2]], ['m' => null], ['m' => ['c' => null]], ['m' => ['d' => 4]]], + ]; + yield 'struct' => [ + Schema::with(NestedColumn::struct('s', [FlatColumn::int32('a'), FlatColumn::string('b')])), + 's', + [ + ['s' => ['a' => 1, 'b' => 'x']], + ['s' => null], + ['s' => ['a' => null, 'b' => 'y']], + ['s' => [ + 'a' => 4, + 'b' => null, + ]], + ], + ]; + } + + /** + * @param array> $rows + */ + #[DataProvider('nested')] + public function test_two_pages_assemble_as_one_merged_page(Schema $schema, string $name, array $rows): void + { + $assembler = new DremelAssembler(DataConverter::initialize(Options::default())); + $column = $schema->get($name); + + $merged = iterator_to_array($assembler->assemble( + $column, + new ReadColumnData($column, FlatColumnPages::merged($schema, $rows)), + )); + + static::assertSame($rows, $merged); + static::assertSame( + $merged, + iterator_to_array($assembler->assemble( + $column, + new ReadColumnData($column, FlatColumnPages::paged($schema, $rows, 2)), + )), + ); + } + + /** + * @param array> $rows + */ + #[DataProvider('nested')] + public function test_two_pages_iterate_as_one_merged_page(Schema $schema, string $name, array $rows): void + { + $merged = FlatColumnPages::merged($schema, $rows); + $paged = FlatColumnPages::paged($schema, $rows, 2); + + foreach ($merged as $index => $page) { + static::assertSame($page->flatPath(), $paged[$index]->flatPath()); + static::assertEquals( + iterator_to_array($page->iterator(), false), + iterator_to_array($paged[$index]->iterator()), + ); + } + } +} diff --git a/src/lib/parquet/tests/Flow/Parquet/Tests/Unit/Dremel/ColumnData/WriteFlatColumnValuesTest.php b/src/lib/parquet/tests/Flow/Parquet/Tests/Unit/Dremel/ColumnData/WriteFlatColumnValuesTest.php index 46a2d5370d..ddc579ef7a 100644 --- a/src/lib/parquet/tests/Flow/Parquet/Tests/Unit/Dremel/ColumnData/WriteFlatColumnValuesTest.php +++ b/src/lib/parquet/tests/Flow/Parquet/Tests/Unit/Dremel/ColumnData/WriteFlatColumnValuesTest.php @@ -92,7 +92,7 @@ public function test_skip_rows_with_struct_containing_list_of_strings(): void )); $shredder = new DremelShredder(new ColumnDataValidator(), DataConverter::initialize(Options::default())); - $result = $shredder->shred($schema, $rows); + $result = $shredder->shred($schema, $rows, 0); $flatColumnValues = $result['struct.list_of_string.list.element']; static::assertSame(10, $flatColumnValues->rowsCount()); @@ -203,7 +203,7 @@ public function test_split_by_rows_with_struct_containing_list_of_strings(): voi )); $shredder = new DremelShredder(new ColumnDataValidator(), DataConverter::initialize(Options::default())); - $result = $shredder->shred($schema, $rows); + $result = $shredder->shred($schema, $rows, 0); $flatColumnValues = $result['struct.list_of_string.list.element']; $splitResult = $flatColumnValues->splitByRows(20); diff --git a/src/lib/parquet/tests/Flow/Parquet/Tests/Unit/Dremel/DremelFlatTest.php b/src/lib/parquet/tests/Flow/Parquet/Tests/Unit/Dremel/DremelFlatTest.php index 4e07edb81d..c1572efa8d 100644 --- a/src/lib/parquet/tests/Flow/Parquet/Tests/Unit/Dremel/DremelFlatTest.php +++ b/src/lib/parquet/tests/Flow/Parquet/Tests/Unit/Dremel/DremelFlatTest.php @@ -52,7 +52,7 @@ public function test_optional_int32(array $row, array $flatData): void static::assertEquals(0, $schema->get('int32')->repetitions()->maxRepetitionLevel()); $shredder = new DremelShredder(new ColumnDataValidator(), DataConverter::initialize(Options::default())); - $shredResult = $shredder->shred($schema, [$row]); + $shredResult = $shredder->shred($schema, [$row], 0); $normalized = []; @@ -118,9 +118,9 @@ public function test_required_int32(array $row, array $flatData, ?string $except if ($exceptionMessage) { $this->expectExceptionMessage($exceptionMessage); - $shredder->shred($schema, [$row]); + $shredder->shred($schema, [$row], 0); } else { - $shredResult = $shredder->shred($schema, [$row]); + $shredResult = $shredder->shred($schema, [$row], 0); $normalized = []; diff --git a/src/lib/parquet/tests/Flow/Parquet/Tests/Unit/Dremel/DremelListsTest.php b/src/lib/parquet/tests/Flow/Parquet/Tests/Unit/Dremel/DremelListsTest.php index 29872a9fbb..35475ba76e 100644 --- a/src/lib/parquet/tests/Flow/Parquet/Tests/Unit/Dremel/DremelListsTest.php +++ b/src/lib/parquet/tests/Flow/Parquet/Tests/Unit/Dremel/DremelListsTest.php @@ -102,7 +102,7 @@ public function test_optional_list_optional_int32(array $rows, array $expectedCo static::assertEquals(3, $schema->get('l.list.element')->repetitions()->maxDefinitionLevel()); static::assertEquals(1, $schema->get('l.list.element')->repetitions()->maxRepetitionLevel()); - $result = $shredder->shred($schema, $rows); + $result = $shredder->shred($schema, $rows, 0); $normalized = []; @@ -267,7 +267,7 @@ public function test_optional_list_optional_list_optional_int32(array $rows, arr static::assertEquals(5, $schema->get('l.list.element.list.element')->repetitions()->maxDefinitionLevel()); static::assertEquals(2, $schema->get('l.list.element.list.element')->repetitions()->maxRepetitionLevel()); - $result = $shredder->shred($schema, $rows); + $result = $shredder->shred($schema, $rows, 0); $normalized = []; @@ -387,7 +387,7 @@ public function test_optional_list_optional_list_optional_map_string_optional_in $schema->get('l.list.element.list.element.key_value.value')->repetitions()->maxRepetitionLevel(), ); - $result = $shredder->shred($schema, $rows); + $result = $shredder->shred($schema, $rows, 0); $normalized = []; @@ -586,7 +586,7 @@ public function test_optional_list_optional_list_optional_map_string_optional_st ->maxRepetitionLevel(), ); - $result = $shredder->shred($schema, $rows); + $result = $shredder->shred($schema, $rows, 0); $normalized = []; @@ -701,7 +701,7 @@ public function test_optional_list_optional_list_optional_struct_optional_int32_ $schema->get('l.list.element.list.element.string')->repetitions()->maxRepetitionLevel(), ); - $result = $shredder->shred($schema, $rows); + $result = $shredder->shred($schema, $rows, 0); $normalized = []; @@ -925,9 +925,9 @@ public function test_optional_list_optional_map_string_optional_int32( if ($exceptionMessage) { $this->expectExceptionMessage($exceptionMessage); - $shredder->shred($schema, $rows); + $shredder->shred($schema, $rows, 0); } else { - $result = $shredder->shred($schema, $rows); + $result = $shredder->shred($schema, $rows, 0); $normalized = []; @@ -1097,7 +1097,7 @@ public function test_optional_list_optional_struct_optional_int32_optional_list_ static::assertEquals(6, $schema->get('l.list.element.l.list.element')->repetitions()->maxDefinitionLevel()); static::assertEquals(2, $schema->get('l.list.element.l.list.element')->repetitions()->maxRepetitionLevel()); - $result = $shredder->shred($schema, $rows); + $result = $shredder->shred($schema, $rows, 0); $readFlatValues = []; @@ -1236,7 +1236,7 @@ public function test_optional_list_optional_struct_optional_int32_optional_map_i static::assertEquals(6, $schema->get('l.list.element.m.key_value.value')->repetitions()->maxDefinitionLevel()); static::assertEquals(2, $schema->get('l.list.element.m.key_value.value')->repetitions()->maxRepetitionLevel()); - $result = $shredder->shred($schema, $rows); + $result = $shredder->shred($schema, $rows, 0); $normalized = []; @@ -1443,7 +1443,7 @@ public function test_optional_list_optional_struct_optional_int32_optional_strin static::assertEquals(4, $schema->get('l.list.element.string')->repetitions()->maxDefinitionLevel()); static::assertEquals(1, $schema->get('l.list.element.string')->repetitions()->maxRepetitionLevel()); - $result = $shredder->shred($schema, $rows); + $result = $shredder->shred($schema, $rows, 0); $normalized = []; @@ -1532,7 +1532,7 @@ public function test_optional_list_optional_struct_optional_struct_optional_int3 static::assertEquals(5, $schema->get('l.list.element.s.string')->repetitions()->maxDefinitionLevel()); static::assertEquals(1, $schema->get('l.list.element.s.string')->repetitions()->maxRepetitionLevel()); - $result = $shredder->shred($schema, $rows); + $result = $shredder->shred($schema, $rows, 0); $normalized = []; @@ -1605,7 +1605,7 @@ public function test_optional_list_required_int32(array $rows, array $expectedCo static::assertEquals(2, $schema->get('l.list.element')->repetitions()->maxDefinitionLevel()); static::assertEquals(1, $schema->get('l.list.element')->repetitions()->maxRepetitionLevel()); - $result = $shredder->shred($schema, $rows); + $result = $shredder->shred($schema, $rows, 0); $normalized = []; @@ -1666,7 +1666,7 @@ public function test_required_list_required_int32(array $rows, array $expectedFl static::assertEquals(1, $schema->get('l.list.element')->repetitions()->maxDefinitionLevel()); static::assertEquals(1, $schema->get('l.list.element')->repetitions()->maxRepetitionLevel()); - $result = $shredder->shred($schema, $rows); + $result = $shredder->shred($schema, $rows, 0); $normalized = []; @@ -1734,7 +1734,7 @@ public function test_required_list_required_list_required_int32(array $rows, arr static::assertEquals(2, $schema->get('l.list.element.list.element')->repetitions()->maxDefinitionLevel()); static::assertEquals(2, $schema->get('l.list.element.list.element')->repetitions()->maxRepetitionLevel()); - $result = $shredder->shred($schema, $rows); + $result = $shredder->shred($schema, $rows, 0); $normalized = []; @@ -1840,9 +1840,9 @@ public function test_required_list_required_map_string_required_int32( if ($exceptionMessage) { $this->expectExceptionMessage($exceptionMessage); - $shredder->shred($schema, $rows); + $shredder->shred($schema, $rows, 0); } else { - $result = $shredder->shred($schema, $rows); + $result = $shredder->shred($schema, $rows, 0); $normalized = []; @@ -1978,9 +1978,9 @@ public function test_required_list_required_struct_required_int32_required_strin if ($exceptMessage) { $this->expectExceptionMessage($exceptMessage); - $shredder->shred($schema, $rows); + $shredder->shred($schema, $rows, 0); } else { - $result = $shredder->shred($schema, $rows); + $result = $shredder->shred($schema, $rows, 0); $normalized = []; diff --git a/src/lib/parquet/tests/Flow/Parquet/Tests/Unit/Dremel/DremelMapsTest.php b/src/lib/parquet/tests/Flow/Parquet/Tests/Unit/Dremel/DremelMapsTest.php index c9864c3415..13c0b7080d 100644 --- a/src/lib/parquet/tests/Flow/Parquet/Tests/Unit/Dremel/DremelMapsTest.php +++ b/src/lib/parquet/tests/Flow/Parquet/Tests/Unit/Dremel/DremelMapsTest.php @@ -141,9 +141,9 @@ public function test_optional_map_string_optional_int32( if ($exceptionMessage) { $this->expectExceptionMessage($exceptionMessage); - $shredder->shred($schema, $rows); + $shredder->shred($schema, $rows, 0); } else { - $result = $shredder->shred($schema, $rows); + $result = $shredder->shred($schema, $rows, 0); $normalized = []; @@ -350,9 +350,9 @@ public function test_optional_map_string_optional_list_optional_int32( if ($exceptionMessage) { $this->expectExceptionMessage($exceptionMessage); - $shredder->shred($schema, $rows); + $shredder->shred($schema, $rows, 0); } else { - $result = $shredder->shred($schema, $rows); + $result = $shredder->shred($schema, $rows, 0); $normalized = []; @@ -612,9 +612,9 @@ public function test_optional_map_string_optional_map_string_optional_int( if ($exceptionMessage) { $this->expectExceptionMessage($exceptionMessage); - $shredder->shred($schema, $rows); + $shredder->shred($schema, $rows, 0); } else { - $result = $shredder->shred($schema, $rows); + $result = $shredder->shred($schema, $rows, 0); $normalized = []; @@ -841,9 +841,9 @@ public function test_optional_map_string_optional_struct_optional_int32_optional if ($exceptionMessage) { $this->expectExceptionMessage($exceptionMessage); - $shredder->shred($schema, $rows); + $shredder->shred($schema, $rows, 0); } else { - $result = $shredder->shred($schema, $rows); + $result = $shredder->shred($schema, $rows, 0); $normalized = []; @@ -958,9 +958,9 @@ public function test_optional_map_string_optional_struct_optional_list_optional_ if ($exceptionMessage) { $this->expectExceptionMessage($exceptionMessage); - $shredder->shred($schema, $rows); + $shredder->shred($schema, $rows, 0); } else { - $result = $shredder->shred($schema, $rows); + $result = $shredder->shred($schema, $rows, 0); $normalized = []; @@ -1055,9 +1055,9 @@ public function test_optional_map_string_required_int32( if ($exceptionMessage) { $this->expectExceptionMessage($exceptionMessage); - $shredder->shred($schema, $rows); + $shredder->shred($schema, $rows, 0); } else { - $result = $shredder->shred($schema, $rows); + $result = $shredder->shred($schema, $rows, 0); $normalized = []; @@ -1149,9 +1149,9 @@ public function test_required_map_string_required_int32( if ($exceptionMessage) { $this->expectExceptionMessage($exceptionMessage); - $shredder->shred($schema, $rows); + $shredder->shred($schema, $rows, 0); } else { - $result = $shredder->shred($schema, $rows); + $result = $shredder->shred($schema, $rows, 0); $normalized = []; @@ -1248,9 +1248,9 @@ public function test_required_map_string_required_list_required_int32( if ($exceptionMessage) { $this->expectExceptionMessage($exceptionMessage); - $shredder->shred($schema, $rows); + $shredder->shred($schema, $rows, 0); } else { - $result = $shredder->shred($schema, $rows); + $result = $shredder->shred($schema, $rows, 0); $normalized = []; @@ -1377,9 +1377,9 @@ public function test_required_map_string_required_map_string_required_int( if ($exceptionMessage) { $this->expectExceptionMessage($exceptionMessage); - $shredder->shred($schema, $rows); + $shredder->shred($schema, $rows, 0); } else { - $result = $shredder->shred($schema, $rows); + $result = $shredder->shred($schema, $rows, 0); $normalized = []; @@ -1528,9 +1528,9 @@ public function test_required_map_string_required_struct_required_int32_required if ($exceptionMessage) { $this->expectExceptionMessage($exceptionMessage); - $shredder->shred($schema, $rows); + $shredder->shred($schema, $rows, 0); } else { - $result = $shredder->shred($schema, $rows); + $result = $shredder->shred($schema, $rows, 0); $normalized = []; diff --git a/src/lib/parquet/tests/Flow/Parquet/Tests/Unit/Dremel/DremelShredderTest.php b/src/lib/parquet/tests/Flow/Parquet/Tests/Unit/Dremel/DremelShredderTest.php index 8409356d32..4046cdeca7 100644 --- a/src/lib/parquet/tests/Flow/Parquet/Tests/Unit/Dremel/DremelShredderTest.php +++ b/src/lib/parquet/tests/Flow/Parquet/Tests/Unit/Dremel/DremelShredderTest.php @@ -5,7 +5,9 @@ namespace Flow\Parquet\Tests\Unit\Dremel; use Flow\Parquet\Dremel\DremelShredder; +use Flow\Parquet\Dremel\Validator\ColumnDataValidator; use Flow\Parquet\Dremel\Validator\DisabledValidator; +use Flow\Parquet\Exception\ValidationException; use Flow\Parquet\Options; use Flow\Parquet\ParquetFile\Data\DataConverter; use Flow\Parquet\ParquetFile\Schema; @@ -55,7 +57,7 @@ public function test_complex_schema(): void ]; $shredder = new DremelShredder(new DisabledValidator(), DataConverter::initialize(Options::default())); - $result = $shredder->shred($schema, $rows); + $result = $shredder->shred($schema, $rows, 0); static::assertSame( [ @@ -87,7 +89,7 @@ public function test_flat_optional_column(): void $rows = [['id' => 1], ['id' => null], ['id' => 3]]; $shredder = new DremelShredder(new DisabledValidator(), DataConverter::initialize(Options::default())); - $result = $shredder->shred($schema, $rows); + $result = $shredder->shred($schema, $rows, 0); static::assertSame(['id'], array_keys($result)); static::assertSame([1, 3], $result['id']->values()); @@ -101,7 +103,7 @@ public function test_flat_optional_column_all_nulls(): void $rows = [['id' => null], ['id' => null], ['id' => null]]; $shredder = new DremelShredder(new DisabledValidator(), DataConverter::initialize(Options::default())); - $result = $shredder->shred($schema, $rows); + $result = $shredder->shred($schema, $rows, 0); static::assertSame(['id'], array_keys($result)); static::assertSame([], $result['id']->values()); @@ -115,7 +117,7 @@ public function test_flat_required_column(): void $rows = [['id' => 1], ['id' => 2], ['id' => 3]]; $shredder = new DremelShredder(new DisabledValidator(), DataConverter::initialize(Options::default())); - $result = $shredder->shred($schema, $rows); + $result = $shredder->shred($schema, $rows, 0); static::assertSame(['id'], array_keys($result)); static::assertSame([1, 2, 3], $result['id']->values()); @@ -134,7 +136,7 @@ public function test_list_of_flat_values(): void ]; $shredder = new DremelShredder(new DisabledValidator(), DataConverter::initialize(Options::default())); - $result = $shredder->shred($schema, $rows); + $result = $shredder->shred($schema, $rows, 0); static::assertSame(['tags.list.element'], array_keys($result)); static::assertSame(['a', 'b', 'c', 'd'], $result['tags.list.element']->values()); @@ -156,7 +158,7 @@ public function test_list_of_structs(): void ]; $shredder = new DremelShredder(new DisabledValidator(), DataConverter::initialize(Options::default())); - $result = $shredder->shred($schema, $rows); + $result = $shredder->shred($schema, $rows, 0); static::assertSame(['items.list.element.id', 'items.list.element.name'], array_keys($result)); static::assertSame([1, 2, 3], $result['items.list.element.id']->values()); @@ -174,7 +176,7 @@ public function test_map_of_flat_values(): void ]; $shredder = new DremelShredder(new DisabledValidator(), DataConverter::initialize(Options::default())); - $result = $shredder->shred($schema, $rows); + $result = $shredder->shred($schema, $rows, 0); static::assertSame(['props.key_value.key', 'props.key_value.value'], array_keys($result)); static::assertSame(['a', 'b', 'c'], $result['props.key_value.key']->values()); @@ -198,7 +200,7 @@ public function test_map_with_struct_values(): void ]; $shredder = new DremelShredder(new DisabledValidator(), DataConverter::initialize(Options::default())); - $result = $shredder->shred($schema, $rows); + $result = $shredder->shred($schema, $rows, 0); static::assertSame( ['data.key_value.key', 'data.key_value.value.x', 'data.key_value.value.y'], @@ -219,7 +221,7 @@ public function test_multiple_flat_columns(): void ]; $shredder = new DremelShredder(new DisabledValidator(), DataConverter::initialize(Options::default())); - $result = $shredder->shred($schema, $rows); + $result = $shredder->shred($schema, $rows, 0); static::assertSame(['id', 'name', 'active'], array_keys($result)); static::assertSame([1, 3], $result['id']->values()); @@ -238,7 +240,7 @@ public function test_nested_list_of_lists(): void ]; $shredder = new DremelShredder(new DisabledValidator(), DataConverter::initialize(Options::default())); - $result = $shredder->shred($schema, $rows); + $result = $shredder->shred($schema, $rows, 0); static::assertSame(['matrix.list.element.list.element'], array_keys($result)); static::assertSame([1, 2, 3, 4], $result['matrix.list.element.list.element']->values()); @@ -257,10 +259,86 @@ public function test_struct_with_flat_children(): void ]; $shredder = new DremelShredder(new DisabledValidator(), DataConverter::initialize(Options::default())); - $result = $shredder->shred($schema, $rows); + $result = $shredder->shred($schema, $rows, 0); static::assertSame(['s.a', 's.b'], array_keys($result)); static::assertSame([1], $result['s.a']->values()); static::assertSame(['x', 'y'], $result['s.b']->values()); } + + public function test_shred_columns_equals_shred_of_the_same_rows(): void + { + $schema = Schema::with( + FlatColumn::int32('id')->makeRequired(), + FlatColumn::string('name'), + NestedColumn::structure('address', [FlatColumn::string('city'), FlatColumn::string('zip')]), + NestedColumn::list('tags', ListElement::string()), + NestedColumn::map('metadata', MapKey::string(), MapValue::int32()), + ); + $rows = [ + [ + 'id' => 1, + 'name' => 'alice', + 'address' => ['city' => 'NYC', 'zip' => '10001'], + 'tags' => ['a', 'b'], + 'metadata' => ['k' => 1], + ], + ['id' => 2, 'name' => null, 'address' => null, 'tags' => null, 'metadata' => null], + [ + 'id' => 3, + 'name' => 'charlie', + 'address' => ['city' => 'LA', 'zip' => null], + 'tags' => [], + 'metadata' => [], + ], + ]; + $shredder = new DremelShredder(new DisabledValidator(), DataConverter::initialize(Options::default())); + + static::assertEquals( + $shredder->shred($schema, $rows, 0), + $shredder->shredColumns( + $schema, + [ + 'id' => [1, 2, 3], + 'name' => ['alice', null, 'charlie'], + 'address' => [['city' => 'NYC', 'zip' => '10001'], null, ['city' => 'LA', 'zip' => null]], + 'tags' => [['a', 'b'], null, []], + 'metadata' => [['k' => 1], null, []], + ], + 3, + 0, + ), + ); + } + + public function test_shred_columns_writes_a_column_the_array_lacks_as_nulls(): void + { + $result = (new DremelShredder( + new DisabledValidator(), + DataConverter::initialize(Options::default()), + ))->shredColumns( + Schema::with(FlatColumn::int32('id'), FlatColumn::string('name')), + ['id' => [1, 2], 'unknown' => ['x', 'y']], + 2, + 0, + ); + + static::assertSame(['id', 'name'], array_keys($result)); + static::assertSame([1, 2], $result['id']->values()); + static::assertSame([], $result['name']->values()); + static::assertSame([0, 0], $result['name']->definitionLevels()); + } + + public function test_shred_columns_names_the_writer_row_in_a_refusal(): void + { + $this->expectException(ValidationException::class); + $this->expectExceptionMessage('Column "id" row 12: '); + + (new DremelShredder(new ColumnDataValidator(), DataConverter::initialize(Options::default())))->shredColumns( + Schema::with(FlatColumn::int32('id')), + ['id' => [1, 2, 4_000_000_000]], + 3, + 10, + ); + } } diff --git a/src/lib/parquet/tests/Flow/Parquet/Tests/Unit/Dremel/DremelStructuresTest.php b/src/lib/parquet/tests/Flow/Parquet/Tests/Unit/Dremel/DremelStructuresTest.php index b4afac63bc..98aebb8fca 100644 --- a/src/lib/parquet/tests/Flow/Parquet/Tests/Unit/Dremel/DremelStructuresTest.php +++ b/src/lib/parquet/tests/Flow/Parquet/Tests/Unit/Dremel/DremelStructuresTest.php @@ -17,6 +17,10 @@ use Flow\Parquet\ParquetFile\Schema\MapKey; use Flow\Parquet\ParquetFile\Schema\MapValue; use Flow\Parquet\ParquetFile\Schema\NestedColumn; +use Flow\Parquet\ParquetFile\Schema\Repetition; +use Flow\Parquet\Tests\Context\FlatColumnPages; +use Generator; +use PHPUnit\Framework\Attributes\DataProvider; use PHPUnit\Framework\Attributes\TestWith; use PHPUnit\Framework\TestCase; @@ -24,6 +28,52 @@ final class DremelStructuresTest extends TestCase { + /** + * @return Generator>}> + */ + public static function structs_with_a_null_optional_child(): Generator + { + $children = static fn(): array => [FlatColumn::int64('id', Repetition::REQUIRED), FlatColumn::string('name')]; + + yield 'required struct' => [ + NestedColumn::struct('s', $children(), Repetition::REQUIRED), + [['s' => ['id' => 1, 'name' => null]], ['s' => ['id' => 2, 'name' => 'x']]], + ]; + yield 'optional struct' => [ + NestedColumn::struct('s', $children()), + [['s' => ['id' => 1, 'name' => null]], ['s' => null], ['s' => ['id' => 2, 'name' => 'x']]], + ]; + yield 'required struct in an optional struct' => [ + NestedColumn::struct('s', [NestedColumn::struct('inner', $children(), Repetition::REQUIRED)]), + [['s' => ['inner' => ['id' => 1, 'name' => null]]], ['s' => null]], + ]; + yield 'required struct in a list' => [ + NestedColumn::list('s', ListElement::structure($children(), required: true)), + [['s' => [['id' => 1, 'name' => null], ['id' => 2, 'name' => 'x']]], ['s' => null]], + ]; + yield 'optional struct as a map value' => [ + NestedColumn::map('s', MapKey::string(), MapValue::structure($children())), + [['s' => ['a' => ['id' => 1, 'name' => null]]], ['s' => ['b' => ['id' => 2, 'name' => 'x']]]], + ]; + } + + /** + * @param array> $rows + */ + #[DataProvider('structs_with_a_null_optional_child')] + public function test_a_null_optional_child_keeps_its_struct(NestedColumn $column, array $rows): void + { + $schema = Schema::with($column); + + static::assertSame( + $rows, + iterator_to_array((new DremelAssembler(DataConverter::initialize(Options::default())))->assemble( + $schema->get('s'), + new ReadColumnData($schema->get('s'), FlatColumnPages::merged($schema, $rows)), + )), + ); + } + /** * @param array $row */ @@ -130,9 +180,9 @@ public function test_optional_struct_optional_int_optional_string_optional_bool( if ($exceptionMessage) { $this->expectExceptionMessage($exceptionMessage); - $shredder->shred($schema, [$row]); + $shredder->shred($schema, [$row], 0); } else { - $shredResult = $shredder->shred($schema, [$row]); + $shredResult = $shredder->shred($schema, [$row], 0); $normalized = []; @@ -264,9 +314,9 @@ public function test_optional_struct_optional_list_optional_int( if ($exceptionMessage) { $this->expectExceptionMessage($exceptionMessage); - $shredder->shred($schema, [$row]); + $shredder->shred($schema, [$row], 0); } else { - $shredResult = $shredder->shred($schema, [$row]); + $shredResult = $shredder->shred($schema, [$row], 0); $normalized = []; @@ -427,9 +477,9 @@ public function test_optional_struct_optional_map_string_optional_int( if ($exceptionMessage) { $this->expectExceptionMessage($exceptionMessage); - $shredder->shred($schema, [$row]); + $shredder->shred($schema, [$row], 0); } else { - $shredResult = $shredder->shred($schema, [$row]); + $shredResult = $shredder->shred($schema, [$row], 0); $normalized = []; @@ -574,9 +624,9 @@ public function test_optional_struct_optional_struct_optional_int32_optional_str if ($exceptionMessage) { $this->expectExceptionMessage($exceptionMessage); - $shredder->shred($schema, [$row]); + $shredder->shred($schema, [$row], 0); } else { - $shredResult = $shredder->shred($schema, [$row]); + $shredResult = $shredder->shred($schema, [$row], 0); $normalized = []; @@ -706,9 +756,9 @@ public function test_required_struct_required_int_required_string_required_bool( if ($exceptionMessage) { $this->expectExceptionMessage($exceptionMessage); - $shredder->shred($schema, [$row]); + $shredder->shred($schema, [$row], 0); } else { - $shredResult = $shredder->shred($schema, [$row]); + $shredResult = $shredder->shred($schema, [$row], 0); $normalized = []; @@ -814,9 +864,9 @@ public function test_required_struct_required_list_required_int( if ($exceptionMessage) { $this->expectExceptionMessage($exceptionMessage); - $shredder->shred($schema, [$row]); + $shredder->shred($schema, [$row], 0); } else { - $shredResult = $shredder->shred($schema, [$row]); + $shredResult = $shredder->shred($schema, [$row], 0); $normalized = []; @@ -937,9 +987,9 @@ public function test_required_struct_required_map_string_required_int( if ($exceptionMessage) { $this->expectExceptionMessage($exceptionMessage); - $shredder->shred($schema, [$row]); + $shredder->shred($schema, [$row], 0); } else { - $shredResult = $shredder->shred($schema, [$row]); + $shredResult = $shredder->shred($schema, [$row], 0); $normalized = []; @@ -1064,9 +1114,9 @@ public function test_required_struct_required_struct_required_int32_required_str if ($exceptionMessage) { $this->expectExceptionMessage($exceptionMessage); - $shredder->shred($schema, [$row]); + $shredder->shred($schema, [$row], 0); } else { - $shredResult = $shredder->shred($schema, [$row]); + $shredResult = $shredder->shred($schema, [$row], 0); $normalized = []; diff --git a/src/lib/parquet/tests/Flow/Parquet/Tests/Unit/Dremel/ReadColumnDataTest.php b/src/lib/parquet/tests/Flow/Parquet/Tests/Unit/Dremel/ReadColumnDataTest.php index dca0f55e29..51c4ae2176 100644 --- a/src/lib/parquet/tests/Flow/Parquet/Tests/Unit/Dremel/ReadColumnDataTest.php +++ b/src/lib/parquet/tests/Flow/Parquet/Tests/Unit/Dremel/ReadColumnDataTest.php @@ -12,6 +12,7 @@ use Flow\Parquet\ParquetFile\Schema\MapKey; use Flow\Parquet\ParquetFile\Schema\MapValue; use Flow\Parquet\ParquetFile\Schema\NestedColumn; +use Flow\Parquet\Tests\Context\FlatColumnPages; use PHPUnit\Framework\TestCase; use function iterator_to_array; @@ -121,4 +122,23 @@ public function test_iterating_over_map_column_data(): void iterator_to_array($columnData->iterator($valueColumn)), ); } + + public function test_iterating_over_a_paged_column_chains_its_pages(): void + { + $schema = Schema::with(NestedColumn::map('m', MapKey::string(), MapValue::int32())); + $rows = [['m' => ['a' => 1]], ['m' => ['b' => 2]], ['m' => ['c' => 3]]]; + $keyColumn = $schema->get('m.key_value.key'); + static::assertInstanceOf(FlatColumn::class, $keyColumn); + + $columnData = new ReadColumnData($schema->get('m'), FlatColumnPages::paged($schema, $rows, 1)); + + static::assertEquals( + [ + new FlatValue($keyColumn, 0, 2, 'a'), + new FlatValue($keyColumn, 0, 2, 'b'), + new FlatValue($keyColumn, 0, 2, 'c'), + ], + iterator_to_array($columnData->iterator($keyColumn)), + ); + } } diff --git a/src/lib/parquet/tests/Flow/Parquet/Tests/Unit/Dremel/Validator/ColumnDataValidatorTest.php b/src/lib/parquet/tests/Flow/Parquet/Tests/Unit/Dremel/Validator/ColumnDataValidatorTest.php new file mode 100644 index 0000000000..b6a40192a8 --- /dev/null +++ b/src/lib/parquet/tests/Flow/Parquet/Tests/Unit/Dremel/Validator/ColumnDataValidatorTest.php @@ -0,0 +1,206 @@ + + */ + public static function integers_in_range(): Generator + { + yield 'INT_8 min' => [new FlatColumn('c', PhysicalType::INT32, ConvertedType::INT_8), -128]; + yield 'INT_16 max' => [new FlatColumn('c', PhysicalType::INT32, ConvertedType::INT_16), 32_767]; + yield 'INT_32 min' => [FlatColumn::int32('c'), -2_147_483_648]; + yield 'plain INT32 max' => [new FlatColumn('c', PhysicalType::INT32), 2_147_483_647]; + yield 'UINT_8 max' => [new FlatColumn('c', PhysicalType::INT32, ConvertedType::UINT_8), 255]; + yield 'UINT_16 max' => [new FlatColumn('c', PhysicalType::INT32, ConvertedType::UINT_16), 65_535]; + yield 'UINT_32 max' => [new FlatColumn('c', PhysicalType::INT32, ConvertedType::UINT_32), 4_294_967_295]; + yield 'UINT_64 max' => [new FlatColumn('c', PhysicalType::INT64, ConvertedType::UINT_64), PHP_INT_MAX]; + yield 'INT_64 min' => [FlatColumn::int64('c'), PHP_INT_MIN]; + yield 'plain INT64 unchecked' => [new FlatColumn('c', PhysicalType::INT64), PHP_INT_MIN]; + yield 'DECIMAL in INT32 unchecked' => [ + new FlatColumn('c', PhysicalType::INT32, logicalType: LogicalType::decimal(2, 9)), + PHP_INT_MAX, + ]; + } + + #[DataProvider('integers_in_range')] + public function test_an_integer_in_the_column_range_is_accepted(FlatColumn $column, int $value): void + { + (new ColumnDataValidator())->validate($column, $value, 0); + + $this->expectNotToPerformAssertions(); + } + + /** + * @return Generator + */ + public static function integers_out_of_range(): Generator + { + yield 'INT_8 above' => [ + new FlatColumn('c', PhysicalType::INT32, ConvertedType::INT_8), + 128, + '128 is out of the INT_8 range [-128, 127]', + ]; + yield 'INT_16 below' => [ + new FlatColumn('c', PhysicalType::INT32, ConvertedType::INT_16), + -32_769, + '-32769 is out of the INT_16 range [-32768, 32767]', + ]; + yield 'INT_32 above' => [ + FlatColumn::int32('c'), + 4_000_000_000, + '4000000000 is out of the INT_32 range [-2147483648, 2147483647]', + ]; + yield 'plain INT32 below' => [ + new FlatColumn('c', PhysicalType::INT32), + -2_147_483_649, + '-2147483649 is out of the INT32 range [-2147483648, 2147483647]', + ]; + yield 'UINT_8 above' => [ + new FlatColumn('c', PhysicalType::INT32, ConvertedType::UINT_8), + 256, + '256 is out of the UINT_8 range [0, 255]', + ]; + yield 'UINT_16 negative' => [ + new FlatColumn('c', PhysicalType::INT32, ConvertedType::UINT_16), + -1, + '-1 is out of the UINT_16 range [0, 65535]', + ]; + yield 'UINT_32 above' => [ + new FlatColumn('c', PhysicalType::INT32, ConvertedType::UINT_32), + 4_294_967_296, + '4294967296 is out of the UINT_32 range [0, 4294967295]', + ]; + yield 'UINT_64 negative' => [ + new FlatColumn('c', PhysicalType::INT64, ConvertedType::UINT_64), + -1, + '-1 is out of the UINT_64 range [0, 9223372036854775807]', + ]; + yield 'logical INTEGER(8, signed) above' => [ + new FlatColumn('c', PhysicalType::INT32, logicalType: LogicalType::integer(8, true)), + 200, + '200 is out of the INT_8 range [-128, 127]', + ]; + } + + #[DataProvider('integers_out_of_range')] + public function test_an_integer_out_of_the_column_range_is_refused_naming_its_row( + FlatColumn $column, + int $value, + string $message, + ): void { + $this->expectException(ValidationException::class); + $this->expectExceptionMessage('Column "c" row 3: ' . $message); + + (new ColumnDataValidator())->validate($column, $value, 3); + } + + /** + * @return Generator + */ + public static function uuids(): Generator + { + yield 'dashed string' => ['0190a6b0-6b1f-7c5e-9a3b-0123456789ab']; + yield 'dash-less string' => ['0190A6B06B1F7C5E9A3B0123456789AB']; + yield 'toString() object' => [new ToStringUuid('0190a6b0-6b1f-7c5e-9a3b-0123456789ab')]; + yield 'Stringable object' => [new StringableUuid('0190a6b0-6b1f-7c5e-9a3b-0123456789ab')]; + } + + #[DataProvider('uuids')] + public function test_a_uuid_of_32_hex_digits_is_accepted(mixed $value): void + { + (new ColumnDataValidator())->validate(FlatColumn::uuid('c'), $value, 0); + + $this->expectNotToPerformAssertions(); + } + + /** + * @return Generator + */ + public static function not_uuids(): Generator + { + yield '16 raw bytes' => ["\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01", 'string']; + yield 'too short' => ['0190a6b0-6b1f-7c5e-9a3b', 'string']; + yield 'not hexadecimal' => ['0190a6b0-6b1f-7c5e-9a3b-0123456789az', 'string']; + yield 'an int' => [1, 'int']; + yield 'an object without text' => [new DateTimeImmutable('2024-01-01'), 'DateTimeImmutable']; + } + + #[DataProvider('not_uuids')] + public function test_a_uuid_that_is_not_32_hex_digits_is_refused(mixed $value, string $type): void + { + $this->expectException(ValidationException::class); + $this->expectExceptionMessage('Column "c" row 1: ' . $type . ' is not a UUID of 32 hexadecimal digits'); + + (new ColumnDataValidator())->validate(FlatColumn::uuid('c'), $value, 1); + } + + public function test_a_fixed_length_value_of_its_length_is_accepted(): void + { + (new ColumnDataValidator())->validate(FlatColumn::fixedSizeByteArray('c', 4), "\x00\x01\x02\x03", 0); + + $this->expectNotToPerformAssertions(); + } + + public function test_a_fixed_length_value_of_another_length_is_refused(): void + { + $this->expectException(ValidationException::class); + $this->expectExceptionMessage('Column "c" row 2: a FIXED_LEN_BYTE_ARRAY(4) value is 5 bytes long'); + + (new ColumnDataValidator())->validate(FlatColumn::fixedSizeByteArray('c', 4), '12345', 2); + } + + /** + * @return Generator + */ + public static function utf8_columns(): Generator + { + yield 'STRING' => [FlatColumn::string('c')]; + yield 'JSON' => [FlatColumn::json('c')]; + yield 'ENUM' => [FlatColumn::enum('c')]; + } + + #[DataProvider('utf8_columns')] + public function test_invalid_utf8_is_refused(FlatColumn $column): void + { + $this->expectException(ValidationException::class); + $this->expectExceptionMessage('Column "c" row 0: the string is not valid UTF-8'); + + (new ColumnDataValidator())->validate($column, "\xFF\xFE", 0); + } + + #[DataProvider('utf8_columns')] + public function test_valid_utf8_is_accepted(FlatColumn $column): void + { + (new ColumnDataValidator())->validate($column, 'zażółć', 0); + + $this->expectNotToPerformAssertions(); + } + + public function test_invalid_utf8_into_a_plain_byte_array_is_accepted(): void + { + (new ColumnDataValidator())->validate(new FlatColumn('c', PhysicalType::BYTE_ARRAY), "\xFF\xFE", 0); + + $this->expectNotToPerformAssertions(); + } +} diff --git a/src/lib/parquet/tests/Flow/Parquet/Tests/Unit/Engine/AdaptiveParquetEngineTest.php b/src/lib/parquet/tests/Flow/Parquet/Tests/Unit/Engine/AdaptiveParquetEngineTest.php index 04becd6bca..cea7cd486b 100644 --- a/src/lib/parquet/tests/Flow/Parquet/Tests/Unit/Engine/AdaptiveParquetEngineTest.php +++ b/src/lib/parquet/tests/Flow/Parquet/Tests/Unit/Engine/AdaptiveParquetEngineTest.php @@ -4,45 +4,122 @@ namespace Flow\Parquet\Tests\Unit\Engine; +use Flow\Filesystem\Stream\MemorySourceStream; +use Flow\Filesystem\Stream\StringDestinationStream; +use Flow\Parquet\Binary\ByteOrder; use Flow\Parquet\Engine\AdaptiveParquetEngine; -use Flow\Parquet\Engine\ArrowParquetEngine; -use Flow\Parquet\Engine\PhpParquetEngine; +use Flow\Parquet\Engine\ArrowExtension; +use Flow\Parquet\Engine\PhpParquetFileReader; +use Flow\Parquet\Engine\PhpParquetFileWriter; +use Flow\Parquet\Engine\RustParquetFileReader; +use Flow\Parquet\Engine\RustParquetFileWriter; +use Flow\Parquet\Exception\RuntimeException; +use Flow\Parquet\Options; +use Flow\Parquet\ParquetFile\Compressions; +use Flow\Parquet\ParquetFile\Schema; +use Flow\Parquet\ParquetFile\Schema\FlatColumn; +use Flow\Parquet\Tests\Context\MemoryParquetFile; +use Flow\Parquet\Tests\Mother\ParquetFileWriterMother; +use PHPUnit\Framework\Attributes\RequiresPhpExtension; use PHPUnit\Framework\TestCase; -use ReflectionClass; use function extension_loaded; +use function Flow\Filesystem\DSL\memory_filesystem; +use function Flow\Filesystem\DSL\path; +use function iterator_to_array; final class AdaptiveParquetEngineTest extends TestCase { - public function test_creates_arrow_engine_when_arrow_extension_loaded(): void + public function test_rows_written_through_the_contract_read_back(): void { - if (!extension_loaded('arrow')) { - static::markTestSkipped('This test requires the arrow extension to be loaded'); - } - + $filesystem = memory_filesystem(); $engine = new AdaptiveParquetEngine(); - static::assertInstanceOf( - ArrowParquetEngine::class, - (new ReflectionClass($engine)) - ->getProperty('delegate') - ->getValue($engine), + $engine->writeRows( + $filesystem->writeTo(path('memory://out.parquet')), + Schema::with(FlatColumn::int32('id')), + Compressions::SNAPPY, + new Options(), + [['id' => 1], ['id' => 2]], + ); + + static::assertSame( + [['id' => [1, 2]]], + iterator_to_array( + $engine->openForRead($filesystem->readFrom(path('memory://out.parquet')))->readColumns( + ['id'], + 10, + null, + null, + ), + false, + ), ); } - public function test_creates_php_engine_when_arrow_extension_not_loaded(): void + public function test_without_arrow_files_are_read_and_written_by_the_php_engine(): void { if (extension_loaded('arrow')) { - static::markTestSkipped('This test requires the arrow extension to NOT be loaded'); + static::markTestSkipped('This test requires arrow not to be loaded'); } $engine = new AdaptiveParquetEngine(); static::assertInstanceOf( - PhpParquetEngine::class, - (new ReflectionClass($engine)) - ->getProperty('delegate') - ->getValue($engine), + PhpParquetFileReader::class, + $engine->openForRead(new MemorySourceStream(MemoryParquetFile::threeRowGroups())), ); + static::assertInstanceOf(PhpParquetFileWriter::class, ParquetFileWriterMother::open( + $engine, + new StringDestinationStream(path('memory://out.parquet')), + )); + } + + #[RequiresPhpExtension('arrow')] + public function test_with_arrow_files_are_read_and_written_by_arrow(): void + { + $engine = new AdaptiveParquetEngine(); + + static::assertInstanceOf( + RustParquetFileReader::class, + $engine->openForRead(new MemorySourceStream(MemoryParquetFile::threeRowGroups())), + ); + static::assertInstanceOf(RustParquetFileWriter::class, ParquetFileWriterMother::open( + $engine, + new StringDestinationStream(path('memory://out.parquet')), + )); + } + + public function test_big_endian_is_read_by_the_php_engine(): void + { + static::assertInstanceOf( + PhpParquetFileReader::class, + (new AdaptiveParquetEngine(ByteOrder::BIG_ENDIAN))->openForRead( + new MemorySourceStream(MemoryParquetFile::threeRowGroups()), + ), + ); + } + + public function test_an_arrow_extension_of_another_abi_is_refused(): void + { + $this->expectException(RuntimeException::class); + $this->expectExceptionMessage('does not match flow-php/parquet'); + + new AdaptiveParquetEngine(extension: new ArrowExtension(true, ArrowExtension::ABI + 1, '0.46.0')); + } + + public function test_without_arrow_files_are_opened_by_the_php_engine_whatever_is_loaded(): void + { + $filesystem = memory_filesystem(); + + static::assertInstanceOf(PhpParquetFileWriter::class, (new AdaptiveParquetEngine(extension: new ArrowExtension( + false, + null, + )))->openForWrite( + $filesystem->writeTo(path('memory://out.parquet')), + Schema::with(FlatColumn::int32('id')), + Compressions::UNCOMPRESSED, + new Options(), + )); } } diff --git a/src/lib/parquet/tests/Flow/Parquet/Tests/Unit/Engine/Arrow/DestinationStreamAdapterTest.php b/src/lib/parquet/tests/Flow/Parquet/Tests/Unit/Engine/Arrow/DestinationStreamAdapterTest.php deleted file mode 100644 index b2e3379483..0000000000 --- a/src/lib/parquet/tests/Flow/Parquet/Tests/Unit/Engine/Arrow/DestinationStreamAdapterTest.php +++ /dev/null @@ -1,37 +0,0 @@ -append('hello '); - $adapter->append('world'); - - static::assertSame('hello world', $stream->content()); - } - - public function test_append_returns_self(): void - { - $handle = fopen('php://memory', 'r+b'); - $stream = new MemoryStream($handle, Path::realpath('/tmp/test')); - - $adapter = new DestinationStreamAdapter($stream); - - static::assertSame($adapter, $adapter->append('data')); - } -} diff --git a/src/lib/parquet/tests/Flow/Parquet/Tests/Unit/Engine/Arrow/OptionsConverterTest.php b/src/lib/parquet/tests/Flow/Parquet/Tests/Unit/Engine/Arrow/OptionsConverterTest.php index 98808e7fb3..4d28e05175 100644 --- a/src/lib/parquet/tests/Flow/Parquet/Tests/Unit/Engine/Arrow/OptionsConverterTest.php +++ b/src/lib/parquet/tests/Flow/Parquet/Tests/Unit/Engine/Arrow/OptionsConverterTest.php @@ -13,6 +13,28 @@ final class OptionsConverterTest extends TestCase { + public function test_engine_options_default_without_options(): void + { + static::assertSame( + [ + 'INT_96_AS_DATETIME' => true, + 'ARROW_WRITE_BATCH_SIZE' => (new Options())->getInt(Option::ARROW_WRITE_BATCH_SIZE), + ], + OptionsConverter::toEngine(null), + ); + } + + public function test_engine_options_are_read_from_the_options(): void + { + static::assertSame( + ['INT_96_AS_DATETIME' => false, 'ARROW_WRITE_BATCH_SIZE' => 7], + OptionsConverter::toEngine((new Options())->set(Option::INT_96_AS_DATETIME, false)->set( + Option::ARROW_WRITE_BATCH_SIZE, + 7, + )), + ); + } + public function test_arrow_batch_size_is_excluded_when_null(): void { $options = new Options(); diff --git a/src/lib/parquet/tests/Flow/Parquet/Tests/Unit/Engine/Arrow/SourceStreamAdapterTest.php b/src/lib/parquet/tests/Flow/Parquet/Tests/Unit/Engine/Arrow/SourceStreamAdapterTest.php deleted file mode 100644 index 7afd3df539..0000000000 --- a/src/lib/parquet/tests/Flow/Parquet/Tests/Unit/Engine/Arrow/SourceStreamAdapterTest.php +++ /dev/null @@ -1,40 +0,0 @@ -read(5, 0)); - static::assertSame('world', $adapter->read(5, 6)); - static::assertSame(' ', $adapter->read(1, 5)); - } - - public function test_size_delegates_to_source_stream(): void - { - $handle = fopen('php://memory', 'r+b'); - fwrite($handle, 'hello world'); - $stream = new MemoryStream($handle, Path::realpath('/tmp/test')); - - $adapter = new SourceStreamAdapter($stream); - - static::assertSame(11, $adapter->size()); - } -} diff --git a/src/lib/parquet/tests/Flow/Parquet/Tests/Unit/Engine/ArrowExtensionTest.php b/src/lib/parquet/tests/Flow/Parquet/Tests/Unit/Engine/ArrowExtensionTest.php new file mode 100644 index 0000000000..ef894a0435 --- /dev/null +++ b/src/lib/parquet/tests/Flow/Parquet/Tests/Unit/Engine/ArrowExtensionTest.php @@ -0,0 +1,51 @@ +available()); + } + + public function test_a_loaded_extension_of_the_expected_abi_is_available(): void + { + static::assertTrue((new ArrowExtension(true, ArrowExtension::ABI))->available()); + } + + public function test_a_loaded_extension_of_another_abi_is_refused(): void + { + $this->expectException(RuntimeException::class); + $this->expectExceptionMessage( + 'The loaded arrow extension (0.46.0, ABI 2) does not match flow-php/parquet (ABI 1). Rebuild or upgrade ' + . 'arrow-ext to the version of flow-php/parquet, or stop loading it to use the PHP implementation.', + ); + + (new ArrowExtension(true, ArrowExtension::ABI + 1, '0.46.0'))->available(); + } + + public function test_a_loaded_extension_without_an_abi_is_refused(): void + { + $this->expectException(RuntimeException::class); + $this->expectExceptionMessage('The loaded arrow extension (0.44.1, ABI none) does not match flow-php/parquet'); + + (new ArrowExtension(true, null, '0.44.1'))->available(); + } + + public function test_detect_reports_whether_arrow_is_loaded(): void + { + $extension = ArrowExtension::detect(); + + static::assertSame(extension_loaded('arrow'), $extension->loaded); + static::assertSame(extension_loaded('arrow') ? ArrowExtension::ABI : null, $extension->abi); + } +} diff --git a/src/lib/parquet/tests/Flow/Parquet/Tests/Unit/Engine/ArrowParquetEngineTest.php b/src/lib/parquet/tests/Flow/Parquet/Tests/Unit/Engine/ArrowParquetEngineTest.php deleted file mode 100644 index 8587c23d7e..0000000000 --- a/src/lib/parquet/tests/Flow/Parquet/Tests/Unit/Engine/ArrowParquetEngineTest.php +++ /dev/null @@ -1,48 +0,0 @@ -expectException(RuntimeException::class); - $this->expectExceptionMessage('arrow extension is required'); - - new ArrowParquetEngine(); - } - - #[TestWith([Compressions::UNCOMPRESSED, 'UNCOMPRESSED'])] - #[TestWith([Compressions::SNAPPY, 'SNAPPY'])] - #[TestWith([Compressions::GZIP, 'GZIP'])] - #[TestWith([Compressions::BROTLI, 'BROTLI'])] - #[TestWith([Compressions::LZ4, 'LZ4_RAW'])] - #[TestWith([Compressions::LZ4_RAW, 'LZ4_RAW'])] - #[TestWith([Compressions::ZSTD, 'ZSTD'])] - public function test_map_compression(Compressions $input, string $expected): void - { - static::assertSame($expected, ArrowParquetEngine::mapCompression($input)); - } - - public function test_map_compression_throws_for_lzo(): void - { - $this->expectException(RuntimeException::class); - $this->expectExceptionMessage('LZO compression is not supported'); - - ArrowParquetEngine::mapCompression(Compressions::LZO); - } -} diff --git a/src/lib/parquet/tests/Flow/Parquet/Tests/Unit/Engine/ArrowParquetFileWriterTest.php b/src/lib/parquet/tests/Flow/Parquet/Tests/Unit/Engine/ArrowParquetFileWriterTest.php deleted file mode 100644 index 75621106cb..0000000000 --- a/src/lib/parquet/tests/Flow/Parquet/Tests/Unit/Engine/ArrowParquetFileWriterTest.php +++ /dev/null @@ -1,140 +0,0 @@ -writeTo(path('memory://file.parquet'))), - ); - - try { - $file->close(); - static::fail('close() was expected to throw'); - } catch (FilesystemRuntimeException $failure) { - static::assertSame('Closing "memory://file.parquet" failed', $failure->getMessage()); - } - - $this->expectException(RuntimeException::class); - $this->expectExceptionMessage('Writer is not open'); - - $file->close(); - } - - public function test_closing_twice_throws(): void - { - $file = ParquetFileWriterMother::open( - new ArrowParquetEngine(), - memory_filesystem()->writeTo(path('memory://file.parquet')), - ); - $file->close(); - - $this->expectException(RuntimeException::class); - $this->expectExceptionMessage('Writer is not open'); - - $file->close(); - } - - public function test_closing_writes_a_file_the_reader_reads_back(): void - { - $memory = memory_filesystem(); - $file = ParquetFileWriterMother::open( - new ArrowParquetEngine(), - $memory->writeTo(path('memory://file.parquet')), - ); - - $file->writeRow(['id' => 1]); - $file->writeBatch([['id' => 2]]); - $file->close(); - - static::assertSame( - [['id' => 1], ['id' => 2]], - iterator_to_array( - (new Reader(engine: new PhpParquetEngine())) - ->readStream($memory->readFrom(path('memory://file.parquet'))) - ->values(), - false, - ), - ); - } - - public function test_writing_a_batch_after_close_throws(): void - { - $file = ParquetFileWriterMother::open( - new ArrowParquetEngine(), - memory_filesystem()->writeTo(path('memory://file.parquet')), - ); - $file->close(); - - $this->expectException(RuntimeException::class); - $this->expectExceptionMessage('Writer is not open'); - - $file->writeBatch([['id' => 1]]); - } - - public function test_writing_a_row_after_close_throws(): void - { - $file = ParquetFileWriterMother::open( - new ArrowParquetEngine(), - memory_filesystem()->writeTo(path('memory://file.parquet')), - ); - $file->close(); - - $this->expectException(RuntimeException::class); - $this->expectExceptionMessage('Writer is not open'); - - $file->writeRow(['id' => 1]); - } - - public function test_rows_beyond_the_batch_size_are_written_once(): void - { - $memory = memory_filesystem(); - $file = ParquetFileWriterMother::open( - new ArrowParquetEngine(Options::default()->set(Option::ARROW_WRITE_BATCH_SIZE, 2)), - $memory->writeTo(path('memory://file.parquet')), - ); - - $file->writeRow(['id' => 1]); - $file->writeRow(['id' => 2]); - $file->writeRow(['id' => 3]); - $file->close(); - - static::assertSame( - [['id' => 1], ['id' => 2], ['id' => 3]], - iterator_to_array( - (new Reader(engine: new PhpParquetEngine())) - ->readStream($memory->readFrom(path('memory://file.parquet'))) - ->values(), - false, - ), - ); - } -} diff --git a/src/lib/parquet/tests/Flow/Parquet/Tests/Unit/Engine/PhpParquetFileReaderTest.php b/src/lib/parquet/tests/Flow/Parquet/Tests/Unit/Engine/PhpParquetFileReaderTest.php new file mode 100644 index 0000000000..7276cc8266 --- /dev/null +++ b/src/lib/parquet/tests/Flow/Parquet/Tests/Unit/Engine/PhpParquetFileReaderTest.php @@ -0,0 +1,113 @@ +metadata(); + $reads = $stream->reads; + + static::assertSame($metadata, $reader->metadata()); + static::assertSame($metadata->schema(), $reader->schema()); + static::assertSame(6, $reader->rowsNumber()); + static::assertSame($reads, $stream->reads); + } + + public function test_total_byte_size_sums_the_row_groups(): void + { + $reader = new PhpParquetFileReader( + new MemorySourceStream(MemoryParquetFile::threeRowGroups()), + ByteOrder::LITTLE_ENDIAN, + new Options(), + ); + $expected = 0; + + foreach ($reader->metadata()->rowGroups()->all() as $rowGroup) { + $expected += $rowGroup->totalByteSize(); + } + + static::assertCount(3, $reader->metadata()->rowGroups()->all()); + static::assertGreaterThan(0, $expected); + static::assertSame($expected, $reader->totalByteSize()); + } + + /** + * @param int<1, max> $batchSize + * @param list> $ids + */ + #[TestWith([4, null, null, [[1, 2, 3, 4], [5, 6]]])] + #[TestWith([2, 3, 2, [[3, 4], [5]]])] + #[TestWith([2, null, 6, []])] + #[TestWith([2, null, 7, []])] + public function test_read_columns_chunks_the_rows(int $batchSize, ?int $limit, ?int $offset, array $ids): void + { + $reader = new PhpParquetFileReader( + new MemorySourceStream(MemoryParquetFile::threeRowGroups()), + ByteOrder::LITTLE_ENDIAN, + new Options(), + ); + $chunks = iterator_to_array($reader->readColumns(['id'], $batchSize, $limit, $offset), false); + + static::assertSame($ids, array_column($chunks, 'id')); + } + + public function test_close_closes_the_stream_and_every_later_call_throws(): void + { + $stream = new ReadCountingSourceStream(new MemorySourceStream(MemoryParquetFile::threeRowGroups())); + $reader = new PhpParquetFileReader($stream, ByteOrder::LITTLE_ENDIAN, new Options()); + $reader->metadata(); + + $reader->close(); + + static::assertTrue($stream->closed); + + foreach ([ + static fn() => $reader->metadata(), + static fn() => $reader->schema(), + static fn() => $reader->rowsNumber(), + static fn() => $reader->totalByteSize(), + static fn() => iterator_to_array($reader->readColumns(['id'], 1, null, null)), + static fn() => $reader->close(), + ] as $call) { + try { + $call(); + static::fail('A call on a closed reader must throw'); + } catch (RuntimeException $exception) { + static::assertSame('Reader is not open', $exception->getMessage()); + } + } + } + + public function test_a_file_without_the_parquet_magic_is_refused(): void + { + $this->expectException(InvalidArgumentException::class); + $this->expectExceptionMessage('Given file is not valid Parquet file'); + + (new PhpParquetFileReader( + new MemorySourceStream('not a parquet file'), + ByteOrder::LITTLE_ENDIAN, + new Options(), + ))->metadata(); + } +} diff --git a/src/lib/parquet/tests/Flow/Parquet/Tests/Unit/Engine/PhpParquetFileWriterTest.php b/src/lib/parquet/tests/Flow/Parquet/Tests/Unit/Engine/PhpParquetFileWriterTest.php index 5e3d51b623..fde6e8cb33 100644 --- a/src/lib/parquet/tests/Flow/Parquet/Tests/Unit/Engine/PhpParquetFileWriterTest.php +++ b/src/lib/parquet/tests/Flow/Parquet/Tests/Unit/Engine/PhpParquetFileWriterTest.php @@ -8,16 +8,24 @@ use Flow\Filesystem\Exception\RuntimeException as FilesystemRuntimeException; use Flow\Filesystem\Tests\Double\FailingCloseDestinationStream; use Flow\Parquet\Engine\PhpParquetEngine; +use Flow\Parquet\Exception\InvalidArgumentException; use Flow\Parquet\Exception\RuntimeException; +use Flow\Parquet\Exception\ValidationException; use Flow\Parquet\Option; use Flow\Parquet\Options; +use Flow\Parquet\ParquetFile\Schema; +use Flow\Parquet\ParquetFile\Schema\FlatColumn; use Flow\Parquet\Reader; +use Flow\Parquet\Tests\Context\ColumnDoor; use Flow\Parquet\Tests\Mother\ParquetFileWriterMother; use PHPUnit\Framework\TestCase; +use function array_map; +use function array_slice; use function Flow\Filesystem\DSL\memory_filesystem; use function Flow\Filesystem\DSL\path; use function iterator_to_array; +use function range; final class PhpParquetFileWriterTest extends TestCase { @@ -140,4 +148,91 @@ public function test_a_non_array_batch_is_split_into_row_groups(): void static::assertSame([['id' => 1], ['id' => 2], ['id' => 3]], iterator_to_array($parquet->values(), false)); static::assertCount(3, $parquet->metadata()->rowGroups()->all()); } + + public function test_columns_of_different_lengths_are_refused(): void + { + $file = ColumnDoor::open(new PhpParquetEngine(), ColumnDoor::stream(), Schema::with(FlatColumn::int32('id'))); + + $this->expectException(InvalidArgumentException::class); + $this->expectExceptionMessage('writeColumns() takes lists of one length, got "id": 2, "other": 1'); + + $file->writeColumns(['id' => [1, 2], 'other' => [1]]); + } + + public function test_a_column_the_array_lacks_is_written_as_nulls_and_an_unknown_key_is_ignored(): void + { + $stream = ColumnDoor::stream(); + $file = ColumnDoor::open( + new PhpParquetEngine(), + $stream, + Schema::with(FlatColumn::int32('id'), FlatColumn::string('name')), + ); + + $file->writeColumns(['id' => [1, 2], 'unknown' => ['a', 'b']]); + $file->writeColumns([]); + $file->close(); + + static::assertSame([['id' => 1, 'name' => null], ['id' => 2, 'name' => null]], ColumnDoor::read($stream)); + } + + public function test_writing_columns_after_close_throws(): void + { + $file = ColumnDoor::open(new PhpParquetEngine(), ColumnDoor::stream(), Schema::with(FlatColumn::int32('id'))); + $file->close(); + + $this->expectException(RuntimeException::class); + $this->expectExceptionMessage('Writer is not open'); + + $file->writeColumns(['id' => [1]]); + } + + public function test_columns_write_the_bytes_a_batch_writes(): void + { + $options = Options::default() + ->set(Option::ROW_GROUP_SIZE_BYTES, 2 * 1024) + ->set(Option::ROW_GROUP_SIZE_CHECK_INTERVAL, 140) + ->set(Option::PAGE_SIZE_BYTES, 256) + ->set(Option::PAGE_SIZE_CHECK_INTERVAL, 26); + $rows = ColumnDoor::rows(600); + $batch = ColumnDoor::stream(); + $columns = ColumnDoor::stream(); + + $file = ColumnDoor::open(new PhpParquetEngine(options: $options), $batch, ColumnDoor::schema(), $options); + $file->writeRow($rows[0]); + $file->writeBatch(array_slice($rows, 1)); + $file->close(); + + $file = ColumnDoor::open(new PhpParquetEngine(options: $options), $columns, ColumnDoor::schema(), $options); + $file->writeRow($rows[0]); + $file->writeColumns(ColumnDoor::columns(array_slice($rows, 1))); + $file->close(); + + static::assertTrue($batch->content() === $columns->content()); + static::assertSame($rows, ColumnDoor::read($columns)); + } + + public function test_columns_refuse_the_cell_a_batch_refuses_and_keep_the_same_rows(): void + { + $schema = Schema::with(FlatColumn::int32('a'), FlatColumn::int32('b'), FlatColumn::int32('c')); + $rows = array_map(static fn(int $i): array => ['a' => $i, 'b' => $i, 'c' => $i], range(0, 7)); + $rows[5]['a'] = 'five'; + $rows[2]['c'] = 'two'; + $outcomes = []; + + foreach (['batch', 'columns'] as $door) { + $stream = ColumnDoor::stream(); + $file = ColumnDoor::open(new PhpParquetEngine(), $stream, $schema); + + try { + $door === 'batch' ? $file->writeBatch($rows) : $file->writeColumns(ColumnDoor::columns($rows)); + static::fail('a string was written into INT32'); + } catch (ValidationException $refusal) { + $file->close(); + $outcomes[$door] = [$refusal->getMessage(), ColumnDoor::read($stream)]; + } + } + + static::assertSame($outcomes['batch'], $outcomes['columns']); + static::assertSame('Column "a" require integer as value, got: string instead', $outcomes['columns'][0]); + } } diff --git a/src/lib/parquet/tests/Flow/Parquet/Tests/Unit/Engine/RustParquetEngineTest.php b/src/lib/parquet/tests/Flow/Parquet/Tests/Unit/Engine/RustParquetEngineTest.php new file mode 100644 index 0000000000..c9f906815f --- /dev/null +++ b/src/lib/parquet/tests/Flow/Parquet/Tests/Unit/Engine/RustParquetEngineTest.php @@ -0,0 +1,49 @@ +set(Option::INT_96_AS_DATETIME, false)); + + $this->expectException(InvalidArgumentException::class); + $this->expectExceptionMessage('Parquet column "ts" holds INT96'); + + $engine->openForRead(NativeLocalSourceStream::open(path_real(__DIR__ + . '/../../Integration/IO/Fixtures/EdgeCases/int96.parquet')))->readColumns(['ts'], 10, null, null); + } + + public function test_lzo_is_refused(): void + { + $this->expectException(RuntimeException::class); + $this->expectExceptionMessage('LZO compression is not supported by the Arrow engine'); + + (new RustParquetEngine())->openForWrite( + new StringDestinationStream(path('memory://out.parquet')), + Schema::with(FlatColumn::int32('id')), + Compressions::LZO, + new Options(), + ); + } +} diff --git a/src/lib/parquet/tests/Flow/Parquet/Tests/Unit/ParquetFile/Data/Converter/Int96DateTimeConverterTest.php b/src/lib/parquet/tests/Flow/Parquet/Tests/Unit/ParquetFile/Data/Converter/Int96DateTimeConverterTest.php new file mode 100644 index 0000000000..4c53b44f29 --- /dev/null +++ b/src/lib/parquet/tests/Flow/Parquet/Tests/Unit/ParquetFile/Data/Converter/Int96DateTimeConverterTest.php @@ -0,0 +1,50 @@ + + */ + public static function int96_values(): Generator + { + yield 'nanoseconds floored' => [ + pack('PV', 45_296_789_012_345, 19_782 + 2_440_588), + '2024-02-29 12:34:56.789012', + ]; + yield 'epoch' => [pack('PV', 0, 2_440_588), '1970-01-01 00:00:00.000000']; + yield 'before the epoch, never rounded up' => [ + pack('PV', 86_399_999_999_999, -3_653 + 2_440_588), + '1960-01-01 23:59:59.999999', + ]; + yield 'first microsecond of year 1' => [pack('PV', 1_000, -719_162 + 2_440_588), '0001-01-01 00:00:00.000001']; + yield 'sub-microsecond of year 9999' => [pack('PV', 999, 2_932_896 + 2_440_588), '9999-12-31 00:00:00.000000']; + yield 'remainder of 999 nanoseconds' => [ + pack('PV', 123_456_789_999, 19_000 + 2_440_588), + '2022-01-08 00:02:03.456789', + ]; + } + + #[DataProvider('int96_values')] + public function test_int96_reads_as_the_microsecond_it_falls_in(string $bytes, string $datetime): void + { + static::assertSame( + $datetime, + (new Int96DateTimeConverter()) + ->fromParquetType($bytes) + ->format('Y-m-d H:i:s.u'), + ); + } +} diff --git a/src/lib/parquet/tests/Flow/Parquet/Tests/Unit/ParquetFile/Data/Converter/UnsignedIntegerConverterTest.php b/src/lib/parquet/tests/Flow/Parquet/Tests/Unit/ParquetFile/Data/Converter/UnsignedIntegerConverterTest.php new file mode 100644 index 0000000000..2907c4e028 --- /dev/null +++ b/src/lib/parquet/tests/Flow/Parquet/Tests/Unit/ParquetFile/Data/Converter/UnsignedIntegerConverterTest.php @@ -0,0 +1,91 @@ +fromParquetType($stored)); + } + + public function test_uint64_up_to_php_int_max_reads_as_is(): void + { + static::assertSame(PHP_INT_MAX, (new UnsignedIntegerConverter('u64', 64))->fromParquetType(PHP_INT_MAX)); + } + + public function test_uint64_above_php_int_max_is_refused(): void + { + $this->expectException(RuntimeException::class); + $this->expectExceptionMessage('Parquet column "u64" holds a UINT_64 value above PHP_INT_MAX'); + + (new UnsignedIntegerConverter('u64', 64))->fromParquetType(-1); + } + + public function test_a_value_that_is_not_an_int_is_refused(): void + { + $this->expectException(InvalidArgumentException::class); + $this->expectExceptionMessage('Expected int, got string'); + + (new UnsignedIntegerConverter('u32', 32))->fromParquetType('1'); + } + + public function test_writes_are_the_value_itself(): void + { + static::assertSame(4_000_000_000, (new UnsignedIntegerConverter('u32', 32))->toParquetType(4_000_000_000)); + } + + #[TestWith([ConvertedType::UINT_32, 32])] + #[TestWith([ConvertedType::UINT_64, 64])] + public function test_for_column_reads_the_converted_type(ConvertedType $convertedType, int $bitWidth): void + { + static::assertEquals( + new UnsignedIntegerConverter('u', $bitWidth), + UnsignedIntegerConverter::forColumn( + new FlatColumn('u', PhysicalType::INT64, $convertedType), + Options::default(), + ), + ); + } + + public function test_for_column_reads_a_logical_unsigned_integer(): void + { + static::assertEquals( + new UnsignedIntegerConverter('u', 32), + UnsignedIntegerConverter::forColumn( + new FlatColumn('u', PhysicalType::INT32, logicalType: LogicalType::integer(32, false)), + Options::default(), + ), + ); + } + + #[TestWith([ConvertedType::UINT_8])] + #[TestWith([ConvertedType::UINT_16])] + #[TestWith([ConvertedType::INT_32])] + #[TestWith([ConvertedType::INT_64])] + public function test_for_column_skips_what_a_signed_int_holds(ConvertedType $convertedType): void + { + static::assertNull(UnsignedIntegerConverter::forColumn( + new FlatColumn('u', PhysicalType::INT32, $convertedType), + Options::default(), + )); + static::assertNull(UnsignedIntegerConverter::forColumn(FlatColumn::int64('i'), Options::default())); + } +} diff --git a/src/lib/parquet/tests/Flow/Parquet/Tests/Unit/ParquetFile/Schema/FlatColumnTest.php b/src/lib/parquet/tests/Flow/Parquet/Tests/Unit/ParquetFile/Schema/FlatColumnTest.php index c2b3a10840..fdd6851eb0 100644 --- a/src/lib/parquet/tests/Flow/Parquet/Tests/Unit/ParquetFile/Schema/FlatColumnTest.php +++ b/src/lib/parquet/tests/Flow/Parquet/Tests/Unit/ParquetFile/Schema/FlatColumnTest.php @@ -8,6 +8,7 @@ use Flow\Parquet\ParquetFile\Schema; use Flow\Parquet\ParquetFile\Schema\ConvertedType; use Flow\Parquet\ParquetFile\Schema\FlatColumn; +use Flow\Parquet\ParquetFile\Schema\LogicalType; use Flow\Parquet\ParquetFile\Schema\PhysicalType; use Flow\Parquet\ParquetFile\Schema\Repetition; use Flow\Parquet\ParquetFile\Schema\TimeUnit; @@ -108,4 +109,13 @@ public function test_repetitions(): void Schema::with(FlatColumn::int32('int32'))->get('int32')->repetitions()->toArray(), ); } + + public function test_enum_is_annotated_enum(): void + { + $column = FlatColumn::enum('color'); + + static::assertSame(ConvertedType::ENUM, $column->convertedType()); + static::assertSame(LogicalType::ENUM, $column->logicalType()?->name()); + static::assertNotNull($column->toThrift()->logicalType->ENUM); + } } diff --git a/src/lib/parquet/tests/Flow/Parquet/Tests/Unit/ParquetFile/Schema/LogicalType/IntegerTest.php b/src/lib/parquet/tests/Flow/Parquet/Tests/Unit/ParquetFile/Schema/LogicalType/IntegerTest.php new file mode 100644 index 0000000000..65350faebe --- /dev/null +++ b/src/lib/parquet/tests/Flow/Parquet/Tests/Unit/ParquetFile/Schema/LogicalType/IntegerTest.php @@ -0,0 +1,78 @@ +toThrift(); + + static::assertEquals(new IntType(['bitWidth' => $bitWidth, 'isSigned' => $isSigned]), $thrift); + static::assertEquals(new Integer($bitWidth, $isSigned), Integer::fromThrift($thrift)); + static::assertSame($bitWidth, Integer::fromThrift($thrift)->bitWidth()); + static::assertSame($isSigned, Integer::fromThrift($thrift)->isSigned()); + } +} diff --git a/src/lib/parquet/tests/Flow/Parquet/Tests/Unit/ParquetFile/Schema/LogicalTypeTest.php b/src/lib/parquet/tests/Flow/Parquet/Tests/Unit/ParquetFile/Schema/LogicalTypeTest.php index 592dc59318..bd043c1c1f 100644 --- a/src/lib/parquet/tests/Flow/Parquet/Tests/Unit/ParquetFile/Schema/LogicalTypeTest.php +++ b/src/lib/parquet/tests/Flow/Parquet/Tests/Unit/ParquetFile/Schema/LogicalTypeTest.php @@ -6,7 +6,9 @@ use Flow\Parquet\ParquetFile\Schema\ConvertedType; use Flow\Parquet\ParquetFile\Schema\LogicalType; +use Flow\Parquet\ParquetFile\Schema\LogicalType\Integer; use Flow\Parquet\ParquetFile\Schema\TimeUnit; +use Flow\Parquet\ThriftModel\IntType; use Generator; use PHPUnit\Framework\Attributes\DataProvider; use PHPUnit\Framework\Attributes\TestWith; @@ -96,4 +98,16 @@ public function test_timestamp_is_an_utc_instant(): void static::assertTrue(LogicalType::timestamp()->timestampData()?->isAdjustedToUTC()); static::assertSame(TimeUnit::MICROSECONDS, LogicalType::timestamp()->timestampData()?->unit()); } + + #[TestWith([8, true])] + #[TestWith([64, true])] + #[TestWith([32, false])] + #[TestWith([64, false])] + public function test_integer_keeps_its_width_and_sign_through_thrift(int $bitWidth, bool $isSigned): void + { + $thrift = LogicalType::integer($bitWidth, $isSigned)->toThrift(); + + static::assertEquals(new IntType(['bitWidth' => $bitWidth, 'isSigned' => $isSigned]), $thrift->INTEGER); + static::assertEquals(new Integer($bitWidth, $isSigned), LogicalType::fromThrift($thrift)?->integerData()); + } } diff --git a/src/lib/parquet/tests/Flow/Parquet/Tests/Unit/ParquetFileTest.php b/src/lib/parquet/tests/Flow/Parquet/Tests/Unit/ParquetFileTest.php new file mode 100644 index 0000000000..0df103c808 --- /dev/null +++ b/src/lib/parquet/tests/Flow/Parquet/Tests/Unit/ParquetFileTest.php @@ -0,0 +1,91 @@ +close(); + $file->close(); + unset($file); + + static::assertSame(1, $reader->closes); + } + + public function test_dropping_the_file_closes_the_reader(): void + { + $reader = new SpyParquetFileReader(Schema::with(FlatColumn::int64('id'))); + $file = new ParquetFile(new MemorySourceStream('PAR1'), new Options(), $reader); + + unset($file); + + static::assertSame(1, $reader->closes); + } + + public function test_metadata_schema_and_reader_come_from_the_reader(): void + { + $schema = Schema::with(FlatColumn::int64('id')); + $reader = new SpyParquetFileReader($schema); + $file = new ParquetFile(new MemorySourceStream('PAR1'), new Options(), $reader); + + static::assertSame($reader, $file->reader()); + static::assertSame($schema, $file->schema()); + static::assertSame($schema, $file->metadata()->schema()); + } + + public function test_columns_reads_the_asked_columns_through_the_reader(): void + { + $reader = new SpyParquetFileReader(Schema::with(FlatColumn::int64('id'), FlatColumn::string('name')), [[ + 'name' => ['a'], + 'id' => [1], + ]]); + $file = new ParquetFile(new MemorySourceStream('PAR1'), new Options(), $reader); + + static::assertSame( + [['name' => ['a'], 'id' => [1]]], + iterator_to_array($file->columns(10, [3 => 'name', 7 => 'id'], 5, 2), false), + ); + static::assertSame([[['name', 'id'], 10, 5, 2]], $reader->reads); + } + + public function test_columns_without_names_reads_every_root_column(): void + { + $reader = new SpyParquetFileReader(Schema::with(FlatColumn::int64('id'), FlatColumn::string('name'))); + $file = new ParquetFile(new MemorySourceStream('PAR1'), new Options(), $reader); + + iterator_to_array($file->columns(10)); + + static::assertSame([[['id', 'name'], 10, null, null]], $reader->reads); + } + + public function test_columns_refuses_a_column_the_schema_lacks(): void + { + $file = new ParquetFile( + new MemorySourceStream('PAR1'), + new Options(), + new SpyParquetFileReader(Schema::with(FlatColumn::int64('id'))), + ); + + $this->expectException(InvalidArgumentException::class); + $this->expectExceptionMessage('Column "missing" does not exist'); + + iterator_to_array($file->columns(10, ['missing'])); + } +} diff --git a/src/lib/parquet/tests/Flow/Parquet/Tests/Unit/Reader/ColumnReaderTest.php b/src/lib/parquet/tests/Flow/Parquet/Tests/Unit/Reader/ColumnReaderTest.php new file mode 100644 index 0000000000..6af172fd28 --- /dev/null +++ b/src/lib/parquet/tests/Flow/Parquet/Tests/Unit/Reader/ColumnReaderTest.php @@ -0,0 +1,68 @@ + $ids + */ + #[TestWith([null, null, [1, 2, 3, 4, 5, 6]])] + #[TestWith([null, 3, [4, 5, 6]])] + #[TestWith([2, 1, [2, 3]])] + #[TestWith([3, 2, [3, 4, 5]])] + #[TestWith([1, 5, [6]])] + #[TestWith([null, 6, []])] + public function test_a_flat_column_reads_across_row_groups(?int $limit, ?int $offset, array $ids): void + { + $stream = new MemorySourceStream(MemoryParquetFile::threeRowGroups()); + $metadata = (new PhpParquetFileReader($stream, ByteOrder::LITTLE_ENDIAN, new Options()))->metadata(); + + $reader = new ColumnReader( + new ColumnChunkReader(new PageReader(ByteOrder::LITTLE_ENDIAN, new Options()), new Options()), + new DremelAssembler(DataConverter::initialize(new Options())), + ); + + static::assertSame( + array_map(static fn(int $id): array => ['id' => $id], $ids), + iterator_to_array( + $reader->read($metadata->schema()->get('id'), $metadata, $stream, $limit, $offset), + false, + ), + ); + } + + public function test_a_nested_column_reads_across_row_groups(): void + { + $stream = new MemorySourceStream(MemoryParquetFile::threeRowGroups()); + $metadata = (new PhpParquetFileReader($stream, ByteOrder::LITTLE_ENDIAN, new Options()))->metadata(); + + $reader = new ColumnReader( + new ColumnChunkReader(new PageReader(ByteOrder::LITTLE_ENDIAN, new Options()), new Options()), + new DremelAssembler(DataConverter::initialize(new Options())), + ); + + static::assertSame( + array_map(static fn(int $id): array => ['tags' => ['t' . $id]], [2, 3, 4]), + iterator_to_array($reader->read($metadata->schema()->get('tags'), $metadata, $stream, 3, 1), false), + ); + } +} diff --git a/src/lib/parquet/tests/Flow/Parquet/Tests/Unit/ReaderTest.php b/src/lib/parquet/tests/Flow/Parquet/Tests/Unit/ReaderTest.php index 42d6fbaccc..627e0732df 100644 --- a/src/lib/parquet/tests/Flow/Parquet/Tests/Unit/ReaderTest.php +++ b/src/lib/parquet/tests/Flow/Parquet/Tests/Unit/ReaderTest.php @@ -5,31 +5,22 @@ namespace Flow\Parquet\Tests\Unit; use Flow\Filesystem\Stream\NativeLocalSourceStream; -use Flow\Parquet\Engine\ArrowParquetEngine; +use Flow\Parquet\Engine\AdaptiveParquetEngine; use Flow\Parquet\Engine\PhpParquetEngine; -use Flow\Parquet\Exception\RuntimeException; use Flow\Parquet\Reader; use PHPUnit\Framework\TestCase; use ReflectionClass; -use function extension_loaded; use function Flow\Filesystem\DSL\path_real; final class ReaderTest extends TestCase { - public function test_arrow_factory_creates_reader_with_arrow_engine(): void + public function test_the_default_engine_is_adaptive(): void { - if (!extension_loaded('arrow')) { - $this->expectException(RuntimeException::class); - Reader::arrow(); - - return; - } - - $reader = Reader::arrow(); + $reader = new Reader(); static::assertInstanceOf( - ArrowParquetEngine::class, + AdaptiveParquetEngine::class, (new ReflectionClass($reader)) ->getProperty('engine') ->getValue($reader), diff --git a/src/lib/parquet/tests/Flow/Parquet/Tests/Unit/Writer/ColumnListsTest.php b/src/lib/parquet/tests/Flow/Parquet/Tests/Unit/Writer/ColumnListsTest.php new file mode 100644 index 0000000000..172206a034 --- /dev/null +++ b/src/lib/parquet/tests/Flow/Parquet/Tests/Unit/Writer/ColumnListsTest.php @@ -0,0 +1,31 @@ +length(['id' => [1, 2], 'name' => ['a', null]])); + static::assertSame(0, (new ColumnLists())->length(['id' => [], 'name' => []])); + } + + public function test_no_lists_hold_no_rows(): void + { + static::assertSame(0, (new ColumnLists())->length([])); + } + + public function test_lists_of_different_lengths_are_refused_with_every_length(): void + { + $this->expectException(InvalidArgumentException::class); + $this->expectExceptionMessage('writeColumns() takes lists of one length, got "id": 3, "name": 2, "tags": 3'); + + (new ColumnLists())->length(['id' => [1, 2, 3], 'name' => ['a', 'b'], 'tags' => [[], [], []]]); + } +} diff --git a/src/lib/parquet/tests/Flow/Parquet/Tests/Unit/Writer/PageBuilder/DictionaryBuilderTest.php b/src/lib/parquet/tests/Flow/Parquet/Tests/Unit/Writer/PageBuilder/DictionaryBuilderTest.php index bc53eb570a..58b5f4aff0 100644 --- a/src/lib/parquet/tests/Flow/Parquet/Tests/Unit/Writer/PageBuilder/DictionaryBuilderTest.php +++ b/src/lib/parquet/tests/Flow/Parquet/Tests/Unit/Writer/PageBuilder/DictionaryBuilderTest.php @@ -320,7 +320,7 @@ public function test_single_unique_value_creates_single_entry_dictionary(): void public function test_unsupported_byte_array_logical_type_throws_exception(): void { - $column = new FlatColumn('test_column', PhysicalType::BYTE_ARRAY, logicalType: LogicalType::integer()); + $column = new FlatColumn('test_column', PhysicalType::BYTE_ARRAY, logicalType: LogicalType::integer(32, true)); $data = new WriteFlatColumnValues($column, values: [1, 2, 3]); $this->expectException(RuntimeException::class); diff --git a/src/lib/parquet/tests/Flow/Parquet/Tests/Unit/WriterTest.php b/src/lib/parquet/tests/Flow/Parquet/Tests/Unit/WriterTest.php index 4ae9418a4a..e3bf3233d2 100644 --- a/src/lib/parquet/tests/Flow/Parquet/Tests/Unit/WriterTest.php +++ b/src/lib/parquet/tests/Flow/Parquet/Tests/Unit/WriterTest.php @@ -4,25 +4,24 @@ namespace Flow\Parquet\Tests\Unit; +use Flow\Parquet\Engine\AdaptiveParquetEngine; use Flow\Parquet\Engine\PhpParquetEngine; -use Flow\Parquet\Exception\RuntimeException; use Flow\Parquet\Writer; use PHPUnit\Framework\TestCase; use ReflectionClass; -use function extension_loaded; - final class WriterTest extends TestCase { - public function test_arrow_factory_throws_when_extension_not_loaded(): void + public function test_the_default_engine_writes(): void { - if (extension_loaded('arrow')) { - static::markTestSkipped('This test requires the arrow extension to NOT be loaded'); - } - - $this->expectException(RuntimeException::class); + $writer = new Writer(); - Writer::arrow(); + static::assertInstanceOf( + AdaptiveParquetEngine::class, + (new ReflectionClass($writer)) + ->getProperty('engine') + ->getValue($writer), + ); } public function test_php_factory_creates_writer_with_php_engine(): void diff --git a/src/lib/postgresql/tests/Flow/PostgreSql/Tests/Unit/QueryBuilder/Schema/Constraint/ForeignKeyConstraintTest.php b/src/lib/postgresql/tests/Flow/PostgreSql/Tests/Unit/QueryBuilder/Schema/Constraint/ForeignKeyConstraintTest.php index 38617f6d24..c955a3c430 100644 --- a/src/lib/postgresql/tests/Flow/PostgreSql/Tests/Unit/QueryBuilder/Schema/Constraint/ForeignKeyConstraintTest.php +++ b/src/lib/postgresql/tests/Flow/PostgreSql/Tests/Unit/QueryBuilder/Schema/Constraint/ForeignKeyConstraintTest.php @@ -134,11 +134,9 @@ public function test_on_delete_restrict(): void public function test_on_delete_set_default(): void { - $constraint = ForeignKeyConstraint::create( - ['user_id'], - 'users', - ['id'], - )->onDelete(ReferentialAction::SET_DEFAULT); + $constraint = ForeignKeyConstraint::create(['user_id'], 'users', ['id'])->onDelete( + ReferentialAction::SET_DEFAULT, + ); $ast = $constraint->toAst(); @@ -168,11 +166,9 @@ public function test_on_update_cascade(): void public function test_on_update_no_action(): void { - $constraint = ForeignKeyConstraint::create( - ['user_id'], - 'users', - ['id'], - )->onUpdate(ReferentialAction::NO_ACTION); + $constraint = ForeignKeyConstraint::create(['user_id'], 'users', ['id'])->onUpdate( + ReferentialAction::NO_ACTION, + ); $ast = $constraint->toAst(); diff --git a/src/lib/telemetry/tests/Flow/Telemetry/Tests/Integration/Telemetry/SignalLimitsIntegrationTest.php b/src/lib/telemetry/tests/Flow/Telemetry/Tests/Integration/Telemetry/SignalLimitsIntegrationTest.php index dca6ea5a78..c7b002aae7 100644 --- a/src/lib/telemetry/tests/Flow/Telemetry/Tests/Integration/Telemetry/SignalLimitsIntegrationTest.php +++ b/src/lib/telemetry/tests/Flow/Telemetry/Tests/Integration/Telemetry/SignalLimitsIntegrationTest.php @@ -129,7 +129,9 @@ public function test_meter_counter_enforces_cardinality_limit(): void static::assertCount(4, $metrics); - $overflowMetrics = array_values(array_filter($metrics, static fn($m) => $m->attributes->has(MetricLimits::OVERFLOW_ATTRIBUTE))); + $overflowMetrics = array_values(array_filter($metrics, static fn($m) => $m->attributes->has( + MetricLimits::OVERFLOW_ATTRIBUTE, + ))); static::assertCount(1, $overflowMetrics); static::assertTrue($overflowMetrics[0]->attributes->get(MetricLimits::OVERFLOW_ATTRIBUTE)); @@ -154,7 +156,9 @@ public function test_meter_histogram_enforces_cardinality_limit(): void static::assertCount(3, $metrics); - $overflowMetrics = array_values(array_filter($metrics, static fn($m) => $m->attributes->has(MetricLimits::OVERFLOW_ATTRIBUTE))); + $overflowMetrics = array_values(array_filter($metrics, static fn($m) => $m->attributes->has( + MetricLimits::OVERFLOW_ATTRIBUTE, + ))); static::assertCount(1, $overflowMetrics); } @@ -175,7 +179,9 @@ public function test_meter_overflow_aggregates_excess_measurements(): void $metrics = $counter->collect(); - $overflowMetrics = array_values(array_filter($metrics, static fn($m) => $m->attributes->has(MetricLimits::OVERFLOW_ATTRIBUTE))); + $overflowMetrics = array_values(array_filter($metrics, static fn($m) => $m->attributes->has( + MetricLimits::OVERFLOW_ATTRIBUTE, + ))); static::assertSame(12, $overflowMetrics[0]->value); } @@ -196,7 +202,9 @@ public function test_meter_overflow_attribute_set_has_correct_attribute(): void static::assertCount(2, $metrics); - $overflowMetrics = array_values(array_filter($metrics, static fn($m) => $m->attributes->has(MetricLimits::OVERFLOW_ATTRIBUTE))); + $overflowMetrics = array_values(array_filter($metrics, static fn($m) => $m->attributes->has( + MetricLimits::OVERFLOW_ATTRIBUTE, + ))); static::assertTrue($overflowMetrics[0]->attributes->get(MetricLimits::OVERFLOW_ATTRIBUTE)); static::assertCount(1, $overflowMetrics[0]->attributes->normalize()); diff --git a/src/lib/telemetry/tests/Flow/Telemetry/Tests/Unit/DSL/FunctionsTest.php b/src/lib/telemetry/tests/Flow/Telemetry/Tests/Unit/DSL/FunctionsTest.php index 90177baa7f..af6bb024d3 100644 --- a/src/lib/telemetry/tests/Flow/Telemetry/Tests/Unit/DSL/FunctionsTest.php +++ b/src/lib/telemetry/tests/Flow/Telemetry/Tests/Unit/DSL/FunctionsTest.php @@ -195,7 +195,9 @@ public function test_meter_provider_with_limits(): void $metrics = $counter->collect(); static::assertCount(6, $metrics); - $overflowMetrics = array_filter($metrics, static fn($m) => $m->attributes->has(MetricLimits::OVERFLOW_ATTRIBUTE)); + $overflowMetrics = array_filter($metrics, static fn($m) => $m->attributes->has( + MetricLimits::OVERFLOW_ATTRIBUTE, + )); static::assertCount(1, $overflowMetrics); } diff --git a/src/lib/telemetry/tests/Flow/Telemetry/Tests/Unit/Logger/Middleware/AttributeFilteringLogMiddlewareTest.php b/src/lib/telemetry/tests/Flow/Telemetry/Tests/Unit/Logger/Middleware/AttributeFilteringLogMiddlewareTest.php index 6a1b214909..402edd78bd 100644 --- a/src/lib/telemetry/tests/Flow/Telemetry/Tests/Unit/Logger/Middleware/AttributeFilteringLogMiddlewareTest.php +++ b/src/lib/telemetry/tests/Flow/Telemetry/Tests/Unit/Logger/Middleware/AttributeFilteringLogMiddlewareTest.php @@ -241,7 +241,9 @@ public function test_does_not_leak_synthetic_severity_keys_onto_kept_entry(): vo static::assertSame($entry, $processed); static::assertFalse($processed->record->attributes->has(AttributeFilteringLogMiddleware::SEVERITY_KEY)); - static::assertFalse($processed->record->attributes->has(AttributeFilteringLogMiddleware::SEVERITY_NAME_KEY)); + static::assertFalse($processed->record->attributes->has( + AttributeFilteringLogMiddleware::SEVERITY_NAME_KEY, + )); } finally { $tmp->remove(); } diff --git a/src/lib/types/src/Flow/Types/Type/Comparator.php b/src/lib/types/src/Flow/Types/Type/Comparator.php index f544e9e58f..e5bec56122 100644 --- a/src/lib/types/src/Flow/Types/Type/Comparator.php +++ b/src/lib/types/src/Flow/Types/Type/Comparator.php @@ -24,12 +24,8 @@ use function Flow\Types\DSL\type_instance_of; use function in_array; -final class Comparator +final readonly class Comparator { - public function __construct( - private StructureComparison $structures = new StructureComparison(), - ) {} - /** * @param Type $left * @param Type $right @@ -125,7 +121,7 @@ public function equals(Type $left, Type $right): bool } if ($left instanceof StructureType && $right instanceof StructureType) { - return $this->structures->identical($left, $right, $this); + return (new StructureComparison())->identical($left, $right, $this); } return $left->toString() === $right->toString(); diff --git a/src/lib/types/src/Flow/Types/Type/Logical/DateTimeType.php b/src/lib/types/src/Flow/Types/Type/Logical/DateTimeType.php index 0ef8566988..18c06bb247 100644 --- a/src/lib/types/src/Flow/Types/Type/Logical/DateTimeType.php +++ b/src/lib/types/src/Flow/Types/Type/Logical/DateTimeType.php @@ -15,6 +15,7 @@ use Flow\Types\Exception\InvalidTypeException; use Flow\Types\Type; use Flow\Types\Type\Native\String\StringTemporalParts; +use Flow\Types\Type\NonFiniteFloat; use Throwable; use function array_combine; @@ -197,6 +198,10 @@ public function cast(mixed $value): DateTimeImmutable return (new DateTimeImmutable($value, $this->zone))->setTimezone($this->zone); } + if (NonFiniteFloat::is($value)) { + throw new CastingException($value, $this, reason: 'value is not a point in time'); + } + if (is_numeric($value)) { return (new DateTimeImmutable('@' . $value))->setTimezone($this->zone); } diff --git a/src/lib/types/src/Flow/Types/Type/Logical/DateType.php b/src/lib/types/src/Flow/Types/Type/Logical/DateType.php index 6bc4afc991..003d0a460e 100644 --- a/src/lib/types/src/Flow/Types/Type/Logical/DateType.php +++ b/src/lib/types/src/Flow/Types/Type/Logical/DateType.php @@ -14,6 +14,7 @@ use Flow\Types\Exception\InvalidTypeException; use Flow\Types\Type; use Flow\Types\Type\Native\String\StringTemporalParts; +use Flow\Types\Type\NonFiniteFloat; use Throwable; use function is_bool; @@ -52,7 +53,7 @@ public function cast(mixed $value): DateTimeInterface try { if ($value instanceof DateTimeImmutable || $value instanceof DateTime) { - return DateTimeImmutable::createFromInterface($value)->setTime(0, 0, 0, 0); + return new DateTimeImmutable($value->format('Y-m-d'), $utc); } if (is_string($value)) { @@ -71,6 +72,10 @@ public function cast(mixed $value): DateTimeInterface return new DateTimeImmutable((new DateTimeImmutable($value))->format('Y-m-d'), $utc); } + if (NonFiniteFloat::is($value)) { + throw new CastingException($value, $this, reason: 'value is not a point in time'); + } + if (is_numeric($value)) { return (new DateTimeImmutable('@' . $value)) ->setTimezone($utc) @@ -98,7 +103,11 @@ public function cast(mixed $value): DateTimeInterface public function isValid(mixed $value): bool { - return $value instanceof DateTimeInterface && $value->format('H:i:s.u') === '00:00:00.000000'; + return ( + $value instanceof DateTimeInterface + && $value->getOffset() === 0 + && $value->format('H:i:s.u') === '00:00:00.000000' + ); } public function normalize(): array diff --git a/src/lib/types/src/Flow/Types/Type/Logical/InstanceOfType.php b/src/lib/types/src/Flow/Types/Type/Logical/InstanceOfType.php index 03d32bcdf1..9e23c0c098 100644 --- a/src/lib/types/src/Flow/Types/Type/Logical/InstanceOfType.php +++ b/src/lib/types/src/Flow/Types/Type/Logical/InstanceOfType.php @@ -18,6 +18,7 @@ use function interface_exists; use function is_a; use function is_object; +use function is_scalar; use function is_string; /** @@ -75,7 +76,8 @@ public function cast(mixed $value): object } try { - $object = (object) $value; + // an explicit wrapper: PHP 8.5 warns on (object) NAN + $object = is_scalar($value) ? (object) ['scalar' => $value] : (object) $value; if (!$object instanceof $this->class) { throw new CastingException($value, type_instance_of($this->class)); diff --git a/src/lib/types/src/Flow/Types/Type/Logical/ListType.php b/src/lib/types/src/Flow/Types/Type/Logical/ListType.php index 58c32c0172..a48fdf0613 100644 --- a/src/lib/types/src/Flow/Types/Type/Logical/ListType.php +++ b/src/lib/types/src/Flow/Types/Type/Logical/ListType.php @@ -105,7 +105,12 @@ public function cast(mixed $value): array $castedList[$key] = $this->element()->cast($item); } - return $this->assert($castedList); + if ($castedList !== [] && !array_is_list($castedList)) { + throw InvalidTypeException::value($castedList, $this); + } + + /** @var T $castedList */ + return $castedList; } catch (Throwable $e) { throw new CastingException($value, $this, $e); } diff --git a/src/lib/types/src/Flow/Types/Type/Logical/MapType.php b/src/lib/types/src/Flow/Types/Type/Logical/MapType.php index 0b9fbe23fe..4f601bea05 100644 --- a/src/lib/types/src/Flow/Types/Type/Logical/MapType.php +++ b/src/lib/types/src/Flow/Types/Type/Logical/MapType.php @@ -14,6 +14,7 @@ use Throwable; use function array_key_exists; +use function array_keys; use function Flow\Types\DSL\type_array; use function Flow\Types\DSL\type_from_array; use function Flow\Types\DSL\type_literal; @@ -103,6 +104,7 @@ public function cast(mixed $value): array } $castedMap = []; + $castedKeys = []; // @mago-ignore analysis:mixed-assignment foreach ($value as $key => $item) { @@ -113,9 +115,16 @@ public function cast(mixed $value): array } $castedMap[$castedKey] = $this->value->cast($item); + $castedKeys[] = $castedKey; } - return $this->assert($castedMap); + // PHP turns a numeric string key back into an integer + if (array_keys($castedMap) !== $castedKeys) { + throw InvalidTypeException::value($castedMap, $this); + } + + /** @var T $castedMap */ + return $castedMap; } catch (Throwable $e) { throw new CastingException($value, $this, $e); } diff --git a/src/lib/types/src/Flow/Types/Type/Logical/NonEmptyStringType.php b/src/lib/types/src/Flow/Types/Type/Logical/NonEmptyStringType.php index efa560667e..a1ee10ec91 100644 --- a/src/lib/types/src/Flow/Types/Type/Logical/NonEmptyStringType.php +++ b/src/lib/types/src/Flow/Types/Type/Logical/NonEmptyStringType.php @@ -11,12 +11,14 @@ use Flow\Types\Exception\CastingException; use Flow\Types\Exception\InvalidTypeException; use Flow\Types\Type; +use Flow\Types\Type\NonFiniteFloat; use Stringable; use Throwable; use function Flow\Types\DSL\dom_element_to_string; use function is_array; use function is_bool; +use function is_float; use function is_object; use function is_scalar; use function is_string; @@ -83,6 +85,10 @@ public function cast(mixed $value): string throw new CastingException($value, $this); } + if (is_float($value) && ($text = NonFiniteFloat::text($value)) !== null) { + return $text; + } + if (is_scalar($value) || is_object($value) && method_exists($value, '__toString')) { return $this->assert((string) $value); } diff --git a/src/lib/types/src/Flow/Types/Type/Logical/NumericStringType.php b/src/lib/types/src/Flow/Types/Type/Logical/NumericStringType.php index c719cbf714..af915c8cb5 100644 --- a/src/lib/types/src/Flow/Types/Type/Logical/NumericStringType.php +++ b/src/lib/types/src/Flow/Types/Type/Logical/NumericStringType.php @@ -7,8 +7,11 @@ use Flow\Types\Exception\CastingException; use Flow\Types\Exception\InvalidTypeException; use Flow\Types\Type; +use Flow\Types\Type\NonFiniteFloat; use Stringable; +use function is_float; +use function is_int; use function is_numeric; use function is_string; @@ -40,7 +43,7 @@ public function cast(mixed $value): string return $value; } - if (is_numeric($value)) { + if (is_int($value) || is_float($value) && !NonFiniteFloat::is($value)) { return (string) $value; } diff --git a/src/lib/types/src/Flow/Types/Type/Logical/StructureType.php b/src/lib/types/src/Flow/Types/Type/Logical/StructureType.php index 760b4dd1d0..7d58960131 100644 --- a/src/lib/types/src/Flow/Types/Type/Logical/StructureType.php +++ b/src/lib/types/src/Flow/Types/Type/Logical/StructureType.php @@ -44,7 +44,7 @@ final readonly class StructureType implements Type { /** - * @var list>> + * @var non-empty-list>> */ private array $elements; @@ -199,7 +199,12 @@ public function cast(mixed $value): array $castedStructure[$element->name] = $element->type->cast($value[$element->name] ?? null); } - return $this->assert($castedStructure); + if ($castedStructure !== [] && array_is_list($castedStructure)) { + throw InvalidTypeException::value($castedStructure, $this); + } + + /** @var T $castedStructure */ + return $castedStructure; } catch (Throwable $e) { throw new CastingException($value, $this, $e); } @@ -220,7 +225,7 @@ public function element(int|string $name): ?StructureElement } /** - * @return list>> + * @return non-empty-list>> */ public function elements(): array { diff --git a/src/lib/types/src/Flow/Types/Type/Logical/XMLElementType.php b/src/lib/types/src/Flow/Types/Type/Logical/XMLElementType.php index 9c8743a9ab..1cc7676b89 100644 --- a/src/lib/types/src/Flow/Types/Type/Logical/XMLElementType.php +++ b/src/lib/types/src/Flow/Types/Type/Logical/XMLElementType.php @@ -38,7 +38,10 @@ public function cast(mixed $value): DOMElement|Element if (is_string($value)) { $dom = new DOMDocument(); - $dom->loadXML($value); + + if (!@$dom->loadXML($value)) { + throw new CastingException($value, $this); + } return type_instance_of(DOMElement::class)->assert($dom->documentElement); } diff --git a/src/lib/types/src/Flow/Types/Type/Native/BooleanType.php b/src/lib/types/src/Flow/Types/Type/Native/BooleanType.php index 6b150bbff8..a535a8bdec 100644 --- a/src/lib/types/src/Flow/Types/Type/Native/BooleanType.php +++ b/src/lib/types/src/Flow/Types/Type/Native/BooleanType.php @@ -8,6 +8,7 @@ use Flow\Types\Exception\CastingException; use Flow\Types\Exception\InvalidTypeException; use Flow\Types\Type; +use Flow\Types\Type\NonFiniteFloat; use Throwable; use function in_array; @@ -54,6 +55,10 @@ public function cast(mixed $value): bool throw new CastingException($value, $this); } + if (NonFiniteFloat::is($value)) { + throw new CastingException($value, $this); + } + if (is_scalar($value)) { return (bool) $value; } diff --git a/src/lib/types/src/Flow/Types/Type/Native/FloatType.php b/src/lib/types/src/Flow/Types/Type/Native/FloatType.php index aeb81f90cd..f0cbe1df12 100644 --- a/src/lib/types/src/Flow/Types/Type/Native/FloatType.php +++ b/src/lib/types/src/Flow/Types/Type/Native/FloatType.php @@ -10,10 +10,12 @@ use Flow\Types\Exception\CastingException; use Flow\Types\Exception\InvalidTypeException; use Flow\Types\Type; +use Flow\Types\Type\NonFiniteFloat; use function is_bool; use function is_float; use function is_numeric; +use function is_string; /** * @template T of float @@ -56,6 +58,12 @@ public function cast(mixed $value): float return ((float) $endTime->format('Uu') - (float) $reference->format('Uu')) / 1e6; } + $nonFinite = is_string($value) ? NonFiniteFloat::fromText($value) : null; + + if ($nonFinite !== null) { + return $nonFinite; + } + if (is_numeric($value)) { return (float) $value; } diff --git a/src/lib/types/src/Flow/Types/Type/Native/ObjectType.php b/src/lib/types/src/Flow/Types/Type/Native/ObjectType.php index 94d00d1920..d8e7748191 100644 --- a/src/lib/types/src/Flow/Types/Type/Native/ObjectType.php +++ b/src/lib/types/src/Flow/Types/Type/Native/ObjectType.php @@ -8,6 +8,7 @@ use Flow\Types\Type; use function is_object; +use function is_scalar; /** * @template T of object @@ -36,7 +37,8 @@ public function cast(mixed $value): object return $value; } - return (object) $value; + // an explicit wrapper: PHP 8.5 warns on (object) NAN + return is_scalar($value) ? (object) ['scalar' => $value] : (object) $value; } public function isValid(mixed $value): bool diff --git a/src/lib/types/src/Flow/Types/Type/Native/StringType.php b/src/lib/types/src/Flow/Types/Type/Native/StringType.php index 3b483b9d9d..f24d75f419 100644 --- a/src/lib/types/src/Flow/Types/Type/Native/StringType.php +++ b/src/lib/types/src/Flow/Types/Type/Native/StringType.php @@ -14,12 +14,14 @@ use Flow\Types\Exception\CastingException; use Flow\Types\Exception\InvalidTypeException; use Flow\Types\Type; +use Flow\Types\Type\NonFiniteFloat; use Stringable; use Throwable; use function Flow\Types\DSL\dom_element_to_string; use function is_array; use function is_bool; +use function is_float; use function is_object; use function is_scalar; use function is_string; @@ -96,6 +98,10 @@ public function cast(mixed $value): string return $value->innerHTML; } + if (is_float($value) && ($text = NonFiniteFloat::text($value)) !== null) { + return $text; + } + if (is_scalar($value) || is_object($value) && method_exists($value, '__toString')) { return (string) $value; } diff --git a/src/lib/types/src/Flow/Types/Type/NonFiniteFloat.php b/src/lib/types/src/Flow/Types/Type/NonFiniteFloat.php new file mode 100644 index 0000000000..63a83b5baf --- /dev/null +++ b/src/lib/types/src/Flow/Types/Type/NonFiniteFloat.php @@ -0,0 +1,40 @@ + NAN, + 'INF' => INF, + '-INF' => -INF, + default => null, + }; + } + + public static function is(mixed $value): bool + { + return is_float($value) && !is_finite($value); + } + + /** + * @return null|'-INF'|'INF'|'NAN' + */ + public static function text(float $value): ?string + { + return match (true) { + is_nan($value) => 'NAN', + $value === INF => 'INF', + $value === -INF => '-INF', + default => null, + }; + } +} diff --git a/src/lib/types/src/Flow/Types/Value/Uuid.php b/src/lib/types/src/Flow/Types/Value/Uuid.php index ca29283897..18fe95cf63 100644 --- a/src/lib/types/src/Flow/Types/Value/Uuid.php +++ b/src/lib/types/src/Flow/Types/Value/Uuid.php @@ -7,12 +7,16 @@ use Flow\Types\Exception\InvalidArgumentException; use Flow\Types\Exception\RuntimeException; use Ramsey\Uuid\UuidInterface; +use ReflectionClass; use Stringable; use Symfony\Component\Uid\Uuid as SymfonyUuid; +use function bin2hex; use function is_string; use function preg_match; +use function sprintf; use function strlen; +use function substr; final readonly class Uuid implements Stringable { @@ -42,6 +46,30 @@ public function __construct(string|UuidInterface|SymfonyUuid $value) } } + /** + * @throws InvalidArgumentException + */ + public static function fromBytes(string $bytes): self + { + if (strlen($bytes) !== 16) { + throw new InvalidArgumentException('Uuid::fromBytes() expects 16 bytes, got ' . strlen($bytes)); + } + + $hex = bin2hex($bytes); + $uuid = (new ReflectionClass(self::class))->newInstanceWithoutConstructor(); + // @mago-ignore analysis:invalid-property-write + $uuid->value = sprintf( + '%s-%s-%s-%s-%s', + substr($hex, 0, 8), + substr($hex, 8, 4), + substr($hex, 12, 4), + substr($hex, 16, 4), + substr($hex, 20), + ); + + return $uuid; + } + public static function fromString(string $value): self { return new self($value); diff --git a/src/lib/types/tests/Flow/Types/Tests/Unit/Type/CastResultIsValidTest.php b/src/lib/types/tests/Flow/Types/Tests/Unit/Type/CastResultIsValidTest.php new file mode 100644 index 0000000000..bf5fed084c --- /dev/null +++ b/src/lib/types/tests/Flow/Types/Tests/Unit/Type/CastResultIsValidTest.php @@ -0,0 +1,124 @@ + [type_integer()]; + yield 'float' => [type_float()]; + yield 'boolean' => [type_boolean()]; + yield 'string' => [type_string()]; + yield 'numeric_string' => [type_numeric_string()]; + yield 'date' => [type_date()]; + yield 'datetime Europe/Warsaw' => [type_datetime('Europe/Warsaw')]; + yield 'uuid' => [type_uuid()]; + yield 'json' => [type_json()]; + yield 'xml' => [type_xml()]; + yield 'optional' => [type_optional(type_date())]; + yield 'list' => [type_list(type_date())]; + yield 'list' => [type_list(type_optional(type_numeric_string()))]; + yield 'list>' => [type_list(type_list(type_integer()))]; + yield 'map' => [type_map(type_string(), type_date())]; + yield 'map' => [type_map(type_integer(), type_string())]; + yield 'structure{a: date}' => [type_structure(['a' => type_date()])]; + yield 'structure{a: numeric_string, b?: numeric_string}' => [type_structure([ + 'a' => type_numeric_string(), + 'b' => structure_element('b', type_numeric_string(), optional: true), + ])]; + yield 'structure{a?: string}' => [type_structure(['a' => structure_element( + 'a', + type_string(), + optional: true, + )])]; + yield 'list' => [type_list(type_structure(['a' => type_float()]))]; + } + + /** + * @param Type $type + */ + #[DataProvider('types')] + public function test_every_cast_result_is_valid_for_its_type(Type $type): void + { + $warsaw = new DateTimeImmutable('2024-01-02 03:04:05.123456', new DateTimeZone('Europe/Warsaw')); + $values = [ + null, + true, + 0, + 5, + 1.5, + NAN, + INF, + '', + '5', + 'abc', + '1e3', + '2024-01-02', + '2024-01-02T03:04:05Z', + '0193f3a0-7b1c-7e6d-9d3a-1c2b3a4d5e6f', + '{"a":1}', + '', + $warsaw, + new DateTime('2024-01-02'), + new Uuid('0193f3a0-7b1c-7e6d-9d3a-1c2b3a4d5e6f'), + new Json('{"a":1}'), + ['5'], + ['a' => '2024-01-02'], + [1 => 'x'], + ['a' => $warsaw], + [[1, 2]], + ['a' => '1', 'b' => INF], + [], + [['a' => 1.5]], + ['1' => 'a', '2' => 'b'], + [null], + ]; + + foreach ($values as $index => $value) { + try { + // @mago-ignore analysis:mixed-assignment + $cast = $type->cast($value); + } catch (Throwable) { + continue; + } + + static::assertTrue( + $type->isValid($cast), + "value #{$index} cast into {$type->toString()} is not valid for it", + ); + } + } +} diff --git a/src/lib/types/tests/Flow/Types/Tests/Unit/Type/Logical/DateTimeTypeTest.php b/src/lib/types/tests/Flow/Types/Tests/Unit/Type/Logical/DateTimeTypeTest.php index 1a4fabe99e..61ccd4635a 100644 --- a/src/lib/types/tests/Flow/Types/Tests/Unit/Type/Logical/DateTimeTypeTest.php +++ b/src/lib/types/tests/Flow/Types/Tests/Unit/Type/Logical/DateTimeTypeTest.php @@ -492,4 +492,14 @@ public function test_cast_does_not_mutate_a_datetime(): void static::assertSame('2021-01-01 00:00:00 UTC', $value->format('Y-m-d H:i:s e')); } + + #[TestWith([NAN])] + #[TestWith([INF])] + #[TestWith([-INF])] + public function test_a_non_finite_float_is_refused(float $value): void + { + $this->expectException(CastingException::class); + + type_datetime()->cast($value); + } } diff --git a/src/lib/types/tests/Flow/Types/Tests/Unit/Type/Logical/DateTypeTest.php b/src/lib/types/tests/Flow/Types/Tests/Unit/Type/Logical/DateTypeTest.php index 39a77a6301..023f4dcb82 100644 --- a/src/lib/types/tests/Flow/Types/Tests/Unit/Type/Logical/DateTypeTest.php +++ b/src/lib/types/tests/Flow/Types/Tests/Unit/Type/Logical/DateTypeTest.php @@ -123,6 +123,16 @@ public static function is_valid_data_provider(): Generator 'expected' => true, ]; + yield 'midnight outside UTC is not a date' => [ + 'value' => new DateTimeImmutable('2026-01-01 00:00:00', new DateTimeZone('Europe/Warsaw')), + 'expected' => false, + ]; + + yield 'midnight in UTC is a date' => [ + 'value' => new DateTimeImmutable('2026-01-01 00:00:00', new DateTimeZone('UTC')), + 'expected' => true, + ]; + yield 'invalid DateTimeImmutable with time' => [ 'value' => new DateTimeImmutable(), 'expected' => false, @@ -274,4 +284,25 @@ public function test_string_date_is_midnight_utc_under_non_utc_default(string $v date_default_timezone_set($previous); } } + + #[TestWith(['2026-01-02 00:30:00', 'Europe/Warsaw'])] + #[TestWith(['2026-01-02 23:30:00', 'America/New_York'])] + public function test_a_zoned_datetime_casts_to_midnight_utc_of_its_calendar_day( + string $datetime, + string $zone, + ): void { + $date = type_date()->cast(new DateTimeImmutable($datetime, new DateTimeZone($zone))); + + static::assertSame('2026-01-02 00:00:00 UTC', $date->format('Y-m-d H:i:s e')); + } + + #[TestWith([NAN])] + #[TestWith([INF])] + #[TestWith([-INF])] + public function test_a_non_finite_float_is_refused(float $value): void + { + $this->expectException(CastingException::class); + + type_date()->cast($value); + } } diff --git a/src/lib/types/tests/Flow/Types/Tests/Unit/Type/Logical/InstanceOfTypeTest.php b/src/lib/types/tests/Flow/Types/Tests/Unit/Type/Logical/InstanceOfTypeTest.php index d7004fe249..27dfe4330e 100644 --- a/src/lib/types/tests/Flow/Types/Tests/Unit/Type/Logical/InstanceOfTypeTest.php +++ b/src/lib/types/tests/Flow/Types/Tests/Unit/Type/Logical/InstanceOfTypeTest.php @@ -161,4 +161,9 @@ public function test_to_string(): void { static::assertSame('object', type_instance_of(DateTimeImmutable::class)->toString()); } + + public function test_a_nan_is_wrapped_as_its_scalar(): void + { + static::assertNan(type_instance_of(stdClass::class)->cast(NAN)->scalar); + } } diff --git a/src/lib/types/tests/Flow/Types/Tests/Unit/Type/Logical/ListTypeTest.php b/src/lib/types/tests/Flow/Types/Tests/Unit/Type/Logical/ListTypeTest.php index 75c50bdd1b..41315c342e 100644 --- a/src/lib/types/tests/Flow/Types/Tests/Unit/Type/Logical/ListTypeTest.php +++ b/src/lib/types/tests/Flow/Types/Tests/Unit/Type/Logical/ListTypeTest.php @@ -4,6 +4,8 @@ namespace Flow\Types\Tests\Unit\Type\Logical; +use DateTimeImmutable; +use DateTimeInterface; use DateTimeZone; use Flow\Types\Exception\CastingException; use Flow\Types\Exception\InvalidTypeException; @@ -13,7 +15,9 @@ use PHPUnit\Framework\TestCase; use stdClass; +use function array_map; use function Flow\Types\DSL\type_boolean; +use function Flow\Types\DSL\type_date; use function Flow\Types\DSL\type_float; use function Flow\Types\DSL\type_from_array; use function Flow\Types\DSL\type_integer; @@ -255,4 +259,32 @@ public function test_to_string(): void { static::assertSame('list', type_list(type_boolean())->toString()); } + + public function test_a_cast_that_keeps_non_list_keys_is_refused(): void + { + try { + type_list(type_string())->cast([1 => 'x']); + static::fail('Expected CastingException'); + } catch (CastingException $e) { + static::assertSame('Can\'t cast "array" into "list" type', $e->getMessage()); + static::assertInstanceOf(InvalidTypeException::class, $e->getPrevious()); + static::assertSame( + 'Expected type "list", got "map".', + $e->getPrevious()->getMessage(), + ); + } + } + + public function test_a_zoned_datetime_casts_to_its_calendar_day_in_a_list_of_dates(): void + { + $dates = type_list(type_date())->cast([new DateTimeImmutable( + '2026-01-02 00:30:00', + new DateTimeZone('Europe/Warsaw'), + )]); + + static::assertSame( + ['2026-01-02 00:00:00 UTC'], + array_map(static fn(DateTimeInterface $date): string => $date->format('Y-m-d H:i:s e'), $dates), + ); + } } diff --git a/src/lib/types/tests/Flow/Types/Tests/Unit/Type/Logical/MapTypeTest.php b/src/lib/types/tests/Flow/Types/Tests/Unit/Type/Logical/MapTypeTest.php index 2c410c87d4..8c18158955 100644 --- a/src/lib/types/tests/Flow/Types/Tests/Unit/Type/Logical/MapTypeTest.php +++ b/src/lib/types/tests/Flow/Types/Tests/Unit/Type/Logical/MapTypeTest.php @@ -209,4 +209,19 @@ public function test_to_string(): void { static::assertSame('map', type_map(type_string(), type_string())->toString()); } + + public function test_a_key_php_coerces_back_is_refused(): void + { + try { + type_map(type_string(), type_integer())->cast([5 => 1]); + static::fail('Expected CastingException'); + } catch (CastingException $e) { + static::assertSame('Can\'t cast "array" into "map" type', $e->getMessage()); + static::assertInstanceOf(InvalidTypeException::class, $e->getPrevious()); + static::assertSame( + 'Expected type "map", got "map".', + $e->getPrevious()->getMessage(), + ); + } + } } diff --git a/src/lib/types/tests/Flow/Types/Tests/Unit/Type/Logical/NonEmptyStringTypeTest.php b/src/lib/types/tests/Flow/Types/Tests/Unit/Type/Logical/NonEmptyStringTypeTest.php index 8fad639c76..737521e74c 100644 --- a/src/lib/types/tests/Flow/Types/Tests/Unit/Type/Logical/NonEmptyStringTypeTest.php +++ b/src/lib/types/tests/Flow/Types/Tests/Unit/Type/Logical/NonEmptyStringTypeTest.php @@ -12,6 +12,7 @@ use Flow\Types\Exception\InvalidTypeException; use Generator; use PHPUnit\Framework\Attributes\DataProvider; +use PHPUnit\Framework\Attributes\TestWith; use PHPUnit\Framework\TestCase; use function Flow\Types\DSL\type_from_array; @@ -160,4 +161,12 @@ public function test_to_string(): void { static::assertSame('non_empty_string', type_non_empty_string()->toString()); } + + #[TestWith([NAN, 'NAN'])] + #[TestWith([INF, 'INF'])] + #[TestWith([-INF, '-INF'])] + public function test_a_non_finite_float_is_spelled_as_the_float_cast_reads_it(float $value, string $text): void + { + static::assertSame($text, type_non_empty_string()->cast($value)); + } } diff --git a/src/lib/types/tests/Flow/Types/Tests/Unit/Type/Logical/NumericStringTypeTest.php b/src/lib/types/tests/Flow/Types/Tests/Unit/Type/Logical/NumericStringTypeTest.php index 1be82b860e..eaef39d2c6 100644 --- a/src/lib/types/tests/Flow/Types/Tests/Unit/Type/Logical/NumericStringTypeTest.php +++ b/src/lib/types/tests/Flow/Types/Tests/Unit/Type/Logical/NumericStringTypeTest.php @@ -10,6 +10,7 @@ use Flow\Types\Tests\Unit\Type\Fixtures\StringableObject; use Generator; use PHPUnit\Framework\Attributes\DataProvider; +use PHPUnit\Framework\Attributes\TestWith; use PHPUnit\Framework\TestCase; use function Flow\Types\DSL\type_from_array; @@ -154,4 +155,14 @@ public function test_to_string(): void { static::assertSame('numeric-string', type_numeric_string()->toString()); } + + #[TestWith([NAN])] + #[TestWith([INF])] + #[TestWith([-INF])] + public function test_a_non_finite_float_is_refused(float $value): void + { + $this->expectException(CastingException::class); + + type_numeric_string()->cast($value); + } } diff --git a/src/lib/types/tests/Flow/Types/Tests/Unit/Type/Logical/StructureTypeTest.php b/src/lib/types/tests/Flow/Types/Tests/Unit/Type/Logical/StructureTypeTest.php index 4716ab3b54..864bb2cafa 100644 --- a/src/lib/types/tests/Flow/Types/Tests/Unit/Type/Logical/StructureTypeTest.php +++ b/src/lib/types/tests/Flow/Types/Tests/Unit/Type/Logical/StructureTypeTest.php @@ -868,4 +868,27 @@ public function test_to_string_with_optional_elements(): void $struct->toString(), ); } + + public function test_a_cast_whose_keys_form_a_list_is_refused(): void + { + try { + type_structure(['0' => structure_element('0', type_integer())])->cast(['0' => 1]); + static::fail('Expected CastingException'); + } catch (CastingException $e) { + static::assertSame('Can\'t cast "array" into "structure{0: integer}" type', $e->getMessage()); + static::assertInstanceOf(InvalidTypeException::class, $e->getPrevious()); + static::assertSame( + 'Expected type "structure{0: integer}", got "list".', + $e->getPrevious()->getMessage(), + ); + } + } + + public function test_an_all_optional_structure_casts_an_empty_array(): void + { + static::assertSame( + [], + type_structure(['a' => structure_element('a', type_string(), optional: true)])->cast([]), + ); + } } diff --git a/src/lib/types/tests/Flow/Types/Tests/Unit/Type/Logical/XMLElementTypeTest.php b/src/lib/types/tests/Flow/Types/Tests/Unit/Type/Logical/XMLElementTypeTest.php index 93c98edf91..112aa4a245 100644 --- a/src/lib/types/tests/Flow/Types/Tests/Unit/Type/Logical/XMLElementTypeTest.php +++ b/src/lib/types/tests/Flow/Types/Tests/Unit/Type/Logical/XMLElementTypeTest.php @@ -7,14 +7,18 @@ use DateTimeImmutable; use DateTimeZone; use DOMElement; +use Flow\Types\Exception\CastingException; use Flow\Types\Exception\InvalidTypeException; use Generator; use PHPUnit\Framework\Attributes\DataProvider; use PHPUnit\Framework\TestCase; use stdClass; +use function error_reporting; use function Flow\Types\DSL\type_from_array; use function Flow\Types\DSL\type_xml_element; +use function restore_error_handler; +use function set_error_handler; final class XMLElementTypeTest extends TestCase { @@ -157,4 +161,27 @@ public function test_to_string(): void { static::assertSame('xml_element', type_xml_element()->toString()); } + + public function test_a_string_that_is_not_xml_is_refused_without_a_warning(): void + { + $warnings = []; + set_error_handler(static function (int $level, string $message) use (&$warnings): bool { + if ((error_reporting() & $level) !== 0) { + $warnings[] = $message; + } + + return true; + }); + + try { + type_xml_element()->cast('not xml'); + static::fail('Expected CastingException'); + } catch (CastingException $e) { + static::assertSame('Can\'t cast "string" into "xml_element" type', $e->getMessage()); + } finally { + restore_error_handler(); + } + + static::assertSame([], $warnings); + } } diff --git a/src/lib/types/tests/Flow/Types/Tests/Unit/Type/Native/BooleanTypeTest.php b/src/lib/types/tests/Flow/Types/Tests/Unit/Type/Native/BooleanTypeTest.php index 9c3437f1e9..9469fd3333 100644 --- a/src/lib/types/tests/Flow/Types/Tests/Unit/Type/Native/BooleanTypeTest.php +++ b/src/lib/types/tests/Flow/Types/Tests/Unit/Type/Native/BooleanTypeTest.php @@ -12,6 +12,7 @@ use Flow\Types\Exception\InvalidTypeException; use Generator; use PHPUnit\Framework\Attributes\DataProvider; +use PHPUnit\Framework\Attributes\TestWith; use PHPUnit\Framework\TestCase; use function Flow\Types\DSL\type_boolean; @@ -245,4 +246,14 @@ public function test_to_string(): void { static::assertSame('boolean', type_boolean()->toString()); } + + #[TestWith([NAN])] + #[TestWith([INF])] + #[TestWith([-INF])] + public function test_a_non_finite_float_is_refused(float $value): void + { + $this->expectException(CastingException::class); + + type_boolean()->cast($value); + } } diff --git a/src/lib/types/tests/Flow/Types/Tests/Unit/Type/Native/FloatTypeTest.php b/src/lib/types/tests/Flow/Types/Tests/Unit/Type/Native/FloatTypeTest.php index 593d629359..b2aeddc971 100644 --- a/src/lib/types/tests/Flow/Types/Tests/Unit/Type/Native/FloatTypeTest.php +++ b/src/lib/types/tests/Flow/Types/Tests/Unit/Type/Native/FloatTypeTest.php @@ -102,6 +102,36 @@ public static function cast_data_provider(): Generator 'exceptionClass' => CastingException::class, ]; + yield 'lower-case nan' => [ + 'value' => 'nan', + 'expected' => null, + 'exceptionClass' => CastingException::class, + ]; + + yield 'mixed-case Inf' => [ + 'value' => 'Inf', + 'expected' => null, + 'exceptionClass' => CastingException::class, + ]; + + yield 'signed +INF' => [ + 'value' => '+INF', + 'expected' => null, + 'exceptionClass' => CastingException::class, + ]; + + yield 'INF' => [ + 'value' => 'INF', + 'expected' => INF, + 'exceptionClass' => null, + ]; + + yield '-INF' => [ + 'value' => '-INF', + 'expected' => -INF, + 'exceptionClass' => null, + ]; + yield 'numeric string' => [ 'value' => '12.9', 'expected' => 12.9, @@ -219,6 +249,11 @@ public function test_cast(mixed $value, mixed $expected, ?string $exceptionClass } } + public function test_cast_of_the_nan_text(): void + { + static::assertNan(type_float()->cast('NAN')); + } + #[DataProvider('is_valid_data_provider')] public function test_is_valid(mixed $value, bool $expected): void { diff --git a/src/lib/types/tests/Flow/Types/Tests/Unit/Type/Native/ObjectTypeTest.php b/src/lib/types/tests/Flow/Types/Tests/Unit/Type/Native/ObjectTypeTest.php index de46d49090..9a05d15ae8 100644 --- a/src/lib/types/tests/Flow/Types/Tests/Unit/Type/Native/ObjectTypeTest.php +++ b/src/lib/types/tests/Flow/Types/Tests/Unit/Type/Native/ObjectTypeTest.php @@ -156,4 +156,12 @@ public function test_to_string(): void { static::assertSame('object', type_object()->toString()); } + + public function test_a_nan_is_wrapped_as_its_scalar(): void + { + $object = type_object()->cast(NAN); + + static::assertInstanceOf(stdClass::class, $object); + static::assertNan($object->scalar); + } } diff --git a/src/lib/types/tests/Flow/Types/Tests/Unit/Type/Native/StringTypeTest.php b/src/lib/types/tests/Flow/Types/Tests/Unit/Type/Native/StringTypeTest.php index c34b521389..29a52d1c74 100644 --- a/src/lib/types/tests/Flow/Types/Tests/Unit/Type/Native/StringTypeTest.php +++ b/src/lib/types/tests/Flow/Types/Tests/Unit/Type/Native/StringTypeTest.php @@ -18,6 +18,7 @@ use Generator; use PHPUnit\Framework\Attributes\DataProvider; use PHPUnit\Framework\Attributes\RequiresPhp; +use PHPUnit\Framework\Attributes\TestWith; use PHPUnit\Framework\TestCase; use stdClass; use Stringable; @@ -290,4 +291,12 @@ public function test_to_string(): void { static::assertSame('string', type_string()->toString()); } + + #[TestWith([NAN, 'NAN'])] + #[TestWith([INF, 'INF'])] + #[TestWith([-INF, '-INF'])] + public function test_a_non_finite_float_is_spelled_as_the_float_cast_reads_it(float $value, string $text): void + { + static::assertSame($text, type_string()->cast($value)); + } } diff --git a/src/lib/types/tests/Flow/Types/Tests/Unit/Type/NonFiniteFloatTest.php b/src/lib/types/tests/Flow/Types/Tests/Unit/Type/NonFiniteFloatTest.php new file mode 100644 index 0000000000..f490aa2e51 --- /dev/null +++ b/src/lib/types/tests/Flow/Types/Tests/Unit/Type/NonFiniteFloatTest.php @@ -0,0 +1,51 @@ +toString()); + static::assertEquals(new Uuid('6c2f1d4e-8b3a-4c5d-9e6f-0a1b2c3d4e5f'), $uuid); + } + + public function test_from_bytes_refuses_fifteen_bytes(): void + { + $this->expectException(InvalidArgumentException::class); + $this->expectExceptionMessage('Uuid::fromBytes() expects 16 bytes, got 15'); + + Uuid::fromBytes(str_repeat('a', 15)); + } + + public function test_from_bytes_refuses_seventeen_bytes(): void + { + $this->expectException(InvalidArgumentException::class); + $this->expectExceptionMessage('Uuid::fromBytes() expects 16 bytes, got 17'); + + Uuid::fromBytes(str_repeat('a', 17)); + } + public function test_construct_with_invalid_string_uuid_throws_exception(): void { $this->expectException(InvalidArgumentException::class); diff --git a/src/tools/documentation/tests/Flow/Documentation/Tests/Integration/DocumentationFencesTest.php b/src/tools/documentation/tests/Flow/Documentation/Tests/Integration/DocumentationFencesTest.php index 5514c4d0fc..d2d650e3ad 100644 --- a/src/tools/documentation/tests/Flow/Documentation/Tests/Integration/DocumentationFencesTest.php +++ b/src/tools/documentation/tests/Flow/Documentation/Tests/Integration/DocumentationFencesTest.php @@ -18,16 +18,17 @@ use function array_filter; use function array_values; +use function fnmatch; use function implode; use function sprintf; final class DocumentationFencesTest extends TestCase { /** - * documentation/upgrading.md is append-only history. Its "Before:" fences are deliberately - * written against APIs that no longer exist, so the whole file is excluded by name. + * documentation/upgrading*.md is append-only history. Its "Before:" fences are deliberately + * written against APIs that no longer exist, so every such file is excluded by pattern. */ - private const EXCLUDED = 'documentation/upgrading.md'; + private const EXCLUDED = 'documentation/upgrading*.md'; public static function documentation_links(): Generator { @@ -43,7 +44,7 @@ public static function documentation_links(): Generator public static function documentation_pages(): Generator { foreach ((new MarkdownCorpusContext())->filesIn(['documentation']) as $file) { - if ($file === self::EXCLUDED) { + if (fnmatch(self::EXCLUDED, $file)) { continue; } @@ -56,7 +57,7 @@ public static function php_fences(): Generator $fences = new MarkdownFences(); foreach ((new MarkdownCorpusContext())->filesIn(['documentation']) as $file) { - if ($file === self::EXCLUDED) { + if (fnmatch(self::EXCLUDED, $file)) { continue; } diff --git a/tools/box/composer.lock b/tools/box/composer.lock index f69c122e6f..8d4b3744c2 100644 --- a/tools/box/composer.lock +++ b/tools/box/composer.lock @@ -902,16 +902,16 @@ }, { "name": "composer/semver", - "version": "3.4.4", + "version": "3.5.0", "source": { "type": "git", "url": "https://github.com/composer/semver.git", - "reference": "198166618906cb2de69b95d7d47e5fa8aa1b2b95" + "reference": "f7a296f4c4cf8cb8bb83e35d6951a406bb11afa5" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/composer/semver/zipball/198166618906cb2de69b95d7d47e5fa8aa1b2b95", - "reference": "198166618906cb2de69b95d7d47e5fa8aa1b2b95", + "url": "https://api.github.com/repos/composer/semver/zipball/f7a296f4c4cf8cb8bb83e35d6951a406bb11afa5", + "reference": "f7a296f4c4cf8cb8bb83e35d6951a406bb11afa5", "shasum": "" }, "require": { @@ -953,7 +953,7 @@ "homepage": "http://robbast.nl" } ], - "description": "Semver library that offers utilities, version constraint parsing and validation.", + "description": "Version comparison library that offers utilities, version constraint parsing and validation.", "keywords": [ "semantic", "semver", @@ -963,7 +963,7 @@ "support": { "irc": "ircs://irc.libera.chat:6697/composer", "issues": "https://github.com/composer/semver/issues", - "source": "https://github.com/composer/semver/tree/3.4.4" + "source": "https://github.com/composer/semver/tree/3.5.0" }, "funding": [ { @@ -975,7 +975,7 @@ "type": "github" } ], - "time": "2025-08-20T19:15:30+00:00" + "time": "2026-09-24T14:38:51+00:00" }, { "name": "composer/xdebug-handler", @@ -1534,16 +1534,16 @@ }, { "name": "justinrainbow/json-schema", - "version": "6.12.0", + "version": "6.13.1", "source": { "type": "git", "url": "https://github.com/jsonrainbow/json-schema.git", - "reference": "8ef236a1a37df364b518b7411a4b5bb95b040d79" + "reference": "e54aacc4c0657b3a4af4f628428d2702146828c6" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/jsonrainbow/json-schema/zipball/8ef236a1a37df364b518b7411a4b5bb95b040d79", - "reference": "8ef236a1a37df364b518b7411a4b5bb95b040d79", + "url": "https://api.github.com/repos/jsonrainbow/json-schema/zipball/e54aacc4c0657b3a4af4f628428d2702146828c6", + "reference": "e54aacc4c0657b3a4af4f628428d2702146828c6", "shasum": "" }, "require": { @@ -1593,9 +1593,9 @@ ], "support": { "issues": "https://github.com/jsonrainbow/json-schema/issues", - "source": "https://github.com/jsonrainbow/json-schema/tree/6.12.0" + "source": "https://github.com/jsonrainbow/json-schema/tree/6.13.1" }, - "time": "2026-09-04T12:54:20+00:00" + "time": "2026-09-30T07:42:28+00:00" }, { "name": "kelunik/certificate", @@ -2021,20 +2021,25 @@ }, { "name": "phpdocumentor/reflection-common", - "version": "2.2.0", + "version": "2.2.1", "source": { "type": "git", "url": "https://github.com/phpDocumentor/ReflectionCommon.git", - "reference": "1d01c49d4ed62f25aa84a747ad35d5a16924662b" + "reference": "ee15a5a2022c8383906b225055e9496a6a5e6e3b" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/phpDocumentor/ReflectionCommon/zipball/1d01c49d4ed62f25aa84a747ad35d5a16924662b", - "reference": "1d01c49d4ed62f25aa84a747ad35d5a16924662b", + "url": "https://api.github.com/repos/phpDocumentor/ReflectionCommon/zipball/ee15a5a2022c8383906b225055e9496a6a5e6e3b", + "reference": "ee15a5a2022c8383906b225055e9496a6a5e6e3b", "shasum": "" }, "require": { - "php": "^7.2 || ^8.0" + "php": "^7.4 || ^8.0" + }, + "require-dev": { + "phpstan/phpstan": "^1.8", + "phpunit/phpunit": "^9.5", + "vimeo/psalm": "^4.25" }, "type": "library", "extra": { @@ -2058,7 +2063,7 @@ } ], "description": "Common reflection classes used by phpdocumentor to reflect the code structure", - "homepage": "http://www.phpdoc.org", + "homepage": "https://www.phpdoc.org", "keywords": [ "FQSEN", "phpDocumentor", @@ -2068,9 +2073,9 @@ ], "support": { "issues": "https://github.com/phpDocumentor/ReflectionCommon/issues", - "source": "https://github.com/phpDocumentor/ReflectionCommon/tree/2.x" + "source": "https://github.com/phpDocumentor/ReflectionCommon/tree/2.2.1" }, - "time": "2020-06-27T09:03:43+00:00" + "time": "2025-11-10T21:21:55+00:00" }, { "name": "phpdocumentor/reflection-docblock", @@ -2196,16 +2201,16 @@ }, { "name": "phpstan/phpdoc-parser", - "version": "2.3.5", + "version": "2.3.6", "source": { "type": "git", "url": "https://github.com/phpstan/phpdoc-parser.git", - "reference": "148cefffaf0233e4c08cc13db8a195a56dd6dfe9" + "reference": "1427af4647235b4a73b13940b1f1258d3584f36e" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/phpstan/phpdoc-parser/zipball/148cefffaf0233e4c08cc13db8a195a56dd6dfe9", - "reference": "148cefffaf0233e4c08cc13db8a195a56dd6dfe9", + "url": "https://api.github.com/repos/phpstan/phpdoc-parser/zipball/1427af4647235b4a73b13940b1f1258d3584f36e", + "reference": "1427af4647235b4a73b13940b1f1258d3584f36e", "shasum": "" }, "require": { @@ -2237,9 +2242,9 @@ "description": "PHPDoc parser with support for nullable, intersection and generic types", "support": { "issues": "https://github.com/phpstan/phpdoc-parser/issues", - "source": "https://github.com/phpstan/phpdoc-parser/tree/2.3.5" + "source": "https://github.com/phpstan/phpdoc-parser/tree/2.3.6" }, - "time": "2026-08-31T16:05:28+00:00" + "time": "2026-09-27T20:07:45+00:00" }, { "name": "psr/container", @@ -2719,16 +2724,16 @@ }, { "name": "seld/phar-utils", - "version": "1.2.2", + "version": "1.2.3", "source": { "type": "git", "url": "https://github.com/Seldaek/phar-utils.git", - "reference": "990bbd0e92caa216d52eca0935f6e35e589bfaa5" + "reference": "65fb381613fd31745d39a740a112a233f978d2d9" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/Seldaek/phar-utils/zipball/990bbd0e92caa216d52eca0935f6e35e589bfaa5", - "reference": "990bbd0e92caa216d52eca0935f6e35e589bfaa5", + "url": "https://api.github.com/repos/Seldaek/phar-utils/zipball/65fb381613fd31745d39a740a112a233f978d2d9", + "reference": "65fb381613fd31745d39a740a112a233f978d2d9", "shasum": "" }, "require": { @@ -2761,22 +2766,22 @@ ], "support": { "issues": "https://github.com/Seldaek/phar-utils/issues", - "source": "https://github.com/Seldaek/phar-utils/tree/1.2.2" + "source": "https://github.com/Seldaek/phar-utils/tree/1.2.3" }, - "time": "2026-08-01T12:48:55+00:00" + "time": "2026-10-01T08:16:23+00:00" }, { "name": "symfony/console", - "version": "v7.4.19", + "version": "v7.4.20", "source": { "type": "git", "url": "https://github.com/symfony/console.git", - "reference": "3a1973458b153f566a8d7499bc06ece323d60063" + "reference": "d7bca2fe05241dc7732d0c8b6d551b879f5d0a7d" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/symfony/console/zipball/3a1973458b153f566a8d7499bc06ece323d60063", - "reference": "3a1973458b153f566a8d7499bc06ece323d60063", + "url": "https://api.github.com/repos/symfony/console/zipball/d7bca2fe05241dc7732d0c8b6d551b879f5d0a7d", + "reference": "d7bca2fe05241dc7732d0c8b6d551b879f5d0a7d", "shasum": "" }, "require": { @@ -2841,7 +2846,7 @@ "terminal" ], "support": { - "source": "https://github.com/symfony/console/tree/v7.4.19" + "source": "https://github.com/symfony/console/tree/v7.4.20" }, "funding": [ { @@ -2861,7 +2866,7 @@ "type": "tidelift" } ], - "time": "2026-09-13T10:38:41+00:00" + "time": "2026-09-18T15:55:00+00:00" }, { "name": "symfony/deprecation-contracts", @@ -3086,16 +3091,16 @@ }, { "name": "symfony/finder", - "version": "v7.4.19", + "version": "v7.4.20", "source": { "type": "git", "url": "https://github.com/symfony/finder.git", - "reference": "1b900bc6ae5ba60c5eee69c11ba44566190576fe" + "reference": "2dd31e5208db7a0fda7bfc998ad727b79c2e4d1e" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/symfony/finder/zipball/1b900bc6ae5ba60c5eee69c11ba44566190576fe", - "reference": "1b900bc6ae5ba60c5eee69c11ba44566190576fe", + "url": "https://api.github.com/repos/symfony/finder/zipball/2dd31e5208db7a0fda7bfc998ad727b79c2e4d1e", + "reference": "2dd31e5208db7a0fda7bfc998ad727b79c2e4d1e", "shasum": "" }, "require": { @@ -3130,7 +3135,7 @@ "description": "Finds files and directories via an intuitive fluent interface", "homepage": "https://symfony.com", "support": { - "source": "https://github.com/symfony/finder/tree/v7.4.19" + "source": "https://github.com/symfony/finder/tree/v7.4.20" }, "funding": [ { @@ -3150,7 +3155,7 @@ "type": "tidelift" } ], - "time": "2026-09-10T19:14:10+00:00" + "time": "2026-09-26T06:31:10+00:00" }, { "name": "symfony/polyfill-ctype", @@ -3237,16 +3242,16 @@ }, { "name": "symfony/polyfill-iconv", - "version": "v1.37.0", + "version": "v1.43.0", "source": { "type": "git", "url": "https://github.com/symfony/polyfill-iconv.git", - "reference": "2c5729fd241b4b22f6e4b436bc3354a4f262df57" + "reference": "254724d95fe01b679cd97d88786e9623ee00bfb7" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/symfony/polyfill-iconv/zipball/2c5729fd241b4b22f6e4b436bc3354a4f262df57", - "reference": "2c5729fd241b4b22f6e4b436bc3354a4f262df57", + "url": "https://api.github.com/repos/symfony/polyfill-iconv/zipball/254724d95fe01b679cd97d88786e9623ee00bfb7", + "reference": "254724d95fe01b679cd97d88786e9623ee00bfb7", "shasum": "" }, "require": { @@ -3297,7 +3302,7 @@ "shim" ], "support": { - "source": "https://github.com/symfony/polyfill-iconv/tree/v1.37.0" + "source": "https://github.com/symfony/polyfill-iconv/tree/v1.43.0" }, "funding": [ { @@ -3317,20 +3322,20 @@ "type": "tidelift" } ], - "time": "2026-04-10T16:19:22+00:00" + "time": "2026-09-29T06:25:14+00:00" }, { "name": "symfony/polyfill-intl-grapheme", - "version": "v1.41.0", + "version": "v1.43.0", "source": { "type": "git", "url": "https://github.com/symfony/polyfill-intl-grapheme.git", - "reference": "bb899c1db0aa8127dc3afe8cda4a67eb24915f8d" + "reference": "5cac8f4db97208823aa353f59a01faad2da52c75" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/symfony/polyfill-intl-grapheme/zipball/bb899c1db0aa8127dc3afe8cda4a67eb24915f8d", - "reference": "bb899c1db0aa8127dc3afe8cda4a67eb24915f8d", + "url": "https://api.github.com/repos/symfony/polyfill-intl-grapheme/zipball/5cac8f4db97208823aa353f59a01faad2da52c75", + "reference": "5cac8f4db97208823aa353f59a01faad2da52c75", "shasum": "" }, "require": { @@ -3379,7 +3384,7 @@ "shim" ], "support": { - "source": "https://github.com/symfony/polyfill-intl-grapheme/tree/v1.41.0" + "source": "https://github.com/symfony/polyfill-intl-grapheme/tree/v1.43.0" }, "funding": [ { @@ -3399,20 +3404,20 @@ "type": "tidelift" } ], - "time": "2026-07-28T08:25:59+00:00" + "time": "2026-09-25T15:46:47+00:00" }, { "name": "symfony/polyfill-intl-normalizer", - "version": "v1.42.0", + "version": "v1.43.0", "source": { "type": "git", "url": "https://github.com/symfony/polyfill-intl-normalizer.git", - "reference": "aa20edea75bd9c48cfecc8360922e5a6e5c44502" + "reference": "ebd57ccc7edb93d632e184efbe7693d2517bab3b" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/symfony/polyfill-intl-normalizer/zipball/aa20edea75bd9c48cfecc8360922e5a6e5c44502", - "reference": "aa20edea75bd9c48cfecc8360922e5a6e5c44502", + "url": "https://api.github.com/repos/symfony/polyfill-intl-normalizer/zipball/ebd57ccc7edb93d632e184efbe7693d2517bab3b", + "reference": "ebd57ccc7edb93d632e184efbe7693d2517bab3b", "shasum": "" }, "require": { @@ -3464,7 +3469,7 @@ "shim" ], "support": { - "source": "https://github.com/symfony/polyfill-intl-normalizer/tree/v1.42.0" + "source": "https://github.com/symfony/polyfill-intl-normalizer/tree/v1.43.0" }, "funding": [ { @@ -3484,20 +3489,20 @@ "type": "tidelift" } ], - "time": "2026-08-07T06:33:24+00:00" + "time": "2026-09-23T22:02:09+00:00" }, { "name": "symfony/polyfill-mbstring", - "version": "v1.38.2", + "version": "v1.43.0", "source": { "type": "git", "url": "https://github.com/symfony/polyfill-mbstring.git", - "reference": "d3d318bad5e7a1bfbd026009c8bfb8d8f99ae6b6" + "reference": "4d5d331bf731e0aedbc6b2f5e61135b7b7996a72" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/symfony/polyfill-mbstring/zipball/d3d318bad5e7a1bfbd026009c8bfb8d8f99ae6b6", - "reference": "d3d318bad5e7a1bfbd026009c8bfb8d8f99ae6b6", + "url": "https://api.github.com/repos/symfony/polyfill-mbstring/zipball/4d5d331bf731e0aedbc6b2f5e61135b7b7996a72", + "reference": "4d5d331bf731e0aedbc6b2f5e61135b7b7996a72", "shasum": "" }, "require": { @@ -3549,7 +3554,7 @@ "shim" ], "support": { - "source": "https://github.com/symfony/polyfill-mbstring/tree/v1.38.2" + "source": "https://github.com/symfony/polyfill-mbstring/tree/v1.43.0" }, "funding": [ { @@ -3569,20 +3574,20 @@ "type": "tidelift" } ], - "time": "2026-05-27T06:59:30+00:00" + "time": "2026-09-24T15:19:04+00:00" }, { "name": "symfony/polyfill-php84", - "version": "v1.38.1", + "version": "v1.43.0", "source": { "type": "git", "url": "https://github.com/symfony/polyfill-php84.git", - "reference": "f4e1dfaee5b74aba5964fe1fd4dfc7ba5e3085fa" + "reference": "22c6548499cbe5c5bb072fc0c42312dff062a015" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/symfony/polyfill-php84/zipball/f4e1dfaee5b74aba5964fe1fd4dfc7ba5e3085fa", - "reference": "f4e1dfaee5b74aba5964fe1fd4dfc7ba5e3085fa", + "url": "https://api.github.com/repos/symfony/polyfill-php84/zipball/22c6548499cbe5c5bb072fc0c42312dff062a015", + "reference": "22c6548499cbe5c5bb072fc0c42312dff062a015", "shasum": "" }, "require": { @@ -3629,7 +3634,7 @@ "shim" ], "support": { - "source": "https://github.com/symfony/polyfill-php84/tree/v1.38.1" + "source": "https://github.com/symfony/polyfill-php84/tree/v1.43.0" }, "funding": [ { @@ -3649,7 +3654,7 @@ "type": "tidelift" } ], - "time": "2026-05-26T12:51:13+00:00" + "time": "2026-09-23T22:01:09+00:00" }, { "name": "symfony/process", diff --git a/tools/infection/composer.lock b/tools/infection/composer.lock index 4fd6ff6c5e..a0a2889e7e 100644 --- a/tools/infection/composer.lock +++ b/tools/infection/composer.lock @@ -239,23 +239,23 @@ }, { "name": "fidry/cpu-core-counter", - "version": "1.3.0", + "version": "1.4.1", "source": { "type": "git", "url": "https://github.com/theofidry/cpu-core-counter.git", - "reference": "db9508f7b1474469d9d3c53b86f817e344732678" + "reference": "d384a7d0d6e67aa497327260ca95023e92dc2032" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/theofidry/cpu-core-counter/zipball/db9508f7b1474469d9d3c53b86f817e344732678", - "reference": "db9508f7b1474469d9d3c53b86f817e344732678", + "url": "https://api.github.com/repos/theofidry/cpu-core-counter/zipball/d384a7d0d6e67aa497327260ca95023e92dc2032", + "reference": "d384a7d0d6e67aa497327260ca95023e92dc2032", "shasum": "" }, "require": { "php": "^7.2 || ^8.0" }, "require-dev": { - "fidry/makefile": "^0.2.0", + "fidry/makefile": "^0.2.0 || ^1.1.2", "fidry/php-cs-fixer-config": "^1.1.2", "phpstan/extension-installer": "^1.2.0", "phpstan/phpstan": "^2.0", @@ -263,6 +263,7 @@ "phpstan/phpstan-phpunit": "^2.0", "phpstan/phpstan-strict-rules": "^2.0", "phpunit/phpunit": "^8.5.31 || ^9.5.26", + "symfony/polyfill-mbstring": "^1.33", "webmozarts/strict-phpunit": "^7.5" }, "type": "library", @@ -288,7 +289,7 @@ ], "support": { "issues": "https://github.com/theofidry/cpu-core-counter/issues", - "source": "https://github.com/theofidry/cpu-core-counter/tree/1.3.0" + "source": "https://github.com/theofidry/cpu-core-counter/tree/1.4.1" }, "funding": [ { @@ -296,7 +297,7 @@ "type": "github" } ], - "time": "2025-08-14T07:29:31+00:00" + "time": "2026-09-29T22:11:39+00:00" }, { "name": "infection/abstract-testframework-adapter", @@ -479,16 +480,16 @@ }, { "name": "infection/infection", - "version": "0.35.4", + "version": "0.35.6", "source": { "type": "git", "url": "https://github.com/infection/infection.git", - "reference": "487bf060228e1272c6a9653468b87a4fe29436df" + "reference": "1b89ae38246fcac2774a8591612f0a863b20e181" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/infection/infection/zipball/487bf060228e1272c6a9653468b87a4fe29436df", - "reference": "487bf060228e1272c6a9653468b87a4fe29436df", + "url": "https://api.github.com/repos/infection/infection/zipball/1b89ae38246fcac2774a8591612f0a863b20e181", + "reference": "1b89ae38246fcac2774a8591612f0a863b20e181", "shasum": "" }, "require": { @@ -519,7 +520,7 @@ "symfony/finder": "^6.4 || ^7.4 || ^8.0", "symfony/polyfill-php85": "^1.33", "symfony/process": "^6.4 || ^7.4 || ^8.0", - "thecodingmachine/safe": "^v3.0", + "thecodingmachine/safe": "^v3.4", "webmozart/assert": "^1.11 || ^2.0" }, "conflict": { @@ -603,7 +604,7 @@ ], "support": { "issues": "https://github.com/infection/infection/issues", - "source": "https://github.com/infection/infection/tree/0.35.4" + "source": "https://github.com/infection/infection/tree/0.35.6" }, "funding": [ { @@ -615,7 +616,7 @@ "type": "open_collective" } ], - "time": "2026-09-02T13:30:25+00:00" + "time": "2026-10-02T12:18:24+00:00" }, { "name": "infection/mutator", @@ -672,16 +673,16 @@ }, { "name": "justinrainbow/json-schema", - "version": "6.12.0", + "version": "6.13.1", "source": { "type": "git", "url": "https://github.com/jsonrainbow/json-schema.git", - "reference": "8ef236a1a37df364b518b7411a4b5bb95b040d79" + "reference": "e54aacc4c0657b3a4af4f628428d2702146828c6" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/jsonrainbow/json-schema/zipball/8ef236a1a37df364b518b7411a4b5bb95b040d79", - "reference": "8ef236a1a37df364b518b7411a4b5bb95b040d79", + "url": "https://api.github.com/repos/jsonrainbow/json-schema/zipball/e54aacc4c0657b3a4af4f628428d2702146828c6", + "reference": "e54aacc4c0657b3a4af4f628428d2702146828c6", "shasum": "" }, "require": { @@ -731,9 +732,9 @@ ], "support": { "issues": "https://github.com/jsonrainbow/json-schema/issues", - "source": "https://github.com/jsonrainbow/json-schema/tree/6.12.0" + "source": "https://github.com/jsonrainbow/json-schema/tree/6.13.1" }, - "time": "2026-09-04T12:54:20+00:00" + "time": "2026-09-30T07:42:28+00:00" }, { "name": "marc-mabe/php-enum", @@ -1302,16 +1303,16 @@ }, { "name": "sanmai/pipeline", - "version": "7.10", + "version": "7.12", "source": { "type": "git", "url": "https://github.com/sanmai/pipeline.git", - "reference": "a8e4e57a6031efc7a92defd13fed9c1c4c73fc71" + "reference": "e7c1f115790ae68ea22f30a646578c587a691399" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/sanmai/pipeline/zipball/a8e4e57a6031efc7a92defd13fed9c1c4c73fc71", - "reference": "a8e4e57a6031efc7a92defd13fed9c1c4c73fc71", + "url": "https://api.github.com/repos/sanmai/pipeline/zipball/e7c1f115790ae68ea22f30a646578c587a691399", + "reference": "e7c1f115790ae68ea22f30a646578c587a691399", "shasum": "" }, "require": { @@ -1328,7 +1329,6 @@ "phpstan/phpstan": "^2", "phpunit/phpunit": "^11 || ^12", "sanmai/phpstan-rules": "^0.3.11", - "sanmai/phpunit-double-colon-syntax": "^0.1.1", "vimeo/psalm": ">=2" }, "type": "library", @@ -1358,7 +1358,7 @@ "description": "General-purpose collections pipeline", "support": { "issues": "https://github.com/sanmai/pipeline/issues", - "source": "https://github.com/sanmai/pipeline/tree/7.10" + "source": "https://github.com/sanmai/pipeline/tree/7.12" }, "funding": [ { @@ -1366,7 +1366,7 @@ "type": "github" } ], - "time": "2026-08-03T04:56:22+00:00" + "time": "2026-09-30T13:46:33+00:00" }, { "name": "sebastian/diff", @@ -1449,16 +1449,16 @@ }, { "name": "symfony/console", - "version": "v7.4.19", + "version": "v7.4.20", "source": { "type": "git", "url": "https://github.com/symfony/console.git", - "reference": "3a1973458b153f566a8d7499bc06ece323d60063" + "reference": "d7bca2fe05241dc7732d0c8b6d551b879f5d0a7d" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/symfony/console/zipball/3a1973458b153f566a8d7499bc06ece323d60063", - "reference": "3a1973458b153f566a8d7499bc06ece323d60063", + "url": "https://api.github.com/repos/symfony/console/zipball/d7bca2fe05241dc7732d0c8b6d551b879f5d0a7d", + "reference": "d7bca2fe05241dc7732d0c8b6d551b879f5d0a7d", "shasum": "" }, "require": { @@ -1523,7 +1523,7 @@ "terminal" ], "support": { - "source": "https://github.com/symfony/console/tree/v7.4.19" + "source": "https://github.com/symfony/console/tree/v7.4.20" }, "funding": [ { @@ -1543,7 +1543,7 @@ "type": "tidelift" } ], - "time": "2026-09-13T10:38:41+00:00" + "time": "2026-09-18T15:55:00+00:00" }, { "name": "symfony/deprecation-contracts", @@ -1688,16 +1688,16 @@ }, { "name": "symfony/finder", - "version": "v7.4.19", + "version": "v7.4.20", "source": { "type": "git", "url": "https://github.com/symfony/finder.git", - "reference": "1b900bc6ae5ba60c5eee69c11ba44566190576fe" + "reference": "2dd31e5208db7a0fda7bfc998ad727b79c2e4d1e" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/symfony/finder/zipball/1b900bc6ae5ba60c5eee69c11ba44566190576fe", - "reference": "1b900bc6ae5ba60c5eee69c11ba44566190576fe", + "url": "https://api.github.com/repos/symfony/finder/zipball/2dd31e5208db7a0fda7bfc998ad727b79c2e4d1e", + "reference": "2dd31e5208db7a0fda7bfc998ad727b79c2e4d1e", "shasum": "" }, "require": { @@ -1732,7 +1732,7 @@ "description": "Finds files and directories via an intuitive fluent interface", "homepage": "https://symfony.com", "support": { - "source": "https://github.com/symfony/finder/tree/v7.4.19" + "source": "https://github.com/symfony/finder/tree/v7.4.20" }, "funding": [ { @@ -1752,7 +1752,7 @@ "type": "tidelift" } ], - "time": "2026-09-10T19:14:10+00:00" + "time": "2026-09-26T06:31:10+00:00" }, { "name": "symfony/polyfill-ctype", @@ -1839,16 +1839,16 @@ }, { "name": "symfony/polyfill-intl-grapheme", - "version": "v1.41.0", + "version": "v1.43.0", "source": { "type": "git", "url": "https://github.com/symfony/polyfill-intl-grapheme.git", - "reference": "bb899c1db0aa8127dc3afe8cda4a67eb24915f8d" + "reference": "5cac8f4db97208823aa353f59a01faad2da52c75" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/symfony/polyfill-intl-grapheme/zipball/bb899c1db0aa8127dc3afe8cda4a67eb24915f8d", - "reference": "bb899c1db0aa8127dc3afe8cda4a67eb24915f8d", + "url": "https://api.github.com/repos/symfony/polyfill-intl-grapheme/zipball/5cac8f4db97208823aa353f59a01faad2da52c75", + "reference": "5cac8f4db97208823aa353f59a01faad2da52c75", "shasum": "" }, "require": { @@ -1897,7 +1897,7 @@ "shim" ], "support": { - "source": "https://github.com/symfony/polyfill-intl-grapheme/tree/v1.41.0" + "source": "https://github.com/symfony/polyfill-intl-grapheme/tree/v1.43.0" }, "funding": [ { @@ -1917,20 +1917,20 @@ "type": "tidelift" } ], - "time": "2026-07-28T08:25:59+00:00" + "time": "2026-09-25T15:46:47+00:00" }, { "name": "symfony/polyfill-intl-normalizer", - "version": "v1.42.0", + "version": "v1.43.0", "source": { "type": "git", "url": "https://github.com/symfony/polyfill-intl-normalizer.git", - "reference": "aa20edea75bd9c48cfecc8360922e5a6e5c44502" + "reference": "ebd57ccc7edb93d632e184efbe7693d2517bab3b" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/symfony/polyfill-intl-normalizer/zipball/aa20edea75bd9c48cfecc8360922e5a6e5c44502", - "reference": "aa20edea75bd9c48cfecc8360922e5a6e5c44502", + "url": "https://api.github.com/repos/symfony/polyfill-intl-normalizer/zipball/ebd57ccc7edb93d632e184efbe7693d2517bab3b", + "reference": "ebd57ccc7edb93d632e184efbe7693d2517bab3b", "shasum": "" }, "require": { @@ -1982,7 +1982,7 @@ "shim" ], "support": { - "source": "https://github.com/symfony/polyfill-intl-normalizer/tree/v1.42.0" + "source": "https://github.com/symfony/polyfill-intl-normalizer/tree/v1.43.0" }, "funding": [ { @@ -2002,20 +2002,20 @@ "type": "tidelift" } ], - "time": "2026-08-07T06:33:24+00:00" + "time": "2026-09-23T22:02:09+00:00" }, { "name": "symfony/polyfill-mbstring", - "version": "v1.38.2", + "version": "v1.43.0", "source": { "type": "git", "url": "https://github.com/symfony/polyfill-mbstring.git", - "reference": "d3d318bad5e7a1bfbd026009c8bfb8d8f99ae6b6" + "reference": "4d5d331bf731e0aedbc6b2f5e61135b7b7996a72" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/symfony/polyfill-mbstring/zipball/d3d318bad5e7a1bfbd026009c8bfb8d8f99ae6b6", - "reference": "d3d318bad5e7a1bfbd026009c8bfb8d8f99ae6b6", + "url": "https://api.github.com/repos/symfony/polyfill-mbstring/zipball/4d5d331bf731e0aedbc6b2f5e61135b7b7996a72", + "reference": "4d5d331bf731e0aedbc6b2f5e61135b7b7996a72", "shasum": "" }, "require": { @@ -2067,7 +2067,7 @@ "shim" ], "support": { - "source": "https://github.com/symfony/polyfill-mbstring/tree/v1.38.2" + "source": "https://github.com/symfony/polyfill-mbstring/tree/v1.43.0" }, "funding": [ { @@ -2087,20 +2087,20 @@ "type": "tidelift" } ], - "time": "2026-05-27T06:59:30+00:00" + "time": "2026-09-24T15:19:04+00:00" }, { "name": "symfony/polyfill-php85", - "version": "v1.41.0", + "version": "v1.43.0", "source": { "type": "git", "url": "https://github.com/symfony/polyfill-php85.git", - "reference": "255fab485aaa1006ed411040c42aecd7b5302d7a" + "reference": "86a42cbcc07adc331481a5ab310a4024bfc227f6" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/symfony/polyfill-php85/zipball/255fab485aaa1006ed411040c42aecd7b5302d7a", - "reference": "255fab485aaa1006ed411040c42aecd7b5302d7a", + "url": "https://api.github.com/repos/symfony/polyfill-php85/zipball/86a42cbcc07adc331481a5ab310a4024bfc227f6", + "reference": "86a42cbcc07adc331481a5ab310a4024bfc227f6", "shasum": "" }, "require": { @@ -2147,7 +2147,7 @@ "shim" ], "support": { - "source": "https://github.com/symfony/polyfill-php85/tree/v1.41.0" + "source": "https://github.com/symfony/polyfill-php85/tree/v1.43.0" }, "funding": [ { @@ -2167,7 +2167,7 @@ "type": "tidelift" } ], - "time": "2026-07-01T12:47:55+00:00" + "time": "2026-09-25T15:46:47+00:00" }, { "name": "symfony/process", diff --git a/tools/infection/phpunit.xml b/tools/infection/phpunit.xml index b150360b53..27447a2060 100644 --- a/tools/infection/phpunit.xml +++ b/tools/infection/phpunit.xml @@ -21,4 +21,7 @@ ../../src/core/etl/src/Flow/ETL/DSL + + + diff --git a/tools/mago/composer.lock b/tools/mago/composer.lock index cb1d1824ca..6124220fd9 100644 --- a/tools/mago/composer.lock +++ b/tools/mago/composer.lock @@ -9,16 +9,16 @@ "packages-dev": [ { "name": "carthage-software/mago", - "version": "1.49.0", + "version": "1.51.2", "source": { "type": "git", "url": "https://github.com/carthage-software/mago.git", - "reference": "bbdaf378e69114f5d9c580862c8533dd3f01c050" + "reference": "5415d6900d330a5a6a70728aaa7f9c52c7672495" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/carthage-software/mago/zipball/bbdaf378e69114f5d9c580862c8533dd3f01c050", - "reference": "bbdaf378e69114f5d9c580862c8533dd3f01c050", + "url": "https://api.github.com/repos/carthage-software/mago/zipball/5415d6900d330a5a6a70728aaa7f9c52c7672495", + "reference": "5415d6900d330a5a6a70728aaa7f9c52c7672495", "shasum": "" }, "require": { @@ -60,7 +60,7 @@ ], "support": { "issues": "https://github.com/carthage-software/mago/issues", - "source": "https://github.com/carthage-software/mago/tree/1.49.0" + "source": "https://github.com/carthage-software/mago/tree/1.51.2" }, "funding": [ { @@ -68,7 +68,7 @@ "type": "github" } ], - "time": "2026-09-16T05:38:39+00:00" + "time": "2026-10-03T09:05:17+00:00" }, { "name": "revolt/event-loop", diff --git a/tools/monorepo/composer.lock b/tools/monorepo/composer.lock index b194b2af7b..ba69c6c40a 100644 --- a/tools/monorepo/composer.lock +++ b/tools/monorepo/composer.lock @@ -462,16 +462,16 @@ }, { "name": "symfony/console", - "version": "v7.4.19", + "version": "v7.4.20", "source": { "type": "git", "url": "https://github.com/symfony/console.git", - "reference": "3a1973458b153f566a8d7499bc06ece323d60063" + "reference": "d7bca2fe05241dc7732d0c8b6d551b879f5d0a7d" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/symfony/console/zipball/3a1973458b153f566a8d7499bc06ece323d60063", - "reference": "3a1973458b153f566a8d7499bc06ece323d60063", + "url": "https://api.github.com/repos/symfony/console/zipball/d7bca2fe05241dc7732d0c8b6d551b879f5d0a7d", + "reference": "d7bca2fe05241dc7732d0c8b6d551b879f5d0a7d", "shasum": "" }, "require": { @@ -536,7 +536,7 @@ "terminal" ], "support": { - "source": "https://github.com/symfony/console/tree/v7.4.19" + "source": "https://github.com/symfony/console/tree/v7.4.20" }, "funding": [ { @@ -556,20 +556,20 @@ "type": "tidelift" } ], - "time": "2026-09-13T10:38:41+00:00" + "time": "2026-09-18T15:55:00+00:00" }, { "name": "symfony/dependency-injection", - "version": "v7.4.17", + "version": "v7.4.20", "source": { "type": "git", "url": "https://github.com/symfony/dependency-injection.git", - "reference": "f318ac9da5aba0be2cf43ecdd562c05a33bc11c0" + "reference": "a7faee783a87b55f27460fc571f8c639bf9d93a2" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/symfony/dependency-injection/zipball/f318ac9da5aba0be2cf43ecdd562c05a33bc11c0", - "reference": "f318ac9da5aba0be2cf43ecdd562c05a33bc11c0", + "url": "https://api.github.com/repos/symfony/dependency-injection/zipball/a7faee783a87b55f27460fc571f8c639bf9d93a2", + "reference": "a7faee783a87b55f27460fc571f8c639bf9d93a2", "shasum": "" }, "require": { @@ -620,7 +620,7 @@ "description": "Allows you to standardize and centralize the way objects are constructed in your application", "homepage": "https://symfony.com", "support": { - "source": "https://github.com/symfony/dependency-injection/tree/v7.4.17" + "source": "https://github.com/symfony/dependency-injection/tree/v7.4.20" }, "funding": [ { @@ -640,7 +640,7 @@ "type": "tidelift" } ], - "time": "2026-08-21T17:40:08+00:00" + "time": "2026-09-29T12:35:16+00:00" }, { "name": "symfony/deprecation-contracts", @@ -715,16 +715,16 @@ }, { "name": "symfony/error-handler", - "version": "v7.4.17", + "version": "v7.4.20", "source": { "type": "git", "url": "https://github.com/symfony/error-handler.git", - "reference": "8373921e231e190a88e2ad526951bbaa791576fa" + "reference": "82239fe00f7cdc9f17c4166fff4ebdc22bd26ec0" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/symfony/error-handler/zipball/8373921e231e190a88e2ad526951bbaa791576fa", - "reference": "8373921e231e190a88e2ad526951bbaa791576fa", + "url": "https://api.github.com/repos/symfony/error-handler/zipball/82239fe00f7cdc9f17c4166fff4ebdc22bd26ec0", + "reference": "82239fe00f7cdc9f17c4166fff4ebdc22bd26ec0", "shasum": "" }, "require": { @@ -773,7 +773,7 @@ "description": "Provides tools to manage errors and ease debugging PHP code", "homepage": "https://symfony.com", "support": { - "source": "https://github.com/symfony/error-handler/tree/v7.4.17" + "source": "https://github.com/symfony/error-handler/tree/v7.4.20" }, "funding": [ { @@ -793,7 +793,7 @@ "type": "tidelift" } ], - "time": "2026-08-21T17:40:08+00:00" + "time": "2026-09-22T18:56:40+00:00" }, { "name": "symfony/event-dispatcher", @@ -1032,16 +1032,16 @@ }, { "name": "symfony/finder", - "version": "v7.4.19", + "version": "v7.4.20", "source": { "type": "git", "url": "https://github.com/symfony/finder.git", - "reference": "1b900bc6ae5ba60c5eee69c11ba44566190576fe" + "reference": "2dd31e5208db7a0fda7bfc998ad727b79c2e4d1e" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/symfony/finder/zipball/1b900bc6ae5ba60c5eee69c11ba44566190576fe", - "reference": "1b900bc6ae5ba60c5eee69c11ba44566190576fe", + "url": "https://api.github.com/repos/symfony/finder/zipball/2dd31e5208db7a0fda7bfc998ad727b79c2e4d1e", + "reference": "2dd31e5208db7a0fda7bfc998ad727b79c2e4d1e", "shasum": "" }, "require": { @@ -1076,7 +1076,7 @@ "description": "Finds files and directories via an intuitive fluent interface", "homepage": "https://symfony.com", "support": { - "source": "https://github.com/symfony/finder/tree/v7.4.19" + "source": "https://github.com/symfony/finder/tree/v7.4.20" }, "funding": [ { @@ -1096,20 +1096,20 @@ "type": "tidelift" } ], - "time": "2026-09-10T19:14:10+00:00" + "time": "2026-09-26T06:31:10+00:00" }, { "name": "symfony/http-foundation", - "version": "v7.4.19", + "version": "v7.4.20", "source": { "type": "git", "url": "https://github.com/symfony/http-foundation.git", - "reference": "f7b953be7801974a5a0685e1228bbc3b12b35936" + "reference": "9d0761c0da6bd5b801b401666834901a9a1ffc33" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/symfony/http-foundation/zipball/f7b953be7801974a5a0685e1228bbc3b12b35936", - "reference": "f7b953be7801974a5a0685e1228bbc3b12b35936", + "url": "https://api.github.com/repos/symfony/http-foundation/zipball/9d0761c0da6bd5b801b401666834901a9a1ffc33", + "reference": "9d0761c0da6bd5b801b401666834901a9a1ffc33", "shasum": "" }, "require": { @@ -1158,7 +1158,7 @@ "description": "Defines an object-oriented layer for the HTTP specification", "homepage": "https://symfony.com", "support": { - "source": "https://github.com/symfony/http-foundation/tree/v7.4.19" + "source": "https://github.com/symfony/http-foundation/tree/v7.4.20" }, "funding": [ { @@ -1178,20 +1178,20 @@ "type": "tidelift" } ], - "time": "2026-09-14T17:41:17+00:00" + "time": "2026-09-24T07:12:44+00:00" }, { "name": "symfony/http-kernel", - "version": "v7.4.19", + "version": "v7.4.20", "source": { "type": "git", "url": "https://github.com/symfony/http-kernel.git", - "reference": "683507424cc84ca85ac1551adb609b6b98216667" + "reference": "392784b5d7b959b8d872542a2530c50c540a788f" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/symfony/http-kernel/zipball/683507424cc84ca85ac1551adb609b6b98216667", - "reference": "683507424cc84ca85ac1551adb609b6b98216667", + "url": "https://api.github.com/repos/symfony/http-kernel/zipball/392784b5d7b959b8d872542a2530c50c540a788f", + "reference": "392784b5d7b959b8d872542a2530c50c540a788f", "shasum": "" }, "require": { @@ -1277,7 +1277,7 @@ "description": "Provides a structured process for converting a Request into a Response", "homepage": "https://symfony.com", "support": { - "source": "https://github.com/symfony/http-kernel/tree/v7.4.19" + "source": "https://github.com/symfony/http-kernel/tree/v7.4.20" }, "funding": [ { @@ -1297,7 +1297,7 @@ "type": "tidelift" } ], - "time": "2026-09-15T07:09:35+00:00" + "time": "2026-09-29T14:05:16+00:00" }, { "name": "symfony/polyfill-ctype", @@ -1384,16 +1384,16 @@ }, { "name": "symfony/polyfill-intl-grapheme", - "version": "v1.41.0", + "version": "v1.43.0", "source": { "type": "git", "url": "https://github.com/symfony/polyfill-intl-grapheme.git", - "reference": "bb899c1db0aa8127dc3afe8cda4a67eb24915f8d" + "reference": "5cac8f4db97208823aa353f59a01faad2da52c75" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/symfony/polyfill-intl-grapheme/zipball/bb899c1db0aa8127dc3afe8cda4a67eb24915f8d", - "reference": "bb899c1db0aa8127dc3afe8cda4a67eb24915f8d", + "url": "https://api.github.com/repos/symfony/polyfill-intl-grapheme/zipball/5cac8f4db97208823aa353f59a01faad2da52c75", + "reference": "5cac8f4db97208823aa353f59a01faad2da52c75", "shasum": "" }, "require": { @@ -1442,7 +1442,7 @@ "shim" ], "support": { - "source": "https://github.com/symfony/polyfill-intl-grapheme/tree/v1.41.0" + "source": "https://github.com/symfony/polyfill-intl-grapheme/tree/v1.43.0" }, "funding": [ { @@ -1462,20 +1462,20 @@ "type": "tidelift" } ], - "time": "2026-07-28T08:25:59+00:00" + "time": "2026-09-25T15:46:47+00:00" }, { "name": "symfony/polyfill-intl-normalizer", - "version": "v1.42.0", + "version": "v1.43.0", "source": { "type": "git", "url": "https://github.com/symfony/polyfill-intl-normalizer.git", - "reference": "aa20edea75bd9c48cfecc8360922e5a6e5c44502" + "reference": "ebd57ccc7edb93d632e184efbe7693d2517bab3b" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/symfony/polyfill-intl-normalizer/zipball/aa20edea75bd9c48cfecc8360922e5a6e5c44502", - "reference": "aa20edea75bd9c48cfecc8360922e5a6e5c44502", + "url": "https://api.github.com/repos/symfony/polyfill-intl-normalizer/zipball/ebd57ccc7edb93d632e184efbe7693d2517bab3b", + "reference": "ebd57ccc7edb93d632e184efbe7693d2517bab3b", "shasum": "" }, "require": { @@ -1527,7 +1527,7 @@ "shim" ], "support": { - "source": "https://github.com/symfony/polyfill-intl-normalizer/tree/v1.42.0" + "source": "https://github.com/symfony/polyfill-intl-normalizer/tree/v1.43.0" }, "funding": [ { @@ -1547,20 +1547,20 @@ "type": "tidelift" } ], - "time": "2026-08-07T06:33:24+00:00" + "time": "2026-09-23T22:02:09+00:00" }, { "name": "symfony/polyfill-mbstring", - "version": "v1.38.2", + "version": "v1.43.0", "source": { "type": "git", "url": "https://github.com/symfony/polyfill-mbstring.git", - "reference": "d3d318bad5e7a1bfbd026009c8bfb8d8f99ae6b6" + "reference": "4d5d331bf731e0aedbc6b2f5e61135b7b7996a72" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/symfony/polyfill-mbstring/zipball/d3d318bad5e7a1bfbd026009c8bfb8d8f99ae6b6", - "reference": "d3d318bad5e7a1bfbd026009c8bfb8d8f99ae6b6", + "url": "https://api.github.com/repos/symfony/polyfill-mbstring/zipball/4d5d331bf731e0aedbc6b2f5e61135b7b7996a72", + "reference": "4d5d331bf731e0aedbc6b2f5e61135b7b7996a72", "shasum": "" }, "require": { @@ -1612,7 +1612,7 @@ "shim" ], "support": { - "source": "https://github.com/symfony/polyfill-mbstring/tree/v1.38.2" + "source": "https://github.com/symfony/polyfill-mbstring/tree/v1.43.0" }, "funding": [ { @@ -1632,20 +1632,20 @@ "type": "tidelift" } ], - "time": "2026-05-27T06:59:30+00:00" + "time": "2026-09-24T15:19:04+00:00" }, { "name": "symfony/polyfill-php85", - "version": "v1.41.0", + "version": "v1.43.0", "source": { "type": "git", "url": "https://github.com/symfony/polyfill-php85.git", - "reference": "255fab485aaa1006ed411040c42aecd7b5302d7a" + "reference": "86a42cbcc07adc331481a5ab310a4024bfc227f6" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/symfony/polyfill-php85/zipball/255fab485aaa1006ed411040c42aecd7b5302d7a", - "reference": "255fab485aaa1006ed411040c42aecd7b5302d7a", + "url": "https://api.github.com/repos/symfony/polyfill-php85/zipball/86a42cbcc07adc331481a5ab310a4024bfc227f6", + "reference": "86a42cbcc07adc331481a5ab310a4024bfc227f6", "shasum": "" }, "require": { @@ -1692,7 +1692,7 @@ "shim" ], "support": { - "source": "https://github.com/symfony/polyfill-php85/tree/v1.41.0" + "source": "https://github.com/symfony/polyfill-php85/tree/v1.43.0" }, "funding": [ { @@ -1712,7 +1712,7 @@ "type": "tidelift" } ], - "time": "2026-07-01T12:47:55+00:00" + "time": "2026-09-25T15:46:47+00:00" }, { "name": "symfony/process", diff --git a/tools/phpbench/composer.lock b/tools/phpbench/composer.lock index acf41e03dd..f48f69f579 100644 --- a/tools/phpbench/composer.lock +++ b/tools/phpbench/composer.lock @@ -86,27 +86,25 @@ }, { "name": "doctrine/lexer", - "version": "3.0.1", + "version": "3.0.3", "source": { "type": "git", "url": "https://github.com/doctrine/lexer.git", - "reference": "31ad66abc0fc9e1a1f2d9bc6a42668d2fbbcd6dd" + "reference": "71d305eaa6480d3d3460c7d053d1da7749fa9cca" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/doctrine/lexer/zipball/31ad66abc0fc9e1a1f2d9bc6a42668d2fbbcd6dd", - "reference": "31ad66abc0fc9e1a1f2d9bc6a42668d2fbbcd6dd", + "url": "https://api.github.com/repos/doctrine/lexer/zipball/71d305eaa6480d3d3460c7d053d1da7749fa9cca", + "reference": "71d305eaa6480d3d3460c7d053d1da7749fa9cca", "shasum": "" }, "require": { "php": "^8.1" }, "require-dev": { - "doctrine/coding-standard": "^12", - "phpstan/phpstan": "^1.10", - "phpunit/phpunit": "^10.5", - "psalm/plugin-phpunit": "^0.18.3", - "vimeo/psalm": "^5.21" + "doctrine/coding-standard": "^14", + "phpstan/phpstan": "^2", + "phpunit/phpunit": "^10.5.58 || ^12.5.4" }, "type": "library", "autoload": { @@ -143,7 +141,7 @@ ], "support": { "issues": "https://github.com/doctrine/lexer/issues", - "source": "https://github.com/doctrine/lexer/tree/3.0.1" + "source": "https://github.com/doctrine/lexer/tree/3.0.3" }, "funding": [ { @@ -159,7 +157,7 @@ "type": "tidelift" } ], - "time": "2024-02-05T11:56:58+00:00" + "time": "2026-10-01T06:59:29+00:00" }, { "name": "phpbench/container", @@ -529,16 +527,16 @@ }, { "name": "symfony/console", - "version": "v7.4.19", + "version": "v7.4.20", "source": { "type": "git", "url": "https://github.com/symfony/console.git", - "reference": "3a1973458b153f566a8d7499bc06ece323d60063" + "reference": "d7bca2fe05241dc7732d0c8b6d551b879f5d0a7d" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/symfony/console/zipball/3a1973458b153f566a8d7499bc06ece323d60063", - "reference": "3a1973458b153f566a8d7499bc06ece323d60063", + "url": "https://api.github.com/repos/symfony/console/zipball/d7bca2fe05241dc7732d0c8b6d551b879f5d0a7d", + "reference": "d7bca2fe05241dc7732d0c8b6d551b879f5d0a7d", "shasum": "" }, "require": { @@ -603,7 +601,7 @@ "terminal" ], "support": { - "source": "https://github.com/symfony/console/tree/v7.4.19" + "source": "https://github.com/symfony/console/tree/v7.4.20" }, "funding": [ { @@ -623,7 +621,7 @@ "type": "tidelift" } ], - "time": "2026-09-13T10:38:41+00:00" + "time": "2026-09-18T15:55:00+00:00" }, { "name": "symfony/deprecation-contracts", @@ -768,16 +766,16 @@ }, { "name": "symfony/finder", - "version": "v7.4.19", + "version": "v7.4.20", "source": { "type": "git", "url": "https://github.com/symfony/finder.git", - "reference": "1b900bc6ae5ba60c5eee69c11ba44566190576fe" + "reference": "2dd31e5208db7a0fda7bfc998ad727b79c2e4d1e" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/symfony/finder/zipball/1b900bc6ae5ba60c5eee69c11ba44566190576fe", - "reference": "1b900bc6ae5ba60c5eee69c11ba44566190576fe", + "url": "https://api.github.com/repos/symfony/finder/zipball/2dd31e5208db7a0fda7bfc998ad727b79c2e4d1e", + "reference": "2dd31e5208db7a0fda7bfc998ad727b79c2e4d1e", "shasum": "" }, "require": { @@ -812,7 +810,7 @@ "description": "Finds files and directories via an intuitive fluent interface", "homepage": "https://symfony.com", "support": { - "source": "https://github.com/symfony/finder/tree/v7.4.19" + "source": "https://github.com/symfony/finder/tree/v7.4.20" }, "funding": [ { @@ -832,7 +830,7 @@ "type": "tidelift" } ], - "time": "2026-09-10T19:14:10+00:00" + "time": "2026-09-26T06:31:10+00:00" }, { "name": "symfony/options-resolver", @@ -990,16 +988,16 @@ }, { "name": "symfony/polyfill-intl-grapheme", - "version": "v1.41.0", + "version": "v1.43.0", "source": { "type": "git", "url": "https://github.com/symfony/polyfill-intl-grapheme.git", - "reference": "bb899c1db0aa8127dc3afe8cda4a67eb24915f8d" + "reference": "5cac8f4db97208823aa353f59a01faad2da52c75" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/symfony/polyfill-intl-grapheme/zipball/bb899c1db0aa8127dc3afe8cda4a67eb24915f8d", - "reference": "bb899c1db0aa8127dc3afe8cda4a67eb24915f8d", + "url": "https://api.github.com/repos/symfony/polyfill-intl-grapheme/zipball/5cac8f4db97208823aa353f59a01faad2da52c75", + "reference": "5cac8f4db97208823aa353f59a01faad2da52c75", "shasum": "" }, "require": { @@ -1048,7 +1046,7 @@ "shim" ], "support": { - "source": "https://github.com/symfony/polyfill-intl-grapheme/tree/v1.41.0" + "source": "https://github.com/symfony/polyfill-intl-grapheme/tree/v1.43.0" }, "funding": [ { @@ -1068,20 +1066,20 @@ "type": "tidelift" } ], - "time": "2026-07-28T08:25:59+00:00" + "time": "2026-09-25T15:46:47+00:00" }, { "name": "symfony/polyfill-intl-normalizer", - "version": "v1.42.0", + "version": "v1.43.0", "source": { "type": "git", "url": "https://github.com/symfony/polyfill-intl-normalizer.git", - "reference": "aa20edea75bd9c48cfecc8360922e5a6e5c44502" + "reference": "ebd57ccc7edb93d632e184efbe7693d2517bab3b" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/symfony/polyfill-intl-normalizer/zipball/aa20edea75bd9c48cfecc8360922e5a6e5c44502", - "reference": "aa20edea75bd9c48cfecc8360922e5a6e5c44502", + "url": "https://api.github.com/repos/symfony/polyfill-intl-normalizer/zipball/ebd57ccc7edb93d632e184efbe7693d2517bab3b", + "reference": "ebd57ccc7edb93d632e184efbe7693d2517bab3b", "shasum": "" }, "require": { @@ -1133,7 +1131,7 @@ "shim" ], "support": { - "source": "https://github.com/symfony/polyfill-intl-normalizer/tree/v1.42.0" + "source": "https://github.com/symfony/polyfill-intl-normalizer/tree/v1.43.0" }, "funding": [ { @@ -1153,20 +1151,20 @@ "type": "tidelift" } ], - "time": "2026-08-07T06:33:24+00:00" + "time": "2026-09-23T22:02:09+00:00" }, { "name": "symfony/polyfill-mbstring", - "version": "v1.38.2", + "version": "v1.43.0", "source": { "type": "git", "url": "https://github.com/symfony/polyfill-mbstring.git", - "reference": "d3d318bad5e7a1bfbd026009c8bfb8d8f99ae6b6" + "reference": "4d5d331bf731e0aedbc6b2f5e61135b7b7996a72" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/symfony/polyfill-mbstring/zipball/d3d318bad5e7a1bfbd026009c8bfb8d8f99ae6b6", - "reference": "d3d318bad5e7a1bfbd026009c8bfb8d8f99ae6b6", + "url": "https://api.github.com/repos/symfony/polyfill-mbstring/zipball/4d5d331bf731e0aedbc6b2f5e61135b7b7996a72", + "reference": "4d5d331bf731e0aedbc6b2f5e61135b7b7996a72", "shasum": "" }, "require": { @@ -1218,7 +1216,7 @@ "shim" ], "support": { - "source": "https://github.com/symfony/polyfill-mbstring/tree/v1.38.2" + "source": "https://github.com/symfony/polyfill-mbstring/tree/v1.43.0" }, "funding": [ { @@ -1238,7 +1236,7 @@ "type": "tidelift" } ], - "time": "2026-05-27T06:59:30+00:00" + "time": "2026-09-24T15:19:04+00:00" }, { "name": "symfony/process", diff --git a/tools/phpunit/composer.lock b/tools/phpunit/composer.lock index f8e0376642..be5401f0f4 100644 --- a/tools/phpunit/composer.lock +++ b/tools/phpunit/composer.lock @@ -244,25 +244,25 @@ }, { "name": "phpunit/php-code-coverage", - "version": "12.5.7", + "version": "12.5.8", "source": { "type": "git", "url": "https://github.com/sebastianbergmann/php-code-coverage.git", - "reference": "186dab580576598076de6818596d12b61801880e" + "reference": "7440368d94738d54f2c1de24dbd3871793bbb0dd" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/sebastianbergmann/php-code-coverage/zipball/186dab580576598076de6818596d12b61801880e", - "reference": "186dab580576598076de6818596d12b61801880e", + "url": "https://api.github.com/repos/sebastianbergmann/php-code-coverage/zipball/7440368d94738d54f2c1de24dbd3871793bbb0dd", + "reference": "7440368d94738d54f2c1de24dbd3871793bbb0dd", "shasum": "" }, "require": { "ext-dom": "*", "ext-libxml": "*", "ext-xmlwriter": "*", - "nikic/php-parser": "^5.7.0", + "nikic/php-parser": "^5.9.0", "php": ">=8.3", - "phpunit/php-text-template": "^5.0", + "phpunit/php-text-template": "^5.0.1", "sebastian/complexity": "^5.0", "sebastian/environment": "^8.1.2", "sebastian/lines-of-code": "^4.0.1", @@ -270,7 +270,7 @@ "theseer/tokenizer": "^2.0.1" }, "require-dev": { - "phpunit/phpunit": "^12.5.28" + "phpunit/phpunit": "^12.5.37" }, "suggest": { "ext-pcov": "PHP extension that provides line coverage", @@ -308,7 +308,7 @@ "support": { "issues": "https://github.com/sebastianbergmann/php-code-coverage/issues", "security": "https://github.com/sebastianbergmann/php-code-coverage/security/policy", - "source": "https://github.com/sebastianbergmann/php-code-coverage/tree/12.5.7" + "source": "https://github.com/sebastianbergmann/php-code-coverage/tree/12.5.8" }, "funding": [ { @@ -328,7 +328,7 @@ "type": "tidelift" } ], - "time": "2026-06-01T13:24:19+00:00" + "time": "2026-10-03T14:03:50+00:00" }, { "name": "phpunit/php-file-iterator", @@ -469,23 +469,23 @@ }, { "name": "phpunit/php-text-template", - "version": "5.0.0", + "version": "5.0.1", "source": { "type": "git", "url": "https://github.com/sebastianbergmann/php-text-template.git", - "reference": "e1367a453f0eda562eedb4f659e13aa900d66c53" + "reference": "babab73b7ef9fae406f2eb3ee11323a61caa63d5" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/sebastianbergmann/php-text-template/zipball/e1367a453f0eda562eedb4f659e13aa900d66c53", - "reference": "e1367a453f0eda562eedb4f659e13aa900d66c53", + "url": "https://api.github.com/repos/sebastianbergmann/php-text-template/zipball/babab73b7ef9fae406f2eb3ee11323a61caa63d5", + "reference": "babab73b7ef9fae406f2eb3ee11323a61caa63d5", "shasum": "" }, "require": { "php": ">=8.3" }, "require-dev": { - "phpunit/phpunit": "^12.0" + "phpunit/phpunit": "^12.5.37" }, "type": "library", "extra": { @@ -517,15 +517,27 @@ "support": { "issues": "https://github.com/sebastianbergmann/php-text-template/issues", "security": "https://github.com/sebastianbergmann/php-text-template/security/policy", - "source": "https://github.com/sebastianbergmann/php-text-template/tree/5.0.0" + "source": "https://github.com/sebastianbergmann/php-text-template/tree/5.0.1" }, "funding": [ { "url": "https://github.com/sebastianbergmann", "type": "github" + }, + { + "url": "https://liberapay.com/sebastianbergmann", + "type": "liberapay" + }, + { + "url": "https://thanks.dev/u/gh/sebastianbergmann", + "type": "thanks_dev" + }, + { + "url": "https://tidelift.com/funding/github/packagist/phpunit/php-text-template", + "type": "tidelift" } ], - "time": "2025-02-07T04:59:16+00:00" + "time": "2026-10-03T04:51:29+00:00" }, { "name": "phpunit/php-timer", @@ -589,16 +601,16 @@ }, { "name": "phpunit/phpunit", - "version": "12.5.35", + "version": "12.5.37", "source": { "type": "git", "url": "https://github.com/sebastianbergmann/phpunit.git", - "reference": "345ac22e42cb14cdc7aa8428655d8bb8c82a4aab" + "reference": "684340c1590e5f3f3b859f5e80619725d88b63ba" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/sebastianbergmann/phpunit/zipball/345ac22e42cb14cdc7aa8428655d8bb8c82a4aab", - "reference": "345ac22e42cb14cdc7aa8428655d8bb8c82a4aab", + "url": "https://api.github.com/repos/sebastianbergmann/phpunit/zipball/684340c1590e5f3f3b859f5e80619725d88b63ba", + "reference": "684340c1590e5f3f3b859f5e80619725d88b63ba", "shasum": "" }, "require": { @@ -667,7 +679,7 @@ "support": { "issues": "https://github.com/sebastianbergmann/phpunit/issues", "security": "https://github.com/sebastianbergmann/phpunit/security/policy", - "source": "https://github.com/sebastianbergmann/phpunit/tree/12.5.35" + "source": "https://github.com/sebastianbergmann/phpunit/tree/12.5.37" }, "funding": [ { @@ -675,7 +687,7 @@ "type": "other" } ], - "time": "2026-09-09T04:48:50+00:00" + "time": "2026-09-29T12:08:54+00:00" }, { "name": "sebastian/cli-parser", diff --git a/wasm/build.sh b/wasm/build.sh index f557fa7ac7..19e5ea71c8 100755 --- a/wasm/build.sh +++ b/wasm/build.sh @@ -208,6 +208,11 @@ export PKG_CONFIG_PATH="$LIBZIP_INSTALL_DIR/lib/pkgconfig:${PKG_CONFIG_PATH:-}" export LIBZIP_CFLAGS="-I$LIBZIP_INSTALL_DIR/include" export LIBZIP_LIBS="-L$LIBZIP_INSTALL_DIR/lib -lzip" +# Without these, pkg-config finds the nix shell's own libxml2 and PHP compiles against its newer headers while linking +# the libxml2 built above - mismatched signatures, and calls into functions the linked library does not have. +export LIBXML_CFLAGS="-I$LIBXML2_INSTALL_DIR/include/libxml2" +export LIBXML_LIBS="-L$LIBXML2_INSTALL_DIR/lib -lxml2" + cd $PHP_PATH # The extension set and the reason for each is in documentation/contributing/wasm.md. @@ -287,7 +292,16 @@ mkdir -p out echo "Compile pib_eval wrapper" emcc $CFLAGS -I . -I Zend -I main -I TSRM/ ../pib_eval.c -c -o pib_eval.o +# --disable-fiber-asm makes Zend/zend_fibers.c switch Fiber stacks with getcontext/makecontext/swapcontext, which +# Emscripten's libc lacks. The shim implements them over Emscripten fibers (needs -sASYNCIFY below). +echo "Compile ucontext shim" +emcc $CFLAGS ../ucontext-emscripten.c -c -o ucontext-emscripten.o + +echo "Compile the libc functions Emscripten leaves out" +emcc $CFLAGS ../libc-emscripten.c -c -o libc-emscripten.o + echo "Link everything together" +# -s DEFAULT_TO_CXX=1 links libc++/libc++abi for the C++ snappy extension: operator new/delete, __cxxabiv1::__class_type_info / __si_class_type_info vtables emcc $CFLAGS $LDFLAGS $WASM64_MODE \ -s ENVIRONMENT=web \ -s EXPORTED_FUNCTIONS='["_pib_eval", "_pib_force_exit", "_pib_heap_bytes"]' \ @@ -301,11 +315,12 @@ emcc $CFLAGS $LDFLAGS $WASM64_MODE \ -s STACK_SIZE=5242880 \ -s ASSERTIONS=0 \ -s INVOKE_RUN=0 \ - -s ERROR_ON_UNDEFINED_SYMBOLS=0 \ + -s ERROR_ON_UNDEFINED_SYMBOLS=1 \ + -s DEFAULT_TO_CXX=1 \ -s ASYNCIFY=1 \ -s STACK_OVERFLOW_CHECK=0 \ -s SAFE_HEAP=0 \ - libs/libphp.a pib_eval.o $LIBXML2_INSTALL_DIR/lib/libxml2.a $LIBPG_QUERY_INSTALL_DIR/libpg_query.a $LIBZIP_INSTALL_DIR/lib/libzip.a -o out/php.js + ucontext-emscripten.o libc-emscripten.o libs/libphp.a pib_eval.o $LIBXML2_INSTALL_DIR/lib/libxml2.a $LIBPG_QUERY_INSTALL_DIR/libpg_query.a $LIBZIP_INSTALL_DIR/lib/libzip.a -o out/php.js echo "Copy outputs to web/landing/assets/wasm" OUTPUT_DIR="$PROJECT_ROOT/../web/landing/assets/wasm" diff --git a/wasm/libc-emscripten.c b/wasm/libc-emscripten.c new file mode 100644 index 0000000000..5739bad0bf --- /dev/null +++ b/wasm/libc-emscripten.c @@ -0,0 +1,41 @@ +/* + * libc functions PHP calls that Emscripten leaves out of its libc build (tools/system_libs.py excludes musl's + * src/legacy and src/process). Defined here so the link runs with -sERROR_ON_UNDEFINED_SYMBOLS=1 instead of turning + * them into stubs that abort the runtime when called. + */ + +#include +#include +#include +#include +#include + +/* musl's src/legacy/getdtablesize.c - php://fd/N checks N against it */ +int getdtablesize(void) +{ + struct rlimit limit; + + getrlimit(RLIMIT_NOFILE, &limit); + + return limit.rlim_cur < INT_MAX ? (int) limit.rlim_cur : INT_MAX; +} + +/* a browser cannot start a process: proc_open() reports ENOSYS as a PHP warning instead of aborting */ +int posix_spawnp( + pid_t *restrict pid, + const char *restrict file, + const posix_spawn_file_actions_t *file_actions, + const posix_spawnattr_t *restrict attributes, + char *const argv[restrict], + char *const envp[restrict] +) +{ + (void) pid; + (void) file; + (void) file_actions; + (void) attributes; + (void) argv; + (void) envp; + + return ENOSYS; +} diff --git a/wasm/ucontext-emscripten.c b/wasm/ucontext-emscripten.c new file mode 100644 index 0000000000..d0be6c4348 --- /dev/null +++ b/wasm/ucontext-emscripten.c @@ -0,0 +1,156 @@ +/* + * getcontext() / makecontext() / swapcontext() for Emscripten, built on its fiber API (needs -sASYNCIFY). + * + * PHP built with --disable-fiber-asm switches Fiber stacks through these three calls (Zend/zend_fibers.c), and + * Emscripten's libc has none of them. Only what zend_fibers.c does is supported; anything else aborts with a message: + * - getcontext() on a context that makecontext() prepares next, + * - makecontext() with no arguments, uc_link == NULL and uc_stack set, + * - swapcontext() from the running context to a prepared or previously saved one. The main context is never + * passed to getcontext() (zend_fiber_init() points it at uninitialised memory), so the running context is + * tracked here, and the main one is captured at the first swap. + */ + +#include +#include +#include +#include +#include + +/* + * At a swap, Asyncify spills the wasm locals of every frame on the stack into the context's buffer: about 1.2 bytes + * per byte of C stack in use (measured on PHP 8.5: ~336 against ~276 bytes per nested internal callback). + * A Fiber gives half of its own stack to the buffer, so nesting is bounded by the stack as on native PHP, at no extra + * memory. The main context is captured at its first swap, from Flow's pipeline loop (~1.3 KB measured); 256 KB covers + * ~750 nested internal callbacks there. Overflowing either buffer traps in Asyncify ("unreachable"), never corrupts. + */ +#define FLOW_UCONTEXT_MAIN_ASYNCIFY_SIZE (256 * 1024) +#define FLOW_UCONTEXT_MIN_STACK_SIZE (64 * 1024) + +typedef struct flow_ucontext { + emscripten_fiber_t fiber; + void (*entry)(void); + size_t asyncify_size; + char *asyncify_stack; +} flow_ucontext; + +_Static_assert(sizeof(((ucontext_t *) 0)->uc_mcontext) >= sizeof(flow_ucontext *), "uc_mcontext cannot hold the state pointer"); + +static flow_ucontext *running = NULL; +static flow_ucontext main_context; +static char main_asyncify_stack[FLOW_UCONTEXT_MAIN_ASYNCIFY_SIZE] __attribute__((aligned(16))); + +static _Noreturn void refuse(const char *what) +{ + fprintf(stderr, "ucontext-emscripten: %s is not supported\n", what); + abort(); +} + +static flow_ucontext **slot(ucontext_t *ucp) +{ + return (flow_ucontext **) &ucp->uc_mcontext; +} + +#ifdef FLOW_UCONTEXT_TRACE +/* the context that swapped away last: until it is resumed, its buffer holds its whole unwound stack */ +static flow_ucontext *unwound = NULL; +static size_t high_water_main = 0; +static size_t high_water_fiber = 0; + +static void trace(void) +{ + size_t used = (size_t) ((char *) unwound->fiber.asyncify_data.stack_ptr - unwound->asyncify_stack); + size_t *mark = unwound == &main_context ? &high_water_main : &high_water_fiber; + + if (used > *mark) { + *mark = used; + fprintf( + stderr, + "ucontext-emscripten: asyncify high water %s %zu bytes, C stack in use %zu bytes\n", + unwound == &main_context ? "main" : "fiber", + used, + (size_t) ((char *) unwound->fiber.stack_base - (char *) unwound->fiber.stack_ptr) + ); + } +} +#endif + +static void enter(void *arg) +{ +#ifdef FLOW_UCONTEXT_TRACE + trace(); +#endif + + ((flow_ucontext *) arg)->entry(); + + refuse("returning from a makecontext() entry (uc_link is NULL)"); +} + +int getcontext(ucontext_t *ucp) +{ + *slot(ucp) = NULL; + + return 0; +} + +void makecontext(ucontext_t *ucp, void (*func)(void), int argc, ...) +{ + if (argc != 0) { + refuse("makecontext() with arguments"); + } + + if (ucp->uc_link != NULL) { + refuse("makecontext() with uc_link"); + } + + char *bottom = (char *) ucp->uc_stack.ss_sp; + size_t size = ucp->uc_stack.ss_size; + + if (bottom == NULL || size < FLOW_UCONTEXT_MIN_STACK_SIZE) { + refuse("makecontext() without a uc_stack of at least 64 KiB"); + } + + /* the state and its Asyncify buffer take the top half of the stack, which grows down, and are freed with it */ + size_t asyncify_size = (size / 2 - sizeof(flow_ucontext) - 32) & ~(size_t) 15; + flow_ucontext *context = (flow_ucontext *) (((uintptr_t) bottom + size - sizeof(flow_ucontext) - asyncify_size - 32) & ~(uintptr_t) 15); + + context->entry = func; + context->asyncify_size = asyncify_size; + context->asyncify_stack = (char *) (((uintptr_t) (context + 1) + 15) & ~(uintptr_t) 15); + + emscripten_fiber_init(&context->fiber, enter, context, bottom, (size_t) ((char *) context - bottom), context->asyncify_stack, context->asyncify_size); + + *slot(ucp) = context; +} + +int swapcontext(ucontext_t *oucp, const ucontext_t *ucp) +{ + if (running == NULL) { + main_context.asyncify_size = sizeof(main_asyncify_stack); + main_context.asyncify_stack = main_asyncify_stack; + emscripten_fiber_init_from_current_context(&main_context.fiber, main_asyncify_stack, sizeof(main_asyncify_stack)); + running = &main_context; + } + + flow_ucontext *target = *slot((ucontext_t *) ucp); + + if (target == NULL) { + refuse("swapcontext() to a context makecontext() did not prepare"); + } + + flow_ucontext *previous = running; + + *slot(oucp) = previous; + running = target; + +#ifdef FLOW_UCONTEXT_TRACE + unwound = previous; +#endif + + emscripten_fiber_swap(&previous->fiber, &target->fiber); + +#ifdef FLOW_UCONTEXT_TRACE + trace(); +#endif + + return 0; +} diff --git a/web/landing/assets/codemirror/completions/dataframe.js b/web/landing/assets/codemirror/completions/dataframe.js index 2ebb22f397..b2af0b323e 100644 --- a/web/landing/assets/codemirror/completions/dataframe.js +++ b/web/landing/assets/codemirror/completions/dataframe.js @@ -1,7 +1,7 @@ /** * CodeMirror Completer for Flow PHP DataFrame Methods * - * DataFrame methods: 47 + * DataFrame methods: 46 * DataFrame-returning methods from classes: 3 * * This completer triggers after DataFrame-returning methods @@ -10,7 +10,7 @@ import { CompletionContext, snippet } from "@codemirror/autocomplete" // Map of DataFrame-returning methods grouped by class -const dataframeReturningMethods = {"flow":["extract","from","process","read"],"groupeddataframe":["aggregate"],"dataframe":["aggregate","batchBy","batchSize","cache","collect","collectRefs","constrain","crossJoin","drop","dropDuplicates","duplicateRow","filter","filters","join","joinEach","limit","load","match","offset","onError","repartition","rename","renameEach","rows","select","sortBy","transform","until","void","with","withEntries","withEntry","write"]}; +const dataframeReturningMethods = {"flow":["extract","from","process","read"],"dataframe":["aggregate","batchBy","batchSize","cache","collect","collectRefs","constrain","crossJoin","drop","dropDuplicates","duplicateRow","filter","filters","join","joinEach","limit","load","match","offset","onError","repartition","rename","renameEach","rows","select","sortBy","transform","until","void","with","withEntries","withEntry","write"],"groupeddataframe":["aggregate"]}; // DataFrame methods const dataframeMethods = [ @@ -253,7 +253,7 @@ const dataframeMethods = [ fetch(int $limit = null) : Rows
- Be aware that fetch is not memory safe and will load all rows into memory.
If you want to safely iterate over Rows use oe of the following methods:.
DataFrame::get() : \\Generator
DataFrame::getAsArray() : \\Generator
DataFrame::getEach() : \\Generator
DataFrame::getEachAsArray() : \\Generator
@trigger
@throws InvalidArgumentException + Be aware that fetch is not memory safe and will load all rows into memory.
If you want to safely iterate over Rows use oe of the following methods:.
DataFrame::get() : \\Generator
DataFrame::getAsArray() : \\Generator
DataFrame::getEachAsArray() : \\Generator
@trigger
@throws InvalidArgumentException
` return div @@ -350,24 +350,6 @@ const dataframeMethods = [ }, apply: snippet("getAsArray()"), boost: 10 - }, { - label: "getEach", - type: "method", - detail: "Flow\\\\ETL\\\\DataFrame", - info: () => { - const div = document.createElement("div") - div.innerHTML = ` -
- getEach() : Generator -
-
- Yield each row as an instance of Row.
@trigger
@return \\Generator -
- ` - return div - }, - apply: snippet("getEach()"), - boost: 10 }, { label: "getEachAsArray", type: "method", diff --git a/web/landing/assets/codemirror/completions/dsl.js b/web/landing/assets/codemirror/completions/dsl.js index 17a9645e32..19c7d7b703 100644 --- a/web/landing/assets/codemirror/completions/dsl.js +++ b/web/landing/assets/codemirror/completions/dsl.js @@ -1,7 +1,7 @@ /** * CodeMirror Completer for Flow PHP DSL Functions * - * Total functions: 814 + * Total functions: 812 * * This completer provides autocompletion for all Flow PHP DSL functions: * - Extractors (flow-extractors) @@ -692,24 +692,6 @@ const dslFunctions = [ }, apply: snippet("\\Flow\\ETL\\Adapter\\Parquet\\array_to_generator(" + "$" + "{" + "1:data" + "}" + ")"), boost: 10 - }, { - label: "array_to_row", - type: "function", - detail: "flow\u002Ddsl\u002Ddata\u002Dframe", - info: () => { - const div = document.createElement("div") - div.innerHTML = ` -
- array_to_row(array $data, Schema $schema, Hydrator $hydrator = Flow\\ETL\\Row\\AdaptiveRowHydrator::..., Partitions|array $partitions = []) : Row -
-
- @param array>|array $data
@param array|Partitions $partitions -
- ` - return div - }, - apply: snippet("\\Flow\\ETL\\DSL\\array_to_row(" + "$" + "{" + "1:data" + "}" + ", " + "$" + "{" + "2:schema" + "}" + ", " + "$" + "{" + "3:hydrator" + "}" + ", " + "$" + "{" + "4:partitions" + "}" + ")"), - boost: 10 }, { label: "array_to_rows", type: "function", @@ -718,7 +700,7 @@ const dslFunctions = [ const div = document.createElement("div") div.innerHTML = `
- array_to_rows(array $data, Schema $schema, Hydrator $hydrator = Flow\\ETL\\Row\\AdaptiveRowHydrator::...) : Rows + array_to_rows(array $data, Schema $schema, Backend $backend = Flow\\ETL\\Column\\AdaptiveBackend::...) : Rows
@param array>|array $data @@ -726,7 +708,7 @@ const dslFunctions = [ ` return div }, - apply: snippet("\\Flow\\ETL\\DSL\\array_to_rows(" + "$" + "{" + "1:data" + "}" + ", " + "$" + "{" + "2:schema" + "}" + ", " + "$" + "{" + "3:hydrator" + "}" + ")"), + apply: snippet("\\Flow\\ETL\\DSL\\array_to_rows(" + "$" + "{" + "1:data" + "}" + ", " + "$" + "{" + "2:schema" + "}" + ", " + "$" + "{" + "3:backend" + "}" + ")"), boost: 10 }, { label: "array_unpack", @@ -895,12 +877,12 @@ const dslFunctions = [ const div = document.createElement("div") div.innerHTML = `
- average(Reference|string $ref, int $scale = 2, Rounding $rounding = Flow\\Calculator\\Rounding::...) : Average + average(Reference|string $ref, int $scale = 2, Rounding $rounding = Flow\\Calculator\\Rounding::..., bool $exact = false) : Average
` return div }, - apply: snippet("\\Flow\\ETL\\DSL\\average(" + "$" + "{" + "1:ref" + "}" + ", " + "$" + "{" + "2:scale" + "}" + ", " + "$" + "{" + "3:rounding" + "}" + ")"), + apply: snippet("\\Flow\\ETL\\DSL\\average(" + "$" + "{" + "1:ref" + "}" + ", " + "$" + "{" + "2:scale" + "}" + ", " + "$" + "{" + "3:rounding" + "}" + ", " + "$" + "{" + "4:exact" + "}" + ")"), boost: 10 }, { label: "aws_s3_client", @@ -3799,12 +3781,12 @@ const dslFunctions = [ const div = document.createElement("div") div.innerHTML = `
- filesystem_cache(Path|string|null $cache_dir = null, Filesystem $filesystem = Flow\\Filesystem\\Local\\NativeLocalFilesystem::..., Serializer $serializer = Flow\\Floe\\FloeSerializer::...) : FilesystemCache + filesystem_cache(Serializer $serializer, Path|string|null $cache_dir = null, Filesystem $filesystem = Flow\\Filesystem\\Local\\NativeLocalFilesystem::...) : FilesystemCache
` return div }, - apply: snippet("\\Flow\\ETL\\DSL\\filesystem_cache(" + "$" + "{" + "1:cache_dir" + "}" + ", " + "$" + "{" + "2:filesystem" + "}" + ", " + "$" + "{" + "3:serializer" + "}" + ")"), + apply: snippet("\\Flow\\ETL\\DSL\\filesystem_cache(" + "$" + "{" + "1:serializer" + "}" + ", " + "$" + "{" + "2:cache_dir" + "}" + ", " + "$" + "{" + "3:filesystem" + "}" + ")"), boost: 10 }, { label: "filesystem_telemetry_config", @@ -4336,7 +4318,7 @@ const dslFunctions = [ const div = document.createElement("div") div.innerHTML = `
- from_floe(Path|string $path, Codec $codec = Flow\\Floe\\Codec\\NoopCodec::..., int $chunk_size = 65536, FloeEngine $engine = Flow\\Floe\\FloeEngine::..., Filesystem $filesystem = Flow\\Filesystem\\Local\\NativeLocalFilesystem::...) : FloeExtractor + from_floe(Path|string $path, Codec $codec = Flow\\Floe\\Codec\\NoopCodec::..., int $chunk_size = 65536, Filesystem $filesystem = Flow\\Filesystem\\Local\\NativeLocalFilesystem::...) : FloeExtractor
@param Path|string $path @@ -4344,7 +4326,7 @@ const dslFunctions = [ ` return div }, - apply: snippet("\\Flow\\Floe\\DSL\\from_floe(" + "$" + "{" + "1:path" + "}" + ", " + "$" + "{" + "2:codec" + "}" + ", " + "$" + "{" + "3:chunk_size" + "}" + ", " + "$" + "{" + "4:engine" + "}" + ", " + "$" + "{" + "5:filesystem" + "}" + ")"), + apply: snippet("\\Flow\\Floe\\DSL\\from_floe(" + "$" + "{" + "1:path" + "}" + ", " + "$" + "{" + "2:codec" + "}" + ", " + "$" + "{" + "3:chunk_size" + "}" + ", " + "$" + "{" + "4:filesystem" + "}" + ")"), boost: 10 }, { label: "from_google_sheet", @@ -4423,7 +4405,7 @@ const dslFunctions = [ const div = document.createElement("div") div.innerHTML = `
- from_json_lines(Path|string $path, Filesystem $filesystem = Flow\\Filesystem\\Local\\NativeLocalFilesystem::...) : JsonLinesExtractor + from_json_lines(Path|string $path, Filesystem $filesystem = Flow\\Filesystem\\Local\\NativeLocalFilesystem::...) : JsonExtractor
Used to read from a JSON lines https://jsonlines.org/ formatted file.
@param Path|string $path - string is internally turned into stream @@ -6415,7 +6397,7 @@ const dslFunctions = [ const div = document.createElement("div") div.innerHTML = `
- merge_floe(array $sources, Path|string $dest, bool $compact = false, Metadata $metadata = null, Filesystem $filesystem = Flow\\Filesystem\\Local\\NativeLocalFilesystem::...) : void + merge_floe(array $sources, Path|string $dest, Backend $backend, bool $compact = false, Metadata $metadata = null, Filesystem $filesystem = Flow\\Filesystem\\Local\\NativeLocalFilesystem::...) : void
Merges several Floe files (same or append-compatible evolving schema) into one. Byte-splices frame
regions by default (O(bytes), no re-encode); compact re-encodes all rows into fewer sections.
@param array $sources @@ -6423,7 +6405,7 @@ const dslFunctions = [ ` return div }, - apply: snippet("\\Flow\\Floe\\DSL\\merge_floe(" + "$" + "{" + "1:sources" + "}" + ", " + "$" + "{" + "2:dest" + "}" + ", " + "$" + "{" + "3:compact" + "}" + ", " + "$" + "{" + "4:metadata" + "}" + ", " + "$" + "{" + "5:filesystem" + "}" + ")"), + apply: snippet("\\Flow\\Floe\\DSL\\merge_floe(" + "$" + "{" + "1:sources" + "}" + ", " + "$" + "{" + "2:dest" + "}" + ", " + "$" + "{" + "3:backend" + "}" + ", " + "$" + "{" + "4:compact" + "}" + ", " + "$" + "{" + "5:metadata" + "}" + ", " + "$" + "{" + "6:filesystem" + "}" + ")"), boost: 10 }, { label: "meter_provider", @@ -8513,24 +8495,6 @@ const dslFunctions = [ }, apply: snippet("\\Flow\\ETL\\DSL\\round(" + "$" + "{" + "1:value" + "}" + ", " + "$" + "{" + "2:precision" + "}" + ", " + "$" + "{" + "3:mode" + "}" + ")"), boost: 10 - }, { - label: "row", - type: "function", - detail: "flow\u002Ddsl\u002Ddata\u002Dframe", - info: () => { - const div = document.createElement("div") - div.innerHTML = ` -
- row(array $values) : Row -
-
- @param array $values -
- ` - return div - }, - apply: snippet("\\Flow\\ETL\\DSL\\row(" + "$" + "{" + "1:values" + "}" + ")"), - boost: 10 }, { label: "rows", type: "function", @@ -8539,12 +8503,12 @@ const dslFunctions = [ const div = document.createElement("div") div.innerHTML = `
- rows(Schema $schema, Row $row) : Rows + rows(Schema $schema, Backend $backend = Flow\\ETL\\Column\\AdaptiveBackend::...) : Rows
` return div }, - apply: snippet("\\Flow\\ETL\\DSL\\rows(" + "$" + "{" + "1:schema" + "}" + ", " + "$" + "{" + "2:row" + "}" + ")"), + apply: snippet("\\Flow\\ETL\\DSL\\rows(" + "$" + "{" + "1:schema" + "}" + ", " + "$" + "{" + "2:backend" + "}" + ")"), boost: 10 }, { label: "row_expr", @@ -10756,7 +10720,7 @@ const dslFunctions = [ const div = document.createElement("div") div.innerHTML = `
- sum(Reference|string $ref, ScalarFunction|bool $exact = false) : Sum + sum(Reference|string $ref, bool $exact = false) : Sum
` return div @@ -11257,7 +11221,7 @@ const dslFunctions = [ const div = document.createElement("div") div.innerHTML = `
- to_floe(Path|string $path, Metadata $metadata = null, Options $options = Flow\\Floe\\Options::..., FloeEngine $engine = Flow\\Floe\\FloeEngine::..., Filesystem $filesystem = Flow\\Filesystem\\Local\\NativeLocalFilesystem::...) : FloeLoader + to_floe(Path|string $path, Metadata $metadata = null, Options $options = Flow\\Floe\\Options::..., Filesystem $filesystem = Flow\\Filesystem\\Local\\NativeLocalFilesystem::...) : FloeLoader
@param Path|string $path @@ -11265,7 +11229,7 @@ const dslFunctions = [ ` return div }, - apply: snippet("\\Flow\\Floe\\DSL\\to_floe(" + "$" + "{" + "1:path" + "}" + ", " + "$" + "{" + "2:metadata" + "}" + ", " + "$" + "{" + "3:options" + "}" + ", " + "$" + "{" + "4:engine" + "}" + ", " + "$" + "{" + "5:filesystem" + "}" + ")"), + apply: snippet("\\Flow\\Floe\\DSL\\to_floe(" + "$" + "{" + "1:path" + "}" + ", " + "$" + "{" + "2:metadata" + "}" + ", " + "$" + "{" + "3:options" + "}" + ", " + "$" + "{" + "4:filesystem" + "}" + ")"), boost: 10 }, { label: "to_json", @@ -11293,10 +11257,10 @@ const dslFunctions = [ const div = document.createElement("div") div.innerHTML = `
- to_json_lines(Path|string $path, Filesystem $filesystem = Flow\\Filesystem\\Local\\NativeLocalFilesystem::...) : JsonLinesLoader + to_json_lines(Path|string $path, Filesystem $filesystem = Flow\\Filesystem\\Local\\NativeLocalFilesystem::...) : JsonLoader
- Used to write to a JSON lines https://jsonlines.org/ formatted file.
@param Path|string $path
@return JsonLinesLoader + Used to write to a JSON lines https://jsonlines.org/ formatted file.
@param Path|string $path
@return JsonLoader
` return div diff --git a/web/landing/assets/codemirror/completions/scalarfunctionchain.js b/web/landing/assets/codemirror/completions/scalarfunctionchain.js index 53f109219a..745e7b229d 100644 --- a/web/landing/assets/codemirror/completions/scalarfunctionchain.js +++ b/web/landing/assets/codemirror/completions/scalarfunctionchain.js @@ -1133,12 +1133,12 @@ const scalarFunctionChainMethods = [ const div = document.createElement("div") div.innerHTML = `
- minus(ScalarFunction|int|float $ref) : Minus + minus(ScalarFunction|int|float $ref, bool $exact = false) : Minus
` return div }, - apply: snippet("minus(" + "$" + "{" + "1:ref" + "}" + ")"), + apply: snippet("minus(" + "$" + "{" + "1:ref" + "}" + ", " + "$" + "{" + "2:exact" + "}" + ")"), boost: 10 }, { label: "mod", @@ -1178,12 +1178,12 @@ const scalarFunctionChainMethods = [ const div = document.createElement("div") div.innerHTML = `
- multiply(ScalarFunction|int|float $value) : Multiply + multiply(ScalarFunction|int|float $value, bool $exact = false) : Multiply
` return div }, - apply: snippet("multiply(" + "$" + "{" + "1:value" + "}" + ")"), + apply: snippet("multiply(" + "$" + "{" + "1:value" + "}" + ", " + "$" + "{" + "2:exact" + "}" + ")"), boost: 10 }, { label: "notEquals", @@ -1286,12 +1286,12 @@ const scalarFunctionChainMethods = [ const div = document.createElement("div") div.innerHTML = `
- plus(ScalarFunction|int|float $ref) : Plus + plus(ScalarFunction|int|float $ref, bool $exact = false) : Plus
` return div }, - apply: snippet("plus(" + "$" + "{" + "1:ref" + "}" + ")"), + apply: snippet("plus(" + "$" + "{" + "1:ref" + "}" + ", " + "$" + "{" + "2:exact" + "}" + ")"), boost: 10 }, { label: "power", @@ -1301,12 +1301,12 @@ const scalarFunctionChainMethods = [ const div = document.createElement("div") div.innerHTML = `
- power(ScalarFunction|int $value) : Power + power(ScalarFunction|int $value, bool $exact = false) : Power
` return div }, - apply: snippet("power(" + "$" + "{" + "1:value" + "}" + ")"), + apply: snippet("power(" + "$" + "{" + "1:value" + "}" + ", " + "$" + "{" + "2:exact" + "}" + ")"), boost: 10 }, { label: "prepend", diff --git a/web/landing/assets/wasm/data/orders.floe b/web/landing/assets/wasm/data/orders.floe index 3136f68f7a..85b16c9bbd 100644 Binary files a/web/landing/assets/wasm/data/orders.floe and b/web/landing/assets/wasm/data/orders.floe differ diff --git a/web/landing/assets/wasm/data/orders.parquet b/web/landing/assets/wasm/data/orders.parquet index cbd326fd8e..6a677bbed5 100644 Binary files a/web/landing/assets/wasm/data/orders.parquet and b/web/landing/assets/wasm/data/orders.parquet differ diff --git a/web/landing/assets/wasm/php.js b/web/landing/assets/wasm/php.js index 4a34d89019..cf6695ad36 100644 --- a/web/landing/assets/wasm/php.js +++ b/web/landing/assets/wasm/php.js @@ -1,2 +1,2 @@ -var PHP=(()=>{var _scriptName=globalThis.document?.currentScript?.src;return async function(moduleArg={}){var moduleRtn;var Module=moduleArg;var ENVIRONMENT_IS_WEB=true;var ENVIRONMENT_IS_WORKER=false;var ENVIRONMENT_IS_NODE=false;var arguments_=[];var thisProgram="./this.program";var quit_=(status,toThrow)=>{throw toThrow};var scriptDirectory="";function locateFile(path){if(Module["locateFile"]){return Module["locateFile"](path,scriptDirectory)}return scriptDirectory+path}var readAsync,readBinary;if(ENVIRONMENT_IS_WEB||ENVIRONMENT_IS_WORKER){try{scriptDirectory=new URL(".",_scriptName).href}catch{}{readAsync=async url=>{var response=await fetch(url,{credentials:"same-origin"});if(response.ok){return response.arrayBuffer()}throw new Error(response.status+" : "+response.url)}}}else{}var out=console.log.bind(console);var err=console.error.bind(console);var wasmBinary;var ABORT=false;var EXITSTATUS;class EmscriptenEH{}class EmscriptenSjLj extends EmscriptenEH{}var readyPromiseResolve,readyPromiseReject;var runtimeInitialized=false;function updateMemoryViews(){var b=wasmMemory.buffer;HEAP8=new Int8Array(b);HEAP16=new Int16Array(b);HEAPU8=new Uint8Array(b);HEAPU16=new Uint16Array(b);HEAP32=new Int32Array(b);HEAPU32=new Uint32Array(b);HEAPF32=new Float32Array(b);HEAPF64=new Float64Array(b);HEAP64=new BigInt64Array(b);HEAPU64=new BigUint64Array(b)}function preRun(){if(Module["preRun"]){if(typeof Module["preRun"]=="function")Module["preRun"]=[Module["preRun"]];while(Module["preRun"].length){addOnPreRun(Module["preRun"].shift())}}callRuntimeCallbacks(onPreRuns)}function initRuntime(){runtimeInitialized=true;SOCKFS.root=FS.mount(SOCKFS,{},null);if(!Module["noFSInit"]&&!FS.initialized)FS.init();TTY.init();PIPEFS.root=FS.mount(PIPEFS,{},null);wasmExports["Lb"]();FS.ignorePermissions=false}function postRun(){if(Module["postRun"]){if(typeof Module["postRun"]=="function")Module["postRun"]=[Module["postRun"]];while(Module["postRun"].length){addOnPostRun(Module["postRun"].shift())}}callRuntimeCallbacks(onPostRuns)}function abort(what){Module["onAbort"]?.(what);what=`Aborted(${what})`;err(what);ABORT=true;what+=". Build with -sASSERTIONS for more info.";var e=new WebAssembly.RuntimeError(what);readyPromiseReject?.(e);throw e}var wasmBinaryFile;function findWasmBinary(){return locateFile("php.wasm")}function getBinarySync(file){if(file==wasmBinaryFile&&wasmBinary){return new Uint8Array(wasmBinary)}if(readBinary){return readBinary(file)}throw"both async and sync fetching of the wasm failed"}async function getWasmBinary(binaryFile){if(!wasmBinary){try{var response=await readAsync(binaryFile);return new Uint8Array(response)}catch{}}return getBinarySync(binaryFile)}async function instantiateArrayBuffer(binaryFile,imports){try{var binary=await getWasmBinary(binaryFile);var instance=await WebAssembly.instantiate(binary,imports);return instance}catch(reason){err(`failed to asynchronously prepare wasm: ${reason}`);abort(reason)}}async function instantiateAsync(binary,binaryFile,imports){if(!binary){try{var response=fetch(binaryFile,{credentials:"same-origin"});var instantiationResult=await WebAssembly.instantiateStreaming(response,imports);return instantiationResult}catch(reason){err(`wasm streaming compile failed: ${reason}`);err("falling back to ArrayBuffer instantiation")}}return instantiateArrayBuffer(binaryFile,imports)}function getWasmImports(){var imports={a:wasmImports};return imports}async function createWasm(){function receiveInstance(instance,module){wasmExports=instance.exports;wasmExports=Asyncify.instrumentWasmExports(wasmExports);wasmExports=applySignatureConversions(wasmExports);assignWasmExports(wasmExports);updateMemoryViews();return wasmExports}function receiveInstantiationResult(result){return receiveInstance(result["instance"])}var info=getWasmImports();if(Module["instantiateWasm"]){return new Promise((resolve,reject)=>{Module["instantiateWasm"](info,(inst,mod)=>{resolve(receiveInstance(inst,mod))})})}wasmBinaryFile??=findWasmBinary();var result=await instantiateAsync(wasmBinary,wasmBinaryFile,info);var exports=receiveInstantiationResult(result);return exports}class ExitStatus{name="ExitStatus";constructor(status){this.message=`Program terminated with exit(${status})`;this.status=status}}var HEAP16;var HEAP32;var HEAP64;var HEAP8;var HEAPF32;var HEAPF64;var HEAPU16;var HEAPU32;var HEAPU64;var HEAPU8;var callRuntimeCallbacks=callbacks=>{while(callbacks.length>0){callbacks.shift()(Module)}};var onPostRuns=[];var addOnPostRun=cb=>onPostRuns.push(cb);var onPreRuns=[];var addOnPreRun=cb=>onPreRuns.push(cb);var dynCalls={};function getValue(ptr,type="i8"){if(type.endsWith("*"))type="*";switch(type){case"i1":return HEAP8[ptr];case"i8":return HEAP8[ptr];case"i16":return HEAP16[ptr>>1];case"i32":return HEAP32[ptr>>2];case"i64":return HEAP64[ptr>>3];case"float":return HEAPF32[ptr>>2];case"double":return HEAPF64[ptr>>3];case"*":return Number(HEAPU64[ptr>>3]);default:abort(`invalid type for getValue: ${type}`)}}var noExitRuntime=true;function setValue(ptr,value,type="i8"){if(type.endsWith("*"))type="*";switch(type){case"i1":HEAP8[ptr]=value;break;case"i8":HEAP8[ptr]=value;break;case"i16":HEAP16[ptr>>1]=value;break;case"i32":HEAP32[ptr>>2]=value;break;case"i64":HEAP64[ptr>>3]=BigInt(value);break;case"float":HEAPF32[ptr>>2]=value;break;case"double":HEAPF64[ptr>>3]=value;break;case"*":HEAPU64[ptr>>3]=BigInt(value);break;default:abort(`invalid type for setValue: ${type}`)}}var stackRestore=val=>__emscripten_stack_restore(val);var stackSave=()=>_emscripten_stack_get_current();var INT53_MAX=9007199254740992;var INT53_MIN=-9007199254740992;var bigintToI53Checked=num=>numINT53_MAX?NaN:Number(num);var UTF8Decoder=new TextDecoder;var findStringEnd=(heapOrArray,idx,maxBytesToRead,ignoreNul)=>{var maxIdx=idx+maxBytesToRead;if(ignoreNul)return maxIdx;while(heapOrArray[idx]&&!(idx>=maxIdx))++idx;return idx};var UTF8ToString=(ptr,maxBytesToRead,ignoreNul)=>{if(!ptr)return"";var end=findStringEnd(HEAPU8,ptr,maxBytesToRead,ignoreNul);return UTF8Decoder.decode(HEAPU8.subarray(ptr,end))};function ___assert_fail(condition,filename,line,func){condition=bigintToI53Checked(condition);filename=bigintToI53Checked(filename);func=bigintToI53Checked(func);return abort(`Assertion failed: ${UTF8ToString(condition)}, at: `+[filename?UTF8ToString(filename):"unknown filename",line,func?UTF8ToString(func):"unknown function"])}var ___call_sighandler=function(fp,sig){fp=bigintToI53Checked(fp);return(a1=>dynCall_vi(fp,a1))(sig)};var initRandomFill=()=>view=>(crypto.getRandomValues(view),0);var randomFill=view=>(randomFill=initRandomFill())(view);var PATH={isAbs:path=>path.charAt(0)==="/",splitPath:filename=>{var splitPathRe=/^(\/?|)([\s\S]*?)((?:\.{1,2}|[^\/]+?|)(\.[^.\/]*|))(?:[\/]*)$/;return splitPathRe.exec(filename).slice(1)},normalizeArray:(parts,allowAboveRoot)=>{var up=0;for(var i=parts.length-1;i>=0;i--){var last=parts[i];if(last==="."){parts.splice(i,1)}else if(last===".."){parts.splice(i,1);up++}else if(up){parts.splice(i,1);up--}}if(allowAboveRoot){for(;up;up--){parts.unshift("..")}}return parts},normalize:path=>{var isAbsolute=PATH.isAbs(path),trailingSlash=path.slice(-1)==="/";path=PATH.normalizeArray(path.split("/").filter(p=>!!p),!isAbsolute).join("/");if(!path&&!isAbsolute){path="."}if(path&&trailingSlash){path+="/"}return(isAbsolute?"/":"")+path},dirname:path=>{var result=PATH.splitPath(path),root=result[0],dir=result[1];if(!root&&!dir){return"."}if(dir){dir=dir.slice(0,-1)}return root+dir},basename:path=>path&&path.match(/([^\/]+|\/)\/*$/)[1],join:(...paths)=>PATH.normalize(paths.join("/")),join2:(l,r)=>PATH.normalize(l+"/"+r)};var PATH_FS={resolve:(...args)=>{var resolvedPath="",resolvedAbsolute=false;for(var i=args.length-1;i>=-1&&!resolvedAbsolute;i--){var path=i>=0?args[i]:FS.cwd();if(typeof path!="string"){throw new TypeError("Arguments to path.resolve must be strings")}else if(!path){return""}resolvedPath=path+"/"+resolvedPath;resolvedAbsolute=PATH.isAbs(path)}resolvedPath=PATH.normalizeArray(resolvedPath.split("/").filter(p=>!!p),!resolvedAbsolute).join("/");return(resolvedAbsolute?"/":"")+resolvedPath||"."},relative:(from,to)=>{from=PATH_FS.resolve(from).slice(1);to=PATH_FS.resolve(to).slice(1);function trim(arr){var start=0;for(;start=0;end--){if(arr[end]!=="")break}if(start>end)return[];return arr.slice(start,end-start+1)}var fromParts=trim(from.split("/"));var toParts=trim(to.split("/"));var length=Math.min(fromParts.length,toParts.length);var samePartsLength=length;for(var i=0;i{var endPtr=findStringEnd(heapOrArray,idx,maxBytesToRead,ignoreNul);return UTF8Decoder.decode(heapOrArray.buffer?heapOrArray.subarray(idx,endPtr):new Uint8Array(heapOrArray.slice(idx,endPtr)))};var FS_stdin_getChar_buffer=[];var lengthBytesUTF8=str=>{var len=0;for(var i=0;i=55296&&c<=57343){len+=4;++i}else{len+=3}}return len};var stringToUTF8Array=(str,heap,outIdx,maxBytesToWrite)=>{if(!(maxBytesToWrite>0))return 0;var startIdx=outIdx;var endIdx=outIdx+maxBytesToWrite-1;for(var i=0;i=endIdx)break;heap[outIdx++]=u}else if(u<=2047){if(outIdx+1>=endIdx)break;heap[outIdx++]=192|u>>6;heap[outIdx++]=128|u&63}else if(u<=65535){if(outIdx+2>=endIdx)break;heap[outIdx++]=224|u>>12;heap[outIdx++]=128|u>>6&63;heap[outIdx++]=128|u&63}else{if(outIdx+3>=endIdx)break;heap[outIdx++]=240|u>>18;heap[outIdx++]=128|u>>12&63;heap[outIdx++]=128|u>>6&63;heap[outIdx++]=128|u&63;i++}}heap[outIdx]=0;return outIdx-startIdx};var intArrayFromString=(stringy,dontAddNull,length)=>{var len=length>0?length:lengthBytesUTF8(stringy)+1;var u8array=new Array(len);var numBytesWritten=stringToUTF8Array(stringy,u8array,0,u8array.length);if(dontAddNull)u8array.length=numBytesWritten;return u8array};var FS_stdin_getChar=()=>{if(!FS_stdin_getChar_buffer.length){var result=null;if(globalThis.window?.prompt){result=window.prompt("Input: ");if(result!==null){result+="\n"}}else{}if(!result){return null}FS_stdin_getChar_buffer=intArrayFromString(result,true)}return FS_stdin_getChar_buffer.shift()};var TTY={ttys:[],init(){},shutdown(){},register(dev,ops){TTY.ttys[dev]={input:[],output:[],ops};FS.registerDevice(dev,TTY.stream_ops)},stream_ops:{open(stream){var tty=TTY.ttys[stream.node.rdev];if(!tty){throw new FS.ErrnoError(43)}stream.tty=tty;stream.seekable=false},close(stream){stream.tty.ops.fsync(stream.tty)},fsync(stream){stream.tty.ops.fsync(stream.tty)},read(stream,buffer,offset,length,pos){if(!stream.tty||!stream.tty.ops.get_char){throw new FS.ErrnoError(60)}var bytesRead=0;for(var i=0;i0){out(UTF8ArrayToString(tty.output));tty.output=[]}},ioctl_tcgets(tty){return{c_iflag:25856,c_oflag:5,c_cflag:191,c_lflag:35387,c_cc:[3,28,127,21,4,0,1,0,17,19,26,0,18,15,23,22,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]}},ioctl_tcsets(tty,optional_actions,data){return 0},ioctl_tiocgwinsz(tty){return[24,80]}},default_tty1_ops:{put_char(tty,val){if(val===null||val===10){err(UTF8ArrayToString(tty.output));tty.output=[]}else{if(val!=0)tty.output.push(val)}},fsync(tty){if(tty.output?.length>0){err(UTF8ArrayToString(tty.output));tty.output=[]}}}};var zeroMemory=(ptr,size)=>HEAPU8.fill(0,ptr,ptr+size);var alignMemory=(size,alignment)=>Math.ceil(size/alignment)*alignment;var mmapAlloc=size=>{size=alignMemory(size,65536);var ptr=_emscripten_builtin_memalign(65536,size);if(ptr)zeroMemory(ptr,size);return ptr};var MEMFS={ops_table:null,mount(mount){return MEMFS.createNode(null,"/",16895,0)},createNode(parent,name,mode,dev){if(FS.isBlkdev(mode)||FS.isFIFO(mode)){throw new FS.ErrnoError(63)}MEMFS.ops_table||={dir:{node:{getattr:MEMFS.node_ops.getattr,setattr:MEMFS.node_ops.setattr,lookup:MEMFS.node_ops.lookup,mknod:MEMFS.node_ops.mknod,rename:MEMFS.node_ops.rename,unlink:MEMFS.node_ops.unlink,rmdir:MEMFS.node_ops.rmdir,readdir:MEMFS.node_ops.readdir,symlink:MEMFS.node_ops.symlink},stream:{llseek:MEMFS.stream_ops.llseek}},file:{node:{getattr:MEMFS.node_ops.getattr,setattr:MEMFS.node_ops.setattr},stream:{llseek:MEMFS.stream_ops.llseek,read:MEMFS.stream_ops.read,write:MEMFS.stream_ops.write,mmap:MEMFS.stream_ops.mmap,msync:MEMFS.stream_ops.msync}},link:{node:{getattr:MEMFS.node_ops.getattr,setattr:MEMFS.node_ops.setattr,readlink:MEMFS.node_ops.readlink},stream:{}},chrdev:{node:{getattr:MEMFS.node_ops.getattr,setattr:MEMFS.node_ops.setattr},stream:FS.chrdev_stream_ops}};var node=FS.createNode(parent,name,mode,dev);if(FS.isDir(node.mode)){node.node_ops=MEMFS.ops_table.dir.node;node.stream_ops=MEMFS.ops_table.dir.stream;node.contents={}}else if(FS.isFile(node.mode)){node.node_ops=MEMFS.ops_table.file.node;node.stream_ops=MEMFS.ops_table.file.stream;node.usedBytes=0;node.contents=MEMFS.emptyFileContents??=new Uint8Array(0)}else if(FS.isLink(node.mode)){node.node_ops=MEMFS.ops_table.link.node;node.stream_ops=MEMFS.ops_table.link.stream}else if(FS.isChrdev(node.mode)){node.node_ops=MEMFS.ops_table.chrdev.node;node.stream_ops=MEMFS.ops_table.chrdev.stream}node.atime=node.mtime=node.ctime=Date.now();if(parent){parent.contents[name]=node;parent.atime=parent.mtime=parent.ctime=node.atime}return node},getFileDataAsTypedArray(node){return node.contents.subarray(0,node.usedBytes)},expandFileStorage(node,newCapacity){var prevCapacity=node.contents.length;if(prevCapacity>=newCapacity)return;var CAPACITY_DOUBLING_MAX=1024*1024;newCapacity=Math.max(newCapacity,prevCapacity*(prevCapacity>>0);if(prevCapacity)newCapacity=Math.max(newCapacity,256);var oldContents=MEMFS.getFileDataAsTypedArray(node);node.contents=new Uint8Array(newCapacity);node.contents.set(oldContents)},resizeFileStorage(node,newSize){if(node.usedBytes==newSize)return;var oldContents=node.contents;node.contents=new Uint8Array(newSize);node.contents.set(oldContents.subarray(0,Math.min(newSize,node.usedBytes)));node.usedBytes=newSize},node_ops:{getattr(node){var attr={};attr.dev=FS.isChrdev(node.mode)?node.id:1;attr.ino=node.id;attr.mode=node.mode;attr.nlink=1;attr.uid=0;attr.gid=0;attr.rdev=node.rdev;if(FS.isDir(node.mode)){attr.size=4096}else if(FS.isFile(node.mode)){attr.size=node.usedBytes}else if(FS.isLink(node.mode)){attr.size=node.link.length}else{attr.size=0}attr.atime=new Date(node.atime);attr.mtime=new Date(node.mtime);attr.ctime=new Date(node.ctime);attr.blksize=4096;attr.blocks=Math.ceil(attr.size/attr.blksize);return attr},setattr(node,attr){for(const key of["mode","atime","mtime","ctime"]){if(attr[key]!=null){node[key]=attr[key]}}if(attr.size!==undefined){MEMFS.resizeFileStorage(node,attr.size)}},lookup(parent,name){if(!MEMFS.doesNotExistError){MEMFS.doesNotExistError=new FS.ErrnoError(44);MEMFS.doesNotExistError.stack=""}throw MEMFS.doesNotExistError},mknod(parent,name,mode,dev){return MEMFS.createNode(parent,name,mode,dev)},rename(old_node,new_dir,new_name){var new_node;try{new_node=FS.lookupNode(new_dir,new_name)}catch(e){}if(new_node){if(FS.isDir(old_node.mode)){for(var i in new_node.contents){throw new FS.ErrnoError(55)}}FS.hashRemoveNode(new_node)}delete old_node.parent.contents[old_node.name];new_dir.contents[new_name]=old_node;old_node.name=new_name;new_dir.ctime=new_dir.mtime=old_node.parent.ctime=old_node.parent.mtime=Date.now()},unlink(parent,name){delete parent.contents[name];parent.ctime=parent.mtime=Date.now()},rmdir(parent,name){var node=FS.lookupNode(parent,name);for(var i in node.contents){throw new FS.ErrnoError(55)}delete parent.contents[name];parent.ctime=parent.mtime=Date.now()},readdir(node){return[".","..",...Object.keys(node.contents)]},symlink(parent,newname,oldpath){var node=MEMFS.createNode(parent,newname,511|40960,0);node.link=oldpath;return node},readlink(node){if(!FS.isLink(node.mode)){throw new FS.ErrnoError(28)}return node.link}},stream_ops:{read(stream,buffer,offset,length,position){var contents=stream.node.contents;if(position>=stream.node.usedBytes)return 0;var size=Math.min(stream.node.usedBytes-position,length);buffer.set(contents.subarray(position,position+size),offset);return size},write(stream,buffer,offset,length,position,canOwn){if(buffer.buffer===HEAP8.buffer){canOwn=false}if(!length)return 0;var node=stream.node;node.mtime=node.ctime=Date.now();if(canOwn){node.contents=buffer.subarray(offset,offset+length);node.usedBytes=length}else if(node.usedBytes===0&&position===0){node.contents=buffer.slice(offset,offset+length);node.usedBytes=length}else{MEMFS.expandFileStorage(node,position+length);node.contents.set(buffer.subarray(offset,offset+length),position);node.usedBytes=Math.max(node.usedBytes,position+length)}return length},llseek(stream,offset,whence){var position=offset;if(whence===1){position+=stream.position}else if(whence===2){if(FS.isFile(stream.node.mode)){position+=stream.node.usedBytes}}if(position<0){throw new FS.ErrnoError(28)}return position},mmap(stream,length,position,prot,flags){if(!FS.isFile(stream.node.mode)){throw new FS.ErrnoError(43)}var ptr;var allocated;var contents=stream.node.contents;if(!(flags&2)&&contents.buffer===HEAP8.buffer){allocated=false;ptr=contents.byteOffset}else{allocated=true;ptr=mmapAlloc(length);if(!ptr){throw new FS.ErrnoError(48)}if(contents){if(position>0||position+length{if(typeof str!="string")return str;var flagModes={r:0,"r+":2,w:512|64|1,"w+":512|64|2,a:1024|64|1,"a+":1024|64|2};var flags=flagModes[str];if(typeof flags=="undefined"){throw new Error(`Unknown file open mode: ${str}`)}return flags};var FS_fileDataToTypedArray=data=>{if(typeof data=="string"){data=intArrayFromString(data,true)}if(!data.subarray){data=new Uint8Array(data)}return data};var FS_getMode=(canRead,canWrite)=>{var mode=0;if(canRead)mode|=292|73;if(canWrite)mode|=146;return mode};var asyncLoad=async url=>{var arrayBuffer=await readAsync(url);return new Uint8Array(arrayBuffer)};var FS_createDataFile=(...args)=>FS.createDataFile(...args);var getUniqueRunDependency=id=>id;var runDependencies=0;var dependenciesFulfilled=null;var removeRunDependency=id=>{runDependencies--;Module["monitorRunDependencies"]?.(runDependencies);if(runDependencies==0){if(dependenciesFulfilled){var callback=dependenciesFulfilled;dependenciesFulfilled=null;callback()}}};var addRunDependency=id=>{runDependencies++;Module["monitorRunDependencies"]?.(runDependencies)};var preloadPlugins=[];var FS_handledByPreloadPlugin=async(byteArray,fullname)=>{if(typeof Browser!="undefined")Browser.init();for(var plugin of preloadPlugins){if(plugin["canHandle"](fullname)){return plugin["handle"](byteArray,fullname)}}return byteArray};var FS_preloadFile=async(parent,name,url,canRead,canWrite,dontCreateFile,canOwn,preFinish)=>{var fullname=name?PATH_FS.resolve(PATH.join2(parent,name)):parent;var dep=getUniqueRunDependency(`cp ${fullname}`);addRunDependency(dep);try{var byteArray=url;if(typeof url=="string"){byteArray=await asyncLoad(url)}byteArray=await FS_handledByPreloadPlugin(byteArray,fullname);preFinish?.();if(!dontCreateFile){FS_createDataFile(parent,name,byteArray,canRead,canWrite,canOwn)}}finally{removeRunDependency(dep)}};var FS_createPreloadedFile=(parent,name,url,canRead,canWrite,onload,onerror,dontCreateFile,canOwn,preFinish)=>{FS_preloadFile(parent,name,url,canRead,canWrite,dontCreateFile,canOwn,preFinish).then(onload).catch(onerror)};var FS={root:null,mounts:[],devices:{},streams:[],nextInode:1,nameTable:null,currentPath:"/",initialized:false,ignorePermissions:true,filesystems:null,syncFSRequests:0,ErrnoError:class{name="ErrnoError";constructor(errno){this.errno=errno}},FSStream:class{shared={};get object(){return this.node}set object(val){this.node=val}get isRead(){return(this.flags&2097155)!==1}get isWrite(){return(this.flags&2097155)!==0}get isAppend(){return this.flags&1024}get flags(){return this.shared.flags}set flags(val){this.shared.flags=val}get position(){return this.shared.position}set position(val){this.shared.position=val}},FSNode:class{node_ops={};stream_ops={};readMode=292|73;writeMode=146;mounted=null;constructor(parent,name,mode,rdev){if(!parent){parent=this}this.parent=parent;this.mount=parent.mount;this.id=FS.nextInode++;this.name=name;this.mode=mode;this.rdev=rdev;this.atime=this.mtime=this.ctime=Date.now()}get read(){return(this.mode&this.readMode)===this.readMode}set read(val){val?this.mode|=this.readMode:this.mode&=~this.readMode}get write(){return(this.mode&this.writeMode)===this.writeMode}set write(val){val?this.mode|=this.writeMode:this.mode&=~this.writeMode}get isFolder(){return FS.isDir(this.mode)}get isDevice(){return FS.isChrdev(this.mode)}},lookupPath(path,opts={}){if(!path){throw new FS.ErrnoError(44)}opts.follow_mount??=true;if(!PATH.isAbs(path)){path=FS.cwd()+"/"+path}linkloop:for(var nlinks=0;nlinks<40;nlinks++){var parts=path.split("/").filter(p=>!!p);var current=FS.root;var current_path="/";for(var i=0;i>>0)%FS.nameTable.length},hashAddNode(node){var hash=FS.hashName(node.parent.id,node.name);node.name_next=FS.nameTable[hash];FS.nameTable[hash]=node},hashRemoveNode(node){var hash=FS.hashName(node.parent.id,node.name);if(FS.nameTable[hash]===node){FS.nameTable[hash]=node.name_next}else{var current=FS.nameTable[hash];while(current){if(current.name_next===node){current.name_next=node.name_next;break}current=current.name_next}}},lookupNode(parent,name){var errCode=FS.mayLookup(parent);if(errCode){throw new FS.ErrnoError(errCode)}var hash=FS.hashName(parent.id,name);for(var node=FS.nameTable[hash];node;node=node.name_next){var nodeName=node.name;if(node.parent.id===parent.id&&nodeName===name){return node}}return FS.lookup(parent,name)},createNode(parent,name,mode,rdev){var node=new FS.FSNode(parent,name,mode,rdev);FS.hashAddNode(node);return node},destroyNode(node){FS.hashRemoveNode(node)},isRoot(node){return node===node.parent},isMountpoint(node){return!!node.mounted},isFile(mode){return(mode&61440)===32768},isDir(mode){return(mode&61440)===16384},isLink(mode){return(mode&61440)===40960},isChrdev(mode){return(mode&61440)===8192},isBlkdev(mode){return(mode&61440)===24576},isFIFO(mode){return(mode&61440)===4096},isSocket(mode){return(mode&49152)===49152},flagsToPermissionString(flag){var perms=["r","w","rw"][flag&3];if(flag&512){perms+="w"}return perms},nodePermissions(node,perms){if(FS.ignorePermissions){return 0}if(perms.includes("r")&&!(node.mode&292)){return 2}if(perms.includes("w")&&!(node.mode&146)){return 2}if(perms.includes("x")&&!(node.mode&73)){return 2}return 0},mayLookup(dir){if(!FS.isDir(dir.mode))return 54;var errCode=FS.nodePermissions(dir,"x");if(errCode)return errCode;if(!dir.node_ops.lookup)return 2;return 0},mayCreate(dir,name){if(!FS.isDir(dir.mode)){return 54}try{var node=FS.lookupNode(dir,name);return 20}catch(e){}return FS.nodePermissions(dir,"wx")},mayDelete(dir,name,isdir){var node;try{node=FS.lookupNode(dir,name)}catch(e){return e.errno}var errCode=FS.nodePermissions(dir,"wx");if(errCode){return errCode}if(isdir){if(!FS.isDir(node.mode)){return 54}if(FS.isRoot(node)||FS.getPath(node)===FS.cwd()){return 10}}else if(FS.isDir(node.mode)){return 31}return 0},mayOpen(node,flags){if(!node){return 44}if(FS.isLink(node.mode)){return 32}var mode=FS.flagsToPermissionString(flags);if(FS.isDir(node.mode)){if(mode!=="r"||flags&(512|64)){return 31}}return FS.nodePermissions(node,mode)},checkOpExists(op,err){if(!op){throw new FS.ErrnoError(err)}return op},MAX_OPEN_FDS:4096,nextfd(){for(var fd=0;fd<=FS.MAX_OPEN_FDS;fd++){if(!FS.streams[fd]){return fd}}throw new FS.ErrnoError(33)},getStreamChecked(fd){var stream=FS.getStream(fd);if(!stream){throw new FS.ErrnoError(8)}return stream},getStream:fd=>FS.streams[fd],createStream(stream,fd=-1){stream=Object.assign(new FS.FSStream,stream);if(fd==-1){fd=FS.nextfd()}stream.fd=fd;FS.streams[fd]=stream;return stream},closeStream(fd){FS.streams[fd]=null},dupStream(origStream,fd=-1){var stream=FS.createStream(origStream,fd);stream.stream_ops?.dup?.(stream);return stream},doSetAttr(stream,node,attr){var setattr=stream?.stream_ops.setattr;var arg=setattr?stream:node;setattr??=node.node_ops.setattr;FS.checkOpExists(setattr,63);setattr(arg,attr)},chrdev_stream_ops:{open(stream){var device=FS.getDevice(stream.node.rdev);stream.stream_ops=device.stream_ops;stream.stream_ops.open?.(stream)},llseek(){throw new FS.ErrnoError(70)}},major:dev=>dev>>8,minor:dev=>dev&255,makedev:(ma,mi)=>ma<<8|mi,registerDevice(dev,ops){FS.devices[dev]={stream_ops:ops}},getDevice:dev=>FS.devices[dev],getMounts(mount){var mounts=[];var check=[mount];while(check.length){var m=check.pop();mounts.push(m);check.push(...m.mounts)}return mounts},syncfs(populate,callback){if(typeof populate=="function"){callback=populate;populate=false}FS.syncFSRequests++;if(FS.syncFSRequests>1){err(`warning: ${FS.syncFSRequests} FS.syncfs operations in flight at once, probably just doing extra work`)}var mounts=FS.getMounts(FS.root.mount);var completed=0;function doCallback(errCode){FS.syncFSRequests--;return callback(errCode)}function done(errCode){if(errCode){if(!done.errored){done.errored=true;return doCallback(errCode)}return}if(++completed>=mounts.length){doCallback(null)}}for(var mount of mounts){if(mount.type.syncfs){mount.type.syncfs(mount,populate,done)}else{done(null)}}},mount(type,opts,mountpoint){var root=mountpoint==="/";var pseudo=!mountpoint;var node;if(root&&FS.root){throw new FS.ErrnoError(10)}else if(!root&&!pseudo){var lookup=FS.lookupPath(mountpoint,{follow_mount:false});mountpoint=lookup.path;node=lookup.node;if(FS.isMountpoint(node)){throw new FS.ErrnoError(10)}if(!FS.isDir(node.mode)){throw new FS.ErrnoError(54)}}var mount={type,opts,mountpoint,mounts:[]};var mountRoot=type.mount(mount);mountRoot.mount=mount;mount.root=mountRoot;if(root){FS.root=mountRoot}else if(node){node.mounted=mount;if(node.mount){node.mount.mounts.push(mount)}}return mountRoot},unmount(mountpoint){var lookup=FS.lookupPath(mountpoint,{follow_mount:false});if(!FS.isMountpoint(lookup.node)){throw new FS.ErrnoError(28)}var node=lookup.node;var mount=node.mounted;var mounts=FS.getMounts(mount);for(var[hash,current]of Object.entries(FS.nameTable)){while(current){var next=current.name_next;if(mounts.includes(current.mount)){FS.destroyNode(current)}current=next}}node.mounted=null;var idx=node.mount.mounts.indexOf(mount);node.mount.mounts.splice(idx,1)},lookup(parent,name){return parent.node_ops.lookup(parent,name)},mknod(path,mode,dev){var lookup=FS.lookupPath(path,{parent:true});var parent=lookup.node;var name=PATH.basename(path);if(!name){throw new FS.ErrnoError(28)}if(name==="."||name===".."){throw new FS.ErrnoError(20)}var errCode=FS.mayCreate(parent,name);if(errCode){throw new FS.ErrnoError(errCode)}if(!parent.node_ops.mknod){throw new FS.ErrnoError(63)}return parent.node_ops.mknod(parent,name,mode,dev)},statfs(path){return FS.statfsNode(FS.lookupPath(path,{follow:true}).node)},statfsStream(stream){return FS.statfsNode(stream.node)},statfsNode(node){var rtn={bsize:4096,frsize:4096,blocks:1e6,bfree:5e5,bavail:5e5,files:FS.nextInode,ffree:FS.nextInode-1,fsid:42,flags:2,namelen:255};if(node.node_ops.statfs){Object.assign(rtn,node.node_ops.statfs(node.mount.opts.root))}return rtn},create(path,mode=438){mode&=4095;mode|=32768;return FS.mknod(path,mode,0)},mkdir(path,mode=511){mode&=511|512;mode|=16384;return FS.mknod(path,mode,0)},mkdirTree(path,mode){var dirs=path.split("/");var d="";for(var dir of dirs){if(!dir)continue;if(d||PATH.isAbs(path))d+="/";d+=dir;try{FS.mkdir(d,mode)}catch(e){if(e.errno!=20)throw e}}},mkdev(path,mode,dev){if(typeof dev=="undefined"){dev=mode;mode=438}mode|=8192;return FS.mknod(path,mode,dev)},symlink(oldpath,newpath){if(!PATH_FS.resolve(oldpath)){throw new FS.ErrnoError(44)}var lookup=FS.lookupPath(newpath,{parent:true});var parent=lookup.node;if(!parent){throw new FS.ErrnoError(44)}var newname=PATH.basename(newpath);var errCode=FS.mayCreate(parent,newname);if(errCode){throw new FS.ErrnoError(errCode)}if(!parent.node_ops.symlink){throw new FS.ErrnoError(63)}return parent.node_ops.symlink(parent,newname,oldpath)},rename(old_path,new_path){var old_dirname=PATH.dirname(old_path);var new_dirname=PATH.dirname(new_path);var old_name=PATH.basename(old_path);var new_name=PATH.basename(new_path);var lookup,old_dir,new_dir;lookup=FS.lookupPath(old_path,{parent:true});old_dir=lookup.node;lookup=FS.lookupPath(new_path,{parent:true});new_dir=lookup.node;if(!old_dir||!new_dir)throw new FS.ErrnoError(44);if(old_dir.mount!==new_dir.mount){throw new FS.ErrnoError(75)}var old_node=FS.lookupNode(old_dir,old_name);var relative=PATH_FS.relative(old_path,new_dirname);if(relative.charAt(0)!=="."){throw new FS.ErrnoError(28)}relative=PATH_FS.relative(new_path,old_dirname);if(relative.charAt(0)!=="."){throw new FS.ErrnoError(55)}var new_node;try{new_node=FS.lookupNode(new_dir,new_name)}catch(e){}if(old_node===new_node){return}var isdir=FS.isDir(old_node.mode);var errCode=FS.mayDelete(old_dir,old_name,isdir);if(errCode){throw new FS.ErrnoError(errCode)}errCode=new_node?FS.mayDelete(new_dir,new_name,isdir):FS.mayCreate(new_dir,new_name);if(errCode){throw new FS.ErrnoError(errCode)}if(!old_dir.node_ops.rename){throw new FS.ErrnoError(63)}if(FS.isMountpoint(old_node)||new_node&&FS.isMountpoint(new_node)){throw new FS.ErrnoError(10)}if(new_dir!==old_dir){errCode=FS.nodePermissions(old_dir,"w");if(errCode){throw new FS.ErrnoError(errCode)}}FS.hashRemoveNode(old_node);try{old_dir.node_ops.rename(old_node,new_dir,new_name);old_node.parent=new_dir}catch(e){throw e}finally{FS.hashAddNode(old_node)}},rmdir(path){var lookup=FS.lookupPath(path,{parent:true});var parent=lookup.node;var name=PATH.basename(path);var node=FS.lookupNode(parent,name);var errCode=FS.mayDelete(parent,name,true);if(errCode){throw new FS.ErrnoError(errCode)}if(!parent.node_ops.rmdir){throw new FS.ErrnoError(63)}if(FS.isMountpoint(node)){throw new FS.ErrnoError(10)}parent.node_ops.rmdir(parent,name);FS.destroyNode(node)},readdir(path){var lookup=FS.lookupPath(path,{follow:true});var node=lookup.node;var readdir=FS.checkOpExists(node.node_ops.readdir,54);return readdir(node)},unlink(path){var lookup=FS.lookupPath(path,{parent:true});var parent=lookup.node;if(!parent){throw new FS.ErrnoError(44)}var name=PATH.basename(path);var node=FS.lookupNode(parent,name);var errCode=FS.mayDelete(parent,name,false);if(errCode){throw new FS.ErrnoError(errCode)}if(!parent.node_ops.unlink){throw new FS.ErrnoError(63)}if(FS.isMountpoint(node)){throw new FS.ErrnoError(10)}parent.node_ops.unlink(parent,name);FS.destroyNode(node)},readlink(path){var lookup=FS.lookupPath(path);var link=lookup.node;if(!link){throw new FS.ErrnoError(44)}if(!link.node_ops.readlink){throw new FS.ErrnoError(28)}return link.node_ops.readlink(link)},stat(path,dontFollow){var lookup=FS.lookupPath(path,{follow:!dontFollow});var node=lookup.node;var getattr=FS.checkOpExists(node.node_ops.getattr,63);return getattr(node)},fstat(fd){var stream=FS.getStreamChecked(fd);var node=stream.node;var getattr=stream.stream_ops.getattr;var arg=getattr?stream:node;getattr??=node.node_ops.getattr;FS.checkOpExists(getattr,63);return getattr(arg)},lstat(path){return FS.stat(path,true)},doChmod(stream,node,mode,dontFollow){FS.doSetAttr(stream,node,{mode:mode&4095|node.mode&~4095,ctime:Date.now(),dontFollow})},chmod(path,mode,dontFollow){var node;if(typeof path=="string"){var lookup=FS.lookupPath(path,{follow:!dontFollow});node=lookup.node}else{node=path}FS.doChmod(null,node,mode,dontFollow)},lchmod(path,mode){FS.chmod(path,mode,true)},fchmod(fd,mode){var stream=FS.getStreamChecked(fd);FS.doChmod(stream,stream.node,mode,false)},doChown(stream,node,dontFollow){FS.doSetAttr(stream,node,{timestamp:Date.now(),dontFollow})},chown(path,uid,gid,dontFollow){var node;if(typeof path=="string"){var lookup=FS.lookupPath(path,{follow:!dontFollow});node=lookup.node}else{node=path}FS.doChown(null,node,dontFollow)},lchown(path,uid,gid){FS.chown(path,uid,gid,true)},fchown(fd,uid,gid){var stream=FS.getStreamChecked(fd);FS.doChown(stream,stream.node,false)},doTruncate(stream,node,len){if(FS.isDir(node.mode)){throw new FS.ErrnoError(31)}if(!FS.isFile(node.mode)){throw new FS.ErrnoError(28)}var errCode=FS.nodePermissions(node,"w");if(errCode){throw new FS.ErrnoError(errCode)}FS.doSetAttr(stream,node,{size:len,timestamp:Date.now()})},truncate(path,len){if(len<0){throw new FS.ErrnoError(28)}var node;if(typeof path=="string"){var lookup=FS.lookupPath(path,{follow:true});node=lookup.node}else{node=path}FS.doTruncate(null,node,len)},ftruncate(fd,len){var stream=FS.getStreamChecked(fd);if(len<0||(stream.flags&2097155)===0){throw new FS.ErrnoError(28)}FS.doTruncate(stream,stream.node,len)},utime(path,atime,mtime){var lookup=FS.lookupPath(path,{follow:true});var node=lookup.node;var setattr=FS.checkOpExists(node.node_ops.setattr,63);setattr(node,{atime,mtime})},open(path,flags,mode=438){if(path===""){throw new FS.ErrnoError(44)}flags=FS_modeStringToFlags(flags);if(flags&64){mode=mode&4095|32768}else{mode=0}var node;var isDirPath;if(typeof path=="object"){node=path}else{isDirPath=path.endsWith("/");var lookup=FS.lookupPath(path,{follow:!(flags&131072),noent_okay:true});node=lookup.node;path=lookup.path}var created=false;if(flags&64){if(node){if(flags&128){throw new FS.ErrnoError(20)}}else if(isDirPath){throw new FS.ErrnoError(31)}else{node=FS.mknod(path,mode|511,0);created=true}}if(!node){throw new FS.ErrnoError(44)}if(FS.isChrdev(node.mode)){flags&=~512}if(flags&65536&&!FS.isDir(node.mode)){throw new FS.ErrnoError(54)}if(!created){var errCode=FS.mayOpen(node,flags);if(errCode){throw new FS.ErrnoError(errCode)}}if(flags&512&&!created){FS.truncate(node,0)}flags&=~(128|512|131072);var stream=FS.createStream({node,path:FS.getPath(node),flags,seekable:true,position:0,stream_ops:node.stream_ops,ungotten:[],error:false});if(stream.stream_ops.open){stream.stream_ops.open(stream)}if(created){FS.chmod(node,mode&511)}return stream},close(stream){if(FS.isClosed(stream)){throw new FS.ErrnoError(8)}if(stream.getdents)stream.getdents=null;try{if(stream.stream_ops.close){stream.stream_ops.close(stream)}}catch(e){throw e}finally{FS.closeStream(stream.fd)}stream.fd=null},isClosed(stream){return stream.fd===null},llseek(stream,offset,whence){if(FS.isClosed(stream)){throw new FS.ErrnoError(8)}if(!stream.seekable||!stream.stream_ops.llseek){throw new FS.ErrnoError(70)}if(whence!=0&&whence!=1&&whence!=2){throw new FS.ErrnoError(28)}stream.position=stream.stream_ops.llseek(stream,offset,whence);stream.ungotten=[];return stream.position},read(stream,buffer,offset,length,position){if(length<0||position<0){throw new FS.ErrnoError(28)}if(FS.isClosed(stream)){throw new FS.ErrnoError(8)}if((stream.flags&2097155)===1){throw new FS.ErrnoError(8)}if(FS.isDir(stream.node.mode)){throw new FS.ErrnoError(31)}if(!stream.stream_ops.read){throw new FS.ErrnoError(28)}var seeking=typeof position!="undefined";if(!seeking){position=stream.position}else if(!stream.seekable){throw new FS.ErrnoError(70)}var bytesRead=stream.stream_ops.read(stream,buffer,offset,length,position);if(!seeking)stream.position+=bytesRead;return bytesRead},write(stream,buffer,offset,length,position,canOwn){if(length<0||position<0){throw new FS.ErrnoError(28)}if(FS.isClosed(stream)){throw new FS.ErrnoError(8)}if((stream.flags&2097155)===0){throw new FS.ErrnoError(8)}if(FS.isDir(stream.node.mode)){throw new FS.ErrnoError(31)}if(!stream.stream_ops.write){throw new FS.ErrnoError(28)}if(stream.seekable&&stream.flags&1024){FS.llseek(stream,0,2)}var seeking=typeof position!="undefined";if(!seeking){position=stream.position}else if(!stream.seekable){throw new FS.ErrnoError(70)}var bytesWritten=stream.stream_ops.write(stream,buffer,offset,length,position,canOwn);if(!seeking)stream.position+=bytesWritten;return bytesWritten},mmap(stream,length,position,prot,flags){if((prot&2)!==0&&(flags&2)===0&&(stream.flags&2097155)!==2){throw new FS.ErrnoError(2)}if((stream.flags&2097155)===1){throw new FS.ErrnoError(2)}if(!stream.stream_ops.mmap){throw new FS.ErrnoError(43)}if(!length){throw new FS.ErrnoError(28)}return stream.stream_ops.mmap(stream,length,position,prot,flags)},msync(stream,buffer,offset,length,mmapFlags){if(!stream.stream_ops.msync){return 0}return stream.stream_ops.msync(stream,buffer,offset,length,mmapFlags)},ioctl(stream,cmd,arg){if(!stream.stream_ops.ioctl){throw new FS.ErrnoError(59)}return stream.stream_ops.ioctl(stream,cmd,arg)},readFile(path,opts={}){opts.flags=opts.flags||0;opts.encoding=opts.encoding||"binary";if(opts.encoding!=="utf8"&&opts.encoding!=="binary"){abort(`Invalid encoding type "${opts.encoding}"`)}var stream=FS.open(path,opts.flags);var stat=FS.stat(path);var length=stat.size;var buf=new Uint8Array(length);FS.read(stream,buf,0,length,0);if(opts.encoding==="utf8"){buf=UTF8ArrayToString(buf)}FS.close(stream);return buf},writeFile(path,data,opts={}){opts.flags=opts.flags||577;var stream=FS.open(path,opts.flags,opts.mode);data=FS_fileDataToTypedArray(data);FS.write(stream,data,0,data.byteLength,undefined,opts.canOwn);FS.close(stream)},cwd:()=>FS.currentPath,chdir(path){var lookup=FS.lookupPath(path,{follow:true});if(lookup.node===null){throw new FS.ErrnoError(44)}if(!FS.isDir(lookup.node.mode)){throw new FS.ErrnoError(54)}var errCode=FS.nodePermissions(lookup.node,"x");if(errCode){throw new FS.ErrnoError(errCode)}FS.currentPath=lookup.path},createDefaultDirectories(){FS.mkdir("/tmp");FS.mkdir("/home");FS.mkdir("/home/web_user")},createDefaultDevices(){FS.mkdir("/dev");FS.registerDevice(FS.makedev(1,3),{read:()=>0,write:(stream,buffer,offset,length,pos)=>length,llseek:()=>0});FS.mkdev("/dev/null",FS.makedev(1,3));TTY.register(FS.makedev(5,0),TTY.default_tty_ops);TTY.register(FS.makedev(6,0),TTY.default_tty1_ops);FS.mkdev("/dev/tty",FS.makedev(5,0));FS.mkdev("/dev/tty1",FS.makedev(6,0));var randomBuffer=new Uint8Array(1024),randomLeft=0;var randomByte=()=>{if(randomLeft===0){randomFill(randomBuffer);randomLeft=randomBuffer.byteLength}return randomBuffer[--randomLeft]};FS.createDevice("/dev","random",randomByte);FS.createDevice("/dev","urandom",randomByte);FS.mkdir("/dev/shm");FS.mkdir("/dev/shm/tmp")},createSpecialDirectories(){FS.mkdir("/proc");var proc_self=FS.mkdir("/proc/self");FS.mkdir("/proc/self/fd");FS.mount({mount(){var node=FS.createNode(proc_self,"fd",16895,73);node.stream_ops={llseek:MEMFS.stream_ops.llseek};node.node_ops={lookup(parent,name){var fd=+name;var stream=FS.getStreamChecked(fd);var ret={parent:null,mount:{mountpoint:"fake"},node_ops:{readlink:()=>stream.path},id:fd+1};ret.parent=ret;return ret},readdir(){return Array.from(FS.streams.entries()).filter(([k,v])=>v).map(([k,v])=>k.toString())}};return node}},{},"/proc/self/fd")},createStandardStreams(input,output,error){if(input){FS.createDevice("/dev","stdin",input)}else{FS.symlink("/dev/tty","/dev/stdin")}if(output){FS.createDevice("/dev","stdout",null,output)}else{FS.symlink("/dev/tty","/dev/stdout")}if(error){FS.createDevice("/dev","stderr",null,error)}else{FS.symlink("/dev/tty1","/dev/stderr")}var stdin=FS.open("/dev/stdin",0);var stdout=FS.open("/dev/stdout",1);var stderr=FS.open("/dev/stderr",1)},staticInit(){FS.nameTable=new Array(4096);FS.mount(MEMFS,{},"/");FS.createDefaultDirectories();FS.createDefaultDevices();FS.createSpecialDirectories();FS.filesystems={MEMFS}},init(input,output,error){FS.initialized=true;input??=Module["stdin"];output??=Module["stdout"];error??=Module["stderr"];FS.createStandardStreams(input,output,error)},quit(){FS.initialized=false;for(var stream of FS.streams){if(stream){FS.close(stream)}}},findObject(path,dontResolveLastLink){var ret=FS.analyzePath(path,dontResolveLastLink);if(!ret.exists){return null}return ret.object},analyzePath(path,dontResolveLastLink){try{var lookup=FS.lookupPath(path,{follow:!dontResolveLastLink});path=lookup.path}catch(e){}var ret={isRoot:false,exists:false,error:0,name:null,path:null,object:null,parentExists:false,parentPath:null,parentObject:null};try{var lookup=FS.lookupPath(path,{parent:true});ret.parentExists=true;ret.parentPath=lookup.path;ret.parentObject=lookup.node;ret.name=PATH.basename(path);lookup=FS.lookupPath(path,{follow:!dontResolveLastLink});ret.exists=true;ret.path=lookup.path;ret.object=lookup.node;ret.name=lookup.node.name;ret.isRoot=lookup.path==="/"}catch(e){ret.error=e.errno}return ret},createPath(parent,path,canRead,canWrite){parent=typeof parent=="string"?parent:FS.getPath(parent);var parts=path.split("/").reverse();while(parts.length){var part=parts.pop();if(!part)continue;var current=PATH.join2(parent,part);try{FS.mkdir(current)}catch(e){if(e.errno!=20)throw e}parent=current}return current},createFile(parent,name,properties,canRead,canWrite){var path=PATH.join2(typeof parent=="string"?parent:FS.getPath(parent),name);var mode=FS_getMode(canRead,canWrite);return FS.create(path,mode)},createDataFile(parent,name,data,canRead,canWrite,canOwn){var path=name;if(parent){parent=typeof parent=="string"?parent:FS.getPath(parent);path=name?PATH.join2(parent,name):parent}var mode=FS_getMode(canRead,canWrite);var node=FS.create(path,mode);if(data){data=FS_fileDataToTypedArray(data);FS.chmod(node,mode|146);var stream=FS.open(node,577);FS.write(stream,data,0,data.length,0,canOwn);FS.close(stream);FS.chmod(node,mode)}},createDevice(parent,name,input,output){var path=PATH.join2(typeof parent=="string"?parent:FS.getPath(parent),name);var mode=FS_getMode(!!input,!!output);FS.createDevice.major??=64;var dev=FS.makedev(FS.createDevice.major++,0);FS.registerDevice(dev,{open(stream){stream.seekable=false},close(stream){if(output?.buffer?.length){output(10)}},read(stream,buffer,offset,length,pos){var bytesRead=0;for(var i=0;ithis.length-1||idx<0){return undefined}var chunkOffset=idx%this.chunkSize;var chunkNum=idx/this.chunkSize|0;return this.getter(chunkNum)[chunkOffset]}setDataGetter(getter){this.getter=getter}cacheLength(){var xhr=new XMLHttpRequest;xhr.open("HEAD",url,false);xhr.send(null);if(!(xhr.status>=200&&xhr.status<300||xhr.status===304))abort("Couldn't load "+url+". Status: "+xhr.status);var datalength=Number(xhr.getResponseHeader("Content-length"));var header;var hasByteServing=(header=xhr.getResponseHeader("Accept-Ranges"))&&header==="bytes";var usesGzip=(header=xhr.getResponseHeader("Content-Encoding"))&&header==="gzip";var chunkSize=1024*1024;if(!hasByteServing)chunkSize=datalength;var doXHR=(from,to)=>{if(from>to)abort("invalid range ("+from+", "+to+") or no bytes requested!");if(to>datalength-1)abort("only "+datalength+" bytes available! programmer error!");var xhr=new XMLHttpRequest;xhr.open("GET",url,false);if(datalength!==chunkSize)xhr.setRequestHeader("Range","bytes="+from+"-"+to);xhr.responseType="arraybuffer";if(xhr.overrideMimeType){xhr.overrideMimeType("text/plain; charset=x-user-defined")}xhr.send(null);if(!(xhr.status>=200&&xhr.status<300||xhr.status===304))abort("Couldn't load "+url+". Status: "+xhr.status);if(xhr.response!==undefined){return new Uint8Array(xhr.response||[])}return intArrayFromString(xhr.responseText||"",true)};var lazyArray=this;lazyArray.setDataGetter(chunkNum=>{var start=chunkNum*chunkSize;var end=(chunkNum+1)*chunkSize-1;end=Math.min(end,datalength-1);if(typeof lazyArray.chunks[chunkNum]=="undefined"){lazyArray.chunks[chunkNum]=doXHR(start,end)}if(typeof lazyArray.chunks[chunkNum]=="undefined")abort("doXHR failed!");return lazyArray.chunks[chunkNum]});if(usesGzip||!datalength){chunkSize=datalength=1;datalength=this.getter(0).length;chunkSize=datalength;out("LazyFiles on gzip forces download of the whole file when length is accessed")}this._length=datalength;this._chunkSize=chunkSize;this.lengthKnown=true}get length(){if(!this.lengthKnown){this.cacheLength()}return this._length}get chunkSize(){if(!this.lengthKnown){this.cacheLength()}return this._chunkSize}}if(globalThis.XMLHttpRequest){if(!ENVIRONMENT_IS_WORKER)abort("Cannot do synchronous binary XHRs outside webworkers in modern browsers. Use --embed-file or --preload-file in emcc");var lazyArray=new LazyUint8Array;var properties={isDevice:false,contents:lazyArray}}else{var properties={isDevice:false,url}}var node=FS.createFile(parent,name,properties,canRead,canWrite);if(properties.contents){node.contents=properties.contents}else if(properties.url){node.contents=null;node.url=properties.url}Object.defineProperties(node,{usedBytes:{get:function(){return this.contents.length}}});var stream_ops={};for(const[key,fn]of Object.entries(node.stream_ops)){stream_ops[key]=(...args)=>{FS.forceLoadFile(node);return fn(...args)}}function writeChunks(stream,buffer,offset,length,position){var contents=stream.node.contents;if(position>=contents.length)return 0;var size=Math.min(contents.length-position,length);if(contents.slice){for(var i=0;i{FS.forceLoadFile(node);return writeChunks(stream,buffer,offset,length,position)};stream_ops.mmap=(stream,length,position,prot,flags)=>{FS.forceLoadFile(node);var ptr=mmapAlloc(length);if(!ptr){throw new FS.ErrnoError(48)}writeChunks(stream,HEAP8,ptr,length,position);return{ptr,allocated:true}};node.stream_ops=stream_ops;return node}};var SOCKFS={websocketArgs:{},callbacks:{},on(event,callback){SOCKFS.callbacks[event]=callback},emit(event,param){SOCKFS.callbacks[event]?.(param)},mount(mount){SOCKFS.websocketArgs=Module["websocket"]||{};(Module["websocket"]??={})["on"]=SOCKFS.on;return FS.createNode(null,"/",16895,0)},createSocket(family,type,protocol){if(family!=2){throw new FS.ErrnoError(5)}type&=~526336;if(type!=1&&type!=2){throw new FS.ErrnoError(28)}var streaming=type==1;if(streaming&&protocol&&protocol!=6){throw new FS.ErrnoError(66)}var sock={family,type,protocol,server:null,error:null,peers:{},pending:[],recv_queue:[],sock_ops:SOCKFS.websocket_sock_ops};var name=SOCKFS.nextname();var node=FS.createNode(SOCKFS.root,name,49152,0);node.sock=sock;var stream=FS.createStream({path:name,node,flags:2,seekable:false,stream_ops:SOCKFS.stream_ops});sock.stream=stream;return sock},getSocket(fd){var stream=FS.getStream(fd);if(!stream||!FS.isSocket(stream.node.mode)){return null}return stream.node.sock},stream_ops:{poll(stream){var sock=stream.node.sock;return sock.sock_ops.poll(sock)},ioctl(stream,request,varargs){var sock=stream.node.sock;return sock.sock_ops.ioctl(sock,request,varargs)},read(stream,buffer,offset,length,position){var sock=stream.node.sock;var msg=sock.sock_ops.recvmsg(sock,length);if(!msg){return 0}buffer.set(msg.buffer,offset);return msg.buffer.length},write(stream,buffer,offset,length,position){var sock=stream.node.sock;return sock.sock_ops.sendmsg(sock,buffer,offset,length)},close(stream){var sock=stream.node.sock;sock.sock_ops.close(sock)}},nextname(){if(!SOCKFS.nextname.current){SOCKFS.nextname.current=0}return`socket[${SOCKFS.nextname.current++}]`},websocket_sock_ops:{createPeer(sock,addr,port){var ws;if(typeof addr=="object"){ws=addr;addr=null;port=null}if(ws){if(ws._socket){addr=ws._socket.remoteAddress;port=ws._socket.remotePort}else{var result=/ws[s]?:\/\/([^:]+):(\d+)/.exec(ws.url);if(!result){throw new Error("WebSocket URL must be in the format ws(s)://address:port")}addr=result[1];port=parseInt(result[2],10)}}else{try{var url="ws://".replace("#","//");var subProtocols="binary";var opts=undefined;if(SOCKFS.websocketArgs["url"]){url=SOCKFS.websocketArgs["url"]}if(SOCKFS.websocketArgs["subprotocol"]){subProtocols=SOCKFS.websocketArgs["subprotocol"]}else if(SOCKFS.websocketArgs["subprotocol"]===null){subProtocols="null"}if(url==="ws://"||url==="wss://"){var parts=addr.split("/");url=url+parts[0]+":"+port+"/"+parts.slice(1).join("/")}if(subProtocols!=="null"){subProtocols=subProtocols.replace(/^ +| +$/g,"").split(/ *, */);opts=subProtocols}var WebSocketConstructor;{WebSocketConstructor=WebSocket}ws=new WebSocketConstructor(url,opts);ws.binaryType="arraybuffer"}catch(e){throw new FS.ErrnoError(23)}}var peer={addr,port,socket:ws,msg_send_queue:[]};SOCKFS.websocket_sock_ops.addPeer(sock,peer);SOCKFS.websocket_sock_ops.handlePeerEvents(sock,peer);if(sock.type===2&&typeof sock.sport!="undefined"){peer.msg_send_queue.push(new Uint8Array([255,255,255,255,"p".charCodeAt(0),"o".charCodeAt(0),"r".charCodeAt(0),"t".charCodeAt(0),(sock.sport&65280)>>8,sock.sport&255]))}return peer},getPeer(sock,addr,port){return sock.peers[addr+":"+port]},addPeer(sock,peer){sock.peers[peer.addr+":"+peer.port]=peer},removePeer(sock,peer){delete sock.peers[peer.addr+":"+peer.port]},handlePeerEvents(sock,peer){var first=true;var handleOpen=function(){sock.connecting=false;SOCKFS.emit("open",sock.stream.fd);try{var queued=peer.msg_send_queue.shift();while(queued){peer.socket.send(queued);queued=peer.msg_send_queue.shift()}}catch(e){peer.socket.close()}};function handleMessage(data){if(typeof data=="string"){var encoder=new TextEncoder;data=encoder.encode(data)}else{if(data.byteLength==0){return}data=new Uint8Array(data)}var wasfirst=first;first=false;if(wasfirst&&data.length===10&&data[0]===255&&data[1]===255&&data[2]===255&&data[3]===255&&data[4]==="p".charCodeAt(0)&&data[5]==="o".charCodeAt(0)&&data[6]==="r".charCodeAt(0)&&data[7]==="t".charCodeAt(0)){var newport=data[8]<<8|data[9];SOCKFS.websocket_sock_ops.removePeer(sock,peer);peer.port=newport;SOCKFS.websocket_sock_ops.addPeer(sock,peer);return}sock.recv_queue.push({addr:peer.addr,port:peer.port,data});SOCKFS.emit("message",sock.stream.fd)}if(ENVIRONMENT_IS_NODE){peer.socket.on("open",handleOpen);peer.socket.on("message",function(data,isBinary){if(!isBinary){return}handleMessage(new Uint8Array(data).buffer)});peer.socket.on("close",function(){SOCKFS.emit("close",sock.stream.fd)});peer.socket.on("error",function(error){sock.error=14;SOCKFS.emit("error",[sock.stream.fd,sock.error,"ECONNREFUSED: Connection refused"])})}else{peer.socket.onopen=handleOpen;peer.socket.onclose=function(){SOCKFS.emit("close",sock.stream.fd)};peer.socket.onmessage=function peer_socket_onmessage(event){handleMessage(event.data)};peer.socket.onerror=function(error){sock.error=14;SOCKFS.emit("error",[sock.stream.fd,sock.error,"ECONNREFUSED: Connection refused"])}}},poll(sock){if(sock.type===1&&sock.server){return sock.pending.length?64|1:0}var mask=0;var dest=sock.type===1?SOCKFS.websocket_sock_ops.getPeer(sock,sock.daddr,sock.dport):null;if(sock.recv_queue.length||!dest||dest&&dest.socket.readyState===dest.socket.CLOSING||dest&&dest.socket.readyState===dest.socket.CLOSED){mask|=64|1}if(!dest||dest&&dest.socket.readyState===dest.socket.OPEN){mask|=4}if(dest&&dest.socket.readyState===dest.socket.CLOSING||dest&&dest.socket.readyState===dest.socket.CLOSED){if(sock.connecting){mask|=4}else{mask|=16}}return mask},ioctl(sock,request,arg){switch(request){case 21531:var bytes=0;if(sock.recv_queue.length){bytes=sock.recv_queue[0].data.length}HEAP32[arg>>2]=bytes;return 0;case 21537:var on=HEAP32[arg>>2];if(on){sock.stream.flags|=2048}else{sock.stream.flags&=~2048}return 0;default:return 28}},close(sock){if(sock.server){try{sock.server.close()}catch(e){}sock.server=null}for(var peer of Object.values(sock.peers)){try{peer.socket.close()}catch(e){}SOCKFS.websocket_sock_ops.removePeer(sock,peer)}return 0},bind(sock,addr,port){if(typeof sock.saddr!="undefined"||typeof sock.sport!="undefined"){throw new FS.ErrnoError(28)}sock.saddr=addr;sock.sport=port;if(sock.type===2){if(sock.server){sock.server.close();sock.server=null}try{sock.sock_ops.listen(sock,0)}catch(e){if(!(e.name==="ErrnoError"))throw e;if(e.errno!==138)throw e}}},connect(sock,addr,port){if(sock.server){throw new FS.ErrnoError(138)}if(typeof sock.daddr!="undefined"&&typeof sock.dport!="undefined"){var dest=SOCKFS.websocket_sock_ops.getPeer(sock,sock.daddr,sock.dport);if(dest){if(dest.socket.readyState===dest.socket.CONNECTING){throw new FS.ErrnoError(7)}else{throw new FS.ErrnoError(30)}}}var peer=SOCKFS.websocket_sock_ops.createPeer(sock,addr,port);sock.daddr=peer.addr;sock.dport=peer.port;sock.connecting=true},listen(sock,backlog){if(!ENVIRONMENT_IS_NODE){throw new FS.ErrnoError(138)}},accept(listensock){if(!listensock.server||!listensock.pending.length){throw new FS.ErrnoError(28)}var newsock=listensock.pending.shift();newsock.stream.flags=listensock.stream.flags;return newsock},getname(sock,peer){var addr,port;if(peer){if(sock.daddr===undefined||sock.dport===undefined){throw new FS.ErrnoError(53)}addr=sock.daddr;port=sock.dport}else{addr=sock.saddr||0;port=sock.sport||0}return{addr,port}},sendmsg(sock,buffer,offset,length,addr,port){if(sock.type===2){if(addr===undefined||port===undefined){addr=sock.daddr;port=sock.dport}if(addr===undefined||port===undefined){throw new FS.ErrnoError(17)}}else{addr=sock.daddr;port=sock.dport}var dest=SOCKFS.websocket_sock_ops.getPeer(sock,addr,port);if(sock.type===1){if(!dest||dest.socket.readyState===dest.socket.CLOSING||dest.socket.readyState===dest.socket.CLOSED){throw new FS.ErrnoError(53)}}if(ArrayBuffer.isView(buffer)){offset+=buffer.byteOffset;buffer=buffer.buffer}var data=buffer.slice(offset,offset+length);if(!dest||dest.socket.readyState!==dest.socket.OPEN){if(sock.type===2){if(!dest||dest.socket.readyState===dest.socket.CLOSING||dest.socket.readyState===dest.socket.CLOSED){dest=SOCKFS.websocket_sock_ops.createPeer(sock,addr,port)}}dest.msg_send_queue.push(data);return length}try{dest.socket.send(data);return length}catch(e){throw new FS.ErrnoError(28)}},recvmsg(sock,length){if(sock.type===1&&sock.server){throw new FS.ErrnoError(53)}var queued=sock.recv_queue.shift();if(!queued){if(sock.type===1){var dest=SOCKFS.websocket_sock_ops.getPeer(sock,sock.daddr,sock.dport);if(!dest){throw new FS.ErrnoError(53)}if(dest.socket.readyState===dest.socket.CLOSING||dest.socket.readyState===dest.socket.CLOSED){return null}throw new FS.ErrnoError(6)}throw new FS.ErrnoError(6)}var queuedLength=queued.data.byteLength||queued.data.length;var queuedOffset=queued.data.byteOffset||0;var queuedBuffer=queued.data.buffer||queued.data;var bytesRead=Math.min(length,queuedLength);var res={buffer:new Uint8Array(queuedBuffer,queuedOffset,bytesRead),addr:queued.addr,port:queued.port};if(sock.type===1&&bytesRead{var socket=SOCKFS.getSocket(fd);if(!socket)throw new FS.ErrnoError(8);return socket};var inetPton4=str=>{var b=str.split(".");for(var i=0;i<4;i++){var tmp=Number(b[i]);if(isNaN(tmp))return null;b[i]=tmp}return(b[0]|b[1]<<8|b[2]<<16|b[3]<<24)>>>0};var inetPton6=str=>{var words;var w,offset,z;var valid6regx=/^((?=.*::)(?!.*::.+::)(::)?([\dA-F]{1,4}:(:|\b)|){5}|([\dA-F]{1,4}:){6})((([\dA-F]{1,4}((?!\3)::|:\b|$))|(?!\2\3)){2}|(((2[0-4]|1\d|[1-9])?\d|25[0-5])\.?\b){4})$/i;var parts=[];if(!valid6regx.test(str)){return null}if(str==="::"){return[0,0,0,0,0,0,0,0]}if(str.startsWith("::")){str=str.replace("::","Z:")}else{str=str.replace("::",":Z:")}if(str.indexOf(".")>0){str=str.replace(new RegExp("[.]","g"),":");words=str.split(":");words[words.length-4]=Number(words[words.length-4])+Number(words[words.length-3])*256;words[words.length-3]=Number(words[words.length-2])+Number(words[words.length-1])*256;words=words.slice(0,words.length-2)}else{words=str.split(":")}offset=0;z=0;for(w=0;w{switch(family){case 2:addr=inetPton4(addr);zeroMemory(sa,16);if(addrlen){HEAP32[addrlen>>2]=16}HEAP16[sa>>1]=family;HEAP32[sa+4>>2]=addr;HEAP16[sa+2>>1]=_htons(port);break;case 10:addr=inetPton6(addr);zeroMemory(sa,28);if(addrlen){HEAP32[addrlen>>2]=28}HEAP32[sa>>2]=family;HEAP32[sa+8>>2]=addr[0];HEAP32[sa+12>>2]=addr[1];HEAP32[sa+16>>2]=addr[2];HEAP32[sa+20>>2]=addr[3];HEAP16[sa+2>>1]=_htons(port);break;default:return 5}return 0};var DNS={address_map:{id:1,addrs:{},names:{}},lookup_name(name){var res=inetPton4(name);if(res!==null){return name}res=inetPton6(name);if(res!==null){return name}var addr;if(DNS.address_map.addrs[name]){addr=DNS.address_map.addrs[name]}else{var id=DNS.address_map.id++;addr="172.29."+(id&255)+"."+(id&65280);DNS.address_map.names[addr]=name;DNS.address_map.addrs[name]=addr}return addr},lookup_addr(addr){if(DNS.address_map.names[addr]){return DNS.address_map.names[addr]}return null}};function ___syscall_accept4(fd,addr,addrlen,flags,d1,d2){addr=bigintToI53Checked(addr);addrlen=bigintToI53Checked(addrlen);try{var sock=getSocketFromFD(fd);var newsock=sock.sock_ops.accept(sock);if(addr){var errno=writeSockaddr(addr,newsock.family,DNS.lookup_name(newsock.daddr),newsock.dport,addrlen)}return newsock.stream.fd}catch(e){if(typeof FS=="undefined"||!(e.name==="ErrnoError"))throw e;return-e.errno}}var inetNtop4=addr=>(addr&255)+"."+(addr>>8&255)+"."+(addr>>16&255)+"."+(addr>>24&255);var inetNtop6=ints=>{var str="";var word=0;var longest=0;var lastzero=0;var zstart=0;var len=0;var i=0;var parts=[ints[0]&65535,ints[0]>>16,ints[1]&65535,ints[1]>>16,ints[2]&65535,ints[2]>>16,ints[3]&65535,ints[3]>>16];var hasipv4=true;var v4part="";for(i=0;i<5;i++){if(parts[i]!==0){hasipv4=false;break}}if(hasipv4){v4part=inetNtop4(parts[6]|parts[7]<<16);if(parts[5]===-1){str="::ffff:";str+=v4part;return str}if(parts[5]===0){str="::";if(v4part==="0.0.0.0")v4part="";if(v4part==="0.0.0.1")v4part="1";str+=v4part;return str}}for(word=0;word<8;word++){if(parts[word]===0){if(word-lastzero>1){len=0}lastzero=word;len++}if(len>longest){longest=len;zstart=word-longest+1}}for(word=0;word<8;word++){if(longest>1){if(parts[word]===0&&word>=zstart&&word{var family=HEAP16[sa>>1];var port=_ntohs(HEAPU16[sa+2>>1]);var addr;switch(family){case 2:if(salen!==16){return{errno:28}}addr=HEAP32[sa+4>>2];addr=inetNtop4(addr);break;case 10:if(salen!==28){return{errno:28}}addr=[HEAP32[sa+8>>2],HEAP32[sa+12>>2],HEAP32[sa+16>>2],HEAP32[sa+20>>2]];addr=inetNtop6(addr);break;default:return{errno:5}}return{family,addr,port}};var getSocketAddress=(addrp,addrlen)=>{var info=readSockaddr(addrp,addrlen);if(info.errno)throw new FS.ErrnoError(info.errno);info.addr=DNS.lookup_addr(info.addr)||info.addr;return info};function ___syscall_bind(fd,addr,addrlen,d1,d2,d3){addr=bigintToI53Checked(addr);addrlen=bigintToI53Checked(addrlen);try{var sock=getSocketFromFD(fd);var info=getSocketAddress(addr,addrlen);sock.sock_ops.bind(sock,info.addr,info.port);return 0}catch(e){if(typeof FS=="undefined"||!(e.name==="ErrnoError"))throw e;return-e.errno}}var SYSCALLS={calculateAt(dirfd,path,allowEmpty){if(PATH.isAbs(path)){return path}var dir;if(dirfd===-100){dir=FS.cwd()}else{var dirstream=SYSCALLS.getStreamFromFD(dirfd);dir=dirstream.path}if(path.length==0){if(!allowEmpty){throw new FS.ErrnoError(44)}return dir}return dir+"/"+path},writeStat(buf,stat){HEAPU32[buf>>2]=stat.dev;HEAPU32[buf+4>>2]=stat.mode;HEAPU64[buf+8>>3]=BigInt(stat.nlink);HEAPU32[buf+16>>2]=stat.uid;HEAPU32[buf+20>>2]=stat.gid;HEAPU32[buf+24>>2]=stat.rdev;HEAP64[buf+32>>3]=BigInt(stat.size);HEAP32[buf+40>>2]=4096;HEAP32[buf+44>>2]=stat.blocks;var atime=stat.atime.getTime();var mtime=stat.mtime.getTime();var ctime=stat.ctime.getTime();HEAP64[buf+48>>3]=BigInt(Math.floor(atime/1e3));HEAPU64[buf+56>>3]=BigInt(atime%1e3*1e3*1e3);HEAP64[buf+64>>3]=BigInt(Math.floor(mtime/1e3));HEAPU64[buf+72>>3]=BigInt(mtime%1e3*1e3*1e3);HEAP64[buf+80>>3]=BigInt(Math.floor(ctime/1e3));HEAPU64[buf+88>>3]=BigInt(ctime%1e3*1e3*1e3);HEAP64[buf+96>>3]=BigInt(stat.ino);return 0},writeStatFs(buf,stats){HEAPU32[buf+8>>2]=stats.bsize;HEAPU32[buf+72>>2]=stats.bsize;HEAP64[buf+16>>3]=BigInt(stats.blocks);HEAP64[buf+24>>3]=BigInt(stats.bfree);HEAP64[buf+32>>3]=BigInt(stats.bavail);HEAP64[buf+40>>3]=BigInt(stats.files);HEAP64[buf+48>>3]=BigInt(stats.ffree);HEAPU32[buf+56>>2]=stats.fsid;HEAPU32[buf+80>>2]=stats.flags;HEAPU32[buf+64>>2]=stats.namelen},doMsync(addr,stream,len,flags,offset){if(!FS.isFile(stream.node.mode)){throw new FS.ErrnoError(43)}if(flags&2){return 0}var buffer=HEAPU8.slice(addr,addr+len);FS.msync(stream,buffer,offset,len,flags)},getStreamFromFD(fd){var stream=FS.getStreamChecked(fd);return stream},varargs:undefined,getStr(ptr){var ret=UTF8ToString(ptr);return ret}};function ___syscall_chdir(path){path=bigintToI53Checked(path);try{path=SYSCALLS.getStr(path);FS.chdir(path);return 0}catch(e){if(typeof FS=="undefined"||!(e.name==="ErrnoError"))throw e;return-e.errno}}function ___syscall_chmod(path,mode){path=bigintToI53Checked(path);try{path=SYSCALLS.getStr(path);FS.chmod(path,mode);return 0}catch(e){if(typeof FS=="undefined"||!(e.name==="ErrnoError"))throw e;return-e.errno}}function ___syscall_connect(fd,addr,addrlen,d1,d2,d3){addr=bigintToI53Checked(addr);addrlen=bigintToI53Checked(addrlen);try{var sock=getSocketFromFD(fd);var info=getSocketAddress(addr,addrlen);sock.sock_ops.connect(sock,info.addr,info.port);return 0}catch(e){if(typeof FS=="undefined"||!(e.name==="ErrnoError"))throw e;return-e.errno}}function ___syscall_dup(fd){try{var old=SYSCALLS.getStreamFromFD(fd);return FS.dupStream(old).fd}catch(e){if(typeof FS=="undefined"||!(e.name==="ErrnoError"))throw e;return-e.errno}}function ___syscall_faccessat(dirfd,path,amode,flags){path=bigintToI53Checked(path);try{path=SYSCALLS.getStr(path);path=SYSCALLS.calculateAt(dirfd,path);if(amode&~7){return-28}var lookup=FS.lookupPath(path,{follow:true});var node=lookup.node;if(!node){return-44}var perms="";if(amode&4)perms+="r";if(amode&2)perms+="w";if(amode&1)perms+="x";if(perms&&FS.nodePermissions(node,perms)){return-2}return 0}catch(e){if(typeof FS=="undefined"||!(e.name==="ErrnoError"))throw e;return-e.errno}}function ___syscall_fchmod(fd,mode){try{FS.fchmod(fd,mode);return 0}catch(e){if(typeof FS=="undefined"||!(e.name==="ErrnoError"))throw e;return-e.errno}}function ___syscall_fchownat(dirfd,path,owner,group,flags){path=bigintToI53Checked(path);try{path=SYSCALLS.getStr(path);var nofollow=flags&256;flags=flags&~256;path=SYSCALLS.calculateAt(dirfd,path);(nofollow?FS.lchown:FS.chown)(path,owner,group);return 0}catch(e){if(typeof FS=="undefined"||!(e.name==="ErrnoError"))throw e;return-e.errno}}var syscallGetVarargP=()=>{var ret=Number(HEAPU64[SYSCALLS.varargs>>3]);SYSCALLS.varargs+=8;return ret};var syscallGetVarargI=()=>{var ret=HEAP32[+SYSCALLS.varargs>>2];SYSCALLS.varargs+=4;return ret};function ___syscall_fcntl64(fd,cmd,varargs){varargs=bigintToI53Checked(varargs);SYSCALLS.varargs=varargs;try{var stream=SYSCALLS.getStreamFromFD(fd);switch(cmd){case 0:{var arg=syscallGetVarargI();if(arg<0){return-28}while(FS.streams[arg]){arg++}var newStream;newStream=FS.dupStream(stream,arg);return newStream.fd}case 1:case 2:return 0;case 3:return stream.flags;case 4:{var arg=syscallGetVarargI();stream.flags|=arg;return 0}case 5:{var arg=syscallGetVarargP();var offset=0;HEAP16[arg+offset>>1]=2;return 0}case 6:case 7:return 0}return-28}catch(e){if(typeof FS=="undefined"||!(e.name==="ErrnoError"))throw e;return-e.errno}}function ___syscall_fdatasync(fd){try{var stream=SYSCALLS.getStreamFromFD(fd);return 0}catch(e){if(typeof FS=="undefined"||!(e.name==="ErrnoError"))throw e;return-e.errno}}function ___syscall_fstat64(fd,buf){buf=bigintToI53Checked(buf);try{return SYSCALLS.writeStat(buf,FS.fstat(fd))}catch(e){if(typeof FS=="undefined"||!(e.name==="ErrnoError"))throw e;return-e.errno}}function ___syscall_ftruncate64(fd,length){length=bigintToI53Checked(length);try{if(isNaN(length))return-61;FS.ftruncate(fd,length);return 0}catch(e){if(typeof FS=="undefined"||!(e.name==="ErrnoError"))throw e;return-e.errno}}var stringToUTF8=(str,outPtr,maxBytesToWrite)=>stringToUTF8Array(str,HEAPU8,outPtr,maxBytesToWrite);function ___syscall_getcwd(buf,size){buf=bigintToI53Checked(buf);size=bigintToI53Checked(size);try{if(size===0)return-28;var cwd=FS.cwd();var cwdLengthInBytes=lengthBytesUTF8(cwd)+1;if(size>3]=BigInt(id);HEAP64[dirp+pos+8>>3]=BigInt((idx+1)*struct_size);HEAP16[dirp+pos+16>>1]=280;HEAP8[dirp+pos+18]=type;stringToUTF8(name,dirp+pos+19,256);pos+=struct_size}FS.llseek(stream,idx*struct_size,0);return pos}catch(e){if(typeof FS=="undefined"||!(e.name==="ErrnoError"))throw e;return-e.errno}}function ___syscall_getpeername(fd,addr,addrlen,d1,d2,d3){addr=bigintToI53Checked(addr);addrlen=bigintToI53Checked(addrlen);try{var sock=getSocketFromFD(fd);if(!sock.daddr){return-53}var errno=writeSockaddr(addr,sock.family,DNS.lookup_name(sock.daddr),sock.dport,addrlen);return 0}catch(e){if(typeof FS=="undefined"||!(e.name==="ErrnoError"))throw e;return-e.errno}}function ___syscall_getsockname(fd,addr,addrlen,d1,d2,d3){addr=bigintToI53Checked(addr);addrlen=bigintToI53Checked(addrlen);try{var sock=getSocketFromFD(fd);var errno=writeSockaddr(addr,sock.family,DNS.lookup_name(sock.saddr||"0.0.0.0"),sock.sport,addrlen);return 0}catch(e){if(typeof FS=="undefined"||!(e.name==="ErrnoError"))throw e;return-e.errno}}function ___syscall_getsockopt(fd,level,optname,optval,optlen,d1){optval=bigintToI53Checked(optval);optlen=bigintToI53Checked(optlen);try{var sock=getSocketFromFD(fd);if(level===1){if(optname===4){HEAP32[optval>>2]=sock.error;HEAP32[optlen>>2]=4;sock.error=null;return 0}}return-50}catch(e){if(typeof FS=="undefined"||!(e.name==="ErrnoError"))throw e;return-e.errno}}function ___syscall_ioctl(fd,op,varargs){varargs=bigintToI53Checked(varargs);SYSCALLS.varargs=varargs;try{var stream=SYSCALLS.getStreamFromFD(fd);switch(op){case 21509:{if(!stream.tty)return-59;return 0}case 21505:{if(!stream.tty)return-59;if(stream.tty.ops.ioctl_tcgets){var termios=stream.tty.ops.ioctl_tcgets(stream);var argp=syscallGetVarargP();HEAP32[argp>>2]=termios.c_iflag||0;HEAP32[argp+4>>2]=termios.c_oflag||0;HEAP32[argp+8>>2]=termios.c_cflag||0;HEAP32[argp+12>>2]=termios.c_lflag||0;for(var i=0;i<32;i++){HEAP8[argp+i+17]=termios.c_cc[i]||0}return 0}return 0}case 21510:case 21511:case 21512:{if(!stream.tty)return-59;return 0}case 21506:case 21507:case 21508:{if(!stream.tty)return-59;if(stream.tty.ops.ioctl_tcsets){var argp=syscallGetVarargP();var c_iflag=HEAP32[argp>>2];var c_oflag=HEAP32[argp+4>>2];var c_cflag=HEAP32[argp+8>>2];var c_lflag=HEAP32[argp+12>>2];var c_cc=[];for(var i=0;i<32;i++){c_cc.push(HEAP8[argp+i+17])}return stream.tty.ops.ioctl_tcsets(stream.tty,op,{c_iflag,c_oflag,c_cflag,c_lflag,c_cc})}return 0}case 21519:{if(!stream.tty)return-59;var argp=syscallGetVarargP();HEAP32[argp>>2]=0;return 0}case 21520:{if(!stream.tty)return-59;return-28}case 21537:case 21531:{var argp=syscallGetVarargP();return FS.ioctl(stream,op,argp)}case 21523:{if(!stream.tty)return-59;if(stream.tty.ops.ioctl_tiocgwinsz){var winsize=stream.tty.ops.ioctl_tiocgwinsz(stream.tty);var argp=syscallGetVarargP();HEAP16[argp>>1]=winsize[0];HEAP16[argp+2>>1]=winsize[1]}return 0}case 21524:{if(!stream.tty)return-59;return 0}case 21515:{if(!stream.tty)return-59;return 0}default:return-28}}catch(e){if(typeof FS=="undefined"||!(e.name==="ErrnoError"))throw e;return-e.errno}}function ___syscall_listen(fd,backlog){try{var sock=getSocketFromFD(fd);sock.sock_ops.listen(sock,backlog);return 0}catch(e){if(typeof FS=="undefined"||!(e.name==="ErrnoError"))throw e;return-e.errno}}function ___syscall_lstat64(path,buf){path=bigintToI53Checked(path);buf=bigintToI53Checked(buf);try{path=SYSCALLS.getStr(path);return SYSCALLS.writeStat(buf,FS.lstat(path))}catch(e){if(typeof FS=="undefined"||!(e.name==="ErrnoError"))throw e;return-e.errno}}function ___syscall_mkdirat(dirfd,path,mode){path=bigintToI53Checked(path);try{path=SYSCALLS.getStr(path);path=SYSCALLS.calculateAt(dirfd,path);FS.mkdir(path,mode,0);return 0}catch(e){if(typeof FS=="undefined"||!(e.name==="ErrnoError"))throw e;return-e.errno}}function ___syscall_newfstatat(dirfd,path,buf,flags){path=bigintToI53Checked(path);buf=bigintToI53Checked(buf);try{path=SYSCALLS.getStr(path);var nofollow=flags&256;var allowEmpty=flags&4096;flags=flags&~6400;path=SYSCALLS.calculateAt(dirfd,path,allowEmpty);return SYSCALLS.writeStat(buf,nofollow?FS.lstat(path):FS.stat(path))}catch(e){if(typeof FS=="undefined"||!(e.name==="ErrnoError"))throw e;return-e.errno}}function ___syscall_openat(dirfd,path,flags,varargs){path=bigintToI53Checked(path);varargs=bigintToI53Checked(varargs);SYSCALLS.varargs=varargs;try{path=SYSCALLS.getStr(path);path=SYSCALLS.calculateAt(dirfd,path);var mode=varargs?syscallGetVarargI():0;return FS.open(path,flags,mode).fd}catch(e){if(typeof FS=="undefined"||!(e.name==="ErrnoError"))throw e;return-e.errno}}var PIPEFS={BUCKET_BUFFER_SIZE:8192,mount(mount){return FS.createNode(null,"/",16384|511,0)},createPipe(){var pipe={buckets:[],refcnt:2,timestamp:new Date,readableHandlers:[],registerReadableHandler:callback=>{callback.registerCleanupFunc(()=>{const i=pipe.readableHandlers.indexOf(callback);if(i!==-1)pipe.readableHandlers.splice(i,1)});pipe.readableHandlers.push(callback)},notifyReadableHandlers:()=>{while(pipe.readableHandlers.length>0){const cb=pipe.readableHandlers.shift();if(cb)cb(64|1)}pipe.readableHandlers=[]}};pipe.buckets.push({buffer:new Uint8Array(PIPEFS.BUCKET_BUFFER_SIZE),offset:0,roffset:0});var rName=PIPEFS.nextname();var wName=PIPEFS.nextname();var rNode=FS.createNode(PIPEFS.root,rName,4096,0);var wNode=FS.createNode(PIPEFS.root,wName,4096,0);rNode.pipe=pipe;wNode.pipe=pipe;var readableStream=FS.createStream({path:rName,node:rNode,flags:0,seekable:false,stream_ops:PIPEFS.stream_ops});rNode.stream=readableStream;var writableStream=FS.createStream({path:wName,node:wNode,flags:1,seekable:false,stream_ops:PIPEFS.stream_ops});wNode.stream=writableStream;return{readable_fd:readableStream.fd,writable_fd:writableStream.fd}},stream_ops:{getattr(stream){var node=stream.node;var timestamp=node.pipe.timestamp;return{dev:14,ino:node.id,mode:4480,nlink:1,uid:0,gid:0,rdev:0,size:0,atime:timestamp,mtime:timestamp,ctime:timestamp,blksize:4096,blocks:0}},poll(stream,timeout,notifyCallback){var pipe=stream.node.pipe;if((stream.flags&2097155)===1){return 256|4}for(var bucket of pipe.buckets){if(bucket.offset-bucket.roffset>0){return 64|1}}if(notifyCallback)pipe.registerReadableHandler(notifyCallback);return 0},dup(stream){stream.node.pipe.refcnt++},ioctl(stream,request,varargs){return 28},fsync(stream){return 28},read(stream,buffer,offset,length,position){var pipe=stream.node.pipe;var currentLength=0;for(var bucket of pipe.buckets){currentLength+=bucket.offset-bucket.roffset}var data=buffer.subarray(offset,offset+length);if(length<=0){return 0}if(currentLength==0){throw new FS.ErrnoError(6)}var toRead=Math.min(currentLength,length);var totalRead=toRead;var toRemove=0;for(var bucket of pipe.buckets){var bucketSize=bucket.offset-bucket.roffset;if(toRead<=bucketSize){var tmpSlice=bucket.buffer.subarray(bucket.roffset,bucket.offset);if(toRead=dataLen){currBucket.buffer.set(data,currBucket.offset);currBucket.offset+=dataLen;pipe.notifyReadableHandlers();return dataLen}else if(freeBytesInCurrBuffer>0){currBucket.buffer.set(data.subarray(0,freeBytesInCurrBuffer),currBucket.offset);currBucket.offset+=freeBytesInCurrBuffer;data=data.subarray(freeBytesInCurrBuffer,data.byteLength)}var numBuckets=data.byteLength/PIPEFS.BUCKET_BUFFER_SIZE|0;var remElements=data.byteLength%PIPEFS.BUCKET_BUFFER_SIZE;for(var i=0;i0){var newBucket={buffer:new Uint8Array(PIPEFS.BUCKET_BUFFER_SIZE),offset:data.byteLength,roffset:0};pipe.buckets.push(newBucket);newBucket.buffer.set(data)}pipe.notifyReadableHandlers();return dataLen},close(stream){var pipe=stream.node.pipe;pipe.refcnt--;if(pipe.refcnt===0){pipe.buckets=null}}},nextname(){if(!PIPEFS.nextname.current){PIPEFS.nextname.current=0}return"pipe["+PIPEFS.nextname.current+++"]"}};function ___syscall_pipe2(fdPtr,flags){fdPtr=bigintToI53Checked(fdPtr);try{if(fdPtr==0){throw new FS.ErrnoError(21)}var validFlags=524288|2048;if(flags&~validFlags){throw new FS.ErrnoError(138)}var res=PIPEFS.createPipe();if(flags&2048){FS.getStream(res.readable_fd).flags|=2048;FS.getStream(res.writable_fd).flags|=2048}HEAP32[fdPtr>>2]=res.readable_fd;HEAP32[fdPtr+4>>2]=res.writable_fd;return 0}catch(e){if(typeof FS=="undefined"||!(e.name==="ErrnoError"))throw e;return-e.errno}}var ___syscall_poll=function(fds,nfds,timeout){let innerFunc=()=>{fds=bigintToI53Checked(fds);try{const isAsyncContext=true;var resolve;var promise=new Promise(resolve_=>{resolve=resolve_});var cleanupFuncs=[];var notifyDone=false;function asyncPollComplete(count){if(notifyDone){return}notifyDone=true;cleanupFuncs.forEach(cb=>cb());resolve(count)}function makeNotifyCallback(stream,pollfd){var cb=flags=>{if(notifyDone){return}var events=HEAP16[pollfd+4>>1];flags&=events|8|16;HEAP16[pollfd+6>>1]=flags;asyncPollComplete(1)};cb.registerCleanupFunc=f=>{if(f)cleanupFuncs.push(f)};return cb}if(isAsyncContext){if(timeout>0){var t=setTimeout(()=>{asyncPollComplete(0)},timeout);cleanupFuncs.push(()=>clearTimeout(t))}}var count=0;for(var i=0;i>2];var events=HEAP16[pollfd+4>>1];var flags=32;var stream=FS.getStream(fd);if(stream){if(stream.stream_ops.poll){if(isAsyncContext&&timeout){flags=stream.stream_ops.poll(stream,timeout,makeNotifyCallback(stream,pollfd))}else flags=stream.stream_ops.poll(stream,-1)}else{flags=5}}flags&=events|8|16;if(flags)count++;HEAP16[pollfd+6>>1]=flags}if(isAsyncContext){if(count||!timeout){asyncPollComplete(count)}return promise}return count}catch(e){if(typeof FS=="undefined"||!(e.name==="ErrnoError"))throw e;return-e.errno}};return Asyncify.handleAsync(innerFunc)};___syscall_poll.isAsync=true;function ___syscall_readlinkat(dirfd,path,buf,bufsize){path=bigintToI53Checked(path);buf=bigintToI53Checked(buf);bufsize=bigintToI53Checked(bufsize);try{path=SYSCALLS.getStr(path);path=SYSCALLS.calculateAt(dirfd,path);if(bufsize<=0)return-28;var ret=FS.readlink(path);var len=Math.min(bufsize,lengthBytesUTF8(ret));var endChar=HEAP8[buf+len];stringToUTF8(ret,buf,bufsize+1);HEAP8[buf+len]=endChar;return len}catch(e){if(typeof FS=="undefined"||!(e.name==="ErrnoError"))throw e;return-e.errno}}function ___syscall_recvfrom(fd,buf,len,flags,addr,addrlen){buf=bigintToI53Checked(buf);len=bigintToI53Checked(len);addr=bigintToI53Checked(addr);addrlen=bigintToI53Checked(addrlen);try{var sock=getSocketFromFD(fd);var msg=sock.sock_ops.recvmsg(sock,len);if(!msg)return 0;if(addr){var errno=writeSockaddr(addr,sock.family,DNS.lookup_name(msg.addr),msg.port,addrlen)}HEAPU8.set(msg.buffer,buf);return msg.buffer.byteLength}catch(e){if(typeof FS=="undefined"||!(e.name==="ErrnoError"))throw e;return-e.errno}}function ___syscall_renameat(olddirfd,oldpath,newdirfd,newpath){oldpath=bigintToI53Checked(oldpath);newpath=bigintToI53Checked(newpath);try{oldpath=SYSCALLS.getStr(oldpath);newpath=SYSCALLS.getStr(newpath);oldpath=SYSCALLS.calculateAt(olddirfd,oldpath);newpath=SYSCALLS.calculateAt(newdirfd,newpath);FS.rename(oldpath,newpath);return 0}catch(e){if(typeof FS=="undefined"||!(e.name==="ErrnoError"))throw e;return-e.errno}}function ___syscall_rmdir(path){path=bigintToI53Checked(path);try{path=SYSCALLS.getStr(path);FS.rmdir(path);return 0}catch(e){if(typeof FS=="undefined"||!(e.name==="ErrnoError"))throw e;return-e.errno}}function ___syscall_sendto(fd,message,length,flags,addr,addr_len){message=bigintToI53Checked(message);length=bigintToI53Checked(length);addr=bigintToI53Checked(addr);addr_len=bigintToI53Checked(addr_len);try{var sock=getSocketFromFD(fd);if(!addr){return FS.write(sock.stream,HEAP8,message,length)}var dest=getSocketAddress(addr,addr_len);return sock.sock_ops.sendmsg(sock,HEAP8,message,length,dest.addr,dest.port)}catch(e){if(typeof FS=="undefined"||!(e.name==="ErrnoError"))throw e;return-e.errno}}function ___syscall_socket(domain,type,protocol){try{var sock=SOCKFS.createSocket(domain,type,protocol);return sock.stream.fd}catch(e){if(typeof FS=="undefined"||!(e.name==="ErrnoError"))throw e;return-e.errno}}function ___syscall_stat64(path,buf){path=bigintToI53Checked(path);buf=bigintToI53Checked(buf);try{path=SYSCALLS.getStr(path);return SYSCALLS.writeStat(buf,FS.stat(path))}catch(e){if(typeof FS=="undefined"||!(e.name==="ErrnoError"))throw e;return-e.errno}}function ___syscall_statfs64(path,size,buf){path=bigintToI53Checked(path);size=bigintToI53Checked(size);buf=bigintToI53Checked(buf);try{SYSCALLS.writeStatFs(buf,FS.statfs(SYSCALLS.getStr(path)));return 0}catch(e){if(typeof FS=="undefined"||!(e.name==="ErrnoError"))throw e;return-e.errno}}function ___syscall_symlinkat(target,dirfd,linkpath){target=bigintToI53Checked(target);linkpath=bigintToI53Checked(linkpath);try{target=SYSCALLS.getStr(target);linkpath=SYSCALLS.getStr(linkpath);linkpath=SYSCALLS.calculateAt(dirfd,linkpath);FS.symlink(target,linkpath);return 0}catch(e){if(typeof FS=="undefined"||!(e.name==="ErrnoError"))throw e;return-e.errno}}function ___syscall_unlinkat(dirfd,path,flags){path=bigintToI53Checked(path);try{path=SYSCALLS.getStr(path);path=SYSCALLS.calculateAt(dirfd,path);if(!flags){FS.unlink(path)}else if(flags===512){FS.rmdir(path)}else{return-28}return 0}catch(e){if(typeof FS=="undefined"||!(e.name==="ErrnoError"))throw e;return-e.errno}}var readI53FromI64=ptr=>HEAPU32[ptr>>2]+HEAP32[ptr+4>>2]*4294967296;function ___syscall_utimensat(dirfd,path,times,flags){path=bigintToI53Checked(path);times=bigintToI53Checked(times);try{path=SYSCALLS.getStr(path);path=SYSCALLS.calculateAt(dirfd,path,true);var now=Date.now(),atime,mtime;if(!times){atime=now;mtime=now}else{var seconds=readI53FromI64(times);var nanoseconds=HEAP32[times+8>>2];if(nanoseconds==1073741823){atime=now}else if(nanoseconds==1073741822){atime=null}else{atime=seconds*1e3+nanoseconds/(1e3*1e3)}times+=16;seconds=readI53FromI64(times);nanoseconds=HEAP32[times+8>>2];if(nanoseconds==1073741823){mtime=now}else if(nanoseconds==1073741822){mtime=null}else{mtime=seconds*1e3+nanoseconds/(1e3*1e3)}}if((mtime??atime)!==null){FS.utime(path,atime,mtime)}return 0}catch(e){if(typeof FS=="undefined"||!(e.name==="ErrnoError"))throw e;return-e.errno}}var __abort_js=()=>abort("");function __emscripten_lookup_name(name){name=bigintToI53Checked(name);var nameString=UTF8ToString(name);return inetPton4(DNS.lookup_name(nameString))}var runtimeKeepaliveCounter=0;var __emscripten_runtime_keepalive_clear=()=>{noExitRuntime=false;runtimeKeepaliveCounter=0};var __emscripten_throw_longjmp=()=>{throw new EmscriptenSjLj};function __gmtime_js(time,tmPtr){time=bigintToI53Checked(time);tmPtr=bigintToI53Checked(tmPtr);var date=new Date(time*1e3);HEAP32[tmPtr>>2]=date.getUTCSeconds();HEAP32[tmPtr+4>>2]=date.getUTCMinutes();HEAP32[tmPtr+8>>2]=date.getUTCHours();HEAP32[tmPtr+12>>2]=date.getUTCDate();HEAP32[tmPtr+16>>2]=date.getUTCMonth();HEAP32[tmPtr+20>>2]=date.getUTCFullYear()-1900;HEAP32[tmPtr+24>>2]=date.getUTCDay();var start=Date.UTC(date.getUTCFullYear(),0,1,0,0,0,0);var yday=(date.getTime()-start)/(1e3*60*60*24)|0;HEAP32[tmPtr+28>>2]=yday}var isLeapYear=year=>year%4===0&&(year%100!==0||year%400===0);var MONTH_DAYS_LEAP_CUMULATIVE=[0,31,60,91,121,152,182,213,244,274,305,335];var MONTH_DAYS_REGULAR_CUMULATIVE=[0,31,59,90,120,151,181,212,243,273,304,334];var ydayFromDate=date=>{var leap=isLeapYear(date.getFullYear());var monthDaysCumulative=leap?MONTH_DAYS_LEAP_CUMULATIVE:MONTH_DAYS_REGULAR_CUMULATIVE;var yday=monthDaysCumulative[date.getMonth()]+date.getDate()-1;return yday};function __localtime_js(time,tmPtr){time=bigintToI53Checked(time);tmPtr=bigintToI53Checked(tmPtr);var date=new Date(time*1e3);HEAP32[tmPtr>>2]=date.getSeconds();HEAP32[tmPtr+4>>2]=date.getMinutes();HEAP32[tmPtr+8>>2]=date.getHours();HEAP32[tmPtr+12>>2]=date.getDate();HEAP32[tmPtr+16>>2]=date.getMonth();HEAP32[tmPtr+20>>2]=date.getFullYear()-1900;HEAP32[tmPtr+24>>2]=date.getDay();var yday=ydayFromDate(date)|0;HEAP32[tmPtr+28>>2]=yday;HEAP64[tmPtr+40>>3]=BigInt(-(date.getTimezoneOffset()*60));var start=new Date(date.getFullYear(),0,1);var summerOffset=new Date(date.getFullYear(),6,1).getTimezoneOffset();var winterOffset=start.getTimezoneOffset();var dst=(summerOffset!=winterOffset&&date.getTimezoneOffset()==Math.min(winterOffset,summerOffset))|0;HEAP32[tmPtr+32>>2]=dst}var __mktime_js=function(tmPtr){tmPtr=bigintToI53Checked(tmPtr);var ret=(()=>{var date=new Date(HEAP32[tmPtr+20>>2]+1900,HEAP32[tmPtr+16>>2],HEAP32[tmPtr+12>>2],HEAP32[tmPtr+8>>2],HEAP32[tmPtr+4>>2],HEAP32[tmPtr>>2],0);var dst=HEAP32[tmPtr+32>>2];var guessedOffset=date.getTimezoneOffset();var start=new Date(date.getFullYear(),0,1);var summerOffset=new Date(date.getFullYear(),6,1).getTimezoneOffset();var winterOffset=start.getTimezoneOffset();var dstOffset=Math.min(winterOffset,summerOffset);if(dst<0){HEAP32[tmPtr+32>>2]=Number(summerOffset!=winterOffset&&dstOffset==guessedOffset)}else if(dst>0!=(dstOffset==guessedOffset)){var nonDstOffset=Math.max(winterOffset,summerOffset);var trueOffset=dst>0?dstOffset:nonDstOffset;date.setTime(date.getTime()+(trueOffset-guessedOffset)*6e4)}HEAP32[tmPtr+24>>2]=date.getDay();var yday=ydayFromDate(date)|0;HEAP32[tmPtr+28>>2]=yday;HEAP32[tmPtr>>2]=date.getSeconds();HEAP32[tmPtr+4>>2]=date.getMinutes();HEAP32[tmPtr+8>>2]=date.getHours();HEAP32[tmPtr+12>>2]=date.getDate();HEAP32[tmPtr+16>>2]=date.getMonth();HEAP32[tmPtr+20>>2]=date.getYear();var timeMs=date.getTime();if(isNaN(timeMs)){return-1}return timeMs/1e3})();return BigInt(ret)};function __mmap_js(len,prot,flags,fd,offset,allocated,addr){len=bigintToI53Checked(len);offset=bigintToI53Checked(offset);allocated=bigintToI53Checked(allocated);addr=bigintToI53Checked(addr);try{var stream=SYSCALLS.getStreamFromFD(fd);var res=FS.mmap(stream,len,offset,prot,flags);var ptr=res.ptr;HEAP32[allocated>>2]=res.allocated;HEAPU64[addr>>3]=BigInt(ptr);return 0}catch(e){if(typeof FS=="undefined"||!(e.name==="ErrnoError"))throw e;return-e.errno}}function __munmap_js(addr,len,prot,flags,fd,offset){addr=bigintToI53Checked(addr);len=bigintToI53Checked(len);offset=bigintToI53Checked(offset);try{var stream=SYSCALLS.getStreamFromFD(fd);if(prot&2){SYSCALLS.doMsync(addr,stream,len,flags,offset)}}catch(e){if(typeof FS=="undefined"||!(e.name==="ErrnoError"))throw e;return-e.errno}}var timers={};var handleException=e=>{if(e instanceof ExitStatus||e=="unwind"){return EXITSTATUS}quit_(1,e)};var keepRuntimeAlive=()=>noExitRuntime||runtimeKeepaliveCounter>0;var _proc_exit=code=>{EXITSTATUS=code;if(!keepRuntimeAlive()){Module["onExit"]?.(code);ABORT=true}quit_(code,new ExitStatus(code))};var exitJS=(status,implicit)=>{EXITSTATUS=status;_proc_exit(status)};var _exit=exitJS;var maybeExit=()=>{if(!keepRuntimeAlive()){try{_exit(EXITSTATUS)}catch(e){handleException(e)}}};var callUserCallback=func=>{if(ABORT){return}try{return func()}catch(e){handleException(e)}finally{maybeExit()}};var _emscripten_get_now=()=>performance.now();var __setitimer_js=(which,timeout_ms)=>{if(timers[which]){clearTimeout(timers[which].id);delete timers[which]}if(!timeout_ms)return 0;var id=setTimeout(()=>{delete timers[which];callUserCallback(()=>__emscripten_timeout(which,_emscripten_get_now()))},timeout_ms);timers[which]={id,timeout_ms};return 0};var __tzset_js=function(timezone,daylight,std_name,dst_name){timezone=bigintToI53Checked(timezone);daylight=bigintToI53Checked(daylight);std_name=bigintToI53Checked(std_name);dst_name=bigintToI53Checked(dst_name);var currentYear=(new Date).getFullYear();var winter=new Date(currentYear,0,1);var summer=new Date(currentYear,6,1);var winterOffset=winter.getTimezoneOffset();var summerOffset=summer.getTimezoneOffset();var stdTimezoneOffset=Math.max(winterOffset,summerOffset);HEAPU64[timezone>>3]=BigInt(stdTimezoneOffset*60);HEAP32[daylight>>2]=Number(winterOffset!=summerOffset);var extractZone=timezoneOffset=>{var sign=timezoneOffset>=0?"-":"+";var absOffset=Math.abs(timezoneOffset);var hours=String(Math.floor(absOffset/60)).padStart(2,"0");var minutes=String(absOffset%60).padStart(2,"0");return`UTC${sign}${hours}${minutes}`};var winterName=extractZone(winterOffset);var summerName=extractZone(summerOffset);if(summerOffsetDate.now();var nowIsMonotonic=1;var checkWasiClock=clock_id=>clock_id>=0&&clock_id<=3;function _clock_time_get(clk_id,ignored_precision,ptime){ignored_precision=bigintToI53Checked(ignored_precision);ptime=bigintToI53Checked(ptime);if(!checkWasiClock(clk_id)){return 28}var now;if(clk_id===0){now=_emscripten_date_now()}else if(nowIsMonotonic){now=_emscripten_get_now()}else{return 52}var nsec=Math.round(now*1e3*1e3);HEAP64[ptime>>3]=BigInt(nsec);return 0}var _emscripten_force_exit=status=>{__emscripten_runtime_keepalive_clear();_exit(status)};var getHeapMax=()=>2147483648;var _emscripten_get_heap_max=()=>BigInt(getHeapMax());var growMemory=size=>{var oldHeapSize=wasmMemory.buffer.byteLength;var pages=(size-oldHeapSize+65535)/65536|0;try{wasmMemory.grow(pages);updateMemoryViews();return 1}catch(e){}};function _emscripten_resize_heap(requestedSize){requestedSize=bigintToI53Checked(requestedSize);var oldSize=HEAPU8.length;var maxHeapSize=getHeapMax();if(requestedSize>maxHeapSize){return false}for(var cutDown=1;cutDown<=4;cutDown*=2){var overGrownHeapSize=oldSize*(1+.2/cutDown);overGrownHeapSize=Math.min(overGrownHeapSize,requestedSize+100663296);var newSize=Math.min(maxHeapSize,alignMemory(Math.max(requestedSize,overGrownHeapSize),65536));var replacement=growMemory(newSize);if(replacement){return true}}return false}var ENV={};var getExecutableName=()=>thisProgram||"./this.program";var getEnvStrings=()=>{if(!getEnvStrings.strings){var lang=(globalThis.navigator?.language??"C").replace("-","_")+".UTF-8";var env={USER:"web_user",LOGNAME:"web_user",PATH:"/",PWD:"/",HOME:"/home/web_user",LANG:lang,_:getExecutableName()};for(var x in ENV){if(ENV[x]===undefined)delete env[x];else env[x]=ENV[x]}var strings=[];for(var x in env){strings.push(`${x}=${env[x]}`)}getEnvStrings.strings=strings}return getEnvStrings.strings};function _environ_get(__environ,environ_buf){__environ=bigintToI53Checked(__environ);environ_buf=bigintToI53Checked(environ_buf);var bufSize=0;var envp=0;for(var string of getEnvStrings()){var ptr=environ_buf+bufSize;HEAPU64[__environ+envp>>3]=BigInt(ptr);bufSize+=stringToUTF8(string,ptr,Infinity)+1;envp+=8}return 0}function _environ_sizes_get(penviron_count,penviron_buf_size){penviron_count=bigintToI53Checked(penviron_count);penviron_buf_size=bigintToI53Checked(penviron_buf_size);var strings=getEnvStrings();HEAPU64[penviron_count>>3]=BigInt(strings.length);var bufSize=0;for(var string of strings){bufSize+=lengthBytesUTF8(string)+1}HEAPU64[penviron_buf_size>>3]=BigInt(bufSize);return 0}function _fd_close(fd){try{var stream=SYSCALLS.getStreamFromFD(fd);FS.close(stream);return 0}catch(e){if(typeof FS=="undefined"||!(e.name==="ErrnoError"))throw e;return e.errno}}function _fd_fdstat_get(fd,pbuf){pbuf=bigintToI53Checked(pbuf);try{var rightsBase=0;var rightsInheriting=0;var flags=0;{var stream=SYSCALLS.getStreamFromFD(fd);var type=stream.tty?2:FS.isDir(stream.mode)?3:FS.isLink(stream.mode)?7:4}HEAP8[pbuf]=type;HEAP16[pbuf+2>>1]=flags;HEAP64[pbuf+8>>3]=BigInt(rightsBase);HEAP64[pbuf+16>>3]=BigInt(rightsInheriting);return 0}catch(e){if(typeof FS=="undefined"||!(e.name==="ErrnoError"))throw e;return e.errno}}var doReadv=(stream,iov,iovcnt,offset)=>{var ret=0;for(var i=0;i>3]);var len=Number(HEAPU64[iov+8>>3]);iov+=16;var curr=FS.read(stream,HEAP8,ptr,len,offset);if(curr<0)return-1;ret+=curr;if(curr>3]=BigInt(num);return 0}catch(e){if(typeof FS=="undefined"||!(e.name==="ErrnoError"))throw e;return e.errno}}function _fd_seek(fd,offset,whence,newOffset){offset=bigintToI53Checked(offset);newOffset=bigintToI53Checked(newOffset);try{if(isNaN(offset))return 61;var stream=SYSCALLS.getStreamFromFD(fd);FS.llseek(stream,offset,whence);HEAP64[newOffset>>3]=BigInt(stream.position);if(stream.getdents&&offset===0&&whence===0)stream.getdents=null;return 0}catch(e){if(typeof FS=="undefined"||!(e.name==="ErrnoError"))throw e;return e.errno}}var _fd_sync=function(fd){let innerFunc=()=>{try{var stream=SYSCALLS.getStreamFromFD(fd);var rtn=stream.stream_ops?.fsync?.(stream);return new Promise(resolve=>{var mount=stream.node.mount;if(mount?.type.syncfs){mount.type.syncfs(mount,false,err=>resolve(err?29:0))}else{resolve(rtn)}})}catch(e){if(typeof FS=="undefined"||!(e.name==="ErrnoError"))throw e;return e.errno}};return Asyncify.handleAsync(innerFunc)};_fd_sync.isAsync=true;var doWritev=(stream,iov,iovcnt,offset)=>{var ret=0;for(var i=0;i>3]);var len=Number(HEAPU64[iov+8>>3]);iov+=16;var curr=FS.write(stream,HEAP8,ptr,len,offset);if(curr<0)return-1;ret+=curr;if(curr>3]=BigInt(num);return 0}catch(e){if(typeof FS=="undefined"||!(e.name==="ErrnoError"))throw e;return e.errno}}function _getaddrinfo(node,service,hint,out){node=bigintToI53Checked(node);service=bigintToI53Checked(service);hint=bigintToI53Checked(hint);out=bigintToI53Checked(out);var addr=0;var port=0;var flags=0;var family=0;var type=0;var proto=0;var ai;function allocaddrinfo(family,type,proto,canon,addr,port){var sa,salen,ai;var errno;salen=family===10?28:16;addr=family===10?inetNtop6(addr):inetNtop4(addr);sa=_malloc(salen);errno=writeSockaddr(sa,family,addr,port);ai=_malloc(48);HEAP32[ai+4>>2]=family;HEAP32[ai+8>>2]=type;HEAP32[ai+12>>2]=proto;HEAPU64[ai+32>>3]=BigInt(canon);HEAPU64[ai+24>>3]=BigInt(sa);if(family===10){HEAP32[ai+16>>2]=28}else{HEAP32[ai+16>>2]=16}HEAP32[ai+40>>2]=0;return ai}if(hint){flags=HEAP32[hint>>2];family=HEAP32[hint+4>>2];type=HEAP32[hint+8>>2];proto=HEAP32[hint+12>>2]}if(type&&!proto){proto=type===2?17:6}if(!type&&proto){type=proto===17?2:1}if(proto===0){proto=6}if(type===0){type=1}if(!node&&!service){return-2}if(flags&~(1|2|4|1024|8|16|32)){return-1}if(hint!==0&&HEAP32[hint>>2]&2&&!node){return-1}if(flags&32){return-2}if(type!==0&&type!==1&&type!==2){return-7}if(family!==0&&family!==2&&family!==10){return-6}if(service){service=UTF8ToString(service);port=parseInt(service,10);if(isNaN(port)){if(flags&1024){return-2}return-8}}if(!node){if(family===0){family=2}if((flags&1)===0){if(family===2){addr=_htonl(2130706433)}else{addr=[0,0,0,_htonl(1)]}}ai=allocaddrinfo(family,type,proto,null,addr,port);HEAPU64[out>>3]=BigInt(ai);return 0}node=UTF8ToString(node);addr=inetPton4(node);if(addr!==null){if(family===0||family===2){family=2}else if(family===10&&flags&8){addr=[0,0,_htonl(65535),addr];family=10}else{return-2}}else{addr=inetPton6(node);if(addr!==null){if(family===0||family===10){family=10}else{return-2}}}if(addr!=null){ai=allocaddrinfo(family,type,proto,node,addr,port);HEAPU64[out>>3]=BigInt(ai);return 0}if(flags&4){return-2}node=DNS.lookup_name(node);addr=inetPton4(node);if(family===0){family=2}else if(family===10){addr=[0,0,_htonl(65535),addr]}ai=allocaddrinfo(family,type,proto,null,addr,port);HEAPU64[out>>3]=BigInt(ai);return 0}function _getcontext(...args){abort("missing function: getcontext")}_getcontext.stub=true;function _getdtablesize(...args){abort("missing function: getdtablesize")}_getdtablesize.stub=true;function _getnameinfo(sa,salen,node,nodelen,serv,servlen,flags){sa=bigintToI53Checked(sa);node=bigintToI53Checked(node);serv=bigintToI53Checked(serv);var info=readSockaddr(sa,salen);if(info.errno){return-6}var port=info.port;var addr=info.addr;var overflowed=false;if(node&&nodelen){var lookup;if(flags&1||!(lookup=DNS.lookup_addr(addr))){if(flags&8){return-2}}else{addr=lookup}var numBytesWrittenExclNull=stringToUTF8(addr,node,nodelen);if(numBytesWrittenExclNull+1>=nodelen){overflowed=true}}if(serv&&servlen){port=""+port;var numBytesWrittenExclNull=stringToUTF8(port,serv,servlen);if(numBytesWrittenExclNull+1>=servlen){overflowed=true}}if(overflowed){return-12}return 0}var Protocols={list:[],map:{}};var stringToAscii=(str,buffer)=>{for(var i=0;i{function allocprotoent(name,proto,aliases){var nameBuf=_malloc(name.length+1);stringToAscii(name,nameBuf);var j=0;var length=aliases.length;var aliasListBuf=_malloc((length+1)*4);for(var i=0;i>3]=BigInt(aliasBuf)}HEAPU64[aliasListBuf+j>>3]=0n;var pe=_malloc(24);HEAPU64[pe>>3]=BigInt(nameBuf);HEAPU64[pe+8>>3]=BigInt(aliasListBuf);HEAP32[pe+16>>2]=proto;return pe}var list=Protocols.list;var map=Protocols.map;if(list.length===0){var entry=allocprotoent("tcp",6,["TCP"]);list.push(entry);map["tcp"]=map["6"]=entry;entry=allocprotoent("udp",17,["UDP"]);list.push(entry);map["udp"]=map["17"]=entry}_setprotoent.index=0};var _getprotobyname=function(name){name=bigintToI53Checked(name);var ret=(()=>{name=UTF8ToString(name);_setprotoent(true);var result=Protocols.map[name];return result})();return BigInt(ret)};var _getprotobynumber=function(number){var ret=(()=>{_setprotoent(true);var result=Protocols.map[number];return result})();return BigInt(ret)};function _makecontext(...args){abort("missing function: makecontext")}_makecontext.stub=true;function _posix_spawnp(...args){abort("missing function: posix_spawnp")}_posix_spawnp.stub=true;var arraySum=(array,index)=>{var sum=0;for(var i=0;i<=index;sum+=array[i++]){}return sum};var MONTH_DAYS_LEAP=[31,29,31,30,31,30,31,31,30,31,30,31];var MONTH_DAYS_REGULAR=[31,28,31,30,31,30,31,31,30,31,30,31];var addDays=(date,days)=>{var newDate=new Date(date.getTime());while(days>0){var leap=isLeapYear(newDate.getFullYear());var currentMonth=newDate.getMonth();var daysInCurrentMonth=(leap?MONTH_DAYS_LEAP:MONTH_DAYS_REGULAR)[currentMonth];if(days>daysInCurrentMonth-newDate.getDate()){days-=daysInCurrentMonth-newDate.getDate()+1;newDate.setDate(1);if(currentMonth<11){newDate.setMonth(currentMonth+1)}else{newDate.setMonth(0);newDate.setFullYear(newDate.getFullYear()+1)}}else{newDate.setDate(newDate.getDate()+days);return newDate}}return newDate};var _strptime=function(buf,format,tm){buf=bigintToI53Checked(buf);format=bigintToI53Checked(format);tm=bigintToI53Checked(tm);var ret=(()=>{var pattern=UTF8ToString(format);var SPECIAL_CHARS="\\!@#$^&*()+=-[]/{}|:<>?,.";for(var i=0,ii=SPECIAL_CHARS.length;iEQUIVALENT_MATCHERS[c]||m).replace(/%(.)/g,(_,c)=>{let pat=DATE_PATTERNS[c];if(pat){capture.push(c);return`(${pat})`}else{return c}}).replace(/\s+/g,"\\s*");var matches=new RegExp("^"+pattern_out,"i").exec(UTF8ToString(buf));function initDate(){function fixup(value,min,max){return typeof value!="number"||isNaN(value)?min:value>=min?value<=max?value:max:min}return{year:fixup(HEAP32[tm+20>>2]+1900,1970,9999),month:fixup(HEAP32[tm+16>>2],0,11),day:fixup(HEAP32[tm+12>>2],1,31),hour:fixup(HEAP32[tm+8>>2],0,23),min:fixup(HEAP32[tm+4>>2],0,59),sec:fixup(HEAP32[tm>>2],0,59),gmtoff:0}}if(matches){var date=initDate();var value;var getMatch=symbol=>{var pos=capture.indexOf(symbol);if(pos>=0){return matches[pos+1]}return};if(value=getMatch("S")){date.sec=Number(value)}if(value=getMatch("M")){date.min=Number(value)}if(value=getMatch("H")){date.hour=Number(value)}else if(value=getMatch("I")){var hour=Number(value);if(value=getMatch("p")){hour+=value.toUpperCase()[0]==="P"?12:0}date.hour=hour}if(value=getMatch("Y")){date.year=Number(value)}else if(value=getMatch("y")){var year=Number(value);if(value=getMatch("C")){year+=Number(value)*100}else{year+=year<69?2e3:1900}date.year=year}if(value=getMatch("m")){date.month=Number(value)-1}else if(value=getMatch("b")){date.month=MONTH_NUMBERS[value.substring(0,3).toUpperCase()]||0}if(value=getMatch("d")){date.day=Number(value)}else if(value=getMatch("j")){var day=Number(value);var leapYear=isLeapYear(date.year);for(var month=0;month<12;++month){var daysUntilMonth=arraySum(leapYear?MONTH_DAYS_LEAP:MONTH_DAYS_REGULAR,month-1);if(day<=daysUntilMonth+(leapYear?MONTH_DAYS_LEAP:MONTH_DAYS_REGULAR)[month]){date.day=day-daysUntilMonth}}}else if(value=getMatch("a")){var weekDay=value.substring(0,3).toUpperCase();if(value=getMatch("U")){var weekDayNumber=DAY_NUMBERS_SUN_FIRST[weekDay];var weekNumber=Number(value);var janFirst=new Date(date.year,0,1);var endDate;if(janFirst.getDay()===0){endDate=addDays(janFirst,weekDayNumber+7*(weekNumber-1))}else{endDate=addDays(janFirst,7-janFirst.getDay()+weekDayNumber+7*(weekNumber-1))}date.day=endDate.getDate();date.month=endDate.getMonth()}else if(value=getMatch("W")){var weekDayNumber=DAY_NUMBERS_MON_FIRST[weekDay];var weekNumber=Number(value);var janFirst=new Date(date.year,0,1);var endDate;if(janFirst.getDay()===1){endDate=addDays(janFirst,weekDayNumber+7*(weekNumber-1))}else{endDate=addDays(janFirst,7-janFirst.getDay()+1+weekDayNumber+7*(weekNumber-1))}date.day=endDate.getDate();date.month=endDate.getMonth()}}if(value=getMatch("z")){if(value.toLowerCase()==="z"){date.gmtoff=0}else{var match=value.match(/^((?:\-|\+)\d\d):?(\d\d)?/);date.gmtoff=match[1]*3600;if(match[2]){date.gmtoff+=date.gmtoff>0?match[2]*60:-match[2]*60}}}var fullDate=new Date(date.year,date.month,date.day,date.hour,date.min,date.sec,0);HEAP32[tm>>2]=fullDate.getSeconds();HEAP32[tm+4>>2]=fullDate.getMinutes();HEAP32[tm+8>>2]=fullDate.getHours();HEAP32[tm+12>>2]=fullDate.getDate();HEAP32[tm+16>>2]=fullDate.getMonth();HEAP32[tm+20>>2]=fullDate.getFullYear()-1900;HEAP32[tm+24>>2]=fullDate.getDay();HEAP32[tm+28>>2]=arraySum(isLeapYear(fullDate.getFullYear())?MONTH_DAYS_LEAP:MONTH_DAYS_REGULAR,fullDate.getMonth()-1)+fullDate.getDate()-1;HEAP32[tm+32>>2]=0;HEAP64[tm+40>>3]=BigInt(date.gmtoff);return buf+lengthBytesUTF8(matches[0])}return 0})();return BigInt(ret)};function _swapcontext(...args){abort("missing function: swapcontext")}_swapcontext.stub=true;function _xmlFreeEntity(...args){abort("missing function: xmlFreeEntity")}_xmlFreeEntity.stub=true;function _xmlNewInputFromIO(...args){abort("missing function: xmlNewInputFromIO")}_xmlNewInputFromIO.stub=true;function _xmlXPathValuePop(...args){abort("missing function: xmlXPathValuePop")}_xmlXPathValuePop.stub=true;function _xmlXPathValuePush(...args){abort("missing function: xmlXPathValuePush")}_xmlXPathValuePush.stub=true;var runAndAbortIfError=func=>{try{return func()}catch(e){abort(e)}};var runtimeKeepalivePush=()=>{runtimeKeepaliveCounter+=1};var runtimeKeepalivePop=()=>{runtimeKeepaliveCounter-=1};var Asyncify={rewindArguments:new Map,instrumentWasmImports(imports){var importPattern=/^(invoke_.*|__asyncjs__.*)$/;for(let[x,original]of Object.entries(imports)){if(typeof original=="function"){let isAsyncifyImport=original.isAsync||importPattern.test(x)}}},saveRewindArguments(func,passedArguments){return Asyncify.rewindArguments.set(func,Array.from(passedArguments))},restoreRewindArguments(func){return Asyncify.rewindArguments.get(func)},instrumentFunction(original){var wrapper=(...args)=>{Asyncify.exportCallStack.push(original);try{Asyncify.saveRewindArguments(original,args);return original(...args)}finally{if(!ABORT){var top=Asyncify.exportCallStack.pop();Asyncify.maybeStopUnwind()}}};Asyncify.funcWrappers.set(original,wrapper);return wrapper},instrumentWasmExports(exports){var ret={};for(let[x,original]of Object.entries(exports)){if(typeof original=="function"){var wrapper=Asyncify.instrumentFunction(original);ret[x]=wrapper}else{ret[x]=original}}return ret},State:{Normal:0,Unwinding:1,Rewinding:2,Disabled:3},state:0,StackSize:4096,currData:null,handleSleepReturnValue:0,exportCallStack:[],callstackFuncToId:new Map,callStackIdToFunc:new Map,funcWrappers:new Map,callStackId:0,asyncPromiseHandlers:null,sleepCallbacks:[],getCallStackId(func){if(!Asyncify.callstackFuncToId.has(func)){var id=Asyncify.callStackId++;Asyncify.callstackFuncToId.set(func,id);Asyncify.callStackIdToFunc.set(id,func)}return Asyncify.callstackFuncToId.get(func)},maybeStopUnwind(){if(Asyncify.currData&&Asyncify.state===Asyncify.State.Unwinding&&Asyncify.exportCallStack.length===0){Asyncify.state=Asyncify.State.Normal;runAndAbortIfError(_asyncify_stop_unwind);if(typeof Fibers!="undefined"){Fibers.trampoline()}}},whenDone(){return new Promise((resolve,reject)=>{Asyncify.asyncPromiseHandlers={resolve,reject}})},allocateData(){var ptr=_malloc(24+Asyncify.StackSize);Asyncify.setDataHeader(ptr,ptr+24,Asyncify.StackSize);Asyncify.setDataRewindFunc(ptr);return ptr},setDataHeader(ptr,stack,stackSize){HEAPU64[ptr>>3]=BigInt(stack);HEAPU64[ptr+8>>3]=BigInt(stack+stackSize)},setDataRewindFunc(ptr){var bottomOfCallStack=Asyncify.exportCallStack[0];var rewindId=Asyncify.getCallStackId(bottomOfCallStack);HEAP32[ptr+16>>2]=rewindId},getDataRewindFunc(ptr){var id=HEAP32[ptr+16>>2];var func=Asyncify.callStackIdToFunc.get(id);return func},doRewind(ptr){var original=Asyncify.getDataRewindFunc(ptr);var func=Asyncify.funcWrappers.get(original);func=func.bind(0,...Asyncify.restoreRewindArguments(original));return callUserCallback(func)},handleSleep(startAsync){if(ABORT)return;if(Asyncify.state===Asyncify.State.Normal){var reachedCallback=false;var reachedAfterCallback=false;startAsync((handleSleepReturnValue=0)=>{if(ABORT)return;Asyncify.handleSleepReturnValue=handleSleepReturnValue;reachedCallback=true;if(!reachedAfterCallback){return}Asyncify.state=Asyncify.State.Rewinding;runAndAbortIfError(()=>_asyncify_start_rewind(Asyncify.currData));if(typeof MainLoop!="undefined"&&MainLoop.func){MainLoop.resume()}var asyncWasmReturnValue,isError=false;try{asyncWasmReturnValue=Asyncify.doRewind(Asyncify.currData)}catch(err){asyncWasmReturnValue=err;isError=true}var handled=false;if(!Asyncify.currData){var asyncPromiseHandlers=Asyncify.asyncPromiseHandlers;if(asyncPromiseHandlers){Asyncify.asyncPromiseHandlers=null;(isError?asyncPromiseHandlers.reject:asyncPromiseHandlers.resolve)(asyncWasmReturnValue);handled=true}}if(isError&&!handled){throw asyncWasmReturnValue}});reachedAfterCallback=true;if(!reachedCallback){Asyncify.state=Asyncify.State.Unwinding;Asyncify.currData=Asyncify.allocateData();if(typeof MainLoop!="undefined"&&MainLoop.func){MainLoop.pause()}runAndAbortIfError(()=>_asyncify_start_unwind(Asyncify.currData))}}else if(Asyncify.state===Asyncify.State.Rewinding){Asyncify.state=Asyncify.State.Normal;runAndAbortIfError(_asyncify_stop_rewind);_free(Asyncify.currData);Asyncify.currData=null;Asyncify.sleepCallbacks.forEach(callUserCallback)}else{abort(`invalid state: ${Asyncify.state}`)}return Asyncify.handleSleepReturnValue},handleAsync:startAsync=>Asyncify.handleSleep(async wakeUp=>{wakeUp(await startAsync())})};var getCFunc=ident=>{var func=Module["_"+ident];return func};var writeArrayToMemory=(array,buffer)=>{HEAP8.set(array,buffer)};var stackAlloc=sz=>__emscripten_stack_alloc(sz);var stringToUTF8OnStack=str=>{var size=lengthBytesUTF8(str)+1;var ret=stackAlloc(size);stringToUTF8(str,ret,size);return ret};var ccall=(ident,returnType,argTypes,args,opts)=>{var toC={pointer:p=>BigInt(p),string:str=>{var ret=0;if(str!==null&&str!==undefined&&str!==0){ret=stringToUTF8OnStack(str)}return BigInt(ret)},array:arr=>{var ret=stackAlloc(arr.length);writeArrayToMemory(arr,ret);return BigInt(ret)}};function convertReturnValue(ret){if(returnType==="string"){return UTF8ToString(Number(ret))}if(returnType==="pointer")return Number(ret);if(returnType==="boolean")return Boolean(ret);return ret}var func=getCFunc(ident);var cArgs=[];var stack=0;if(args){for(var i=0;i0){Module["preInit"].shift()()}}}Module["ENV"]=ENV;Module["ccall"]=ccall;Module["setValue"]=setValue;Module["getValue"]=getValue;Module["UTF8ToString"]=UTF8ToString;Module["stringToUTF8"]=stringToUTF8;Module["lengthBytesUTF8"]=lengthBytesUTF8;Module["FS"]=FS;var _pib_heap_bytes,_pib_eval,_free,_malloc,_htons,_ntohs,_htonl,_pib_force_exit,_emscripten_builtin_memalign,__emscripten_timeout,_setThrew,__emscripten_stack_restore,__emscripten_stack_alloc,_emscripten_stack_get_current,dynCall_vjj,dynCall_jjj,dynCall_ijjjj,dynCall_vjjj,dynCall_ijjj,dynCall_vj,dynCall_jj,dynCall_vjjjj,dynCall_ij,dynCall_jjjj,dynCall_vjjijjj,dynCall_vjjjiijj,dynCall_vjjij,dynCall_vjjjjj,dynCall_vjji,dynCall_vjjjjijiij,dynCall_ijj,dynCall_ijji,dynCall_jji,dynCall_jjji,dynCall_ii,dynCall_ijjjjj,dynCall_ijjjjjjj,dynCall_ijjjjjj,dynCall_vji,dynCall_j,dynCall_v,dynCall_jjjjj,dynCall_vjjiijjjjjjjjj,dynCall_jjjji,dynCall_iij,dynCall_vjij,dynCall_ji,dynCall_jjii,dynCall_vjjjji,dynCall_vjjii,dynCall_vjjji,dynCall_iji,dynCall_jjjiij,dynCall_jiij,dynCall_jjjjji,dynCall_ijjji,dynCall_iijj,dynCall_vjiji,dynCall_i,dynCall_vijj,dynCall_ijjii,dynCall_vi,dynCall_vii,dynCall_jjij,dynCall_vijij,dynCall_jjjjii,dynCall_jjjiiii,dynCall_vjjdi,dynCall_ijiijj,dynCall_vjjjij,dynCall_vjii,dynCall_iijjjj,dynCall_jjjjjjiijj,dynCall_ijjij,dynCall_jjjijj,dynCall_jjjij,dynCall_jijj,dynCall_vjjjjij,dynCall_vjjjjjj,dynCall_jjjjjj,dynCall_ijjiii,dynCall_jjjjijj,dynCall_jii,dynCall_vij,dynCall_ijjjjjji,dynCall_ijjjji,dynCall_ijiij,dynCall_vjijj,dynCall_vjiijijjj,dynCall_jjijj,dynCall_vjijjj,dynCall_iijjj,dynCall_ijij,dynCall_iiji,dynCall_ijdiiii,dynCall_jij,dynCall_ijjjjijj,dynCall_ijjjjji,dynCall_vijijj,dynCall_ijii,dynCall_viiijjj,dynCall_vjdj,dynCall_iii,dynCall_jjjjjjj,dynCall_ijjijj,dynCall_ijjjij,dynCall_ijjiij,dynCall_jjjjij,dynCall_vjjjjii,dynCall_vjjjii,_asyncify_start_unwind,_asyncify_stop_unwind,_asyncify_start_rewind,_asyncify_stop_rewind,memory,__indirect_function_table,wasmMemory,wasmTable;function assignWasmExports(wasmExports){_pib_heap_bytes=Module["_pib_heap_bytes"]=wasmExports["Mb"];_pib_eval=Module["_pib_eval"]=wasmExports["Nb"];_free=wasmExports["Ob"];_malloc=wasmExports["Pb"];_htons=wasmExports["Qb"];_ntohs=wasmExports["Rb"];_htonl=wasmExports["Sb"];_pib_force_exit=Module["_pib_force_exit"]=wasmExports["Tb"];_emscripten_builtin_memalign=wasmExports["Vb"];__emscripten_timeout=wasmExports["Wb"];_setThrew=wasmExports["Xb"];__emscripten_stack_restore=wasmExports["Yb"];__emscripten_stack_alloc=wasmExports["Zb"];_emscripten_stack_get_current=wasmExports["_b"];dynCall_vjj=dynCalls["vjj"]=wasmExports["$b"];dynCall_jjj=dynCalls["jjj"]=wasmExports["ac"];dynCall_ijjjj=dynCalls["ijjjj"]=wasmExports["bc"];dynCall_vjjj=dynCalls["vjjj"]=wasmExports["cc"];dynCall_ijjj=dynCalls["ijjj"]=wasmExports["dc"];dynCall_vj=dynCalls["vj"]=wasmExports["ec"];dynCall_jj=dynCalls["jj"]=wasmExports["fc"];dynCall_vjjjj=dynCalls["vjjjj"]=wasmExports["gc"];dynCall_ij=dynCalls["ij"]=wasmExports["hc"];dynCall_jjjj=dynCalls["jjjj"]=wasmExports["ic"];dynCall_vjjijjj=dynCalls["vjjijjj"]=wasmExports["jc"];dynCall_vjjjiijj=dynCalls["vjjjiijj"]=wasmExports["kc"];dynCall_vjjij=dynCalls["vjjij"]=wasmExports["lc"];dynCall_vjjjjj=dynCalls["vjjjjj"]=wasmExports["mc"];dynCall_vjji=dynCalls["vjji"]=wasmExports["nc"];dynCall_vjjjjijiij=dynCalls["vjjjjijiij"]=wasmExports["oc"];dynCall_ijj=dynCalls["ijj"]=wasmExports["pc"];dynCall_ijji=dynCalls["ijji"]=wasmExports["qc"];dynCall_jji=dynCalls["jji"]=wasmExports["rc"];dynCall_jjji=dynCalls["jjji"]=wasmExports["sc"];dynCall_ii=dynCalls["ii"]=wasmExports["tc"];dynCall_ijjjjj=dynCalls["ijjjjj"]=wasmExports["uc"];dynCall_ijjjjjjj=dynCalls["ijjjjjjj"]=wasmExports["vc"];dynCall_ijjjjjj=dynCalls["ijjjjjj"]=wasmExports["wc"];dynCall_vji=dynCalls["vji"]=wasmExports["xc"];dynCall_j=dynCalls["j"]=wasmExports["yc"];dynCall_v=dynCalls["v"]=wasmExports["zc"];dynCall_jjjjj=dynCalls["jjjjj"]=wasmExports["Ac"];dynCall_vjjiijjjjjjjjj=dynCalls["vjjiijjjjjjjjj"]=wasmExports["Bc"];dynCall_jjjji=dynCalls["jjjji"]=wasmExports["Cc"];dynCall_iij=dynCalls["iij"]=wasmExports["Dc"];dynCall_vjij=dynCalls["vjij"]=wasmExports["Ec"];dynCall_ji=dynCalls["ji"]=wasmExports["Fc"];dynCall_jjii=dynCalls["jjii"]=wasmExports["Gc"];dynCall_vjjjji=dynCalls["vjjjji"]=wasmExports["Hc"];dynCall_vjjii=dynCalls["vjjii"]=wasmExports["Ic"];dynCall_vjjji=dynCalls["vjjji"]=wasmExports["Jc"];dynCall_iji=dynCalls["iji"]=wasmExports["Kc"];dynCall_jjjiij=dynCalls["jjjiij"]=wasmExports["Lc"];dynCall_jiij=dynCalls["jiij"]=wasmExports["Mc"];dynCall_jjjjji=dynCalls["jjjjji"]=wasmExports["Nc"];dynCall_ijjji=dynCalls["ijjji"]=wasmExports["Oc"];dynCall_iijj=dynCalls["iijj"]=wasmExports["Pc"];dynCall_vjiji=dynCalls["vjiji"]=wasmExports["Qc"];dynCall_i=dynCalls["i"]=wasmExports["Rc"];dynCall_vijj=dynCalls["vijj"]=wasmExports["Sc"];dynCall_ijjii=dynCalls["ijjii"]=wasmExports["Tc"];dynCall_vi=dynCalls["vi"]=wasmExports["Uc"];dynCall_vii=dynCalls["vii"]=wasmExports["Vc"];dynCall_jjij=dynCalls["jjij"]=wasmExports["Wc"];dynCall_vijij=dynCalls["vijij"]=wasmExports["Xc"];dynCall_jjjjii=dynCalls["jjjjii"]=wasmExports["Yc"];dynCall_jjjiiii=dynCalls["jjjiiii"]=wasmExports["Zc"];dynCall_vjjdi=dynCalls["vjjdi"]=wasmExports["_c"];dynCall_ijiijj=dynCalls["ijiijj"]=wasmExports["$c"];dynCall_vjjjij=dynCalls["vjjjij"]=wasmExports["ad"];dynCall_vjii=dynCalls["vjii"]=wasmExports["bd"];dynCall_iijjjj=dynCalls["iijjjj"]=wasmExports["cd"];dynCall_jjjjjjiijj=dynCalls["jjjjjjiijj"]=wasmExports["dd"];dynCall_ijjij=dynCalls["ijjij"]=wasmExports["ed"];dynCall_jjjijj=dynCalls["jjjijj"]=wasmExports["fd"];dynCall_jjjij=dynCalls["jjjij"]=wasmExports["gd"];dynCall_jijj=dynCalls["jijj"]=wasmExports["hd"];dynCall_vjjjjij=dynCalls["vjjjjij"]=wasmExports["id"];dynCall_vjjjjjj=dynCalls["vjjjjjj"]=wasmExports["jd"];dynCall_jjjjjj=dynCalls["jjjjjj"]=wasmExports["kd"];dynCall_ijjiii=dynCalls["ijjiii"]=wasmExports["ld"];dynCall_jjjjijj=dynCalls["jjjjijj"]=wasmExports["md"];dynCall_jii=dynCalls["jii"]=wasmExports["nd"];dynCall_vij=dynCalls["vij"]=wasmExports["od"];dynCall_ijjjjjji=dynCalls["ijjjjjji"]=wasmExports["pd"];dynCall_ijjjji=dynCalls["ijjjji"]=wasmExports["qd"];dynCall_ijiij=dynCalls["ijiij"]=wasmExports["rd"];dynCall_vjijj=dynCalls["vjijj"]=wasmExports["sd"];dynCall_vjiijijjj=dynCalls["vjiijijjj"]=wasmExports["td"];dynCall_jjijj=dynCalls["jjijj"]=wasmExports["ud"];dynCall_vjijjj=dynCalls["vjijjj"]=wasmExports["vd"];dynCall_iijjj=dynCalls["iijjj"]=wasmExports["wd"];dynCall_ijij=dynCalls["ijij"]=wasmExports["xd"];dynCall_iiji=dynCalls["iiji"]=wasmExports["yd"];dynCall_ijdiiii=dynCalls["ijdiiii"]=wasmExports["zd"];dynCall_jij=dynCalls["jij"]=wasmExports["Ad"];dynCall_ijjjjijj=dynCalls["ijjjjijj"]=wasmExports["Bd"];dynCall_ijjjjji=dynCalls["ijjjjji"]=wasmExports["Cd"];dynCall_vijijj=dynCalls["vijijj"]=wasmExports["Dd"];dynCall_ijii=dynCalls["ijii"]=wasmExports["Ed"];dynCall_viiijjj=dynCalls["viiijjj"]=wasmExports["Fd"];dynCall_vjdj=dynCalls["vjdj"]=wasmExports["Gd"];dynCall_iii=dynCalls["iii"]=wasmExports["Hd"];dynCall_jjjjjjj=dynCalls["jjjjjjj"]=wasmExports["Id"];dynCall_ijjijj=dynCalls["ijjijj"]=wasmExports["Jd"];dynCall_ijjjij=dynCalls["ijjjij"]=wasmExports["Kd"];dynCall_ijjiij=dynCalls["ijjiij"]=wasmExports["Ld"];dynCall_jjjjij=dynCalls["jjjjij"]=wasmExports["Md"];dynCall_vjjjjii=dynCalls["vjjjjii"]=wasmExports["Nd"];dynCall_vjjjii=dynCalls["vjjjii"]=wasmExports["Od"];_asyncify_start_unwind=wasmExports["Pd"];_asyncify_stop_unwind=wasmExports["Qd"];_asyncify_start_rewind=wasmExports["Rd"];_asyncify_stop_rewind=wasmExports["Sd"];memory=wasmMemory=wasmExports["Kb"];__indirect_function_table=wasmTable=wasmExports["Ub"]}var wasmImports={m:___assert_fail,Ja:___call_sighandler,ya:___syscall_accept4,xa:___syscall_bind,eb:___syscall_chdir,fa:___syscall_chmod,P:___syscall_connect,cb:___syscall_dup,fb:___syscall_faccessat,bb:___syscall_fchmod,ea:___syscall_fchownat,q:___syscall_fcntl64,ab:___syscall_fdatasync,_a:___syscall_fstat64,Va:___syscall_ftruncate64,Ua:___syscall_getcwd,Ia:___syscall_getdents64,va:___syscall_getpeername,ua:___syscall_getsockname,ta:___syscall_getsockopt,ha:___syscall_ioctl,Gb:___syscall_listen,Ya:___syscall_lstat64,Ra:___syscall_mkdirat,Xa:___syscall_newfstatat,Q:___syscall_openat,La:___syscall_pipe2,ba:___syscall_poll,Ha:___syscall_readlinkat,Z:___syscall_recvfrom,Ga:___syscall_renameat,$:___syscall_rmdir,I:___syscall_sendto,sa:___syscall_socket,Za:___syscall_stat64,Fa:___syscall_statfs64,Ea:___syscall_symlinkat,aa:___syscall_unlinkat,Ba:___syscall_utimensat,gb:__abort_js,wa:__emscripten_lookup_name,Ka:__emscripten_runtime_keepalive_clear,za:__emscripten_throw_longjmp,Oa:__gmtime_js,Pa:__localtime_js,Qa:__mktime_js,Ma:__mmap_js,Na:__munmap_js,_:__setitimer_js,jb:__tzset_js,db:_clock_time_get,da:_emscripten_date_now,vb:_emscripten_force_exit,Da:_emscripten_get_heap_max,K:_emscripten_get_now,Aa:_emscripten_resize_heap,kb:_environ_get,lb:_environ_sizes_get,z:_exit,D:_fd_close,ca:_fd_fdstat_get,ga:_fd_read,Sa:_fd_seek,Wa:_fd_sync,R:_fd_write,sb:_getaddrinfo,Jb:_getcontext,Ca:_getdtablesize,J:_getnameinfo,hb:_getprotobyname,$a:_getprotobynumber,C:invoke_i,u:invoke_ii,X:invoke_iii,y:invoke_iij,Fb:invoke_iijj,t:invoke_ij,E:invoke_iji,tb:invoke_ijii,M:invoke_ijiij,U:invoke_ijiijj,w:invoke_ijij,e:invoke_ijj,Y:invoke_ijji,Eb:invoke_ijjii,na:invoke_ijjiii,xb:invoke_ijjijj,k:invoke_ijjj,H:invoke_ijjji,p:invoke_ijjjj,la:invoke_ijjjjijj,Bb:invoke_ijjjjji,zb:invoke_ijjjjjji,o:invoke_j,ia:invoke_ji,wb:invoke_jij,qa:invoke_jijj,b:invoke_jj,i:invoke_jji,ob:invoke_jjii,h:invoke_jjj,s:invoke_jjji,Cb:invoke_jjjiiii,l:invoke_jjjj,pb:invoke_jjjji,d:invoke_jjjjii,Ab:invoke_jjjjijj,n:invoke_jjjjj,ub:invoke_jjjjji,oa:invoke_jjjjjj,yb:invoke_jjjjjjiijj,c:invoke_v,O:invoke_vi,ra:invoke_vii,T:invoke_vij,S:invoke_vijijj,g:invoke_vijj,a:invoke_vj,x:invoke_vji,B:invoke_vjii,r:invoke_vjij,j:invoke_vjiji,F:invoke_vjijj,f:invoke_vjj,V:invoke_vjjdi,ma:invoke_vjji,A:invoke_vjjii,qb:invoke_vjjiijjjjjjjjj,L:invoke_vjjij,v:invoke_vjjj,ja:invoke_vjjjj,nb:invoke_vjjjji,pa:invoke_vjjjjij,ka:invoke_vjjjjj,N:invoke_vjjjjjj,Ib:_makecontext,mb:_posix_spawnp,ib:_proc_exit,Ta:_strptime,rb:_swapcontext,Hb:_xmlFreeEntity,Db:_xmlNewInputFromIO,W:_xmlXPathValuePop,G:_xmlXPathValuePush};function invoke_j(index){var sp=stackSave();try{return dynCall_j(Number(index))}catch(e){stackRestore(sp);if(!(e instanceof EmscriptenEH))throw e;_setThrew(1,0);return 0n}}function invoke_jj(index,a1){var sp=stackSave();try{return dynCall_jj(Number(index),a1)}catch(e){stackRestore(sp);if(!(e instanceof EmscriptenEH))throw e;_setThrew(1,0);return 0n}}function invoke_vj(index,a1){var sp=stackSave();try{dynCall_vj(Number(index),a1)}catch(e){stackRestore(sp);if(!(e instanceof EmscriptenEH))throw e;_setThrew(1,0)}}function invoke_vjjj(index,a1,a2,a3){var sp=stackSave();try{dynCall_vjjj(Number(index),a1,a2,a3)}catch(e){stackRestore(sp);if(!(e instanceof EmscriptenEH))throw e;_setThrew(1,0)}}function invoke_vjj(index,a1,a2){var sp=stackSave();try{dynCall_vjj(Number(index),a1,a2)}catch(e){stackRestore(sp);if(!(e instanceof EmscriptenEH))throw e;_setThrew(1,0)}}function invoke_v(index){var sp=stackSave();try{dynCall_v(Number(index))}catch(e){stackRestore(sp);if(!(e instanceof EmscriptenEH))throw e;_setThrew(1,0)}}function invoke_jji(index,a1,a2){var sp=stackSave();try{return dynCall_jji(Number(index),a1,a2)}catch(e){stackRestore(sp);if(!(e instanceof EmscriptenEH))throw e;_setThrew(1,0);return 0n}}function invoke_ijj(index,a1,a2){var sp=stackSave();try{return dynCall_ijj(Number(index),a1,a2)}catch(e){stackRestore(sp);if(!(e instanceof EmscriptenEH))throw e;_setThrew(1,0)}}function invoke_vjjjj(index,a1,a2,a3,a4){var sp=stackSave();try{dynCall_vjjjj(Number(index),a1,a2,a3,a4)}catch(e){stackRestore(sp);if(!(e instanceof EmscriptenEH))throw e;_setThrew(1,0)}}function invoke_jjjjj(index,a1,a2,a3,a4){var sp=stackSave();try{return dynCall_jjjjj(Number(index),a1,a2,a3,a4)}catch(e){stackRestore(sp);if(!(e instanceof EmscriptenEH))throw e;_setThrew(1,0);return 0n}}function invoke_ijjj(index,a1,a2,a3){var sp=stackSave();try{return dynCall_ijjj(Number(index),a1,a2,a3)}catch(e){stackRestore(sp);if(!(e instanceof EmscriptenEH))throw e;_setThrew(1,0)}}function invoke_ii(index,a1){var sp=stackSave();try{return dynCall_ii(Number(index),a1)}catch(e){stackRestore(sp);if(!(e instanceof EmscriptenEH))throw e;_setThrew(1,0)}}function invoke_vji(index,a1,a2){var sp=stackSave();try{dynCall_vji(Number(index),a1,a2)}catch(e){stackRestore(sp);if(!(e instanceof EmscriptenEH))throw e;_setThrew(1,0)}}function invoke_jjj(index,a1,a2){var sp=stackSave();try{return dynCall_jjj(Number(index),a1,a2)}catch(e){stackRestore(sp);if(!(e instanceof EmscriptenEH))throw e;_setThrew(1,0);return 0n}}function invoke_vjjij(index,a1,a2,a3,a4){var sp=stackSave();try{dynCall_vjjij(Number(index),a1,a2,a3,a4)}catch(e){stackRestore(sp);if(!(e instanceof EmscriptenEH))throw e;_setThrew(1,0)}}function invoke_vjjiijjjjjjjjj(index,a1,a2,a3,a4,a5,a6,a7,a8,a9,a10,a11,a12,a13){var sp=stackSave();try{dynCall_vjjiijjjjjjjjj(Number(index),a1,a2,a3,a4,a5,a6,a7,a8,a9,a10,a11,a12,a13)}catch(e){stackRestore(sp);if(!(e instanceof EmscriptenEH))throw e;_setThrew(1,0)}}function invoke_jjjji(index,a1,a2,a3,a4){var sp=stackSave();try{return dynCall_jjjji(Number(index),a1,a2,a3,a4)}catch(e){stackRestore(sp);if(!(e instanceof EmscriptenEH))throw e;_setThrew(1,0);return 0n}}function invoke_iij(index,a1,a2){var sp=stackSave();try{return dynCall_iij(Number(index),a1,a2)}catch(e){stackRestore(sp);if(!(e instanceof EmscriptenEH))throw e;_setThrew(1,0)}}function invoke_vjij(index,a1,a2,a3){var sp=stackSave();try{dynCall_vjij(Number(index),a1,a2,a3)}catch(e){stackRestore(sp);if(!(e instanceof EmscriptenEH))throw e;_setThrew(1,0)}}function invoke_ij(index,a1){var sp=stackSave();try{return dynCall_ij(Number(index),a1)}catch(e){stackRestore(sp);if(!(e instanceof EmscriptenEH))throw e;_setThrew(1,0)}}function invoke_ji(index,a1){var sp=stackSave();try{return dynCall_ji(Number(index),a1)}catch(e){stackRestore(sp);if(!(e instanceof EmscriptenEH))throw e;_setThrew(1,0);return 0n}}function invoke_jjii(index,a1,a2,a3){var sp=stackSave();try{return dynCall_jjii(Number(index),a1,a2,a3)}catch(e){stackRestore(sp);if(!(e instanceof EmscriptenEH))throw e;_setThrew(1,0);return 0n}}function invoke_vjjjji(index,a1,a2,a3,a4,a5){var sp=stackSave();try{dynCall_vjjjji(Number(index),a1,a2,a3,a4,a5)}catch(e){stackRestore(sp);if(!(e instanceof EmscriptenEH))throw e;_setThrew(1,0)}}function invoke_vjjii(index,a1,a2,a3,a4){var sp=stackSave();try{dynCall_vjjii(Number(index),a1,a2,a3,a4)}catch(e){stackRestore(sp);if(!(e instanceof EmscriptenEH))throw e;_setThrew(1,0)}}function invoke_iijj(index,a1,a2,a3){var sp=stackSave();try{return dynCall_iijj(Number(index),a1,a2,a3)}catch(e){stackRestore(sp);if(!(e instanceof EmscriptenEH))throw e;_setThrew(1,0)}}function invoke_vjiji(index,a1,a2,a3,a4){var sp=stackSave();try{dynCall_vjiji(Number(index),a1,a2,a3,a4)}catch(e){stackRestore(sp);if(!(e instanceof EmscriptenEH))throw e;_setThrew(1,0)}}function invoke_i(index){var sp=stackSave();try{return dynCall_i(Number(index))}catch(e){stackRestore(sp);if(!(e instanceof EmscriptenEH))throw e;_setThrew(1,0)}}function invoke_jjji(index,a1,a2,a3){var sp=stackSave();try{return dynCall_jjji(Number(index),a1,a2,a3)}catch(e){stackRestore(sp);if(!(e instanceof EmscriptenEH))throw e;_setThrew(1,0);return 0n}}function invoke_vijj(index,a1,a2,a3){var sp=stackSave();try{dynCall_vijj(Number(index),a1,a2,a3)}catch(e){stackRestore(sp);if(!(e instanceof EmscriptenEH))throw e;_setThrew(1,0)}}function invoke_ijjii(index,a1,a2,a3,a4){var sp=stackSave();try{return dynCall_ijjii(Number(index),a1,a2,a3,a4)}catch(e){stackRestore(sp);if(!(e instanceof EmscriptenEH))throw e;_setThrew(1,0)}}function invoke_ijji(index,a1,a2,a3){var sp=stackSave();try{return dynCall_ijji(Number(index),a1,a2,a3)}catch(e){stackRestore(sp);if(!(e instanceof EmscriptenEH))throw e;_setThrew(1,0)}}function invoke_iii(index,a1,a2){var sp=stackSave();try{return dynCall_iii(Number(index),a1,a2)}catch(e){stackRestore(sp);if(!(e instanceof EmscriptenEH))throw e;_setThrew(1,0)}}function invoke_vi(index,a1){var sp=stackSave();try{dynCall_vi(Number(index),a1)}catch(e){stackRestore(sp);if(!(e instanceof EmscriptenEH))throw e;_setThrew(1,0)}}function invoke_vii(index,a1,a2){var sp=stackSave();try{dynCall_vii(Number(index),a1,a2)}catch(e){stackRestore(sp);if(!(e instanceof EmscriptenEH))throw e;_setThrew(1,0)}}function invoke_jjjjii(index,a1,a2,a3,a4,a5){var sp=stackSave();try{return dynCall_jjjjii(Number(index),a1,a2,a3,a4,a5)}catch(e){stackRestore(sp);if(!(e instanceof EmscriptenEH))throw e;_setThrew(1,0);return 0n}}function invoke_jjjiiii(index,a1,a2,a3,a4,a5,a6){var sp=stackSave();try{return dynCall_jjjiiii(Number(index),a1,a2,a3,a4,a5,a6)}catch(e){stackRestore(sp);if(!(e instanceof EmscriptenEH))throw e;_setThrew(1,0);return 0n}}function invoke_jjjj(index,a1,a2,a3){var sp=stackSave();try{return dynCall_jjjj(Number(index),a1,a2,a3)}catch(e){stackRestore(sp);if(!(e instanceof EmscriptenEH))throw e;_setThrew(1,0);return 0n}}function invoke_ijjji(index,a1,a2,a3,a4){var sp=stackSave();try{return dynCall_ijjji(Number(index),a1,a2,a3,a4)}catch(e){stackRestore(sp);if(!(e instanceof EmscriptenEH))throw e;_setThrew(1,0)}}function invoke_ijjjj(index,a1,a2,a3,a4){var sp=stackSave();try{return dynCall_ijjjj(Number(index),a1,a2,a3,a4)}catch(e){stackRestore(sp);if(!(e instanceof EmscriptenEH))throw e;_setThrew(1,0)}}function invoke_iji(index,a1,a2){var sp=stackSave();try{return dynCall_iji(Number(index),a1,a2)}catch(e){stackRestore(sp);if(!(e instanceof EmscriptenEH))throw e;_setThrew(1,0)}}function invoke_vjjdi(index,a1,a2,a3,a4){var sp=stackSave();try{dynCall_vjjdi(Number(index),a1,a2,a3,a4)}catch(e){stackRestore(sp);if(!(e instanceof EmscriptenEH))throw e;_setThrew(1,0)}}function invoke_ijiijj(index,a1,a2,a3,a4,a5){var sp=stackSave();try{return dynCall_ijiijj(Number(index),a1,a2,a3,a4,a5)}catch(e){stackRestore(sp);if(!(e instanceof EmscriptenEH))throw e;_setThrew(1,0)}}function invoke_vjii(index,a1,a2,a3){var sp=stackSave();try{dynCall_vjii(Number(index),a1,a2,a3)}catch(e){stackRestore(sp);if(!(e instanceof EmscriptenEH))throw e;_setThrew(1,0)}}function invoke_ijjjjji(index,a1,a2,a3,a4,a5,a6){var sp=stackSave();try{return dynCall_ijjjjji(Number(index),a1,a2,a3,a4,a5,a6)}catch(e){stackRestore(sp);if(!(e instanceof EmscriptenEH))throw e;_setThrew(1,0)}}function invoke_jijj(index,a1,a2,a3){var sp=stackSave();try{return dynCall_jijj(Number(index),a1,a2,a3)}catch(e){stackRestore(sp);if(!(e instanceof EmscriptenEH))throw e;_setThrew(1,0);return 0n}}function invoke_vjjjjij(index,a1,a2,a3,a4,a5,a6){var sp=stackSave();try{dynCall_vjjjjij(Number(index),a1,a2,a3,a4,a5,a6)}catch(e){stackRestore(sp);if(!(e instanceof EmscriptenEH))throw e;_setThrew(1,0)}}function invoke_vjjjjjj(index,a1,a2,a3,a4,a5,a6){var sp=stackSave();try{dynCall_vjjjjjj(Number(index),a1,a2,a3,a4,a5,a6)}catch(e){stackRestore(sp);if(!(e instanceof EmscriptenEH))throw e;_setThrew(1,0)}}function invoke_jjjjjj(index,a1,a2,a3,a4,a5){var sp=stackSave();try{return dynCall_jjjjjj(Number(index),a1,a2,a3,a4,a5)}catch(e){stackRestore(sp);if(!(e instanceof EmscriptenEH))throw e;_setThrew(1,0);return 0n}}function invoke_ijjiii(index,a1,a2,a3,a4,a5){var sp=stackSave();try{return dynCall_ijjiii(Number(index),a1,a2,a3,a4,a5)}catch(e){stackRestore(sp);if(!(e instanceof EmscriptenEH))throw e;_setThrew(1,0)}}function invoke_jjjjijj(index,a1,a2,a3,a4,a5,a6){var sp=stackSave();try{return dynCall_jjjjijj(Number(index),a1,a2,a3,a4,a5,a6)}catch(e){stackRestore(sp);if(!(e instanceof EmscriptenEH))throw e;_setThrew(1,0);return 0n}}function invoke_vjji(index,a1,a2,a3){var sp=stackSave();try{dynCall_vjji(Number(index),a1,a2,a3)}catch(e){stackRestore(sp);if(!(e instanceof EmscriptenEH))throw e;_setThrew(1,0)}}function invoke_vij(index,a1,a2){var sp=stackSave();try{dynCall_vij(Number(index),a1,a2)}catch(e){stackRestore(sp);if(!(e instanceof EmscriptenEH))throw e;_setThrew(1,0)}}function invoke_ijjjjjji(index,a1,a2,a3,a4,a5,a6,a7){var sp=stackSave();try{return dynCall_ijjjjjji(Number(index),a1,a2,a3,a4,a5,a6,a7)}catch(e){stackRestore(sp);if(!(e instanceof EmscriptenEH))throw e;_setThrew(1,0)}}function invoke_ijiij(index,a1,a2,a3,a4){var sp=stackSave();try{return dynCall_ijiij(Number(index),a1,a2,a3,a4)}catch(e){stackRestore(sp);if(!(e instanceof EmscriptenEH))throw e;_setThrew(1,0)}}function invoke_vjijj(index,a1,a2,a3,a4){var sp=stackSave();try{dynCall_vjijj(Number(index),a1,a2,a3,a4)}catch(e){stackRestore(sp);if(!(e instanceof EmscriptenEH))throw e;_setThrew(1,0)}}function invoke_jjjjjjiijj(index,a1,a2,a3,a4,a5,a6,a7,a8,a9){var sp=stackSave();try{return dynCall_jjjjjjiijj(Number(index),a1,a2,a3,a4,a5,a6,a7,a8,a9)}catch(e){stackRestore(sp);if(!(e instanceof EmscriptenEH))throw e;_setThrew(1,0);return 0n}}function invoke_ijij(index,a1,a2,a3){var sp=stackSave();try{return dynCall_ijij(Number(index),a1,a2,a3)}catch(e){stackRestore(sp);if(!(e instanceof EmscriptenEH))throw e;_setThrew(1,0)}}function invoke_ijjijj(index,a1,a2,a3,a4,a5){var sp=stackSave();try{return dynCall_ijjijj(Number(index),a1,a2,a3,a4,a5)}catch(e){stackRestore(sp);if(!(e instanceof EmscriptenEH))throw e;_setThrew(1,0)}}function invoke_jij(index,a1,a2){var sp=stackSave();try{return dynCall_jij(Number(index),a1,a2)}catch(e){stackRestore(sp);if(!(e instanceof EmscriptenEH))throw e;_setThrew(1,0);return 0n}}function invoke_ijjjjijj(index,a1,a2,a3,a4,a5,a6,a7){var sp=stackSave();try{return dynCall_ijjjjijj(Number(index),a1,a2,a3,a4,a5,a6,a7)}catch(e){stackRestore(sp);if(!(e instanceof EmscriptenEH))throw e;_setThrew(1,0)}}function invoke_vijijj(index,a1,a2,a3,a4,a5){var sp=stackSave();try{dynCall_vijijj(Number(index),a1,a2,a3,a4,a5)}catch(e){stackRestore(sp);if(!(e instanceof EmscriptenEH))throw e;_setThrew(1,0)}}function invoke_vjjjjj(index,a1,a2,a3,a4,a5){var sp=stackSave();try{dynCall_vjjjjj(Number(index),a1,a2,a3,a4,a5)}catch(e){stackRestore(sp);if(!(e instanceof EmscriptenEH))throw e;_setThrew(1,0)}}function invoke_jjjjji(index,a1,a2,a3,a4,a5){var sp=stackSave();try{return dynCall_jjjjji(Number(index),a1,a2,a3,a4,a5)}catch(e){stackRestore(sp);if(!(e instanceof EmscriptenEH))throw e;_setThrew(1,0);return 0n}}function invoke_ijii(index,a1,a2,a3){var sp=stackSave();try{return dynCall_ijii(Number(index),a1,a2,a3)}catch(e){stackRestore(sp);if(!(e instanceof EmscriptenEH))throw e;_setThrew(1,0)}}function applySignatureConversions(wasmExports){wasmExports=Object.assign({},wasmExports);var makeWrapper__p=f=>a0=>f(BigInt(a0));var makeWrapper_pp=f=>a0=>Number(f(BigInt(a0)));var makeWrapper_ppp=f=>(a0,a1)=>Number(f(BigInt(a0),BigInt(a1)));var makeWrapper_p=f=>()=>Number(f());var makeWrapper__p__=f=>(a0,a1,a2)=>f(BigInt(a0),a1,a2);var makeWrapper__p____=f=>(a0,a1,a2,a3,a4)=>f(BigInt(a0),a1,a2,a3,a4);var makeWrapper__p___=f=>(a0,a1,a2,a3)=>f(BigInt(a0),a1,a2,a3);var makeWrapper__p_=f=>(a0,a1)=>f(BigInt(a0),a1);var makeWrapper__p______=f=>(a0,a1,a2,a3,a4,a5,a6)=>f(BigInt(a0),a1,a2,a3,a4,a5,a6);var makeWrapper__p_______=f=>(a0,a1,a2,a3,a4,a5,a6,a7)=>f(BigInt(a0),a1,a2,a3,a4,a5,a6,a7);var makeWrapper__p_____=f=>(a0,a1,a2,a3,a4,a5)=>f(BigInt(a0),a1,a2,a3,a4,a5);var makeWrapper__p_________=f=>(a0,a1,a2,a3,a4,a5,a6,a7,a8,a9)=>f(BigInt(a0),a1,a2,a3,a4,a5,a6,a7,a8,a9);var makeWrapper__p_____________=f=>(a0,a1,a2,a3,a4,a5,a6,a7,a8,a9,a10,a11,a12,a13)=>f(BigInt(a0),a1,a2,a3,a4,a5,a6,a7,a8,a9,a10,a11,a12,a13);var makeWrapper__p________=f=>(a0,a1,a2,a3,a4,a5,a6,a7,a8)=>f(BigInt(a0),a1,a2,a3,a4,a5,a6,a7,a8);wasmExports["Ob"]=makeWrapper__p(wasmExports["Ob"]);wasmExports["Pb"]=makeWrapper_pp(wasmExports["Pb"]);wasmExports["Vb"]=makeWrapper_ppp(wasmExports["Vb"]);wasmExports["Xb"]=makeWrapper__p(wasmExports["Xb"]);wasmExports["Yb"]=makeWrapper__p(wasmExports["Yb"]);wasmExports["Zb"]=makeWrapper_pp(wasmExports["Zb"]);wasmExports["_b"]=makeWrapper_p(wasmExports["_b"]);wasmExports["$b"]=makeWrapper__p__(wasmExports["$b"]);wasmExports["ac"]=makeWrapper__p__(wasmExports["ac"]);wasmExports["bc"]=makeWrapper__p____(wasmExports["bc"]);wasmExports["cc"]=makeWrapper__p___(wasmExports["cc"]);wasmExports["dc"]=makeWrapper__p___(wasmExports["dc"]);wasmExports["ec"]=makeWrapper__p_(wasmExports["ec"]);wasmExports["fc"]=makeWrapper__p_(wasmExports["fc"]);wasmExports["gc"]=makeWrapper__p____(wasmExports["gc"]);wasmExports["hc"]=makeWrapper__p_(wasmExports["hc"]);wasmExports["ic"]=makeWrapper__p___(wasmExports["ic"]);wasmExports["jc"]=makeWrapper__p______(wasmExports["jc"]);wasmExports["kc"]=makeWrapper__p_______(wasmExports["kc"]);wasmExports["lc"]=makeWrapper__p____(wasmExports["lc"]);wasmExports["mc"]=makeWrapper__p_____(wasmExports["mc"]);wasmExports["nc"]=makeWrapper__p___(wasmExports["nc"]);wasmExports["oc"]=makeWrapper__p_________(wasmExports["oc"]);wasmExports["pc"]=makeWrapper__p__(wasmExports["pc"]);wasmExports["qc"]=makeWrapper__p___(wasmExports["qc"]);wasmExports["rc"]=makeWrapper__p__(wasmExports["rc"]);wasmExports["sc"]=makeWrapper__p___(wasmExports["sc"]);wasmExports["tc"]=makeWrapper__p_(wasmExports["tc"]);wasmExports["uc"]=makeWrapper__p_____(wasmExports["uc"]);wasmExports["vc"]=makeWrapper__p_______(wasmExports["vc"]);wasmExports["wc"]=makeWrapper__p______(wasmExports["wc"]);wasmExports["xc"]=makeWrapper__p__(wasmExports["xc"]);wasmExports["yc"]=makeWrapper__p(wasmExports["yc"]);wasmExports["zc"]=makeWrapper__p(wasmExports["zc"]);wasmExports["Ac"]=makeWrapper__p____(wasmExports["Ac"]);wasmExports["Bc"]=makeWrapper__p_____________(wasmExports["Bc"]);wasmExports["Cc"]=makeWrapper__p____(wasmExports["Cc"]);wasmExports["Dc"]=makeWrapper__p__(wasmExports["Dc"]);wasmExports["Ec"]=makeWrapper__p___(wasmExports["Ec"]);wasmExports["Fc"]=makeWrapper__p_(wasmExports["Fc"]);wasmExports["Gc"]=makeWrapper__p___(wasmExports["Gc"]);wasmExports["Hc"]=makeWrapper__p_____(wasmExports["Hc"]);wasmExports["Ic"]=makeWrapper__p____(wasmExports["Ic"]);wasmExports["Jc"]=makeWrapper__p____(wasmExports["Jc"]);wasmExports["Kc"]=makeWrapper__p__(wasmExports["Kc"]);wasmExports["Lc"]=makeWrapper__p_____(wasmExports["Lc"]);wasmExports["Mc"]=makeWrapper__p___(wasmExports["Mc"]);wasmExports["Nc"]=makeWrapper__p_____(wasmExports["Nc"]);wasmExports["Oc"]=makeWrapper__p____(wasmExports["Oc"]);wasmExports["Pc"]=makeWrapper__p___(wasmExports["Pc"]);wasmExports["Qc"]=makeWrapper__p____(wasmExports["Qc"]);wasmExports["Rc"]=makeWrapper__p(wasmExports["Rc"]);wasmExports["Sc"]=makeWrapper__p___(wasmExports["Sc"]);wasmExports["Tc"]=makeWrapper__p____(wasmExports["Tc"]);wasmExports["Uc"]=makeWrapper__p_(wasmExports["Uc"]);wasmExports["Vc"]=makeWrapper__p__(wasmExports["Vc"]);wasmExports["Wc"]=makeWrapper__p___(wasmExports["Wc"]);wasmExports["Xc"]=makeWrapper__p____(wasmExports["Xc"]);wasmExports["Yc"]=makeWrapper__p_____(wasmExports["Yc"]);wasmExports["Zc"]=makeWrapper__p______(wasmExports["Zc"]);wasmExports["_c"]=makeWrapper__p____(wasmExports["_c"]);wasmExports["$c"]=makeWrapper__p_____(wasmExports["$c"]);wasmExports["ad"]=makeWrapper__p_____(wasmExports["ad"]);wasmExports["bd"]=makeWrapper__p___(wasmExports["bd"]);wasmExports["cd"]=makeWrapper__p_____(wasmExports["cd"]);wasmExports["dd"]=makeWrapper__p_________(wasmExports["dd"]);wasmExports["ed"]=makeWrapper__p____(wasmExports["ed"]);wasmExports["fd"]=makeWrapper__p_____(wasmExports["fd"]);wasmExports["gd"]=makeWrapper__p____(wasmExports["gd"]);wasmExports["hd"]=makeWrapper__p___(wasmExports["hd"]);wasmExports["id"]=makeWrapper__p______(wasmExports["id"]);wasmExports["jd"]=makeWrapper__p______(wasmExports["jd"]);wasmExports["kd"]=makeWrapper__p_____(wasmExports["kd"]);wasmExports["ld"]=makeWrapper__p_____(wasmExports["ld"]);wasmExports["md"]=makeWrapper__p______(wasmExports["md"]);wasmExports["nd"]=makeWrapper__p__(wasmExports["nd"]);wasmExports["od"]=makeWrapper__p__(wasmExports["od"]);wasmExports["pd"]=makeWrapper__p_______(wasmExports["pd"]);wasmExports["qd"]=makeWrapper__p_____(wasmExports["qd"]);wasmExports["rd"]=makeWrapper__p____(wasmExports["rd"]);wasmExports["sd"]=makeWrapper__p____(wasmExports["sd"]);wasmExports["td"]=makeWrapper__p________(wasmExports["td"]);wasmExports["ud"]=makeWrapper__p____(wasmExports["ud"]);wasmExports["vd"]=makeWrapper__p_____(wasmExports["vd"]);wasmExports["wd"]=makeWrapper__p____(wasmExports["wd"]);wasmExports["xd"]=makeWrapper__p___(wasmExports["xd"]);wasmExports["yd"]=makeWrapper__p___(wasmExports["yd"]);wasmExports["zd"]=makeWrapper__p______(wasmExports["zd"]);wasmExports["Ad"]=makeWrapper__p__(wasmExports["Ad"]);wasmExports["Bd"]=makeWrapper__p_______(wasmExports["Bd"]);wasmExports["Cd"]=makeWrapper__p______(wasmExports["Cd"]);wasmExports["Dd"]=makeWrapper__p_____(wasmExports["Dd"]);wasmExports["Ed"]=makeWrapper__p___(wasmExports["Ed"]);wasmExports["Fd"]=makeWrapper__p______(wasmExports["Fd"]);wasmExports["Gd"]=makeWrapper__p___(wasmExports["Gd"]);wasmExports["Hd"]=makeWrapper__p__(wasmExports["Hd"]);wasmExports["Id"]=makeWrapper__p______(wasmExports["Id"]);wasmExports["Jd"]=makeWrapper__p_____(wasmExports["Jd"]);wasmExports["Kd"]=makeWrapper__p_____(wasmExports["Kd"]);wasmExports["Ld"]=makeWrapper__p_____(wasmExports["Ld"]);wasmExports["Md"]=makeWrapper__p_____(wasmExports["Md"]);wasmExports["Nd"]=makeWrapper__p______(wasmExports["Nd"]);wasmExports["Od"]=makeWrapper__p_____(wasmExports["Od"]);wasmExports["Pd"]=makeWrapper__p(wasmExports["Pd"]);wasmExports["Rd"]=makeWrapper__p(wasmExports["Rd"]);return wasmExports}function run(){if(runDependencies>0){dependenciesFulfilled=run;return}preRun();if(runDependencies>0){dependenciesFulfilled=run;return}function doRun(){Module["calledRun"]=true;if(ABORT)return;initRuntime();readyPromiseResolve?.(Module);Module["onRuntimeInitialized"]?.();postRun()}if(Module["setStatus"]){Module["setStatus"]("Running...");setTimeout(()=>{setTimeout(()=>Module["setStatus"](""),1);doRun()},1)}else{doRun()}}var wasmExports;wasmExports=await (createWasm());run();if(runtimeInitialized){moduleRtn=Module}else{moduleRtn=new Promise((resolve,reject)=>{readyPromiseResolve=resolve;readyPromiseReject=reject})} -;return moduleRtn}})();if(typeof exports==="object"&&typeof module==="object"){module.exports=PHP;module.exports.default=PHP}else if(typeof define==="function"&&define["amd"])define([],()=>PHP); +var PHP=(()=>{var _scriptName=globalThis.document?.currentScript?.src;return async function(moduleArg={}){var Module=moduleArg;var ENVIRONMENT_IS_WEB=true;var ENVIRONMENT_IS_WORKER=false;var ENVIRONMENT_IS_NODE=false;var programArgs=[];var thisProgram="./this.program";var quit_=(status,toThrow)=>{throw toThrow};var scriptDirectory="";function locateFile(path){if(Module["locateFile"]){return Module["locateFile"](path,scriptDirectory)}return scriptDirectory+path}var readAsync,readBinary;if(ENVIRONMENT_IS_WEB||ENVIRONMENT_IS_WORKER){try{scriptDirectory=new URL(".",_scriptName).href}catch{}{readAsync=async url=>{var response=await fetch(url,{credentials:"same-origin"});if(response.ok){return response.arrayBuffer()}throw new Error(response.status+" : "+response.url)}}}else{}var out=console.log.bind(console);var err=console.error.bind(console);var wasmBinary;var ABORT=false;var EXITSTATUS;class EmscriptenEH{}class EmscriptenSjLj extends EmscriptenEH{}var runtimeInitialized=false;function getMemoryBuffer(){return wasmMemory.buffer}function updateMemoryViews(){if(HEAP8?.buffer?.resizable)return;var b=getMemoryBuffer();HEAP8=new Int8Array(b);HEAP16=new Int16Array(b);HEAPU8=new Uint8Array(b);HEAPU16=new Uint16Array(b);HEAP32=new Int32Array(b);HEAPU32=new Uint32Array(b);HEAPF32=new Float32Array(b);HEAPF64=new Float64Array(b);HEAP64=new BigInt64Array(b);HEAPU64=new BigUint64Array(b)}function preRun(){var preRun=Module["preRun"];if(preRun){if(typeof preRun=="function")preRun=[preRun];onPreRuns.push(...preRun)}callRuntimeCallbacks(onPreRuns)}function initRuntime(){runtimeInitialized=true;SOCKFS.root=FS.mount(SOCKFS,{},null);if(!Module["noFSInit"]&&!FS.initialized)FS.init();TTY.init();PIPEFS.root=FS.mount(PIPEFS,{},null);wasmExports["Lb"]();FS.ignorePermissions=false}function postRun(){var postRun=Module["postRun"];if(postRun){if(typeof postRun=="function")postRun=[postRun];onPostRuns.push(...postRun)}callRuntimeCallbacks(onPostRuns)}function abort(what){Module["onAbort"]?.(what);what=`Aborted(${what})`;err(what);ABORT=true;what+=". Build with -sASSERTIONS for more info.";var e=new WebAssembly.RuntimeError(what);throw e}var wasmBinaryFile;function findWasmBinary(){return locateFile("php.wasm")}function getBinarySync(file){if(readBinary){return readBinary(file)}throw"both async and sync fetching of the wasm failed"}async function getWasmBinary(binaryFile){if(!wasmBinary){try{var response=await readAsync(binaryFile);return new Uint8Array(response)}catch{}}return getBinarySync(binaryFile)}async function instantiateArrayBuffer(binaryFile,imports){try{var binary=await getWasmBinary(binaryFile);var instance=await WebAssembly.instantiate(binary,imports);return instance}catch(reason){err(`failed to asynchronously prepare wasm: ${reason}`);abort(reason)}}async function instantiateAsync(binary,binaryFile,imports){if(!binary){try{var response=fetch(binaryFile,{credentials:"same-origin"});var instantiationResult=await WebAssembly.instantiateStreaming(response,imports);return instantiationResult}catch(reason){err(`wasm streaming compile failed: ${reason}`);err("falling back to ArrayBuffer instantiation")}}return instantiateArrayBuffer(binaryFile,imports)}function getWasmImports(){var imports={a:wasmImports};return imports}async function createWasm(){function receiveInstance(instance){wasmExports=instance.exports;wasmExports=Asyncify.instrumentWasmExports(wasmExports);wasmExports=applySignatureConversions(wasmExports);assignWasmExports(wasmExports);updateMemoryViews();return wasmExports}function receiveInstantiationResult(result){return receiveInstance(result["instance"])}var info=getWasmImports();var instantiateWasm=Module["instantiateWasm"];if(instantiateWasm){return new Promise(resolve=>{instantiateWasm(info,inst=>resolve(receiveInstance(inst)))})}wasmBinaryFile??=findWasmBinary();var result=await instantiateAsync(wasmBinary,wasmBinaryFile,info);var exports=receiveInstantiationResult(result);return exports}class ExitStatus{name="ExitStatus";constructor(status){this.message=`Program terminated with exit(${status})`;this.status=status}}var HEAP8;var callRuntimeCallbacks=callbacks=>{while(callbacks.length>0){callbacks.shift()(Module)}};var onPostRuns=[];var onPreRuns=[];var dynCalls={};var noExitRuntime=true;var stackRestore=val=>__emscripten_stack_restore(val);var stackSave=()=>_emscripten_stack_get_current();var INT53_MAX=9007199254740992;var INT53_MIN=-9007199254740992;var bigintToI53Checked=num=>numINT53_MAX?NaN:Number(num);var UTF8Decoder=new TextDecoder;var findStringEnd=(heapOrArray,idx,maxBytesToRead,ignoreNul)=>{var maxIdx=idx+maxBytesToRead;if(ignoreNul)return maxIdx;while(heapOrArray[idx]&&!(idx>=maxIdx))++idx;return idx};var HEAPU8;var UTF8ToString=(ptr,maxBytesToRead,ignoreNul)=>{if(!ptr)return"";var end=findStringEnd(HEAPU8,ptr,maxBytesToRead,ignoreNul);return UTF8Decoder.decode(HEAPU8.subarray(ptr,end))};function ___assert_fail(condition,filename,line,func){condition=bigintToI53Checked(condition);filename=bigintToI53Checked(filename);func=bigintToI53Checked(func);return abort(`Assertion failed: ${UTF8ToString(condition)}, at: `+[filename?UTF8ToString(filename):"unknown filename",line,func?UTF8ToString(func):"unknown function"])}var ___call_sighandler=function(fp,sig){fp=bigintToI53Checked(fp);return(a1=>dynCall_vi(fp,a1))(sig)};var initRandomFill=()=>view=>(crypto.getRandomValues(view),0);var randomFill=view=>(randomFill=initRandomFill())(view);var PATH={isAbs:path=>path.charAt(0)==="/",splitPath:filename=>{var splitPathRe=/^(\/?|)([\s\S]*?)((?:\.{1,2}|[^\/]+?|)(\.[^.\/]*|))(?:[\/]*)$/;return splitPathRe.exec(filename).slice(1)},normalizeArray:(parts,allowAboveRoot)=>{var up=0;for(var i=parts.length-1;i>=0;i--){var last=parts[i];if(last==="."){parts.splice(i,1)}else if(last===".."){parts.splice(i,1);up++}else if(up){parts.splice(i,1);up--}}if(allowAboveRoot){for(;up;up--){parts.unshift("..")}}return parts},normalize:path=>{var isAbsolute=PATH.isAbs(path),trailingSlash=path.slice(-1)==="/";path=PATH.normalizeArray(path.split("/").filter(p=>!!p),!isAbsolute).join("/");if(!path&&!isAbsolute){path="."}if(path&&trailingSlash){path+="/"}return(isAbsolute?"/":"")+path},dirname:path=>{var result=PATH.splitPath(path),root=result[0],dir=result[1];if(!root&&!dir){return"."}if(dir){dir=dir.slice(0,-1)}return root+dir},basename:path=>path&&path.match(/([^\/]+|\/)\/*$/)[1],join:(...paths)=>PATH.normalize(paths.join("/")),join2:(l,r)=>PATH.normalize(l+"/"+r)};var PATH_FS={resolve:(...args)=>{var resolvedPath="",resolvedAbsolute=false;for(var i=args.length-1;i>=-1&&!resolvedAbsolute;i--){var path=i>=0?args[i]:FS.cwd();if(typeof path!="string"){throw new TypeError("Arguments to path.resolve must be strings")}else if(!path){return""}resolvedPath=path+"/"+resolvedPath;resolvedAbsolute=PATH.isAbs(path)}resolvedPath=PATH.normalizeArray(resolvedPath.split("/").filter(p=>!!p),!resolvedAbsolute).join("/");return(resolvedAbsolute?"/":"")+resolvedPath||"."},relative:(from,to)=>{from=PATH_FS.resolve(from).slice(1);to=PATH_FS.resolve(to).slice(1);function trim(arr){var start=0;for(;start=0;end--){if(arr[end]!=="")break}if(start>end)return[];return arr.slice(start,end-start+1)}var fromParts=trim(from.split("/"));var toParts=trim(to.split("/"));var length=Math.min(fromParts.length,toParts.length);var samePartsLength=length;for(var i=0;i{var endPtr=findStringEnd(heapOrArray,idx,maxBytesToRead,ignoreNul);return UTF8Decoder.decode(heapOrArray.buffer?heapOrArray.subarray(idx,endPtr):new Uint8Array(heapOrArray.slice(idx,endPtr)))};var FS_stdin_getChar_buffer=[];var lengthBytesUTF8=str=>{var len=0;for(var i=0;i=55296&&c<=57343){len+=4;++i}else{len+=3}}return len};var stringToUTF8Array=(str,heap,outIdx,maxBytesToWrite)=>{if(!(maxBytesToWrite>0))return 0;var startIdx=outIdx;var endIdx=outIdx+maxBytesToWrite-1;for(var i=0;i=endIdx)break;heap[outIdx++]=u}else if(u<=2047){if(outIdx+1>=endIdx)break;heap[outIdx++]=192|u>>6;heap[outIdx++]=128|u&63}else if(u<=65535){if(outIdx+2>=endIdx)break;heap[outIdx++]=224|u>>12;heap[outIdx++]=128|u>>6&63;heap[outIdx++]=128|u&63}else{if(outIdx+3>=endIdx)break;heap[outIdx++]=240|u>>18;heap[outIdx++]=128|u>>12&63;heap[outIdx++]=128|u>>6&63;heap[outIdx++]=128|u&63;i++}}heap[outIdx]=0;return outIdx-startIdx};var intArrayFromString=(stringy,dontAddNull,length)=>{var len=length>0?length:lengthBytesUTF8(stringy)+1;var u8array=new Array(len);var numBytesWritten=stringToUTF8Array(stringy,u8array,0,u8array.length);if(dontAddNull)u8array.length=numBytesWritten;return u8array};var FS_stdin_getChar=()=>{if(!FS_stdin_getChar_buffer.length){var result=null;if(globalThis.window?.prompt){result=window.prompt("Input: ");if(result!==null){result+="\n"}}else{}if(!result){return null}FS_stdin_getChar_buffer=intArrayFromString(result,true)}return FS_stdin_getChar_buffer.shift()};var TTY={ttys:[],init(){},shutdown(){},register(dev,ops){TTY.ttys[dev]={input:[],output:[],ops};FS.registerDevice(dev,TTY.stream_ops)},stream_ops:{open(stream){var tty=TTY.ttys[stream.node.rdev];if(!tty){throw new FS.ErrnoError(43)}stream.tty=tty;stream.seekable=false},close(stream){stream.tty.ops.fsync(stream.tty)},fsync(stream){stream.tty.ops.fsync(stream.tty)},read(stream,buffer,offset,length,pos){if(!stream.tty||!stream.tty.ops.get_char){throw new FS.ErrnoError(60)}var bytesRead=0;for(var i=0;i0){out(UTF8ArrayToString(tty.output));tty.output=[]}},ioctl_tcgets(tty){return{c_iflag:25856,c_oflag:5,c_cflag:191,c_lflag:35387,c_cc:[3,28,127,21,4,0,1,0,17,19,26,0,18,15,23,22,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]}},ioctl_tcsets(tty,optional_actions,data){return 0},ioctl_tiocgwinsz(tty){return[24,80]}},default_tty1_ops:{put_char(tty,val){if(val===null||val===10){err(UTF8ArrayToString(tty.output));tty.output=[]}else{if(val!=0)tty.output.push(val)}},fsync(tty){if(tty.output?.length>0){err(UTF8ArrayToString(tty.output));tty.output=[]}}}};var zeroMemory=(ptr,size)=>HEAPU8.fill(0,ptr,ptr+size);var alignMemory=(size,alignment)=>Math.ceil(size/alignment)*alignment;var mmapAlloc=size=>{size=alignMemory(size,65536);var ptr=_emscripten_builtin_memalign(65536,size);if(ptr)zeroMemory(ptr,size);return ptr};var MEMFS={ops_table:null,mount(mount){return MEMFS.createNode(null,"/",16895,0)},createNode(parent,name,mode,dev){if(FS.isBlkdev(mode)||FS.isFIFO(mode)){throw new FS.ErrnoError(63)}MEMFS.ops_table||={dir:{node:{getattr:MEMFS.node_ops.getattr,setattr:MEMFS.node_ops.setattr,lookup:MEMFS.node_ops.lookup,mknod:MEMFS.node_ops.mknod,rename:MEMFS.node_ops.rename,unlink:MEMFS.node_ops.unlink,rmdir:MEMFS.node_ops.rmdir,readdir:MEMFS.node_ops.readdir,symlink:MEMFS.node_ops.symlink},stream:{llseek:MEMFS.stream_ops.llseek}},file:{node:{getattr:MEMFS.node_ops.getattr,setattr:MEMFS.node_ops.setattr},stream:{llseek:MEMFS.stream_ops.llseek,read:MEMFS.stream_ops.read,write:MEMFS.stream_ops.write,mmap:MEMFS.stream_ops.mmap,msync:MEMFS.stream_ops.msync}},link:{node:{getattr:MEMFS.node_ops.getattr,setattr:MEMFS.node_ops.setattr,readlink:MEMFS.node_ops.readlink},stream:{}},chrdev:{node:{getattr:MEMFS.node_ops.getattr,setattr:MEMFS.node_ops.setattr},stream:FS.chrdev_stream_ops}};var node=FS.createNode(parent,name,mode,dev);if(FS.isDir(node.mode)){node.node_ops=MEMFS.ops_table.dir.node;node.stream_ops=MEMFS.ops_table.dir.stream;node.contents={}}else if(FS.isFile(node.mode)){node.node_ops=MEMFS.ops_table.file.node;node.stream_ops=MEMFS.ops_table.file.stream;node.usedBytes=0;node.contents=MEMFS.emptyFileContents??=new Uint8Array(0)}else if(FS.isLink(node.mode)){node.node_ops=MEMFS.ops_table.link.node;node.stream_ops=MEMFS.ops_table.link.stream}else if(FS.isChrdev(node.mode)){node.node_ops=MEMFS.ops_table.chrdev.node;node.stream_ops=MEMFS.ops_table.chrdev.stream}node.atime=node.mtime=node.ctime=Date.now();if(parent){parent.contents[name]=node;parent.atime=parent.mtime=parent.ctime=node.atime}return node},getFileDataAsTypedArray(node){return node.contents.subarray(0,node.usedBytes)},expandFileStorage(node,newCapacity){var prevCapacity=node.contents.length;if(prevCapacity>=newCapacity)return;var CAPACITY_DOUBLING_MAX=1024*1024;newCapacity=Math.max(newCapacity,prevCapacity*(prevCapacity>>0);if(prevCapacity)newCapacity=Math.max(newCapacity,256);var oldContents=MEMFS.getFileDataAsTypedArray(node);node.contents=new Uint8Array(newCapacity);node.contents.set(oldContents)},resizeFileStorage(node,newSize){if(node.usedBytes==newSize)return;var oldContents=node.contents;node.contents=new Uint8Array(newSize);node.contents.set(oldContents.subarray(0,Math.min(newSize,node.usedBytes)));node.usedBytes=newSize},node_ops:{getattr(node){var attr={};attr.dev=FS.isChrdev(node.mode)?node.id:1;attr.ino=node.id;attr.mode=node.mode;attr.nlink=1;attr.uid=0;attr.gid=0;attr.rdev=node.rdev;if(FS.isDir(node.mode)){attr.size=4096}else if(FS.isFile(node.mode)){attr.size=node.usedBytes}else if(FS.isLink(node.mode)){attr.size=node.link.length}else{attr.size=0}attr.atime=new Date(node.atime);attr.mtime=new Date(node.mtime);attr.ctime=new Date(node.ctime);attr.blksize=4096;attr.blocks=Math.ceil(attr.size/attr.blksize);return attr},setattr(node,attr){for(const key of["mode","atime","mtime","ctime"]){if(attr[key]!=null){node[key]=attr[key]}}if(attr.size!==undefined){MEMFS.resizeFileStorage(node,attr.size)}},lookup(parent,name){if(!MEMFS.doesNotExistError){MEMFS.doesNotExistError=new FS.ErrnoError(44);MEMFS.doesNotExistError.stack=""}throw MEMFS.doesNotExistError},mknod(parent,name,mode,dev){return MEMFS.createNode(parent,name,mode,dev)},rename(old_node,new_dir,new_name){var new_node;try{new_node=FS.lookupNode(new_dir,new_name)}catch(e){}if(new_node){if(FS.isDir(old_node.mode)){for(var i in new_node.contents){throw new FS.ErrnoError(55)}}FS.hashRemoveNode(new_node)}delete old_node.parent.contents[old_node.name];new_dir.contents[new_name]=old_node;old_node.name=new_name;new_dir.ctime=new_dir.mtime=old_node.parent.ctime=old_node.parent.mtime=Date.now()},unlink(parent,name){delete parent.contents[name];parent.ctime=parent.mtime=Date.now()},rmdir(parent,name){var node=FS.lookupNode(parent,name);for(var i in node.contents){throw new FS.ErrnoError(55)}delete parent.contents[name];parent.ctime=parent.mtime=Date.now()},readdir(node){return[".","..",...Object.keys(node.contents)]},symlink(parent,newname,oldpath){var node=MEMFS.createNode(parent,newname,511|40960,0);node.link=oldpath;return node},readlink(node){if(!FS.isLink(node.mode)){throw new FS.ErrnoError(28)}return node.link}},stream_ops:{read(stream,buffer,offset,length,position){var contents=stream.node.contents;if(position>=stream.node.usedBytes)return 0;var size=Math.min(stream.node.usedBytes-position,length);buffer.set(contents.subarray(position,position+size),offset);return size},write(stream,buffer,offset,length,position,canOwn){if(buffer.buffer===HEAP8.buffer){canOwn=false}if(!length)return 0;var node=stream.node;node.mtime=node.ctime=Date.now();if(canOwn){node.contents=buffer.subarray(offset,offset+length);node.usedBytes=length}else if(!node.usedBytes&&!position){node.contents=buffer.slice(offset,offset+length);node.usedBytes=length}else{MEMFS.expandFileStorage(node,position+length);node.contents.set(buffer.subarray(offset,offset+length),position);node.usedBytes=Math.max(node.usedBytes,position+length)}return length},llseek(stream,offset,whence){var position=offset;if(whence===1){position+=stream.position}else if(whence===2){if(FS.isFile(stream.node.mode)){position+=stream.node.usedBytes}}if(position<0){throw new FS.ErrnoError(28)}return position},mmap(stream,length,position,prot,flags){if(!FS.isFile(stream.node.mode)){throw new FS.ErrnoError(43)}var ptr;var allocated;var contents=stream.node.contents;if(!(flags&2)&&contents.buffer===HEAP8.buffer){allocated=false;ptr=contents.byteOffset}else{allocated=true;ptr=mmapAlloc(length);if(!ptr){throw new FS.ErrnoError(48)}if(contents){if(position>0||position+length{if(typeof str!="string")return str;var flagModes={r:0,"r+":2,w:512|64|1,"w+":512|64|2,a:1024|64|1,"a+":1024|64|2};var flags=flagModes[str];if(typeof flags=="undefined"){throw new Error(`Unknown file open mode: ${str}`)}return flags};var FS_fileDataToTypedArray=data=>{if(typeof data=="string"){data=intArrayFromString(data,true)}if(!data.subarray){data=new Uint8Array(data)}return data};var FS_getMode=(canRead,canWrite)=>{var mode=0;if(canRead)mode|=292|73;if(canWrite)mode|=146;return mode};var asyncLoad=async url=>{var arrayBuffer=await readAsync(url);return new Uint8Array(arrayBuffer)};var FS_createDataFile=(...args)=>FS.createDataFile(...args);var getUniqueRunDependency=id=>id;var dependenciesPromise=null;var resolveRunDependencies=async()=>dependenciesPromise;var runDependencies=0;var dependenciesPromiseResolve=null;var removeRunDependency=id=>{runDependencies--;Module["monitorRunDependencies"]?.(runDependencies);if(!runDependencies){dependenciesPromiseResolve()}};var addRunDependency=id=>{if(!runDependencies){dependenciesPromise=new Promise(resolve=>dependenciesPromiseResolve=resolve)}runDependencies++;Module["monitorRunDependencies"]?.(runDependencies)};var preloadPlugins=[];var FS_handledByPreloadPlugin=async(byteArray,fullname)=>{if(typeof Browser!="undefined")Browser.init();for(var plugin of preloadPlugins){if(plugin["canHandle"](fullname)){return plugin["handle"](byteArray,fullname)}}return byteArray};var FS_preloadFile=async(parent,name,url,canRead,canWrite,dontCreateFile,canOwn,preFinish)=>{var fullname=name?PATH_FS.resolve(PATH.join2(parent,name)):parent;var dep=getUniqueRunDependency(`cp ${fullname}`);addRunDependency(dep);try{var byteArray=url;if(typeof url=="string"){byteArray=await asyncLoad(url)}byteArray=await FS_handledByPreloadPlugin(byteArray,fullname);preFinish?.();if(!dontCreateFile){FS_createDataFile(parent,name,byteArray,canRead,canWrite,canOwn)}}finally{removeRunDependency(dep)}};var FS_createPreloadedFile=(parent,name,url,canRead,canWrite,onload,onerror,dontCreateFile,canOwn,preFinish)=>{FS_preloadFile(parent,name,url,canRead,canWrite,dontCreateFile,canOwn,preFinish).then(onload).catch(onerror)};var FS={root:null,mounts:[],devices:{},streams:[],nextInode:1,nameTable:null,currentPath:"/",initialized:false,ignorePermissions:true,filesystems:null,syncFSRequests:0,ErrnoError:class{name="ErrnoError";constructor(errno){this.errno=errno}},FSStream:class{shared={};get object(){return this.node}set object(val){this.node=val}get isRead(){return(this.flags&2097155)!==1}get isWrite(){return(this.flags&2097155)!==0}get isAppend(){return this.flags&1024}get flags(){return this.shared.flags}set flags(val){this.shared.flags=val}get position(){return this.shared.position}set position(val){this.shared.position=val}},FSNode:class{node_ops={};stream_ops={};readMode=292|73;writeMode=146;mounted=null;constructor(parent,name,mode,rdev){if(!parent){parent=this}this.parent=parent;this.mount=parent.mount;this.id=FS.nextInode++;this.name=name;this.mode=mode;this.rdev=rdev;this.atime=this.mtime=this.ctime=Date.now()}get read(){return(this.mode&this.readMode)===this.readMode}set read(val){val?this.mode|=this.readMode:this.mode&=~this.readMode}get write(){return(this.mode&this.writeMode)===this.writeMode}set write(val){val?this.mode|=this.writeMode:this.mode&=~this.writeMode}get isFolder(){return FS.isDir(this.mode)}get isDevice(){return FS.isChrdev(this.mode)}addListener(cb,exclusive=false){var entry={cb,exclusive};var listeners=this.listeners??=new Set;listeners.add(entry);return{listeners,entry}}notifyListeners(flags){if(!this.listeners)return;var excl;for(var entry of this.listeners){if(entry.exclusive)(excl||=[]).push(entry);else entry.cb(flags)}if(excl){var i=(this.exclTurn||0)%excl.length;this.exclTurn=i+1;excl[i].cb(flags)}}},lookupPath(path,opts={}){if(!path){throw new FS.ErrnoError(44)}opts.follow_mount??=true;if(!PATH.isAbs(path)){path=FS.cwd()+"/"+path}linkloop:for(var nlinks=0;nlinks<40;nlinks++){var parts=path.split("/").filter(p=>!!p);var current=FS.root;var current_path="/";for(var i=0;i>>0)%FS.nameTable.length},hashAddNode(node){var hash=FS.hashName(node.parent.id,node.name);node.name_next=FS.nameTable[hash];FS.nameTable[hash]=node},hashRemoveNode(node){var hash=FS.hashName(node.parent.id,node.name);if(FS.nameTable[hash]===node){FS.nameTable[hash]=node.name_next}else{var current=FS.nameTable[hash];while(current){if(current.name_next===node){current.name_next=node.name_next;break}current=current.name_next}}},lookupNode(parent,name){var errCode=FS.mayLookup(parent);if(errCode){throw new FS.ErrnoError(errCode)}var hash=FS.hashName(parent.id,name);for(var node=FS.nameTable[hash];node;node=node.name_next){var nodeName=node.name;if(node.parent.id===parent.id&&nodeName===name){return node}}return FS.lookup(parent,name)},createNode(parent,name,mode,rdev){var node=new FS.FSNode(parent,name,mode,rdev);FS.hashAddNode(node);return node},destroyNode(node){FS.hashRemoveNode(node)},isRoot(node){return node===node.parent},isMountpoint(node){return!!node.mounted},isFile(mode){return(mode&61440)===32768},isDir(mode){return(mode&61440)===16384},isLink(mode){return(mode&61440)===40960},isChrdev(mode){return(mode&61440)===8192},isBlkdev(mode){return(mode&61440)===24576},isFIFO(mode){return(mode&61440)===4096},isSocket(mode){return(mode&49152)===49152},flagsToPermissionString(flag){var perms=["r","w","rw"][flag&3];if(flag&512){perms+="w"}return perms},nodePermissions(node,perms){if(FS.ignorePermissions){return 0}if(perms.includes("r")&&!(node.mode&292)){return 2}if(perms.includes("w")&&!(node.mode&146)){return 2}if(perms.includes("x")&&!(node.mode&73)){return 2}return 0},mayLookup(dir){if(!FS.isDir(dir.mode))return 54;var errCode=FS.nodePermissions(dir,"x");if(errCode)return errCode;if(!dir.node_ops.lookup)return 2;return 0},mayCreate(dir,name){if(!FS.isDir(dir.mode)){return 54}try{var node=FS.lookupNode(dir,name);return 20}catch(e){}return FS.nodePermissions(dir,"wx")},mayDelete(dir,name,isdir){var node;try{node=FS.lookupNode(dir,name)}catch(e){return e.errno}var errCode=FS.nodePermissions(dir,"wx");if(errCode){return errCode}if(isdir){if(!FS.isDir(node.mode)){return 54}if(FS.isRoot(node)||FS.getPath(node)===FS.cwd()){return 10}}else if(FS.isDir(node.mode)){return 31}return 0},mayOpen(node,flags){if(!node){return 44}if(FS.isLink(node.mode)){return 32}var mode=FS.flagsToPermissionString(flags);if(FS.isDir(node.mode)){if(mode!=="r"||flags&(512|64)){return 31}}return FS.nodePermissions(node,mode)},checkOpExists(op,err){if(!op){throw new FS.ErrnoError(err)}return op},MAX_OPEN_FDS:4096,nextfd(){for(var fd=0;fd<=FS.MAX_OPEN_FDS;fd++){if(!FS.streams[fd]){return fd}}throw new FS.ErrnoError(33)},getStreamChecked(fd){var stream=FS.getStream(fd);if(!stream){throw new FS.ErrnoError(8)}return stream},getStream:fd=>FS.streams[fd],createStream(stream,fd=-1){stream=Object.assign(new FS.FSStream,stream);if(fd==-1){fd=FS.nextfd()}stream.fd=fd;FS.streams[fd]=stream;return stream},closeStream(fd){FS.streams[fd]=null},dupStream(origStream,fd=-1){var stream=FS.createStream(origStream,fd);stream.stream_ops?.dup?.(stream);return stream},doSetAttr(stream,node,attr){var setattr=stream?.stream_ops.setattr;var arg=setattr?stream:node;setattr??=node.node_ops.setattr;FS.checkOpExists(setattr,63);try{setattr(arg,attr)}catch(e){if(e instanceof RangeError){throw new FS.ErrnoError(22)}throw e}},chrdev_stream_ops:{open(stream){var device=FS.getDevice(stream.node.rdev);stream.stream_ops=device.stream_ops;stream.stream_ops.open?.(stream)},llseek(){throw new FS.ErrnoError(70)}},major:dev=>dev>>8,minor:dev=>dev&255,makedev:(ma,mi)=>ma<<8|mi,registerDevice(dev,ops){FS.devices[dev]={stream_ops:ops}},getDevice:dev=>FS.devices[dev],getMounts(mount){var mounts=[];var check=[mount];while(check.length){var m=check.pop();mounts.push(m);check.push(...m.mounts)}return mounts},syncfs(populate,callback){if(typeof populate=="function"){callback=populate;populate=false}FS.syncFSRequests++;if(FS.syncFSRequests>1){err(`warning: ${FS.syncFSRequests} FS.syncfs operations in flight at once, probably just doing extra work`)}var mounts=FS.getMounts(FS.root.mount);var completed=0;function doCallback(errCode){FS.syncFSRequests--;return callback(errCode)}function done(errCode){if(errCode){if(!done.errored){done.errored=true;return doCallback(errCode)}return}if(++completed>=mounts.length){doCallback(null)}}for(var mount of mounts){if(mount.type.syncfs){mount.type.syncfs(mount,populate,done)}else{done(null)}}},mount(type,opts,mountpoint){var root=mountpoint==="/";var pseudo=!mountpoint;var node;if(root&&FS.root){throw new FS.ErrnoError(10)}else if(!root&&!pseudo){var lookup=FS.lookupPath(mountpoint,{follow_mount:false});mountpoint=lookup.path;node=lookup.node;if(FS.isMountpoint(node)){throw new FS.ErrnoError(10)}if(!FS.isDir(node.mode)){throw new FS.ErrnoError(54)}}var mount={type,opts,mountpoint,mounts:[]};var mountRoot=type.mount(mount);mountRoot.mount=mount;mount.root=mountRoot;if(root){FS.root=mountRoot}else if(node){node.mounted=mount;if(node.mount){node.mount.mounts.push(mount)}}return mountRoot},unmount(mountpoint){var lookup=FS.lookupPath(mountpoint,{follow_mount:false});if(!FS.isMountpoint(lookup.node)){throw new FS.ErrnoError(28)}var node=lookup.node;var mount=node.mounted;var mounts=FS.getMounts(mount);for(var[hash,current]of Object.entries(FS.nameTable)){while(current){var next=current.name_next;if(mounts.includes(current.mount)){FS.destroyNode(current)}current=next}}node.mounted=null;var idx=node.mount.mounts.indexOf(mount);node.mount.mounts.splice(idx,1)},lookup(parent,name){return parent.node_ops.lookup(parent,name)},mknod(path,mode,dev){var lookup=FS.lookupPath(path,{parent:true});var parent=lookup.node;var name=PATH.basename(path);if(!name){throw new FS.ErrnoError(28)}if(name==="."||name===".."){throw new FS.ErrnoError(20)}var errCode=FS.mayCreate(parent,name);if(errCode){throw new FS.ErrnoError(errCode)}if(!parent.node_ops.mknod){throw new FS.ErrnoError(63)}return parent.node_ops.mknod(parent,name,mode,dev)},statfs(path){return FS.statfsNode(FS.lookupPath(path,{follow:true}).node)},statfsStream(stream){return FS.statfsNode(stream.node)},statfsNode(node){var rtn={bsize:4096,frsize:4096,blocks:1e6,bfree:5e5,bavail:5e5,files:FS.nextInode,ffree:FS.nextInode-1,fsid:42,flags:2,namelen:255};if(node.node_ops.statfs){Object.assign(rtn,node.node_ops.statfs(node.mount.opts.root))}return rtn},create(path,mode=438){mode&=4095;mode|=32768;return FS.mknod(path,mode,0)},mkdir(path,mode=511){mode&=511|512;mode|=16384;return FS.mknod(path,mode,0)},mkdirTree(path,mode){var dirs=path.split("/");var d="";for(var dir of dirs){if(!dir)continue;if(d||PATH.isAbs(path))d+="/";d+=dir;try{FS.mkdir(d,mode)}catch(e){if(e.errno!=20)throw e}}},mkdev(path,mode,dev){if(typeof dev=="undefined"){dev=mode;mode=438}mode|=8192;return FS.mknod(path,mode,dev)},symlink(oldpath,newpath){if(!PATH_FS.resolve(oldpath)){throw new FS.ErrnoError(44)}var lookup=FS.lookupPath(newpath,{parent:true});var parent=lookup.node;if(!parent){throw new FS.ErrnoError(44)}var newname=PATH.basename(newpath);var errCode=FS.mayCreate(parent,newname);if(errCode){throw new FS.ErrnoError(errCode)}if(!parent.node_ops.symlink){throw new FS.ErrnoError(63)}return parent.node_ops.symlink(parent,newname,oldpath)},link(oldpath,newpath,flags){var lookup=FS.lookupPath(newpath,{parent:true});var parent=lookup.node;if(!parent){throw new FS.ErrnoError(44)}var newname=PATH.basename(newpath);var errCode=FS.mayCreate(parent,newname);if(errCode){throw new FS.ErrnoError(errCode)}if(!parent.node_ops.link){throw new FS.ErrnoError(34)}return parent.node_ops.link(parent,newname,oldpath,flags)},rename(old_path,new_path){var old_dirname=PATH.dirname(old_path);var new_dirname=PATH.dirname(new_path);var old_name=PATH.basename(old_path);var new_name=PATH.basename(new_path);var lookup,old_dir,new_dir;lookup=FS.lookupPath(old_path,{parent:true});old_dir=lookup.node;lookup=FS.lookupPath(new_path,{parent:true});new_dir=lookup.node;if(!old_dir||!new_dir)throw new FS.ErrnoError(44);if(old_dir.mount!==new_dir.mount){throw new FS.ErrnoError(75)}var old_node=FS.lookupNode(old_dir,old_name);var relative=PATH_FS.relative(old_path,new_dirname);if(relative.charAt(0)!=="."){throw new FS.ErrnoError(28)}relative=PATH_FS.relative(new_path,old_dirname);if(relative.charAt(0)!=="."){throw new FS.ErrnoError(55)}var new_node;try{new_node=FS.lookupNode(new_dir,new_name)}catch(e){}if(old_node===new_node){return}var isdir=FS.isDir(old_node.mode);var errCode=FS.mayDelete(old_dir,old_name,isdir);if(errCode){throw new FS.ErrnoError(errCode)}errCode=new_node?FS.mayDelete(new_dir,new_name,isdir):FS.mayCreate(new_dir,new_name);if(errCode){throw new FS.ErrnoError(errCode)}if(!old_dir.node_ops.rename){throw new FS.ErrnoError(63)}if(FS.isMountpoint(old_node)||new_node&&FS.isMountpoint(new_node)){throw new FS.ErrnoError(10)}if(new_dir!==old_dir){errCode=FS.nodePermissions(old_dir,"w");if(errCode){throw new FS.ErrnoError(errCode)}}FS.hashRemoveNode(old_node);try{old_dir.node_ops.rename(old_node,new_dir,new_name);old_node.parent=new_dir}catch(e){throw e}finally{FS.hashAddNode(old_node)}},rmdir(path){var lookup=FS.lookupPath(path,{parent:true});var parent=lookup.node;var name=PATH.basename(path);var node=FS.lookupNode(parent,name);var errCode=FS.mayDelete(parent,name,true);if(errCode){throw new FS.ErrnoError(errCode)}if(!parent.node_ops.rmdir){throw new FS.ErrnoError(63)}if(FS.isMountpoint(node)){throw new FS.ErrnoError(10)}parent.node_ops.rmdir(parent,name);FS.destroyNode(node)},readdir(path){var lookup=FS.lookupPath(path,{follow:true});var node=lookup.node;var readdir=FS.checkOpExists(node.node_ops.readdir,54);return readdir(node)},unlink(path){var lookup=FS.lookupPath(path,{parent:true});var parent=lookup.node;if(!parent){throw new FS.ErrnoError(44)}var name=PATH.basename(path);var node=FS.lookupNode(parent,name);var errCode=FS.mayDelete(parent,name,false);if(errCode){throw new FS.ErrnoError(errCode)}if(!parent.node_ops.unlink){throw new FS.ErrnoError(63)}if(FS.isMountpoint(node)){throw new FS.ErrnoError(10)}parent.node_ops.unlink(parent,name);FS.destroyNode(node)},readlink(path){var lookup=FS.lookupPath(path);var link=lookup.node;if(!link){throw new FS.ErrnoError(44)}if(!link.node_ops.readlink){throw new FS.ErrnoError(28)}return link.node_ops.readlink(link)},stat(path,dontFollow){var lookup=FS.lookupPath(path,{follow:!dontFollow});var node=lookup.node;var getattr=FS.checkOpExists(node.node_ops.getattr,63);return getattr(node)},fstat(fd){var stream=FS.getStreamChecked(fd);var node=stream.node;var getattr=stream.stream_ops.getattr;var arg=getattr?stream:node;getattr??=node.node_ops.getattr;FS.checkOpExists(getattr,63);return getattr(arg)},lstat(path){return FS.stat(path,true)},doChmod(stream,node,mode,dontFollow){FS.doSetAttr(stream,node,{mode:mode&4095|node.mode&~4095,ctime:Date.now(),dontFollow})},chmod(path,mode,dontFollow){var node;if(typeof path=="string"){var lookup=FS.lookupPath(path,{follow:!dontFollow});node=lookup.node}else{node=path}FS.doChmod(null,node,mode,dontFollow)},lchmod(path,mode){FS.chmod(path,mode,true)},fchmod(fd,mode){var stream=FS.getStreamChecked(fd);FS.doChmod(stream,stream.node,mode,false)},doChown(stream,node,dontFollow){FS.doSetAttr(stream,node,{timestamp:Date.now(),dontFollow})},chown(path,uid,gid,dontFollow){var node;if(typeof path=="string"){var lookup=FS.lookupPath(path,{follow:!dontFollow});node=lookup.node}else{node=path}FS.doChown(null,node,dontFollow)},lchown(path,uid,gid){FS.chown(path,uid,gid,true)},fchown(fd,uid,gid){var stream=FS.getStreamChecked(fd);FS.doChown(stream,stream.node,false)},doTruncate(stream,node,len){if(FS.isDir(node.mode)){throw new FS.ErrnoError(31)}if(!FS.isFile(node.mode)){throw new FS.ErrnoError(28)}var errCode=FS.nodePermissions(node,"w");if(errCode){throw new FS.ErrnoError(errCode)}FS.doSetAttr(stream,node,{size:len,timestamp:Date.now()})},truncate(path,len){if(len<0){throw new FS.ErrnoError(28)}var node;if(typeof path=="string"){var lookup=FS.lookupPath(path,{follow:true});node=lookup.node}else{node=path}FS.doTruncate(null,node,len)},ftruncate(fd,len){var stream=FS.getStreamChecked(fd);if(len<0||(stream.flags&2097155)===0){throw new FS.ErrnoError(28)}FS.doTruncate(stream,stream.node,len)},utime(path,atime,mtime,dontFollow){var lookup=FS.lookupPath(path,{follow:!dontFollow});FS.doSetAttr(null,lookup.node,{atime,mtime,dontFollow})},open(path,flags,mode=438){if(path===""){throw new FS.ErrnoError(44)}flags=FS_modeStringToFlags(flags);if(flags&64){mode=mode&4095|32768}else{mode=0}var node;var isDirPath;if(typeof path=="object"){node=path}else{isDirPath=path.endsWith("/");var lookup=FS.lookupPath(path,{follow:!(flags&131072),noent_okay:true});node=lookup.node;path=lookup.path}var created=false;if(flags&64){if(node){if(flags&128){throw new FS.ErrnoError(20)}}else if(isDirPath){throw new FS.ErrnoError(31)}else{node=FS.mknod(path,mode|511,0);created=true}}if(!node){throw new FS.ErrnoError(44)}if(FS.isChrdev(node.mode)){flags&=~512}if(flags&65536&&!FS.isDir(node.mode)){throw new FS.ErrnoError(54)}if(!created){var errCode=FS.mayOpen(node,flags);if(errCode){throw new FS.ErrnoError(errCode)}}if(flags&512&&!created){FS.truncate(node,0)}flags&=~(128|512|131072);var stream=FS.createStream({node,path:FS.getPath(node),flags,seekable:true,position:0,stream_ops:node.stream_ops,ungotten:[],error:false});if(stream.stream_ops.open){stream.stream_ops.open(stream)}if(created){FS.chmod(node,mode&511)}return stream},close(stream){if(FS.isClosed(stream)){throw new FS.ErrnoError(8)}if(stream.getdents)stream.getdents=null;stream.node?.notifyListeners(32);try{if(stream.stream_ops.close){stream.stream_ops.close(stream)}}catch(e){throw e}finally{FS.closeStream(stream.fd)}stream.fd=null},isClosed(stream){return stream.fd===null},llseek(stream,offset,whence){if(FS.isClosed(stream)){throw new FS.ErrnoError(8)}if(!stream.seekable||!stream.stream_ops.llseek){throw new FS.ErrnoError(70)}if(whence!=0&&whence!=1&&whence!=2){throw new FS.ErrnoError(28)}stream.position=stream.stream_ops.llseek(stream,offset,whence);stream.ungotten=[];return stream.position},read(stream,buffer,offset,length,position){if(length<0||position<0){throw new FS.ErrnoError(28)}if(FS.isClosed(stream)){throw new FS.ErrnoError(8)}if((stream.flags&2097155)===1){throw new FS.ErrnoError(8)}if(FS.isDir(stream.node.mode)){throw new FS.ErrnoError(31)}if(!stream.stream_ops.read){throw new FS.ErrnoError(28)}var seeking=typeof position!="undefined";if(!seeking){position=stream.position}else if(!stream.seekable){throw new FS.ErrnoError(70)}var bytesRead=stream.stream_ops.read(stream,buffer,offset,length,position);if(!seeking)stream.position+=bytesRead;return bytesRead},write(stream,buffer,offset,length,position,canOwn){if(length<0||position<0){throw new FS.ErrnoError(28)}if(FS.isClosed(stream)){throw new FS.ErrnoError(8)}if((stream.flags&2097155)===0){throw new FS.ErrnoError(8)}if(FS.isDir(stream.node.mode)){throw new FS.ErrnoError(31)}if(!stream.stream_ops.write){throw new FS.ErrnoError(28)}if(stream.seekable&&stream.flags&1024){FS.llseek(stream,0,2)}var seeking=typeof position!="undefined";if(!seeking){position=stream.position}else if(!stream.seekable){throw new FS.ErrnoError(70)}var bytesWritten=stream.stream_ops.write(stream,buffer,offset,length,position,canOwn);if(!seeking)stream.position+=bytesWritten;return bytesWritten},mmap(stream,length,position,prot,flags){if(prot&2&&!(flags&2)&&(stream.flags&2097155)!==2){throw new FS.ErrnoError(2)}if((stream.flags&2097155)===1){throw new FS.ErrnoError(2)}if(!stream.stream_ops.mmap){throw new FS.ErrnoError(43)}if(!length){throw new FS.ErrnoError(28)}return stream.stream_ops.mmap(stream,length,position,prot,flags)},msync(stream,buffer,offset,length,mmapFlags){if(!stream.stream_ops.msync){return 0}return stream.stream_ops.msync(stream,buffer,offset,length,mmapFlags)},ioctl(stream,cmd,arg){if(!stream.stream_ops.ioctl){throw new FS.ErrnoError(59)}return stream.stream_ops.ioctl(stream,cmd,arg)},readFile(path,opts={}){opts.flags=opts.flags??0;opts.encoding=opts.encoding??"binary";if(opts.encoding!=="utf8"&&opts.encoding!=="binary"){abort(`Invalid encoding type "${opts.encoding}"`)}var stream=FS.open(path,opts.flags);var stat=FS.stat(path);var length=stat.size;var buf=new Uint8Array(length);FS.read(stream,buf,0,length,0);if(opts.encoding==="utf8"){buf=UTF8ArrayToString(buf)}FS.close(stream);return buf},writeFile(path,data,opts={}){opts.flags=opts.flags??577;var stream=FS.open(path,opts.flags,opts.mode);data=FS_fileDataToTypedArray(data);FS.write(stream,data,0,data.byteLength,undefined,opts.canOwn);FS.close(stream)},cwd:()=>FS.currentPath,chdir(path){var lookup=FS.lookupPath(path,{follow:true});if(lookup.node===null){throw new FS.ErrnoError(44)}if(!FS.isDir(lookup.node.mode)){throw new FS.ErrnoError(54)}var errCode=FS.nodePermissions(lookup.node,"x");if(errCode){throw new FS.ErrnoError(errCode)}FS.currentPath=lookup.path},createDefaultDirectories(){FS.mkdir("/tmp");FS.mkdir("/home");FS.mkdir("/home/web_user")},createDefaultDevices(){FS.mkdir("/dev");FS.registerDevice(FS.makedev(1,3),{read:()=>0,write:(stream,buffer,offset,length,pos)=>length,llseek:()=>0});FS.mkdev("/dev/null",FS.makedev(1,3));TTY.register(FS.makedev(5,0),TTY.default_tty_ops);TTY.register(FS.makedev(6,0),TTY.default_tty1_ops);FS.mkdev("/dev/tty",FS.makedev(5,0));FS.mkdev("/dev/tty1",FS.makedev(6,0));var randomBuffer=new Uint8Array(1024),randomLeft=0;var randomByte=()=>{if(!randomLeft){randomFill(randomBuffer);randomLeft=randomBuffer.byteLength}return randomBuffer[--randomLeft]};FS.createDevice("/dev","random",randomByte);FS.createDevice("/dev","urandom",randomByte);FS.mkdir("/dev/shm");FS.mkdir("/dev/shm/tmp")},createSpecialDirectories(){FS.mkdir("/proc");var proc_self=FS.mkdir("/proc/self");FS.mkdir("/proc/self/fd");FS.mount({mount(){var node=FS.createNode(proc_self,"fd",16895,73);node.stream_ops={llseek:MEMFS.stream_ops.llseek};node.node_ops={lookup(parent,name){var fd=+name;var stream=FS.getStreamChecked(fd);var ret={parent:null,mount:{mountpoint:"fake"},node_ops:{readlink:()=>stream.path},id:fd+1};ret.parent=ret;return ret},readdir(){return Array.from(FS.streams.entries()).filter(([k,v])=>v).map(([k,v])=>k.toString())}};return node}},{},"/proc/self/fd")},createStandardStreams(input,output,error){if(input){FS.createDevice("/dev","stdin",input)}else{FS.symlink("/dev/tty","/dev/stdin")}if(output){FS.createDevice("/dev","stdout",null,output)}else{FS.symlink("/dev/tty","/dev/stdout")}if(error){FS.createDevice("/dev","stderr",null,error)}else{FS.symlink("/dev/tty1","/dev/stderr")}var stdin=FS.open("/dev/stdin",0);var stdout=FS.open("/dev/stdout",1);var stderr=FS.open("/dev/stderr",1)},staticInit(){FS.nameTable=new Array(4096);FS.mount(MEMFS,{},"/");FS.createDefaultDirectories();FS.createDefaultDevices();FS.createSpecialDirectories();FS.filesystems={MEMFS}},init(input,output,error){FS.initialized=true;input??=Module["stdin"];output??=Module["stdout"];error??=Module["stderr"];FS.createStandardStreams(input,output,error)},quit(){FS.initialized=false;for(var stream of FS.streams){if(stream){FS.close(stream)}}},findObject(path,dontResolveLastLink){var ret=FS.analyzePath(path,dontResolveLastLink);if(!ret.exists){return null}return ret.object},analyzePath(path,dontResolveLastLink){try{var lookup=FS.lookupPath(path,{follow:!dontResolveLastLink});path=lookup.path}catch(e){}var ret={isRoot:false,exists:false,error:0,name:null,path:null,object:null,parentExists:false,parentPath:null,parentObject:null};try{var lookup=FS.lookupPath(path,{parent:true});ret.parentExists=true;ret.parentPath=lookup.path;ret.parentObject=lookup.node;ret.name=PATH.basename(path);lookup=FS.lookupPath(path,{follow:!dontResolveLastLink});ret.exists=true;ret.path=lookup.path;ret.object=lookup.node;ret.name=lookup.node.name;ret.isRoot=lookup.path==="/"}catch(e){ret.error=e.errno}return ret},createPath(parent,path,canRead,canWrite){parent=typeof parent=="string"?parent:FS.getPath(parent);var parts=path.split("/").reverse();while(parts.length){var part=parts.pop();if(!part)continue;var current=PATH.join2(parent,part);try{FS.mkdir(current)}catch(e){if(e.errno!=20)throw e}parent=current}return current},createFile(parent,name,properties,canRead,canWrite){var path=PATH.join2(typeof parent=="string"?parent:FS.getPath(parent),name);var mode=FS_getMode(canRead,canWrite);return FS.create(path,mode)},createDataFile(parent,name,data,canRead,canWrite,canOwn){var path=name;if(parent){parent=typeof parent=="string"?parent:FS.getPath(parent);path=name?PATH.join2(parent,name):parent}var mode=FS_getMode(canRead,canWrite);var node=FS.create(path,mode);if(data){data=FS_fileDataToTypedArray(data);FS.chmod(node,mode|146);var stream=FS.open(node,577);FS.write(stream,data,0,data.length,0,canOwn);FS.close(stream);FS.chmod(node,mode)}},createDevice(parent,name,input,output){var path=PATH.join2(typeof parent=="string"?parent:FS.getPath(parent),name);var mode=FS_getMode(!!input,!!output);FS.createDevice.major??=64;var dev=FS.makedev(FS.createDevice.major++,0);FS.registerDevice(dev,{open(stream){stream.seekable=false},close(stream){if(output?.buffer?.length){output(10)}},read(stream,buffer,offset,length,pos){var bytesRead=0;for(var i=0;ithis.length-1||idx<0){return undefined}var chunkOffset=idx%this.chunkSize;var chunkNum=idx/this.chunkSize|0;return this.getter(chunkNum)[chunkOffset]}setDataGetter(getter){this.getter=getter}cacheLength(){var xhr=new XMLHttpRequest;xhr.open("HEAD",url,false);xhr.send(null);if(!(xhr.status>=200&&xhr.status<300||xhr.status===304))abort(`Couldn't load ${url}. Status: ${xhr.status}`);var datalength=Number(xhr.getResponseHeader("Content-length"));var header;var hasByteServing=(header=xhr.getResponseHeader("Accept-Ranges"))&&header==="bytes";var usesGzip=(header=xhr.getResponseHeader("Content-Encoding"))&&header==="gzip";var chunkSize=1024*1024;if(!hasByteServing)chunkSize=datalength;var doXHR=(from,to)=>{if(from>to)abort(`invalid range (${from}, ${to}) or no bytes requested!`);if(to>datalength-1)abort(`only ${datalength} bytes available! programmer error!`);var xhr=new XMLHttpRequest;xhr.open("GET",url,false);if(datalength!==chunkSize)xhr.setRequestHeader("Range",`bytes=${from}-${to}`);xhr.responseType="arraybuffer";if(xhr.overrideMimeType){xhr.overrideMimeType("text/plain; charset=x-user-defined")}xhr.send(null);if(!(xhr.status>=200&&xhr.status<300||xhr.status===304))abort(`Couldn't load ${url}. Status: ${xhr.status}`);if(xhr.response!==undefined){return new Uint8Array(xhr.response||[])}return intArrayFromString(xhr.responseText??"",true)};var lazyArray=this;lazyArray.setDataGetter(chunkNum=>{var start=chunkNum*chunkSize;var end=(chunkNum+1)*chunkSize-1;end=Math.min(end,datalength-1);if(typeof lazyArray.chunks[chunkNum]=="undefined"){lazyArray.chunks[chunkNum]=doXHR(start,end)}if(typeof lazyArray.chunks[chunkNum]=="undefined")abort("doXHR failed!");return lazyArray.chunks[chunkNum]});if(usesGzip||!datalength){chunkSize=datalength=1;datalength=this.getter(0).length;chunkSize=datalength;out("LazyFiles on gzip forces download of the whole file when length is accessed")}this._length=datalength;this._chunkSize=chunkSize;this.lengthKnown=true}get length(){if(!this.lengthKnown){this.cacheLength()}return this._length}get chunkSize(){if(!this.lengthKnown){this.cacheLength()}return this._chunkSize}}if(globalThis.XMLHttpRequest){if(!ENVIRONMENT_IS_WORKER)abort("Cannot do synchronous binary XHRs outside webworkers in modern browsers. Use --embed-file or --preload-file in emcc");var lazyArray=new LazyUint8Array;var properties={isDevice:false,contents:lazyArray}}else{var properties={isDevice:false,url}}var node=FS.createFile(parent,name,properties,canRead,canWrite);if(properties.contents){node.contents=properties.contents}else if(properties.url){node.contents=null;node.url=properties.url}Object.defineProperties(node,{usedBytes:{get:function(){return this.contents.length}}});var stream_ops={};for(const[key,fn]of Object.entries(node.stream_ops)){stream_ops[key]=(...args)=>{FS.forceLoadFile(node);return fn(...args)}}function writeChunks(stream,buffer,offset,length,position){var contents=stream.node.contents;if(position>=contents.length)return 0;var size=Math.min(contents.length-position,length);if(contents.slice){for(var i=0;i{FS.forceLoadFile(node);return writeChunks(stream,buffer,offset,length,position)};stream_ops.mmap=(stream,length,position,prot,flags)=>{FS.forceLoadFile(node);var ptr=mmapAlloc(length);if(!ptr){throw new FS.ErrnoError(48)}writeChunks(stream,HEAP8,ptr,length,position);return{ptr,allocated:true}};node.stream_ops=stream_ops;return node}};var HEAP32;var SOCKFS={websocketArgs:{},callbacks:{},on(event,callback){SOCKFS.callbacks[event]=callback},emit(event,param){SOCKFS.callbacks[event]?.(param);var fd=event==="error"?param[0]:param;var flags={message:64|1,open:4,connection:64|1,close:1|16,error:8}[event];if(flags)FS.getStream(fd)?.node.notifyListeners(flags)},mount(mount){SOCKFS.websocketArgs=Module["websocket"]||{};(Module["websocket"]??={})["on"]=SOCKFS.on;return FS.createNode(null,"/",16895,0)},createSocket(family,type,protocol){if(family!=2){throw new FS.ErrnoError(5)}type&=~526336;if(type!=1&&type!=2){throw new FS.ErrnoError(28)}var streaming=type==1;if(streaming&&protocol&&protocol!=6){throw new FS.ErrnoError(66)}var sock={family,type,protocol,server:null,error:null,peers:{},pending:[],recv_queue:[],sock_ops:SOCKFS.websocket_sock_ops};var name=SOCKFS.nextname();var node=FS.createNode(SOCKFS.root,name,49152,0);node.sock=sock;var stream=FS.createStream({path:name,node,flags:2,seekable:false,stream_ops:SOCKFS.stream_ops});sock.stream=stream;return sock},getSocket(fd){var stream=FS.getStream(fd);if(!stream||!FS.isSocket(stream.node.mode)){return null}return stream.node.sock},stream_ops:{getattr(stream){var node=stream.node;return{dev:1,ino:node.id,mode:49152|511,nlink:1,uid:0,gid:0,rdev:0,size:0,atime:new Date(0),mtime:new Date(0),ctime:new Date(0),blksize:4096,blocks:0}},poll(stream){var sock=stream.node.sock;return sock.sock_ops.poll(sock)},ioctl(stream,request,varargs){var sock=stream.node.sock;return sock.sock_ops.ioctl(sock,request,varargs)},read(stream,buffer,offset,length,position){var sock=stream.node.sock;var msg=sock.sock_ops.recvmsg(sock,length);if(!msg){return 0}buffer.set(msg.buffer,offset);return msg.buffer.length},write(stream,buffer,offset,length,position){var sock=stream.node.sock;return sock.sock_ops.sendmsg(sock,buffer,offset,length)},close(stream){var sock=stream.node.sock;sock.sock_ops.close(sock)}},nextname(){if(!SOCKFS.nextname.current){SOCKFS.nextname.current=0}return`socket[${SOCKFS.nextname.current++}]`},websocket_sock_ops:{createPeer(sock,addr,port){var ws;if(typeof addr=="object"){ws=addr;addr=null;port=null}if(ws){if(ws._socket){addr=ws._socket.remoteAddress;port=ws._socket.remotePort}else{var result=/ws[s]?:\/\/([^:]+):(\d+)/.exec(ws.url);if(!result){throw new Error("WebSocket URL must be in the format ws(s)://address:port")}addr=result[1];port=parseInt(result[2],10)}}else{try{var url="ws://".replace("#","//");var subProtocols="binary";var opts=undefined;if(SOCKFS.websocketArgs["url"]){url=SOCKFS.websocketArgs["url"]}if(SOCKFS.websocketArgs["subprotocol"]){subProtocols=SOCKFS.websocketArgs["subprotocol"]}else if(SOCKFS.websocketArgs["subprotocol"]===null){subProtocols="null"}if(url==="ws://"||url==="wss://"){var parts=addr.split("/");url=url+parts[0]+":"+port+"/"+parts.slice(1).join("/")}if(subProtocols!=="null"){subProtocols=subProtocols.replace(/^ +| +$/g,"").split(/ *, */);opts=subProtocols}var WebSocketConstructor;{WebSocketConstructor=WebSocket}ws=new WebSocketConstructor(url,opts);ws.binaryType="arraybuffer"}catch(e){throw new FS.ErrnoError(23)}}var peer={addr,port,socket:ws,msg_send_queue:[]};SOCKFS.websocket_sock_ops.addPeer(sock,peer);SOCKFS.websocket_sock_ops.handlePeerEvents(sock,peer);if(sock.type===2&&typeof sock.sport!="undefined"){peer.msg_send_queue.push(new Uint8Array([255,255,255,255,"p".charCodeAt(0),"o".charCodeAt(0),"r".charCodeAt(0),"t".charCodeAt(0),(sock.sport&65280)>>8,sock.sport&255]))}return peer},getPeer(sock,addr,port){return sock.peers[addr+":"+port]},addPeer(sock,peer){sock.peers[peer.addr+":"+peer.port]=peer},removePeer(sock,peer){delete sock.peers[peer.addr+":"+peer.port]},handlePeerEvents(sock,peer){var first=true;function handleOpen(){sock.connecting=false;SOCKFS.emit("open",sock.stream.fd);try{var queued=peer.msg_send_queue.shift();while(queued){peer.socket.send(queued);queued=peer.msg_send_queue.shift()}}catch(e){peer.socket.close()}}function handleMessage(data){if(typeof data=="string"){var encoder=new TextEncoder;data=encoder.encode(data)}else{if(data.byteLength==0){return}data=new Uint8Array(data)}var wasfirst=first;first=false;if(wasfirst&&data.length===10&&data[0]===255&&data[1]===255&&data[2]===255&&data[3]===255&&data[4]==="p".charCodeAt(0)&&data[5]==="o".charCodeAt(0)&&data[6]==="r".charCodeAt(0)&&data[7]==="t".charCodeAt(0)){var newport=data[8]<<8|data[9];SOCKFS.websocket_sock_ops.removePeer(sock,peer);peer.port=newport;SOCKFS.websocket_sock_ops.addPeer(sock,peer);return}sock.recv_queue.push({addr:peer.addr,port:peer.port,data});SOCKFS.emit("message",sock.stream.fd)}peer.socket.onopen=handleOpen;peer.socket.onclose=()=>SOCKFS.emit("close",sock.stream.fd);peer.socket.onmessage=event=>handleMessage(event.data);peer.socket.onerror=error=>{sock.error=14;SOCKFS.emit("error",[sock.stream.fd,sock.error,"ECONNREFUSED: Connection refused"])}},poll(sock){if(sock.type===1&&sock.server){return sock.pending.length?64|1:0}var mask=0;var dest=sock.type===1?SOCKFS.websocket_sock_ops.getPeer(sock,sock.daddr,sock.dport):null;if(sock.recv_queue.length||!dest||dest&&dest.socket.readyState===dest.socket.CLOSING||dest&&dest.socket.readyState===dest.socket.CLOSED){mask|=64|1}if(!dest||dest&&dest.socket.readyState===dest.socket.OPEN){mask|=4}if(dest&&dest.socket.readyState===dest.socket.CLOSING||dest&&dest.socket.readyState===dest.socket.CLOSED){if(sock.connecting){mask|=4}else{mask|=16|8192}}return mask},ioctl(sock,request,arg){switch(request){case 21531:var bytes=0;if(sock.recv_queue.length){bytes=sock.recv_queue[0].data.length}HEAP32[arg>>2]=bytes;return 0;case 21537:var on=HEAP32[arg>>2];if(on){sock.stream.flags|=2048}else{sock.stream.flags&=~2048}return 0;default:return 28}},close(sock){if(sock.server){try{sock.server.close()}catch(e){}sock.server=null}for(var peer of Object.values(sock.peers)){try{peer.socket.close()}catch(e){}SOCKFS.websocket_sock_ops.removePeer(sock,peer)}return 0},bind(sock,addr,port){if(typeof sock.saddr!="undefined"||typeof sock.sport!="undefined"){throw new FS.ErrnoError(28)}sock.saddr=addr;sock.sport=port;if(sock.type===2){if(sock.server){sock.server.close();sock.server=null}try{sock.sock_ops.listen(sock,0)}catch(e){if(!(e.name==="ErrnoError"))throw e;if(e.errno!==138)throw e}}},connect(sock,addr,port){if(sock.server){throw new FS.ErrnoError(138)}if(typeof sock.daddr!="undefined"&&typeof sock.dport!="undefined"){var dest=SOCKFS.websocket_sock_ops.getPeer(sock,sock.daddr,sock.dport);if(dest){if(dest.socket.readyState===dest.socket.CONNECTING){throw new FS.ErrnoError(7)}else{throw new FS.ErrnoError(30)}}}var peer=SOCKFS.websocket_sock_ops.createPeer(sock,addr,port);sock.daddr=peer.addr;sock.dport=peer.port;sock.connecting=true},listen(sock,backlog){if(!ENVIRONMENT_IS_NODE){throw new FS.ErrnoError(138)}},accept(listensock){if(!listensock.server||!listensock.pending.length){throw new FS.ErrnoError(28)}var newsock=listensock.pending.shift();newsock.stream.flags=listensock.stream.flags;return newsock},getname(sock,peer){var addr,port;if(peer){if(sock.daddr===undefined||sock.dport===undefined){throw new FS.ErrnoError(53)}addr=sock.daddr;port=sock.dport}else{addr=sock.saddr||0;port=sock.sport||0}return{addr,port}},sendmsg(sock,buffer,offset,length,addr,port){if(sock.type===2){if(addr===undefined||port===undefined){addr=sock.daddr;port=sock.dport}if(addr===undefined||port===undefined){throw new FS.ErrnoError(17)}}else{addr=sock.daddr;port=sock.dport}var dest=SOCKFS.websocket_sock_ops.getPeer(sock,addr,port);if(sock.type===1){if(!dest||dest.socket.readyState===dest.socket.CLOSING||dest.socket.readyState===dest.socket.CLOSED){throw new FS.ErrnoError(53)}}if(ArrayBuffer.isView(buffer)){offset+=buffer.byteOffset;buffer=buffer.buffer}var data=buffer.slice(offset,offset+length);if(!dest||dest.socket.readyState!==dest.socket.OPEN){if(sock.type===2){if(!dest||dest.socket.readyState===dest.socket.CLOSING||dest.socket.readyState===dest.socket.CLOSED){dest=SOCKFS.websocket_sock_ops.createPeer(sock,addr,port)}}dest.msg_send_queue.push(data);return length}try{dest.socket.send(data);return length}catch(e){throw new FS.ErrnoError(28)}},recvmsg(sock,length,flags){if(sock.type===1&&sock.server){throw new FS.ErrnoError(53)}var peek=flags&2;var queued=sock.recv_queue[0];if(!queued){if(sock.type===1){var dest=SOCKFS.websocket_sock_ops.getPeer(sock,sock.daddr,sock.dport);if(!dest){throw new FS.ErrnoError(53)}if(dest.socket.readyState===dest.socket.CLOSING||dest.socket.readyState===dest.socket.CLOSED){return null}throw new FS.ErrnoError(6)}throw new FS.ErrnoError(6)}var queuedLength=queued.data.byteLength||queued.data.length;var queuedOffset=queued.data.byteOffset||0;var queuedBuffer=queued.data.buffer||queued.data;var bytesRead=Math.min(length,queuedLength);var res={buffer:new Uint8Array(queuedBuffer,queuedOffset,bytesRead),addr:queued.addr,port:queued.port};if(peek)return res;sock.recv_queue.shift();if(sock.type===1&&bytesRead{var socket=SOCKFS.getSocket(fd);if(!socket)throw new FS.ErrnoError(8);return socket};var inetPton4=str=>{var b=str.split(".");for(var i=0;i<4;i++){var tmp=Number(b[i]);if(isNaN(tmp))return null;b[i]=tmp}return(b[0]|b[1]<<8|b[2]<<16|b[3]<<24)>>>0};var inetPton6=str=>{var words;var w,offset,z;var valid6regx=/^((?=.*::)(?!.*::.+::)(::)?([\dA-F]{1,4}:(:|\b)|){5}|([\dA-F]{1,4}:){6})((([\dA-F]{1,4}((?!\3)::|:\b|$))|(?!\2\3)){2}|(((2[0-4]|1\d|[1-9])?\d|25[0-5])\.?\b){4})$/i;var parts=[];if(!valid6regx.test(str)){return null}if(str==="::"){return[0,0,0,0,0,0,0,0]}if(str.startsWith("::")){str=str.replace("::","Z:")}else{str=str.replace("::",":Z:")}if(str.indexOf(".")>0){str=str.replace(new RegExp("[.]","g"),":");words=str.split(":");words[words.length-4]=Number(words[words.length-4])+Number(words[words.length-3])*256;words[words.length-3]=Number(words[words.length-2])+Number(words[words.length-1])*256;words=words.slice(0,words.length-2)}else{words=str.split(":")}offset=0;z=0;for(w=0;w{switch(family){case 2:addr=inetPton4(DNS.lookup_name(addr));zeroMemory(sa,16);if(addrlen){HEAP32[addrlen>>2]=16}HEAP16[sa>>1]=family;HEAP32[sa+4>>2]=addr;HEAP16[sa+2>>1]=_htons(port);break;case 10:addr=inetPton6(DNS.lookup_name(addr));zeroMemory(sa,28);if(addrlen){HEAP32[addrlen>>2]=28}HEAP32[sa>>2]=family;HEAP32[sa+8>>2]=addr[0];HEAP32[sa+12>>2]=addr[1];HEAP32[sa+16>>2]=addr[2];HEAP32[sa+20>>2]=addr[3];HEAP16[sa+2>>1]=_htons(port);break;default:return 5}return 0};function ___syscall_accept4(fd,addr,len,flags,u1,u2){addr=bigintToI53Checked(addr);len=bigintToI53Checked(len);try{var sock=getSocketFromFD(fd);var newsock=sock.sock_ops.accept(sock);if(addr){var errno=writeSockaddr(addr,newsock.family,newsock.daddr,newsock.dport,len)}return newsock.stream.fd}catch(e){if(typeof FS=="undefined"||!(e.name==="ErrnoError"))throw e;return-e.errno}}var inetNtop4=addr=>(addr&255)+"."+(addr>>8&255)+"."+(addr>>16&255)+"."+(addr>>24&255);var inetNtop6=ints=>{var str="";var word=0;var longest=0;var lastzero=0;var zstart=0;var len=0;var i=0;var parts=[ints[0]&65535,ints[0]>>16,ints[1]&65535,ints[1]>>16,ints[2]&65535,ints[2]>>16,ints[3]&65535,ints[3]>>16];var hasipv4=true;var v4part="";for(i=0;i<5;i++){if(parts[i]){hasipv4=false;break}}if(hasipv4){v4part=inetNtop4(parts[6]|parts[7]<<16);if(parts[5]===-1){str="::ffff:";str+=v4part;return str}if(!parts[5]){str="::";if(v4part==="0.0.0.0")v4part="";if(v4part==="0.0.0.1")v4part="1";str+=v4part;return str}}for(word=0;word<8;word++){if(!parts[word]){if(word-lastzero>1){len=0}lastzero=word;len++}if(len>longest){longest=len;zstart=word-longest+1}}for(word=0;word<8;word++){if(longest>1){if(!parts[word]&&word>=zstart&&word{var family=HEAP16[sa>>1];var port=_ntohs(HEAPU16[sa+2>>1]);var addr;switch(family){case 2:if(salen!==16){return{errno:28}}addr=HEAP32[sa+4>>2];addr=inetNtop4(addr);break;case 10:if(salen!==28){return{errno:28}}addr=[HEAP32[sa+8>>2],HEAP32[sa+12>>2],HEAP32[sa+16>>2],HEAP32[sa+20>>2]];addr=inetNtop6(addr);break;default:return{errno:5}}return{family,addr,port}};var getSocketAddress=(addrp,addrlen)=>{var info=readSockaddr(addrp,addrlen);if(info.errno)throw new FS.ErrnoError(info.errno);info.addr=DNS.lookup_addr(info.addr)||info.addr;return info};function ___syscall_bind(fd,addr,len,u1,u2,u3){addr=bigintToI53Checked(addr);try{var sock=getSocketFromFD(fd);var info=getSocketAddress(addr,len);sock.sock_ops.bind(sock,info.addr,info.port);return 0}catch(e){if(typeof FS=="undefined"||!(e.name==="ErrnoError"))throw e;return-e.errno}}var HEAPU32;var HEAP64;var HEAPU64;var SYSCALLS={currentUmask:18,calculateAt(dirfd,path,allowEmpty){if(PATH.isAbs(path)){return path}var dir;if(dirfd===-100){dir=FS.cwd()}else{var dirstream=SYSCALLS.getStreamFromFD(dirfd);dir=dirstream.path}if(path.length==0){if(!allowEmpty){throw new FS.ErrnoError(44)}return dir}return dir+"/"+path},writeStat(buf,stat){HEAPU32[buf>>2]=stat.dev;HEAPU32[buf+4>>2]=stat.mode;HEAPU64[buf+8>>3]=BigInt(stat.nlink);HEAPU32[buf+16>>2]=stat.uid;HEAPU32[buf+20>>2]=stat.gid;HEAPU32[buf+24>>2]=stat.rdev;HEAP64[buf+32>>3]=BigInt(stat.size);HEAP32[buf+40>>2]=4096;HEAP32[buf+44>>2]=stat.blocks;var atime=stat.atime.getTime();var mtime=stat.mtime.getTime();var ctime=stat.ctime.getTime();HEAP64[buf+48>>3]=BigInt(Math.floor(atime/1e3));HEAPU64[buf+56>>3]=BigInt(atime%1e3*1e3*1e3);HEAP64[buf+64>>3]=BigInt(Math.floor(mtime/1e3));HEAPU64[buf+72>>3]=BigInt(mtime%1e3*1e3*1e3);HEAP64[buf+80>>3]=BigInt(Math.floor(ctime/1e3));HEAPU64[buf+88>>3]=BigInt(ctime%1e3*1e3*1e3);HEAP64[buf+96>>3]=BigInt(stat.ino);return 0},writeStatFs(buf,stats){HEAPU32[buf+8>>2]=stats.bsize;HEAPU32[buf+72>>2]=stats.bsize;HEAP64[buf+16>>3]=BigInt(stats.blocks);HEAP64[buf+24>>3]=BigInt(stats.bfree);HEAP64[buf+32>>3]=BigInt(stats.bavail);HEAP64[buf+40>>3]=BigInt(stats.files);HEAP64[buf+48>>3]=BigInt(stats.ffree);HEAPU32[buf+56>>2]=stats.fsid;HEAPU32[buf+80>>2]=stats.flags;HEAPU32[buf+64>>2]=stats.namelen},doMsync(addr,stream,len,flags,offset){if(!FS.isFile(stream.node.mode)){throw new FS.ErrnoError(43)}if(flags&2){return 0}var buffer=HEAPU8.subarray(addr,addr+len);FS.msync(stream,buffer,offset,len,flags)},getStreamFromFD(fd){var stream=FS.getStreamChecked(fd);return stream},varargs:undefined,getStr(ptr){var ret=UTF8ToString(ptr);return ret}};function ___syscall_chdir(path){path=bigintToI53Checked(path);try{path=SYSCALLS.getStr(path);FS.chdir(path);return 0}catch(e){if(typeof FS=="undefined"||!(e.name==="ErrnoError"))throw e;return-e.errno}}function ___syscall_chmod(path,mode){path=bigintToI53Checked(path);try{path=SYSCALLS.getStr(path);FS.chmod(path,mode);return 0}catch(e){if(typeof FS=="undefined"||!(e.name==="ErrnoError"))throw e;return-e.errno}}function ___syscall_connect(fd,addr,len,u1,u2,u3){addr=bigintToI53Checked(addr);try{var sock=getSocketFromFD(fd);var info=getSocketAddress(addr,len);sock.sock_ops.connect(sock,info.addr,info.port);return 0}catch(e){if(typeof FS=="undefined"||!(e.name==="ErrnoError"))throw e;return-e.errno}}function ___syscall_dup(fd){try{var old=SYSCALLS.getStreamFromFD(fd);return FS.dupStream(old).fd}catch(e){if(typeof FS=="undefined"||!(e.name==="ErrnoError"))throw e;return-e.errno}}function ___syscall_faccessat(dirfd,path,amode,flags){path=bigintToI53Checked(path);try{path=SYSCALLS.getStr(path);path=SYSCALLS.calculateAt(dirfd,path);if(amode&~7){return-28}var lookup=FS.lookupPath(path,{follow:true});var node=lookup.node;if(!node){return-44}var perms="";if(amode&4)perms+="r";if(amode&2)perms+="w";if(amode&1)perms+="x";if(perms&&FS.nodePermissions(node,perms)){return-2}return 0}catch(e){if(typeof FS=="undefined"||!(e.name==="ErrnoError"))throw e;return-e.errno}}function ___syscall_fchmod(fd,mode){try{FS.fchmod(fd,mode);return 0}catch(e){if(typeof FS=="undefined"||!(e.name==="ErrnoError"))throw e;return-e.errno}}function ___syscall_fchownat(dirfd,path,owner,group,flags){path=bigintToI53Checked(path);try{path=SYSCALLS.getStr(path);var nofollow=flags&256;flags=flags&~256;path=SYSCALLS.calculateAt(dirfd,path);(nofollow?FS.lchown:FS.chown)(path,owner,group);return 0}catch(e){if(typeof FS=="undefined"||!(e.name==="ErrnoError"))throw e;return-e.errno}}var syscallGetVarargP=()=>{var ret=Number(HEAPU64[SYSCALLS.varargs>>3]);SYSCALLS.varargs+=8;return ret};var syscallGetVarargI=()=>{var ret=HEAP32[+SYSCALLS.varargs>>2];SYSCALLS.varargs+=4;return ret};function ___syscall_fcntl64(fd,cmd,varargs){varargs=bigintToI53Checked(varargs);SYSCALLS.varargs=varargs;try{var stream=SYSCALLS.getStreamFromFD(fd);switch(cmd){case 0:{var arg=syscallGetVarargI();if(arg<0){return-28}while(FS.streams[arg]){arg++}var newStream;newStream=FS.dupStream(stream,arg);return newStream.fd}case 1:case 2:return 0;case 3:return stream.flags;case 4:{var arg=syscallGetVarargI();var mask=289792;stream.flags=stream.flags&~mask|arg&mask;return 0}case 5:{var arg=syscallGetVarargP();var offset=0;HEAP16[arg+offset>>1]=2;return 0}case 6:case 7:return 0}return-28}catch(e){if(typeof FS=="undefined"||!(e.name==="ErrnoError"))throw e;return-e.errno}}function ___syscall_fdatasync(fd){try{var stream=SYSCALLS.getStreamFromFD(fd);return 0}catch(e){if(typeof FS=="undefined"||!(e.name==="ErrnoError"))throw e;return-e.errno}}function ___syscall_fstat64(fd,buf){buf=bigintToI53Checked(buf);try{return SYSCALLS.writeStat(buf,FS.fstat(fd))}catch(e){if(typeof FS=="undefined"||!(e.name==="ErrnoError"))throw e;return-e.errno}}function ___syscall_ftruncate64(fd,length){length=bigintToI53Checked(length);try{if(isNaN(length))return-22;FS.ftruncate(fd,length);return 0}catch(e){if(typeof FS=="undefined"||!(e.name==="ErrnoError"))throw e;return-e.errno}}var stringToUTF8=(str,outPtr,maxBytesToWrite)=>stringToUTF8Array(str,HEAPU8,outPtr,maxBytesToWrite);function ___syscall_getcwd(buf,size){buf=bigintToI53Checked(buf);size=bigintToI53Checked(size);try{if(!size)return-28;var cwd=FS.cwd();var cwdLengthInBytes=lengthBytesUTF8(cwd)+1;if(size>3]=BigInt(id);HEAP64[dirp+pos+8>>3]=BigInt((idx+1)*struct_size);HEAP16[dirp+pos+16>>1]=280;HEAP8[dirp+pos+18]=type;stringToUTF8(name,dirp+pos+19,256);pos+=struct_size}FS.llseek(stream,idx*struct_size,0);return pos}catch(e){if(typeof FS=="undefined"||!(e.name==="ErrnoError"))throw e;return-e.errno}}var ___syscall_geteuid32=()=>0;var ___syscall_getgid32=()=>0;function ___syscall_getpeername(fd,addr,len,u1,u2,u3){addr=bigintToI53Checked(addr);len=bigintToI53Checked(len);try{var sock=getSocketFromFD(fd);if(!sock.daddr){return-53}var errno=writeSockaddr(addr,sock.family,sock.daddr,sock.dport,len);return 0}catch(e){if(typeof FS=="undefined"||!(e.name==="ErrnoError"))throw e;return-e.errno}}function ___syscall_getsockname(fd,addr,len,u1,u2,u3){addr=bigintToI53Checked(addr);len=bigintToI53Checked(len);try{var sock=getSocketFromFD(fd);var defaultAddr="0.0.0.0";var errno=writeSockaddr(addr,sock.family,sock.saddr||defaultAddr,sock.sport,len);return 0}catch(e){if(typeof FS=="undefined"||!(e.name==="ErrnoError"))throw e;return-e.errno}}function ___syscall_getsockopt(fd,level,optname,optval,optlen,unused){optval=bigintToI53Checked(optval);optlen=bigintToI53Checked(optlen);try{var sock=getSocketFromFD(fd);if(level===1){if(optname===4){HEAP32[optval>>2]=sock.error;HEAP32[optlen>>2]=4;sock.error=null;return 0}}return-50}catch(e){if(typeof FS=="undefined"||!(e.name==="ErrnoError"))throw e;return-e.errno}}var ___syscall_getuid32=()=>0;function ___syscall_ioctl(fd,op,varargs){varargs=bigintToI53Checked(varargs);SYSCALLS.varargs=varargs;try{var stream=SYSCALLS.getStreamFromFD(fd);switch(op){case 21509:{if(!stream.tty)return-59;return 0}case 21505:{if(!stream.tty)return-59;if(stream.tty.ops.ioctl_tcgets){var termios=stream.tty.ops.ioctl_tcgets(stream);var argp=syscallGetVarargP();HEAP32[argp>>2]=termios.c_iflag||0;HEAP32[argp+4>>2]=termios.c_oflag||0;HEAP32[argp+8>>2]=termios.c_cflag||0;HEAP32[argp+12>>2]=termios.c_lflag||0;for(var i=0;i<32;i++){HEAP8[argp+i+17]=termios.c_cc[i]||0}return 0}return 0}case 21510:case 21511:case 21512:{if(!stream.tty)return-59;return 0}case 21506:case 21507:case 21508:{if(!stream.tty)return-59;if(stream.tty.ops.ioctl_tcsets){var argp=syscallGetVarargP();var c_iflag=HEAP32[argp>>2];var c_oflag=HEAP32[argp+4>>2];var c_cflag=HEAP32[argp+8>>2];var c_lflag=HEAP32[argp+12>>2];var c_cc=[];for(var i=0;i<32;i++){c_cc.push(HEAP8[argp+i+17])}return stream.tty.ops.ioctl_tcsets(stream.tty,op,{c_iflag,c_oflag,c_cflag,c_lflag,c_cc})}return 0}case 21519:{if(!stream.tty)return-59;var argp=syscallGetVarargP();HEAP32[argp>>2]=0;return 0}case 21520:{if(!stream.tty)return-59;return-28}case 21537:case 21531:{var argp=syscallGetVarargP();return FS.ioctl(stream,op,argp)}case 21523:{if(!stream.tty)return-59;if(stream.tty.ops.ioctl_tiocgwinsz){var winsize=stream.tty.ops.ioctl_tiocgwinsz(stream.tty);var argp=syscallGetVarargP();HEAP16[argp>>1]=winsize[0];HEAP16[argp+2>>1]=winsize[1]}return 0}case 21524:{if(!stream.tty)return-59;return 0}case 21515:{if(!stream.tty)return-59;return 0}default:return-28}}catch(e){if(typeof FS=="undefined"||!(e.name==="ErrnoError"))throw e;return-e.errno}}function ___syscall_linkat(olddirfd,oldpath,newdirfd,newpath,flags){oldpath=bigintToI53Checked(oldpath);newpath=bigintToI53Checked(newpath);try{oldpath=SYSCALLS.getStr(oldpath);newpath=SYSCALLS.getStr(newpath);oldpath=SYSCALLS.calculateAt(olddirfd,oldpath);newpath=SYSCALLS.calculateAt(newdirfd,newpath);FS.link(oldpath,newpath,flags);return 0}catch(e){if(typeof FS=="undefined"||!(e.name==="ErrnoError"))throw e;return-e.errno}}function ___syscall_listen(fd,backlog,u1,u2,u3,u4){try{var sock=getSocketFromFD(fd);sock.sock_ops.listen(sock,backlog);return 0}catch(e){if(typeof FS=="undefined"||!(e.name==="ErrnoError"))throw e;return-e.errno}}function ___syscall_lstat64(path,buf){path=bigintToI53Checked(path);buf=bigintToI53Checked(buf);try{path=SYSCALLS.getStr(path);return SYSCALLS.writeStat(buf,FS.lstat(path))}catch(e){if(typeof FS=="undefined"||!(e.name==="ErrnoError"))throw e;return-e.errno}}function ___syscall_mkdirat(dirfd,path,mode){path=bigintToI53Checked(path);try{path=SYSCALLS.getStr(path);path=SYSCALLS.calculateAt(dirfd,path);mode&=~SYSCALLS.currentUmask;FS.mkdir(path,mode,0);return 0}catch(e){if(typeof FS=="undefined"||!(e.name==="ErrnoError"))throw e;return-e.errno}}function ___syscall_newfstatat(dirfd,path,buf,flags){path=bigintToI53Checked(path);buf=bigintToI53Checked(buf);try{path=SYSCALLS.getStr(path);var nofollow=flags&256;var allowEmpty=flags&4096;flags=flags&~6400;path=SYSCALLS.calculateAt(dirfd,path,allowEmpty);return SYSCALLS.writeStat(buf,nofollow?FS.lstat(path):FS.stat(path))}catch(e){if(typeof FS=="undefined"||!(e.name==="ErrnoError"))throw e;return-e.errno}}function ___syscall_openat(dirfd,path,flags,varargs){path=bigintToI53Checked(path);varargs=bigintToI53Checked(varargs);SYSCALLS.varargs=varargs;try{path=SYSCALLS.getStr(path);path=SYSCALLS.calculateAt(dirfd,path);var mode=varargs?syscallGetVarargI():0;if(flags&64){mode&=~SYSCALLS.currentUmask}return FS.open(path,flags,mode).fd}catch(e){if(typeof FS=="undefined"||!(e.name==="ErrnoError"))throw e;return-e.errno}}var PIPEFS={BUCKET_BUFFER_SIZE:8192,mount(mount){return FS.createNode(null,"/",16384|511,0)},createPipe(){var pipe={buckets:[],writerCount:1,writeClosed:false,readerCount:1,readClosed:false,timestamp:new Date};pipe.buckets.push({buffer:new Uint8Array(PIPEFS.BUCKET_BUFFER_SIZE),offset:0,roffset:0});var rName=PIPEFS.nextname();var wName=PIPEFS.nextname();var rNode=FS.createNode(PIPEFS.root,rName,4096,0);var wNode=FS.createNode(PIPEFS.root,wName,4096,0);rNode.pipe=pipe;wNode.pipe=pipe;pipe.readNode=rNode;pipe.writeNode=wNode;var readableStream=FS.createStream({path:rName,node:rNode,flags:0,seekable:false,stream_ops:PIPEFS.stream_ops});rNode.stream=readableStream;var writableStream=FS.createStream({path:wName,node:wNode,flags:1,seekable:false,stream_ops:PIPEFS.stream_ops});wNode.stream=writableStream;return{readable_fd:readableStream.fd,writable_fd:writableStream.fd}},stream_ops:{getattr(stream){var node=stream.node;var timestamp=node.pipe.timestamp;return{dev:14,ino:node.id,mode:4480,nlink:1,uid:0,gid:0,rdev:0,size:0,atime:timestamp,mtime:timestamp,ctime:timestamp,blksize:4096,blocks:0}},poll(stream){var pipe=stream.node.pipe;if((stream.flags&2097155)===1){var mask=256|4;if(pipe.readClosed){mask|=8}return mask}var mask=0;for(var bucket of pipe.buckets){if(bucket.offset-bucket.roffset>0){mask=64|1;break}}if(pipe.writeClosed){mask|=16|1}return mask},dup(stream){var pipe=stream.node.pipe;if((stream.flags&2097155)===1){pipe.writerCount++}else{pipe.readerCount++}},ioctl(stream,request,argp){if(request==21531){var pipe=stream.node.pipe;var currentLength=0;for(var bucket of pipe.buckets){currentLength+=bucket.offset-bucket.roffset}HEAP32[argp>>2]=currentLength;return 0}return 28},fsync(stream){return 28},read(stream,buffer,offset,length,position){var pipe=stream.node.pipe;var currentLength=0;for(var bucket of pipe.buckets){currentLength+=bucket.offset-bucket.roffset}var data=buffer.subarray(offset,offset+length);if(length<=0){return 0}if(currentLength==0){throw new FS.ErrnoError(6)}var toRead=Math.min(currentLength,length);var totalRead=toRead;var toRemove=0;for(var bucket of pipe.buckets){var bucketSize=bucket.offset-bucket.roffset;if(toRead<=bucketSize){var tmpSlice=bucket.buffer.subarray(bucket.roffset,bucket.offset);if(toRead=dataLen){currBucket.buffer.set(data,currBucket.offset);currBucket.offset+=dataLen;pipe.readNode.notifyListeners(64|1);return dataLen}else if(freeBytesInCurrBuffer>0){currBucket.buffer.set(data.subarray(0,freeBytesInCurrBuffer),currBucket.offset);currBucket.offset+=freeBytesInCurrBuffer;data=data.subarray(freeBytesInCurrBuffer,data.byteLength)}var numBuckets=data.byteLength/PIPEFS.BUCKET_BUFFER_SIZE|0;var remElements=data.byteLength%PIPEFS.BUCKET_BUFFER_SIZE;for(var i=0;i0){var newBucket={buffer:new Uint8Array(PIPEFS.BUCKET_BUFFER_SIZE),offset:data.byteLength,roffset:0};pipe.buckets.push(newBucket);newBucket.buffer.set(data)}pipe.readNode.notifyListeners(64|1);return dataLen},close(stream){var pipe=stream.node.pipe;if((stream.flags&2097155)===1){if(!--pipe.writerCount){pipe.writeClosed=true;pipe.readNode.notifyListeners(16|64|1)}}else if(!--pipe.readerCount){pipe.readClosed=true;pipe.writeNode.notifyListeners(8|256|4)}if(!pipe.readerCount&&!pipe.writerCount){pipe.buckets=null}}},nextname(){if(!PIPEFS.nextname.current){PIPEFS.nextname.current=0}return"pipe["+PIPEFS.nextname.current+++"]"}};function ___syscall_pipe2(fdPtr,flags){fdPtr=bigintToI53Checked(fdPtr);try{if(fdPtr==0){throw new FS.ErrnoError(21)}var validFlags=524288|2048;if(flags&~validFlags){throw new FS.ErrnoError(138)}var res=PIPEFS.createPipe();if(flags&2048){FS.getStream(res.readable_fd).flags|=2048;FS.getStream(res.writable_fd).flags|=2048}HEAP32[fdPtr>>2]=res.readable_fd;HEAP32[fdPtr+4>>2]=res.writable_fd;return 0}catch(e){if(typeof FS=="undefined"||!(e.name==="ErrnoError"))throw e;return-e.errno}}var pollOne=(fd,events)=>{var stream=FS.getStream(fd);if(!stream)return 32;var flags=stream.stream_ops?.poll?.(stream)??5;return flags&(events|8|16|32)};var doPollSync=(fds,nfds)=>{var count=0;for(var i=0,pollfd=fds;i>2],HEAP16[pollfd+4>>1]);if(revents)count++;HEAP16[pollfd+6>>1]=revents}return count};var doPollAsync=(fds,nfds,timeout)=>new Promise(resolve=>{var regs=[];var timer;var done=false;function derive(){var count=0;for(var i=0,pollfd=fds;i>2],HEAP16[pollfd+4>>1]);if(revents)count++;HEAP16[pollfd+6>>1]=revents}return count}function finish(count){if(done)return;done=true;for(var r of regs)r.listeners.delete(r.entry);if(timer)clearTimeout(timer);resolve(count)}var count=derive();if(count||!timeout){finish(count)}else{function recheck(){if(done)return;var c=derive();if(c)finish(c)}for(var i=0,pollfd=fds;i>2]);if(stream)regs.push(stream.node.addListener(recheck))}if(timeout>0)timer=setTimeout(()=>finish(0),timeout)}});var ___syscall_poll=function(fds,nfds,timeout){let innerFunc=()=>{fds=bigintToI53Checked(fds);try{const isAsyncContext=true;if(isAsyncContext){return doPollAsync(fds,nfds,timeout)}var count=doPollSync(fds,nfds);return count}catch(e){if(typeof FS=="undefined"||!(e.name==="ErrnoError"))throw e;return-e.errno}};return Asyncify.handleAsync(innerFunc)};___syscall_poll.isAsync=true;function ___syscall_poll_nonblocking(fds,nfds){fds=bigintToI53Checked(fds);try{return doPollSync(fds,nfds)}catch(e){if(typeof FS=="undefined"||!(e.name==="ErrnoError"))throw e;return-e.errno}}function ___syscall_readlinkat(dirfd,path,buf,bufsize){path=bigintToI53Checked(path);buf=bigintToI53Checked(buf);bufsize=bigintToI53Checked(bufsize);try{path=SYSCALLS.getStr(path);path=SYSCALLS.calculateAt(dirfd,path);if(bufsize<=0)return-28;var ret=FS.readlink(path);var len=Math.min(bufsize,lengthBytesUTF8(ret));var endChar=HEAP8[buf+len];stringToUTF8(ret,buf,bufsize+1);HEAP8[buf+len]=endChar;return len}catch(e){if(typeof FS=="undefined"||!(e.name==="ErrnoError"))throw e;return-e.errno}}function ___syscall_recvfrom(fd,buf,len,flags,addr,alen){buf=bigintToI53Checked(buf);len=bigintToI53Checked(len);addr=bigintToI53Checked(addr);alen=bigintToI53Checked(alen);try{var sock=getSocketFromFD(fd);var msg=sock.sock_ops.recvmsg(sock,len,flags);if(!msg)return 0;if(addr){var errno=writeSockaddr(addr,sock.family,msg.addr,msg.port,alen)}HEAPU8.set(msg.buffer,buf);return msg.buffer.byteLength}catch(e){if(typeof FS=="undefined"||!(e.name==="ErrnoError"))throw e;return-e.errno}}function ___syscall_renameat(olddirfd,oldpath,newdirfd,newpath){oldpath=bigintToI53Checked(oldpath);newpath=bigintToI53Checked(newpath);try{oldpath=SYSCALLS.getStr(oldpath);newpath=SYSCALLS.getStr(newpath);oldpath=SYSCALLS.calculateAt(olddirfd,oldpath);newpath=SYSCALLS.calculateAt(newdirfd,newpath);FS.rename(oldpath,newpath);return 0}catch(e){if(typeof FS=="undefined"||!(e.name==="ErrnoError"))throw e;return-e.errno}}function ___syscall_rmdir(path){path=bigintToI53Checked(path);try{path=SYSCALLS.getStr(path);FS.rmdir(path);return 0}catch(e){if(typeof FS=="undefined"||!(e.name==="ErrnoError"))throw e;return-e.errno}}function ___syscall_sendto(fd,buf,len,flags,addr,alen){buf=bigintToI53Checked(buf);len=bigintToI53Checked(len);addr=bigintToI53Checked(addr);try{var sock=getSocketFromFD(fd);if(!addr){return FS.write(sock.stream,HEAP8,buf,len)}var dest=getSocketAddress(addr,alen);return sock.sock_ops.sendmsg(sock,HEAP8,buf,len,dest.addr,dest.port)}catch(e){if(typeof FS=="undefined"||!(e.name==="ErrnoError"))throw e;return-e.errno}}function ___syscall_setsockopt(fd,level,optname,optval,optlen,unused){optval=bigintToI53Checked(optval);try{getSocketFromFD(fd);return-50}catch(e){if(typeof FS=="undefined"||!(e.name==="ErrnoError"))throw e;return-e.errno}}function ___syscall_shutdown(fd,how,u1,u2,u3,u4){try{var sock=getSocketFromFD(fd);return-52}catch(e){if(typeof FS=="undefined"||!(e.name==="ErrnoError"))throw e;return-e.errno}}function ___syscall_socket(domain,type,protocol,u1,u2,u3){try{var sock=SOCKFS.createSocket(domain,type,protocol);return sock.stream.fd}catch(e){if(typeof FS=="undefined"||!(e.name==="ErrnoError"))throw e;return-e.errno}}function ___syscall_stat64(path,buf){path=bigintToI53Checked(path);buf=bigintToI53Checked(buf);try{path=SYSCALLS.getStr(path);return SYSCALLS.writeStat(buf,FS.stat(path))}catch(e){if(typeof FS=="undefined"||!(e.name==="ErrnoError"))throw e;return-e.errno}}function ___syscall_statfs64(path,size,buf){path=bigintToI53Checked(path);size=bigintToI53Checked(size);buf=bigintToI53Checked(buf);try{SYSCALLS.writeStatFs(buf,FS.statfs(SYSCALLS.getStr(path)));return 0}catch(e){if(typeof FS=="undefined"||!(e.name==="ErrnoError"))throw e;return-e.errno}}function ___syscall_symlinkat(target,dirfd,linkpath){target=bigintToI53Checked(target);linkpath=bigintToI53Checked(linkpath);try{target=SYSCALLS.getStr(target);linkpath=SYSCALLS.getStr(linkpath);linkpath=SYSCALLS.calculateAt(dirfd,linkpath);FS.symlink(target,linkpath);return 0}catch(e){if(typeof FS=="undefined"||!(e.name==="ErrnoError"))throw e;return-e.errno}}function ___syscall_umask(mask){try{var old=SYSCALLS.currentUmask;SYSCALLS.currentUmask=mask;return old}catch(e){if(typeof FS=="undefined"||!(e.name==="ErrnoError"))throw e;return-e.errno}}function ___syscall_unlinkat(dirfd,path,flags){path=bigintToI53Checked(path);try{path=SYSCALLS.getStr(path);path=SYSCALLS.calculateAt(dirfd,path);if(!flags){FS.unlink(path)}else if(flags===512){FS.rmdir(path)}else{return-28}return 0}catch(e){if(typeof FS=="undefined"||!(e.name==="ErrnoError"))throw e;return-e.errno}}var readI53FromI64=ptr=>HEAPU32[ptr>>2]+HEAP32[ptr+4>>2]*4294967296;function ___syscall_utimensat(dirfd,path,times,flags){path=bigintToI53Checked(path);times=bigintToI53Checked(times);try{var nofollow=flags&256;path=SYSCALLS.getStr(path);path=SYSCALLS.calculateAt(dirfd,path,true);var now=Date.now(),atime,mtime;if(!times){atime=now;mtime=now}else{var seconds=readI53FromI64(times);var nanoseconds=HEAP32[times+8>>2];if(nanoseconds==1073741823){atime=now}else if(nanoseconds==1073741822){atime=null}else{atime=seconds*1e3+nanoseconds/(1e3*1e3)}times+=16;seconds=readI53FromI64(times);nanoseconds=HEAP32[times+8>>2];if(nanoseconds==1073741823){mtime=now}else if(nanoseconds==1073741822){mtime=null}else{mtime=seconds*1e3+nanoseconds/(1e3*1e3)}}if((mtime??atime)!==null){FS.utime(path,atime,mtime,nofollow)}return 0}catch(e){if(typeof FS=="undefined"||!(e.name==="ErrnoError"))throw e;return-e.errno}}var __abort_js=()=>abort("");function __emscripten_lookup_name(name){name=bigintToI53Checked(name);var nameString=UTF8ToString(name);return inetPton4(DNS.lookup_name(nameString))}var runtimeKeepaliveCounter=0;var __emscripten_runtime_keepalive_clear=()=>{noExitRuntime=false;runtimeKeepaliveCounter=0};var __emscripten_throw_longjmp=()=>{throw new EmscriptenSjLj};function __gmtime_js(time,tmPtr){time=bigintToI53Checked(time);tmPtr=bigintToI53Checked(tmPtr);var date=new Date(time*1e3);if(isNaN(date.getTime())){return 1}HEAP32[tmPtr>>2]=date.getUTCSeconds();HEAP32[tmPtr+4>>2]=date.getUTCMinutes();HEAP32[tmPtr+8>>2]=date.getUTCHours();HEAP32[tmPtr+12>>2]=date.getUTCDate();HEAP32[tmPtr+16>>2]=date.getUTCMonth();HEAP32[tmPtr+20>>2]=date.getUTCFullYear()-1900;HEAP32[tmPtr+24>>2]=date.getUTCDay();var start=Date.UTC(date.getUTCFullYear(),0,1,0,0,0,0);var yday=(date.getTime()-start)/(1e3*60*60*24)|0;HEAP32[tmPtr+28>>2]=yday;return 0}var isLeapYear=year=>year%4===0&&(year%100!==0||year%400===0);var MONTH_DAYS_LEAP_CUMULATIVE=[0,31,60,91,121,152,182,213,244,274,305,335];var MONTH_DAYS_REGULAR_CUMULATIVE=[0,31,59,90,120,151,181,212,243,273,304,334];var ydayFromDate=date=>{var leap=isLeapYear(date.getFullYear());var monthDaysCumulative=leap?MONTH_DAYS_LEAP_CUMULATIVE:MONTH_DAYS_REGULAR_CUMULATIVE;var yday=monthDaysCumulative[date.getMonth()]+date.getDate()-1;return yday};function __localtime_js(time,tmPtr){time=bigintToI53Checked(time);tmPtr=bigintToI53Checked(tmPtr);var date=new Date(time*1e3);if(isNaN(date.getTime())){return 1}HEAP32[tmPtr>>2]=date.getSeconds();HEAP32[tmPtr+4>>2]=date.getMinutes();HEAP32[tmPtr+8>>2]=date.getHours();HEAP32[tmPtr+12>>2]=date.getDate();HEAP32[tmPtr+16>>2]=date.getMonth();HEAP32[tmPtr+20>>2]=date.getFullYear()-1900;HEAP32[tmPtr+24>>2]=date.getDay();var yday=ydayFromDate(date)|0;HEAP32[tmPtr+28>>2]=yday;HEAP64[tmPtr+40>>3]=BigInt(-(date.getTimezoneOffset()*60));var start=new Date(date.getFullYear(),0,1);var summerOffset=new Date(date.getFullYear(),6,1).getTimezoneOffset();var winterOffset=start.getTimezoneOffset();var dst=(summerOffset!=winterOffset&&date.getTimezoneOffset()==Math.min(winterOffset,summerOffset))|0;HEAP32[tmPtr+32>>2]=dst;return 0}var __mktime_js=function(tmPtr){tmPtr=bigintToI53Checked(tmPtr);var ret=(()=>{var date=new Date(HEAP32[tmPtr+20>>2]+1900,HEAP32[tmPtr+16>>2],HEAP32[tmPtr+12>>2],HEAP32[tmPtr+8>>2],HEAP32[tmPtr+4>>2],HEAP32[tmPtr>>2],0);if(isNaN(date.getTime())){return-1}var dst=HEAP32[tmPtr+32>>2];var guessedOffset=date.getTimezoneOffset();var start=new Date(date.getFullYear(),0,1);var summerOffset=new Date(date.getFullYear(),6,1).getTimezoneOffset();var winterOffset=start.getTimezoneOffset();var dstOffset=Math.min(winterOffset,summerOffset);if(dst<0){dst=Number(summerOffset!=winterOffset&&dstOffset==guessedOffset)}else if(dst>0!=(dstOffset==guessedOffset)){var nonDstOffset=Math.max(winterOffset,summerOffset);var trueOffset=dst>0?dstOffset:nonDstOffset;date.setTime(date.getTime()+(trueOffset-guessedOffset)*6e4);if(isNaN(date.getTime())){return-1}}HEAP32[tmPtr+32>>2]=dst;HEAP32[tmPtr+24>>2]=date.getDay();var yday=ydayFromDate(date)|0;HEAP32[tmPtr+28>>2]=yday;HEAP32[tmPtr>>2]=date.getSeconds();HEAP32[tmPtr+4>>2]=date.getMinutes();HEAP32[tmPtr+8>>2]=date.getHours();HEAP32[tmPtr+12>>2]=date.getDate();HEAP32[tmPtr+16>>2]=date.getMonth();HEAP32[tmPtr+20>>2]=date.getYear();return date.getTime()/1e3})();return BigInt(ret)};function __mmap_js(len,prot,flags,fd,offset,allocated,addr){len=bigintToI53Checked(len);offset=bigintToI53Checked(offset);allocated=bigintToI53Checked(allocated);addr=bigintToI53Checked(addr);try{var stream=SYSCALLS.getStreamFromFD(fd);var res=FS.mmap(stream,len,offset,prot,flags);var ptr=res.ptr;HEAP32[allocated>>2]=res.allocated;HEAPU64[addr>>3]=BigInt(ptr);return 0}catch(e){if(typeof FS=="undefined"||!(e.name==="ErrnoError"))throw e;return-e.errno}}function __munmap_js(addr,len,prot,flags,fd,offset){addr=bigintToI53Checked(addr);len=bigintToI53Checked(len);offset=bigintToI53Checked(offset);try{var stream=SYSCALLS.getStreamFromFD(fd);if(prot&2){SYSCALLS.doMsync(addr,stream,len,flags,offset)}}catch(e){if(typeof FS=="undefined"||!(e.name==="ErrnoError"))throw e;return-e.errno}}var timers={};var handleException=e=>{if(e instanceof ExitStatus||e=="unwind"){return EXITSTATUS}quit_(1,e)};var keepRuntimeAlive=()=>noExitRuntime||runtimeKeepaliveCounter>0;var _proc_exit=code=>{EXITSTATUS=code;if(!keepRuntimeAlive()){Module["onExit"]?.(code);ABORT=true}quit_(code,new ExitStatus(code))};var exitJS=(status,implicit)=>{EXITSTATUS=status;_proc_exit(status)};var _exit=exitJS;var maybeExit=()=>{if(!keepRuntimeAlive()){try{_exit(EXITSTATUS)}catch(e){handleException(e)}}};var callUserCallback=func=>{if(ABORT){return}try{return func()}catch(e){handleException(e)}finally{maybeExit()}};var _emscripten_get_now=()=>performance.now();var __setitimer_js=(which,timeout_ms)=>{if(timers[which]){clearTimeout(timers[which].id);delete timers[which]}if(!timeout_ms)return 0;var id=setTimeout(()=>{delete timers[which];callUserCallback(()=>__emscripten_timeout(which,_emscripten_get_now()))},timeout_ms);timers[which]={id,timeout_ms};return 0};var __tzset_js=function(timezone,daylight,std_name,dst_name){timezone=bigintToI53Checked(timezone);daylight=bigintToI53Checked(daylight);std_name=bigintToI53Checked(std_name);dst_name=bigintToI53Checked(dst_name);var currentYear=(new Date).getFullYear();var winter=new Date(currentYear,0,1);var summer=new Date(currentYear,6,1);var winterOffset=winter.getTimezoneOffset();var summerOffset=summer.getTimezoneOffset();var stdTimezoneOffset=Math.max(winterOffset,summerOffset);HEAPU64[timezone>>3]=BigInt(stdTimezoneOffset*60);HEAP32[daylight>>2]=Number(winterOffset!=summerOffset);var extractZone=timezoneOffset=>{var sign=timezoneOffset>=0?"-":"+";var absOffset=Math.abs(timezoneOffset);var hours=String(Math.floor(absOffset/60)).padStart(2,"0");var minutes=String(absOffset%60).padStart(2,"0");return`UTC${sign}${hours}${minutes}`};var winterName=extractZone(winterOffset);var summerName=extractZone(summerOffset);if(summerOffsetDate.now();var nowIsMonotonic=1;var checkWasiClock=clock_id=>clock_id>=0&&clock_id<=3;function _clock_time_get(clk_id,ignored_precision,ptime){ignored_precision=bigintToI53Checked(ignored_precision);ptime=bigintToI53Checked(ptime);if(!checkWasiClock(clk_id)){return 28}var now;if(clk_id===0){now=_emscripten_date_now()}else if(nowIsMonotonic){now=_emscripten_get_now()}else{return 52}var nsec=Math.round(now*1e3*1e3);HEAP64[ptime>>3]=BigInt(nsec);return 0}var runAndAbortIfError=func=>{try{return func()}catch(e){abort(e)}};var runtimeKeepalivePush=()=>{runtimeKeepaliveCounter+=1};var runtimeKeepalivePop=()=>{runtimeKeepaliveCounter-=1};var Asyncify={rewindArguments:new Map,instrumentWasmImports(imports){var importPattern=/^(invoke_.*|__asyncjs__.*)$/;for(let[x,original]of Object.entries(imports)){if(typeof original=="function"){let isAsyncifyImport=original.isAsync||importPattern.test(x)}}},saveRewindArguments(func,passedArguments){return Asyncify.rewindArguments.set(func,Array.from(passedArguments))},restoreRewindArguments(func){return Asyncify.rewindArguments.get(func)},instrumentFunction(original){var wrapper=(...args)=>{Asyncify.exportCallStack.push(original);try{Asyncify.saveRewindArguments(original,args);return original(...args)}finally{if(!ABORT){var top=Asyncify.exportCallStack.pop();Asyncify.maybeStopUnwind()}}};Asyncify.funcWrappers.set(original,wrapper);return wrapper},instrumentWasmExports(exports){var ret={};for(let[x,original]of Object.entries(exports)){if(typeof original=="function"){var wrapper=Asyncify.instrumentFunction(original);ret[x]=wrapper}else{ret[x]=original}}return ret},State:{Normal:0,Unwinding:1,Rewinding:2,Disabled:3},state:0,StackSize:4096,currData:null,handleSleepReturnValue:0,exportCallStack:[],callstackFuncToId:new Map,callStackIdToFunc:new Map,funcWrappers:new Map,callStackId:0,asyncPromiseHandlers:null,sleepCallbacks:[],getCallStackId(func){if(!Asyncify.callstackFuncToId.has(func)){var id=Asyncify.callStackId++;Asyncify.callstackFuncToId.set(func,id);Asyncify.callStackIdToFunc.set(id,func)}return Asyncify.callstackFuncToId.get(func)},maybeStopUnwind(){if(Asyncify.currData&&Asyncify.state===Asyncify.State.Unwinding&&!Asyncify.exportCallStack.length){Asyncify.state=Asyncify.State.Normal;runAndAbortIfError(_asyncify_stop_unwind);if(typeof Fibers!="undefined"){Fibers.trampoline()}}},whenDone(){return new Promise((resolve,reject)=>{Asyncify.asyncPromiseHandlers={resolve,reject}})},allocateData(){var ptr=_malloc(24+Asyncify.StackSize);Asyncify.setDataHeader(ptr,ptr+24,Asyncify.StackSize);Asyncify.setDataRewindFunc(ptr);return ptr},setDataHeader(ptr,stack,stackSize){HEAPU64[ptr>>3]=BigInt(stack);HEAPU64[ptr+8>>3]=BigInt(stack+stackSize)},setDataRewindFunc(ptr){var bottomOfCallStack=Asyncify.exportCallStack[0];var rewindId=Asyncify.getCallStackId(bottomOfCallStack);HEAP32[ptr+16>>2]=rewindId},getDataRewindFunc(ptr){var id=HEAP32[ptr+16>>2];var func=Asyncify.callStackIdToFunc.get(id);return func},doRewind(ptr){var original=Asyncify.getDataRewindFunc(ptr);var func=Asyncify.funcWrappers.get(original);func=func.bind(0,...Asyncify.restoreRewindArguments(original));return callUserCallback(func)},handleSleep(startAsync){if(ABORT)return;if(Asyncify.state===Asyncify.State.Normal){var reachedCallback=false;var reachedAfterCallback=false;startAsync((handleSleepReturnValue=0)=>{if(ABORT)return;Asyncify.handleSleepReturnValue=handleSleepReturnValue;reachedCallback=true;if(!reachedAfterCallback){return}Asyncify.state=Asyncify.State.Rewinding;runAndAbortIfError(()=>_asyncify_start_rewind(Asyncify.currData));if(typeof MainLoop!="undefined"&&MainLoop.func){MainLoop.resume()}var asyncWasmReturnValue,isError=false;try{asyncWasmReturnValue=Asyncify.doRewind(Asyncify.currData)}catch(err){asyncWasmReturnValue=err;isError=true}var handled=false;if(!Asyncify.currData){var asyncPromiseHandlers=Asyncify.asyncPromiseHandlers;if(asyncPromiseHandlers){Asyncify.asyncPromiseHandlers=null;(isError?asyncPromiseHandlers.reject:asyncPromiseHandlers.resolve)(asyncWasmReturnValue);handled=true}}if(isError&&!handled){throw asyncWasmReturnValue}});reachedAfterCallback=true;if(!reachedCallback){Asyncify.state=Asyncify.State.Unwinding;Asyncify.currData=Asyncify.allocateData();if(typeof MainLoop!="undefined"&&MainLoop.func){MainLoop.pause()}runAndAbortIfError(()=>_asyncify_start_unwind(Asyncify.currData))}}else if(Asyncify.state===Asyncify.State.Rewinding){Asyncify.state=Asyncify.State.Normal;runAndAbortIfError(_asyncify_stop_rewind);_free(Asyncify.currData);Asyncify.currData=null;Asyncify.sleepCallbacks.forEach(callUserCallback)}else{abort(`invalid state: ${Asyncify.state}`)}return Asyncify.handleSleepReturnValue},handleAsync:startAsync=>Asyncify.handleSleep(async wakeUp=>{wakeUp(await startAsync())})};var Fibers={nextFiber:0,trampolineRunning:false,trampoline(){if(!Fibers.trampolineRunning&&Fibers.nextFiber){Fibers.trampolineRunning=true;do{var fiber=Fibers.nextFiber;Fibers.nextFiber=0;Fibers.finishContextSwitch(fiber)}while(Fibers.nextFiber);Fibers.trampolineRunning=false}},finishContextSwitch(newFiber){var stack_base=Number(HEAPU64[newFiber>>3]);var stack_max=Number(HEAPU64[newFiber+8>>3]);_emscripten_stack_set_limits(stack_base,stack_max);stackRestore(Number(HEAPU64[newFiber+16>>3]));var entryPoint=Number(HEAPU64[newFiber+24>>3]);if(entryPoint){Asyncify.currData=null;HEAPU64[newFiber+24>>3]=0n;var userData=Number(HEAPU64[newFiber+32>>3]);(a1=>dynCall_vj(entryPoint,BigInt(a1)))(userData)}else{var asyncifyData=newFiber+40;Asyncify.currData=asyncifyData;Asyncify.state=Asyncify.State.Rewinding;_asyncify_start_rewind(asyncifyData);Asyncify.doRewind(asyncifyData)}}};function _emscripten_fiber_swap(oldFiber,newFiber){oldFiber=bigintToI53Checked(oldFiber);newFiber=bigintToI53Checked(newFiber);if(ABORT)return;if(Asyncify.state===Asyncify.State.Normal){Asyncify.state=Asyncify.State.Unwinding;var asyncifyData=oldFiber+40;Asyncify.setDataRewindFunc(asyncifyData);Asyncify.currData=asyncifyData;_asyncify_start_unwind(asyncifyData);var stackTop=stackSave();HEAPU64[oldFiber+16>>3]=BigInt(stackTop);Fibers.nextFiber=newFiber}else{Asyncify.state=Asyncify.State.Normal;_asyncify_stop_rewind();Asyncify.currData=null}}_emscripten_fiber_swap.isAsync=true;var _emscripten_force_exit=status=>{__emscripten_runtime_keepalive_clear();_exit(status)};var getHeapMax=()=>2147483648;var _emscripten_get_heap_max=()=>BigInt(getHeapMax());var growMemory=size=>{var oldHeapSize=wasmMemory.buffer.byteLength;var pages=(size-oldHeapSize+65535)/65536|0;try{wasmMemory.grow(pages);updateMemoryViews();return 1}catch(e){}};function _emscripten_resize_heap(requestedSize){requestedSize=bigintToI53Checked(requestedSize);var oldSize=HEAPU8.length;var maxHeapSize=getHeapMax();if(requestedSize>maxHeapSize){return false}for(var cutDown=1;cutDown<=4;cutDown*=2){var overGrownHeapSize=oldSize*(1+.2/cutDown);overGrownHeapSize=Math.min(overGrownHeapSize,requestedSize+100663296);var newSize=Math.min(maxHeapSize,alignMemory(Math.max(requestedSize,overGrownHeapSize),65536));var replacement=growMemory(newSize);if(replacement){return true}}return false}var ENV={};var getExecutableName=()=>thisProgram;var getEnvStrings=()=>{if(!getEnvStrings.strings){var lang=(globalThis.navigator?.language??"C").replace("-","_")+".UTF-8";var env={USER:"web_user",LOGNAME:"web_user",PATH:"/",PWD:"/",HOME:"/home/web_user",LANG:lang,_:getExecutableName()};for(var x in ENV){if(ENV[x]===undefined)delete env[x];else env[x]=ENV[x]}var strings=[];for(var x in env){strings.push(`${x}=${env[x]}`)}getEnvStrings.strings=strings}return getEnvStrings.strings};function _environ_get(__environ,environ_buf){__environ=bigintToI53Checked(__environ);environ_buf=bigintToI53Checked(environ_buf);var bufSize=0;var envp=0;for(var string of getEnvStrings()){var ptr=environ_buf+bufSize;HEAPU64[__environ+envp>>3]=BigInt(ptr);bufSize+=stringToUTF8(string,ptr,Infinity)+1;envp+=8}return 0}function _environ_sizes_get(penviron_count,penviron_buf_size){penviron_count=bigintToI53Checked(penviron_count);penviron_buf_size=bigintToI53Checked(penviron_buf_size);var strings=getEnvStrings();HEAPU64[penviron_count>>3]=BigInt(strings.length);var bufSize=0;for(var string of strings){bufSize+=lengthBytesUTF8(string)+1}HEAPU64[penviron_buf_size>>3]=BigInt(bufSize);return 0}function _fd_close(fd){try{var stream=SYSCALLS.getStreamFromFD(fd);FS.close(stream);return 0}catch(e){if(typeof FS=="undefined"||!(e.name==="ErrnoError"))throw e;return e.errno}}function _fd_fdstat_get(fd,pbuf){pbuf=bigintToI53Checked(pbuf);try{var rightsBase=0;var rightsInheriting=0;var flags=0;{var stream=SYSCALLS.getStreamFromFD(fd);var type=stream.tty?2:FS.isDir(stream.mode)?3:FS.isLink(stream.mode)?7:4}HEAP8[pbuf]=type;HEAP16[pbuf+2>>1]=flags;HEAP64[pbuf+8>>3]=BigInt(rightsBase);HEAP64[pbuf+16>>3]=BigInt(rightsInheriting);return 0}catch(e){if(typeof FS=="undefined"||!(e.name==="ErrnoError"))throw e;return e.errno}}var doReadv=(stream,iov,iovcnt,offset)=>{var ret=0;for(var i=0;i>3]);var len=Number(HEAPU64[iov+8>>3]);iov+=16;try{var curr=FS.read(stream,HEAP8,ptr,len,offset)}catch(e){if(ret>0&&e instanceof FS.ErrnoError&&(e.errno==6||e.errno==6)){break}throw e}if(curr<0)return-1;ret+=curr;if(curr>3]=BigInt(num);return 0}catch(e){if(typeof FS=="undefined"||!(e.name==="ErrnoError"))throw e;return e.errno}}function _fd_seek(fd,offset,whence,newOffset){offset=bigintToI53Checked(offset);newOffset=bigintToI53Checked(newOffset);try{if(isNaN(offset))return 22;var stream=SYSCALLS.getStreamFromFD(fd);FS.llseek(stream,offset,whence);HEAP64[newOffset>>3]=BigInt(stream.position);if(stream.getdents&&!offset&&whence===0)stream.getdents=null;return 0}catch(e){if(typeof FS=="undefined"||!(e.name==="ErrnoError"))throw e;return e.errno}}var _fd_sync=function(fd){let innerFunc=()=>{try{var stream=SYSCALLS.getStreamFromFD(fd);var rtn=stream.stream_ops?.fsync?.(stream);return new Promise(resolve=>{var mount=stream.node.mount;if(mount?.type.syncfs){mount.type.syncfs(mount,false,err=>resolve(err?29:0))}else{resolve(rtn)}})}catch(e){if(typeof FS=="undefined"||!(e.name==="ErrnoError"))throw e;return e.errno}};return Asyncify.handleAsync(innerFunc)};_fd_sync.isAsync=true;var doWritev=(stream,iov,iovcnt,offset)=>{if(iovcnt==1){return FS.write(stream,HEAP8,Number(HEAPU64[iov>>3]),Number(HEAPU64[iov+8>>3]),offset)}var total=0;for(var i=0,p=iov;i>3])}var view=new Uint8Array(total);var voff=0;for(var i=0;i>3]);var len=Number(HEAPU64[iov+8>>3]);view.set(HEAPU8.subarray(ptr,ptr+len),voff);voff+=len}return FS.write(stream,view,0,total,offset)};function _fd_write(fd,iov,iovcnt,pnum){iov=bigintToI53Checked(iov);iovcnt=bigintToI53Checked(iovcnt);pnum=bigintToI53Checked(pnum);try{var stream=SYSCALLS.getStreamFromFD(fd);var num=doWritev(stream,iov,iovcnt);HEAPU64[pnum>>3]=BigInt(num);return 0}catch(e){if(typeof FS=="undefined"||!(e.name==="ErrnoError"))throw e;return e.errno}}function _getaddrinfo(node,service,hint,out){node=bigintToI53Checked(node);service=bigintToI53Checked(service);hint=bigintToI53Checked(hint);out=bigintToI53Checked(out);var addr=0;var port=0;var flags=0;var family=0;var type=0;var proto=0;var ai;function allocaddrinfo(family,type,proto,canon,addr,port){var sa,salen,ai;var errno;salen=family===10?28:16;addr=family===10?inetNtop6(addr):inetNtop4(addr);sa=_malloc(salen);errno=writeSockaddr(sa,family,addr,port);ai=_malloc(48);HEAP32[ai+4>>2]=family;HEAP32[ai+8>>2]=type;HEAP32[ai+12>>2]=proto;HEAPU64[ai+32>>3]=BigInt(canon);HEAPU64[ai+24>>3]=BigInt(sa);if(family===10){HEAP32[ai+16>>2]=28}else{HEAP32[ai+16>>2]=16}HEAP32[ai+40>>2]=0;return ai}if(hint){flags=HEAP32[hint>>2];family=HEAP32[hint+4>>2];type=HEAP32[hint+8>>2];proto=HEAP32[hint+12>>2]}if(type&&!proto){proto=type===2?17:6}if(!type&&proto){type=proto===17?2:1}if(!proto){proto=6}if(!type){type=1}if(!node&&!service){return-2}if(flags&~(1|2|4|1024|8|16|32)){return-1}if(hint&&HEAP32[hint>>2]&2&&!node){return-1}if(flags&32){return-2}if(type&&type!==1&&type!==2){return-7}if(family!==0&&family!==2&&family!==10){return-6}if(service){service=UTF8ToString(service);port=parseInt(service,10);if(isNaN(port)){if(flags&1024){return-2}return-8}}if(!node){if(family===0){family=2}if(!(flags&1)){if(family===2){addr=_htonl(2130706433)}else{addr=[0,0,0,_htonl(1)]}}ai=allocaddrinfo(family,type,proto,null,addr,port);HEAPU64[out>>3]=BigInt(ai);return 0}node=UTF8ToString(node);addr=inetPton4(node);if(addr!==null){if(family===0||family===2){family=2}else if(family===10&&flags&8){addr=[0,0,_htonl(65535),addr];family=10}else{return-2}}else{addr=inetPton6(node);if(addr!==null){if(family===0||family===10){family=10}else{return-2}}}if(addr!=null){ai=allocaddrinfo(family,type,proto,node,addr,port);HEAPU64[out>>3]=BigInt(ai);return 0}if(flags&4){return-2}node=DNS.lookup_name(node);addr=inetPton4(node);if(family===0){family=2}else if(family===10){addr=[0,0,_htonl(65535),addr]}ai=allocaddrinfo(family,type,proto,null,addr,port);HEAPU64[out>>3]=BigInt(ai);return 0}function _getnameinfo(sa,salen,node,nodelen,serv,servlen,flags){sa=bigintToI53Checked(sa);node=bigintToI53Checked(node);serv=bigintToI53Checked(serv);var info=readSockaddr(sa,salen);if(info.errno){return-6}var port=info.port;var addr=info.addr;var overflowed=false;if(node&&nodelen){var lookup;if(flags&1||!(lookup=DNS.lookup_addr(addr))){if(flags&8){return-2}}else{addr=lookup}var numBytesWrittenExclNull=stringToUTF8(addr,node,nodelen);if(numBytesWrittenExclNull+1>=nodelen){overflowed=true}}if(serv&&servlen){port=""+port;var numBytesWrittenExclNull=stringToUTF8(port,serv,servlen);if(numBytesWrittenExclNull+1>=servlen){overflowed=true}}if(overflowed){return-12}return 0}var Protocols={list:[],map:{}};var stringToAscii=(str,buffer)=>{for(var i=0;i{function allocprotoent(name,proto,aliases){var nameBuf=_malloc(name.length+1);stringToAscii(name,nameBuf);var j=0;var length=aliases.length;var aliasListBuf=_malloc((length+1)*4);for(var i=0;i>3]=BigInt(aliasBuf)}HEAPU64[aliasListBuf+j>>3]=0n;var pe=_malloc(24);HEAPU64[pe>>3]=BigInt(nameBuf);HEAPU64[pe+8>>3]=BigInt(aliasListBuf);HEAP32[pe+16>>2]=proto;return pe}var list=Protocols.list;var map=Protocols.map;if(!list.length){var entry=allocprotoent("tcp",6,["TCP"]);list.push(entry);map["tcp"]=map["6"]=entry;entry=allocprotoent("udp",17,["UDP"]);list.push(entry);map["udp"]=map["17"]=entry}_setprotoent.index=0};var _getprotobyname=function(name){name=bigintToI53Checked(name);var ret=(()=>{name=UTF8ToString(name);_setprotoent(true);var result=Protocols.map[name];return result})();return BigInt(ret)};var _getprotobynumber=function(number){var ret=(()=>{_setprotoent(true);var result=Protocols.map[number];return result})();return BigInt(ret)};var arraySum=(array,index)=>{var sum=0;for(var i=0;i<=index;sum+=array[i++]){}return sum};var MONTH_DAYS_LEAP=[31,29,31,30,31,30,31,31,30,31,30,31];var MONTH_DAYS_REGULAR=[31,28,31,30,31,30,31,31,30,31,30,31];var addDays=(date,days)=>{var newDate=new Date(date.getTime());while(days>0){var leap=isLeapYear(newDate.getFullYear());var currentMonth=newDate.getMonth();var daysInCurrentMonth=(leap?MONTH_DAYS_LEAP:MONTH_DAYS_REGULAR)[currentMonth];if(days>daysInCurrentMonth-newDate.getDate()){days-=daysInCurrentMonth-newDate.getDate()+1;newDate.setDate(1);if(currentMonth<11){newDate.setMonth(currentMonth+1)}else{newDate.setMonth(0);newDate.setFullYear(newDate.getFullYear()+1)}}else{newDate.setDate(newDate.getDate()+days);return newDate}}return newDate};var _strptime=function(buf,format,tm){buf=bigintToI53Checked(buf);format=bigintToI53Checked(format);tm=bigintToI53Checked(tm);var ret=(()=>{var pattern=UTF8ToString(format);var SPECIAL_CHARS="\\!@#$^&*()+=-[]/{}|:<>?,.";for(var i=0,ii=SPECIAL_CHARS.length;iEQUIVALENT_MATCHERS[c]||m).replace(/%(.)/g,(_,c)=>{let pat=DATE_PATTERNS[c];if(pat){capture.push(c);return`(${pat})`}else{return c}}).replace(/\s+/g,"\\s*");var matches=new RegExp("^"+pattern_out,"i").exec(UTF8ToString(buf));function initDate(){function fixup(value,min,max){return typeof value!="number"||isNaN(value)?min:value>=min?value<=max?value:max:min}return{year:fixup(HEAP32[tm+20>>2]+1900,1970,9999),month:fixup(HEAP32[tm+16>>2],0,11),day:fixup(HEAP32[tm+12>>2],1,31),hour:fixup(HEAP32[tm+8>>2],0,23),min:fixup(HEAP32[tm+4>>2],0,59),sec:fixup(HEAP32[tm>>2],0,59),gmtoff:0}}if(matches){var date=initDate();var value;var getMatch=symbol=>{var pos=capture.indexOf(symbol);if(pos>=0){return matches[pos+1]}return};if(value=getMatch("S")){date.sec=Number(value)}if(value=getMatch("M")){date.min=Number(value)}if(value=getMatch("H")){date.hour=Number(value)}else if(value=getMatch("I")){var hour=Number(value);if(value=getMatch("p")){hour+=value.toUpperCase()[0]==="P"?12:0}date.hour=hour}if(value=getMatch("Y")){date.year=Number(value)}else if(value=getMatch("y")){var year=Number(value);if(value=getMatch("C")){year+=Number(value)*100}else{year+=year<69?2e3:1900}date.year=year}if(value=getMatch("m")){date.month=Number(value)-1}else if(value=getMatch("b")){date.month=MONTH_NUMBERS[value.substring(0,3).toUpperCase()]||0}if(value=getMatch("d")){date.day=Number(value)}else if(value=getMatch("j")){var day=Number(value);var leapYear=isLeapYear(date.year);for(var month=0;month<12;++month){var daysUntilMonth=arraySum(leapYear?MONTH_DAYS_LEAP:MONTH_DAYS_REGULAR,month-1);if(day<=daysUntilMonth+(leapYear?MONTH_DAYS_LEAP:MONTH_DAYS_REGULAR)[month]){date.day=day-daysUntilMonth}}}else if(value=getMatch("a")){var weekDay=value.substring(0,3).toUpperCase();if(value=getMatch("U")){var weekDayNumber=DAY_NUMBERS_SUN_FIRST[weekDay];var weekNumber=Number(value);var janFirst=new Date(date.year,0,1);var endDate;if(janFirst.getDay()===0){endDate=addDays(janFirst,weekDayNumber+7*(weekNumber-1))}else{endDate=addDays(janFirst,7-janFirst.getDay()+weekDayNumber+7*(weekNumber-1))}date.day=endDate.getDate();date.month=endDate.getMonth()}else if(value=getMatch("W")){var weekDayNumber=DAY_NUMBERS_MON_FIRST[weekDay];var weekNumber=Number(value);var janFirst=new Date(date.year,0,1);var endDate;if(janFirst.getDay()===1){endDate=addDays(janFirst,weekDayNumber+7*(weekNumber-1))}else{endDate=addDays(janFirst,7-janFirst.getDay()+1+weekDayNumber+7*(weekNumber-1))}date.day=endDate.getDate();date.month=endDate.getMonth()}}if(value=getMatch("z")){if(value.toLowerCase()==="z"){date.gmtoff=0}else{var match=value.match(/^((?:\-|\+)\d\d):?(\d\d)?/);date.gmtoff=match[1]*3600;if(match[2]){date.gmtoff+=date.gmtoff>0?match[2]*60:-match[2]*60}}}var fullDate=new Date(date.year,date.month,date.day,date.hour,date.min,date.sec,0);HEAP32[tm>>2]=fullDate.getSeconds();HEAP32[tm+4>>2]=fullDate.getMinutes();HEAP32[tm+8>>2]=fullDate.getHours();HEAP32[tm+12>>2]=fullDate.getDate();HEAP32[tm+16>>2]=fullDate.getMonth();HEAP32[tm+20>>2]=fullDate.getFullYear()-1900;HEAP32[tm+24>>2]=fullDate.getDay();HEAP32[tm+28>>2]=arraySum(isLeapYear(fullDate.getFullYear())?MONTH_DAYS_LEAP:MONTH_DAYS_REGULAR,fullDate.getMonth()-1)+fullDate.getDate()-1;HEAP32[tm+32>>2]=0;HEAP64[tm+40>>3]=BigInt(date.gmtoff);return buf+lengthBytesUTF8(matches[0])}return 0})();return BigInt(ret)};var getCFunc=ident=>{var func=Module["_"+ident];return func};var writeArrayToMemory=(array,buffer)=>{HEAP8.set(array,buffer)};var stackAlloc=sz=>__emscripten_stack_alloc(sz);var stringToUTF8OnStack=str=>{var size=lengthBytesUTF8(str)+1;var ret=stackAlloc(size);stringToUTF8(str,ret,size);return ret};var ccall=(ident,returnType,argTypes,args,opts)=>{var toC={pointer:p=>BigInt(p),string:str=>{var ret=0;if(str!==null&&str!==undefined&&str!==0){ret=stringToUTF8OnStack(str)}return BigInt(ret)},array:arr=>{var ret=stackAlloc(arr.length);writeArrayToMemory(arr,ret);return BigInt(ret)}};function convertReturnValue(ret){if(returnType==="string"){return UTF8ToString(Number(ret))}if(returnType==="pointer")return Number(ret);if(returnType==="boolean")return Boolean(ret);return ret}var func=getCFunc(ident);var cArgs=[];var stack=0;if(args){for(var i=0;i>1];case"i32":return HEAP32[ptr>>2];case"i64":return HEAP64[ptr>>3];case"float":return HEAPF32[ptr>>2];case"double":return HEAPF64[ptr>>3];case"*":return Number(HEAPU64[ptr>>3]);default:abort(`invalid type for getValue: ${type}`)}}function setValue(ptr,value,type="i8"){if(type.endsWith("*"))type="*";switch(type){case"i1":HEAP8[ptr]=value;break;case"i8":HEAP8[ptr]=value;break;case"i16":HEAP16[ptr>>1]=value;break;case"i32":HEAP32[ptr>>2]=value;break;case"i64":HEAP64[ptr>>3]=BigInt(value);break;case"float":HEAPF32[ptr>>2]=value;break;case"double":HEAPF64[ptr>>3]=value;break;case"*":HEAPU64[ptr>>3]=BigInt(value);break;default:abort(`invalid type for setValue: ${type}`)}}FS.createPreloadedFile=FS_createPreloadedFile;FS.preloadFile=FS_preloadFile;FS.staticInit();{if(Module["noExitRuntime"])noExitRuntime=Module["noExitRuntime"];if(Module["print"])out=Module["print"];if(Module["printErr"])err=Module["printErr"];if(Module["arguments"])programArgs=Module["arguments"];if(Module["thisProgram"])thisProgram=Module["thisProgram"];var preInit=Module["preInit"];if(preInit){if(typeof preInit=="function")Module["preInit"]=preInit=[preInit];while(preInit.length>0){preInit.shift()()}}}Module["ENV"]=ENV;Module["ccall"]=ccall;Module["setValue"]=setValue;Module["getValue"]=getValue;Module["UTF8ToString"]=UTF8ToString;Module["stringToUTF8"]=stringToUTF8;Module["lengthBytesUTF8"]=lengthBytesUTF8;Module["FS"]=FS;var _pib_heap_bytes,_pib_eval,_free,_malloc,_htons,_ntohs,_htonl,_pib_force_exit,_emscripten_builtin_memalign,__emscripten_timeout,_setThrew,_emscripten_stack_set_limits,__emscripten_stack_restore,__emscripten_stack_alloc,_emscripten_stack_get_current,dynCall_vjj,dynCall_jjj,dynCall_ijjjj,dynCall_vjjj,dynCall_ijjj,dynCall_vj,dynCall_jj,dynCall_vjjjj,dynCall_ij,dynCall_jjjj,dynCall_vjjijjj,dynCall_vjjjiijj,dynCall_vjjij,dynCall_vjjjjj,dynCall_vjji,dynCall_vjjjjijiij,dynCall_ijj,dynCall_ijji,dynCall_jji,dynCall_jjji,dynCall_ii,dynCall_ijjjjj,dynCall_ijjjjjjj,dynCall_ijjjjjj,dynCall_vji,dynCall_j,dynCall_v,dynCall_jjjjj,dynCall_vjjiijjjjjjjjj,dynCall_jjjji,dynCall_iij,dynCall_vjij,dynCall_ji,dynCall_jjii,dynCall_vjjjji,dynCall_vjjii,dynCall_vjjji,dynCall_iji,dynCall_jjjiij,dynCall_jiij,dynCall_jjjjji,dynCall_ijjji,dynCall_iijj,dynCall_vjiji,dynCall_i,dynCall_vijj,dynCall_ijjii,dynCall_vi,dynCall_vii,dynCall_jjij,dynCall_vijij,dynCall_jjjjii,dynCall_jjjiiii,dynCall_vjjdi,dynCall_ijiijj,dynCall_vjjjij,dynCall_vjii,dynCall_iijjjj,dynCall_jjjjjjiijj,dynCall_ijjij,dynCall_jjjijj,dynCall_jjjij,dynCall_jijj,dynCall_vjjjjij,dynCall_vjjjjjj,dynCall_jjjjjj,dynCall_ijjiii,dynCall_jjjjijj,dynCall_jii,dynCall_vij,dynCall_ijjjjjji,dynCall_ijjjji,dynCall_ijiij,dynCall_vjijj,dynCall_vjiijijjj,dynCall_jjijj,dynCall_vjijjj,dynCall_iijjj,dynCall_ijij,dynCall_iiji,dynCall_ijdiiiii,dynCall_jij,dynCall_ijjjjijj,dynCall_ijjjjji,dynCall_vijijj,dynCall_ijii,dynCall_viiijjj,dynCall_vjdj,dynCall_iii,dynCall_jjjjjjj,dynCall_ijjijj,dynCall_ijjjij,dynCall_ijjiij,dynCall_jjjjij,dynCall_vjjjjii,dynCall_vjjjii,_asyncify_start_unwind,_asyncify_stop_unwind,_asyncify_start_rewind,_asyncify_stop_rewind,memory,__indirect_function_table,wasmMemory,wasmTable;function assignWasmExports(wasmExports){_pib_heap_bytes=Module["_pib_heap_bytes"]=wasmExports["Mb"];_pib_eval=Module["_pib_eval"]=wasmExports["Nb"];_free=wasmExports["Ob"];_malloc=wasmExports["Pb"];_htons=wasmExports["Qb"];_ntohs=wasmExports["Rb"];_htonl=wasmExports["Sb"];_pib_force_exit=Module["_pib_force_exit"]=wasmExports["Tb"];_emscripten_builtin_memalign=wasmExports["Vb"];__emscripten_timeout=wasmExports["Wb"];_setThrew=wasmExports["Xb"];_emscripten_stack_set_limits=wasmExports["Yb"];__emscripten_stack_restore=wasmExports["Zb"];__emscripten_stack_alloc=wasmExports["_b"];_emscripten_stack_get_current=wasmExports["$b"];dynCall_vjj=dynCalls["vjj"]=wasmExports["ac"];dynCall_jjj=dynCalls["jjj"]=wasmExports["bc"];dynCall_ijjjj=dynCalls["ijjjj"]=wasmExports["cc"];dynCall_vjjj=dynCalls["vjjj"]=wasmExports["dc"];dynCall_ijjj=dynCalls["ijjj"]=wasmExports["ec"];dynCall_vj=dynCalls["vj"]=wasmExports["fc"];dynCall_jj=dynCalls["jj"]=wasmExports["gc"];dynCall_vjjjj=dynCalls["vjjjj"]=wasmExports["hc"];dynCall_ij=dynCalls["ij"]=wasmExports["ic"];dynCall_jjjj=dynCalls["jjjj"]=wasmExports["jc"];dynCall_vjjijjj=dynCalls["vjjijjj"]=wasmExports["kc"];dynCall_vjjjiijj=dynCalls["vjjjiijj"]=wasmExports["lc"];dynCall_vjjij=dynCalls["vjjij"]=wasmExports["mc"];dynCall_vjjjjj=dynCalls["vjjjjj"]=wasmExports["nc"];dynCall_vjji=dynCalls["vjji"]=wasmExports["oc"];dynCall_vjjjjijiij=dynCalls["vjjjjijiij"]=wasmExports["pc"];dynCall_ijj=dynCalls["ijj"]=wasmExports["qc"];dynCall_ijji=dynCalls["ijji"]=wasmExports["rc"];dynCall_jji=dynCalls["jji"]=wasmExports["sc"];dynCall_jjji=dynCalls["jjji"]=wasmExports["tc"];dynCall_ii=dynCalls["ii"]=wasmExports["uc"];dynCall_ijjjjj=dynCalls["ijjjjj"]=wasmExports["vc"];dynCall_ijjjjjjj=dynCalls["ijjjjjjj"]=wasmExports["wc"];dynCall_ijjjjjj=dynCalls["ijjjjjj"]=wasmExports["xc"];dynCall_vji=dynCalls["vji"]=wasmExports["yc"];dynCall_j=dynCalls["j"]=wasmExports["zc"];dynCall_v=dynCalls["v"]=wasmExports["Ac"];dynCall_jjjjj=dynCalls["jjjjj"]=wasmExports["Bc"];dynCall_vjjiijjjjjjjjj=dynCalls["vjjiijjjjjjjjj"]=wasmExports["Cc"];dynCall_jjjji=dynCalls["jjjji"]=wasmExports["Dc"];dynCall_iij=dynCalls["iij"]=wasmExports["Ec"];dynCall_vjij=dynCalls["vjij"]=wasmExports["Fc"];dynCall_ji=dynCalls["ji"]=wasmExports["Gc"];dynCall_jjii=dynCalls["jjii"]=wasmExports["Hc"];dynCall_vjjjji=dynCalls["vjjjji"]=wasmExports["Ic"];dynCall_vjjii=dynCalls["vjjii"]=wasmExports["Jc"];dynCall_vjjji=dynCalls["vjjji"]=wasmExports["Kc"];dynCall_iji=dynCalls["iji"]=wasmExports["Lc"];dynCall_jjjiij=dynCalls["jjjiij"]=wasmExports["Mc"];dynCall_jiij=dynCalls["jiij"]=wasmExports["Nc"];dynCall_jjjjji=dynCalls["jjjjji"]=wasmExports["Oc"];dynCall_ijjji=dynCalls["ijjji"]=wasmExports["Pc"];dynCall_iijj=dynCalls["iijj"]=wasmExports["Qc"];dynCall_vjiji=dynCalls["vjiji"]=wasmExports["Rc"];dynCall_i=dynCalls["i"]=wasmExports["Sc"];dynCall_vijj=dynCalls["vijj"]=wasmExports["Tc"];dynCall_ijjii=dynCalls["ijjii"]=wasmExports["Uc"];dynCall_vi=dynCalls["vi"]=wasmExports["Vc"];dynCall_vii=dynCalls["vii"]=wasmExports["Wc"];dynCall_jjij=dynCalls["jjij"]=wasmExports["Xc"];dynCall_vijij=dynCalls["vijij"]=wasmExports["Yc"];dynCall_jjjjii=dynCalls["jjjjii"]=wasmExports["Zc"];dynCall_jjjiiii=dynCalls["jjjiiii"]=wasmExports["_c"];dynCall_vjjdi=dynCalls["vjjdi"]=wasmExports["$c"];dynCall_ijiijj=dynCalls["ijiijj"]=wasmExports["ad"];dynCall_vjjjij=dynCalls["vjjjij"]=wasmExports["bd"];dynCall_vjii=dynCalls["vjii"]=wasmExports["cd"];dynCall_iijjjj=dynCalls["iijjjj"]=wasmExports["dd"];dynCall_jjjjjjiijj=dynCalls["jjjjjjiijj"]=wasmExports["ed"];dynCall_ijjij=dynCalls["ijjij"]=wasmExports["fd"];dynCall_jjjijj=dynCalls["jjjijj"]=wasmExports["gd"];dynCall_jjjij=dynCalls["jjjij"]=wasmExports["hd"];dynCall_jijj=dynCalls["jijj"]=wasmExports["id"];dynCall_vjjjjij=dynCalls["vjjjjij"]=wasmExports["jd"];dynCall_vjjjjjj=dynCalls["vjjjjjj"]=wasmExports["kd"];dynCall_jjjjjj=dynCalls["jjjjjj"]=wasmExports["ld"];dynCall_ijjiii=dynCalls["ijjiii"]=wasmExports["md"];dynCall_jjjjijj=dynCalls["jjjjijj"]=wasmExports["nd"];dynCall_jii=dynCalls["jii"]=wasmExports["od"];dynCall_vij=dynCalls["vij"]=wasmExports["pd"];dynCall_ijjjjjji=dynCalls["ijjjjjji"]=wasmExports["qd"];dynCall_ijjjji=dynCalls["ijjjji"]=wasmExports["rd"];dynCall_ijiij=dynCalls["ijiij"]=wasmExports["sd"];dynCall_vjijj=dynCalls["vjijj"]=wasmExports["td"];dynCall_vjiijijjj=dynCalls["vjiijijjj"]=wasmExports["ud"];dynCall_jjijj=dynCalls["jjijj"]=wasmExports["vd"];dynCall_vjijjj=dynCalls["vjijjj"]=wasmExports["wd"];dynCall_iijjj=dynCalls["iijjj"]=wasmExports["xd"];dynCall_ijij=dynCalls["ijij"]=wasmExports["yd"];dynCall_iiji=dynCalls["iiji"]=wasmExports["zd"];dynCall_ijdiiiii=dynCalls["ijdiiiii"]=wasmExports["Ad"];dynCall_jij=dynCalls["jij"]=wasmExports["Bd"];dynCall_ijjjjijj=dynCalls["ijjjjijj"]=wasmExports["Cd"];dynCall_ijjjjji=dynCalls["ijjjjji"]=wasmExports["Dd"];dynCall_vijijj=dynCalls["vijijj"]=wasmExports["Ed"];dynCall_ijii=dynCalls["ijii"]=wasmExports["Fd"];dynCall_viiijjj=dynCalls["viiijjj"]=wasmExports["Gd"];dynCall_vjdj=dynCalls["vjdj"]=wasmExports["Hd"];dynCall_iii=dynCalls["iii"]=wasmExports["Id"];dynCall_jjjjjjj=dynCalls["jjjjjjj"]=wasmExports["Jd"];dynCall_ijjijj=dynCalls["ijjijj"]=wasmExports["Kd"];dynCall_ijjjij=dynCalls["ijjjij"]=wasmExports["Ld"];dynCall_ijjiij=dynCalls["ijjiij"]=wasmExports["Md"];dynCall_jjjjij=dynCalls["jjjjij"]=wasmExports["Nd"];dynCall_vjjjjii=dynCalls["vjjjjii"]=wasmExports["Od"];dynCall_vjjjii=dynCalls["vjjjii"]=wasmExports["Pd"];_asyncify_start_unwind=wasmExports["Qd"];_asyncify_stop_unwind=wasmExports["Rd"];_asyncify_start_rewind=wasmExports["Sd"];_asyncify_stop_rewind=wasmExports["Td"];memory=wasmMemory=wasmExports["Kb"];__indirect_function_table=wasmTable=wasmExports["Ub"]}var wasmImports={m:___assert_fail,Ka:___call_sighandler,za:___syscall_accept4,ya:___syscall_bind,ib:___syscall_chdir,fa:___syscall_chmod,Q:___syscall_connect,gb:___syscall_dup,jb:___syscall_faccessat,fb:___syscall_fchmod,ea:___syscall_fchownat,q:___syscall_fcntl64,eb:___syscall_fdatasync,db:___syscall_fstat64,_a:___syscall_ftruncate64,Za:___syscall_getcwd,Ja:___syscall_getdents64,Xa:___syscall_geteuid32,R:___syscall_getgid32,wa:___syscall_getpeername,va:___syscall_getsockname,ua:___syscall_getsockopt,F:___syscall_getuid32,ha:___syscall_ioctl,Wa:___syscall_linkat,ta:___syscall_listen,bb:___syscall_lstat64,Ua:___syscall_mkdirat,ab:___syscall_newfstatat,S:___syscall_openat,Na:___syscall_pipe2,ba:___syscall_poll,Ma:___syscall_poll_nonblocking,Ia:___syscall_readlinkat,P:___syscall_recvfrom,Ha:___syscall_renameat,$:___syscall_rmdir,I:___syscall_sendto,Ib:___syscall_setsockopt,Hb:___syscall_shutdown,sa:___syscall_socket,cb:___syscall_stat64,Ga:___syscall_statfs64,Fa:___syscall_symlinkat,Da:___syscall_umask,aa:___syscall_unlinkat,Ca:___syscall_utimensat,kb:__abort_js,xa:__emscripten_lookup_name,La:__emscripten_runtime_keepalive_clear,Aa:__emscripten_throw_longjmp,Qa:__gmtime_js,Sa:__localtime_js,Ta:__mktime_js,Oa:__mmap_js,Pa:__munmap_js,_:__setitimer_js,mb:__tzset_js,hb:_clock_time_get,da:_emscripten_date_now,Jb:_emscripten_fiber_swap,Ya:_emscripten_force_exit,Ea:_emscripten_get_heap_max,J:_emscripten_get_now,Ba:_emscripten_resize_heap,ob:_environ_get,pb:_environ_sizes_get,z:_exit,D:_fd_close,ca:_fd_fdstat_get,ga:_fd_read,Va:_fd_seek,$a:_fd_sync,T:_fd_write,Ra:_getaddrinfo,K:_getnameinfo,vb:_getprotobyname,sb:_getprotobynumber,C:invoke_i,u:invoke_ii,Y:invoke_iii,y:invoke_iij,Gb:invoke_iijj,t:invoke_ij,E:invoke_iji,wb:invoke_ijii,M:invoke_ijiij,W:invoke_ijiijj,v:invoke_ijij,e:invoke_ijj,Z:invoke_ijji,Fb:invoke_ijjii,na:invoke_ijjiii,zb:invoke_ijjijj,k:invoke_ijjj,H:invoke_ijjji,p:invoke_ijjjj,la:invoke_ijjjjijj,Db:invoke_ijjjjji,Bb:invoke_ijjjjjji,o:invoke_j,ia:invoke_ji,yb:invoke_jij,qa:invoke_jijj,b:invoke_jj,i:invoke_jji,rb:invoke_jjii,h:invoke_jjj,s:invoke_jjji,Eb:invoke_jjjiiii,l:invoke_jjjj,tb:invoke_jjjji,d:invoke_jjjjii,Cb:invoke_jjjjijj,n:invoke_jjjjj,xb:invoke_jjjjji,oa:invoke_jjjjjj,Ab:invoke_jjjjjjiijj,c:invoke_v,O:invoke_vi,ra:invoke_vii,V:invoke_vij,U:invoke_vijijj,g:invoke_vijj,a:invoke_vj,x:invoke_vji,B:invoke_vjii,r:invoke_vjij,j:invoke_vjiji,G:invoke_vjijj,f:invoke_vjj,X:invoke_vjjdi,ma:invoke_vjji,A:invoke_vjjii,ub:invoke_vjjiijjjjjjjjj,L:invoke_vjjij,w:invoke_vjjj,ja:invoke_vjjjj,qb:invoke_vjjjji,pa:invoke_vjjjjij,ka:invoke_vjjjjj,N:invoke_vjjjjjj,lb:_proc_exit,nb:_strptime};function invoke_j(index){var sp=stackSave();try{return dynCall_j(Number(index))}catch(e){stackRestore(sp);if(!(e instanceof EmscriptenEH))throw e;_setThrew(1,0);return 0n}}function invoke_jj(index,a1){var sp=stackSave();try{return dynCall_jj(Number(index),a1)}catch(e){stackRestore(sp);if(!(e instanceof EmscriptenEH))throw e;_setThrew(1,0);return 0n}}function invoke_vj(index,a1){var sp=stackSave();try{dynCall_vj(Number(index),a1)}catch(e){stackRestore(sp);if(!(e instanceof EmscriptenEH))throw e;_setThrew(1,0)}}function invoke_vjjj(index,a1,a2,a3){var sp=stackSave();try{dynCall_vjjj(Number(index),a1,a2,a3)}catch(e){stackRestore(sp);if(!(e instanceof EmscriptenEH))throw e;_setThrew(1,0)}}function invoke_vjj(index,a1,a2){var sp=stackSave();try{dynCall_vjj(Number(index),a1,a2)}catch(e){stackRestore(sp);if(!(e instanceof EmscriptenEH))throw e;_setThrew(1,0)}}function invoke_v(index){var sp=stackSave();try{dynCall_v(Number(index))}catch(e){stackRestore(sp);if(!(e instanceof EmscriptenEH))throw e;_setThrew(1,0)}}function invoke_jji(index,a1,a2){var sp=stackSave();try{return dynCall_jji(Number(index),a1,a2)}catch(e){stackRestore(sp);if(!(e instanceof EmscriptenEH))throw e;_setThrew(1,0);return 0n}}function invoke_ijj(index,a1,a2){var sp=stackSave();try{return dynCall_ijj(Number(index),a1,a2)}catch(e){stackRestore(sp);if(!(e instanceof EmscriptenEH))throw e;_setThrew(1,0)}}function invoke_vjjjj(index,a1,a2,a3,a4){var sp=stackSave();try{dynCall_vjjjj(Number(index),a1,a2,a3,a4)}catch(e){stackRestore(sp);if(!(e instanceof EmscriptenEH))throw e;_setThrew(1,0)}}function invoke_jjjjj(index,a1,a2,a3,a4){var sp=stackSave();try{return dynCall_jjjjj(Number(index),a1,a2,a3,a4)}catch(e){stackRestore(sp);if(!(e instanceof EmscriptenEH))throw e;_setThrew(1,0);return 0n}}function invoke_ijjj(index,a1,a2,a3){var sp=stackSave();try{return dynCall_ijjj(Number(index),a1,a2,a3)}catch(e){stackRestore(sp);if(!(e instanceof EmscriptenEH))throw e;_setThrew(1,0)}}function invoke_ii(index,a1){var sp=stackSave();try{return dynCall_ii(Number(index),a1)}catch(e){stackRestore(sp);if(!(e instanceof EmscriptenEH))throw e;_setThrew(1,0)}}function invoke_vji(index,a1,a2){var sp=stackSave();try{dynCall_vji(Number(index),a1,a2)}catch(e){stackRestore(sp);if(!(e instanceof EmscriptenEH))throw e;_setThrew(1,0)}}function invoke_jjj(index,a1,a2){var sp=stackSave();try{return dynCall_jjj(Number(index),a1,a2)}catch(e){stackRestore(sp);if(!(e instanceof EmscriptenEH))throw e;_setThrew(1,0);return 0n}}function invoke_vjjij(index,a1,a2,a3,a4){var sp=stackSave();try{dynCall_vjjij(Number(index),a1,a2,a3,a4)}catch(e){stackRestore(sp);if(!(e instanceof EmscriptenEH))throw e;_setThrew(1,0)}}function invoke_vjjiijjjjjjjjj(index,a1,a2,a3,a4,a5,a6,a7,a8,a9,a10,a11,a12,a13){var sp=stackSave();try{dynCall_vjjiijjjjjjjjj(Number(index),a1,a2,a3,a4,a5,a6,a7,a8,a9,a10,a11,a12,a13)}catch(e){stackRestore(sp);if(!(e instanceof EmscriptenEH))throw e;_setThrew(1,0)}}function invoke_jjjji(index,a1,a2,a3,a4){var sp=stackSave();try{return dynCall_jjjji(Number(index),a1,a2,a3,a4)}catch(e){stackRestore(sp);if(!(e instanceof EmscriptenEH))throw e;_setThrew(1,0);return 0n}}function invoke_iij(index,a1,a2){var sp=stackSave();try{return dynCall_iij(Number(index),a1,a2)}catch(e){stackRestore(sp);if(!(e instanceof EmscriptenEH))throw e;_setThrew(1,0)}}function invoke_vjij(index,a1,a2,a3){var sp=stackSave();try{dynCall_vjij(Number(index),a1,a2,a3)}catch(e){stackRestore(sp);if(!(e instanceof EmscriptenEH))throw e;_setThrew(1,0)}}function invoke_ij(index,a1){var sp=stackSave();try{return dynCall_ij(Number(index),a1)}catch(e){stackRestore(sp);if(!(e instanceof EmscriptenEH))throw e;_setThrew(1,0)}}function invoke_ji(index,a1){var sp=stackSave();try{return dynCall_ji(Number(index),a1)}catch(e){stackRestore(sp);if(!(e instanceof EmscriptenEH))throw e;_setThrew(1,0);return 0n}}function invoke_jjii(index,a1,a2,a3){var sp=stackSave();try{return dynCall_jjii(Number(index),a1,a2,a3)}catch(e){stackRestore(sp);if(!(e instanceof EmscriptenEH))throw e;_setThrew(1,0);return 0n}}function invoke_vjjjji(index,a1,a2,a3,a4,a5){var sp=stackSave();try{dynCall_vjjjji(Number(index),a1,a2,a3,a4,a5)}catch(e){stackRestore(sp);if(!(e instanceof EmscriptenEH))throw e;_setThrew(1,0)}}function invoke_vjjii(index,a1,a2,a3,a4){var sp=stackSave();try{dynCall_vjjii(Number(index),a1,a2,a3,a4)}catch(e){stackRestore(sp);if(!(e instanceof EmscriptenEH))throw e;_setThrew(1,0)}}function invoke_iijj(index,a1,a2,a3){var sp=stackSave();try{return dynCall_iijj(Number(index),a1,a2,a3)}catch(e){stackRestore(sp);if(!(e instanceof EmscriptenEH))throw e;_setThrew(1,0)}}function invoke_vjiji(index,a1,a2,a3,a4){var sp=stackSave();try{dynCall_vjiji(Number(index),a1,a2,a3,a4)}catch(e){stackRestore(sp);if(!(e instanceof EmscriptenEH))throw e;_setThrew(1,0)}}function invoke_i(index){var sp=stackSave();try{return dynCall_i(Number(index))}catch(e){stackRestore(sp);if(!(e instanceof EmscriptenEH))throw e;_setThrew(1,0)}}function invoke_jjji(index,a1,a2,a3){var sp=stackSave();try{return dynCall_jjji(Number(index),a1,a2,a3)}catch(e){stackRestore(sp);if(!(e instanceof EmscriptenEH))throw e;_setThrew(1,0);return 0n}}function invoke_vijj(index,a1,a2,a3){var sp=stackSave();try{dynCall_vijj(Number(index),a1,a2,a3)}catch(e){stackRestore(sp);if(!(e instanceof EmscriptenEH))throw e;_setThrew(1,0)}}function invoke_ijjii(index,a1,a2,a3,a4){var sp=stackSave();try{return dynCall_ijjii(Number(index),a1,a2,a3,a4)}catch(e){stackRestore(sp);if(!(e instanceof EmscriptenEH))throw e;_setThrew(1,0)}}function invoke_ijji(index,a1,a2,a3){var sp=stackSave();try{return dynCall_ijji(Number(index),a1,a2,a3)}catch(e){stackRestore(sp);if(!(e instanceof EmscriptenEH))throw e;_setThrew(1,0)}}function invoke_iii(index,a1,a2){var sp=stackSave();try{return dynCall_iii(Number(index),a1,a2)}catch(e){stackRestore(sp);if(!(e instanceof EmscriptenEH))throw e;_setThrew(1,0)}}function invoke_vi(index,a1){var sp=stackSave();try{dynCall_vi(Number(index),a1)}catch(e){stackRestore(sp);if(!(e instanceof EmscriptenEH))throw e;_setThrew(1,0)}}function invoke_vii(index,a1,a2){var sp=stackSave();try{dynCall_vii(Number(index),a1,a2)}catch(e){stackRestore(sp);if(!(e instanceof EmscriptenEH))throw e;_setThrew(1,0)}}function invoke_jjjjii(index,a1,a2,a3,a4,a5){var sp=stackSave();try{return dynCall_jjjjii(Number(index),a1,a2,a3,a4,a5)}catch(e){stackRestore(sp);if(!(e instanceof EmscriptenEH))throw e;_setThrew(1,0);return 0n}}function invoke_jjjiiii(index,a1,a2,a3,a4,a5,a6){var sp=stackSave();try{return dynCall_jjjiiii(Number(index),a1,a2,a3,a4,a5,a6)}catch(e){stackRestore(sp);if(!(e instanceof EmscriptenEH))throw e;_setThrew(1,0);return 0n}}function invoke_jjjj(index,a1,a2,a3){var sp=stackSave();try{return dynCall_jjjj(Number(index),a1,a2,a3)}catch(e){stackRestore(sp);if(!(e instanceof EmscriptenEH))throw e;_setThrew(1,0);return 0n}}function invoke_ijjji(index,a1,a2,a3,a4){var sp=stackSave();try{return dynCall_ijjji(Number(index),a1,a2,a3,a4)}catch(e){stackRestore(sp);if(!(e instanceof EmscriptenEH))throw e;_setThrew(1,0)}}function invoke_ijjjj(index,a1,a2,a3,a4){var sp=stackSave();try{return dynCall_ijjjj(Number(index),a1,a2,a3,a4)}catch(e){stackRestore(sp);if(!(e instanceof EmscriptenEH))throw e;_setThrew(1,0)}}function invoke_iji(index,a1,a2){var sp=stackSave();try{return dynCall_iji(Number(index),a1,a2)}catch(e){stackRestore(sp);if(!(e instanceof EmscriptenEH))throw e;_setThrew(1,0)}}function invoke_vjjdi(index,a1,a2,a3,a4){var sp=stackSave();try{dynCall_vjjdi(Number(index),a1,a2,a3,a4)}catch(e){stackRestore(sp);if(!(e instanceof EmscriptenEH))throw e;_setThrew(1,0)}}function invoke_ijiijj(index,a1,a2,a3,a4,a5){var sp=stackSave();try{return dynCall_ijiijj(Number(index),a1,a2,a3,a4,a5)}catch(e){stackRestore(sp);if(!(e instanceof EmscriptenEH))throw e;_setThrew(1,0)}}function invoke_vjii(index,a1,a2,a3){var sp=stackSave();try{dynCall_vjii(Number(index),a1,a2,a3)}catch(e){stackRestore(sp);if(!(e instanceof EmscriptenEH))throw e;_setThrew(1,0)}}function invoke_ijjjjji(index,a1,a2,a3,a4,a5,a6){var sp=stackSave();try{return dynCall_ijjjjji(Number(index),a1,a2,a3,a4,a5,a6)}catch(e){stackRestore(sp);if(!(e instanceof EmscriptenEH))throw e;_setThrew(1,0)}}function invoke_jijj(index,a1,a2,a3){var sp=stackSave();try{return dynCall_jijj(Number(index),a1,a2,a3)}catch(e){stackRestore(sp);if(!(e instanceof EmscriptenEH))throw e;_setThrew(1,0);return 0n}}function invoke_vjjjjij(index,a1,a2,a3,a4,a5,a6){var sp=stackSave();try{dynCall_vjjjjij(Number(index),a1,a2,a3,a4,a5,a6)}catch(e){stackRestore(sp);if(!(e instanceof EmscriptenEH))throw e;_setThrew(1,0)}}function invoke_vjjjjjj(index,a1,a2,a3,a4,a5,a6){var sp=stackSave();try{dynCall_vjjjjjj(Number(index),a1,a2,a3,a4,a5,a6)}catch(e){stackRestore(sp);if(!(e instanceof EmscriptenEH))throw e;_setThrew(1,0)}}function invoke_jjjjjj(index,a1,a2,a3,a4,a5){var sp=stackSave();try{return dynCall_jjjjjj(Number(index),a1,a2,a3,a4,a5)}catch(e){stackRestore(sp);if(!(e instanceof EmscriptenEH))throw e;_setThrew(1,0);return 0n}}function invoke_ijjiii(index,a1,a2,a3,a4,a5){var sp=stackSave();try{return dynCall_ijjiii(Number(index),a1,a2,a3,a4,a5)}catch(e){stackRestore(sp);if(!(e instanceof EmscriptenEH))throw e;_setThrew(1,0)}}function invoke_jjjjijj(index,a1,a2,a3,a4,a5,a6){var sp=stackSave();try{return dynCall_jjjjijj(Number(index),a1,a2,a3,a4,a5,a6)}catch(e){stackRestore(sp);if(!(e instanceof EmscriptenEH))throw e;_setThrew(1,0);return 0n}}function invoke_vjji(index,a1,a2,a3){var sp=stackSave();try{dynCall_vjji(Number(index),a1,a2,a3)}catch(e){stackRestore(sp);if(!(e instanceof EmscriptenEH))throw e;_setThrew(1,0)}}function invoke_vij(index,a1,a2){var sp=stackSave();try{dynCall_vij(Number(index),a1,a2)}catch(e){stackRestore(sp);if(!(e instanceof EmscriptenEH))throw e;_setThrew(1,0)}}function invoke_ijjjjjji(index,a1,a2,a3,a4,a5,a6,a7){var sp=stackSave();try{return dynCall_ijjjjjji(Number(index),a1,a2,a3,a4,a5,a6,a7)}catch(e){stackRestore(sp);if(!(e instanceof EmscriptenEH))throw e;_setThrew(1,0)}}function invoke_ijiij(index,a1,a2,a3,a4){var sp=stackSave();try{return dynCall_ijiij(Number(index),a1,a2,a3,a4)}catch(e){stackRestore(sp);if(!(e instanceof EmscriptenEH))throw e;_setThrew(1,0)}}function invoke_vjijj(index,a1,a2,a3,a4){var sp=stackSave();try{dynCall_vjijj(Number(index),a1,a2,a3,a4)}catch(e){stackRestore(sp);if(!(e instanceof EmscriptenEH))throw e;_setThrew(1,0)}}function invoke_jjjjjjiijj(index,a1,a2,a3,a4,a5,a6,a7,a8,a9){var sp=stackSave();try{return dynCall_jjjjjjiijj(Number(index),a1,a2,a3,a4,a5,a6,a7,a8,a9)}catch(e){stackRestore(sp);if(!(e instanceof EmscriptenEH))throw e;_setThrew(1,0);return 0n}}function invoke_ijij(index,a1,a2,a3){var sp=stackSave();try{return dynCall_ijij(Number(index),a1,a2,a3)}catch(e){stackRestore(sp);if(!(e instanceof EmscriptenEH))throw e;_setThrew(1,0)}}function invoke_ijjijj(index,a1,a2,a3,a4,a5){var sp=stackSave();try{return dynCall_ijjijj(Number(index),a1,a2,a3,a4,a5)}catch(e){stackRestore(sp);if(!(e instanceof EmscriptenEH))throw e;_setThrew(1,0)}}function invoke_jij(index,a1,a2){var sp=stackSave();try{return dynCall_jij(Number(index),a1,a2)}catch(e){stackRestore(sp);if(!(e instanceof EmscriptenEH))throw e;_setThrew(1,0);return 0n}}function invoke_ijjjjijj(index,a1,a2,a3,a4,a5,a6,a7){var sp=stackSave();try{return dynCall_ijjjjijj(Number(index),a1,a2,a3,a4,a5,a6,a7)}catch(e){stackRestore(sp);if(!(e instanceof EmscriptenEH))throw e;_setThrew(1,0)}}function invoke_vijijj(index,a1,a2,a3,a4,a5){var sp=stackSave();try{dynCall_vijijj(Number(index),a1,a2,a3,a4,a5)}catch(e){stackRestore(sp);if(!(e instanceof EmscriptenEH))throw e;_setThrew(1,0)}}function invoke_vjjjjj(index,a1,a2,a3,a4,a5){var sp=stackSave();try{dynCall_vjjjjj(Number(index),a1,a2,a3,a4,a5)}catch(e){stackRestore(sp);if(!(e instanceof EmscriptenEH))throw e;_setThrew(1,0)}}function invoke_jjjjji(index,a1,a2,a3,a4,a5){var sp=stackSave();try{return dynCall_jjjjji(Number(index),a1,a2,a3,a4,a5)}catch(e){stackRestore(sp);if(!(e instanceof EmscriptenEH))throw e;_setThrew(1,0);return 0n}}function invoke_ijii(index,a1,a2,a3){var sp=stackSave();try{return dynCall_ijii(Number(index),a1,a2,a3)}catch(e){stackRestore(sp);if(!(e instanceof EmscriptenEH))throw e;_setThrew(1,0)}}function applySignatureConversions(wasmExports){wasmExports=Object.assign({},wasmExports);var makeWrapper__p=f=>a0=>f(BigInt(a0));var makeWrapper_pp=f=>a0=>Number(f(BigInt(a0)));var makeWrapper_ppp=f=>(a0,a1)=>Number(f(BigInt(a0),BigInt(a1)));var makeWrapper__p_=f=>(a0,a1)=>f(BigInt(a0),a1);var makeWrapper__pp=f=>(a0,a1)=>f(BigInt(a0),BigInt(a1));var makeWrapper_p=f=>()=>Number(f());var makeWrapper__p__=f=>(a0,a1,a2)=>f(BigInt(a0),a1,a2);var makeWrapper__p____=f=>(a0,a1,a2,a3,a4)=>f(BigInt(a0),a1,a2,a3,a4);var makeWrapper__p___=f=>(a0,a1,a2,a3)=>f(BigInt(a0),a1,a2,a3);var makeWrapper__p______=f=>(a0,a1,a2,a3,a4,a5,a6)=>f(BigInt(a0),a1,a2,a3,a4,a5,a6);var makeWrapper__p_______=f=>(a0,a1,a2,a3,a4,a5,a6,a7)=>f(BigInt(a0),a1,a2,a3,a4,a5,a6,a7);var makeWrapper__p_____=f=>(a0,a1,a2,a3,a4,a5)=>f(BigInt(a0),a1,a2,a3,a4,a5);var makeWrapper__p_________=f=>(a0,a1,a2,a3,a4,a5,a6,a7,a8,a9)=>f(BigInt(a0),a1,a2,a3,a4,a5,a6,a7,a8,a9);var makeWrapper__p_____________=f=>(a0,a1,a2,a3,a4,a5,a6,a7,a8,a9,a10,a11,a12,a13)=>f(BigInt(a0),a1,a2,a3,a4,a5,a6,a7,a8,a9,a10,a11,a12,a13);var makeWrapper__p________=f=>(a0,a1,a2,a3,a4,a5,a6,a7,a8)=>f(BigInt(a0),a1,a2,a3,a4,a5,a6,a7,a8);wasmExports["Ob"]=makeWrapper__p(wasmExports["Ob"]);wasmExports["Pb"]=makeWrapper_pp(wasmExports["Pb"]);wasmExports["Vb"]=makeWrapper_ppp(wasmExports["Vb"]);wasmExports["Xb"]=makeWrapper__p_(wasmExports["Xb"]);wasmExports["Yb"]=makeWrapper__pp(wasmExports["Yb"]);wasmExports["Zb"]=makeWrapper__p(wasmExports["Zb"]);wasmExports["_b"]=makeWrapper_pp(wasmExports["_b"]);wasmExports["$b"]=makeWrapper_p(wasmExports["$b"]);wasmExports["ac"]=makeWrapper__p__(wasmExports["ac"]);wasmExports["bc"]=makeWrapper__p__(wasmExports["bc"]);wasmExports["cc"]=makeWrapper__p____(wasmExports["cc"]);wasmExports["dc"]=makeWrapper__p___(wasmExports["dc"]);wasmExports["ec"]=makeWrapper__p___(wasmExports["ec"]);wasmExports["fc"]=makeWrapper__p_(wasmExports["fc"]);wasmExports["gc"]=makeWrapper__p_(wasmExports["gc"]);wasmExports["hc"]=makeWrapper__p____(wasmExports["hc"]);wasmExports["ic"]=makeWrapper__p_(wasmExports["ic"]);wasmExports["jc"]=makeWrapper__p___(wasmExports["jc"]);wasmExports["kc"]=makeWrapper__p______(wasmExports["kc"]);wasmExports["lc"]=makeWrapper__p_______(wasmExports["lc"]);wasmExports["mc"]=makeWrapper__p____(wasmExports["mc"]);wasmExports["nc"]=makeWrapper__p_____(wasmExports["nc"]);wasmExports["oc"]=makeWrapper__p___(wasmExports["oc"]);wasmExports["pc"]=makeWrapper__p_________(wasmExports["pc"]);wasmExports["qc"]=makeWrapper__p__(wasmExports["qc"]);wasmExports["rc"]=makeWrapper__p___(wasmExports["rc"]);wasmExports["sc"]=makeWrapper__p__(wasmExports["sc"]);wasmExports["tc"]=makeWrapper__p___(wasmExports["tc"]);wasmExports["uc"]=makeWrapper__p_(wasmExports["uc"]);wasmExports["vc"]=makeWrapper__p_____(wasmExports["vc"]);wasmExports["wc"]=makeWrapper__p_______(wasmExports["wc"]);wasmExports["xc"]=makeWrapper__p______(wasmExports["xc"]);wasmExports["yc"]=makeWrapper__p__(wasmExports["yc"]);wasmExports["zc"]=makeWrapper__p(wasmExports["zc"]);wasmExports["Ac"]=makeWrapper__p(wasmExports["Ac"]);wasmExports["Bc"]=makeWrapper__p____(wasmExports["Bc"]);wasmExports["Cc"]=makeWrapper__p_____________(wasmExports["Cc"]);wasmExports["Dc"]=makeWrapper__p____(wasmExports["Dc"]);wasmExports["Ec"]=makeWrapper__p__(wasmExports["Ec"]);wasmExports["Fc"]=makeWrapper__p___(wasmExports["Fc"]);wasmExports["Gc"]=makeWrapper__p_(wasmExports["Gc"]);wasmExports["Hc"]=makeWrapper__p___(wasmExports["Hc"]);wasmExports["Ic"]=makeWrapper__p_____(wasmExports["Ic"]);wasmExports["Jc"]=makeWrapper__p____(wasmExports["Jc"]);wasmExports["Kc"]=makeWrapper__p____(wasmExports["Kc"]);wasmExports["Lc"]=makeWrapper__p__(wasmExports["Lc"]);wasmExports["Mc"]=makeWrapper__p_____(wasmExports["Mc"]);wasmExports["Nc"]=makeWrapper__p___(wasmExports["Nc"]);wasmExports["Oc"]=makeWrapper__p_____(wasmExports["Oc"]);wasmExports["Pc"]=makeWrapper__p____(wasmExports["Pc"]);wasmExports["Qc"]=makeWrapper__p___(wasmExports["Qc"]);wasmExports["Rc"]=makeWrapper__p____(wasmExports["Rc"]);wasmExports["Sc"]=makeWrapper__p(wasmExports["Sc"]);wasmExports["Tc"]=makeWrapper__p___(wasmExports["Tc"]);wasmExports["Uc"]=makeWrapper__p____(wasmExports["Uc"]);wasmExports["Vc"]=makeWrapper__p_(wasmExports["Vc"]);wasmExports["Wc"]=makeWrapper__p__(wasmExports["Wc"]);wasmExports["Xc"]=makeWrapper__p___(wasmExports["Xc"]);wasmExports["Yc"]=makeWrapper__p____(wasmExports["Yc"]);wasmExports["Zc"]=makeWrapper__p_____(wasmExports["Zc"]);wasmExports["_c"]=makeWrapper__p______(wasmExports["_c"]);wasmExports["$c"]=makeWrapper__p____(wasmExports["$c"]);wasmExports["ad"]=makeWrapper__p_____(wasmExports["ad"]);wasmExports["bd"]=makeWrapper__p_____(wasmExports["bd"]);wasmExports["cd"]=makeWrapper__p___(wasmExports["cd"]);wasmExports["dd"]=makeWrapper__p_____(wasmExports["dd"]);wasmExports["ed"]=makeWrapper__p_________(wasmExports["ed"]);wasmExports["fd"]=makeWrapper__p____(wasmExports["fd"]);wasmExports["gd"]=makeWrapper__p_____(wasmExports["gd"]);wasmExports["hd"]=makeWrapper__p____(wasmExports["hd"]);wasmExports["id"]=makeWrapper__p___(wasmExports["id"]);wasmExports["jd"]=makeWrapper__p______(wasmExports["jd"]);wasmExports["kd"]=makeWrapper__p______(wasmExports["kd"]);wasmExports["ld"]=makeWrapper__p_____(wasmExports["ld"]);wasmExports["md"]=makeWrapper__p_____(wasmExports["md"]);wasmExports["nd"]=makeWrapper__p______(wasmExports["nd"]);wasmExports["od"]=makeWrapper__p__(wasmExports["od"]);wasmExports["pd"]=makeWrapper__p__(wasmExports["pd"]);wasmExports["qd"]=makeWrapper__p_______(wasmExports["qd"]);wasmExports["rd"]=makeWrapper__p_____(wasmExports["rd"]);wasmExports["sd"]=makeWrapper__p____(wasmExports["sd"]);wasmExports["td"]=makeWrapper__p____(wasmExports["td"]);wasmExports["ud"]=makeWrapper__p________(wasmExports["ud"]);wasmExports["vd"]=makeWrapper__p____(wasmExports["vd"]);wasmExports["wd"]=makeWrapper__p_____(wasmExports["wd"]);wasmExports["xd"]=makeWrapper__p____(wasmExports["xd"]);wasmExports["yd"]=makeWrapper__p___(wasmExports["yd"]);wasmExports["zd"]=makeWrapper__p___(wasmExports["zd"]);wasmExports["Ad"]=makeWrapper__p_______(wasmExports["Ad"]);wasmExports["Bd"]=makeWrapper__p__(wasmExports["Bd"]);wasmExports["Cd"]=makeWrapper__p_______(wasmExports["Cd"]);wasmExports["Dd"]=makeWrapper__p______(wasmExports["Dd"]);wasmExports["Ed"]=makeWrapper__p_____(wasmExports["Ed"]);wasmExports["Fd"]=makeWrapper__p___(wasmExports["Fd"]);wasmExports["Gd"]=makeWrapper__p______(wasmExports["Gd"]);wasmExports["Hd"]=makeWrapper__p___(wasmExports["Hd"]);wasmExports["Id"]=makeWrapper__p__(wasmExports["Id"]);wasmExports["Jd"]=makeWrapper__p______(wasmExports["Jd"]);wasmExports["Kd"]=makeWrapper__p_____(wasmExports["Kd"]);wasmExports["Ld"]=makeWrapper__p_____(wasmExports["Ld"]);wasmExports["Md"]=makeWrapper__p_____(wasmExports["Md"]);wasmExports["Nd"]=makeWrapper__p_____(wasmExports["Nd"]);wasmExports["Od"]=makeWrapper__p______(wasmExports["Od"]);wasmExports["Pd"]=makeWrapper__p_____(wasmExports["Pd"]);wasmExports["Qd"]=makeWrapper__p(wasmExports["Qd"]);wasmExports["Sd"]=makeWrapper__p(wasmExports["Sd"]);return wasmExports}async function run(){preRun();if(runDependencies){await resolveRunDependencies()}var setStatus=Module["setStatus"];if(setStatus){setStatus("Running...");await new Promise(resolve=>setTimeout(resolve,1));setTimeout(setStatus,1,"")}if(ABORT)return;initRuntime();Module["onRuntimeInitialized"]?.();postRun()}var wasmExports;wasmExports=await createWasm();await run(); +;return Module}})();if(typeof exports==="object"&&typeof module==="object"){module.exports=PHP;module.exports.default=PHP}else if(typeof define==="function"&&define["amd"])define([],()=>PHP); diff --git a/web/landing/assets/wasm/php.wasm b/web/landing/assets/wasm/php.wasm index 733fb6246e..9f38206971 100755 Binary files a/web/landing/assets/wasm/php.wasm and b/web/landing/assets/wasm/php.wasm differ diff --git a/web/landing/assets/wasm/tools/flow.phar b/web/landing/assets/wasm/tools/flow.phar index d0c4b09847..806e5b5fe4 100755 Binary files a/web/landing/assets/wasm/tools/flow.phar and b/web/landing/assets/wasm/tools/flow.phar differ diff --git a/web/landing/content/examples/topics/batching/batch_by/description.md b/web/landing/content/examples/topics/batching/batch_by/description.md index 0176669956..3ed58f132e 100644 --- a/web/landing/content/examples/topics/batching/batch_by/description.md +++ b/web/landing/content/examples/topics/batching/batch_by/description.md @@ -1,3 +1,3 @@ -Group rows based on column values, ensuring related records stay together in the same batch. This is useful when processing hierarchical data (like orders with line items) where splitting related records would cause integrity issues. - -**Important:** Data must be sorted by the grouping column before using batch_by. +`batchBy('order_id')` cuts batches only where `order_id` changes, so rows sharing a value always land +in the same batch. The input must already be ordered by that column - `constraint_sorted_by()` fails +the run if it is not. diff --git a/web/landing/content/examples/topics/batching/collect/description.md b/web/landing/content/examples/topics/batching/collect/description.md index aaa95a30e2..c508a7fe70 100644 --- a/web/landing/content/examples/topics/batching/collect/description.md +++ b/web/landing/content/examples/topics/batching/collect/description.md @@ -1,3 +1,3 @@ -Load all rows into memory and process them at once. This is useful for debugging and working with small datasets. - -**Warning:** Avoid using collect with large datasets as it loads everything into memory. For controlled memory consumption, use [batchSize](/data_frame/batch_size/#example) instead. +`collect()` merges every batch into one, so the next step sees the whole frame at once. Memory grows +with the data - keep it for small frames, or bound it with +[batchSize()](/batching/constant_size/#example). diff --git a/web/landing/content/examples/topics/batching/column_backend/_meta.yaml b/web/landing/content/examples/topics/batching/column_backend/_meta.yaml new file mode 100644 index 0000000000..d3f9270d46 --- /dev/null +++ b/web/landing/content/examples/topics/batching/column_backend/_meta.yaml @@ -0,0 +1,2 @@ +priority: 4 +hidden: false diff --git a/web/landing/content/examples/topics/batching/column_backend/code.php b/web/landing/content/examples/topics/batching/column_backend/code.php new file mode 100644 index 0000000000..08fe170aba --- /dev/null +++ b/web/landing/content/examples/topics/batching/column_backend/code.php @@ -0,0 +1,20 @@ +backend(new PhpBackend())) + ->read(from_array([['id' => 1], ['id' => 2], ['id' => 3]])) + ->withEntry('double', ref('id')->multiply(lit(2))) + ->fetch(); + +foreach (['id', 'double'] as $name) { + $column = $rows->column($name); + + echo $name . ': ' . $column::class . ' ' . json_encode($column->values()) . "\n"; +} diff --git a/web/landing/content/examples/topics/batching/column_backend/description.md b/web/landing/content/examples/topics/batching/column_backend/description.md new file mode 100644 index 0000000000..8032dfab84 --- /dev/null +++ b/web/landing/content/examples/topics/batching/column_backend/description.md @@ -0,0 +1,4 @@ +A batch stores one column per schema definition, and the backend set with +`config_builder()->backend()` builds every one of them. `AdaptiveBackend`, the default, picks +`RustBackend` when the flow_php extension is loaded and `PhpBackend` otherwise. Set `PhpBackend` +to opt out of the extension; both give the same values. diff --git a/web/landing/content/examples/topics/batching/column_backend/documentation.md b/web/landing/content/examples/topics/batching/column_backend/documentation.md new file mode 100644 index 0000000000..9ed84a4c52 --- /dev/null +++ b/web/landing/content/examples/topics/batching/column_backend/documentation.md @@ -0,0 +1,5 @@ +Batch size decides how many rows travel through the pipeline together. + +- [Batch Processing](/documentation/components/core/batch-processing) +- [Column Backend](/documentation/components/core/column-backend) +- [Building Blocks](/documentation/components/core/building-blocks) diff --git a/web/landing/content/examples/topics/batching/constant_size/description.md b/web/landing/content/examples/topics/batching/constant_size/description.md index 5627357f11..15deed1e3a 100644 --- a/web/landing/content/examples/topics/batching/constant_size/description.md +++ b/web/landing/content/examples/topics/batching/constant_size/description.md @@ -1,3 +1,3 @@ -Control how many rows are processed at once. Larger batch sizes can improve processing speed but require more memory. There is no universal optimal batch size-it depends on your dataset and transformations. - -To process all rows at once, use [collect](/data_frame/collect/#example). +`batchSize(2)` splits the frame into batches of two rows; `to_output()` prints each batch on its own. +Larger batches mean fewer steps and more memory. `batchSize(-1)` or +[collect()](/batching/collect/#example) puts the whole frame in one batch. diff --git a/web/landing/content/examples/topics/caching/cache/code.php b/web/landing/content/examples/topics/caching/cache/code.php index e3080eb2f4..50d07112fe 100644 --- a/web/landing/content/examples/topics/caching/cache/code.php +++ b/web/landing/content/examples/topics/caching/cache/code.php @@ -2,7 +2,7 @@ declare(strict_types=1); -use function Flow\ETL\DSL\{config_builder, data_frame, datetime_schema, filesystem_cache, from_cache, int_schema, ref, rename_replace, schema, str_schema, to_output}; +use function Flow\ETL\DSL\{config_builder, data_frame, datetime_schema, from_cache, int_schema, ref, rename_replace, schema, str_schema, to_output}; use Flow\ETL\Adapter\Http\DynamicExtractor\NextRequestFactory; use Flow\ETL\Adapter\Http\PsrHttpClientDynamicExtractor; use Http\Client\Curl\Client; @@ -30,7 +30,7 @@ public function create(?ResponseInterface $previousResponse = null) : ?RequestIn } }); -data_frame(config_builder()->cache(filesystem_cache(__DIR__ . '/output/cache'))) +data_frame(config_builder()->cacheDir(__DIR__ . '/output/cache')) ->read( from_cache( id: 'github_api', diff --git a/web/landing/content/examples/topics/columns/create/description.md b/web/landing/content/examples/topics/columns/create/description.md index b1940a9df5..1acd9a47f9 100644 --- a/web/landing/content/examples/topics/columns/create/description.md +++ b/web/landing/content/examples/topics/columns/create/description.md @@ -1 +1,2 @@ -Add new columns to your data or replace existing ones. New columns can be created from constant values or computed dynamically based on other column values. +`withEntry()` adds a column, or replaces one with the same name. The value is any expression: +computed from other columns (`ref('id')->isOdd()`) or a constant (`lit(5)`). diff --git a/web/landing/content/examples/topics/columns/select/description.md b/web/landing/content/examples/topics/columns/select/description.md index b33a190041..dd4bbac544 100644 --- a/web/landing/content/examples/topics/columns/select/description.md +++ b/web/landing/content/examples/topics/columns/select/description.md @@ -1 +1,2 @@ -Choose specific columns from your data, discarding the rest. Transformations like select help keep pipelines readable and explicit about which data is being processed. +`select()` keeps the named columns and drops the rest. As a transformation it can be passed to +`with()`, or called directly as `->select('id')`. diff --git a/web/landing/content/examples/topics/errors/ignore/code.php b/web/landing/content/examples/topics/errors/ignore/code.php index 164e595ed5..731c00d396 100644 --- a/web/landing/content/examples/topics/errors/ignore/code.php +++ b/web/landing/content/examples/topics/errors/ignore/code.php @@ -6,15 +6,17 @@ require __DIR__ . '/vendor/autoload.php'; -// ignore_error_handler() keeps the row that threw, without the column that failed - so the -// batches no longer share a schema and cannot be collected into one table +// ignore_error_handler() drops the batch a transformation failed in and keeps going: +// with batchSize(1), only id 2 is lost data_frame() ->read(from_array([ ['id' => 1, 'divisor' => 2], ['id' => 2, 'divisor' => 0], ['id' => 3, 'divisor' => 5], ], schema(int_schema('id'), int_schema('divisor')))) + ->batchSize(1) ->onError(ignore_error_handler()) ->withEntry('result', lit(100)->divide(ref('divisor'))) + ->collect() ->write(to_output(truncate: false)) ->run(); diff --git a/web/landing/content/examples/topics/errors/ignore/description.md b/web/landing/content/examples/topics/errors/ignore/description.md index dc552a93c7..613e252e81 100644 --- a/web/landing/content/examples/topics/errors/ignore/description.md +++ b/web/landing/content/examples/topics/errors/ignore/description.md @@ -1,2 +1,3 @@ -`ignore_error_handler()` keeps the row that threw, without the column that failed. The batches -then no longer share a schema, which is why the output arrives as separate tables. +`ignore_error_handler()` drops the batch a transformation failed in and continues with the next one. +With `batchSize(1)` that is only the failing row. Unlike `skip_rows_handler()`, it also survives a +failing loader: that loader misses the batch, the other writes still run. diff --git a/web/landing/content/examples/topics/filesystem/local/description.md b/web/landing/content/examples/topics/filesystem/local/description.md index 844cf484d7..4b4be50f91 100644 --- a/web/landing/content/examples/topics/filesystem/local/description.md +++ b/web/landing/content/examples/topics/filesystem/local/description.md @@ -1 +1,2 @@ -Flow includes a dedicated filesystem abstraction for working with files. The local filesystem provides operations for listing files (with glob pattern support), reading data (in full or chunks), and writing data. +`fstab()->for('file')` is the local filesystem every reader and writer goes through. It writes +streams, lists paths by glob pattern and reads files back, in full or in chunks. diff --git a/web/landing/content/examples/topics/filesystem/stdout/description.md b/web/landing/content/examples/topics/filesystem/stdout/description.md index ff60c4c390..cb35e1ade9 100644 --- a/web/landing/content/examples/topics/filesystem/stdout/description.md +++ b/web/landing/content/examples/topics/filesystem/stdout/description.md @@ -1,3 +1,2 @@ -Write data directly to the process stdout. This is useful for streaming data to web clients without buffering in memory. - -**Note:** Stdout is write-only-reading is not supported. +The `stdout://` filesystem streams writes straight to the process output, with nothing buffered in +memory. It is write-only: reading from it is not supported. diff --git a/web/landing/content/examples/topics/getting_started/03_inferred_or_declared/code.php b/web/landing/content/examples/topics/getting_started/03_inferred_or_declared/code.php index 97731e4394..98bbe4e199 100644 --- a/web/landing/content/examples/topics/getting_started/03_inferred_or_declared/code.php +++ b/web/landing/content/examples/topics/getting_started/03_inferred_or_declared/code.php @@ -9,7 +9,7 @@ $path = __DIR__ . '/data/orders.csv'; -// no schema given: Flow reads the file's own description of itself +// no schema given: the reader infers one from a sample of the file echo "inferred:\n" . schema_to_ascii( data_frame()->read(from_csv($path))->select('order_id', 'discount')->schema(), ); diff --git a/web/landing/content/examples/topics/inspecting/fetch/code.php b/web/landing/content/examples/topics/inspecting/fetch/code.php index 0bcd5d1b0d..4fcdc638c0 100644 --- a/web/landing/content/examples/topics/inspecting/fetch/code.php +++ b/web/landing/content/examples/topics/inspecting/fetch/code.php @@ -16,8 +16,8 @@ ->fetch(2); echo 'fetched ' . $rows->count() . " rows\n"; -echo 'first name: ' . $rows->first()->get('name') . "\n"; +echo 'first name: ' . $rows->column('name')->value(0) . "\n"; -foreach ($rows as $row) { - echo " {$row->get('id')}: {$row->get('name')}\n"; +foreach ($rows->toArray() as $row) { + echo " {$row['id']}: {$row['name']}\n"; } diff --git a/web/landing/content/examples/topics/inspecting/for_each/_meta.yaml b/web/landing/content/examples/topics/inspecting/for_each/_meta.yaml new file mode 100644 index 0000000000..8608dd76b4 --- /dev/null +++ b/web/landing/content/examples/topics/inspecting/for_each/_meta.yaml @@ -0,0 +1,2 @@ +priority: 3 +hidden: false diff --git a/web/landing/content/examples/topics/inspecting/for_each/code.php b/web/landing/content/examples/topics/inspecting/for_each/code.php new file mode 100644 index 0000000000..9484374f77 --- /dev/null +++ b/web/landing/content/examples/topics/inspecting/for_each/code.php @@ -0,0 +1,23 @@ +read(from_array([ + ['id' => 1, 'name' => 'Norbert'], + ['id' => 2, 'name' => 'Jane'], + ['id' => 3, 'name' => 'John'], + ['id' => 4, 'name' => 'Ann'], + ['id' => 5, 'name' => 'Bob'], + ], schema(int_schema('id'), str_schema('name')))) + ->batchSize(2) + ->forEach(static function (Rows $rows): void { + echo $rows->count() . ' rows, names: ' . implode(', ', $rows->column('name')->values()) . "\n"; + }); diff --git a/web/landing/content/examples/topics/inspecting/for_each/description.md b/web/landing/content/examples/topics/inspecting/for_each/description.md new file mode 100644 index 0000000000..da068ff5ac --- /dev/null +++ b/web/landing/content/examples/topics/inspecting/for_each/description.md @@ -0,0 +1,2 @@ +`forEach()` runs the pipeline and hands each batch (`Rows`) to a callback. A batch is read by +column - `$rows->column('name')->values()` - or as rows with `toArray()`. diff --git a/web/landing/content/examples/topics/inspecting/for_each/documentation.md b/web/landing/content/examples/topics/inspecting/for_each/documentation.md new file mode 100644 index 0000000000..38319fcb58 --- /dev/null +++ b/web/landing/content/examples/topics/inspecting/for_each/documentation.md @@ -0,0 +1,4 @@ +Inspecting a frame is how you see what a pipeline is actually producing. + +- [Display](/documentation/components/core/display) +- [Data Retrieval](/documentation/components/core/data-retrieval) diff --git a/web/landing/content/examples/topics/joins/join/description.md b/web/landing/content/examples/topics/joins/join/description.md index d413c909fb..2fb89a8f9c 100644 --- a/web/landing/content/examples/topics/joins/join/description.md +++ b/web/landing/content/examples/topics/joins/join/description.md @@ -1,3 +1,4 @@ -Combine two datasets based on matching column values, similar to SQL JOIN operations. Supports inner, left, right, and left anti join types. - -For large datasets that don't fit in memory, consider using [joinEach](/joins/join_each/#example) instead. +`join()` matches rows of two frames on column values, as SQL does: `Join::inner`, `left`, `right` or +`left_anti`. `join_prefix` renames the right side's columns, so names never collide. The right +frame is read whole (held in memory, spilled past the memory limit); to look up only what each +batch needs, use [joinEach()](/joins/join_each/#example). diff --git a/web/landing/content/examples/topics/joins/join_each/code.php b/web/landing/content/examples/topics/joins/join_each/code.php index 7ca7ccce64..52774c6c3c 100644 --- a/web/landing/content/examples/topics/joins/join_each/code.php +++ b/web/landing/content/examples/topics/joins/join_each/code.php @@ -2,7 +2,7 @@ declare(strict_types=1); -use function Flow\ETL\DSL\{data_frame, df, equal, from_array, int_schema, join_on, row, rows, schema, str_schema, to_output}; +use function Flow\ETL\DSL\{array_to_rows, data_frame, df, equal, from_array, int_schema, join_on, schema, str_schema, to_output}; use Flow\ETL\{DataFrame, DataFrameFactory, Extractor, FlowContext, Rows}; use Flow\ETL\Extractor\Statistics; use Flow\ETL\Join\Join; @@ -19,9 +19,9 @@ public function __construct(private Schema $schema) public function extract(FlowContext $context, ?int $limit = null): Generator { - yield rows($this->schema, row(['id' => 1, 'sku' => 'PRODUCT01']), row(['id' => 2, 'sku' => 'PRODUCT02'])); + yield array_to_rows([['id' => 1, 'sku' => 'PRODUCT01'], ['id' => 2, 'sku' => 'PRODUCT02']], $this->schema); - yield rows($this->schema, row(['id' => 10_001, 'sku' => 'PRODUCT10_001'])); + yield array_to_rows([['id' => 10_001, 'sku' => 'PRODUCT10_001']], $this->schema); } public function schema(): Schema @@ -36,10 +36,7 @@ public function statistics(): Statistics public function withSchema(Schema $schema): static { - $clone = clone $this; - $clone->schema = $schema; - - return $clone; + return new static($schema); } }; diff --git a/web/landing/content/examples/topics/joins/join_each/description.md b/web/landing/content/examples/topics/joins/join_each/description.md index 322e72c7c2..722b235b51 100644 --- a/web/landing/content/examples/topics/joins/join_each/description.md +++ b/web/landing/content/examples/topics/joins/join_each/description.md @@ -1 +1,3 @@ -Join datasets that are too large to fit in memory. Unlike regular [join](/joins/join/#example), joinEach fetches only the relevant rows from storage for each batch, making it suitable for large right-side datasets. +`joinEach()` asks a `DataFrameFactory` for the right side once per left batch, so the right side is +never loaded whole - here, the rows the database does not know about yet (`Join::left_anti`). Use +it when the right side is too large for [join()](/joins/join/#example). diff --git a/web/landing/content/examples/topics/partitioning/partition_placeholders/description.md b/web/landing/content/examples/topics/partitioning/partition_placeholders/description.md index 5e69eb0f5f..8f0887eb40 100644 --- a/web/landing/content/examples/topics/partitioning/partition_placeholders/description.md +++ b/web/landing/content/examples/topics/partitioning/partition_placeholders/description.md @@ -1,14 +1,14 @@ -Partition into a flat directory structure using `{column}` placeholders in the destination path instead of Hive-style `column=value` directories. Every partition consumed by a placeholder becomes part of the file or directory name; remaining partitions still create `column=value` directories. +`{column}` placeholders in the destination path name files and directories after the partition +values, instead of Hive `column=value` directories. A partition column without a placeholder still +gets a `column=value` directory. -```bash -output -├── blue -│ └── PRODUCT02.csv -├── green -│ └── PRODUCT01.csv -└── red - ├── PRODUCT01.csv - └── PRODUCT02.csv +``` +output/blue/PRODUCT02.csv +output/green/PRODUCT01.csv +output/red/PRODUCT01.csv +output/red/PRODUCT02.csv ``` -Reading with the same placeholder pattern recreates the partitions from the path, including support for partition pruning through `filter()`. Keep in mind that this layout is not self-describing - a plain glob like `output/**/*.csv` will read the data but won't recognize any partitions. +Reading with the same pattern restores `color` and `sku` from the path, and `filter()` on them +prunes files. The layout is not self-describing: a plain glob such as `output/**/*.csv` reads the +data without the partition columns. diff --git a/web/landing/content/examples/topics/partitioning/partition_pruning/description.md b/web/landing/content/examples/topics/partitioning/partition_pruning/description.md index 73d660de3c..38ccf2bd5a 100644 --- a/web/landing/content/examples/topics/partitioning/partition_pruning/description.md +++ b/web/landing/content/examples/topics/partitioning/partition_pruning/description.md @@ -1 +1,2 @@ -Skip entire partitions without reading their data: when a filter() predicate touches only partition columns, the planner prunes automatically - partition metadata is evaluated first and only matching partitions are read - dramatically improving performance for large datasets. +A `filter()` that reads only partition columns is pushed into the reader: directories whose path +fails the predicate (`color=green`) are never opened. diff --git a/web/landing/content/examples/topics/partitioning/partitioning/append/description.md b/web/landing/content/examples/topics/partitioning/partitioning/append/description.md index e6ede43264..d82fa4b773 100644 --- a/web/landing/content/examples/topics/partitioning/partitioning/append/description.md +++ b/web/landing/content/examples/topics/partitioning/partitioning/append/description.md @@ -1 +1,2 @@ -Write partitioned data with append mode. New files with randomized suffixes are created in partition directories without removing existing data. Useful for incremental updates. +`append()` adds a file to every partition that already has one, under a random suffix +(`products_.csv`), and never removes existing data. For incremental loads. diff --git a/web/landing/content/examples/topics/partitioning/partitioning/description.md b/web/landing/content/examples/topics/partitioning/partitioning/description.md index 0bc21dbbc2..c219d6588c 100644 --- a/web/landing/content/examples/topics/partitioning/partitioning/description.md +++ b/web/landing/content/examples/topics/partitioning/partitioning/description.md @@ -1,24 +1,15 @@ -Divide large datasets into smaller, organized parts based on column values. Flow uses Hive partitioning convention, creating a directory structure where each folder represents a partition value. +`partitionBy(partition_by(...))` on a file loader writes one directory per value, in the Hive +`column=value` layout. The partition columns move into the path: -```bash -output -├── color=blue -│ ├── sku=PRODUCT01 -│ │ └── products.csv -│ └── sku=PRODUCT02 -│ └── products.csv -├── color=green -│ ├── sku=PRODUCT01 -│ │ └── products.csv -│ ├── sku=PRODUCT02 -│ │ └── products.csv -│ └── sku=PRODUCT03 -│ └── products.csv -└── color=red - ├── sku=PRODUCT01 - │ └── products.csv - ├── sku=PRODUCT02 - │ └── products.csv - └── sku=PRODUCT03 - └── products.csv ``` +output/color=blue/sku=PRODUCT01/products.csv +output/color=blue/sku=PRODUCT02/products.csv +output/color=green/sku=PRODUCT01/products.csv +output/color=green/sku=PRODUCT02/products.csv +output/color=green/sku=PRODUCT03/products.csv +output/color=red/sku=PRODUCT01/products.csv +output/color=red/sku=PRODUCT02/products.csv +output/color=red/sku=PRODUCT03/products.csv +``` + +The save mode decides what happens when a partition file already exists. diff --git a/web/landing/content/examples/topics/partitioning/partitioning/exception_if_exists/description.md b/web/landing/content/examples/topics/partitioning/partitioning/exception_if_exists/description.md index 4ab8a3fc59..03c1832bb5 100644 --- a/web/landing/content/examples/topics/partitioning/partitioning/exception_if_exists/description.md +++ b/web/landing/content/examples/topics/partitioning/partitioning/exception_if_exists/description.md @@ -1 +1,2 @@ -Write partitioned data with exception_if_exists mode (default). Throws RuntimeException if any partition path already exists. Prevents accidental data overwrites. +`exception_if_exists()` is the default save mode: when a partition file already exists, the write +throws `RuntimeException` instead of touching it. diff --git a/web/landing/content/examples/topics/partitioning/partitioning/ignore/description.md b/web/landing/content/examples/topics/partitioning/partitioning/ignore/description.md index a814446cac..8f1a29c4dc 100644 --- a/web/landing/content/examples/topics/partitioning/partitioning/ignore/description.md +++ b/web/landing/content/examples/topics/partitioning/partitioning/ignore/description.md @@ -1 +1,2 @@ -Write partitioned data with ignore mode. If a partition path already exists, writing is silently skipped. Useful for idempotent pipelines where re-running should not duplicate data. +`ignore()` skips the write into any partition file that already exists. Re-running the pipeline +never duplicates data. diff --git a/web/landing/content/examples/topics/partitioning/partitioning/overwrite/description.md b/web/landing/content/examples/topics/partitioning/partitioning/overwrite/description.md index 86748aa2b2..17d6c3f9e0 100644 --- a/web/landing/content/examples/topics/partitioning/partitioning/overwrite/description.md +++ b/web/landing/content/examples/topics/partitioning/partitioning/overwrite/description.md @@ -1 +1,2 @@ -Write partitioned data with overwrite mode. When writing, ALL files within each affected partition directory are removed and replaced. Partitions NOT in the current dataset remain untouched. +`overwrite()` replaces every file inside each partition the frame writes to. Partitions the frame +does not touch stay as they are. diff --git a/web/landing/content/examples/topics/partitioning/path_partitions/description.md b/web/landing/content/examples/topics/partitioning/path_partitions/description.md index 06fbf1de10..f45d9c4814 100644 --- a/web/landing/content/examples/topics/partitioning/path_partitions/description.md +++ b/web/landing/content/examples/topics/partitioning/path_partitions/description.md @@ -1 +1,2 @@ -Extract partition metadata from file paths without reading file contents using from_path_partitions(). Returns the file path and a map of partition key-value pairs. Useful for discovering available partitions or building file manifests. +`from_path_partitions()` lists files without reading them: one row per file with its `path`, the +`partitions` map, and one column per partition. Use it to discover partitions or build a manifest. diff --git a/web/landing/content/examples/topics/partitioning/reading/after_append/description.md b/web/landing/content/examples/topics/partitioning/reading/after_append/description.md index 999119e5b6..9c98c4c4e1 100644 --- a/web/landing/content/examples/topics/partitioning/reading/after_append/description.md +++ b/web/landing/content/examples/topics/partitioning/reading/after_append/description.md @@ -1 +1,2 @@ -Read partitioned data after multiple writes with append mode. Flow automatically combines all files matching `filename*.ext` pattern in each partition. +Two `append()` runs leave two files in each partition (`products.csv` and +`products_.csv`). The glob `color=*/*.csv` reads both, and `color` comes back from the path. diff --git a/web/landing/content/examples/topics/partitioning/reading/after_overwrite/description.md b/web/landing/content/examples/topics/partitioning/reading/after_overwrite/description.md index 09e5c0222e..54a9b3afcb 100644 --- a/web/landing/content/examples/topics/partitioning/reading/after_overwrite/description.md +++ b/web/landing/content/examples/topics/partitioning/reading/after_overwrite/description.md @@ -1 +1,2 @@ -Read partitioned data after writing with overwrite mode. Each partition contains exactly one file with the latest data. +After an `overwrite()` run each partition holds exactly one file, with the data of the last run. +`color` comes back from the path. diff --git a/web/landing/content/examples/topics/partitioning/reading/basic/description.md b/web/landing/content/examples/topics/partitioning/reading/basic/description.md index eab6ec8477..0b15eb85b8 100644 --- a/web/landing/content/examples/topics/partitioning/reading/basic/description.md +++ b/web/landing/content/examples/topics/partitioning/reading/basic/description.md @@ -1 +1,2 @@ -Read data from Hive-style partitioned directories using glob patterns. Flow automatically discovers all partitions matching `column=*/` patterns. +A glob over Hive-style directories (`color=*/sku=*/*.csv`) reads every partition and adds `color` +and `sku` as columns, taken from the path. diff --git a/web/landing/content/examples/topics/reading/array/description.md b/web/landing/content/examples/topics/reading/array/description.md index 3480f4e8f0..31d9262ec4 100644 --- a/web/landing/content/examples/topics/reading/array/description.md +++ b/web/landing/content/examples/topics/reading/array/description.md @@ -1 +1,2 @@ -Read data directly from a PHP array. This is useful when your data is already in memory, such as API responses or test fixtures. +`from_array()` reads rows that are already in memory - API responses, fixtures. Without a schema, +the column types are inferred from the values. diff --git a/web/landing/content/examples/topics/reading/csv/description.md b/web/landing/content/examples/topics/reading/csv/description.md index d4edcd10ed..754a085e93 100644 --- a/web/landing/content/examples/topics/reading/csv/description.md +++ b/web/landing/content/examples/topics/reading/csv/description.md @@ -1 +1,2 @@ -Extract data from CSV files. Supports automatic delimiter detection, header row handling, and schema inference. +`from_csv()` with its options spelled out. Left out, the separator, enclosure and escape are +detected from the file, and with no schema given, the column types are inferred from a sample. diff --git a/web/landing/content/examples/topics/reading/csv_cells/_meta.yaml b/web/landing/content/examples/topics/reading/csv_cells/_meta.yaml new file mode 100644 index 0000000000..3c24b8b7fa --- /dev/null +++ b/web/landing/content/examples/topics/reading/csv_cells/_meta.yaml @@ -0,0 +1,2 @@ +priority: 9 +hidden: false diff --git a/web/landing/content/examples/topics/reading/csv_cells/code.php b/web/landing/content/examples/topics/reading/csv_cells/code.php new file mode 100644 index 0000000000..718fe9d18b --- /dev/null +++ b/web/landing/content/examples/topics/reading/csv_cells/code.php @@ -0,0 +1,16 @@ +read(from_csv(__DIR__ . '/input.csv', empty_to_null: false))->fetch()->toArray() as $row) { + echo json_encode($row) . "\n"; +} diff --git a/web/landing/content/examples/topics/reading/csv_cells/description.md b/web/landing/content/examples/topics/reading/csv_cells/description.md new file mode 100644 index 0000000000..07037ecc72 --- /dev/null +++ b/web/landing/content/examples/topics/reading/csv_cells/description.md @@ -0,0 +1,2 @@ +Header names are trimmed, and a blank header cell is named `e` + its position (`e01`). A cell missing +from a short line is `null`; `empty_to_null` only decides what a present empty cell becomes. diff --git a/web/landing/content/examples/topics/reading/csv_cells/documentation.md b/web/landing/content/examples/topics/reading/csv_cells/documentation.md new file mode 100644 index 0000000000..d37be23355 --- /dev/null +++ b/web/landing/content/examples/topics/reading/csv_cells/documentation.md @@ -0,0 +1,4 @@ +Readers are extractors: they yield `Rows` and describe the schema they produce. + +- [Data Retrieval](/documentation/components/core/data-retrieval) +- [CSV Adapter](/documentation/components/adapters/csv) diff --git a/web/landing/content/examples/topics/reading/dbal/key_set/_meta.yaml b/web/landing/content/examples/topics/reading/dbal/key_set/_meta.yaml index fdb92cc742..89ae9e74cd 100644 --- a/web/landing/content/examples/topics/reading/dbal/key_set/_meta.yaml +++ b/web/landing/content/examples/topics/reading/dbal/key_set/_meta.yaml @@ -1,4 +1,4 @@ priority: 2 hidden: false run: - skip: "needs a database connection" + skip: "uses pdo_sqlite, which the playground PHP build does not include" diff --git a/web/landing/content/examples/topics/reading/dbal/limit_offset/_meta.yaml b/web/landing/content/examples/topics/reading/dbal/limit_offset/_meta.yaml index 95f468aae3..a450768fac 100644 --- a/web/landing/content/examples/topics/reading/dbal/limit_offset/_meta.yaml +++ b/web/landing/content/examples/topics/reading/dbal/limit_offset/_meta.yaml @@ -1,4 +1,4 @@ priority: 1 hidden: false run: - skip: "needs a database connection" + skip: "uses pdo_sqlite, which the playground PHP build does not include" diff --git a/web/landing/content/examples/topics/reading/dbal/queries/_meta.yaml b/web/landing/content/examples/topics/reading/dbal/queries/_meta.yaml index 78d02b88fe..a1a2b907d1 100644 --- a/web/landing/content/examples/topics/reading/dbal/queries/_meta.yaml +++ b/web/landing/content/examples/topics/reading/dbal/queries/_meta.yaml @@ -1,4 +1,4 @@ priority: 4 hidden: false run: - skip: "needs a database connection" + skip: "uses pdo_sqlite, which the playground PHP build does not include" diff --git a/web/landing/content/examples/topics/reading/dbal/query/_meta.yaml b/web/landing/content/examples/topics/reading/dbal/query/_meta.yaml index 3da2a54d4b..817645f8ac 100644 --- a/web/landing/content/examples/topics/reading/dbal/query/_meta.yaml +++ b/web/landing/content/examples/topics/reading/dbal/query/_meta.yaml @@ -1,4 +1,4 @@ priority: 3 hidden: false run: - skip: "needs a database connection" + skip: "uses pdo_sqlite, which the playground PHP build does not include" diff --git a/web/landing/content/examples/topics/reading/floe/description.md b/web/landing/content/examples/topics/reading/floe/description.md index 41506fbe83..00259b7927 100644 --- a/web/landing/content/examples/topics/reading/floe/description.md +++ b/web/landing/content/examples/topics/reading/floe/description.md @@ -1 +1,2 @@ -Extract data from Floe files. Floe is Flow's native self-describing binary format that stores the schema inside the file and evolves it across appended sections. +`from_floe()` reads Floe, Flow's own binary format. The file carries its schema, so nothing is +inferred and every type, nested ones included, comes back exactly as written. diff --git a/web/landing/content/examples/topics/reading/json/description.md b/web/landing/content/examples/topics/reading/json/description.md index d166b50aa3..c6ba600f51 100644 --- a/web/landing/content/examples/topics/reading/json/description.md +++ b/web/landing/content/examples/topics/reading/json/description.md @@ -1 +1,2 @@ -Extract data from JSON files. Supports reading entire files or targeting specific subtrees using JSON pointers. +`from_json()` reads a document holding an array of objects, one row per object. A JSON pointer +reads a nested array instead of the top level. diff --git a/web/landing/content/examples/topics/reading/jsonl/description.md b/web/landing/content/examples/topics/reading/jsonl/description.md index 7092190b5b..5ea4bcb15f 100644 --- a/web/landing/content/examples/topics/reading/jsonl/description.md +++ b/web/landing/content/examples/topics/reading/jsonl/description.md @@ -1 +1,2 @@ -Extract data from [JSON Lines](https://jsonlines.org/) formatted files, where each line contains a separate JSON object. This format is ideal for streaming large datasets. +`from_json_lines()` reads [JSON Lines](https://jsonlines.org/): one JSON object per line, read line +by line, so the file never has to fit in memory. diff --git a/web/landing/content/examples/topics/reading/parquet/description.md b/web/landing/content/examples/topics/reading/parquet/description.md index 39b33c0469..fd029bfdfd 100644 --- a/web/landing/content/examples/topics/reading/parquet/description.md +++ b/web/landing/content/examples/topics/reading/parquet/description.md @@ -1 +1,2 @@ -Extract data from Parquet files. Parquet is a columnar storage format optimized for analytical workloads, offering efficient compression and fast read performance for large datasets. +`from_parquet()` reads a columnar Parquet file. The schema comes from the file itself; `columns:` +limits which columns are decoded at all. diff --git a/web/landing/content/examples/topics/reading/sequence_date/description.md b/web/landing/content/examples/topics/reading/sequence_date/description.md index 22e9de2879..edde608fa5 100644 --- a/web/landing/content/examples/topics/reading/sequence_date/description.md +++ b/web/landing/content/examples/topics/reading/sequence_date/description.md @@ -1,2 +1,2 @@ -`from_sequence_date_period()` generates one row per interval between two dates, so a gap in the data -still produces a row. +`from_sequence_date_period()` generates one row per interval from the start date up to, but not +including, the end date - here 60 days. A gap in the data still produces a row. diff --git a/web/landing/content/examples/topics/reading/sequence_date_recurrences/description.md b/web/landing/content/examples/topics/reading/sequence_date_recurrences/description.md index c3c0716c87..2e7b3efc42 100644 --- a/web/landing/content/examples/topics/reading/sequence_date_recurrences/description.md +++ b/web/landing/content/examples/topics/reading/sequence_date_recurrences/description.md @@ -1 +1,2 @@ -The same date sequence bounded by a number of recurrences instead of an end date. +The same date sequence bounded by a number of recurrences instead of an end date: the start date +plus 60 recurrences is 61 rows. diff --git a/web/landing/content/examples/topics/reading/xml/code.php b/web/landing/content/examples/topics/reading/xml/code.php index 9635ea30a4..fa570735b7 100644 --- a/web/landing/content/examples/topics/reading/xml/code.php +++ b/web/landing/content/examples/topics/reading/xml/code.php @@ -13,12 +13,9 @@ ->withXMLNodePath('rows/row') ) ->withEntry('id', ref('node')->xpath('order_id')->domElementValue()) - ->withEntry('seller_id', ref('node')->xpath('seller_id')->domElementValue()) ->withEntry('created_at', ref('node')->xpath('created_at')->domElementValue()) - ->withEntry('cancelled_at', ref('node')->xpath('cancelled_at')->domElementValue()) ->withEntry('discount', ref('node')->xpath('discount')->domElementValue()) ->drop('node') - ->select('id', 'created_at', 'discount') ->limit(3) ->collect() ->write(to_output(truncate: false)) diff --git a/web/landing/content/examples/topics/schema/apply/code.php b/web/landing/content/examples/topics/schema/apply/code.php index e4063d3214..0f3bd38c49 100644 --- a/web/landing/content/examples/topics/schema/apply/code.php +++ b/web/landing/content/examples/topics/schema/apply/code.php @@ -9,9 +9,9 @@ require __DIR__ . '/vendor/autoload.php'; $schema = schema( - int_schema('id', $nullable = false), - str_schema('name', $nullable = true), - bool_schema('active', $nullable = false, Metadata::empty()->add('key', 'value')), + int_schema('id', nullable: false), + str_schema('name', nullable: true), + bool_schema('active', nullable: false, metadata: Metadata::empty()->add('key', 'value')), ); data_frame() diff --git a/web/landing/content/examples/topics/schema/apply/description.md b/web/landing/content/examples/topics/schema/apply/description.md index 3be56e25a4..f58c02c7b0 100644 --- a/web/landing/content/examples/topics/schema/apply/description.md +++ b/web/landing/content/examples/topics/schema/apply/description.md @@ -1 +1,2 @@ -Define explicit column types for data sources that lack strict schema support (like CSV, XML, or JSON). This ensures consistent type handling and avoids issues when early rows contain null or empty values. +`withSchema()` on a reader declares the column types instead of inferring them. Formats that carry +no types (CSV, JSON, XML, arrays) then never guess from a sample, and NOT NULL columns stay NOT NULL. diff --git a/web/landing/content/examples/topics/schema/display/description.md b/web/landing/content/examples/topics/schema/display/description.md index ee5b2c90d1..5abc1fec6a 100644 --- a/web/landing/content/examples/topics/schema/display/description.md +++ b/web/landing/content/examples/topics/schema/display/description.md @@ -1,2 +1,3 @@ -`printSchema()` renders the schema as a tree, nesting structure fields under their parent and -marking nullable columns with `?`. +`to_output(output: Output::schema)` prints the schema instead of the rows: a tree with structure +fields nested under their parent. Here the schema is inferred, so a key some rows lack becomes a +nullable column (`?`). diff --git a/web/landing/content/examples/topics/schema/inferring/description.md b/web/landing/content/examples/topics/schema/inferring/description.md index 06959a3af7..73c3266d8f 100644 --- a/web/landing/content/examples/topics/schema/inferring/description.md +++ b/web/landing/content/examples/topics/schema/inferring/description.md @@ -1 +1,2 @@ -Automatically detect the schema from your data. Inferred schemas can be saved and reused to speed up subsequent processing by avoiding repeated schema detection. +`schema()` infers the schema from a sample of the file. Saved with `schema_to_json()` and loaded with +`schema_from_json()`, it is passed to the next read, which then skips inference. diff --git a/web/landing/content/examples/topics/schema/strict_or_evolving/code.php b/web/landing/content/examples/topics/schema/strict_or_evolving/code.php index 9092999c77..d341d7c22c 100644 --- a/web/landing/content/examples/topics/schema/strict_or_evolving/code.php +++ b/web/landing/content/examples/topics/schema/strict_or_evolving/code.php @@ -28,5 +28,5 @@ ->write(to_output(truncate: false)) ->run(); } catch (Throwable $e) { - echo "\nstrict rejects it: " . explode("\n", $e->getMessage())[0] . "\n"; + echo "\nstrict rejects it: " . $e->getMessage() . "\n"; } diff --git a/web/landing/content/examples/topics/schema/validate/code.php b/web/landing/content/examples/topics/schema/validate/code.php index 90879d7631..27200d9e5e 100644 --- a/web/landing/content/examples/topics/schema/validate/code.php +++ b/web/landing/content/examples/topics/schema/validate/code.php @@ -9,9 +9,9 @@ require __DIR__ . '/vendor/autoload.php'; $schema = schema( - int_schema('id', $nullable = false), - str_schema('name', $nullable = true), - bool_schema('active', $nullable = false, Metadata::empty()->add('key', 'value')), + int_schema('id', nullable: false), + str_schema('name', nullable: true), + bool_schema('active', nullable: false, metadata: Metadata::empty()->add('key', 'value')), ); data_frame() diff --git a/web/landing/content/examples/topics/transformations/call/description.md b/web/landing/content/examples/topics/transformations/call/description.md index f7655bc1ac..beae1c3c88 100644 --- a/web/landing/content/examples/topics/transformations/call/description.md +++ b/web/landing/content/examples/topics/transformations/call/description.md @@ -1 +1,2 @@ -Apply any PHP callable (function or method) to your data. This is useful when no built-in scalar function exists for your specific transformation needs. +`call()` applies any PHP callable to a column when no built-in function fits. The callable's result +type is declared (`type_list(type_integer())`), so the schema knows it before a row is read. diff --git a/web/landing/content/examples/topics/transformations/custom/description.md b/web/landing/content/examples/topics/transformations/custom/description.md index 9ba7c95e80..c1cfabacb0 100644 --- a/web/landing/content/examples/topics/transformations/custom/description.md +++ b/web/landing/content/examples/topics/transformations/custom/description.md @@ -1 +1,2 @@ -Create reusable transformation blocks that can be applied across multiple pipelines. Custom transformations help organize complex logic and improve code reuse. +A `Transformation` packs several steps into one reusable class. `with()` hands it the frame, and +whatever it returns continues the pipeline. diff --git a/web/landing/content/examples/topics/transformations/filtering/basic/description.md b/web/landing/content/examples/topics/transformations/filtering/basic/description.md index d6f8e1b386..9844299934 100644 --- a/web/landing/content/examples/topics/transformations/filtering/basic/description.md +++ b/web/landing/content/examples/topics/transformations/filtering/basic/description.md @@ -1 +1,2 @@ -Remove rows from your dataset based on conditions. Only rows matching the filter criteria are kept in the result. This is one of the most common operations for selecting a subset of data. +`filter()` keeps the rows where the condition is true and drops the rest, including rows where it +is `null`. diff --git a/web/landing/content/examples/topics/transformations/size/code.php b/web/landing/content/examples/topics/transformations/size/code.php index c78af52485..642acebe68 100644 --- a/web/landing/content/examples/topics/transformations/size/code.php +++ b/web/landing/content/examples/topics/transformations/size/code.php @@ -2,14 +2,15 @@ declare(strict_types=1); -use function Flow\ETL\DSL\{data_frame, from_array, int_schema, json_schema, ref, schema, to_output}; +use function Flow\ETL\DSL\{data_frame, from_array, int_schema, map_schema, ref, schema, to_output}; +use function Flow\Types\DSL\{type_integer, type_map, type_string}; require __DIR__ . '/vendor/autoload.php'; data_frame() ->read(from_array( [['id' => 1, 'array' => ['a' => 1, 'b' => 2, 'c' => 3]]], - schema(int_schema('id'), json_schema('array')), + schema(int_schema('id'), map_schema('array', type_map(type_string(), type_integer()))), )) ->withEntry('array_size', ref('array')->size()) ->write(to_output(truncate: false)) diff --git a/web/landing/content/examples/topics/transformations/size/description.md b/web/landing/content/examples/topics/transformations/size/description.md index bee9da3f5d..a9866de322 100644 --- a/web/landing/content/examples/topics/transformations/size/description.md +++ b/web/landing/content/examples/topics/transformations/size/description.md @@ -1,2 +1,2 @@ -`size()` counts the elements of an array column without unpacking it, so the array itself stays in -the row. +`size()` counts the elements of a list or map column (the characters of a string) without unpacking +it, so the array itself stays in the row. diff --git a/web/landing/content/examples/topics/transformations/when/basic/description.md b/web/landing/content/examples/topics/transformations/when/basic/description.md index aa186e8cd6..6c388fc3a7 100644 --- a/web/landing/content/examples/topics/transformations/when/basic/description.md +++ b/web/landing/content/examples/topics/transformations/when/basic/description.md @@ -1 +1,2 @@ -Add conditional logic to your transformations. Evaluate a condition and return different values based on whether it's true or false-similar to an if/else statement. +`when(condition, then, else)` is an if/else per row: here `true` for active users tagged `foo`, +`false` for everyone else. diff --git a/web/landing/content/examples/topics/transformations/when/else_when/description.md b/web/landing/content/examples/topics/transformations/when/else_when/description.md index cb1196ee82..6fe993ccc1 100644 --- a/web/landing/content/examples/topics/transformations/when/else_when/description.md +++ b/web/landing/content/examples/topics/transformations/when/else_when/description.md @@ -1 +1,2 @@ -Chain multiple conditions for multi-tier logic, similar to if-elseif-else chains. Useful for categorizing data into multiple levels, implementing grading systems, or determining status based on several criteria. +Nested `when()` calls make an if/elseif/else chain, checked top to bottom - the first match decides +the grade. diff --git a/web/landing/content/examples/topics/transformations/when/null/code.php b/web/landing/content/examples/topics/transformations/when/null/code.php index f0e8baf5f6..1ebe08992b 100644 --- a/web/landing/content/examples/topics/transformations/when/null/code.php +++ b/web/landing/content/examples/topics/transformations/when/null/code.php @@ -19,7 +19,7 @@ )) ->withEntry( 'value', - when(ref('value')->isNull(), then: lit(0)) + when(ref('value')->isNull(), then: lit(0), else: ref('value')) ) ->write(to_output(truncate: false)) ->run(); diff --git a/web/landing/content/examples/topics/transformations/when/null/description.md b/web/landing/content/examples/topics/transformations/when/null/description.md index ee9c68aab0..c955e13d8e 100644 --- a/web/landing/content/examples/topics/transformations/when/null/description.md +++ b/web/landing/content/examples/topics/transformations/when/null/description.md @@ -1,2 +1,2 @@ -`when()` with only a `then` branch: rows that do not match keep the value they had. The nullable -column is declared on the source, so the intent is explicit rather than inferred. +`when()` replaces nulls with `0`. Without `else`, a row that does not match gets `null`, so +`else: ref('value')` keeps the value it had. diff --git a/web/landing/content/examples/topics/types/is_valid/code.php b/web/landing/content/examples/topics/types/is_valid/code.php index 2274a90f83..8040e01c7f 100644 --- a/web/landing/content/examples/topics/types/is_valid/code.php +++ b/web/landing/content/examples/topics/types/is_valid/code.php @@ -11,6 +11,8 @@ echo var_export($value, true) . ' is integer: ' . (type_integer()->isValid($value) ? 'yes' : 'no') . "\n"; } -foreach (['{"name":"John"}', 'hello', ['name' => 'John']] as $value) { - echo var_export($value, true) . ' is json: ' . (type_json()->isValid($value) ? 'yes' : 'no') . "\n"; -} +// json is a value object: JSON text is only a string until cast() turns it into one +$text = '{"name":"John"}'; + +echo var_export($text, true) . ' is json: ' . (type_json()->isValid($text) ? 'yes' : 'no') . "\n"; +echo 'type_json()->cast(' . var_export($text, true) . ') is json: ' . (type_json()->isValid(type_json()->cast($text)) ? 'yes' : 'no') . "\n"; diff --git a/web/landing/content/examples/topics/types/is_valid/description.md b/web/landing/content/examples/topics/types/is_valid/description.md index d41aecbb39..10b2f98650 100644 --- a/web/landing/content/examples/topics/types/is_valid/description.md +++ b/web/landing/content/examples/topics/types/is_valid/description.md @@ -1 +1,3 @@ `isValid()` asks the same question as `assert()` without throwing, so it can be used in a condition. +It checks the value as it is: `'42'` is not an integer, and JSON text is not `json` until `cast()` +turns it into a `Json` value. diff --git a/web/landing/content/examples/topics/window_functions/dense_rank/code.php b/web/landing/content/examples/topics/window_functions/dense_rank/code.php index 6a64a600b7..3aa2222e05 100644 --- a/web/landing/content/examples/topics/window_functions/dense_rank/code.php +++ b/web/landing/content/examples/topics/window_functions/dense_rank/code.php @@ -6,7 +6,7 @@ require __DIR__ . '/vendor/autoload.php'; -$df = data_frame() +data_frame() ->read( from_array([ ['id' => 1, 'name' => 'Greg', 'department' => 'IT', 'salary' => 6000], @@ -15,6 +15,7 @@ ['id' => 4, 'name' => 'John', 'department' => 'Finances', 'salary' => 9000], ['id' => 5, 'name' => 'Jane', 'department' => 'Finances', 'salary' => 14_000], ['id' => 6, 'name' => 'Janet', 'department' => 'Finances', 'salary' => 9000], + ['id' => 7, 'name' => 'Ann', 'department' => 'Finances', 'salary' => 8000], ]) ) ->withEntry('dense_rank', dense_rank()->over(window()->partitionBy(ref('department'))->orderBy(ref('salary')->desc()))) diff --git a/web/landing/content/examples/topics/window_functions/dense_rank/description.md b/web/landing/content/examples/topics/window_functions/dense_rank/description.md index 14a0d0e670..9864da5890 100644 --- a/web/landing/content/examples/topics/window_functions/dense_rank/description.md +++ b/web/landing/content/examples/topics/window_functions/dense_rank/description.md @@ -1,3 +1,2 @@ -The `dense_rank()` function assigns a rank to each row within a partition, without gaps in ranking when there are ties. If two rows have the same value, they get the same rank, but the next rank is consecutive (no gaps). - -For example, if two employees have the same salary and both get rank 2, the next employee gets rank 3 (not 4 like with regular `rank()`). +`dense_rank()` gives tied rows the same rank without leaving a gap: John and Janet are both 3, and +Ann is 4, where `rank()` would give 5. diff --git a/web/landing/content/examples/topics/window_functions/description.md b/web/landing/content/examples/topics/window_functions/description.md index cea93b3522..5672caf93a 100644 --- a/web/landing/content/examples/topics/window_functions/description.md +++ b/web/landing/content/examples/topics/window_functions/description.md @@ -1,3 +1,3 @@ -Window functions perform calculations across a set of rows that are related to the current row. Unlike regular aggregations, window functions don't collapse rows - each row retains its identity while gaining access to aggregate information about its "window" of related rows. - -Use `window()` to define partitions and ordering, then apply window functions like `rank()`, `dense_rank()`, or `row_number()` to compute values within each partition. +A window function computes over related rows without collapsing them: every row stays and gains a +value. `window()` defines the partitions and their order, and `over()` applies `rank()`, +`dense_rank()`, `row_number()` or an aggregate to it. diff --git a/web/landing/content/examples/topics/window_functions/rank/code.php b/web/landing/content/examples/topics/window_functions/rank/code.php index eb0f30f14b..1d75277729 100644 --- a/web/landing/content/examples/topics/window_functions/rank/code.php +++ b/web/landing/content/examples/topics/window_functions/rank/code.php @@ -6,7 +6,7 @@ require __DIR__ . '/vendor/autoload.php'; -$df = data_frame() +data_frame() ->read( from_array([ ['id' => 1, 'name' => 'Greg', 'department' => 'IT', 'salary' => 6000], @@ -15,6 +15,7 @@ ['id' => 4, 'name' => 'John', 'department' => 'Finances', 'salary' => 9000], ['id' => 5, 'name' => 'Jane', 'department' => 'Finances', 'salary' => 14_000], ['id' => 6, 'name' => 'Janet', 'department' => 'Finances', 'salary' => 9000], + ['id' => 7, 'name' => 'Ann', 'department' => 'Finances', 'salary' => 8000], ]) ) ->withEntry('rank', rank()->over(window()->partitionBy(ref('department'))->orderBy(ref('salary')->desc()))) diff --git a/web/landing/content/examples/topics/window_functions/rank/description.md b/web/landing/content/examples/topics/window_functions/rank/description.md index bb24ee5aac..6b982e0246 100644 --- a/web/landing/content/examples/topics/window_functions/rank/description.md +++ b/web/landing/content/examples/topics/window_functions/rank/description.md @@ -1,3 +1,2 @@ -The `rank()` function assigns a rank to each row within a partition, with gaps in ranking when there are ties. If two rows have the same value, they get the same rank, and the next rank is skipped. - -For example, if two employees have the same salary and both get rank 2, the next employee gets rank 4 (rank 3 is skipped). +`rank()` numbers the rows of each partition in window order. Tied rows share a rank and the next +rank skips ahead: John and Janet are both 3, so Ann is 5. diff --git a/web/landing/content/examples/topics/window_functions/row_number/code.php b/web/landing/content/examples/topics/window_functions/row_number/code.php index 5cea34ef77..55b3b67d30 100644 --- a/web/landing/content/examples/topics/window_functions/row_number/code.php +++ b/web/landing/content/examples/topics/window_functions/row_number/code.php @@ -6,7 +6,7 @@ require __DIR__ . '/vendor/autoload.php'; -$df = data_frame() +data_frame() ->read( from_array([ ['id' => 1, 'name' => 'Greg', 'department' => 'IT', 'salary' => 6000], @@ -15,6 +15,7 @@ ['id' => 4, 'name' => 'John', 'department' => 'Finances', 'salary' => 9000], ['id' => 5, 'name' => 'Jane', 'department' => 'Finances', 'salary' => 14_000], ['id' => 6, 'name' => 'Janet', 'department' => 'Finances', 'salary' => 9000], + ['id' => 7, 'name' => 'Ann', 'department' => 'Finances', 'salary' => 8000], ]) ) ->withEntry('row_num', row_number()->over(window()->partitionBy(ref('department'))->orderBy(ref('salary')->desc()))) diff --git a/web/landing/content/examples/topics/window_functions/row_number/description.md b/web/landing/content/examples/topics/window_functions/row_number/description.md index 6957c56e3c..4b10ce9673 100644 --- a/web/landing/content/examples/topics/window_functions/row_number/description.md +++ b/web/landing/content/examples/topics/window_functions/row_number/description.md @@ -1,3 +1,2 @@ -The `row_number()` function assigns a unique sequential number to each row within a partition, starting at 1. Unlike `rank()` and `dense_rank()`, `row_number()` always assigns different numbers to each row, even when values are tied. - -This is useful for pagination, selecting top N rows per group, or creating unique identifiers within partitions. +`row_number()` numbers the rows of each partition 1, 2, 3, ... with no ties: John and Janet earn the +same but get 3 and 4. Use it for top-N per group or for a per-partition id. diff --git a/web/landing/content/examples/topics/writing/array/code.php b/web/landing/content/examples/topics/writing/array/code.php index 41e9f86c4a..2117533773 100644 --- a/web/landing/content/examples/topics/writing/array/code.php +++ b/web/landing/content/examples/topics/writing/array/code.php @@ -20,4 +20,4 @@ ->write(to_array($array)) ->run(); -\var_dump($array, true); +\var_dump($array); diff --git a/web/landing/content/examples/topics/writing/array/description.md b/web/landing/content/examples/topics/writing/array/description.md index 5c2f56cc99..3d3f73de4a 100644 --- a/web/landing/content/examples/topics/writing/array/description.md +++ b/web/landing/content/examples/topics/writing/array/description.md @@ -1,3 +1,2 @@ -Write data directly to a PHP array. Useful for testing, debugging, or when you need results in memory for further processing. - -**Warning:** Large datasets may cause memory overflow since all data is held in memory. +`to_array()` writes the rows into a PHP array passed by reference - for tests, debugging, or handing +results to code outside the pipeline. Every row is held in memory. diff --git a/web/landing/content/examples/topics/writing/dbal/delete/_meta.yaml b/web/landing/content/examples/topics/writing/dbal/delete/_meta.yaml index 3da2a54d4b..817645f8ac 100644 --- a/web/landing/content/examples/topics/writing/dbal/delete/_meta.yaml +++ b/web/landing/content/examples/topics/writing/dbal/delete/_meta.yaml @@ -1,4 +1,4 @@ priority: 3 hidden: false run: - skip: "needs a database connection" + skip: "uses pdo_sqlite, which the playground PHP build does not include" diff --git a/web/landing/content/examples/topics/writing/dbal/delete/description.md b/web/landing/content/examples/topics/writing/dbal/delete/description.md index 3b2d9b03ee..1bb059147e 100644 --- a/web/landing/content/examples/topics/writing/dbal/delete/description.md +++ b/web/landing/content/examples/topics/writing/dbal/delete/description.md @@ -1 +1,2 @@ -Delete rows from a database table that match records in your DataFrame. This is useful for synchronizing data or removing outdated entries. +`to_dbal_table_delete()` deletes the table rows that match a frame row on all of its columns +(`WHERE (order_id) IN (...)` here). The frame holds every order id, so nothing is left. diff --git a/web/landing/content/examples/topics/writing/dbal/insert/_meta.yaml b/web/landing/content/examples/topics/writing/dbal/insert/_meta.yaml index 95f468aae3..a450768fac 100644 --- a/web/landing/content/examples/topics/writing/dbal/insert/_meta.yaml +++ b/web/landing/content/examples/topics/writing/dbal/insert/_meta.yaml @@ -1,4 +1,4 @@ priority: 1 hidden: false run: - skip: "needs a database connection" + skip: "uses pdo_sqlite, which the playground PHP build does not include" diff --git a/web/landing/content/examples/topics/writing/dbal/insert/description.md b/web/landing/content/examples/topics/writing/dbal/insert/description.md index 5bc421e1d5..3745c305f2 100644 --- a/web/landing/content/examples/topics/writing/dbal/insert/description.md +++ b/web/landing/content/examples/topics/writing/dbal/insert/description.md @@ -1,2 +1,2 @@ -Writing to a table with `to_dbal_table_insert()`, then reading it straight back - in the playground -a written file is invisible, so the write has to show its result. +`to_dbal_table_insert()` inserts each batch into a table through Doctrine DBAL, then the table is read +back to show the result. diff --git a/web/landing/content/examples/topics/writing/dbal/upsert/_meta.yaml b/web/landing/content/examples/topics/writing/dbal/upsert/_meta.yaml index fdb92cc742..89ae9e74cd 100644 --- a/web/landing/content/examples/topics/writing/dbal/upsert/_meta.yaml +++ b/web/landing/content/examples/topics/writing/dbal/upsert/_meta.yaml @@ -1,4 +1,4 @@ priority: 2 hidden: false run: - skip: "needs a database connection" + skip: "uses pdo_sqlite, which the playground PHP build does not include" diff --git a/web/landing/content/examples/topics/writing/dbal/upsert/description.md b/web/landing/content/examples/topics/writing/dbal/upsert/description.md index 68580ed698..9b706883c3 100644 --- a/web/landing/content/examples/topics/writing/dbal/upsert/description.md +++ b/web/landing/content/examples/topics/writing/dbal/upsert/description.md @@ -1 +1,3 @@ -Insert new records or update existing ones in a database. Supports MySQL, PostgreSQL, and SQLite with platform-specific conflict resolution strategies. +`sqlite_insert_options(conflict_columns:)` turns the insert into an upsert, so writing the same ten +orders twice leaves ten rows instead of failing. MySQL and PostgreSQL have their own +`*_insert_options()`. diff --git a/web/landing/content/examples/topics/writing/excel/description.md b/web/landing/content/examples/topics/writing/excel/description.md index f13341c1fb..e23a623b58 100644 --- a/web/landing/content/examples/topics/writing/excel/description.md +++ b/web/landing/content/examples/topics/writing/excel/description.md @@ -1,2 +1 @@ -Writing a spreadsheet, then reading it back through `from_excel()` - the round trip is the point, -since a written file is invisible in the playground. +`to_excel()` writes a spreadsheet with a header row from the schema; `from_excel()` reads it back. diff --git a/web/landing/content/examples/topics/writing/floe/description.md b/web/landing/content/examples/topics/writing/floe/description.md index 74eba27abe..02f5da58c6 100644 --- a/web/landing/content/examples/topics/writing/floe/description.md +++ b/web/landing/content/examples/topics/writing/floe/description.md @@ -1 +1,2 @@ -Write data to Floe files. Floe is Flow's native self-describing binary format - the schema travels inside the file, so it reads back through the DataFrame API with no external metadata. +`to_floe()` writes Floe, Flow's own binary format. The schema is stored in the file, so +`from_floe()` reads every type back exactly, with nothing inferred. diff --git a/web/landing/content/examples/topics/writing/jsonl/description.md b/web/landing/content/examples/topics/writing/jsonl/description.md index 835a07f978..f27cef462e 100644 --- a/web/landing/content/examples/topics/writing/jsonl/description.md +++ b/web/landing/content/examples/topics/writing/jsonl/description.md @@ -1 +1,2 @@ -Write data to [JSON Lines](https://jsonlines.org/) files, where each row becomes a separate JSON object on its own line. +`to_json_lines()` writes [JSON Lines](https://jsonlines.org/): one JSON object per row, one row per +line. diff --git a/web/landing/content/examples/topics/writing/overwrite/description.md b/web/landing/content/examples/topics/writing/overwrite/description.md index 9d267428f9..2f97166ddc 100644 --- a/web/landing/content/examples/topics/writing/overwrite/description.md +++ b/web/landing/content/examples/topics/writing/overwrite/description.md @@ -1 +1,2 @@ -`overwrite()` replaces what is already at the destination. Running twice leaves one copy, not two. +`overwrite()` replaces what is already at the destination: the second pipeline reads `file.csv` +and writes it back without the `customer` column. diff --git a/web/landing/content/examples/topics/writing/parquet/description.md b/web/landing/content/examples/topics/writing/parquet/description.md index d364e9b9e9..5588814f21 100644 --- a/web/landing/content/examples/topics/writing/parquet/description.md +++ b/web/landing/content/examples/topics/writing/parquet/description.md @@ -1 +1,2 @@ -Write data to Parquet files. Parquet is a columnar format that provides efficient compression and is ideal for storing large analytical datasets. +`to_parquet()` writes a columnar, compressed Parquet file with the frame's schema; `from_parquet()` +reads it back. diff --git a/web/landing/resources/api.json b/web/landing/resources/api.json index 41752dda5e..430255d623 100644 --- a/web/landing/resources/api.json +++ b/web/landing/resources/api.json @@ -1 +1 @@ -[{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Function\/ScalarFunctionChain.php","start_line_in_file":31,"slug":"and","name":"and","class":"Flow\\ETL\\Function\\ScalarFunctionChain","class_slug":"scalarfunctionchain","parameters":[{"name":"function","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"All","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Function\/ScalarFunctionChain.php","start_line_in_file":36,"slug":"andnot","name":"andNot","class":"Flow\\ETL\\Function\\ScalarFunctionChain","class_slug":"scalarfunctionchain","parameters":[{"name":"function","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"All","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Function\/ScalarFunctionChain.php","start_line_in_file":41,"slug":"append","name":"append","class":"Flow\\ETL\\Function\\ScalarFunctionChain","class_slug":"scalarfunctionchain","parameters":[{"name":"suffix","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"Append","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":true,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Function\/ScalarFunctionChain.php","start_line_in_file":54,"slug":"arrayfilter","name":"arrayFilter","class":"Flow\\ETL\\Function\\ScalarFunctionChain","class_slug":"scalarfunctionchain","parameters":[{"name":"value","type":[{"name":"mixed","namespace":null,"is_nullable":true,"is_variadic":false}],"has_default_value":false,"is_nullable":true,"is_variadic":false,"default_value":null}],"return_type":[{"name":"ArrayFilter","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":true,"doc_comment":"LyoqCiAgICAgKiBGaWx0ZXJzIGFuIGFycmF5IGJ5IHJlbW92aW5nIGFsbCBlbGVtZW50cyB0aGF0IG1hdGNoZXMgcGFzc2VkIHZhbHVlLgogICAgICogQXBwbGljYWJsZSB0byBhbGwgZGF0YSBzdHJ1Y3R1cmVzIHRoYXQgY2FuIGJlIGNvbnZlcnRlZCB0byBhbiBhcnJheToKICAgICAqICAgIC0ganNvbgogICAgICogICAgLSBsaXN0CiAgICAgKiAgICAtIG1hcAogICAgICogICAgLSBzdHJ1Y3R1cmUuCiAgICAgKi8="},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Function\/ScalarFunctionChain.php","start_line_in_file":59,"slug":"arrayget","name":"arrayGet","class":"Flow\\ETL\\Function\\ScalarFunctionChain","class_slug":"scalarfunctionchain","parameters":[{"name":"path","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"ArrayGet","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":true,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Function\/ScalarFunctionChain.php","start_line_in_file":67,"slug":"arraygetcollection","name":"arrayGetCollection","class":"Flow\\ETL\\Function\\ScalarFunctionChain","class_slug":"scalarfunctionchain","parameters":[{"name":"keys","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"array","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"ArrayGetCollection","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":true,"doc_comment":"LyoqCiAgICAgKiBAcGFyYW0gU2NhbGFyRnVuY3Rpb258YXJyYXk8YXJyYXkta2V5LCBtaXhlZD4gJGtleXMKICAgICAqLw=="},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Function\/ScalarFunctionChain.php","start_line_in_file":72,"slug":"arraygetcollectionfirst","name":"arrayGetCollectionFirst","class":"Flow\\ETL\\Function\\ScalarFunctionChain","class_slug":"scalarfunctionchain","parameters":[{"name":"keys","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":true,"default_value":null}],"return_type":[{"name":"ArrayGetCollection","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":true,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Function\/ScalarFunctionChain.php","start_line_in_file":85,"slug":"arraykeep","name":"arrayKeep","class":"Flow\\ETL\\Function\\ScalarFunctionChain","class_slug":"scalarfunctionchain","parameters":[{"name":"value","type":[{"name":"mixed","namespace":null,"is_nullable":true,"is_variadic":false}],"has_default_value":false,"is_nullable":true,"is_variadic":false,"default_value":null}],"return_type":[{"name":"ArrayKeep","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":true,"doc_comment":"LyoqCiAgICAgKiBGaWx0ZXJzIGFuIGFycmF5IGJ5IGtlZXBpbmcgb25seSBlbGVtZW50cyB0aGF0IG1hdGNoZXMgcGFzc2VkIHZhbHVlLgogICAgICogQXBwbGljYWJsZSB0byBhbGwgZGF0YSBzdHJ1Y3R1cmVzIHRoYXQgY2FuIGJlIGNvbnZlcnRlZCB0byBhbiBhcnJheToKICAgICAqICAgLSBqc29uCiAgICAgKiAgIC0gbGlzdAogICAgICogICAtIG1hcAogICAgICogICAtIHN0cnVjdHVyZS4KICAgICAqLw=="},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Function\/ScalarFunctionChain.php","start_line_in_file":98,"slug":"arraykeys","name":"arrayKeys","class":"Flow\\ETL\\Function\\ScalarFunctionChain","class_slug":"scalarfunctionchain","parameters":[],"return_type":[{"name":"ArrayKeys","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":true,"doc_comment":"LyoqCiAgICAgKiBSZXR1cm5zIGFsbCBrZXlzIGZyb20gYW4gYXJyYXksIGlnbm9yaW5nIHRoZSB2YWx1ZXMuCiAgICAgKiBBcHBsaWNhYmxlIHRvIGFsbCBkYXRhIHN0cnVjdHVyZXMgdGhhdCBjYW4gYmUgY29udmVydGVkIHRvIGFuIGFycmF5OgogICAgICogICAtIGpzb24KICAgICAqICAgLSBsaXN0CiAgICAgKiAgIC0gbWFwCiAgICAgKiAgIC0gc3RydWN0dXJlLgogICAgICov"},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Function\/ScalarFunctionChain.php","start_line_in_file":106,"slug":"arraymerge","name":"arrayMerge","class":"Flow\\ETL\\Function\\ScalarFunctionChain","class_slug":"scalarfunctionchain","parameters":[{"name":"ref","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"array","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"ArrayMerge","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":true,"doc_comment":"LyoqCiAgICAgKiBAcGFyYW0gU2NhbGFyRnVuY3Rpb258YXJyYXk8YXJyYXkta2V5LCBtaXhlZD4gJHJlZgogICAgICov"},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Function\/ScalarFunctionChain.php","start_line_in_file":111,"slug":"arraymergecollection","name":"arrayMergeCollection","class":"Flow\\ETL\\Function\\ScalarFunctionChain","class_slug":"scalarfunctionchain","parameters":[],"return_type":[{"name":"ArrayMergeCollection","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":true,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Function\/ScalarFunctionChain.php","start_line_in_file":116,"slug":"arraypathexists","name":"arrayPathExists","class":"Flow\\ETL\\Function\\ScalarFunctionChain","class_slug":"scalarfunctionchain","parameters":[{"name":"path","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"ArrayPathExists","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":true,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Function\/ScalarFunctionChain.php","start_line_in_file":121,"slug":"arrayreverse","name":"arrayReverse","class":"Flow\\ETL\\Function\\ScalarFunctionChain","class_slug":"scalarfunctionchain","parameters":[{"name":"preserveKeys","type":[{"name":"bool","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"false"}],"return_type":[{"name":"ArrayReverse","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":true,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Function\/ScalarFunctionChain.php","start_line_in_file":126,"slug":"arraysort","name":"arraySort","class":"Flow\\ETL\\Function\\ScalarFunctionChain","class_slug":"scalarfunctionchain","parameters":[{"name":"sortFunction","type":[{"name":"Sort","namespace":"Flow\\ETL\\Function\\ArraySort","is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"null"},{"name":"flags","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"int","namespace":null,"is_nullable":false,"is_variadic":false},{"name":"null","namespace":null,"is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"null"},{"name":"recursive","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"bool","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"true"}],"return_type":[{"name":"ArraySort","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":true,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Function\/ScalarFunctionChain.php","start_line_in_file":142,"slug":"arrayvalues","name":"arrayValues","class":"Flow\\ETL\\Function\\ScalarFunctionChain","class_slug":"scalarfunctionchain","parameters":[],"return_type":[{"name":"ArrayValues","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":true,"doc_comment":"LyoqCiAgICAgKiBSZXR1cm5zIGFsbCB2YWx1ZXMgZnJvbSBhbiBhcnJheSwgaWdub3JpbmcgdGhlIGtleXMuCiAgICAgKiBBcHBsaWNhYmxlIHRvIGFsbCBkYXRhIHN0cnVjdHVyZXMgdGhhdCBjYW4gYmUgY29udmVydGVkIHRvIGFuIGFycmF5OgogICAgICogICAtIGpzb24KICAgICAqICAgLSBsaXN0CiAgICAgKiAgIC0gbWFwCiAgICAgKiAgIC0gc3RydWN0dXJlLgogICAgICov"},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Function\/ScalarFunctionChain.php","start_line_in_file":147,"slug":"ascii","name":"ascii","class":"Flow\\ETL\\Function\\ScalarFunctionChain","class_slug":"scalarfunctionchain","parameters":[],"return_type":[{"name":"Ascii","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":true,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Function\/ScalarFunctionChain.php","start_line_in_file":157,"slug":"between","name":"between","class":"Flow\\ETL\\Function\\ScalarFunctionChain","class_slug":"scalarfunctionchain","parameters":[{"name":"lowerBoundRef","type":[{"name":"mixed","namespace":null,"is_nullable":true,"is_variadic":false}],"has_default_value":false,"is_nullable":true,"is_variadic":false,"default_value":null},{"name":"upperBoundRef","type":[{"name":"mixed","namespace":null,"is_nullable":true,"is_variadic":false}],"has_default_value":false,"is_nullable":true,"is_variadic":false,"default_value":null},{"name":"boundary","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"Boundary","namespace":"Flow\\ETL\\Function\\Between","is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"Flow\\ETL\\Function\\Between\\Boundary::..."}],"return_type":[{"name":"Between","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":true,"doc_comment":"LyoqCiAgICAgKiBAcGFyYW0gbWl4ZWR8U2NhbGFyRnVuY3Rpb24gJGxvd2VyQm91bmRSZWYKICAgICAqIEBwYXJhbSBtaXhlZHxTY2FsYXJGdW5jdGlvbiAkdXBwZXJCb3VuZFJlZgogICAgICogQHBhcmFtIEJvdW5kYXJ5fFNjYWxhckZ1bmN0aW9uICRib3VuZGFyeQogICAgICov"},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Function\/ScalarFunctionChain.php","start_line_in_file":165,"slug":"binarylength","name":"binaryLength","class":"Flow\\ETL\\Function\\ScalarFunctionChain","class_slug":"scalarfunctionchain","parameters":[],"return_type":[{"name":"BinaryLength","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":true,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Function\/ScalarFunctionChain.php","start_line_in_file":174,"slug":"call","name":"call","class":"Flow\\ETL\\Function\\ScalarFunctionChain","class_slug":"scalarfunctionchain","parameters":[{"name":"callable","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"returnType","type":[{"name":"Type","namespace":"Flow\\Types","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"arguments","type":[{"name":"array","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"[]"},{"name":"refAlias","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false},{"name":"int","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"0"}],"return_type":[{"name":"CallUserFunc","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":true,"doc_comment":"LyoqCiAgICAgKiBAcGFyYW0gYXJyYXk8YXJyYXkta2V5LCBtaXhlZD4gJGFyZ3VtZW50cwogICAgICogQHBhcmFtIFR5cGU8bWl4ZWQ+ICRyZXR1cm5UeXBlCiAgICAgKi8="},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Function\/ScalarFunctionChain.php","start_line_in_file":183,"slug":"capitalize","name":"capitalize","class":"Flow\\ETL\\Function\\ScalarFunctionChain","class_slug":"scalarfunctionchain","parameters":[],"return_type":[{"name":"Capitalize","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":true,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Function\/ScalarFunctionChain.php","start_line_in_file":191,"slug":"cast","name":"cast","class":"Flow\\ETL\\Function\\ScalarFunctionChain","class_slug":"scalarfunctionchain","parameters":[{"name":"type","type":[{"name":"Type","namespace":"Flow\\Types","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"Cast","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":true,"doc_comment":"LyoqCiAgICAgKiBAcGFyYW0gc3RyaW5nfFR5cGU8bWl4ZWQ+ICR0eXBlCiAgICAgKi8="},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Function\/ScalarFunctionChain.php","start_line_in_file":196,"slug":"chunk","name":"chunk","class":"Flow\\ETL\\Function\\ScalarFunctionChain","class_slug":"scalarfunctionchain","parameters":[{"name":"size","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"int","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"Chunk","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":true,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Function\/ScalarFunctionChain.php","start_line_in_file":201,"slug":"coalesce","name":"coalesce","class":"Flow\\ETL\\Function\\ScalarFunctionChain","class_slug":"scalarfunctionchain","parameters":[{"name":"params","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":true,"default_value":null}],"return_type":[{"name":"Coalesce","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":true,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Function\/ScalarFunctionChain.php","start_line_in_file":206,"slug":"codepointlength","name":"codePointLength","class":"Flow\\ETL\\Function\\ScalarFunctionChain","class_slug":"scalarfunctionchain","parameters":[],"return_type":[{"name":"CodePointLength","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":true,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Function\/ScalarFunctionChain.php","start_line_in_file":211,"slug":"collapsewhitespace","name":"collapseWhitespace","class":"Flow\\ETL\\Function\\ScalarFunctionChain","class_slug":"scalarfunctionchain","parameters":[],"return_type":[{"name":"CollapseWhitespace","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":true,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Function\/ScalarFunctionChain.php","start_line_in_file":216,"slug":"concat","name":"concat","class":"Flow\\ETL\\Function\\ScalarFunctionChain","class_slug":"scalarfunctionchain","parameters":[{"name":"params","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":true,"default_value":null}],"return_type":[{"name":"Concat","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":true,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Function\/ScalarFunctionChain.php","start_line_in_file":221,"slug":"concatwithseparator","name":"concatWithSeparator","class":"Flow\\ETL\\Function\\ScalarFunctionChain","class_slug":"scalarfunctionchain","parameters":[{"name":"separator","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"params","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":true,"default_value":null}],"return_type":[{"name":"ConcatWithSeparator","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":true,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Function\/ScalarFunctionChain.php","start_line_in_file":228,"slug":"contains","name":"contains","class":"Flow\\ETL\\Function\\ScalarFunctionChain","class_slug":"scalarfunctionchain","parameters":[{"name":"needle","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"Contains","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":true,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Function\/ScalarFunctionChain.php","start_line_in_file":233,"slug":"dateformat","name":"dateFormat","class":"Flow\\ETL\\Function\\ScalarFunctionChain","class_slug":"scalarfunctionchain","parameters":[{"name":"format","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"'Y-m-d'"}],"return_type":[{"name":"DateTimeFormat","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":true,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Function\/ScalarFunctionChain.php","start_line_in_file":238,"slug":"datetimeformat","name":"dateTimeFormat","class":"Flow\\ETL\\Function\\ScalarFunctionChain","class_slug":"scalarfunctionchain","parameters":[{"name":"format","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"'Y-m-d H:i:s'"}],"return_type":[{"name":"DateTimeFormat","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":true,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Function\/ScalarFunctionChain.php","start_line_in_file":243,"slug":"divide","name":"divide","class":"Flow\\ETL\\Function\\ScalarFunctionChain","class_slug":"scalarfunctionchain","parameters":[{"name":"value","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false},{"name":"int","namespace":null,"is_nullable":false,"is_variadic":false},{"name":"float","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"scale","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"int","namespace":null,"is_nullable":false,"is_variadic":false},{"name":"null","namespace":null,"is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"null"},{"name":"rounding","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"Rounding","namespace":"Flow\\Calculator","is_nullable":false,"is_variadic":false},{"name":"null","namespace":null,"is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"null"}],"return_type":[{"name":"Divide","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":true,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Function\/ScalarFunctionChain.php","start_line_in_file":251,"slug":"domelementattributescount","name":"domElementAttributesCount","class":"Flow\\ETL\\Function\\ScalarFunctionChain","class_slug":"scalarfunctionchain","parameters":[],"return_type":[{"name":"DOMElementAttributesCount","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":true,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Function\/ScalarFunctionChain.php","start_line_in_file":256,"slug":"domelementattributevalue","name":"domElementAttributeValue","class":"Flow\\ETL\\Function\\ScalarFunctionChain","class_slug":"scalarfunctionchain","parameters":[{"name":"attribute","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"DOMElementAttributeValue","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":true,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Function\/ScalarFunctionChain.php","start_line_in_file":261,"slug":"domelementnamespace","name":"domElementNamespace","class":"Flow\\ETL\\Function\\ScalarFunctionChain","class_slug":"scalarfunctionchain","parameters":[{"name":"attribute","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false},{"name":"null","namespace":null,"is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"null"}],"return_type":[{"name":"DOMElementNamespaceValue","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":true,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Function\/ScalarFunctionChain.php","start_line_in_file":266,"slug":"domelementnextsibling","name":"domElementNextSibling","class":"Flow\\ETL\\Function\\ScalarFunctionChain","class_slug":"scalarfunctionchain","parameters":[],"return_type":[{"name":"DOMElementNextSibling","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":true,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Function\/ScalarFunctionChain.php","start_line_in_file":271,"slug":"domelementparent","name":"domElementParent","class":"Flow\\ETL\\Function\\ScalarFunctionChain","class_slug":"scalarfunctionchain","parameters":[],"return_type":[{"name":"DOMElementParent","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":true,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Function\/ScalarFunctionChain.php","start_line_in_file":276,"slug":"domelementprevioussibling","name":"domElementPreviousSibling","class":"Flow\\ETL\\Function\\ScalarFunctionChain","class_slug":"scalarfunctionchain","parameters":[],"return_type":[{"name":"DOMElementPreviousSibling","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":true,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Function\/ScalarFunctionChain.php","start_line_in_file":281,"slug":"domelementvalue","name":"domElementValue","class":"Flow\\ETL\\Function\\ScalarFunctionChain","class_slug":"scalarfunctionchain","parameters":[],"return_type":[{"name":"DOMElementValue","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":true,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Function\/ScalarFunctionChain.php","start_line_in_file":286,"slug":"endswith","name":"endsWith","class":"Flow\\ETL\\Function\\ScalarFunctionChain","class_slug":"scalarfunctionchain","parameters":[{"name":"needle","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"EndsWith","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":true,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Function\/ScalarFunctionChain.php","start_line_in_file":291,"slug":"ensureend","name":"ensureEnd","class":"Flow\\ETL\\Function\\ScalarFunctionChain","class_slug":"scalarfunctionchain","parameters":[{"name":"suffix","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"EnsureEnd","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":true,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Function\/ScalarFunctionChain.php","start_line_in_file":296,"slug":"ensurestart","name":"ensureStart","class":"Flow\\ETL\\Function\\ScalarFunctionChain","class_slug":"scalarfunctionchain","parameters":[{"name":"prefix","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"EnsureStart","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":true,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Function\/ScalarFunctionChain.php","start_line_in_file":301,"slug":"enumname","name":"enumName","class":"Flow\\ETL\\Function\\ScalarFunctionChain","class_slug":"scalarfunctionchain","parameters":[],"return_type":[{"name":"EnumName","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":true,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Function\/ScalarFunctionChain.php","start_line_in_file":306,"slug":"enumvalue","name":"enumValue","class":"Flow\\ETL\\Function\\ScalarFunctionChain","class_slug":"scalarfunctionchain","parameters":[],"return_type":[{"name":"EnumValue","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":true,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Function\/ScalarFunctionChain.php","start_line_in_file":311,"slug":"equals","name":"equals","class":"Flow\\ETL\\Function\\ScalarFunctionChain","class_slug":"scalarfunctionchain","parameters":[{"name":"ref","type":[{"name":"mixed","namespace":null,"is_nullable":true,"is_variadic":false}],"has_default_value":false,"is_nullable":true,"is_variadic":false,"default_value":null}],"return_type":[{"name":"Equals","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":true,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Function\/ScalarFunctionChain.php","start_line_in_file":316,"slug":"exists","name":"exists","class":"Flow\\ETL\\Function\\ScalarFunctionChain","class_slug":"scalarfunctionchain","parameters":[],"return_type":[{"name":"Exists","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":true,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Function\/ScalarFunctionChain.php","start_line_in_file":345,"slug":"expand","name":"expand","class":"Flow\\ETL\\Function\\ScalarFunctionChain","class_slug":"scalarfunctionchain","parameters":[{"name":"expand","type":[{"name":"ArrayExpand","namespace":"Flow\\ETL\\Function\\ArrayExpand","is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"Flow\\ETL\\Function\\ArrayExpand\\ArrayExpand::..."}],"return_type":[{"name":"ArrayExpand","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":true,"doc_comment":"LyoqCiAgICAgKiBFeHBhbmRzIGVhY2ggdmFsdWUgaW50byBlbnRyeSwgaWYgdGhlcmUgYXJlIG1vcmUgdGhhbiBvbmUgdmFsdWUsIG11bHRpcGxlIHJvd3Mgd2lsbCBiZSBjcmVhdGVkLgogICAgICogQXJyYXkga2V5cyBhcmUgaWdub3JlZCwgb25seSB2YWx1ZXMgYXJlIHVzZWQgdG8gY3JlYXRlIG5ldyByb3dzLgogICAgICogTmVzdGVkIGluIGFub3RoZXIgZnVuY3Rpb24gKHN0cnVjdHVyZSgpLCBjb25jYXQoKSwgLi4uKSBpdCBzdGlsbCBnaXZlcyBvbmUgcm93IHBlciBlbGVtZW50LiBTZXZlcmFsCiAgICAgKiBleHBhbmRzIGluIG9uZSBleHByZXNzaW9uIGFyZSB6aXBwZWQgdG8gdGhlIGxvbmdlc3QgbGlzdDsgYSBzaG9ydGVyIG9uZSBnaXZlcyBudWxsLCBzbyBpdHMgZWxlbWVudAogICAgICogdHlwZSBiZWNvbWVzIG51bGxhYmxlLiBJdCBpcyByZWZ1c2VkIGluc2lkZSBhbm90aGVyIGFycmF5X2V4cGFuZCgpIGFuZCBpbiBmaWx0ZXIoKSwgdW50aWwoKSwKICAgICAqIGR1cGxpY2F0ZVJvdygpLCBhZ2dyZWdhdGUoKSwgb3ZlcigpIGFuZCBvbkVhY2goKS4KICAgICAqCiAgICAgKiBCZWZvcmU6CiAgICAgKiAgICstLSstLS0tLS0tLS0tLS0tLS0tLS0tKwogICAgICogICB8aWR8ICAgICAgICAgICAgICBhcnJheXwKICAgICAqICAgKy0tKy0tLS0tLS0tLS0tLS0tLS0tLS0rCiAgICAgKiAgIHwgMXx7ImEiOjEsImIiOjIsImMiOjN9fAogICAgICogICArLS0rLS0tLS0tLS0tLS0tLS0tLS0tLSsKICAgICAqCiAgICAgKiBBZnRlcjoKICAgICAqICAgKy0tKy0tLS0tLS0tKwogICAgICogICB8aWR8ZXhwYW5kZWR8CiAgICAgKiAgICstLSstLS0tLS0tLSsKICAgICAqICAgfCAxfCAgICAgICAxfAogICAgICogICB8IDF8ICAgICAgIDJ8CiAgICAgKiAgIHwgMXwgICAgICAgM3wKICAgICAqICAgKy0tKy0tLS0tLS0tKwogICAgICov"},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Function\/ScalarFunctionChain.php","start_line_in_file":350,"slug":"greaterthan","name":"greaterThan","class":"Flow\\ETL\\Function\\ScalarFunctionChain","class_slug":"scalarfunctionchain","parameters":[{"name":"ref","type":[{"name":"mixed","namespace":null,"is_nullable":true,"is_variadic":false}],"has_default_value":false,"is_nullable":true,"is_variadic":false,"default_value":null}],"return_type":[{"name":"GreaterThan","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":true,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Function\/ScalarFunctionChain.php","start_line_in_file":355,"slug":"greaterthanequal","name":"greaterThanEqual","class":"Flow\\ETL\\Function\\ScalarFunctionChain","class_slug":"scalarfunctionchain","parameters":[{"name":"ref","type":[{"name":"mixed","namespace":null,"is_nullable":true,"is_variadic":false}],"has_default_value":false,"is_nullable":true,"is_variadic":false,"default_value":null}],"return_type":[{"name":"GreaterThanEqual","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":true,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Function\/ScalarFunctionChain.php","start_line_in_file":360,"slug":"hash","name":"hash","class":"Flow\\ETL\\Function\\ScalarFunctionChain","class_slug":"scalarfunctionchain","parameters":[{"name":"algorithm","type":[{"name":"Algorithm","namespace":"Flow\\ETL\\Hash","is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"Flow\\ETL\\Hash\\NativePHPHash::..."}],"return_type":[{"name":"Hash","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":true,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Function\/ScalarFunctionChain.php","start_line_in_file":365,"slug":"htmlqueryselector","name":"htmlQuerySelector","class":"Flow\\ETL\\Function\\ScalarFunctionChain","class_slug":"scalarfunctionchain","parameters":[{"name":"path","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"HTMLQuerySelector","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":true,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Function\/ScalarFunctionChain.php","start_line_in_file":370,"slug":"htmlqueryselectorall","name":"htmlQuerySelectorAll","class":"Flow\\ETL\\Function\\ScalarFunctionChain","class_slug":"scalarfunctionchain","parameters":[{"name":"path","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"HTMLQuerySelectorAll","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":true,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Function\/ScalarFunctionChain.php","start_line_in_file":378,"slug":"indexof","name":"indexOf","class":"Flow\\ETL\\Function\\ScalarFunctionChain","class_slug":"scalarfunctionchain","parameters":[{"name":"needle","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"ignoreCase","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"bool","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"false"},{"name":"offset","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"int","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"0"}],"return_type":[{"name":"IndexOf","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":true,"doc_comment":"LyoqCiAgICAgKiBSZXR1cm5zIHRoZSBpbmRleCBvZiBnaXZlbiAkbmVlZGxlIGluIHN0cmluZy4KICAgICAqLw=="},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Function\/ScalarFunctionChain.php","start_line_in_file":389,"slug":"indexoflast","name":"indexOfLast","class":"Flow\\ETL\\Function\\ScalarFunctionChain","class_slug":"scalarfunctionchain","parameters":[{"name":"needle","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"ignoreCase","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"bool","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"false"},{"name":"offset","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"int","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"0"}],"return_type":[{"name":"IndexOfLast","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":true,"doc_comment":"LyoqCiAgICAgKiBSZXR1cm5zIHRoZSBsYXN0IGluZGV4IG9mIGdpdmVuICRuZWVkbGUgaW4gc3RyaW5nLgogICAgICov"},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Function\/ScalarFunctionChain.php","start_line_in_file":397,"slug":"isempty","name":"isEmpty","class":"Flow\\ETL\\Function\\ScalarFunctionChain","class_slug":"scalarfunctionchain","parameters":[],"return_type":[{"name":"IsEmpty","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":true,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Function\/ScalarFunctionChain.php","start_line_in_file":402,"slug":"iseven","name":"isEven","class":"Flow\\ETL\\Function\\ScalarFunctionChain","class_slug":"scalarfunctionchain","parameters":[],"return_type":[{"name":"Equals","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":true,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Function\/ScalarFunctionChain.php","start_line_in_file":407,"slug":"isfalse","name":"isFalse","class":"Flow\\ETL\\Function\\ScalarFunctionChain","class_slug":"scalarfunctionchain","parameters":[],"return_type":[{"name":"Same","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":true,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Function\/ScalarFunctionChain.php","start_line_in_file":415,"slug":"isin","name":"isIn","class":"Flow\\ETL\\Function\\ScalarFunctionChain","class_slug":"scalarfunctionchain","parameters":[{"name":"haystack","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"array","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"IsIn","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":true,"doc_comment":"LyoqCiAgICAgKiBAcGFyYW0gU2NhbGFyRnVuY3Rpb258YXJyYXk8YXJyYXkta2V5LCBtaXhlZD4gJGhheXN0YWNrCiAgICAgKi8="},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Function\/ScalarFunctionChain.php","start_line_in_file":420,"slug":"isnotnull","name":"isNotNull","class":"Flow\\ETL\\Function\\ScalarFunctionChain","class_slug":"scalarfunctionchain","parameters":[],"return_type":[{"name":"IsNotNull","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":true,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Function\/ScalarFunctionChain.php","start_line_in_file":425,"slug":"isnotnumeric","name":"isNotNumeric","class":"Flow\\ETL\\Function\\ScalarFunctionChain","class_slug":"scalarfunctionchain","parameters":[],"return_type":[{"name":"IsNotNumeric","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":true,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Function\/ScalarFunctionChain.php","start_line_in_file":430,"slug":"isnull","name":"isNull","class":"Flow\\ETL\\Function\\ScalarFunctionChain","class_slug":"scalarfunctionchain","parameters":[],"return_type":[{"name":"IsNull","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":true,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Function\/ScalarFunctionChain.php","start_line_in_file":435,"slug":"isnumeric","name":"isNumeric","class":"Flow\\ETL\\Function\\ScalarFunctionChain","class_slug":"scalarfunctionchain","parameters":[],"return_type":[{"name":"IsNumeric","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":true,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Function\/ScalarFunctionChain.php","start_line_in_file":440,"slug":"isodd","name":"isOdd","class":"Flow\\ETL\\Function\\ScalarFunctionChain","class_slug":"scalarfunctionchain","parameters":[],"return_type":[{"name":"NotEquals","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":true,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Function\/ScalarFunctionChain.php","start_line_in_file":445,"slug":"istrue","name":"isTrue","class":"Flow\\ETL\\Function\\ScalarFunctionChain","class_slug":"scalarfunctionchain","parameters":[],"return_type":[{"name":"Same","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":true,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Function\/ScalarFunctionChain.php","start_line_in_file":453,"slug":"istype","name":"isType","class":"Flow\\ETL\\Function\\ScalarFunctionChain","class_slug":"scalarfunctionchain","parameters":[{"name":"types","type":[{"name":"Type","namespace":"Flow\\Types","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":true,"default_value":null}],"return_type":[{"name":"IsType","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":true,"doc_comment":"LyoqCiAgICAgKiBAcGFyYW0gc3RyaW5nfFR5cGU8bWl4ZWQ+ICR0eXBlcwogICAgICov"},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Function\/ScalarFunctionChain.php","start_line_in_file":465,"slug":"isutf8","name":"isUtf8","class":"Flow\\ETL\\Function\\ScalarFunctionChain","class_slug":"scalarfunctionchain","parameters":[],"return_type":[{"name":"IsUtf8","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":true,"doc_comment":"LyoqCiAgICAgKiBDaGVjayBzdHJpbmcgaXMgdXRmOCBhbmQgcmV0dXJucyB0cnVlIG9yIGZhbHNlLgogICAgICov"},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Function\/ScalarFunctionChain.php","start_line_in_file":470,"slug":"jsondecode","name":"jsonDecode","class":"Flow\\ETL\\Function\\ScalarFunctionChain","class_slug":"scalarfunctionchain","parameters":[{"name":"flags","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"int","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"4194304"}],"return_type":[{"name":"JsonDecode","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":true,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Function\/ScalarFunctionChain.php","start_line_in_file":475,"slug":"jsonencode","name":"jsonEncode","class":"Flow\\ETL\\Function\\ScalarFunctionChain","class_slug":"scalarfunctionchain","parameters":[{"name":"flags","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"int","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"4194304"}],"return_type":[{"name":"JsonEncode","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":true,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Function\/ScalarFunctionChain.php","start_line_in_file":480,"slug":"lessthan","name":"lessThan","class":"Flow\\ETL\\Function\\ScalarFunctionChain","class_slug":"scalarfunctionchain","parameters":[{"name":"ref","type":[{"name":"mixed","namespace":null,"is_nullable":true,"is_variadic":false}],"has_default_value":false,"is_nullable":true,"is_variadic":false,"default_value":null}],"return_type":[{"name":"LessThan","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":true,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Function\/ScalarFunctionChain.php","start_line_in_file":485,"slug":"lessthanequal","name":"lessThanEqual","class":"Flow\\ETL\\Function\\ScalarFunctionChain","class_slug":"scalarfunctionchain","parameters":[{"name":"ref","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"LessThanEqual","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":true,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Function\/ScalarFunctionChain.php","start_line_in_file":490,"slug":"literal","name":"literal","class":"Flow\\ETL\\Function\\ScalarFunctionChain","class_slug":"scalarfunctionchain","parameters":[{"name":"value","type":[{"name":"mixed","namespace":null,"is_nullable":true,"is_variadic":false}],"has_default_value":false,"is_nullable":true,"is_variadic":false,"default_value":null}],"return_type":[{"name":"Literal","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":true,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Function\/ScalarFunctionChain.php","start_line_in_file":495,"slug":"lower","name":"lower","class":"Flow\\ETL\\Function\\ScalarFunctionChain","class_slug":"scalarfunctionchain","parameters":[],"return_type":[{"name":"ToLower","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":true,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Function\/ScalarFunctionChain.php","start_line_in_file":500,"slug":"minus","name":"minus","class":"Flow\\ETL\\Function\\ScalarFunctionChain","class_slug":"scalarfunctionchain","parameters":[{"name":"ref","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"int","namespace":null,"is_nullable":false,"is_variadic":false},{"name":"float","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"Minus","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":true,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Function\/ScalarFunctionChain.php","start_line_in_file":505,"slug":"mod","name":"mod","class":"Flow\\ETL\\Function\\ScalarFunctionChain","class_slug":"scalarfunctionchain","parameters":[{"name":"value","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"int","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"Mod","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":true,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Function\/ScalarFunctionChain.php","start_line_in_file":510,"slug":"modifydatetime","name":"modifyDateTime","class":"Flow\\ETL\\Function\\ScalarFunctionChain","class_slug":"scalarfunctionchain","parameters":[{"name":"modifier","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"ModifyDateTime","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":true,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Function\/ScalarFunctionChain.php","start_line_in_file":515,"slug":"multiply","name":"multiply","class":"Flow\\ETL\\Function\\ScalarFunctionChain","class_slug":"scalarfunctionchain","parameters":[{"name":"value","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"int","namespace":null,"is_nullable":false,"is_variadic":false},{"name":"float","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"Multiply","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":true,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Function\/ScalarFunctionChain.php","start_line_in_file":520,"slug":"notequals","name":"notEquals","class":"Flow\\ETL\\Function\\ScalarFunctionChain","class_slug":"scalarfunctionchain","parameters":[{"name":"value","type":[{"name":"mixed","namespace":null,"is_nullable":true,"is_variadic":false}],"has_default_value":false,"is_nullable":true,"is_variadic":false,"default_value":null}],"return_type":[{"name":"NotEquals","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":true,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Function\/ScalarFunctionChain.php","start_line_in_file":525,"slug":"notsame","name":"notSame","class":"Flow\\ETL\\Function\\ScalarFunctionChain","class_slug":"scalarfunctionchain","parameters":[{"name":"value","type":[{"name":"mixed","namespace":null,"is_nullable":true,"is_variadic":false}],"has_default_value":false,"is_nullable":true,"is_variadic":false,"default_value":null}],"return_type":[{"name":"NotSame","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":true,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Function\/ScalarFunctionChain.php","start_line_in_file":530,"slug":"numberformat","name":"numberFormat","class":"Flow\\ETL\\Function\\ScalarFunctionChain","class_slug":"scalarfunctionchain","parameters":[{"name":"decimals","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"int","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"2"},{"name":"decimalSeparator","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"'.'"},{"name":"thousandsSeparator","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"','"}],"return_type":[{"name":"NumberFormat","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":true,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Function\/ScalarFunctionChain.php","start_line_in_file":544,"slug":"oneach","name":"onEach","class":"Flow\\ETL\\Function\\ScalarFunctionChain","class_slug":"scalarfunctionchain","parameters":[{"name":"function","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"preserveKeys","type":[{"name":"bool","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"true"}],"return_type":[{"name":"OnEach","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":true,"doc_comment":"LyoqCiAgICAgKiBFeGVjdXRlIGEgc2NhbGFyIGZ1bmN0aW9uIG9uIGVhY2ggZWxlbWVudCBvZiBhbiBhcnJheS9saXN0L21hcC9zdHJ1Y3R1cmUgZW50cnkuCiAgICAgKiBJbiBvcmRlciB0byB1c2UgdGhpcyBmdW5jdGlvbiwgeW91IG5lZWQgdG8gcHJvdmlkZSBhIHJlZmVyZW5jZSB0byB0aGUgImVsZW1lbnQiIHRoYXQgd2lsbCBiZSB1c2VkIGluIHRoZSBmdW5jdGlvbi4KICAgICAqCiAgICAgKiBFeGFtcGxlOiAkZGYtPndpdGhFbnRyeSgnYXJyYXknLCByZWYoJ2FycmF5JyktPm9uRWFjaChyZWYoJ2VsZW1lbnQnKS0+Y2FzdCh0eXBlX3N0cmluZygpKSkpCiAgICAgKi8="},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Function\/ScalarFunctionChain.php","start_line_in_file":549,"slug":"or","name":"or","class":"Flow\\ETL\\Function\\ScalarFunctionChain","class_slug":"scalarfunctionchain","parameters":[{"name":"function","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"Any","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Function\/ScalarFunctionChain.php","start_line_in_file":554,"slug":"ornot","name":"orNot","class":"Flow\\ETL\\Function\\ScalarFunctionChain","class_slug":"scalarfunctionchain","parameters":[{"name":"function","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"Any","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Function\/ScalarFunctionChain.php","start_line_in_file":559,"slug":"plus","name":"plus","class":"Flow\\ETL\\Function\\ScalarFunctionChain","class_slug":"scalarfunctionchain","parameters":[{"name":"ref","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"int","namespace":null,"is_nullable":false,"is_variadic":false},{"name":"float","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"Plus","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":true,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Function\/ScalarFunctionChain.php","start_line_in_file":564,"slug":"power","name":"power","class":"Flow\\ETL\\Function\\ScalarFunctionChain","class_slug":"scalarfunctionchain","parameters":[{"name":"value","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"int","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"Power","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":true,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Function\/ScalarFunctionChain.php","start_line_in_file":569,"slug":"prepend","name":"prepend","class":"Flow\\ETL\\Function\\ScalarFunctionChain","class_slug":"scalarfunctionchain","parameters":[{"name":"prefix","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"Prepend","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":true,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Function\/ScalarFunctionChain.php","start_line_in_file":574,"slug":"regex","name":"regex","class":"Flow\\ETL\\Function\\ScalarFunctionChain","class_slug":"scalarfunctionchain","parameters":[{"name":"pattern","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"flags","type":[{"name":"int","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"0"},{"name":"offset","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"int","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"0"}],"return_type":[{"name":"Regex","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":true,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Function\/ScalarFunctionChain.php","start_line_in_file":579,"slug":"regexall","name":"regexAll","class":"Flow\\ETL\\Function\\ScalarFunctionChain","class_slug":"scalarfunctionchain","parameters":[{"name":"pattern","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"flags","type":[{"name":"int","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"0"},{"name":"offset","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"int","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"0"}],"return_type":[{"name":"RegexAll","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":true,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Function\/ScalarFunctionChain.php","start_line_in_file":584,"slug":"regexmatch","name":"regexMatch","class":"Flow\\ETL\\Function\\ScalarFunctionChain","class_slug":"scalarfunctionchain","parameters":[{"name":"pattern","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"flags","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"int","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"0"},{"name":"offset","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"int","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"0"}],"return_type":[{"name":"RegexMatch","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":true,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Function\/ScalarFunctionChain.php","start_line_in_file":592,"slug":"regexmatchall","name":"regexMatchAll","class":"Flow\\ETL\\Function\\ScalarFunctionChain","class_slug":"scalarfunctionchain","parameters":[{"name":"pattern","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"flags","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"int","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"0"},{"name":"offset","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"int","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"0"}],"return_type":[{"name":"RegexMatchAll","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":true,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Function\/ScalarFunctionChain.php","start_line_in_file":600,"slug":"regexreplace","name":"regexReplace","class":"Flow\\ETL\\Function\\ScalarFunctionChain","class_slug":"scalarfunctionchain","parameters":[{"name":"pattern","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"replacement","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"limit","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"int","namespace":null,"is_nullable":false,"is_variadic":false},{"name":"null","namespace":null,"is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"null"}],"return_type":[{"name":"RegexReplace","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":true,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Function\/ScalarFunctionChain.php","start_line_in_file":608,"slug":"repeat","name":"repeat","class":"Flow\\ETL\\Function\\ScalarFunctionChain","class_slug":"scalarfunctionchain","parameters":[{"name":"times","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"int","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"Repeat","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":true,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Function\/ScalarFunctionChain.php","start_line_in_file":613,"slug":"reverse","name":"reverse","class":"Flow\\ETL\\Function\\ScalarFunctionChain","class_slug":"scalarfunctionchain","parameters":[],"return_type":[{"name":"Reverse","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":true,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Function\/ScalarFunctionChain.php","start_line_in_file":618,"slug":"round","name":"round","class":"Flow\\ETL\\Function\\ScalarFunctionChain","class_slug":"scalarfunctionchain","parameters":[{"name":"precision","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"int","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"2"},{"name":"mode","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"int","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"1"}],"return_type":[{"name":"Round","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":true,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Function\/ScalarFunctionChain.php","start_line_in_file":623,"slug":"same","name":"same","class":"Flow\\ETL\\Function\\ScalarFunctionChain","class_slug":"scalarfunctionchain","parameters":[{"name":"value","type":[{"name":"mixed","namespace":null,"is_nullable":true,"is_variadic":false}],"has_default_value":false,"is_nullable":true,"is_variadic":false,"default_value":null}],"return_type":[{"name":"Same","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":true,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Function\/ScalarFunctionChain.php","start_line_in_file":628,"slug":"sanitize","name":"sanitize","class":"Flow\\ETL\\Function\\ScalarFunctionChain","class_slug":"scalarfunctionchain","parameters":[{"name":"placeholder","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"'*'"},{"name":"skipCharacters","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"int","namespace":null,"is_nullable":false,"is_variadic":false},{"name":"null","namespace":null,"is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"null"}],"return_type":[{"name":"Sanitize","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":true,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Function\/ScalarFunctionChain.php","start_line_in_file":635,"slug":"size","name":"size","class":"Flow\\ETL\\Function\\ScalarFunctionChain","class_slug":"scalarfunctionchain","parameters":[],"return_type":[{"name":"Size","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":true,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Function\/ScalarFunctionChain.php","start_line_in_file":643,"slug":"slug","name":"slug","class":"Flow\\ETL\\Function\\ScalarFunctionChain","class_slug":"scalarfunctionchain","parameters":[{"name":"separator","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"'-'"},{"name":"locale","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false},{"name":"null","namespace":null,"is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"null"},{"name":"symbolsMap","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"array","namespace":null,"is_nullable":false,"is_variadic":false},{"name":"null","namespace":null,"is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"null"}],"return_type":[{"name":"Slug","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":true,"doc_comment":"LyoqCiAgICAgKiBAcGFyYW0gU2NhbGFyRnVuY3Rpb258bnVsbHxhcnJheTxhcnJheS1rZXksIG1peGVkPiAkc3ltYm9sc01hcAogICAgICov"},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Function\/ScalarFunctionChain.php","start_line_in_file":651,"slug":"split","name":"split","class":"Flow\\ETL\\Function\\ScalarFunctionChain","class_slug":"scalarfunctionchain","parameters":[{"name":"separator","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"limit","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"int","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"9223372036854775807"}],"return_type":[{"name":"Split","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":true,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Function\/ScalarFunctionChain.php","start_line_in_file":656,"slug":"sprintf","name":"sprintf","class":"Flow\\ETL\\Function\\ScalarFunctionChain","class_slug":"scalarfunctionchain","parameters":[{"name":"params","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false},{"name":"int","namespace":null,"is_nullable":false,"is_variadic":false},{"name":"float","namespace":null,"is_nullable":false,"is_variadic":false},{"name":"null","namespace":null,"is_nullable":true,"is_variadic":false}],"has_default_value":false,"is_nullable":true,"is_variadic":true,"default_value":null}],"return_type":[{"name":"Sprintf","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":true,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Function\/ScalarFunctionChain.php","start_line_in_file":661,"slug":"startswith","name":"startsWith","class":"Flow\\ETL\\Function\\ScalarFunctionChain","class_slug":"scalarfunctionchain","parameters":[{"name":"needle","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"StartsWith","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":true,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Function\/ScalarFunctionChain.php","start_line_in_file":669,"slug":"stringafter","name":"stringAfter","class":"Flow\\ETL\\Function\\ScalarFunctionChain","class_slug":"scalarfunctionchain","parameters":[{"name":"needle","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"includeNeedle","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"bool","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"false"}],"return_type":[{"name":"StringAfter","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":true,"doc_comment":"LyoqCiAgICAgKiBSZXR1cm5zIHRoZSBjb250ZW50cyBmb3VuZCBhZnRlciB0aGUgZmlyc3Qgb2NjdXJyZW5jZSBvZiB0aGUgZ2l2ZW4gc3RyaW5nLgogICAgICov"},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Function\/ScalarFunctionChain.php","start_line_in_file":677,"slug":"stringafterlast","name":"stringAfterLast","class":"Flow\\ETL\\Function\\ScalarFunctionChain","class_slug":"scalarfunctionchain","parameters":[{"name":"needle","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"includeNeedle","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"bool","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"false"}],"return_type":[{"name":"StringAfterLast","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":true,"doc_comment":"LyoqCiAgICAgKiBSZXR1cm5zIHRoZSBjb250ZW50cyBmb3VuZCBhZnRlciB0aGUgbGFzdCBvY2N1cnJlbmNlIG9mIHRoZSBnaXZlbiBzdHJpbmcuCiAgICAgKi8="},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Function\/ScalarFunctionChain.php","start_line_in_file":687,"slug":"stringbefore","name":"stringBefore","class":"Flow\\ETL\\Function\\ScalarFunctionChain","class_slug":"scalarfunctionchain","parameters":[{"name":"needle","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"includeNeedle","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"bool","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"false"}],"return_type":[{"name":"StringBefore","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":true,"doc_comment":"LyoqCiAgICAgKiBSZXR1cm5zIHRoZSBjb250ZW50cyBmb3VuZCBiZWZvcmUgdGhlIGZpcnN0IG9jY3VycmVuY2Ugb2YgdGhlIGdpdmVuIHN0cmluZy4KICAgICAqLw=="},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Function\/ScalarFunctionChain.php","start_line_in_file":697,"slug":"stringbeforelast","name":"stringBeforeLast","class":"Flow\\ETL\\Function\\ScalarFunctionChain","class_slug":"scalarfunctionchain","parameters":[{"name":"needle","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"includeNeedle","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"bool","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"false"}],"return_type":[{"name":"StringBeforeLast","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":true,"doc_comment":"LyoqCiAgICAgKiBSZXR1cm5zIHRoZSBjb250ZW50cyBmb3VuZCBiZWZvcmUgdGhlIGxhc3Qgb2NjdXJyZW5jZSBvZiB0aGUgZ2l2ZW4gc3RyaW5nLgogICAgICov"},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Function\/ScalarFunctionChain.php","start_line_in_file":707,"slug":"stringcontainsany","name":"stringContainsAny","class":"Flow\\ETL\\Function\\ScalarFunctionChain","class_slug":"scalarfunctionchain","parameters":[{"name":"needles","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"array","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"StringContainsAny","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":true,"doc_comment":"LyoqCiAgICAgKiBAcGFyYW0gYXJyYXk8c3RyaW5nPnxTY2FsYXJGdW5jdGlvbiAkbmVlZGxlcwogICAgICov"},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Function\/ScalarFunctionChain.php","start_line_in_file":712,"slug":"stringequalsto","name":"stringEqualsTo","class":"Flow\\ETL\\Function\\ScalarFunctionChain","class_slug":"scalarfunctionchain","parameters":[{"name":"string","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"StringEqualsTo","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":true,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Function\/ScalarFunctionChain.php","start_line_in_file":720,"slug":"stringfold","name":"stringFold","class":"Flow\\ETL\\Function\\ScalarFunctionChain","class_slug":"scalarfunctionchain","parameters":[],"return_type":[{"name":"StringFold","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":true,"doc_comment":"LyoqCiAgICAgKiBSZXR1cm5zIGEgc3RyaW5nIHRoYXQgeW91IGNhbiB1c2UgaW4gY2FzZS1pbnNlbnNpdGl2ZSBjb21wYXJpc29ucy4KICAgICAqLw=="},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Function\/ScalarFunctionChain.php","start_line_in_file":725,"slug":"stringmatch","name":"stringMatch","class":"Flow\\ETL\\Function\\ScalarFunctionChain","class_slug":"scalarfunctionchain","parameters":[{"name":"pattern","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"StringMatch","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":true,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Function\/ScalarFunctionChain.php","start_line_in_file":730,"slug":"stringmatchall","name":"stringMatchAll","class":"Flow\\ETL\\Function\\ScalarFunctionChain","class_slug":"scalarfunctionchain","parameters":[{"name":"pattern","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"StringMatchAll","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":true,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Function\/ScalarFunctionChain.php","start_line_in_file":735,"slug":"stringnormalize","name":"stringNormalize","class":"Flow\\ETL\\Function\\ScalarFunctionChain","class_slug":"scalarfunctionchain","parameters":[{"name":"form","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"int","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"16"}],"return_type":[{"name":"StringNormalize","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":true,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Function\/ScalarFunctionChain.php","start_line_in_file":744,"slug":"stringstyle","name":"stringStyle","class":"Flow\\ETL\\Function\\ScalarFunctionChain","class_slug":"scalarfunctionchain","parameters":[{"name":"style","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"StringStyles","namespace":"Flow\\ETL\\String","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"StringStyle","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":true,"doc_comment":"LyoqCiAgICAgKiBDb3ZlcnQgc3RyaW5nIHRvIGEgc3R5bGUgZnJvbSBlbnVtIGxpc3QsIHBhc3NlZCBpbiBwYXJhbWV0ZXIuCiAgICAgKiBDYW4gYmUgc3RyaW5nICJ1cHBlciIgb3IgU3RyaW5nU3R5bGVzOjpVUFBFUiBmb3IgVXBwZXIgKGV4YW1wbGUpLgogICAgICov"},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Function\/ScalarFunctionChain.php","start_line_in_file":752,"slug":"stringtitle","name":"stringTitle","class":"Flow\\ETL\\Function\\ScalarFunctionChain","class_slug":"scalarfunctionchain","parameters":[{"name":"allWords","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"bool","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"false"}],"return_type":[{"name":"StringTitle","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":true,"doc_comment":"LyoqCiAgICAgKiBDaGFuZ2VzIGFsbCBncmFwaGVtZXMvY29kZSBwb2ludHMgdG8gInRpdGxlIGNhc2UiLgogICAgICov"},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Function\/ScalarFunctionChain.php","start_line_in_file":757,"slug":"stringwidth","name":"stringWidth","class":"Flow\\ETL\\Function\\ScalarFunctionChain","class_slug":"scalarfunctionchain","parameters":[],"return_type":[{"name":"StringWidth","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":true,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Function\/ScalarFunctionChain.php","start_line_in_file":762,"slug":"strpad","name":"strPad","class":"Flow\\ETL\\Function\\ScalarFunctionChain","class_slug":"scalarfunctionchain","parameters":[{"name":"length","type":[{"name":"int","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"pad_string","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"' '"},{"name":"type","type":[{"name":"int","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"1"}],"return_type":[{"name":"StrPad","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":true,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Function\/ScalarFunctionChain.php","start_line_in_file":767,"slug":"strpadboth","name":"strPadBoth","class":"Flow\\ETL\\Function\\ScalarFunctionChain","class_slug":"scalarfunctionchain","parameters":[{"name":"length","type":[{"name":"int","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"pad_string","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"' '"}],"return_type":[{"name":"StrPad","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":true,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Function\/ScalarFunctionChain.php","start_line_in_file":772,"slug":"strpadleft","name":"strPadLeft","class":"Flow\\ETL\\Function\\ScalarFunctionChain","class_slug":"scalarfunctionchain","parameters":[{"name":"length","type":[{"name":"int","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"pad_string","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"' '"}],"return_type":[{"name":"StrPad","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":true,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Function\/ScalarFunctionChain.php","start_line_in_file":777,"slug":"strpadright","name":"strPadRight","class":"Flow\\ETL\\Function\\ScalarFunctionChain","class_slug":"scalarfunctionchain","parameters":[{"name":"length","type":[{"name":"int","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"pad_string","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"' '"}],"return_type":[{"name":"StrPad","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":true,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Function\/ScalarFunctionChain.php","start_line_in_file":786,"slug":"strreplace","name":"strReplace","class":"Flow\\ETL\\Function\\ScalarFunctionChain","class_slug":"scalarfunctionchain","parameters":[{"name":"search","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"array","namespace":null,"is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"replace","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"array","namespace":null,"is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"StrReplace","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":true,"doc_comment":"LyoqCiAgICAgKiBAcGFyYW0gYXJyYXk8c3RyaW5nPnxTY2FsYXJGdW5jdGlvbnxzdHJpbmcgJHNlYXJjaAogICAgICogQHBhcmFtIGFycmF5PHN0cmluZz58U2NhbGFyRnVuY3Rpb258c3RyaW5nICRyZXBsYWNlCiAgICAgKi8="},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Function\/ScalarFunctionChain.php","start_line_in_file":794,"slug":"todate","name":"toDate","class":"Flow\\ETL\\Function\\ScalarFunctionChain","class_slug":"scalarfunctionchain","parameters":[{"name":"format","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"'Y-m-d\\\\TH:i:sP'"},{"name":"timeZone","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"DateTimeZone","namespace":"","is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"DateTimeZone::..."}],"return_type":[{"name":"ToDate","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":true,"doc_comment":"LyoqCiAgICAgKiBAcGFyYW0gU2NhbGFyRnVuY3Rpb258c3RyaW5nICRmb3JtYXQgLSBjdXJyZW50IGZvcm1hdCBvZiB0aGUgZGF0ZSB0aGF0IHdpbGwgYmUgdXNlZCB0byBjcmVhdGUgRGF0ZVRpbWVJbW11dGFibGUgaW5zdGFuY2UKICAgICAqLw=="},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Function\/ScalarFunctionChain.php","start_line_in_file":805,"slug":"todatetime","name":"toDateTime","class":"Flow\\ETL\\Function\\ScalarFunctionChain","class_slug":"scalarfunctionchain","parameters":[{"name":"format","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"'Y-m-d H:i:s'"},{"name":"timeZone","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"DateTimeZone","namespace":"","is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"DateTimeZone::..."}],"return_type":[{"name":"ToDateTime","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":true,"doc_comment":"LyoqCiAgICAgKiBAcGFyYW0gU2NhbGFyRnVuY3Rpb258c3RyaW5nICRmb3JtYXQgLSBjdXJyZW50IGZvcm1hdCBvZiB0aGUgZGF0ZSB0aGF0IHdpbGwgYmUgdXNlZCB0byBjcmVhdGUgRGF0ZVRpbWVJbW11dGFibGUgaW5zdGFuY2UKICAgICAqIEBwYXJhbSBcRGF0ZVRpbWVab25lfFNjYWxhckZ1bmN0aW9uICR0aW1lWm9uZQogICAgICov"},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Function\/ScalarFunctionChain.php","start_line_in_file":812,"slug":"trim","name":"trim","class":"Flow\\ETL\\Function\\ScalarFunctionChain","class_slug":"scalarfunctionchain","parameters":[{"name":"type","type":[{"name":"Type","namespace":"Flow\\ETL\\Function\\Trim","is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"Flow\\ETL\\Function\\Trim\\Type::..."},{"name":"characters","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"' \\t\\n\\r\\0\u000b'"}],"return_type":[{"name":"Trim","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":true,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Function\/ScalarFunctionChain.php","start_line_in_file":817,"slug":"truncate","name":"truncate","class":"Flow\\ETL\\Function\\ScalarFunctionChain","class_slug":"scalarfunctionchain","parameters":[{"name":"length","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"int","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"ellipsis","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"'...'"}],"return_type":[{"name":"Truncate","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":true,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Function\/ScalarFunctionChain.php","start_line_in_file":822,"slug":"unicodelength","name":"unicodeLength","class":"Flow\\ETL\\Function\\ScalarFunctionChain","class_slug":"scalarfunctionchain","parameters":[],"return_type":[{"name":"UnicodeLength","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":true,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Function\/ScalarFunctionChain.php","start_line_in_file":846,"slug":"unpack","name":"unpack","class":"Flow\\ETL\\Function\\ScalarFunctionChain","class_slug":"scalarfunctionchain","parameters":[{"name":"schema","type":[{"name":"Schema","namespace":"Flow\\ETL","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"ArrayUnpack","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":true,"doc_comment":"LyoqCiAgICAgKiBVbnBhY2tzIGVhY2ggZWxlbWVudCBvZiBhbiBhcnJheSBpbnRvIGEgbmV3IGVudHJ5LCB1c2luZyB0aGUgYXJyYXkga2V5IGFzIHRoZSBlbnRyeSBuYW1lLgogICAgICoKICAgICAqIEJlZm9yZToKICAgICAqICAgKy0tKy0tLS0tLS0tLS0tLS0tLS0tLS0rCiAgICAgKiAgIHxpZHwgICAgICAgICAgICAgIGFycmF5fAogICAgICogICArLS0rLS0tLS0tLS0tLS0tLS0tLS0tLSsKICAgICAqICAgfCAxfHsiYSI6MSwiYiI6MiwiYyI6M318CiAgICAgKiAgIHwgMnx7ImQiOjQsImUiOjUsImYiOjZ9fAogICAgICogICArLS0rLS0tLS0tLS0tLS0tLS0tLS0tLSsKICAgICAqCiAgICAgKiBBZnRlcjoKICAgICAqICAgKy0tKy0tLS0tKy0tLS0tKy0tLS0tKy0tLS0tKy0tLS0tKwogICAgICogICB8aWR8YXJyLmJ8YXJyLmN8YXJyLmR8YXJyLmV8YXJyLmZ8CiAgICAgKiAgICstLSstLS0tLSstLS0tLSstLS0tLSstLS0tLSstLS0tLSsKICAgICAqICAgfCAxfCAgICAyfCAgICAzfCAgICAgfCAgICAgfCAgICAgfAogICAgICogICB8IDJ8ICAgICB8ICAgICB8ICAgIDR8ICAgIDV8ICAgIDZ8CiAgICAgKiAgICstLSstLS0tLSstLS0tLSstLS0tLSstLS0tLSstLS0tLSsKICAgICAqLw=="},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Function\/ScalarFunctionChain.php","start_line_in_file":851,"slug":"upper","name":"upper","class":"Flow\\ETL\\Function\\ScalarFunctionChain","class_slug":"scalarfunctionchain","parameters":[],"return_type":[{"name":"ToUpper","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":true,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Function\/ScalarFunctionChain.php","start_line_in_file":856,"slug":"wordwrap","name":"wordwrap","class":"Flow\\ETL\\Function\\ScalarFunctionChain","class_slug":"scalarfunctionchain","parameters":[{"name":"width","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"int","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"break","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"'\\n'"},{"name":"cut","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"bool","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"false"}],"return_type":[{"name":"Wordwrap","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":true,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Function\/ScalarFunctionChain.php","start_line_in_file":864,"slug":"xpath","name":"xpath","class":"Flow\\ETL\\Function\\ScalarFunctionChain","class_slug":"scalarfunctionchain","parameters":[{"name":"string","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"XPath","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":true,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Function\/ResolvesFromChildren.php","start_line_in_file":12,"slug":"resolved","name":"resolved","class":"Flow\\ETL\\Function\\ScalarFunctionChain","class_slug":"scalarfunctionchain","parameters":[],"return_type":[{"name":"bool","namespace":null,"is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Function\/ResolvesFromChildren.php","start_line_in_file":23,"slug":"deterministic","name":"deterministic","class":"Flow\\ETL\\Function\\ScalarFunctionChain","class_slug":"scalarfunctionchain","parameters":[],"return_type":[{"name":"bool","namespace":null,"is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Flow.php","start_line_in_file":23,"slug":"setup","name":"setUp","class":"Flow\\ETL\\Flow","class_slug":"flow","parameters":[{"name":"config","type":[{"name":"ConfigBuilder","namespace":"Flow\\ETL\\Config","is_nullable":false,"is_variadic":false},{"name":"Config","namespace":"Flow\\ETL","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"self","namespace":null,"is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Flow.php","start_line_in_file":28,"slug":"extract","name":"extract","class":"Flow\\ETL\\Flow","class_slug":"flow","parameters":[{"name":"extractor","type":[{"name":"Extractor","namespace":"Flow\\ETL","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"DataFrame","namespace":"Flow\\ETL","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Flow.php","start_line_in_file":33,"slug":"from","name":"from","class":"Flow\\ETL\\Flow","class_slug":"flow","parameters":[{"name":"extractor","type":[{"name":"Extractor","namespace":"Flow\\ETL","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"DataFrame","namespace":"Flow\\ETL","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Flow.php","start_line_in_file":38,"slug":"process","name":"process","class":"Flow\\ETL\\Flow","class_slug":"flow","parameters":[{"name":"rows","type":[{"name":"Rows","namespace":"Flow\\ETL","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":true,"default_value":null}],"return_type":[{"name":"DataFrame","namespace":"Flow\\ETL","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Flow.php","start_line_in_file":46,"slug":"read","name":"read","class":"Flow\\ETL\\Flow","class_slug":"flow","parameters":[{"name":"extractor","type":[{"name":"Extractor","namespace":"Flow\\ETL","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"DataFrame","namespace":"Flow\\ETL","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":false,"doc_comment":"LyoqCiAgICAgKiBBbGlhcyBmb3IgRmxvdzo6ZXh0cmFjdCBmdW5jdGlvbi4KICAgICAqLw=="},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DataFrame.php","start_line_in_file":74,"slug":"aggregate","name":"aggregate","class":"Flow\\ETL\\DataFrame","class_slug":"dataframe","parameters":[{"name":"aggregations","type":[{"name":"array","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"algorithm","type":[{"name":"GroupByAlgorithmBuilder","namespace":"Flow\\ETL\\Config\\Grouping","is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"null"}],"return_type":[{"name":"self","namespace":null,"is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":false,"doc_comment":"LyoqCiAgICAgKiBAbGF6eQogICAgICoKICAgICAqIEBwYXJhbSBBZ2dyZWdhdGlvbnMgJGFnZ3JlZ2F0aW9ucwogICAgICogQHBhcmFtIG51bGx8R3JvdXBCeUFsZ29yaXRobUJ1aWxkZXIgJGFsZ29yaXRobSBudWxsIGRlZmVycyB0byBjb25maWd1cmF0aW9uOyBhIGJ1aWxkZXIgcGlucyB0aGUKICAgICAqICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICBhbGdvcml0aG0gZm9yIHRoaXMgb3BlcmF0aW9uIGFuZCBza2lwcyBhbnkgYXV0b21hdGljIGNob2ljZQogICAgICov"},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DataFrame.php","start_line_in_file":98,"slug":"batchby","name":"batchBy","class":"Flow\\ETL\\DataFrame","class_slug":"dataframe","parameters":[{"name":"column","type":[{"name":"Reference","namespace":"Flow\\ETL\\Row","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"minSize","type":[{"name":"int","namespace":null,"is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"null"}],"return_type":[{"name":"self","namespace":null,"is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":false,"doc_comment":"LyoqCiAgICAgKiBNZXJnZS9TcGxpdCBSb3dzIHlpZWxkZWQgYnkgRXh0cmFjdG9yIGludG8gYmF0Y2hlcyBidXQga2VlcCB0aG9zZSB3aXRoIGNvbW1vbiB2YWx1ZSBpbiBnaXZlbiBjb2x1bW4gdG9nZXRoZXIuCiAgICAgKiBUaGlzIHdvcmtzIHByb3Blcmx5IG9ubHkgb24gc29ydGVkIGRhdGFzZXRzLgogICAgICoKICAgICAqIFdoZW4gbWluU2l6ZSBpcyBub3QgcHJvdmlkZWQsIGJhdGNoZXMgd2lsbCBiZSBjcmVhdGVkIG9ubHkgd2hlbiB0aGVyZSBpcyBhIGNoYW5nZSBpbiB2YWx1ZSBvZiB0aGUgY29sdW1uLgogICAgICogV2hlbiBtaW5TaXplIGlzIHByb3ZpZGVkLCBiYXRjaGVzIHdpbGwgYmUgY3JlYXRlZCBvbmx5IHdoZW4gdGhlcmUgaXMgYSBjaGFuZ2UgaW4gdmFsdWUgb2YgdGhlIGNvbHVtbiBvcgogICAgICogd2hlbiB0aGVyZSBhcmUgYXQgbGVhc3QgbWluU2l6ZSByb3dzIGluIHRoZSBiYXRjaC4KICAgICAqCiAgICAgKiBAcGFyYW0gUmVmZXJlbmNlfHN0cmluZyAkY29sdW1uIC0gY29sdW1uIHRvIGdyb3VwIGJ5IChhbGwgcm93cyB3aXRoIHNhbWUgdmFsdWUgc3RheSB0b2dldGhlcikKICAgICAqIEBwYXJhbSBudWxsfGludDwxLCBtYXg+ICRtaW5TaXplIC0gb3B0aW9uYWwgbWluaW11bSByb3dzIHBlciBiYXRjaCBmb3IgZWZmaWNpZW5jeQogICAgICoKICAgICAqIEBsYXp5CiAgICAgKgogICAgICogQHRocm93cyBJbnZhbGlkQXJndW1lbnRFeGNlcHRpb24KICAgICAqLw=="},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DataFrame.php","start_line_in_file":118,"slug":"batchsize","name":"batchSize","class":"Flow\\ETL\\DataFrame","class_slug":"dataframe","parameters":[{"name":"size","type":[{"name":"int","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"self","namespace":null,"is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":false,"doc_comment":"LyoqCiAgICAgKiBNZXJnZS9TcGxpdCBSb3dzIHlpZWxkZWQgYnkgRXh0cmFjdG9yIGludG8gYmF0Y2hlcyBvZiBnaXZlbiBzaXplLgogICAgICogRm9yIGV4YW1wbGUsIHdoZW4gRXh0cmFjdG9yIGlzIHlpZWxkaW5nIG9uZSByb3cgYXQgdGltZSwgdGhpcyBtZXRob2Qgd2lsbCBtZXJnZSB0aGVtIGludG8gYmF0Y2hlcyBvZiBnaXZlbiBzaXplCiAgICAgKiBiZWZvcmUgcGFzc2luZyB0aGVtIHRvIHRoZSBuZXh0IHBpcGVsaW5lIGVsZW1lbnQuCiAgICAgKiBTaW1pbGFybHkgd2hlbiBFeHRyYWN0b3IgaXMgeWllbGRpbmcgYmF0Y2hlcyBvZiByb3dzLCB0aGlzIG1ldGhvZCB3aWxsIHNwbGl0IHRoZW0gaW50byBzbWFsbGVyIGJhdGNoZXMgb2YgZ2l2ZW4KICAgICAqIHNpemUuCiAgICAgKgogICAgICogSW4gb3JkZXIgdG8gbWVyZ2UgYWxsIFJvd3MgaW50byBhIHNpbmdsZSBiYXRjaCB1c2UgRGF0YUZyYW1lOjpjb2xsZWN0KCkgbWV0aG9kIG9yIHNldCBzaXplIHRvIC0xIG9yIDAuCiAgICAgKgogICAgICogQHBhcmFtIGludDwtMSwgbWF4PiAkc2l6ZQogICAgICoKICAgICAqIEBsYXp5CiAgICAgKi8="},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DataFrame.php","start_line_in_file":150,"slug":"cache","name":"cache","class":"Flow\\ETL\\DataFrame","class_slug":"dataframe","parameters":[{"name":"id","type":[{"name":"string","namespace":null,"is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"null"},{"name":"cacheBatchSize","type":[{"name":"int","namespace":null,"is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"null"},{"name":"cache","type":[{"name":"Cache","namespace":"Flow\\ETL","is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"null"}],"return_type":[{"name":"self","namespace":null,"is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":false,"doc_comment":"LyoqCiAgICAgKiBTdGFydCBwcm9jZXNzaW5nIHJvd3MgdXAgdG8gdGhpcyBtb21lbnQgYW5kIHB1dCBlYWNoIGluc3RhbmNlIG9mIFJvd3MKICAgICAqIGludG8gcHJldmlvdXNseSBkZWZpbmVkIGNhY2hlLgogICAgICogQ2FjaGUgdHlwZSBjYW4gYmUgc2V0IHRocm91Z2ggQ29uZmlnQnVpbGRlci4KICAgICAqIEJ5IGRlZmF1bHQgZXZlcnl0aGluZyBpcyBjYWNoZWQgaW4gc3lzdGVtIHRtcCBkaXIuCiAgICAgKgogICAgICogSW1wb3J0YW50OiBjYWNoZSBiYXRjaCBzaXplIG1pZ2h0IHNpZ25pZmljYW50bHkgaW1wcm92ZSBwZXJmb3JtYW5jZSB3aGVuIHByb2Nlc3NpbmcgbGFyZ2UgYW1vdW50IG9mIHJvd3MuCiAgICAgKiBMYXJnZXIgYmF0Y2ggc2l6ZSB3aWxsIGluY3JlYXNlIG1lbW9yeSBjb25zdW1wdGlvbiBidXQgd2lsbCByZWR1Y2UgbnVtYmVyIG9mIElPIG9wZXJhdGlvbnMuCiAgICAgKiBXaGVuIG5vdCBzZXQsIHRoZSBiYXRjaCBzaXplIGlzIHRha2VuIGZyb20gdGhlIGxhc3QgRGF0YUZyYW1lOjpiYXRjaFNpemUoKSBjYWxsLgogICAgICoKICAgICAqIEBsYXp5CiAgICAgKgogICAgICogQHBhcmFtIG51bGx8c3RyaW5nICRpZAogICAgICogQHBhcmFtIG51bGx8Q2FjaGUgJGNhY2hlIHJlYWRzIG9mIHRoaXMgY2FjaGUgbXVzdCBwYXNzIHRoZSBzYW1lIGluc3RhbmNlIHRvIGZyb21fY2FjaGUoKQogICAgICoKICAgICAqIEB0aHJvd3MgSW52YWxpZEFyZ3VtZW50RXhjZXB0aW9uCiAgICAgKi8="},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DataFrame.php","start_line_in_file":167,"slug":"collect","name":"collect","class":"Flow\\ETL\\DataFrame","class_slug":"dataframe","parameters":[],"return_type":[{"name":"self","namespace":null,"is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":false,"doc_comment":"LyoqCiAgICAgKiBCZWZvcmUgdHJhbnNmb3JtaW5nIHJvd3MsIGNvbGxlY3QgdGhlbSBhbmQgbWVyZ2UgaW50byBzaW5nbGUgUm93cyBpbnN0YW5jZS4KICAgICAqIFRoaXMgbWlnaHQgbGVhZCB0byBtZW1vcnkgaXNzdWVzIHdoZW4gcHJvY2Vzc2luZyBsYXJnZSBhbW91bnQgb2Ygcm93cywgdXNlIHdpdGggY2F1dGlvbi4KICAgICAqCiAgICAgKiBAbGF6eQogICAgICov"},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DataFrame.php","start_line_in_file":186,"slug":"collectrefs","name":"collectRefs","class":"Flow\\ETL\\DataFrame","class_slug":"dataframe","parameters":[{"name":"references","type":[{"name":"References","namespace":"Flow\\ETL\\Row","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"self","namespace":null,"is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":false,"doc_comment":"LyoqCiAgICAgKiBUaGlzIG1ldGhvZCBhbGxvd3MgdG8gY29sbGVjdCByZWZlcmVuY2VzIHRvIGFsbCBlbnRyaWVzIHVzZWQgaW4gdGhpcyBwaXBlbGluZS4KICAgICAqCiAgICAgKiBgYGBwaHAKICAgICAqIChuZXcgRmxvdygpKQogICAgICogICAtPnJlYWQoRnJvbTo6Y2hhaW4oKSkKICAgICAqICAgLT5jb2xsZWN0UmVmcygkcmVmcyA9IHJlZnMoKSkKICAgICAqICAgLT5ydW4oKTsKICAgICAqIGBgYAogICAgICoKICAgICAqIEBsYXp5CiAgICAgKi8="},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DataFrame.php","start_line_in_file":193,"slug":"constrain","name":"constrain","class":"Flow\\ETL\\DataFrame","class_slug":"dataframe","parameters":[{"name":"constraint","type":[{"name":"Constraint","namespace":"Flow\\ETL","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"constraints","type":[{"name":"Constraint","namespace":"Flow\\ETL","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":true,"default_value":null}],"return_type":[{"name":"self","namespace":null,"is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DataFrame.php","start_line_in_file":204,"slug":"count","name":"count","class":"Flow\\ETL\\DataFrame","class_slug":"dataframe","parameters":[],"return_type":[{"name":"int","namespace":null,"is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":false,"doc_comment":"LyoqCiAgICAgKiBAdHJpZ2dlcgogICAgICogUmV0dXJuIHRvdGFsIGNvdW50IG9mIHJvd3MgcHJvY2Vzc2VkIGJ5IHRoaXMgcGlwZWxpbmUuCiAgICAgKi8="},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DataFrame.php","start_line_in_file":224,"slug":"crossjoin","name":"crossJoin","class":"Flow\\ETL\\DataFrame","class_slug":"dataframe","parameters":[{"name":"dataFrame","type":[{"name":"self","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"prefix","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"''"}],"return_type":[{"name":"self","namespace":null,"is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":false,"doc_comment":"LyoqCiAgICAgKiBAbGF6eQogICAgICov"},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DataFrame.php","start_line_in_file":241,"slug":"display","name":"display","class":"Flow\\ETL\\DataFrame","class_slug":"dataframe","parameters":[{"name":"limit","type":[{"name":"int","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"20"},{"name":"truncate","type":[{"name":"int","namespace":null,"is_nullable":false,"is_variadic":false},{"name":"bool","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"20"},{"name":"formatter","type":[{"name":"Formatter","namespace":"Flow\\ETL","is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"Flow\\ETL\\Formatter\\AsciiTableFormatter::..."}],"return_type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":false,"doc_comment":"LyoqCiAgICAgKiBAcGFyYW0gaW50ICRsaW1pdCBtYXhpbXVtIG51bWJlcnMgb2Ygcm93cyB0byBkaXNwbGF5CiAgICAgKiBAcGFyYW0gYm9vbHxpbnQgJHRydW5jYXRlIGZhbHNlIG9yIGlmIHNldCB0byAwIGNvbHVtbnMgYXJlIG5vdCB0cnVuY2F0ZWQsIG90aGVyd2lzZSBkZWZhdWx0IHRydW5jYXRlIHRvIDIwCiAgICAgKiAgICAgICAgICAgICAgICAgICAgICAgICAgIGNoYXJhY3RlcnMKICAgICAqIEBwYXJhbSBGb3JtYXR0ZXIgJGZvcm1hdHRlcgogICAgICoKICAgICAqIEB0cmlnZ2VyCiAgICAgKgogICAgICogQHRocm93cyBJbnZhbGlkQXJndW1lbnRFeGNlcHRpb24KICAgICAqLw=="},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DataFrame.php","start_line_in_file":268,"slug":"drop","name":"drop","class":"Flow\\ETL\\DataFrame","class_slug":"dataframe","parameters":[{"name":"entries","type":[{"name":"Reference","namespace":"Flow\\ETL\\Row","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":true,"default_value":null}],"return_type":[{"name":"self","namespace":null,"is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":false,"doc_comment":"LyoqCiAgICAgKiBEcm9wIGdpdmVuIGVudHJpZXMuCiAgICAgKgogICAgICogQGxhenkKICAgICAqLw=="},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DataFrame.php","start_line_in_file":282,"slug":"dropduplicates","name":"dropDuplicates","class":"Flow\\ETL\\DataFrame","class_slug":"dataframe","parameters":[{"name":"entries","type":[{"name":"Reference","namespace":"Flow\\ETL\\Row","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":true,"default_value":null}],"return_type":[{"name":"self","namespace":null,"is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":false,"doc_comment":"LyoqCiAgICAgKiBAcGFyYW0gUmVmZXJlbmNlfHN0cmluZyAuLi4kZW50cmllcwogICAgICoKICAgICAqIEBsYXp5CiAgICAgKgogICAgICogQHJldHVybiAkdGhpcwogICAgICov"},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DataFrame.php","start_line_in_file":289,"slug":"duplicaterow","name":"duplicateRow","class":"Flow\\ETL\\DataFrame","class_slug":"dataframe","parameters":[{"name":"condition","type":[{"name":"mixed","namespace":null,"is_nullable":true,"is_variadic":false}],"has_default_value":false,"is_nullable":true,"is_variadic":false,"default_value":null},{"name":"entries","type":[{"name":"WithEntry","namespace":"Flow\\ETL","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":true,"default_value":null}],"return_type":[{"name":"self","namespace":null,"is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DataFrame.php","start_line_in_file":309,"slug":"fetch","name":"fetch","class":"Flow\\ETL\\DataFrame","class_slug":"dataframe","parameters":[{"name":"limit","type":[{"name":"int","namespace":null,"is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"null"}],"return_type":[{"name":"Rows","namespace":"Flow\\ETL","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":false,"doc_comment":"LyoqCiAgICAgKiBCZSBhd2FyZSB0aGF0IGZldGNoIGlzIG5vdCBtZW1vcnkgc2FmZSBhbmQgd2lsbCBsb2FkIGFsbCByb3dzIGludG8gbWVtb3J5LgogICAgICogSWYgeW91IHdhbnQgdG8gc2FmZWx5IGl0ZXJhdGUgb3ZlciBSb3dzIHVzZSBvZSBvZiB0aGUgZm9sbG93aW5nIG1ldGhvZHM6LgogICAgICoKICAgICAqIERhdGFGcmFtZTo6Z2V0KCkgOiBcR2VuZXJhdG9yCiAgICAgKiBEYXRhRnJhbWU6OmdldEFzQXJyYXkoKSA6IFxHZW5lcmF0b3IKICAgICAqIERhdGFGcmFtZTo6Z2V0RWFjaCgpIDogXEdlbmVyYXRvcgogICAgICogRGF0YUZyYW1lOjpnZXRFYWNoQXNBcnJheSgpIDogXEdlbmVyYXRvcgogICAgICoKICAgICAqIEB0cmlnZ2VyCiAgICAgKgogICAgICogQHRocm93cyBJbnZhbGlkQXJndW1lbnRFeGNlcHRpb24KICAgICAqLw=="},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DataFrame.php","start_line_in_file":326,"slug":"filter","name":"filter","class":"Flow\\ETL\\DataFrame","class_slug":"dataframe","parameters":[{"name":"function","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"self","namespace":null,"is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":false,"doc_comment":"LyoqCiAgICAgKiBAbGF6eQogICAgICov"},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DataFrame.php","start_line_in_file":338,"slug":"filters","name":"filters","class":"Flow\\ETL\\DataFrame","class_slug":"dataframe","parameters":[{"name":"functions","type":[{"name":"array","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"self","namespace":null,"is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":false,"doc_comment":"LyoqCiAgICAgKiBAbGF6eQogICAgICoKICAgICAqIEBwYXJhbSBhcnJheTxTY2FsYXJGdW5jdGlvbj4gJGZ1bmN0aW9ucwogICAgICov"},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DataFrame.php","start_line_in_file":352,"slug":"foreach","name":"forEach","class":"Flow\\ETL\\DataFrame","class_slug":"dataframe","parameters":[{"name":"callback","type":[{"name":"callable","namespace":null,"is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"null"}],"return_type":[{"name":"void","namespace":null,"is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":false,"doc_comment":"LyoqCiAgICAgKiBAdHJpZ2dlcgogICAgICoKICAgICAqIEBwYXJhbSBudWxsfGNhbGxhYmxlKFJvd3MgJHJvd3MsIEZsb3dDb250ZXh0ICRjb250ZXh0KSA6IHZvaWQgJGNhbGxiYWNrCiAgICAgKi8="},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DataFrame.php","start_line_in_file":373,"slug":"get","name":"get","class":"Flow\\ETL\\DataFrame","class_slug":"dataframe","parameters":[],"return_type":[{"name":"Generator","namespace":"","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":false,"doc_comment":"LyoqCiAgICAgKiBZaWVsZHMgZWFjaCByb3cgYXMgYW4gaW5zdGFuY2Ugb2YgUm93cy4KICAgICAqCiAgICAgKiBAdHJpZ2dlcgogICAgICoKICAgICAqIEByZXR1cm4gXEdlbmVyYXRvcjxSb3dzPgogICAgICov"},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DataFrame.php","start_line_in_file":392,"slug":"getasarray","name":"getAsArray","class":"Flow\\ETL\\DataFrame","class_slug":"dataframe","parameters":[],"return_type":[{"name":"Generator","namespace":"","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":false,"doc_comment":"LyoqCiAgICAgKiBZaWVsZHMgZWFjaCByb3cgYXMgYW4gYXJyYXkuCiAgICAgKgogICAgICogQHRyaWdnZXIKICAgICAqCiAgICAgKiBAcmV0dXJuIFxHZW5lcmF0b3I8YXJyYXk8YXJyYXk8bWl4ZWQ+Pj4KICAgICAqLw=="},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DataFrame.php","start_line_in_file":411,"slug":"geteach","name":"getEach","class":"Flow\\ETL\\DataFrame","class_slug":"dataframe","parameters":[],"return_type":[{"name":"Generator","namespace":"","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":false,"doc_comment":"LyoqCiAgICAgKiBZaWVsZCBlYWNoIHJvdyBhcyBhbiBpbnN0YW5jZSBvZiBSb3cuCiAgICAgKgogICAgICogQHRyaWdnZXIKICAgICAqCiAgICAgKiBAcmV0dXJuIFxHZW5lcmF0b3I8Um93PgogICAgICov"},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DataFrame.php","start_line_in_file":432,"slug":"geteachasarray","name":"getEachAsArray","class":"Flow\\ETL\\DataFrame","class_slug":"dataframe","parameters":[],"return_type":[{"name":"Generator","namespace":"","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":false,"doc_comment":"LyoqCiAgICAgKiBZaWVsZCBlYWNoIHJvdyBhcyBhbiBhcnJheS4KICAgICAqCiAgICAgKiBAdHJpZ2dlcgogICAgICoKICAgICAqIEByZXR1cm4gXEdlbmVyYXRvcjxhcnJheTxtaXhlZD4+CiAgICAgKi8="},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DataFrame.php","start_line_in_file":453,"slug":"groupby","name":"groupBy","class":"Flow\\ETL\\DataFrame","class_slug":"dataframe","parameters":[{"name":"entries","type":[{"name":"Reference","namespace":"Flow\\ETL\\Row","is_nullable":false,"is_variadic":false},{"name":"array","namespace":null,"is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"algorithm","type":[{"name":"GroupByAlgorithmBuilder","namespace":"Flow\\ETL\\Config\\Grouping","is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"null"}],"return_type":[{"name":"GroupedDataFrame","namespace":"Flow\\ETL\\DataFrame","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":false,"doc_comment":"LyoqCiAgICAgKiBAbGF6eQogICAgICoKICAgICAqIEBwYXJhbSBHcm91cEJ5UmVmZXJlbmNlc3xSZWZlcmVuY2V8c3RyaW5nICRlbnRyaWVzIGEgc2luZ2xlIGNvbHVtbiBpcyBncm91cGVkIGJ5IG9uIGl0cyBvd24KICAgICAqIEBwYXJhbSBudWxsfEdyb3VwQnlBbGdvcml0aG1CdWlsZGVyICRhbGdvcml0aG0gbnVsbCBkZWZlcnMgdG8gY29uZmlndXJhdGlvbjsgYSBidWlsZGVyIHBpbnMgdGhlCiAgICAgKiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgYWxnb3JpdGhtIGZvciB0aGlzIG9wZXJhdGlvbiBhbmQgc2tpcHMgYW55IGF1dG9tYXRpYyBjaG9pY2UKICAgICAqLw=="},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DataFrame.php","start_line_in_file":472,"slug":"join","name":"join","class":"Flow\\ETL\\DataFrame","class_slug":"dataframe","parameters":[{"name":"dataFrame","type":[{"name":"self","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"on","type":[{"name":"Expression","namespace":"Flow\\ETL\\Join","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"type","type":[{"name":"Join","namespace":"Flow\\ETL\\Join","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"Flow\\ETL\\Join\\Join::..."},{"name":"algorithm","type":[{"name":"JoinAlgorithmBuilder","namespace":"Flow\\ETL\\Config\\Join","is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"null"}],"return_type":[{"name":"self","namespace":null,"is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":false,"doc_comment":"LyoqCiAgICAgKiBAbGF6eQogICAgICoKICAgICAqIEBwYXJhbSBudWxsfEpvaW5BbGdvcml0aG1CdWlsZGVyICRhbGdvcml0aG0gbnVsbCBkZWZlcnMgdG8gY29uZmlndXJhdGlvbjsgYSBidWlsZGVyIHBpbnMgdGhlIGFsZ29yaXRobQogICAgICogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIGZvciB0aGlzIG9wZXJhdGlvbiBhbmQgc2tpcHMgYW55IGF1dG9tYXRpYyBjaG9pY2UKICAgICAqLw=="},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DataFrame.php","start_line_in_file":500,"slug":"joineach","name":"joinEach","class":"Flow\\ETL\\DataFrame","class_slug":"dataframe","parameters":[{"name":"factory","type":[{"name":"DataFrameFactory","namespace":"Flow\\ETL","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"on","type":[{"name":"Expression","namespace":"Flow\\ETL\\Join","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"type","type":[{"name":"Join","namespace":"Flow\\ETL\\Join","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"Flow\\ETL\\Join\\Join::..."}],"return_type":[{"name":"self","namespace":null,"is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":false,"doc_comment":"LyoqCiAgICAgKiBKb2lucyBpbiBtZW1vcnkgcGVyIGJhdGNoOyBpdCBpcyBub3QgZ292ZXJuZWQgYnkgdGhlIGpvaW4gYWxnb3JpdGhtIGFuZCB0YWtlcyBubyBhbGdvcml0aG0gb3ZlcnJpZGUuCiAgICAgKgogICAgICogQGxhenkKICAgICAqCiAgICAgKiBAcGFyYW0gc3RyaW5nfEpvaW4gJHR5cGUKICAgICAqLw=="},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DataFrame.php","start_line_in_file":516,"slug":"limit","name":"limit","class":"Flow\\ETL\\DataFrame","class_slug":"dataframe","parameters":[{"name":"limit","type":[{"name":"int","namespace":null,"is_nullable":true,"is_variadic":false}],"has_default_value":false,"is_nullable":true,"is_variadic":false,"default_value":null}],"return_type":[{"name":"self","namespace":null,"is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":false,"doc_comment":"LyoqCiAgICAgKiBAbGF6eQogICAgICoKICAgICAqIEB0aHJvd3MgSW52YWxpZEFyZ3VtZW50RXhjZXB0aW9uCiAgICAgKi8="},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DataFrame.php","start_line_in_file":530,"slug":"load","name":"load","class":"Flow\\ETL\\DataFrame","class_slug":"dataframe","parameters":[{"name":"sink","type":[{"name":"Loader","namespace":"Flow\\ETL","is_nullable":false,"is_variadic":false},{"name":"Sink","namespace":"Flow\\ETL","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"self","namespace":null,"is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":false,"doc_comment":"LyoqCiAgICAgKiBAbGF6eQogICAgICov"},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DataFrame.php","start_line_in_file":576,"slug":"match","name":"match","class":"Flow\\ETL\\DataFrame","class_slug":"dataframe","parameters":[{"name":"schema","type":[{"name":"Schema","namespace":"Flow\\ETL","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"validator","type":[{"name":"SchemaValidator","namespace":"Flow\\ETL","is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"null"}],"return_type":[{"name":"self","namespace":null,"is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":false,"doc_comment":"LyoqCiAgICAgKiBAbGF6eQogICAgICoKICAgICAqIEBwYXJhbSBudWxsfFNjaGVtYVZhbGlkYXRvciAkdmFsaWRhdG9yIC0gd2hlbiBudWxsLCBTdHJpY3RWYWxpZGF0b3IgZ2V0cyBpbml0aWFsaXplZAogICAgICov"},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DataFrame.php","start_line_in_file":597,"slug":"offset","name":"offset","class":"Flow\\ETL\\DataFrame","class_slug":"dataframe","parameters":[{"name":"offset","type":[{"name":"int","namespace":null,"is_nullable":true,"is_variadic":false}],"has_default_value":false,"is_nullable":true,"is_variadic":false,"default_value":null}],"return_type":[{"name":"self","namespace":null,"is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":false,"doc_comment":"LyoqCiAgICAgKiBTa2lwIGdpdmVuIG51bWJlciBvZiByb3dzIGZyb20gdGhlIGJlZ2lubmluZyBvZiB0aGUgZGF0YXNldC4KICAgICAqIFdoZW4gJG9mZnNldCBpcyBudWxsLCBub3RoaW5nIGhhcHBlbnMgKG5vIHJvd3MgYXJlIHNraXBwZWQpLgogICAgICoKICAgICAqIFBlcmZvcm1hbmNlIE5vdGU6IERhdGFGcmFtZSBtdXN0IGl0ZXJhdGUgdGhyb3VnaCBhbmQgcHJvY2VzcyBhbGwgc2tpcHBlZCByb3dzCiAgICAgKiB0byByZWFjaCB0aGUgb2Zmc2V0IHBvc2l0aW9uLiBGb3IgbGFyZ2Ugb2Zmc2V0cywgdGhpcyBjYW4gaW1wYWN0IHBlcmZvcm1hbmNlCiAgICAgKiBhcyB0aGUgZGF0YSBzb3VyY2Ugc3RpbGwgbmVlZHMgdG8gYmUgcmVhZCBhbmQgcHJvY2Vzc2VkIHVwIHRvIHRoZSBvZmZzZXQgcG9pbnQuCiAgICAgKgogICAgICogQHBhcmFtID9pbnQ8MCwgbWF4PiAkb2Zmc2V0CiAgICAgKgogICAgICogQGxhenkKICAgICAqCiAgICAgKiBAdGhyb3dzIEludmFsaWRBcmd1bWVudEV4Y2VwdGlvbgogICAgICov"},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DataFrame.php","start_line_in_file":611,"slug":"onerror","name":"onError","class":"Flow\\ETL\\DataFrame","class_slug":"dataframe","parameters":[{"name":"handler","type":[{"name":"ErrorHandler","namespace":"Flow\\ETL","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"self","namespace":null,"is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":false,"doc_comment":"LyoqCiAgICAgKiBAbGF6eQogICAgICov"},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DataFrame.php","start_line_in_file":623,"slug":"repartition","name":"repartition","class":"Flow\\ETL\\DataFrame","class_slug":"dataframe","parameters":[{"name":"entry","type":[{"name":"Reference","namespace":"Flow\\ETL\\Row","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"entries","type":[{"name":"Reference","namespace":"Flow\\ETL\\Row","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":true,"default_value":null}],"return_type":[{"name":"self","namespace":null,"is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":false,"doc_comment":"LyoqCiAgICAgKiBAbGF6eQogICAgICogU2h1ZmZsZXMgdGhlIHN0cmVhbSBzbyBldmVyeSByb3cgc2hhcmluZyB0aGUgZ2l2ZW4gY29sdW1ucyBhcnJpdmVzIGluIG9uZSBiYXRjaC4gSXQgZG9lcyBub3QKICAgICAqIHdyaXRlIGRpcmVjdG9yaWVzIC0gdGhhdCBpcyBkZWNsYXJlZCBvbiB0aGUgbG9hZGVyLCBgdG9fY3N2KC4uLiktPnBhcnRpdGlvbkJ5KCdyZWdpb24nKWAuCiAgICAgKi8="},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DataFrame.php","start_line_in_file":635,"slug":"printrows","name":"printRows","class":"Flow\\ETL\\DataFrame","class_slug":"dataframe","parameters":[{"name":"limit","type":[{"name":"int","namespace":null,"is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"20"},{"name":"truncate","type":[{"name":"int","namespace":null,"is_nullable":false,"is_variadic":false},{"name":"bool","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"20"},{"name":"formatter","type":[{"name":"Formatter","namespace":"Flow\\ETL","is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"Flow\\ETL\\Formatter\\AsciiTableFormatter::..."}],"return_type":[{"name":"void","namespace":null,"is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":false,"doc_comment":"LyoqCiAgICAgKiBAdHJpZ2dlcgogICAgICov"},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DataFrame.php","start_line_in_file":655,"slug":"explain","name":"explain","class":"Flow\\ETL\\DataFrame","class_slug":"dataframe","parameters":[{"name":"trigger","type":[{"name":"Trigger","namespace":"Flow\\ETL\\Plan","is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"Flow\\ETL\\Plan\\Trigger::..."}],"return_type":[{"name":"Plan","namespace":"Flow\\ETL","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":false,"doc_comment":"LyoqCiAgICAgKiBUaGUgcGxhbiAkdHJpZ2dlciB3b3VsZCBydW4gb3ZlciB0aGlzIGZyYW1lLCBmcm96ZW46IGxhdGVyIHZlcmJzIG9uIHRoaXMgZnJhbWUgZG8gbm90IHJlYWNoIGl0LiBUaGUgZGVmYXVsdAogICAgICogYWRkcyBubyBjb25zdW1lciBvZiBpdHMgb3duIC0gaXQgZHJhd3MgdGhlIGZyYW1lIGFzIGJ1aWx0LCBpdHMgY2hhaW4gYW5kIGl0cyBzaW5rcy4gdG9TdHJpbmcoKSBwcmludHMgaXQgYXMgYQogICAgICogdHJlZS4gQW5zd2VycyBmcm9tIHRoZSBwbGFuIHdpdGhvdXQgcmVhZGluZyBhIHJvdy4KICAgICAqLw=="},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DataFrame.php","start_line_in_file":665,"slug":"printschema","name":"printSchema","class":"Flow\\ETL\\DataFrame","class_slug":"dataframe","parameters":[{"name":"formatter","type":[{"name":"SchemaFormatter","namespace":"Flow\\ETL\\Schema","is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"Flow\\ETL\\Row\\Formatter\\ASCIISchemaFormatter::..."}],"return_type":[{"name":"void","namespace":null,"is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":false,"doc_comment":"LyoqCiAgICAgKiBAbGF6eQogICAgICoKICAgICAqIEB0aHJvd3MgU2NoZW1hTm90RGVyaXZhYmxlRXhjZXB0aW9uCiAgICAgKi8="},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DataFrame.php","start_line_in_file":673,"slug":"rename","name":"rename","class":"Flow\\ETL\\DataFrame","class_slug":"dataframe","parameters":[{"name":"from","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"to","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"self","namespace":null,"is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":false,"doc_comment":"LyoqCiAgICAgKiBAbGF6eQogICAgICov"},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DataFrame.php","start_line_in_file":680,"slug":"renameeach","name":"renameEach","class":"Flow\\ETL\\DataFrame","class_slug":"dataframe","parameters":[{"name":"strategies","type":[{"name":"RenameEntryStrategy","namespace":"Flow\\ETL\\Transformer\\Rename","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":true,"default_value":null}],"return_type":[{"name":"self","namespace":null,"is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DataFrame.php","start_line_in_file":691,"slug":"rows","name":"rows","class":"Flow\\ETL\\DataFrame","class_slug":"dataframe","parameters":[{"name":"transformer","type":[{"name":"Transformer","namespace":"Flow\\ETL","is_nullable":false,"is_variadic":false},{"name":"Transformation","namespace":"Flow\\ETL","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"self","namespace":null,"is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":false,"doc_comment":"LyoqCiAgICAgKiBAbGF6eQogICAgICogQWxpYXMgZm9yIEVUTDo6dHJhbnNmb3JtIG1ldGhvZC4KICAgICAqLw=="},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DataFrame.php","start_line_in_file":708,"slug":"run","name":"run","class":"Flow\\ETL\\DataFrame","class_slug":"dataframe","parameters":[{"name":"analyze","type":[{"name":"Analyze","namespace":"Flow\\ETL","is_nullable":false,"is_variadic":false},{"name":"bool","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"false"}],"return_type":[{"name":"Report","namespace":"Flow\\ETL\\Dataset","is_nullable":true,"is_variadic":false}],"attributes":[],"scalar_function_chain":false,"doc_comment":"LyoqCiAgICAgKiBAdHJpZ2dlcgogICAgICoKICAgICAqIFdoZW4gYW5hbHl6aW5nIHBpcGVsaW5lIGV4ZWN1dGlvbiB3ZSBjYW4gY2hvc2UgdG8gY29sbGVjdCB2YXJpb3VzIG1ldHJpY3MgdGhyb3VnaCBhbmFseXplKCktPndpdGgqKCkgbWV0aG9kCiAgICAgKgogICAgICogLSBjb2x1bW4gc3RhdGlzdGljcyAtIGFuYWx5emUoKS0+d2l0aENvbHVtblN0YXRpc3RpY3MoKQogICAgICogLSBzY2hlbWEgLSBhbmFseXplKCktPndpdGhTY2hlbWEoKQogICAgICoKICAgICAqIEBwYXJhbSBBbmFseXplfGJvb2wgJGFuYWx5emUgLSB3aGVuIHNldCBydW4gd2lsbCByZXR1cm4gUmVwb3J0CiAgICAgKgogICAgICogQHJldHVybiAoJGFuYWx5emUgaXMgQW5hbHl6ZXx0cnVlID8gUmVwb3J0IDogbnVsbCkKICAgICAqLw=="},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DataFrame.php","start_line_in_file":736,"slug":"schema","name":"schema","class":"Flow\\ETL\\DataFrame","class_slug":"dataframe","parameters":[],"return_type":[{"name":"Schema","namespace":"Flow\\ETL","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":false,"doc_comment":"LyoqCiAgICAgKiBAbGF6eQogICAgICoKICAgICAqIEB0aHJvd3MgU2NoZW1hTm90RGVyaXZhYmxlRXhjZXB0aW9uCiAgICAgKi8="},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DataFrame.php","start_line_in_file":749,"slug":"select","name":"select","class":"Flow\\ETL\\DataFrame","class_slug":"dataframe","parameters":[{"name":"entries","type":[{"name":"Reference","namespace":"Flow\\ETL\\Row","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":true,"default_value":null}],"return_type":[{"name":"self","namespace":null,"is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":false,"doc_comment":"LyoqCiAgICAgKiBAbGF6eQogICAgICogS2VlcCBvbmx5IGdpdmVuIGVudHJpZXMuCiAgICAgKi8="},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DataFrame.php","start_line_in_file":763,"slug":"sortby","name":"sortBy","class":"Flow\\ETL\\DataFrame","class_slug":"dataframe","parameters":[{"name":"entries","type":[{"name":"Reference","namespace":"Flow\\ETL\\Row","is_nullable":false,"is_variadic":false},{"name":"array","namespace":null,"is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"algorithm","type":[{"name":"SortAlgorithmBuilder","namespace":"Flow\\ETL\\Config\\Sort","is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"null"}],"return_type":[{"name":"self","namespace":null,"is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":false,"doc_comment":"LyoqCiAgICAgKiBAbGF6eQogICAgICoKICAgICAqIEBwYXJhbSBSZWZlcmVuY2V8U29ydFJlZmVyZW5jZXN8c3RyaW5nICRlbnRyaWVzIGEgc2luZ2xlIGNvbHVtbiBpcyBzb3J0ZWQgYnkgb24gaXRzIG93bgogICAgICogQHBhcmFtIG51bGx8U29ydEFsZ29yaXRobUJ1aWxkZXIgJGFsZ29yaXRobSBudWxsIGRlZmVycyB0byBjb25maWd1cmF0aW9uOyBhIGJ1aWxkZXIgcGlucyB0aGUgYWxnb3JpdGhtCiAgICAgKiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgZm9yIHRoaXMgb3BlcmF0aW9uIGFuZCBza2lwcyBhbnkgYXV0b21hdGljIGNob2ljZQogICAgICov"},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DataFrame.php","start_line_in_file":777,"slug":"transform","name":"transform","class":"Flow\\ETL\\DataFrame","class_slug":"dataframe","parameters":[{"name":"transformer","type":[{"name":"Transformer","namespace":"Flow\\ETL","is_nullable":false,"is_variadic":false},{"name":"Transformation","namespace":"Flow\\ETL","is_nullable":false,"is_variadic":false},{"name":"Transformations","namespace":"Flow\\ETL","is_nullable":false,"is_variadic":false},{"name":"WithEntry","namespace":"Flow\\ETL","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"self","namespace":null,"is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":false,"doc_comment":"LyoqCiAgICAgKiBBbGlhcyBmb3IgRGF0YUZyYW1lOjp3aXRoKCkuCiAgICAgKgogICAgICogQGxhenkKICAgICAqLw=="},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DataFrame.php","start_line_in_file":788,"slug":"until","name":"until","class":"Flow\\ETL\\DataFrame","class_slug":"dataframe","parameters":[{"name":"function","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"self","namespace":null,"is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":false,"doc_comment":"LyoqCiAgICAgKiBUaGUgZGlmZmVyZW5jZSBiZXR3ZWVuIGZpbHRlciBhbmQgdW50aWwgaXMgdGhhdCBmaWx0ZXIgd2lsbCBrZWVwIGZpbHRlcmluZyByb3dzIHVudGlsIGV4dHJhY3RvcnMgZmluaXNoIHlpZWxkaW5nCiAgICAgKiByb3dzLiBVbnRpbCB3aWxsIHNlbmQgYSBTVE9QIHNpZ25hbCB0byB0aGUgRXh0cmFjdG9yIHdoZW4gdGhlIGNvbmRpdGlvbiBpcyBub3QgbWV0LgogICAgICoKICAgICAqIEBsYXp5CiAgICAgKi8="},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DataFrame.php","start_line_in_file":802,"slug":"void","name":"void","class":"Flow\\ETL\\DataFrame","class_slug":"dataframe","parameters":[],"return_type":[{"name":"self","namespace":null,"is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":false,"doc_comment":"LyoqCiAgICAgKiBAbGF6eQogICAgICogVGhpcyBtZXRob2QgaXMgdXNlZnVsIG1vc3RseSBpbiBkZXZlbG9wbWVudCB3aGVuCiAgICAgKiB5b3Ugd2FudCB0byBwYXVzZSBwcm9jZXNzaW5nIGF0IGNlcnRhaW4gbW9tZW50IHdpdGhvdXQKICAgICAqIHJlbW92aW5nIGNvZGUuIEFsbCBvcGVyYXRpb25zIHdpbGwgZ2V0IHByb2Nlc3NlZCB1cCB0byB0aGlzIHBvaW50LAogICAgICogZnJvbSBoZXJlIG5vIHJvd3MgYXJlIHBhc3NlZCBmb3J3YXJkLgogICAgICov"},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DataFrame.php","start_line_in_file":812,"slug":"with","name":"with","class":"Flow\\ETL\\DataFrame","class_slug":"dataframe","parameters":[{"name":"transformer","type":[{"name":"Transformer","namespace":"Flow\\ETL","is_nullable":false,"is_variadic":false},{"name":"Transformation","namespace":"Flow\\ETL","is_nullable":false,"is_variadic":false},{"name":"Transformations","namespace":"Flow\\ETL","is_nullable":false,"is_variadic":false},{"name":"WithEntry","namespace":"Flow\\ETL","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"self","namespace":null,"is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":false,"doc_comment":"LyoqCiAgICAgKiBAbGF6eQogICAgICov"},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DataFrame.php","start_line_in_file":840,"slug":"withentries","name":"withEntries","class":"Flow\\ETL\\DataFrame","class_slug":"dataframe","parameters":[{"name":"references","type":[{"name":"array","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"self","namespace":null,"is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":false,"doc_comment":"LyoqCiAgICAgKiBAbGF6eQogICAgICoKICAgICAqIEBwYXJhbSBhcnJheTxpbnQsIFdpdGhFbnRyeT58YXJyYXk8c3RyaW5nLCBTY2FsYXJGdW5jdGlvbnxXaW5kb3dGdW5jdGlvbnxXaXRoRW50cnk+ICRyZWZlcmVuY2VzCiAgICAgKi8="},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DataFrame.php","start_line_in_file":858,"slug":"withentry","name":"withEntry","class":"Flow\\ETL\\DataFrame","class_slug":"dataframe","parameters":[{"name":"entry","type":[{"name":"Definition","namespace":"Flow\\ETL\\Schema","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"reference","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"WindowFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"self","namespace":null,"is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":false,"doc_comment":"LyoqCiAgICAgKiBAcGFyYW0gRGVmaW5pdGlvbjxtaXhlZD58c3RyaW5nICRlbnRyeQogICAgICoKICAgICAqIEBsYXp5CiAgICAgKi8="},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DataFrame.php","start_line_in_file":871,"slug":"write","name":"write","class":"Flow\\ETL\\DataFrame","class_slug":"dataframe","parameters":[{"name":"sink","type":[{"name":"Loader","namespace":"Flow\\ETL","is_nullable":false,"is_variadic":false},{"name":"Sink","namespace":"Flow\\ETL","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"self","namespace":null,"is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":false,"doc_comment":"LyoqCiAgICAgKiBAbGF6eQogICAgICogQWxpYXMgZm9yIEVUTDo6bG9hZCBmdW5jdGlvbi4KICAgICAqLw=="},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DataFrame\/GroupedDataFrame.php","start_line_in_file":21,"slug":"aggregate","name":"aggregate","class":"Flow\\ETL\\DataFrame\\GroupedDataFrame","class_slug":"groupeddataframe","parameters":[{"name":"aggregations","type":[{"name":"AggregatingFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":true,"default_value":null}],"return_type":[{"name":"DataFrame","namespace":"Flow\\ETL","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DataFrame\/GroupedDataFrame.php","start_line_in_file":28,"slug":"pivot","name":"pivot","class":"Flow\\ETL\\DataFrame\\GroupedDataFrame","class_slug":"groupeddataframe","parameters":[{"name":"ref","type":[{"name":"Reference","namespace":"Flow\\ETL\\Row","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"values","type":[{"name":"PivotValues","namespace":"Flow\\ETL\\GroupBy","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"self","namespace":null,"is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":false,"doc_comment":null}] \ No newline at end of file +[{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Function\/ScalarFunctionChain.php","start_line_in_file":31,"slug":"and","name":"and","class":"Flow\\ETL\\Function\\ScalarFunctionChain","class_slug":"scalarfunctionchain","parameters":[{"name":"function","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"All","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Function\/ScalarFunctionChain.php","start_line_in_file":36,"slug":"andnot","name":"andNot","class":"Flow\\ETL\\Function\\ScalarFunctionChain","class_slug":"scalarfunctionchain","parameters":[{"name":"function","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"All","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Function\/ScalarFunctionChain.php","start_line_in_file":41,"slug":"append","name":"append","class":"Flow\\ETL\\Function\\ScalarFunctionChain","class_slug":"scalarfunctionchain","parameters":[{"name":"suffix","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"Append","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":true,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Function\/ScalarFunctionChain.php","start_line_in_file":54,"slug":"arrayfilter","name":"arrayFilter","class":"Flow\\ETL\\Function\\ScalarFunctionChain","class_slug":"scalarfunctionchain","parameters":[{"name":"value","type":[{"name":"mixed","namespace":null,"is_nullable":true,"is_variadic":false}],"has_default_value":false,"is_nullable":true,"is_variadic":false,"default_value":null}],"return_type":[{"name":"ArrayFilter","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":true,"doc_comment":"LyoqCiAgICAgKiBGaWx0ZXJzIGFuIGFycmF5IGJ5IHJlbW92aW5nIGFsbCBlbGVtZW50cyB0aGF0IG1hdGNoZXMgcGFzc2VkIHZhbHVlLgogICAgICogQXBwbGljYWJsZSB0byBhbGwgZGF0YSBzdHJ1Y3R1cmVzIHRoYXQgY2FuIGJlIGNvbnZlcnRlZCB0byBhbiBhcnJheToKICAgICAqICAgIC0ganNvbgogICAgICogICAgLSBsaXN0CiAgICAgKiAgICAtIG1hcAogICAgICogICAgLSBzdHJ1Y3R1cmUuCiAgICAgKi8="},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Function\/ScalarFunctionChain.php","start_line_in_file":59,"slug":"arrayget","name":"arrayGet","class":"Flow\\ETL\\Function\\ScalarFunctionChain","class_slug":"scalarfunctionchain","parameters":[{"name":"path","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"ArrayGet","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":true,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Function\/ScalarFunctionChain.php","start_line_in_file":67,"slug":"arraygetcollection","name":"arrayGetCollection","class":"Flow\\ETL\\Function\\ScalarFunctionChain","class_slug":"scalarfunctionchain","parameters":[{"name":"keys","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"array","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"ArrayGetCollection","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":true,"doc_comment":"LyoqCiAgICAgKiBAcGFyYW0gU2NhbGFyRnVuY3Rpb258YXJyYXk8YXJyYXkta2V5LCBtaXhlZD4gJGtleXMKICAgICAqLw=="},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Function\/ScalarFunctionChain.php","start_line_in_file":72,"slug":"arraygetcollectionfirst","name":"arrayGetCollectionFirst","class":"Flow\\ETL\\Function\\ScalarFunctionChain","class_slug":"scalarfunctionchain","parameters":[{"name":"keys","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":true,"default_value":null}],"return_type":[{"name":"ArrayGetCollection","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":true,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Function\/ScalarFunctionChain.php","start_line_in_file":85,"slug":"arraykeep","name":"arrayKeep","class":"Flow\\ETL\\Function\\ScalarFunctionChain","class_slug":"scalarfunctionchain","parameters":[{"name":"value","type":[{"name":"mixed","namespace":null,"is_nullable":true,"is_variadic":false}],"has_default_value":false,"is_nullable":true,"is_variadic":false,"default_value":null}],"return_type":[{"name":"ArrayKeep","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":true,"doc_comment":"LyoqCiAgICAgKiBGaWx0ZXJzIGFuIGFycmF5IGJ5IGtlZXBpbmcgb25seSBlbGVtZW50cyB0aGF0IG1hdGNoZXMgcGFzc2VkIHZhbHVlLgogICAgICogQXBwbGljYWJsZSB0byBhbGwgZGF0YSBzdHJ1Y3R1cmVzIHRoYXQgY2FuIGJlIGNvbnZlcnRlZCB0byBhbiBhcnJheToKICAgICAqICAgLSBqc29uCiAgICAgKiAgIC0gbGlzdAogICAgICogICAtIG1hcAogICAgICogICAtIHN0cnVjdHVyZS4KICAgICAqLw=="},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Function\/ScalarFunctionChain.php","start_line_in_file":98,"slug":"arraykeys","name":"arrayKeys","class":"Flow\\ETL\\Function\\ScalarFunctionChain","class_slug":"scalarfunctionchain","parameters":[],"return_type":[{"name":"ArrayKeys","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":true,"doc_comment":"LyoqCiAgICAgKiBSZXR1cm5zIGFsbCBrZXlzIGZyb20gYW4gYXJyYXksIGlnbm9yaW5nIHRoZSB2YWx1ZXMuCiAgICAgKiBBcHBsaWNhYmxlIHRvIGFsbCBkYXRhIHN0cnVjdHVyZXMgdGhhdCBjYW4gYmUgY29udmVydGVkIHRvIGFuIGFycmF5OgogICAgICogICAtIGpzb24KICAgICAqICAgLSBsaXN0CiAgICAgKiAgIC0gbWFwCiAgICAgKiAgIC0gc3RydWN0dXJlLgogICAgICov"},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Function\/ScalarFunctionChain.php","start_line_in_file":106,"slug":"arraymerge","name":"arrayMerge","class":"Flow\\ETL\\Function\\ScalarFunctionChain","class_slug":"scalarfunctionchain","parameters":[{"name":"ref","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"array","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"ArrayMerge","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":true,"doc_comment":"LyoqCiAgICAgKiBAcGFyYW0gU2NhbGFyRnVuY3Rpb258YXJyYXk8YXJyYXkta2V5LCBtaXhlZD4gJHJlZgogICAgICov"},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Function\/ScalarFunctionChain.php","start_line_in_file":111,"slug":"arraymergecollection","name":"arrayMergeCollection","class":"Flow\\ETL\\Function\\ScalarFunctionChain","class_slug":"scalarfunctionchain","parameters":[],"return_type":[{"name":"ArrayMergeCollection","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":true,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Function\/ScalarFunctionChain.php","start_line_in_file":116,"slug":"arraypathexists","name":"arrayPathExists","class":"Flow\\ETL\\Function\\ScalarFunctionChain","class_slug":"scalarfunctionchain","parameters":[{"name":"path","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"ArrayPathExists","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":true,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Function\/ScalarFunctionChain.php","start_line_in_file":121,"slug":"arrayreverse","name":"arrayReverse","class":"Flow\\ETL\\Function\\ScalarFunctionChain","class_slug":"scalarfunctionchain","parameters":[{"name":"preserveKeys","type":[{"name":"bool","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"false"}],"return_type":[{"name":"ArrayReverse","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":true,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Function\/ScalarFunctionChain.php","start_line_in_file":126,"slug":"arraysort","name":"arraySort","class":"Flow\\ETL\\Function\\ScalarFunctionChain","class_slug":"scalarfunctionchain","parameters":[{"name":"sortFunction","type":[{"name":"Sort","namespace":"Flow\\ETL\\Function\\ArraySort","is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"null"},{"name":"flags","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"int","namespace":null,"is_nullable":false,"is_variadic":false},{"name":"null","namespace":null,"is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"null"},{"name":"recursive","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"bool","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"true"}],"return_type":[{"name":"ArraySort","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":true,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Function\/ScalarFunctionChain.php","start_line_in_file":142,"slug":"arrayvalues","name":"arrayValues","class":"Flow\\ETL\\Function\\ScalarFunctionChain","class_slug":"scalarfunctionchain","parameters":[],"return_type":[{"name":"ArrayValues","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":true,"doc_comment":"LyoqCiAgICAgKiBSZXR1cm5zIGFsbCB2YWx1ZXMgZnJvbSBhbiBhcnJheSwgaWdub3JpbmcgdGhlIGtleXMuCiAgICAgKiBBcHBsaWNhYmxlIHRvIGFsbCBkYXRhIHN0cnVjdHVyZXMgdGhhdCBjYW4gYmUgY29udmVydGVkIHRvIGFuIGFycmF5OgogICAgICogICAtIGpzb24KICAgICAqICAgLSBsaXN0CiAgICAgKiAgIC0gbWFwCiAgICAgKiAgIC0gc3RydWN0dXJlLgogICAgICov"},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Function\/ScalarFunctionChain.php","start_line_in_file":147,"slug":"ascii","name":"ascii","class":"Flow\\ETL\\Function\\ScalarFunctionChain","class_slug":"scalarfunctionchain","parameters":[],"return_type":[{"name":"Ascii","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":true,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Function\/ScalarFunctionChain.php","start_line_in_file":157,"slug":"between","name":"between","class":"Flow\\ETL\\Function\\ScalarFunctionChain","class_slug":"scalarfunctionchain","parameters":[{"name":"lowerBoundRef","type":[{"name":"mixed","namespace":null,"is_nullable":true,"is_variadic":false}],"has_default_value":false,"is_nullable":true,"is_variadic":false,"default_value":null},{"name":"upperBoundRef","type":[{"name":"mixed","namespace":null,"is_nullable":true,"is_variadic":false}],"has_default_value":false,"is_nullable":true,"is_variadic":false,"default_value":null},{"name":"boundary","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"Boundary","namespace":"Flow\\ETL\\Function\\Between","is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"Flow\\ETL\\Function\\Between\\Boundary::..."}],"return_type":[{"name":"Between","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":true,"doc_comment":"LyoqCiAgICAgKiBAcGFyYW0gbWl4ZWR8U2NhbGFyRnVuY3Rpb24gJGxvd2VyQm91bmRSZWYKICAgICAqIEBwYXJhbSBtaXhlZHxTY2FsYXJGdW5jdGlvbiAkdXBwZXJCb3VuZFJlZgogICAgICogQHBhcmFtIEJvdW5kYXJ5fFNjYWxhckZ1bmN0aW9uICRib3VuZGFyeQogICAgICov"},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Function\/ScalarFunctionChain.php","start_line_in_file":165,"slug":"binarylength","name":"binaryLength","class":"Flow\\ETL\\Function\\ScalarFunctionChain","class_slug":"scalarfunctionchain","parameters":[],"return_type":[{"name":"BinaryLength","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":true,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Function\/ScalarFunctionChain.php","start_line_in_file":174,"slug":"call","name":"call","class":"Flow\\ETL\\Function\\ScalarFunctionChain","class_slug":"scalarfunctionchain","parameters":[{"name":"callable","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"returnType","type":[{"name":"Type","namespace":"Flow\\Types","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"arguments","type":[{"name":"array","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"[]"},{"name":"refAlias","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false},{"name":"int","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"0"}],"return_type":[{"name":"CallUserFunc","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":true,"doc_comment":"LyoqCiAgICAgKiBAcGFyYW0gYXJyYXk8YXJyYXkta2V5LCBtaXhlZD4gJGFyZ3VtZW50cwogICAgICogQHBhcmFtIFR5cGU8bWl4ZWQ+ICRyZXR1cm5UeXBlCiAgICAgKi8="},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Function\/ScalarFunctionChain.php","start_line_in_file":183,"slug":"capitalize","name":"capitalize","class":"Flow\\ETL\\Function\\ScalarFunctionChain","class_slug":"scalarfunctionchain","parameters":[],"return_type":[{"name":"Capitalize","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":true,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Function\/ScalarFunctionChain.php","start_line_in_file":191,"slug":"cast","name":"cast","class":"Flow\\ETL\\Function\\ScalarFunctionChain","class_slug":"scalarfunctionchain","parameters":[{"name":"type","type":[{"name":"Type","namespace":"Flow\\Types","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"Cast","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":true,"doc_comment":"LyoqCiAgICAgKiBAcGFyYW0gc3RyaW5nfFR5cGU8bWl4ZWQ+ICR0eXBlCiAgICAgKi8="},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Function\/ScalarFunctionChain.php","start_line_in_file":196,"slug":"chunk","name":"chunk","class":"Flow\\ETL\\Function\\ScalarFunctionChain","class_slug":"scalarfunctionchain","parameters":[{"name":"size","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"int","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"Chunk","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":true,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Function\/ScalarFunctionChain.php","start_line_in_file":201,"slug":"coalesce","name":"coalesce","class":"Flow\\ETL\\Function\\ScalarFunctionChain","class_slug":"scalarfunctionchain","parameters":[{"name":"params","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":true,"default_value":null}],"return_type":[{"name":"Coalesce","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":true,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Function\/ScalarFunctionChain.php","start_line_in_file":206,"slug":"codepointlength","name":"codePointLength","class":"Flow\\ETL\\Function\\ScalarFunctionChain","class_slug":"scalarfunctionchain","parameters":[],"return_type":[{"name":"CodePointLength","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":true,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Function\/ScalarFunctionChain.php","start_line_in_file":211,"slug":"collapsewhitespace","name":"collapseWhitespace","class":"Flow\\ETL\\Function\\ScalarFunctionChain","class_slug":"scalarfunctionchain","parameters":[],"return_type":[{"name":"CollapseWhitespace","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":true,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Function\/ScalarFunctionChain.php","start_line_in_file":216,"slug":"concat","name":"concat","class":"Flow\\ETL\\Function\\ScalarFunctionChain","class_slug":"scalarfunctionchain","parameters":[{"name":"params","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":true,"default_value":null}],"return_type":[{"name":"Concat","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":true,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Function\/ScalarFunctionChain.php","start_line_in_file":221,"slug":"concatwithseparator","name":"concatWithSeparator","class":"Flow\\ETL\\Function\\ScalarFunctionChain","class_slug":"scalarfunctionchain","parameters":[{"name":"separator","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"params","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":true,"default_value":null}],"return_type":[{"name":"ConcatWithSeparator","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":true,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Function\/ScalarFunctionChain.php","start_line_in_file":228,"slug":"contains","name":"contains","class":"Flow\\ETL\\Function\\ScalarFunctionChain","class_slug":"scalarfunctionchain","parameters":[{"name":"needle","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"Contains","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":true,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Function\/ScalarFunctionChain.php","start_line_in_file":233,"slug":"dateformat","name":"dateFormat","class":"Flow\\ETL\\Function\\ScalarFunctionChain","class_slug":"scalarfunctionchain","parameters":[{"name":"format","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"'Y-m-d'"}],"return_type":[{"name":"DateTimeFormat","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":true,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Function\/ScalarFunctionChain.php","start_line_in_file":238,"slug":"datetimeformat","name":"dateTimeFormat","class":"Flow\\ETL\\Function\\ScalarFunctionChain","class_slug":"scalarfunctionchain","parameters":[{"name":"format","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"'Y-m-d H:i:s'"}],"return_type":[{"name":"DateTimeFormat","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":true,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Function\/ScalarFunctionChain.php","start_line_in_file":243,"slug":"divide","name":"divide","class":"Flow\\ETL\\Function\\ScalarFunctionChain","class_slug":"scalarfunctionchain","parameters":[{"name":"value","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false},{"name":"int","namespace":null,"is_nullable":false,"is_variadic":false},{"name":"float","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"scale","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"int","namespace":null,"is_nullable":false,"is_variadic":false},{"name":"null","namespace":null,"is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"null"},{"name":"rounding","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"Rounding","namespace":"Flow\\Calculator","is_nullable":false,"is_variadic":false},{"name":"null","namespace":null,"is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"null"}],"return_type":[{"name":"Divide","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":true,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Function\/ScalarFunctionChain.php","start_line_in_file":251,"slug":"domelementattributescount","name":"domElementAttributesCount","class":"Flow\\ETL\\Function\\ScalarFunctionChain","class_slug":"scalarfunctionchain","parameters":[],"return_type":[{"name":"DOMElementAttributesCount","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":true,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Function\/ScalarFunctionChain.php","start_line_in_file":256,"slug":"domelementattributevalue","name":"domElementAttributeValue","class":"Flow\\ETL\\Function\\ScalarFunctionChain","class_slug":"scalarfunctionchain","parameters":[{"name":"attribute","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"DOMElementAttributeValue","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":true,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Function\/ScalarFunctionChain.php","start_line_in_file":261,"slug":"domelementnamespace","name":"domElementNamespace","class":"Flow\\ETL\\Function\\ScalarFunctionChain","class_slug":"scalarfunctionchain","parameters":[{"name":"attribute","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false},{"name":"null","namespace":null,"is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"null"}],"return_type":[{"name":"DOMElementNamespaceValue","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":true,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Function\/ScalarFunctionChain.php","start_line_in_file":266,"slug":"domelementnextsibling","name":"domElementNextSibling","class":"Flow\\ETL\\Function\\ScalarFunctionChain","class_slug":"scalarfunctionchain","parameters":[],"return_type":[{"name":"DOMElementNextSibling","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":true,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Function\/ScalarFunctionChain.php","start_line_in_file":271,"slug":"domelementparent","name":"domElementParent","class":"Flow\\ETL\\Function\\ScalarFunctionChain","class_slug":"scalarfunctionchain","parameters":[],"return_type":[{"name":"DOMElementParent","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":true,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Function\/ScalarFunctionChain.php","start_line_in_file":276,"slug":"domelementprevioussibling","name":"domElementPreviousSibling","class":"Flow\\ETL\\Function\\ScalarFunctionChain","class_slug":"scalarfunctionchain","parameters":[],"return_type":[{"name":"DOMElementPreviousSibling","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":true,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Function\/ScalarFunctionChain.php","start_line_in_file":281,"slug":"domelementvalue","name":"domElementValue","class":"Flow\\ETL\\Function\\ScalarFunctionChain","class_slug":"scalarfunctionchain","parameters":[],"return_type":[{"name":"DOMElementValue","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":true,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Function\/ScalarFunctionChain.php","start_line_in_file":286,"slug":"endswith","name":"endsWith","class":"Flow\\ETL\\Function\\ScalarFunctionChain","class_slug":"scalarfunctionchain","parameters":[{"name":"needle","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"EndsWith","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":true,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Function\/ScalarFunctionChain.php","start_line_in_file":291,"slug":"ensureend","name":"ensureEnd","class":"Flow\\ETL\\Function\\ScalarFunctionChain","class_slug":"scalarfunctionchain","parameters":[{"name":"suffix","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"EnsureEnd","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":true,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Function\/ScalarFunctionChain.php","start_line_in_file":296,"slug":"ensurestart","name":"ensureStart","class":"Flow\\ETL\\Function\\ScalarFunctionChain","class_slug":"scalarfunctionchain","parameters":[{"name":"prefix","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"EnsureStart","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":true,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Function\/ScalarFunctionChain.php","start_line_in_file":301,"slug":"enumname","name":"enumName","class":"Flow\\ETL\\Function\\ScalarFunctionChain","class_slug":"scalarfunctionchain","parameters":[],"return_type":[{"name":"EnumName","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":true,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Function\/ScalarFunctionChain.php","start_line_in_file":306,"slug":"enumvalue","name":"enumValue","class":"Flow\\ETL\\Function\\ScalarFunctionChain","class_slug":"scalarfunctionchain","parameters":[],"return_type":[{"name":"EnumValue","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":true,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Function\/ScalarFunctionChain.php","start_line_in_file":311,"slug":"equals","name":"equals","class":"Flow\\ETL\\Function\\ScalarFunctionChain","class_slug":"scalarfunctionchain","parameters":[{"name":"ref","type":[{"name":"mixed","namespace":null,"is_nullable":true,"is_variadic":false}],"has_default_value":false,"is_nullable":true,"is_variadic":false,"default_value":null}],"return_type":[{"name":"Equals","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":true,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Function\/ScalarFunctionChain.php","start_line_in_file":316,"slug":"exists","name":"exists","class":"Flow\\ETL\\Function\\ScalarFunctionChain","class_slug":"scalarfunctionchain","parameters":[],"return_type":[{"name":"Exists","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":true,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Function\/ScalarFunctionChain.php","start_line_in_file":345,"slug":"expand","name":"expand","class":"Flow\\ETL\\Function\\ScalarFunctionChain","class_slug":"scalarfunctionchain","parameters":[{"name":"expand","type":[{"name":"ArrayExpand","namespace":"Flow\\ETL\\Function\\ArrayExpand","is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"Flow\\ETL\\Function\\ArrayExpand\\ArrayExpand::..."}],"return_type":[{"name":"ArrayExpand","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":true,"doc_comment":"LyoqCiAgICAgKiBFeHBhbmRzIGVhY2ggdmFsdWUgaW50byBlbnRyeSwgaWYgdGhlcmUgYXJlIG1vcmUgdGhhbiBvbmUgdmFsdWUsIG11bHRpcGxlIHJvd3Mgd2lsbCBiZSBjcmVhdGVkLgogICAgICogQXJyYXkga2V5cyBhcmUgaWdub3JlZCwgb25seSB2YWx1ZXMgYXJlIHVzZWQgdG8gY3JlYXRlIG5ldyByb3dzLgogICAgICogTmVzdGVkIGluIGFub3RoZXIgZnVuY3Rpb24gKHN0cnVjdHVyZSgpLCBjb25jYXQoKSwgLi4uKSBpdCBzdGlsbCBnaXZlcyBvbmUgcm93IHBlciBlbGVtZW50LiBTZXZlcmFsCiAgICAgKiBleHBhbmRzIGluIG9uZSBleHByZXNzaW9uIGFyZSB6aXBwZWQgdG8gdGhlIGxvbmdlc3QgbGlzdDsgYSBzaG9ydGVyIG9uZSBnaXZlcyBudWxsLCBzbyBpdHMgZWxlbWVudAogICAgICogdHlwZSBiZWNvbWVzIG51bGxhYmxlLiBJdCBpcyByZWZ1c2VkIGluc2lkZSBhbm90aGVyIGFycmF5X2V4cGFuZCgpIGFuZCBpbiBmaWx0ZXIoKSwgdW50aWwoKSwKICAgICAqIGR1cGxpY2F0ZVJvdygpLCBhZ2dyZWdhdGUoKSwgb3ZlcigpIGFuZCBvbkVhY2goKS4KICAgICAqCiAgICAgKiBCZWZvcmU6CiAgICAgKiAgICstLSstLS0tLS0tLS0tLS0tLS0tLS0tKwogICAgICogICB8aWR8ICAgICAgICAgICAgICBhcnJheXwKICAgICAqICAgKy0tKy0tLS0tLS0tLS0tLS0tLS0tLS0rCiAgICAgKiAgIHwgMXx7ImEiOjEsImIiOjIsImMiOjN9fAogICAgICogICArLS0rLS0tLS0tLS0tLS0tLS0tLS0tLSsKICAgICAqCiAgICAgKiBBZnRlcjoKICAgICAqICAgKy0tKy0tLS0tLS0tKwogICAgICogICB8aWR8ZXhwYW5kZWR8CiAgICAgKiAgICstLSstLS0tLS0tLSsKICAgICAqICAgfCAxfCAgICAgICAxfAogICAgICogICB8IDF8ICAgICAgIDJ8CiAgICAgKiAgIHwgMXwgICAgICAgM3wKICAgICAqICAgKy0tKy0tLS0tLS0tKwogICAgICov"},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Function\/ScalarFunctionChain.php","start_line_in_file":350,"slug":"greaterthan","name":"greaterThan","class":"Flow\\ETL\\Function\\ScalarFunctionChain","class_slug":"scalarfunctionchain","parameters":[{"name":"ref","type":[{"name":"mixed","namespace":null,"is_nullable":true,"is_variadic":false}],"has_default_value":false,"is_nullable":true,"is_variadic":false,"default_value":null}],"return_type":[{"name":"GreaterThan","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":true,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Function\/ScalarFunctionChain.php","start_line_in_file":355,"slug":"greaterthanequal","name":"greaterThanEqual","class":"Flow\\ETL\\Function\\ScalarFunctionChain","class_slug":"scalarfunctionchain","parameters":[{"name":"ref","type":[{"name":"mixed","namespace":null,"is_nullable":true,"is_variadic":false}],"has_default_value":false,"is_nullable":true,"is_variadic":false,"default_value":null}],"return_type":[{"name":"GreaterThanEqual","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":true,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Function\/ScalarFunctionChain.php","start_line_in_file":360,"slug":"hash","name":"hash","class":"Flow\\ETL\\Function\\ScalarFunctionChain","class_slug":"scalarfunctionchain","parameters":[{"name":"algorithm","type":[{"name":"Algorithm","namespace":"Flow\\ETL\\Hash","is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"Flow\\ETL\\Hash\\NativePHPHash::..."}],"return_type":[{"name":"Hash","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":true,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Function\/ScalarFunctionChain.php","start_line_in_file":365,"slug":"htmlqueryselector","name":"htmlQuerySelector","class":"Flow\\ETL\\Function\\ScalarFunctionChain","class_slug":"scalarfunctionchain","parameters":[{"name":"path","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"HTMLQuerySelector","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":true,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Function\/ScalarFunctionChain.php","start_line_in_file":370,"slug":"htmlqueryselectorall","name":"htmlQuerySelectorAll","class":"Flow\\ETL\\Function\\ScalarFunctionChain","class_slug":"scalarfunctionchain","parameters":[{"name":"path","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"HTMLQuerySelectorAll","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":true,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Function\/ScalarFunctionChain.php","start_line_in_file":378,"slug":"indexof","name":"indexOf","class":"Flow\\ETL\\Function\\ScalarFunctionChain","class_slug":"scalarfunctionchain","parameters":[{"name":"needle","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"ignoreCase","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"bool","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"false"},{"name":"offset","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"int","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"0"}],"return_type":[{"name":"IndexOf","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":true,"doc_comment":"LyoqCiAgICAgKiBSZXR1cm5zIHRoZSBpbmRleCBvZiBnaXZlbiAkbmVlZGxlIGluIHN0cmluZy4KICAgICAqLw=="},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Function\/ScalarFunctionChain.php","start_line_in_file":389,"slug":"indexoflast","name":"indexOfLast","class":"Flow\\ETL\\Function\\ScalarFunctionChain","class_slug":"scalarfunctionchain","parameters":[{"name":"needle","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"ignoreCase","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"bool","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"false"},{"name":"offset","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"int","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"0"}],"return_type":[{"name":"IndexOfLast","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":true,"doc_comment":"LyoqCiAgICAgKiBSZXR1cm5zIHRoZSBsYXN0IGluZGV4IG9mIGdpdmVuICRuZWVkbGUgaW4gc3RyaW5nLgogICAgICov"},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Function\/ScalarFunctionChain.php","start_line_in_file":397,"slug":"isempty","name":"isEmpty","class":"Flow\\ETL\\Function\\ScalarFunctionChain","class_slug":"scalarfunctionchain","parameters":[],"return_type":[{"name":"IsEmpty","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":true,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Function\/ScalarFunctionChain.php","start_line_in_file":402,"slug":"iseven","name":"isEven","class":"Flow\\ETL\\Function\\ScalarFunctionChain","class_slug":"scalarfunctionchain","parameters":[],"return_type":[{"name":"Equals","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":true,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Function\/ScalarFunctionChain.php","start_line_in_file":407,"slug":"isfalse","name":"isFalse","class":"Flow\\ETL\\Function\\ScalarFunctionChain","class_slug":"scalarfunctionchain","parameters":[],"return_type":[{"name":"Same","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":true,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Function\/ScalarFunctionChain.php","start_line_in_file":415,"slug":"isin","name":"isIn","class":"Flow\\ETL\\Function\\ScalarFunctionChain","class_slug":"scalarfunctionchain","parameters":[{"name":"haystack","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"array","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"IsIn","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":true,"doc_comment":"LyoqCiAgICAgKiBAcGFyYW0gU2NhbGFyRnVuY3Rpb258YXJyYXk8YXJyYXkta2V5LCBtaXhlZD4gJGhheXN0YWNrCiAgICAgKi8="},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Function\/ScalarFunctionChain.php","start_line_in_file":420,"slug":"isnotnull","name":"isNotNull","class":"Flow\\ETL\\Function\\ScalarFunctionChain","class_slug":"scalarfunctionchain","parameters":[],"return_type":[{"name":"IsNotNull","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":true,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Function\/ScalarFunctionChain.php","start_line_in_file":425,"slug":"isnotnumeric","name":"isNotNumeric","class":"Flow\\ETL\\Function\\ScalarFunctionChain","class_slug":"scalarfunctionchain","parameters":[],"return_type":[{"name":"IsNotNumeric","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":true,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Function\/ScalarFunctionChain.php","start_line_in_file":430,"slug":"isnull","name":"isNull","class":"Flow\\ETL\\Function\\ScalarFunctionChain","class_slug":"scalarfunctionchain","parameters":[],"return_type":[{"name":"IsNull","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":true,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Function\/ScalarFunctionChain.php","start_line_in_file":435,"slug":"isnumeric","name":"isNumeric","class":"Flow\\ETL\\Function\\ScalarFunctionChain","class_slug":"scalarfunctionchain","parameters":[],"return_type":[{"name":"IsNumeric","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":true,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Function\/ScalarFunctionChain.php","start_line_in_file":440,"slug":"isodd","name":"isOdd","class":"Flow\\ETL\\Function\\ScalarFunctionChain","class_slug":"scalarfunctionchain","parameters":[],"return_type":[{"name":"NotEquals","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":true,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Function\/ScalarFunctionChain.php","start_line_in_file":445,"slug":"istrue","name":"isTrue","class":"Flow\\ETL\\Function\\ScalarFunctionChain","class_slug":"scalarfunctionchain","parameters":[],"return_type":[{"name":"Same","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":true,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Function\/ScalarFunctionChain.php","start_line_in_file":453,"slug":"istype","name":"isType","class":"Flow\\ETL\\Function\\ScalarFunctionChain","class_slug":"scalarfunctionchain","parameters":[{"name":"types","type":[{"name":"Type","namespace":"Flow\\Types","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":true,"default_value":null}],"return_type":[{"name":"IsType","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":true,"doc_comment":"LyoqCiAgICAgKiBAcGFyYW0gc3RyaW5nfFR5cGU8bWl4ZWQ+ICR0eXBlcwogICAgICov"},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Function\/ScalarFunctionChain.php","start_line_in_file":465,"slug":"isutf8","name":"isUtf8","class":"Flow\\ETL\\Function\\ScalarFunctionChain","class_slug":"scalarfunctionchain","parameters":[],"return_type":[{"name":"IsUtf8","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":true,"doc_comment":"LyoqCiAgICAgKiBDaGVjayBzdHJpbmcgaXMgdXRmOCBhbmQgcmV0dXJucyB0cnVlIG9yIGZhbHNlLgogICAgICov"},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Function\/ScalarFunctionChain.php","start_line_in_file":470,"slug":"jsondecode","name":"jsonDecode","class":"Flow\\ETL\\Function\\ScalarFunctionChain","class_slug":"scalarfunctionchain","parameters":[{"name":"flags","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"int","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"4194304"}],"return_type":[{"name":"JsonDecode","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":true,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Function\/ScalarFunctionChain.php","start_line_in_file":475,"slug":"jsonencode","name":"jsonEncode","class":"Flow\\ETL\\Function\\ScalarFunctionChain","class_slug":"scalarfunctionchain","parameters":[{"name":"flags","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"int","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"4194304"}],"return_type":[{"name":"JsonEncode","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":true,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Function\/ScalarFunctionChain.php","start_line_in_file":480,"slug":"lessthan","name":"lessThan","class":"Flow\\ETL\\Function\\ScalarFunctionChain","class_slug":"scalarfunctionchain","parameters":[{"name":"ref","type":[{"name":"mixed","namespace":null,"is_nullable":true,"is_variadic":false}],"has_default_value":false,"is_nullable":true,"is_variadic":false,"default_value":null}],"return_type":[{"name":"LessThan","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":true,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Function\/ScalarFunctionChain.php","start_line_in_file":485,"slug":"lessthanequal","name":"lessThanEqual","class":"Flow\\ETL\\Function\\ScalarFunctionChain","class_slug":"scalarfunctionchain","parameters":[{"name":"ref","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"LessThanEqual","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":true,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Function\/ScalarFunctionChain.php","start_line_in_file":490,"slug":"literal","name":"literal","class":"Flow\\ETL\\Function\\ScalarFunctionChain","class_slug":"scalarfunctionchain","parameters":[{"name":"value","type":[{"name":"mixed","namespace":null,"is_nullable":true,"is_variadic":false}],"has_default_value":false,"is_nullable":true,"is_variadic":false,"default_value":null}],"return_type":[{"name":"Literal","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":true,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Function\/ScalarFunctionChain.php","start_line_in_file":495,"slug":"lower","name":"lower","class":"Flow\\ETL\\Function\\ScalarFunctionChain","class_slug":"scalarfunctionchain","parameters":[],"return_type":[{"name":"ToLower","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":true,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Function\/ScalarFunctionChain.php","start_line_in_file":500,"slug":"minus","name":"minus","class":"Flow\\ETL\\Function\\ScalarFunctionChain","class_slug":"scalarfunctionchain","parameters":[{"name":"ref","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"int","namespace":null,"is_nullable":false,"is_variadic":false},{"name":"float","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"exact","type":[{"name":"bool","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"false"}],"return_type":[{"name":"Minus","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":true,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Function\/ScalarFunctionChain.php","start_line_in_file":505,"slug":"mod","name":"mod","class":"Flow\\ETL\\Function\\ScalarFunctionChain","class_slug":"scalarfunctionchain","parameters":[{"name":"value","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"int","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"Mod","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":true,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Function\/ScalarFunctionChain.php","start_line_in_file":510,"slug":"modifydatetime","name":"modifyDateTime","class":"Flow\\ETL\\Function\\ScalarFunctionChain","class_slug":"scalarfunctionchain","parameters":[{"name":"modifier","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"ModifyDateTime","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":true,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Function\/ScalarFunctionChain.php","start_line_in_file":515,"slug":"multiply","name":"multiply","class":"Flow\\ETL\\Function\\ScalarFunctionChain","class_slug":"scalarfunctionchain","parameters":[{"name":"value","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"int","namespace":null,"is_nullable":false,"is_variadic":false},{"name":"float","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"exact","type":[{"name":"bool","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"false"}],"return_type":[{"name":"Multiply","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":true,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Function\/ScalarFunctionChain.php","start_line_in_file":520,"slug":"notequals","name":"notEquals","class":"Flow\\ETL\\Function\\ScalarFunctionChain","class_slug":"scalarfunctionchain","parameters":[{"name":"value","type":[{"name":"mixed","namespace":null,"is_nullable":true,"is_variadic":false}],"has_default_value":false,"is_nullable":true,"is_variadic":false,"default_value":null}],"return_type":[{"name":"NotEquals","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":true,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Function\/ScalarFunctionChain.php","start_line_in_file":525,"slug":"notsame","name":"notSame","class":"Flow\\ETL\\Function\\ScalarFunctionChain","class_slug":"scalarfunctionchain","parameters":[{"name":"value","type":[{"name":"mixed","namespace":null,"is_nullable":true,"is_variadic":false}],"has_default_value":false,"is_nullable":true,"is_variadic":false,"default_value":null}],"return_type":[{"name":"NotSame","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":true,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Function\/ScalarFunctionChain.php","start_line_in_file":530,"slug":"numberformat","name":"numberFormat","class":"Flow\\ETL\\Function\\ScalarFunctionChain","class_slug":"scalarfunctionchain","parameters":[{"name":"decimals","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"int","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"2"},{"name":"decimalSeparator","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"'.'"},{"name":"thousandsSeparator","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"','"}],"return_type":[{"name":"NumberFormat","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":true,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Function\/ScalarFunctionChain.php","start_line_in_file":544,"slug":"oneach","name":"onEach","class":"Flow\\ETL\\Function\\ScalarFunctionChain","class_slug":"scalarfunctionchain","parameters":[{"name":"function","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"preserveKeys","type":[{"name":"bool","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"true"}],"return_type":[{"name":"OnEach","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":true,"doc_comment":"LyoqCiAgICAgKiBFeGVjdXRlIGEgc2NhbGFyIGZ1bmN0aW9uIG9uIGVhY2ggZWxlbWVudCBvZiBhbiBhcnJheS9saXN0L21hcC9zdHJ1Y3R1cmUgZW50cnkuCiAgICAgKiBJbiBvcmRlciB0byB1c2UgdGhpcyBmdW5jdGlvbiwgeW91IG5lZWQgdG8gcHJvdmlkZSBhIHJlZmVyZW5jZSB0byB0aGUgImVsZW1lbnQiIHRoYXQgd2lsbCBiZSB1c2VkIGluIHRoZSBmdW5jdGlvbi4KICAgICAqCiAgICAgKiBFeGFtcGxlOiAkZGYtPndpdGhFbnRyeSgnYXJyYXknLCByZWYoJ2FycmF5JyktPm9uRWFjaChyZWYoJ2VsZW1lbnQnKS0+Y2FzdCh0eXBlX3N0cmluZygpKSkpCiAgICAgKi8="},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Function\/ScalarFunctionChain.php","start_line_in_file":549,"slug":"or","name":"or","class":"Flow\\ETL\\Function\\ScalarFunctionChain","class_slug":"scalarfunctionchain","parameters":[{"name":"function","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"Any","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Function\/ScalarFunctionChain.php","start_line_in_file":554,"slug":"ornot","name":"orNot","class":"Flow\\ETL\\Function\\ScalarFunctionChain","class_slug":"scalarfunctionchain","parameters":[{"name":"function","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"Any","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Function\/ScalarFunctionChain.php","start_line_in_file":559,"slug":"plus","name":"plus","class":"Flow\\ETL\\Function\\ScalarFunctionChain","class_slug":"scalarfunctionchain","parameters":[{"name":"ref","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"int","namespace":null,"is_nullable":false,"is_variadic":false},{"name":"float","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"exact","type":[{"name":"bool","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"false"}],"return_type":[{"name":"Plus","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":true,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Function\/ScalarFunctionChain.php","start_line_in_file":564,"slug":"power","name":"power","class":"Flow\\ETL\\Function\\ScalarFunctionChain","class_slug":"scalarfunctionchain","parameters":[{"name":"value","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"int","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"exact","type":[{"name":"bool","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"false"}],"return_type":[{"name":"Power","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":true,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Function\/ScalarFunctionChain.php","start_line_in_file":569,"slug":"prepend","name":"prepend","class":"Flow\\ETL\\Function\\ScalarFunctionChain","class_slug":"scalarfunctionchain","parameters":[{"name":"prefix","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"Prepend","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":true,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Function\/ScalarFunctionChain.php","start_line_in_file":574,"slug":"regex","name":"regex","class":"Flow\\ETL\\Function\\ScalarFunctionChain","class_slug":"scalarfunctionchain","parameters":[{"name":"pattern","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"flags","type":[{"name":"int","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"0"},{"name":"offset","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"int","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"0"}],"return_type":[{"name":"Regex","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":true,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Function\/ScalarFunctionChain.php","start_line_in_file":579,"slug":"regexall","name":"regexAll","class":"Flow\\ETL\\Function\\ScalarFunctionChain","class_slug":"scalarfunctionchain","parameters":[{"name":"pattern","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"flags","type":[{"name":"int","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"0"},{"name":"offset","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"int","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"0"}],"return_type":[{"name":"RegexAll","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":true,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Function\/ScalarFunctionChain.php","start_line_in_file":584,"slug":"regexmatch","name":"regexMatch","class":"Flow\\ETL\\Function\\ScalarFunctionChain","class_slug":"scalarfunctionchain","parameters":[{"name":"pattern","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"flags","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"int","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"0"},{"name":"offset","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"int","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"0"}],"return_type":[{"name":"RegexMatch","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":true,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Function\/ScalarFunctionChain.php","start_line_in_file":592,"slug":"regexmatchall","name":"regexMatchAll","class":"Flow\\ETL\\Function\\ScalarFunctionChain","class_slug":"scalarfunctionchain","parameters":[{"name":"pattern","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"flags","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"int","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"0"},{"name":"offset","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"int","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"0"}],"return_type":[{"name":"RegexMatchAll","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":true,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Function\/ScalarFunctionChain.php","start_line_in_file":600,"slug":"regexreplace","name":"regexReplace","class":"Flow\\ETL\\Function\\ScalarFunctionChain","class_slug":"scalarfunctionchain","parameters":[{"name":"pattern","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"replacement","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"limit","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"int","namespace":null,"is_nullable":false,"is_variadic":false},{"name":"null","namespace":null,"is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"null"}],"return_type":[{"name":"RegexReplace","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":true,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Function\/ScalarFunctionChain.php","start_line_in_file":608,"slug":"repeat","name":"repeat","class":"Flow\\ETL\\Function\\ScalarFunctionChain","class_slug":"scalarfunctionchain","parameters":[{"name":"times","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"int","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"Repeat","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":true,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Function\/ScalarFunctionChain.php","start_line_in_file":613,"slug":"reverse","name":"reverse","class":"Flow\\ETL\\Function\\ScalarFunctionChain","class_slug":"scalarfunctionchain","parameters":[],"return_type":[{"name":"Reverse","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":true,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Function\/ScalarFunctionChain.php","start_line_in_file":618,"slug":"round","name":"round","class":"Flow\\ETL\\Function\\ScalarFunctionChain","class_slug":"scalarfunctionchain","parameters":[{"name":"precision","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"int","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"2"},{"name":"mode","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"int","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"1"}],"return_type":[{"name":"Round","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":true,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Function\/ScalarFunctionChain.php","start_line_in_file":623,"slug":"same","name":"same","class":"Flow\\ETL\\Function\\ScalarFunctionChain","class_slug":"scalarfunctionchain","parameters":[{"name":"value","type":[{"name":"mixed","namespace":null,"is_nullable":true,"is_variadic":false}],"has_default_value":false,"is_nullable":true,"is_variadic":false,"default_value":null}],"return_type":[{"name":"Same","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":true,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Function\/ScalarFunctionChain.php","start_line_in_file":628,"slug":"sanitize","name":"sanitize","class":"Flow\\ETL\\Function\\ScalarFunctionChain","class_slug":"scalarfunctionchain","parameters":[{"name":"placeholder","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"'*'"},{"name":"skipCharacters","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"int","namespace":null,"is_nullable":false,"is_variadic":false},{"name":"null","namespace":null,"is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"null"}],"return_type":[{"name":"Sanitize","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":true,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Function\/ScalarFunctionChain.php","start_line_in_file":635,"slug":"size","name":"size","class":"Flow\\ETL\\Function\\ScalarFunctionChain","class_slug":"scalarfunctionchain","parameters":[],"return_type":[{"name":"Size","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":true,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Function\/ScalarFunctionChain.php","start_line_in_file":643,"slug":"slug","name":"slug","class":"Flow\\ETL\\Function\\ScalarFunctionChain","class_slug":"scalarfunctionchain","parameters":[{"name":"separator","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"'-'"},{"name":"locale","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false},{"name":"null","namespace":null,"is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"null"},{"name":"symbolsMap","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"array","namespace":null,"is_nullable":false,"is_variadic":false},{"name":"null","namespace":null,"is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"null"}],"return_type":[{"name":"Slug","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":true,"doc_comment":"LyoqCiAgICAgKiBAcGFyYW0gU2NhbGFyRnVuY3Rpb258bnVsbHxhcnJheTxhcnJheS1rZXksIG1peGVkPiAkc3ltYm9sc01hcAogICAgICov"},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Function\/ScalarFunctionChain.php","start_line_in_file":651,"slug":"split","name":"split","class":"Flow\\ETL\\Function\\ScalarFunctionChain","class_slug":"scalarfunctionchain","parameters":[{"name":"separator","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"limit","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"int","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"9223372036854775807"}],"return_type":[{"name":"Split","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":true,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Function\/ScalarFunctionChain.php","start_line_in_file":656,"slug":"sprintf","name":"sprintf","class":"Flow\\ETL\\Function\\ScalarFunctionChain","class_slug":"scalarfunctionchain","parameters":[{"name":"params","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false},{"name":"int","namespace":null,"is_nullable":false,"is_variadic":false},{"name":"float","namespace":null,"is_nullable":false,"is_variadic":false},{"name":"null","namespace":null,"is_nullable":true,"is_variadic":false}],"has_default_value":false,"is_nullable":true,"is_variadic":true,"default_value":null}],"return_type":[{"name":"Sprintf","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":true,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Function\/ScalarFunctionChain.php","start_line_in_file":661,"slug":"startswith","name":"startsWith","class":"Flow\\ETL\\Function\\ScalarFunctionChain","class_slug":"scalarfunctionchain","parameters":[{"name":"needle","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"StartsWith","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":true,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Function\/ScalarFunctionChain.php","start_line_in_file":669,"slug":"stringafter","name":"stringAfter","class":"Flow\\ETL\\Function\\ScalarFunctionChain","class_slug":"scalarfunctionchain","parameters":[{"name":"needle","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"includeNeedle","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"bool","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"false"}],"return_type":[{"name":"StringAfter","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":true,"doc_comment":"LyoqCiAgICAgKiBSZXR1cm5zIHRoZSBjb250ZW50cyBmb3VuZCBhZnRlciB0aGUgZmlyc3Qgb2NjdXJyZW5jZSBvZiB0aGUgZ2l2ZW4gc3RyaW5nLgogICAgICov"},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Function\/ScalarFunctionChain.php","start_line_in_file":677,"slug":"stringafterlast","name":"stringAfterLast","class":"Flow\\ETL\\Function\\ScalarFunctionChain","class_slug":"scalarfunctionchain","parameters":[{"name":"needle","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"includeNeedle","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"bool","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"false"}],"return_type":[{"name":"StringAfterLast","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":true,"doc_comment":"LyoqCiAgICAgKiBSZXR1cm5zIHRoZSBjb250ZW50cyBmb3VuZCBhZnRlciB0aGUgbGFzdCBvY2N1cnJlbmNlIG9mIHRoZSBnaXZlbiBzdHJpbmcuCiAgICAgKi8="},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Function\/ScalarFunctionChain.php","start_line_in_file":687,"slug":"stringbefore","name":"stringBefore","class":"Flow\\ETL\\Function\\ScalarFunctionChain","class_slug":"scalarfunctionchain","parameters":[{"name":"needle","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"includeNeedle","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"bool","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"false"}],"return_type":[{"name":"StringBefore","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":true,"doc_comment":"LyoqCiAgICAgKiBSZXR1cm5zIHRoZSBjb250ZW50cyBmb3VuZCBiZWZvcmUgdGhlIGZpcnN0IG9jY3VycmVuY2Ugb2YgdGhlIGdpdmVuIHN0cmluZy4KICAgICAqLw=="},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Function\/ScalarFunctionChain.php","start_line_in_file":697,"slug":"stringbeforelast","name":"stringBeforeLast","class":"Flow\\ETL\\Function\\ScalarFunctionChain","class_slug":"scalarfunctionchain","parameters":[{"name":"needle","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"includeNeedle","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"bool","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"false"}],"return_type":[{"name":"StringBeforeLast","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":true,"doc_comment":"LyoqCiAgICAgKiBSZXR1cm5zIHRoZSBjb250ZW50cyBmb3VuZCBiZWZvcmUgdGhlIGxhc3Qgb2NjdXJyZW5jZSBvZiB0aGUgZ2l2ZW4gc3RyaW5nLgogICAgICov"},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Function\/ScalarFunctionChain.php","start_line_in_file":707,"slug":"stringcontainsany","name":"stringContainsAny","class":"Flow\\ETL\\Function\\ScalarFunctionChain","class_slug":"scalarfunctionchain","parameters":[{"name":"needles","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"array","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"StringContainsAny","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":true,"doc_comment":"LyoqCiAgICAgKiBAcGFyYW0gYXJyYXk8c3RyaW5nPnxTY2FsYXJGdW5jdGlvbiAkbmVlZGxlcwogICAgICov"},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Function\/ScalarFunctionChain.php","start_line_in_file":712,"slug":"stringequalsto","name":"stringEqualsTo","class":"Flow\\ETL\\Function\\ScalarFunctionChain","class_slug":"scalarfunctionchain","parameters":[{"name":"string","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"StringEqualsTo","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":true,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Function\/ScalarFunctionChain.php","start_line_in_file":720,"slug":"stringfold","name":"stringFold","class":"Flow\\ETL\\Function\\ScalarFunctionChain","class_slug":"scalarfunctionchain","parameters":[],"return_type":[{"name":"StringFold","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":true,"doc_comment":"LyoqCiAgICAgKiBSZXR1cm5zIGEgc3RyaW5nIHRoYXQgeW91IGNhbiB1c2UgaW4gY2FzZS1pbnNlbnNpdGl2ZSBjb21wYXJpc29ucy4KICAgICAqLw=="},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Function\/ScalarFunctionChain.php","start_line_in_file":725,"slug":"stringmatch","name":"stringMatch","class":"Flow\\ETL\\Function\\ScalarFunctionChain","class_slug":"scalarfunctionchain","parameters":[{"name":"pattern","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"StringMatch","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":true,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Function\/ScalarFunctionChain.php","start_line_in_file":730,"slug":"stringmatchall","name":"stringMatchAll","class":"Flow\\ETL\\Function\\ScalarFunctionChain","class_slug":"scalarfunctionchain","parameters":[{"name":"pattern","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"StringMatchAll","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":true,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Function\/ScalarFunctionChain.php","start_line_in_file":735,"slug":"stringnormalize","name":"stringNormalize","class":"Flow\\ETL\\Function\\ScalarFunctionChain","class_slug":"scalarfunctionchain","parameters":[{"name":"form","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"int","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"16"}],"return_type":[{"name":"StringNormalize","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":true,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Function\/ScalarFunctionChain.php","start_line_in_file":744,"slug":"stringstyle","name":"stringStyle","class":"Flow\\ETL\\Function\\ScalarFunctionChain","class_slug":"scalarfunctionchain","parameters":[{"name":"style","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"StringStyles","namespace":"Flow\\ETL\\String","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"StringStyle","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":true,"doc_comment":"LyoqCiAgICAgKiBDb3ZlcnQgc3RyaW5nIHRvIGEgc3R5bGUgZnJvbSBlbnVtIGxpc3QsIHBhc3NlZCBpbiBwYXJhbWV0ZXIuCiAgICAgKiBDYW4gYmUgc3RyaW5nICJ1cHBlciIgb3IgU3RyaW5nU3R5bGVzOjpVUFBFUiBmb3IgVXBwZXIgKGV4YW1wbGUpLgogICAgICov"},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Function\/ScalarFunctionChain.php","start_line_in_file":752,"slug":"stringtitle","name":"stringTitle","class":"Flow\\ETL\\Function\\ScalarFunctionChain","class_slug":"scalarfunctionchain","parameters":[{"name":"allWords","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"bool","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"false"}],"return_type":[{"name":"StringTitle","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":true,"doc_comment":"LyoqCiAgICAgKiBDaGFuZ2VzIGFsbCBncmFwaGVtZXMvY29kZSBwb2ludHMgdG8gInRpdGxlIGNhc2UiLgogICAgICov"},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Function\/ScalarFunctionChain.php","start_line_in_file":757,"slug":"stringwidth","name":"stringWidth","class":"Flow\\ETL\\Function\\ScalarFunctionChain","class_slug":"scalarfunctionchain","parameters":[],"return_type":[{"name":"StringWidth","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":true,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Function\/ScalarFunctionChain.php","start_line_in_file":762,"slug":"strpad","name":"strPad","class":"Flow\\ETL\\Function\\ScalarFunctionChain","class_slug":"scalarfunctionchain","parameters":[{"name":"length","type":[{"name":"int","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"pad_string","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"' '"},{"name":"type","type":[{"name":"int","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"1"}],"return_type":[{"name":"StrPad","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":true,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Function\/ScalarFunctionChain.php","start_line_in_file":767,"slug":"strpadboth","name":"strPadBoth","class":"Flow\\ETL\\Function\\ScalarFunctionChain","class_slug":"scalarfunctionchain","parameters":[{"name":"length","type":[{"name":"int","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"pad_string","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"' '"}],"return_type":[{"name":"StrPad","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":true,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Function\/ScalarFunctionChain.php","start_line_in_file":772,"slug":"strpadleft","name":"strPadLeft","class":"Flow\\ETL\\Function\\ScalarFunctionChain","class_slug":"scalarfunctionchain","parameters":[{"name":"length","type":[{"name":"int","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"pad_string","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"' '"}],"return_type":[{"name":"StrPad","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":true,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Function\/ScalarFunctionChain.php","start_line_in_file":777,"slug":"strpadright","name":"strPadRight","class":"Flow\\ETL\\Function\\ScalarFunctionChain","class_slug":"scalarfunctionchain","parameters":[{"name":"length","type":[{"name":"int","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"pad_string","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"' '"}],"return_type":[{"name":"StrPad","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":true,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Function\/ScalarFunctionChain.php","start_line_in_file":786,"slug":"strreplace","name":"strReplace","class":"Flow\\ETL\\Function\\ScalarFunctionChain","class_slug":"scalarfunctionchain","parameters":[{"name":"search","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"array","namespace":null,"is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"replace","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"array","namespace":null,"is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"StrReplace","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":true,"doc_comment":"LyoqCiAgICAgKiBAcGFyYW0gYXJyYXk8c3RyaW5nPnxTY2FsYXJGdW5jdGlvbnxzdHJpbmcgJHNlYXJjaAogICAgICogQHBhcmFtIGFycmF5PHN0cmluZz58U2NhbGFyRnVuY3Rpb258c3RyaW5nICRyZXBsYWNlCiAgICAgKi8="},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Function\/ScalarFunctionChain.php","start_line_in_file":794,"slug":"todate","name":"toDate","class":"Flow\\ETL\\Function\\ScalarFunctionChain","class_slug":"scalarfunctionchain","parameters":[{"name":"format","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"'Y-m-d\\\\TH:i:sP'"},{"name":"timeZone","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"DateTimeZone","namespace":"","is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"DateTimeZone::..."}],"return_type":[{"name":"ToDate","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":true,"doc_comment":"LyoqCiAgICAgKiBAcGFyYW0gU2NhbGFyRnVuY3Rpb258c3RyaW5nICRmb3JtYXQgLSBjdXJyZW50IGZvcm1hdCBvZiB0aGUgZGF0ZSB0aGF0IHdpbGwgYmUgdXNlZCB0byBjcmVhdGUgRGF0ZVRpbWVJbW11dGFibGUgaW5zdGFuY2UKICAgICAqLw=="},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Function\/ScalarFunctionChain.php","start_line_in_file":805,"slug":"todatetime","name":"toDateTime","class":"Flow\\ETL\\Function\\ScalarFunctionChain","class_slug":"scalarfunctionchain","parameters":[{"name":"format","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"'Y-m-d H:i:s'"},{"name":"timeZone","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"DateTimeZone","namespace":"","is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"DateTimeZone::..."}],"return_type":[{"name":"ToDateTime","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":true,"doc_comment":"LyoqCiAgICAgKiBAcGFyYW0gU2NhbGFyRnVuY3Rpb258c3RyaW5nICRmb3JtYXQgLSBjdXJyZW50IGZvcm1hdCBvZiB0aGUgZGF0ZSB0aGF0IHdpbGwgYmUgdXNlZCB0byBjcmVhdGUgRGF0ZVRpbWVJbW11dGFibGUgaW5zdGFuY2UKICAgICAqIEBwYXJhbSBcRGF0ZVRpbWVab25lfFNjYWxhckZ1bmN0aW9uICR0aW1lWm9uZQogICAgICov"},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Function\/ScalarFunctionChain.php","start_line_in_file":812,"slug":"trim","name":"trim","class":"Flow\\ETL\\Function\\ScalarFunctionChain","class_slug":"scalarfunctionchain","parameters":[{"name":"type","type":[{"name":"Type","namespace":"Flow\\ETL\\Function\\Trim","is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"Flow\\ETL\\Function\\Trim\\Type::..."},{"name":"characters","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"' \\t\\n\\r\\0\u000b'"}],"return_type":[{"name":"Trim","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":true,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Function\/ScalarFunctionChain.php","start_line_in_file":817,"slug":"truncate","name":"truncate","class":"Flow\\ETL\\Function\\ScalarFunctionChain","class_slug":"scalarfunctionchain","parameters":[{"name":"length","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"int","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"ellipsis","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"'...'"}],"return_type":[{"name":"Truncate","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":true,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Function\/ScalarFunctionChain.php","start_line_in_file":822,"slug":"unicodelength","name":"unicodeLength","class":"Flow\\ETL\\Function\\ScalarFunctionChain","class_slug":"scalarfunctionchain","parameters":[],"return_type":[{"name":"UnicodeLength","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":true,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Function\/ScalarFunctionChain.php","start_line_in_file":846,"slug":"unpack","name":"unpack","class":"Flow\\ETL\\Function\\ScalarFunctionChain","class_slug":"scalarfunctionchain","parameters":[{"name":"schema","type":[{"name":"Schema","namespace":"Flow\\ETL","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"ArrayUnpack","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":true,"doc_comment":"LyoqCiAgICAgKiBVbnBhY2tzIGVhY2ggZWxlbWVudCBvZiBhbiBhcnJheSBpbnRvIGEgbmV3IGVudHJ5LCB1c2luZyB0aGUgYXJyYXkga2V5IGFzIHRoZSBlbnRyeSBuYW1lLgogICAgICoKICAgICAqIEJlZm9yZToKICAgICAqICAgKy0tKy0tLS0tLS0tLS0tLS0tLS0tLS0rCiAgICAgKiAgIHxpZHwgICAgICAgICAgICAgIGFycmF5fAogICAgICogICArLS0rLS0tLS0tLS0tLS0tLS0tLS0tLSsKICAgICAqICAgfCAxfHsiYSI6MSwiYiI6MiwiYyI6M318CiAgICAgKiAgIHwgMnx7ImQiOjQsImUiOjUsImYiOjZ9fAogICAgICogICArLS0rLS0tLS0tLS0tLS0tLS0tLS0tLSsKICAgICAqCiAgICAgKiBBZnRlcjoKICAgICAqICAgKy0tKy0tLS0tKy0tLS0tKy0tLS0tKy0tLS0tKy0tLS0tKwogICAgICogICB8aWR8YXJyLmJ8YXJyLmN8YXJyLmR8YXJyLmV8YXJyLmZ8CiAgICAgKiAgICstLSstLS0tLSstLS0tLSstLS0tLSstLS0tLSstLS0tLSsKICAgICAqICAgfCAxfCAgICAyfCAgICAzfCAgICAgfCAgICAgfCAgICAgfAogICAgICogICB8IDJ8ICAgICB8ICAgICB8ICAgIDR8ICAgIDV8ICAgIDZ8CiAgICAgKiAgICstLSstLS0tLSstLS0tLSstLS0tLSstLS0tLSstLS0tLSsKICAgICAqLw=="},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Function\/ScalarFunctionChain.php","start_line_in_file":851,"slug":"upper","name":"upper","class":"Flow\\ETL\\Function\\ScalarFunctionChain","class_slug":"scalarfunctionchain","parameters":[],"return_type":[{"name":"ToUpper","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":true,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Function\/ScalarFunctionChain.php","start_line_in_file":856,"slug":"wordwrap","name":"wordwrap","class":"Flow\\ETL\\Function\\ScalarFunctionChain","class_slug":"scalarfunctionchain","parameters":[{"name":"width","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"int","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"break","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"'\\n'"},{"name":"cut","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"bool","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"false"}],"return_type":[{"name":"Wordwrap","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":true,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Function\/ScalarFunctionChain.php","start_line_in_file":864,"slug":"xpath","name":"xpath","class":"Flow\\ETL\\Function\\ScalarFunctionChain","class_slug":"scalarfunctionchain","parameters":[{"name":"string","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"XPath","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":true,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Function\/ResolvesFromChildren.php","start_line_in_file":12,"slug":"resolved","name":"resolved","class":"Flow\\ETL\\Function\\ScalarFunctionChain","class_slug":"scalarfunctionchain","parameters":[],"return_type":[{"name":"bool","namespace":null,"is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Function\/ResolvesFromChildren.php","start_line_in_file":23,"slug":"deterministic","name":"deterministic","class":"Flow\\ETL\\Function\\ScalarFunctionChain","class_slug":"scalarfunctionchain","parameters":[],"return_type":[{"name":"bool","namespace":null,"is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Flow.php","start_line_in_file":23,"slug":"setup","name":"setUp","class":"Flow\\ETL\\Flow","class_slug":"flow","parameters":[{"name":"config","type":[{"name":"ConfigBuilder","namespace":"Flow\\ETL\\Config","is_nullable":false,"is_variadic":false},{"name":"Config","namespace":"Flow\\ETL","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"self","namespace":null,"is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Flow.php","start_line_in_file":28,"slug":"extract","name":"extract","class":"Flow\\ETL\\Flow","class_slug":"flow","parameters":[{"name":"extractor","type":[{"name":"Extractor","namespace":"Flow\\ETL","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"DataFrame","namespace":"Flow\\ETL","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Flow.php","start_line_in_file":33,"slug":"from","name":"from","class":"Flow\\ETL\\Flow","class_slug":"flow","parameters":[{"name":"extractor","type":[{"name":"Extractor","namespace":"Flow\\ETL","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"DataFrame","namespace":"Flow\\ETL","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Flow.php","start_line_in_file":38,"slug":"process","name":"process","class":"Flow\\ETL\\Flow","class_slug":"flow","parameters":[{"name":"rows","type":[{"name":"Rows","namespace":"Flow\\ETL","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":true,"default_value":null}],"return_type":[{"name":"DataFrame","namespace":"Flow\\ETL","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/Flow.php","start_line_in_file":46,"slug":"read","name":"read","class":"Flow\\ETL\\Flow","class_slug":"flow","parameters":[{"name":"extractor","type":[{"name":"Extractor","namespace":"Flow\\ETL","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"DataFrame","namespace":"Flow\\ETL","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":false,"doc_comment":"LyoqCiAgICAgKiBBbGlhcyBmb3IgRmxvdzo6ZXh0cmFjdCBmdW5jdGlvbi4KICAgICAqLw=="},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DataFrame.php","start_line_in_file":74,"slug":"aggregate","name":"aggregate","class":"Flow\\ETL\\DataFrame","class_slug":"dataframe","parameters":[{"name":"aggregations","type":[{"name":"array","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"algorithm","type":[{"name":"GroupByAlgorithmBuilder","namespace":"Flow\\ETL\\Config\\Grouping","is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"null"}],"return_type":[{"name":"self","namespace":null,"is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":false,"doc_comment":"LyoqCiAgICAgKiBAbGF6eQogICAgICoKICAgICAqIEBwYXJhbSBBZ2dyZWdhdGlvbnMgJGFnZ3JlZ2F0aW9ucwogICAgICogQHBhcmFtIG51bGx8R3JvdXBCeUFsZ29yaXRobUJ1aWxkZXIgJGFsZ29yaXRobSBudWxsIGRlZmVycyB0byBjb25maWd1cmF0aW9uOyBhIGJ1aWxkZXIgcGlucyB0aGUKICAgICAqICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICBhbGdvcml0aG0gZm9yIHRoaXMgb3BlcmF0aW9uIGFuZCBza2lwcyBhbnkgYXV0b21hdGljIGNob2ljZQogICAgICov"},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DataFrame.php","start_line_in_file":98,"slug":"batchby","name":"batchBy","class":"Flow\\ETL\\DataFrame","class_slug":"dataframe","parameters":[{"name":"column","type":[{"name":"Reference","namespace":"Flow\\ETL\\Row","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"minSize","type":[{"name":"int","namespace":null,"is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"null"}],"return_type":[{"name":"self","namespace":null,"is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":false,"doc_comment":"LyoqCiAgICAgKiBNZXJnZS9TcGxpdCBSb3dzIHlpZWxkZWQgYnkgRXh0cmFjdG9yIGludG8gYmF0Y2hlcyBidXQga2VlcCB0aG9zZSB3aXRoIGNvbW1vbiB2YWx1ZSBpbiBnaXZlbiBjb2x1bW4gdG9nZXRoZXIuCiAgICAgKiBUaGlzIHdvcmtzIHByb3Blcmx5IG9ubHkgb24gc29ydGVkIGRhdGFzZXRzLgogICAgICoKICAgICAqIFdoZW4gbWluU2l6ZSBpcyBub3QgcHJvdmlkZWQsIGJhdGNoZXMgd2lsbCBiZSBjcmVhdGVkIG9ubHkgd2hlbiB0aGVyZSBpcyBhIGNoYW5nZSBpbiB2YWx1ZSBvZiB0aGUgY29sdW1uLgogICAgICogV2hlbiBtaW5TaXplIGlzIHByb3ZpZGVkLCBiYXRjaGVzIHdpbGwgYmUgY3JlYXRlZCBvbmx5IHdoZW4gdGhlcmUgaXMgYSBjaGFuZ2UgaW4gdmFsdWUgb2YgdGhlIGNvbHVtbiBvcgogICAgICogd2hlbiB0aGVyZSBhcmUgYXQgbGVhc3QgbWluU2l6ZSByb3dzIGluIHRoZSBiYXRjaC4KICAgICAqCiAgICAgKiBAcGFyYW0gUmVmZXJlbmNlfHN0cmluZyAkY29sdW1uIC0gY29sdW1uIHRvIGdyb3VwIGJ5IChhbGwgcm93cyB3aXRoIHNhbWUgdmFsdWUgc3RheSB0b2dldGhlcikKICAgICAqIEBwYXJhbSBudWxsfGludDwxLCBtYXg+ICRtaW5TaXplIC0gb3B0aW9uYWwgbWluaW11bSByb3dzIHBlciBiYXRjaCBmb3IgZWZmaWNpZW5jeQogICAgICoKICAgICAqIEBsYXp5CiAgICAgKgogICAgICogQHRocm93cyBJbnZhbGlkQXJndW1lbnRFeGNlcHRpb24KICAgICAqLw=="},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DataFrame.php","start_line_in_file":118,"slug":"batchsize","name":"batchSize","class":"Flow\\ETL\\DataFrame","class_slug":"dataframe","parameters":[{"name":"size","type":[{"name":"int","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"self","namespace":null,"is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":false,"doc_comment":"LyoqCiAgICAgKiBNZXJnZS9TcGxpdCBSb3dzIHlpZWxkZWQgYnkgRXh0cmFjdG9yIGludG8gYmF0Y2hlcyBvZiBnaXZlbiBzaXplLgogICAgICogRm9yIGV4YW1wbGUsIHdoZW4gRXh0cmFjdG9yIGlzIHlpZWxkaW5nIG9uZSByb3cgYXQgdGltZSwgdGhpcyBtZXRob2Qgd2lsbCBtZXJnZSB0aGVtIGludG8gYmF0Y2hlcyBvZiBnaXZlbiBzaXplCiAgICAgKiBiZWZvcmUgcGFzc2luZyB0aGVtIHRvIHRoZSBuZXh0IHBpcGVsaW5lIGVsZW1lbnQuCiAgICAgKiBTaW1pbGFybHkgd2hlbiBFeHRyYWN0b3IgaXMgeWllbGRpbmcgYmF0Y2hlcyBvZiByb3dzLCB0aGlzIG1ldGhvZCB3aWxsIHNwbGl0IHRoZW0gaW50byBzbWFsbGVyIGJhdGNoZXMgb2YgZ2l2ZW4KICAgICAqIHNpemUuCiAgICAgKgogICAgICogSW4gb3JkZXIgdG8gbWVyZ2UgYWxsIFJvd3MgaW50byBhIHNpbmdsZSBiYXRjaCB1c2UgRGF0YUZyYW1lOjpjb2xsZWN0KCkgbWV0aG9kIG9yIHNldCBzaXplIHRvIC0xIG9yIDAuCiAgICAgKgogICAgICogQHBhcmFtIGludDwtMSwgbWF4PiAkc2l6ZQogICAgICoKICAgICAqIEBsYXp5CiAgICAgKi8="},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DataFrame.php","start_line_in_file":150,"slug":"cache","name":"cache","class":"Flow\\ETL\\DataFrame","class_slug":"dataframe","parameters":[{"name":"id","type":[{"name":"string","namespace":null,"is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"null"},{"name":"cacheBatchSize","type":[{"name":"int","namespace":null,"is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"null"},{"name":"cache","type":[{"name":"Cache","namespace":"Flow\\ETL","is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"null"}],"return_type":[{"name":"self","namespace":null,"is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":false,"doc_comment":"LyoqCiAgICAgKiBTdGFydCBwcm9jZXNzaW5nIHJvd3MgdXAgdG8gdGhpcyBtb21lbnQgYW5kIHB1dCBlYWNoIGluc3RhbmNlIG9mIFJvd3MKICAgICAqIGludG8gcHJldmlvdXNseSBkZWZpbmVkIGNhY2hlLgogICAgICogQ2FjaGUgdHlwZSBjYW4gYmUgc2V0IHRocm91Z2ggQ29uZmlnQnVpbGRlci4KICAgICAqIEJ5IGRlZmF1bHQgZXZlcnl0aGluZyBpcyBjYWNoZWQgaW4gc3lzdGVtIHRtcCBkaXIuCiAgICAgKgogICAgICogSW1wb3J0YW50OiBjYWNoZSBiYXRjaCBzaXplIG1pZ2h0IHNpZ25pZmljYW50bHkgaW1wcm92ZSBwZXJmb3JtYW5jZSB3aGVuIHByb2Nlc3NpbmcgbGFyZ2UgYW1vdW50IG9mIHJvd3MuCiAgICAgKiBMYXJnZXIgYmF0Y2ggc2l6ZSB3aWxsIGluY3JlYXNlIG1lbW9yeSBjb25zdW1wdGlvbiBidXQgd2lsbCByZWR1Y2UgbnVtYmVyIG9mIElPIG9wZXJhdGlvbnMuCiAgICAgKiBXaGVuIG5vdCBzZXQsIHRoZSBiYXRjaCBzaXplIGlzIHRha2VuIGZyb20gdGhlIGxhc3QgRGF0YUZyYW1lOjpiYXRjaFNpemUoKSBjYWxsLgogICAgICoKICAgICAqIEBsYXp5CiAgICAgKgogICAgICogQHBhcmFtIG51bGx8c3RyaW5nICRpZAogICAgICogQHBhcmFtIG51bGx8Q2FjaGUgJGNhY2hlIHJlYWRzIG9mIHRoaXMgY2FjaGUgbXVzdCBwYXNzIHRoZSBzYW1lIGluc3RhbmNlIHRvIGZyb21fY2FjaGUoKQogICAgICoKICAgICAqIEB0aHJvd3MgSW52YWxpZEFyZ3VtZW50RXhjZXB0aW9uCiAgICAgKi8="},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DataFrame.php","start_line_in_file":167,"slug":"collect","name":"collect","class":"Flow\\ETL\\DataFrame","class_slug":"dataframe","parameters":[],"return_type":[{"name":"self","namespace":null,"is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":false,"doc_comment":"LyoqCiAgICAgKiBCZWZvcmUgdHJhbnNmb3JtaW5nIHJvd3MsIGNvbGxlY3QgdGhlbSBhbmQgbWVyZ2UgaW50byBzaW5nbGUgUm93cyBpbnN0YW5jZS4KICAgICAqIFRoaXMgbWlnaHQgbGVhZCB0byBtZW1vcnkgaXNzdWVzIHdoZW4gcHJvY2Vzc2luZyBsYXJnZSBhbW91bnQgb2Ygcm93cywgdXNlIHdpdGggY2F1dGlvbi4KICAgICAqCiAgICAgKiBAbGF6eQogICAgICov"},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DataFrame.php","start_line_in_file":186,"slug":"collectrefs","name":"collectRefs","class":"Flow\\ETL\\DataFrame","class_slug":"dataframe","parameters":[{"name":"references","type":[{"name":"References","namespace":"Flow\\ETL\\Row","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"self","namespace":null,"is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":false,"doc_comment":"LyoqCiAgICAgKiBUaGlzIG1ldGhvZCBhbGxvd3MgdG8gY29sbGVjdCByZWZlcmVuY2VzIHRvIGFsbCBlbnRyaWVzIHVzZWQgaW4gdGhpcyBwaXBlbGluZS4KICAgICAqCiAgICAgKiBgYGBwaHAKICAgICAqIChuZXcgRmxvdygpKQogICAgICogICAtPnJlYWQoRnJvbTo6Y2hhaW4oKSkKICAgICAqICAgLT5jb2xsZWN0UmVmcygkcmVmcyA9IHJlZnMoKSkKICAgICAqICAgLT5ydW4oKTsKICAgICAqIGBgYAogICAgICoKICAgICAqIEBsYXp5CiAgICAgKi8="},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DataFrame.php","start_line_in_file":193,"slug":"constrain","name":"constrain","class":"Flow\\ETL\\DataFrame","class_slug":"dataframe","parameters":[{"name":"constraint","type":[{"name":"Constraint","namespace":"Flow\\ETL","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"constraints","type":[{"name":"Constraint","namespace":"Flow\\ETL","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":true,"default_value":null}],"return_type":[{"name":"self","namespace":null,"is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DataFrame.php","start_line_in_file":204,"slug":"count","name":"count","class":"Flow\\ETL\\DataFrame","class_slug":"dataframe","parameters":[],"return_type":[{"name":"int","namespace":null,"is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":false,"doc_comment":"LyoqCiAgICAgKiBAdHJpZ2dlcgogICAgICogUmV0dXJuIHRvdGFsIGNvdW50IG9mIHJvd3MgcHJvY2Vzc2VkIGJ5IHRoaXMgcGlwZWxpbmUuCiAgICAgKi8="},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DataFrame.php","start_line_in_file":224,"slug":"crossjoin","name":"crossJoin","class":"Flow\\ETL\\DataFrame","class_slug":"dataframe","parameters":[{"name":"dataFrame","type":[{"name":"self","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"prefix","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"''"}],"return_type":[{"name":"self","namespace":null,"is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":false,"doc_comment":"LyoqCiAgICAgKiBAbGF6eQogICAgICov"},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DataFrame.php","start_line_in_file":241,"slug":"display","name":"display","class":"Flow\\ETL\\DataFrame","class_slug":"dataframe","parameters":[{"name":"limit","type":[{"name":"int","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"20"},{"name":"truncate","type":[{"name":"int","namespace":null,"is_nullable":false,"is_variadic":false},{"name":"bool","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"20"},{"name":"formatter","type":[{"name":"Formatter","namespace":"Flow\\ETL","is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"Flow\\ETL\\Formatter\\AsciiTableFormatter::..."}],"return_type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":false,"doc_comment":"LyoqCiAgICAgKiBAcGFyYW0gaW50ICRsaW1pdCBtYXhpbXVtIG51bWJlcnMgb2Ygcm93cyB0byBkaXNwbGF5CiAgICAgKiBAcGFyYW0gYm9vbHxpbnQgJHRydW5jYXRlIGZhbHNlIG9yIGlmIHNldCB0byAwIGNvbHVtbnMgYXJlIG5vdCB0cnVuY2F0ZWQsIG90aGVyd2lzZSBkZWZhdWx0IHRydW5jYXRlIHRvIDIwCiAgICAgKiAgICAgICAgICAgICAgICAgICAgICAgICAgIGNoYXJhY3RlcnMKICAgICAqIEBwYXJhbSBGb3JtYXR0ZXIgJGZvcm1hdHRlcgogICAgICoKICAgICAqIEB0cmlnZ2VyCiAgICAgKgogICAgICogQHRocm93cyBJbnZhbGlkQXJndW1lbnRFeGNlcHRpb24KICAgICAqLw=="},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DataFrame.php","start_line_in_file":268,"slug":"drop","name":"drop","class":"Flow\\ETL\\DataFrame","class_slug":"dataframe","parameters":[{"name":"entries","type":[{"name":"Reference","namespace":"Flow\\ETL\\Row","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":true,"default_value":null}],"return_type":[{"name":"self","namespace":null,"is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":false,"doc_comment":"LyoqCiAgICAgKiBEcm9wIGdpdmVuIGVudHJpZXMuCiAgICAgKgogICAgICogQGxhenkKICAgICAqLw=="},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DataFrame.php","start_line_in_file":282,"slug":"dropduplicates","name":"dropDuplicates","class":"Flow\\ETL\\DataFrame","class_slug":"dataframe","parameters":[{"name":"entries","type":[{"name":"Reference","namespace":"Flow\\ETL\\Row","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":true,"default_value":null}],"return_type":[{"name":"self","namespace":null,"is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":false,"doc_comment":"LyoqCiAgICAgKiBAcGFyYW0gUmVmZXJlbmNlfHN0cmluZyAuLi4kZW50cmllcwogICAgICoKICAgICAqIEBsYXp5CiAgICAgKgogICAgICogQHJldHVybiAkdGhpcwogICAgICov"},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DataFrame.php","start_line_in_file":289,"slug":"duplicaterow","name":"duplicateRow","class":"Flow\\ETL\\DataFrame","class_slug":"dataframe","parameters":[{"name":"condition","type":[{"name":"mixed","namespace":null,"is_nullable":true,"is_variadic":false}],"has_default_value":false,"is_nullable":true,"is_variadic":false,"default_value":null},{"name":"entries","type":[{"name":"WithEntry","namespace":"Flow\\ETL","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":true,"default_value":null}],"return_type":[{"name":"self","namespace":null,"is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DataFrame.php","start_line_in_file":308,"slug":"fetch","name":"fetch","class":"Flow\\ETL\\DataFrame","class_slug":"dataframe","parameters":[{"name":"limit","type":[{"name":"int","namespace":null,"is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"null"}],"return_type":[{"name":"Rows","namespace":"Flow\\ETL","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":false,"doc_comment":"LyoqCiAgICAgKiBCZSBhd2FyZSB0aGF0IGZldGNoIGlzIG5vdCBtZW1vcnkgc2FmZSBhbmQgd2lsbCBsb2FkIGFsbCByb3dzIGludG8gbWVtb3J5LgogICAgICogSWYgeW91IHdhbnQgdG8gc2FmZWx5IGl0ZXJhdGUgb3ZlciBSb3dzIHVzZSBvZSBvZiB0aGUgZm9sbG93aW5nIG1ldGhvZHM6LgogICAgICoKICAgICAqIERhdGFGcmFtZTo6Z2V0KCkgOiBcR2VuZXJhdG9yCiAgICAgKiBEYXRhRnJhbWU6OmdldEFzQXJyYXkoKSA6IFxHZW5lcmF0b3IKICAgICAqIERhdGFGcmFtZTo6Z2V0RWFjaEFzQXJyYXkoKSA6IFxHZW5lcmF0b3IKICAgICAqCiAgICAgKiBAdHJpZ2dlcgogICAgICoKICAgICAqIEB0aHJvd3MgSW52YWxpZEFyZ3VtZW50RXhjZXB0aW9uCiAgICAgKi8="},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DataFrame.php","start_line_in_file":325,"slug":"filter","name":"filter","class":"Flow\\ETL\\DataFrame","class_slug":"dataframe","parameters":[{"name":"function","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"self","namespace":null,"is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":false,"doc_comment":"LyoqCiAgICAgKiBAbGF6eQogICAgICov"},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DataFrame.php","start_line_in_file":337,"slug":"filters","name":"filters","class":"Flow\\ETL\\DataFrame","class_slug":"dataframe","parameters":[{"name":"functions","type":[{"name":"array","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"self","namespace":null,"is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":false,"doc_comment":"LyoqCiAgICAgKiBAbGF6eQogICAgICoKICAgICAqIEBwYXJhbSBhcnJheTxTY2FsYXJGdW5jdGlvbj4gJGZ1bmN0aW9ucwogICAgICov"},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DataFrame.php","start_line_in_file":351,"slug":"foreach","name":"forEach","class":"Flow\\ETL\\DataFrame","class_slug":"dataframe","parameters":[{"name":"callback","type":[{"name":"callable","namespace":null,"is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"null"}],"return_type":[{"name":"void","namespace":null,"is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":false,"doc_comment":"LyoqCiAgICAgKiBAdHJpZ2dlcgogICAgICoKICAgICAqIEBwYXJhbSBudWxsfGNhbGxhYmxlKFJvd3MgJHJvd3MsIEZsb3dDb250ZXh0ICRjb250ZXh0KSA6IHZvaWQgJGNhbGxiYWNrCiAgICAgKi8="},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DataFrame.php","start_line_in_file":372,"slug":"get","name":"get","class":"Flow\\ETL\\DataFrame","class_slug":"dataframe","parameters":[],"return_type":[{"name":"Generator","namespace":"","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":false,"doc_comment":"LyoqCiAgICAgKiBZaWVsZHMgZWFjaCByb3cgYXMgYW4gaW5zdGFuY2Ugb2YgUm93cy4KICAgICAqCiAgICAgKiBAdHJpZ2dlcgogICAgICoKICAgICAqIEByZXR1cm4gXEdlbmVyYXRvcjxSb3dzPgogICAgICov"},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DataFrame.php","start_line_in_file":391,"slug":"getasarray","name":"getAsArray","class":"Flow\\ETL\\DataFrame","class_slug":"dataframe","parameters":[],"return_type":[{"name":"Generator","namespace":"","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":false,"doc_comment":"LyoqCiAgICAgKiBZaWVsZHMgZWFjaCByb3cgYXMgYW4gYXJyYXkuCiAgICAgKgogICAgICogQHRyaWdnZXIKICAgICAqCiAgICAgKiBAcmV0dXJuIFxHZW5lcmF0b3I8YXJyYXk8YXJyYXk8bWl4ZWQ+Pj4KICAgICAqLw=="},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DataFrame.php","start_line_in_file":410,"slug":"geteachasarray","name":"getEachAsArray","class":"Flow\\ETL\\DataFrame","class_slug":"dataframe","parameters":[],"return_type":[{"name":"Generator","namespace":"","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":false,"doc_comment":"LyoqCiAgICAgKiBZaWVsZCBlYWNoIHJvdyBhcyBhbiBhcnJheS4KICAgICAqCiAgICAgKiBAdHJpZ2dlcgogICAgICoKICAgICAqIEByZXR1cm4gXEdlbmVyYXRvcjxhcnJheTxtaXhlZD4+CiAgICAgKi8="},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DataFrame.php","start_line_in_file":431,"slug":"groupby","name":"groupBy","class":"Flow\\ETL\\DataFrame","class_slug":"dataframe","parameters":[{"name":"entries","type":[{"name":"Reference","namespace":"Flow\\ETL\\Row","is_nullable":false,"is_variadic":false},{"name":"array","namespace":null,"is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"algorithm","type":[{"name":"GroupByAlgorithmBuilder","namespace":"Flow\\ETL\\Config\\Grouping","is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"null"}],"return_type":[{"name":"GroupedDataFrame","namespace":"Flow\\ETL\\DataFrame","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":false,"doc_comment":"LyoqCiAgICAgKiBAbGF6eQogICAgICoKICAgICAqIEBwYXJhbSBHcm91cEJ5UmVmZXJlbmNlc3xSZWZlcmVuY2V8c3RyaW5nICRlbnRyaWVzIGEgc2luZ2xlIGNvbHVtbiBpcyBncm91cGVkIGJ5IG9uIGl0cyBvd24KICAgICAqIEBwYXJhbSBudWxsfEdyb3VwQnlBbGdvcml0aG1CdWlsZGVyICRhbGdvcml0aG0gbnVsbCBkZWZlcnMgdG8gY29uZmlndXJhdGlvbjsgYSBidWlsZGVyIHBpbnMgdGhlCiAgICAgKiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgYWxnb3JpdGhtIGZvciB0aGlzIG9wZXJhdGlvbiBhbmQgc2tpcHMgYW55IGF1dG9tYXRpYyBjaG9pY2UKICAgICAqLw=="},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DataFrame.php","start_line_in_file":450,"slug":"join","name":"join","class":"Flow\\ETL\\DataFrame","class_slug":"dataframe","parameters":[{"name":"dataFrame","type":[{"name":"self","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"on","type":[{"name":"Expression","namespace":"Flow\\ETL\\Join","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"type","type":[{"name":"Join","namespace":"Flow\\ETL\\Join","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"Flow\\ETL\\Join\\Join::..."},{"name":"algorithm","type":[{"name":"JoinAlgorithmBuilder","namespace":"Flow\\ETL\\Config\\Join","is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"null"}],"return_type":[{"name":"self","namespace":null,"is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":false,"doc_comment":"LyoqCiAgICAgKiBAbGF6eQogICAgICoKICAgICAqIEBwYXJhbSBudWxsfEpvaW5BbGdvcml0aG1CdWlsZGVyICRhbGdvcml0aG0gbnVsbCBkZWZlcnMgdG8gY29uZmlndXJhdGlvbjsgYSBidWlsZGVyIHBpbnMgdGhlIGFsZ29yaXRobQogICAgICogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIGZvciB0aGlzIG9wZXJhdGlvbiBhbmQgc2tpcHMgYW55IGF1dG9tYXRpYyBjaG9pY2UKICAgICAqLw=="},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DataFrame.php","start_line_in_file":478,"slug":"joineach","name":"joinEach","class":"Flow\\ETL\\DataFrame","class_slug":"dataframe","parameters":[{"name":"factory","type":[{"name":"DataFrameFactory","namespace":"Flow\\ETL","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"on","type":[{"name":"Expression","namespace":"Flow\\ETL\\Join","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"type","type":[{"name":"Join","namespace":"Flow\\ETL\\Join","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"Flow\\ETL\\Join\\Join::..."}],"return_type":[{"name":"self","namespace":null,"is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":false,"doc_comment":"LyoqCiAgICAgKiBKb2lucyBpbiBtZW1vcnkgcGVyIGJhdGNoOyBpdCBpcyBub3QgZ292ZXJuZWQgYnkgdGhlIGpvaW4gYWxnb3JpdGhtIGFuZCB0YWtlcyBubyBhbGdvcml0aG0gb3ZlcnJpZGUuCiAgICAgKgogICAgICogQGxhenkKICAgICAqCiAgICAgKiBAcGFyYW0gc3RyaW5nfEpvaW4gJHR5cGUKICAgICAqLw=="},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DataFrame.php","start_line_in_file":494,"slug":"limit","name":"limit","class":"Flow\\ETL\\DataFrame","class_slug":"dataframe","parameters":[{"name":"limit","type":[{"name":"int","namespace":null,"is_nullable":true,"is_variadic":false}],"has_default_value":false,"is_nullable":true,"is_variadic":false,"default_value":null}],"return_type":[{"name":"self","namespace":null,"is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":false,"doc_comment":"LyoqCiAgICAgKiBAbGF6eQogICAgICoKICAgICAqIEB0aHJvd3MgSW52YWxpZEFyZ3VtZW50RXhjZXB0aW9uCiAgICAgKi8="},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DataFrame.php","start_line_in_file":508,"slug":"load","name":"load","class":"Flow\\ETL\\DataFrame","class_slug":"dataframe","parameters":[{"name":"sink","type":[{"name":"Loader","namespace":"Flow\\ETL","is_nullable":false,"is_variadic":false},{"name":"Sink","namespace":"Flow\\ETL","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"self","namespace":null,"is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":false,"doc_comment":"LyoqCiAgICAgKiBAbGF6eQogICAgICov"},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DataFrame.php","start_line_in_file":554,"slug":"match","name":"match","class":"Flow\\ETL\\DataFrame","class_slug":"dataframe","parameters":[{"name":"schema","type":[{"name":"Schema","namespace":"Flow\\ETL","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"validator","type":[{"name":"SchemaValidator","namespace":"Flow\\ETL","is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"null"}],"return_type":[{"name":"self","namespace":null,"is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":false,"doc_comment":"LyoqCiAgICAgKiBAbGF6eQogICAgICoKICAgICAqIEBwYXJhbSBudWxsfFNjaGVtYVZhbGlkYXRvciAkdmFsaWRhdG9yIC0gd2hlbiBudWxsLCBTdHJpY3RWYWxpZGF0b3IgZ2V0cyBpbml0aWFsaXplZAogICAgICov"},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DataFrame.php","start_line_in_file":575,"slug":"offset","name":"offset","class":"Flow\\ETL\\DataFrame","class_slug":"dataframe","parameters":[{"name":"offset","type":[{"name":"int","namespace":null,"is_nullable":true,"is_variadic":false}],"has_default_value":false,"is_nullable":true,"is_variadic":false,"default_value":null}],"return_type":[{"name":"self","namespace":null,"is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":false,"doc_comment":"LyoqCiAgICAgKiBTa2lwIGdpdmVuIG51bWJlciBvZiByb3dzIGZyb20gdGhlIGJlZ2lubmluZyBvZiB0aGUgZGF0YXNldC4KICAgICAqIFdoZW4gJG9mZnNldCBpcyBudWxsLCBub3RoaW5nIGhhcHBlbnMgKG5vIHJvd3MgYXJlIHNraXBwZWQpLgogICAgICoKICAgICAqIFBlcmZvcm1hbmNlIE5vdGU6IERhdGFGcmFtZSBtdXN0IGl0ZXJhdGUgdGhyb3VnaCBhbmQgcHJvY2VzcyBhbGwgc2tpcHBlZCByb3dzCiAgICAgKiB0byByZWFjaCB0aGUgb2Zmc2V0IHBvc2l0aW9uLiBGb3IgbGFyZ2Ugb2Zmc2V0cywgdGhpcyBjYW4gaW1wYWN0IHBlcmZvcm1hbmNlCiAgICAgKiBhcyB0aGUgZGF0YSBzb3VyY2Ugc3RpbGwgbmVlZHMgdG8gYmUgcmVhZCBhbmQgcHJvY2Vzc2VkIHVwIHRvIHRoZSBvZmZzZXQgcG9pbnQuCiAgICAgKgogICAgICogQHBhcmFtID9pbnQ8MCwgbWF4PiAkb2Zmc2V0CiAgICAgKgogICAgICogQGxhenkKICAgICAqCiAgICAgKiBAdGhyb3dzIEludmFsaWRBcmd1bWVudEV4Y2VwdGlvbgogICAgICov"},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DataFrame.php","start_line_in_file":589,"slug":"onerror","name":"onError","class":"Flow\\ETL\\DataFrame","class_slug":"dataframe","parameters":[{"name":"handler","type":[{"name":"ErrorHandler","namespace":"Flow\\ETL","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"self","namespace":null,"is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":false,"doc_comment":"LyoqCiAgICAgKiBAbGF6eQogICAgICov"},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DataFrame.php","start_line_in_file":601,"slug":"repartition","name":"repartition","class":"Flow\\ETL\\DataFrame","class_slug":"dataframe","parameters":[{"name":"entry","type":[{"name":"Reference","namespace":"Flow\\ETL\\Row","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"entries","type":[{"name":"Reference","namespace":"Flow\\ETL\\Row","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":true,"default_value":null}],"return_type":[{"name":"self","namespace":null,"is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":false,"doc_comment":"LyoqCiAgICAgKiBAbGF6eQogICAgICogU2h1ZmZsZXMgdGhlIHN0cmVhbSBzbyBldmVyeSByb3cgc2hhcmluZyB0aGUgZ2l2ZW4gY29sdW1ucyBhcnJpdmVzIGluIG9uZSBiYXRjaC4gSXQgZG9lcyBub3QKICAgICAqIHdyaXRlIGRpcmVjdG9yaWVzIC0gdGhhdCBpcyBkZWNsYXJlZCBvbiB0aGUgbG9hZGVyLCBgdG9fY3N2KC4uLiktPnBhcnRpdGlvbkJ5KCdyZWdpb24nKWAuCiAgICAgKi8="},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DataFrame.php","start_line_in_file":613,"slug":"printrows","name":"printRows","class":"Flow\\ETL\\DataFrame","class_slug":"dataframe","parameters":[{"name":"limit","type":[{"name":"int","namespace":null,"is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"20"},{"name":"truncate","type":[{"name":"int","namespace":null,"is_nullable":false,"is_variadic":false},{"name":"bool","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"20"},{"name":"formatter","type":[{"name":"Formatter","namespace":"Flow\\ETL","is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"Flow\\ETL\\Formatter\\AsciiTableFormatter::..."}],"return_type":[{"name":"void","namespace":null,"is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":false,"doc_comment":"LyoqCiAgICAgKiBAdHJpZ2dlcgogICAgICov"},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DataFrame.php","start_line_in_file":633,"slug":"explain","name":"explain","class":"Flow\\ETL\\DataFrame","class_slug":"dataframe","parameters":[{"name":"trigger","type":[{"name":"Trigger","namespace":"Flow\\ETL\\Plan","is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"Flow\\ETL\\Plan\\Trigger::..."}],"return_type":[{"name":"Plan","namespace":"Flow\\ETL","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":false,"doc_comment":"LyoqCiAgICAgKiBUaGUgcGxhbiAkdHJpZ2dlciB3b3VsZCBydW4gb3ZlciB0aGlzIGZyYW1lLCBmcm96ZW46IGxhdGVyIHZlcmJzIG9uIHRoaXMgZnJhbWUgZG8gbm90IHJlYWNoIGl0LiBUaGUgZGVmYXVsdAogICAgICogYWRkcyBubyBjb25zdW1lciBvZiBpdHMgb3duIC0gaXQgZHJhd3MgdGhlIGZyYW1lIGFzIGJ1aWx0LCBpdHMgY2hhaW4gYW5kIGl0cyBzaW5rcy4gdG9TdHJpbmcoKSBwcmludHMgaXQgYXMgYQogICAgICogdHJlZS4gQW5zd2VycyBmcm9tIHRoZSBwbGFuIHdpdGhvdXQgcmVhZGluZyBhIHJvdy4KICAgICAqLw=="},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DataFrame.php","start_line_in_file":643,"slug":"printschema","name":"printSchema","class":"Flow\\ETL\\DataFrame","class_slug":"dataframe","parameters":[{"name":"formatter","type":[{"name":"SchemaFormatter","namespace":"Flow\\ETL\\Schema","is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"Flow\\ETL\\Row\\Formatter\\ASCIISchemaFormatter::..."}],"return_type":[{"name":"void","namespace":null,"is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":false,"doc_comment":"LyoqCiAgICAgKiBAbGF6eQogICAgICoKICAgICAqIEB0aHJvd3MgU2NoZW1hTm90RGVyaXZhYmxlRXhjZXB0aW9uCiAgICAgKi8="},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DataFrame.php","start_line_in_file":651,"slug":"rename","name":"rename","class":"Flow\\ETL\\DataFrame","class_slug":"dataframe","parameters":[{"name":"from","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"to","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"self","namespace":null,"is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":false,"doc_comment":"LyoqCiAgICAgKiBAbGF6eQogICAgICov"},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DataFrame.php","start_line_in_file":658,"slug":"renameeach","name":"renameEach","class":"Flow\\ETL\\DataFrame","class_slug":"dataframe","parameters":[{"name":"strategies","type":[{"name":"RenameEntryStrategy","namespace":"Flow\\ETL\\Transformer\\Rename","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":true,"default_value":null}],"return_type":[{"name":"self","namespace":null,"is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DataFrame.php","start_line_in_file":669,"slug":"rows","name":"rows","class":"Flow\\ETL\\DataFrame","class_slug":"dataframe","parameters":[{"name":"transformer","type":[{"name":"Transformer","namespace":"Flow\\ETL","is_nullable":false,"is_variadic":false},{"name":"Transformation","namespace":"Flow\\ETL","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"self","namespace":null,"is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":false,"doc_comment":"LyoqCiAgICAgKiBAbGF6eQogICAgICogQWxpYXMgZm9yIEVUTDo6dHJhbnNmb3JtIG1ldGhvZC4KICAgICAqLw=="},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DataFrame.php","start_line_in_file":686,"slug":"run","name":"run","class":"Flow\\ETL\\DataFrame","class_slug":"dataframe","parameters":[{"name":"analyze","type":[{"name":"Analyze","namespace":"Flow\\ETL","is_nullable":false,"is_variadic":false},{"name":"bool","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"false"}],"return_type":[{"name":"Report","namespace":"Flow\\ETL\\Dataset","is_nullable":true,"is_variadic":false}],"attributes":[],"scalar_function_chain":false,"doc_comment":"LyoqCiAgICAgKiBAdHJpZ2dlcgogICAgICoKICAgICAqIFdoZW4gYW5hbHl6aW5nIHBpcGVsaW5lIGV4ZWN1dGlvbiB3ZSBjYW4gY2hvc2UgdG8gY29sbGVjdCB2YXJpb3VzIG1ldHJpY3MgdGhyb3VnaCBhbmFseXplKCktPndpdGgqKCkgbWV0aG9kCiAgICAgKgogICAgICogLSBjb2x1bW4gc3RhdGlzdGljcyAtIGFuYWx5emUoKS0+d2l0aENvbHVtblN0YXRpc3RpY3MoKQogICAgICogLSBzY2hlbWEgLSBhbmFseXplKCktPndpdGhTY2hlbWEoKQogICAgICoKICAgICAqIEBwYXJhbSBBbmFseXplfGJvb2wgJGFuYWx5emUgLSB3aGVuIHNldCBydW4gd2lsbCByZXR1cm4gUmVwb3J0CiAgICAgKgogICAgICogQHJldHVybiAoJGFuYWx5emUgaXMgQW5hbHl6ZXx0cnVlID8gUmVwb3J0IDogbnVsbCkKICAgICAqLw=="},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DataFrame.php","start_line_in_file":714,"slug":"schema","name":"schema","class":"Flow\\ETL\\DataFrame","class_slug":"dataframe","parameters":[],"return_type":[{"name":"Schema","namespace":"Flow\\ETL","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":false,"doc_comment":"LyoqCiAgICAgKiBAbGF6eQogICAgICoKICAgICAqIEB0aHJvd3MgU2NoZW1hTm90RGVyaXZhYmxlRXhjZXB0aW9uCiAgICAgKi8="},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DataFrame.php","start_line_in_file":727,"slug":"select","name":"select","class":"Flow\\ETL\\DataFrame","class_slug":"dataframe","parameters":[{"name":"entries","type":[{"name":"Reference","namespace":"Flow\\ETL\\Row","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":true,"default_value":null}],"return_type":[{"name":"self","namespace":null,"is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":false,"doc_comment":"LyoqCiAgICAgKiBAbGF6eQogICAgICogS2VlcCBvbmx5IGdpdmVuIGVudHJpZXMuCiAgICAgKi8="},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DataFrame.php","start_line_in_file":741,"slug":"sortby","name":"sortBy","class":"Flow\\ETL\\DataFrame","class_slug":"dataframe","parameters":[{"name":"entries","type":[{"name":"Reference","namespace":"Flow\\ETL\\Row","is_nullable":false,"is_variadic":false},{"name":"array","namespace":null,"is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"algorithm","type":[{"name":"SortAlgorithmBuilder","namespace":"Flow\\ETL\\Config\\Sort","is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"null"}],"return_type":[{"name":"self","namespace":null,"is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":false,"doc_comment":"LyoqCiAgICAgKiBAbGF6eQogICAgICoKICAgICAqIEBwYXJhbSBSZWZlcmVuY2V8U29ydFJlZmVyZW5jZXN8c3RyaW5nICRlbnRyaWVzIGEgc2luZ2xlIGNvbHVtbiBpcyBzb3J0ZWQgYnkgb24gaXRzIG93bgogICAgICogQHBhcmFtIG51bGx8U29ydEFsZ29yaXRobUJ1aWxkZXIgJGFsZ29yaXRobSBudWxsIGRlZmVycyB0byBjb25maWd1cmF0aW9uOyBhIGJ1aWxkZXIgcGlucyB0aGUgYWxnb3JpdGhtCiAgICAgKiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgZm9yIHRoaXMgb3BlcmF0aW9uIGFuZCBza2lwcyBhbnkgYXV0b21hdGljIGNob2ljZQogICAgICov"},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DataFrame.php","start_line_in_file":755,"slug":"transform","name":"transform","class":"Flow\\ETL\\DataFrame","class_slug":"dataframe","parameters":[{"name":"transformer","type":[{"name":"Transformer","namespace":"Flow\\ETL","is_nullable":false,"is_variadic":false},{"name":"Transformation","namespace":"Flow\\ETL","is_nullable":false,"is_variadic":false},{"name":"Transformations","namespace":"Flow\\ETL","is_nullable":false,"is_variadic":false},{"name":"WithEntry","namespace":"Flow\\ETL","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"self","namespace":null,"is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":false,"doc_comment":"LyoqCiAgICAgKiBBbGlhcyBmb3IgRGF0YUZyYW1lOjp3aXRoKCkuCiAgICAgKgogICAgICogQGxhenkKICAgICAqLw=="},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DataFrame.php","start_line_in_file":766,"slug":"until","name":"until","class":"Flow\\ETL\\DataFrame","class_slug":"dataframe","parameters":[{"name":"function","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"self","namespace":null,"is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":false,"doc_comment":"LyoqCiAgICAgKiBUaGUgZGlmZmVyZW5jZSBiZXR3ZWVuIGZpbHRlciBhbmQgdW50aWwgaXMgdGhhdCBmaWx0ZXIgd2lsbCBrZWVwIGZpbHRlcmluZyByb3dzIHVudGlsIGV4dHJhY3RvcnMgZmluaXNoIHlpZWxkaW5nCiAgICAgKiByb3dzLiBVbnRpbCB3aWxsIHNlbmQgYSBTVE9QIHNpZ25hbCB0byB0aGUgRXh0cmFjdG9yIHdoZW4gdGhlIGNvbmRpdGlvbiBpcyBub3QgbWV0LgogICAgICoKICAgICAqIEBsYXp5CiAgICAgKi8="},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DataFrame.php","start_line_in_file":780,"slug":"void","name":"void","class":"Flow\\ETL\\DataFrame","class_slug":"dataframe","parameters":[],"return_type":[{"name":"self","namespace":null,"is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":false,"doc_comment":"LyoqCiAgICAgKiBAbGF6eQogICAgICogVGhpcyBtZXRob2QgaXMgdXNlZnVsIG1vc3RseSBpbiBkZXZlbG9wbWVudCB3aGVuCiAgICAgKiB5b3Ugd2FudCB0byBwYXVzZSBwcm9jZXNzaW5nIGF0IGNlcnRhaW4gbW9tZW50IHdpdGhvdXQKICAgICAqIHJlbW92aW5nIGNvZGUuIEFsbCBvcGVyYXRpb25zIHdpbGwgZ2V0IHByb2Nlc3NlZCB1cCB0byB0aGlzIHBvaW50LAogICAgICogZnJvbSBoZXJlIG5vIHJvd3MgYXJlIHBhc3NlZCBmb3J3YXJkLgogICAgICov"},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DataFrame.php","start_line_in_file":790,"slug":"with","name":"with","class":"Flow\\ETL\\DataFrame","class_slug":"dataframe","parameters":[{"name":"transformer","type":[{"name":"Transformer","namespace":"Flow\\ETL","is_nullable":false,"is_variadic":false},{"name":"Transformation","namespace":"Flow\\ETL","is_nullable":false,"is_variadic":false},{"name":"Transformations","namespace":"Flow\\ETL","is_nullable":false,"is_variadic":false},{"name":"WithEntry","namespace":"Flow\\ETL","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"self","namespace":null,"is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":false,"doc_comment":"LyoqCiAgICAgKiBAbGF6eQogICAgICov"},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DataFrame.php","start_line_in_file":818,"slug":"withentries","name":"withEntries","class":"Flow\\ETL\\DataFrame","class_slug":"dataframe","parameters":[{"name":"references","type":[{"name":"array","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"self","namespace":null,"is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":false,"doc_comment":"LyoqCiAgICAgKiBAbGF6eQogICAgICoKICAgICAqIEBwYXJhbSBhcnJheTxpbnQsIFdpdGhFbnRyeT58YXJyYXk8c3RyaW5nLCBTY2FsYXJGdW5jdGlvbnxXaW5kb3dGdW5jdGlvbnxXaXRoRW50cnk+ICRyZWZlcmVuY2VzCiAgICAgKi8="},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DataFrame.php","start_line_in_file":836,"slug":"withentry","name":"withEntry","class":"Flow\\ETL\\DataFrame","class_slug":"dataframe","parameters":[{"name":"entry","type":[{"name":"Definition","namespace":"Flow\\ETL\\Schema","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"reference","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"WindowFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"self","namespace":null,"is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":false,"doc_comment":"LyoqCiAgICAgKiBAcGFyYW0gRGVmaW5pdGlvbjxtaXhlZD58c3RyaW5nICRlbnRyeQogICAgICoKICAgICAqIEBsYXp5CiAgICAgKi8="},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DataFrame.php","start_line_in_file":849,"slug":"write","name":"write","class":"Flow\\ETL\\DataFrame","class_slug":"dataframe","parameters":[{"name":"sink","type":[{"name":"Loader","namespace":"Flow\\ETL","is_nullable":false,"is_variadic":false},{"name":"Sink","namespace":"Flow\\ETL","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"self","namespace":null,"is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":false,"doc_comment":"LyoqCiAgICAgKiBAbGF6eQogICAgICogQWxpYXMgZm9yIEVUTDo6bG9hZCBmdW5jdGlvbi4KICAgICAqLw=="},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DataFrame\/GroupedDataFrame.php","start_line_in_file":21,"slug":"aggregate","name":"aggregate","class":"Flow\\ETL\\DataFrame\\GroupedDataFrame","class_slug":"groupeddataframe","parameters":[{"name":"aggregations","type":[{"name":"AggregatingFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":true,"default_value":null}],"return_type":[{"name":"DataFrame","namespace":"Flow\\ETL","is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DataFrame\/GroupedDataFrame.php","start_line_in_file":28,"slug":"pivot","name":"pivot","class":"Flow\\ETL\\DataFrame\\GroupedDataFrame","class_slug":"groupeddataframe","parameters":[{"name":"ref","type":[{"name":"Reference","namespace":"Flow\\ETL\\Row","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"values","type":[{"name":"PivotValues","namespace":"Flow\\ETL\\GroupBy","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"self","namespace":null,"is_nullable":false,"is_variadic":false}],"attributes":[],"scalar_function_chain":false,"doc_comment":null}] \ No newline at end of file diff --git a/web/landing/resources/dsl.json b/web/landing/resources/dsl.json index 2c3197ee47..8da3bd4846 100644 --- a/web/landing/resources/dsl.json +++ b/web/landing/resources/dsl.json @@ -1 +1 @@ -[{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DSL\/functions.php","start_line_in_file":280,"slug":"df","name":"df","namespace":"Flow\\ETL\\DSL","parameters":[{"name":"config","type":[{"name":"Config","namespace":"Flow\\ETL","is_nullable":false,"is_variadic":false},{"name":"ConfigBuilder","namespace":"Flow\\ETL\\Config","is_nullable":false,"is_variadic":false},{"name":"null","namespace":null,"is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"null"}],"return_type":[{"name":"Flow","namespace":"Flow\\ETL","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"CORE","type":"DATA_FRAME"}},{"name":"DocumentationExample","namespace":"Flow\\Documentation\\Attribute","arguments":{"topic":"reading","example":"data_frame"}},{"name":"DocumentationExample","namespace":"Flow\\Documentation\\Attribute","arguments":{"topic":"writing","example":"overwrite"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIEFsaWFzIGZvciBkYXRhX2ZyYW1lKCkgOiBGbG93LgogKi8="},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DSL\/functions.php","start_line_in_file":288,"slug":"data-frame","name":"data_frame","namespace":"Flow\\ETL\\DSL","parameters":[{"name":"config","type":[{"name":"Config","namespace":"Flow\\ETL","is_nullable":false,"is_variadic":false},{"name":"ConfigBuilder","namespace":"Flow\\ETL\\Config","is_nullable":false,"is_variadic":false},{"name":"null","namespace":null,"is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"null"}],"return_type":[{"name":"Flow","namespace":"Flow\\ETL","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"CORE","type":"DATA_FRAME"}},{"name":"DocumentationExample","namespace":"Flow\\Documentation\\Attribute","arguments":{"topic":"reading","example":"data_frame"}},{"name":"DocumentationExample","namespace":"Flow\\Documentation\\Attribute","arguments":{"topic":"writing","example":"overwrite"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DSL\/functions.php","start_line_in_file":294,"slug":"telemetry-options","name":"telemetry_options","namespace":"Flow\\ETL\\DSL","parameters":[{"name":"trace_loading","type":[{"name":"bool","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"false"},{"name":"trace_transformations","type":[{"name":"bool","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"false"},{"name":"trace_cache","type":[{"name":"bool","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"false"},{"name":"collect_metrics","type":[{"name":"bool","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"false"}],"return_type":[{"name":"TelemetryOptions","namespace":"Flow\\ETL\\Config\\Telemetry","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"CORE","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DSL\/functions.php","start_line_in_file":306,"slug":"from-rows","name":"from_rows","namespace":"Flow\\ETL\\DSL","parameters":[{"name":"rows","type":[{"name":"Rows","namespace":"Flow\\ETL","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":true,"default_value":null}],"return_type":[{"name":"RowsExtractor","namespace":"Flow\\ETL\\Extractor","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"CORE","type":"EXTRACTOR"}},{"name":"DocumentationExample","namespace":"Flow\\Documentation\\Attribute","arguments":{"topic":"reading","example":"data_frame"}},{"name":"DocumentationExample","namespace":"Flow\\Documentation\\Attribute","arguments":{"topic":"writing","example":"overwrite"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DSL\/functions.php","start_line_in_file":313,"slug":"from-path-partitions","name":"from_path_partitions","namespace":"Flow\\ETL\\DSL","parameters":[{"name":"path","type":[{"name":"Path","namespace":"Flow\\Filesystem","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"filesystem","type":[{"name":"Filesystem","namespace":"Flow\\Filesystem","is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"Flow\\Filesystem\\Local\\NativeLocalFilesystem::..."}],"return_type":[{"name":"PathPartitionsExtractor","namespace":"Flow\\ETL\\Extractor","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"CORE","type":"EXTRACTOR"}},{"name":"DocumentationExample","namespace":"Flow\\Documentation\\Attribute","arguments":{"topic":"partitioning","example":"path_partitions"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DSL\/functions.php","start_line_in_file":329,"slug":"from-array","name":"from_array","namespace":"Flow\\ETL\\DSL","parameters":[{"name":"array","type":[{"name":"iterable","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"schema","type":[{"name":"Schema","namespace":"Flow\\ETL","is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"null"},{"name":"filesystem","type":[{"name":"Filesystem","namespace":"Flow\\Filesystem","is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"Flow\\Filesystem\\Local\\NativeLocalFilesystem::..."},{"name":"spillRoot","type":[{"name":"Path","namespace":"Flow\\Filesystem","is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"null"}],"return_type":[{"name":"ArrayExtractor","namespace":"Flow\\ETL\\Extractor","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"CORE","type":"EXTRACTOR"}},{"name":"DocumentationExample","namespace":"Flow\\Documentation\\Attribute","arguments":{"topic":"reading","example":"array"}},{"name":"DocumentationExample","namespace":"Flow\\Documentation\\Attribute","arguments":{"topic":"reading","example":"data_frame"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIEBwYXJhbSBpdGVyYWJsZTxhcnJheTxtaXhlZD4+ICRhcnJheQogKiBAcGFyYW0gbnVsbHxTY2hlbWEgJHNjaGVtYSAtIEBkZXByZWNhdGVkIHVzZSB3aXRoU2NoZW1hKCkgbWV0aG9kIGluc3RlYWQKICogQHBhcmFtIG51bGx8UGF0aCAkc3BpbGxSb290IC0gd2hlcmUgYSBub24tYXJyYXkgJGFycmF5IGlzIHNwaWxsZWQgd2hpbGUgaXQgaXMgZGVzY3JpYmVkOyBudWxsIHJlc29sdmVzIHRvCiAqICAgICAgICAgICAgICAgICAgICAgICAgICAkZmlsZXN5c3RlbS0+Z2V0U3lzdGVtVG1wRGlyKCkgYW5kIG9ubHkgb24gdGhhdCBwYXRoCiAqLw=="},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DSL\/functions.php","start_line_in_file":350,"slug":"from-cache","name":"from_cache","namespace":"Flow\\ETL\\DSL","parameters":[{"name":"id","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"fallback_extractor","type":[{"name":"Extractor","namespace":"Flow\\ETL","is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"null"},{"name":"clear","type":[{"name":"bool","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"false"},{"name":"cache","type":[{"name":"Cache","namespace":"Flow\\ETL","is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"null"}],"return_type":[{"name":"CacheExtractor","namespace":"Flow\\ETL\\Extractor","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"CORE","type":"EXTRACTOR"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIEBwYXJhbSBzdHJpbmcgJGlkIC0gY2FjaGUgaWQgZnJvbSB3aGljaCBkYXRhIHdpbGwgYmUgZXh0cmFjdGVkCiAqIEBwYXJhbSBudWxsfEV4dHJhY3RvciAkZmFsbGJhY2tfZXh0cmFjdG9yIC0gZXh0cmFjdG9yIHRoYXQgd2lsbCBiZSB1c2VkIHdoZW4gY2FjaGUgaXMgZW1wdHkgLSBAZGVwcmVjYXRlZCB1c2Ugd2l0aEZhbGxiYWNrRXh0cmFjdG9yKCkgbWV0aG9kIGluc3RlYWQKICogQHBhcmFtIGJvb2wgJGNsZWFyIC0gY2xlYXIgY2FjaGUgYWZ0ZXIgZXh0cmFjdGlvbiAtIEBkZXByZWNhdGVkIHVzZSB3aXRoQ2xlYXJPbkZpbmlzaCgpIG1ldGhvZCBpbnN0ZWFkCiAqLw=="},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DSL\/functions.php","start_line_in_file":370,"slug":"from-all","name":"from_all","namespace":"Flow\\ETL\\DSL","parameters":[{"name":"extractors","type":[{"name":"Extractor","namespace":"Flow\\ETL","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":true,"default_value":null}],"return_type":[{"name":"ChainExtractor","namespace":"Flow\\ETL\\Extractor","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"CORE","type":"EXTRACTOR"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DSL\/functions.php","start_line_in_file":376,"slug":"from-memory","name":"from_memory","namespace":"Flow\\ETL\\DSL","parameters":[{"name":"memory","type":[{"name":"Memory","namespace":"Flow\\ETL\\Memory","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"MemoryExtractor","namespace":"Flow\\ETL\\Extractor","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"CORE","type":"EXTRACTOR"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DSL\/functions.php","start_line_in_file":382,"slug":"files","name":"files","namespace":"Flow\\ETL\\DSL","parameters":[{"name":"directory","type":[{"name":"Path","namespace":"Flow\\Filesystem","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"filesystem","type":[{"name":"Filesystem","namespace":"Flow\\Filesystem","is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"Flow\\Filesystem\\Local\\NativeLocalFilesystem::..."}],"return_type":[{"name":"FilesExtractor","namespace":"Flow\\ETL\\Extractor","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"CORE","type":"EXTRACTOR"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DSL\/functions.php","start_line_in_file":388,"slug":"filesystem-cache","name":"filesystem_cache","namespace":"Flow\\ETL\\DSL","parameters":[{"name":"cache_dir","type":[{"name":"Path","namespace":"Flow\\Filesystem","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false},{"name":"null","namespace":null,"is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"null"},{"name":"filesystem","type":[{"name":"Filesystem","namespace":"Flow\\Filesystem","is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"Flow\\Filesystem\\Local\\NativeLocalFilesystem::..."},{"name":"serializer","type":[{"name":"Serializer","namespace":"Flow\\Serializer","is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"Flow\\Floe\\FloeSerializer::..."}],"return_type":[{"name":"FilesystemCache","namespace":"Flow\\ETL\\Cache\\Implementation","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"CORE","type":"DATA_FRAME"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DSL\/functions.php","start_line_in_file":400,"slug":"batched-by","name":"batched_by","namespace":"Flow\\ETL\\DSL","parameters":[{"name":"extractor","type":[{"name":"Extractor","namespace":"Flow\\ETL","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"column","type":[{"name":"Reference","namespace":"Flow\\ETL\\Row","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"min_size","type":[{"name":"int","namespace":null,"is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"null"}],"return_type":[{"name":"BatchByExtractor","namespace":"Flow\\ETL\\Extractor","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"CORE","type":"EXTRACTOR"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIEBwYXJhbSBudWxsfGludDwxLCBtYXg+ICRtaW5fc2l6ZQogKi8="},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DSL\/functions.php","start_line_in_file":414,"slug":"batches","name":"batches","namespace":"Flow\\ETL\\DSL","parameters":[{"name":"extractor","type":[{"name":"Extractor","namespace":"Flow\\ETL","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"size","type":[{"name":"int","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"BatchExtractor","namespace":"Flow\\ETL\\Extractor","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"CORE","type":"EXTRACTOR"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIEBwYXJhbSBpbnQ8MSwgbWF4PiAkc2l6ZQogKi8="},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DSL\/functions.php","start_line_in_file":420,"slug":"from-data-frame","name":"from_data_frame","namespace":"Flow\\ETL\\DSL","parameters":[{"name":"data_frame","type":[{"name":"DataFrame","namespace":"Flow\\ETL","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"DataFrameExtractor","namespace":"Flow\\ETL\\Extractor","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"CORE","type":"EXTRACTOR"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DSL\/functions.php","start_line_in_file":426,"slug":"from-sequence-date-period","name":"from_sequence_date_period","namespace":"Flow\\ETL\\DSL","parameters":[{"name":"entry_name","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"start","type":[{"name":"DateTimeInterface","namespace":"","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"interval","type":[{"name":"DateInterval","namespace":"","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"end","type":[{"name":"DateTimeInterface","namespace":"","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"options","type":[{"name":"int","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"0"}],"return_type":[{"name":"SequenceExtractor","namespace":"Flow\\ETL\\Extractor","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"CORE","type":"EXTRACTOR"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DSL\/functions.php","start_line_in_file":440,"slug":"from-sequence-date-period-recurrences","name":"from_sequence_date_period_recurrences","namespace":"Flow\\ETL\\DSL","parameters":[{"name":"entry_name","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"start","type":[{"name":"DateTimeInterface","namespace":"","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"interval","type":[{"name":"DateInterval","namespace":"","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"recurrences","type":[{"name":"int","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"options","type":[{"name":"int","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"0"}],"return_type":[{"name":"SequenceExtractor","namespace":"Flow\\ETL\\Extractor","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"CORE","type":"EXTRACTOR"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DSL\/functions.php","start_line_in_file":454,"slug":"from-sequence-number","name":"from_sequence_number","namespace":"Flow\\ETL\\DSL","parameters":[{"name":"entry_name","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"start","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false},{"name":"int","namespace":null,"is_nullable":false,"is_variadic":false},{"name":"float","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"end","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false},{"name":"int","namespace":null,"is_nullable":false,"is_variadic":false},{"name":"float","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"step","type":[{"name":"int","namespace":null,"is_nullable":false,"is_variadic":false},{"name":"float","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"1"}],"return_type":[{"name":"SequenceExtractor","namespace":"Flow\\ETL\\Extractor","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"CORE","type":"EXTRACTOR"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DSL\/functions.php","start_line_in_file":464,"slug":"to-memory","name":"to_memory","namespace":"Flow\\ETL\\DSL","parameters":[{"name":"memory","type":[{"name":"Memory","namespace":"Flow\\ETL\\Memory","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"MemoryLoader","namespace":"Flow\\ETL\\Loader","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"CORE","type":"LOADER"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DSL\/functions.php","start_line_in_file":478,"slug":"to-array","name":"to_array","namespace":"Flow\\ETL\\DSL","parameters":[{"name":"array","type":[{"name":"array","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"ArrayLoader","namespace":"Flow\\ETL\\Loader","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"CORE","type":"LOADER"}},{"name":"DocumentationExample","namespace":"Flow\\Documentation\\Attribute","arguments":{"topic":"writing","example":"array"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENvbnZlcnQgcm93cyB0byBhbiBhcnJheSBhbmQgc3RvcmUgdGhlbSBpbiBwYXNzZWQgYXJyYXkgdmFyaWFibGUuCiAqCiAqIEBwYXJhbSBhcnJheTxhcnJheS1rZXksIG1peGVkPiAkYXJyYXkKICoKICogQHBhcmFtLW91dCBhcnJheTxhcnJheTxtaXhlZD4+ICRhcnJheQogKi8="},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DSL\/functions.php","start_line_in_file":486,"slug":"to-output","name":"to_output","namespace":"Flow\\ETL\\DSL","parameters":[{"name":"truncate","type":[{"name":"int","namespace":null,"is_nullable":false,"is_variadic":false},{"name":"bool","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"20"},{"name":"output","type":[{"name":"Output","namespace":"Flow\\ETL\\Loader\\StreamLoader","is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"Flow\\ETL\\Loader\\StreamLoader\\Output::..."},{"name":"formatter","type":[{"name":"Formatter","namespace":"Flow\\ETL","is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"Flow\\ETL\\Formatter\\AsciiTableFormatter::..."},{"name":"schemaFormatter","type":[{"name":"SchemaFormatter","namespace":"Flow\\ETL\\Schema","is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"Flow\\ETL\\Row\\Formatter\\ASCIISchemaFormatter::..."}],"return_type":[{"name":"StreamLoader","namespace":"Flow\\ETL\\Loader","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"CORE","type":"LOADER"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DSL\/functions.php","start_line_in_file":496,"slug":"to-stderr","name":"to_stderr","namespace":"Flow\\ETL\\DSL","parameters":[{"name":"truncate","type":[{"name":"int","namespace":null,"is_nullable":false,"is_variadic":false},{"name":"bool","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"20"},{"name":"output","type":[{"name":"Output","namespace":"Flow\\ETL\\Loader\\StreamLoader","is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"Flow\\ETL\\Loader\\StreamLoader\\Output::..."},{"name":"formatter","type":[{"name":"Formatter","namespace":"Flow\\ETL","is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"Flow\\ETL\\Formatter\\AsciiTableFormatter::..."},{"name":"schemaFormatter","type":[{"name":"SchemaFormatter","namespace":"Flow\\ETL\\Schema","is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"Flow\\ETL\\Row\\Formatter\\ASCIISchemaFormatter::..."}],"return_type":[{"name":"StreamLoader","namespace":"Flow\\ETL\\Loader","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"CORE","type":"LOADER"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DSL\/functions.php","start_line_in_file":506,"slug":"to-stdout","name":"to_stdout","namespace":"Flow\\ETL\\DSL","parameters":[{"name":"truncate","type":[{"name":"int","namespace":null,"is_nullable":false,"is_variadic":false},{"name":"bool","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"20"},{"name":"output","type":[{"name":"Output","namespace":"Flow\\ETL\\Loader\\StreamLoader","is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"Flow\\ETL\\Loader\\StreamLoader\\Output::..."},{"name":"formatter","type":[{"name":"Formatter","namespace":"Flow\\ETL","is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"Flow\\ETL\\Formatter\\AsciiTableFormatter::..."},{"name":"schemaFormatter","type":[{"name":"SchemaFormatter","namespace":"Flow\\ETL\\Schema","is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"Flow\\ETL\\Row\\Formatter\\ASCIISchemaFormatter::..."}],"return_type":[{"name":"StreamLoader","namespace":"Flow\\ETL\\Loader","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"CORE","type":"LOADER"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DSL\/functions.php","start_line_in_file":516,"slug":"to-stream","name":"to_stream","namespace":"Flow\\ETL\\DSL","parameters":[{"name":"uri","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"truncate","type":[{"name":"int","namespace":null,"is_nullable":false,"is_variadic":false},{"name":"bool","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"20"},{"name":"output","type":[{"name":"Output","namespace":"Flow\\ETL\\Loader\\StreamLoader","is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"Flow\\ETL\\Loader\\StreamLoader\\Output::..."},{"name":"mode","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"'w'"},{"name":"formatter","type":[{"name":"Formatter","namespace":"Flow\\ETL","is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"Flow\\ETL\\Formatter\\AsciiTableFormatter::..."},{"name":"schemaFormatter","type":[{"name":"SchemaFormatter","namespace":"Flow\\ETL\\Schema","is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"Flow\\ETL\\Row\\Formatter\\ASCIISchemaFormatter::..."}],"return_type":[{"name":"StreamLoader","namespace":"Flow\\ETL\\Loader","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"CORE","type":"LOADER"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DSL\/functions.php","start_line_in_file":536,"slug":"to-transformation","name":"to_transformation","namespace":"Flow\\ETL\\DSL","parameters":[{"name":"transformer","type":[{"name":"Transformer","namespace":"Flow\\ETL","is_nullable":false,"is_variadic":false},{"name":"Transformation","namespace":"Flow\\ETL","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"sink","type":[{"name":"Loader","namespace":"Flow\\ETL","is_nullable":false,"is_variadic":false},{"name":"Sink","namespace":"Flow\\ETL","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"Transformed","namespace":"Flow\\ETL\\Sink","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"CORE","type":"LOADER"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DSL\/functions.php","start_line_in_file":542,"slug":"to-branch","name":"to_branch","namespace":"Flow\\ETL\\DSL","parameters":[{"name":"condition","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"sink","type":[{"name":"Loader","namespace":"Flow\\ETL","is_nullable":false,"is_variadic":false},{"name":"Sink","namespace":"Flow\\ETL","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"Branched","namespace":"Flow\\ETL\\Sink","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"CORE","type":"LOADER"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DSL\/functions.php","start_line_in_file":548,"slug":"rename-style","name":"rename_style","namespace":"Flow\\ETL\\DSL","parameters":[{"name":"style","type":[{"name":"StringStyles","namespace":"Flow\\ETL\\String","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"RenameCaseEntryStrategy","namespace":"Flow\\ETL\\Transformer\\Rename","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"CORE","type":"TRANSFORMER"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DSL\/functions.php","start_line_in_file":558,"slug":"rename-replace","name":"rename_replace","namespace":"Flow\\ETL\\DSL","parameters":[{"name":"search","type":[{"name":"array","namespace":null,"is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"replace","type":[{"name":"array","namespace":null,"is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"RenameReplaceEntryStrategy","namespace":"Flow\\ETL\\Transformer\\Rename","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"CORE","type":"TRANSFORMER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIEBwYXJhbSBhcnJheTxzdHJpbmc+fHN0cmluZyAkc2VhcmNoCiAqIEBwYXJhbSBhcnJheTxzdHJpbmc+fHN0cmluZyAkcmVwbGFjZQogKi8="},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DSL\/functions.php","start_line_in_file":567,"slug":"rename-map","name":"rename_map","namespace":"Flow\\ETL\\DSL","parameters":[{"name":"renames","type":[{"name":"array","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"RenameMapEntryStrategy","namespace":"Flow\\ETL\\Transformer\\Rename","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"CORE","type":"TRANSFORMER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIEBwYXJhbSBhcnJheTxhcnJheS1rZXksIHN0cmluZz4gJHJlbmFtZXMgTWFwIG9mIG9sZF9uYW1lID0+IG5ld19uYW1lCiAqLw=="},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DSL\/functions.php","start_line_in_file":576,"slug":"row","name":"row","namespace":"Flow\\ETL\\DSL","parameters":[{"name":"values","type":[{"name":"array","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"Row","namespace":"Flow\\ETL","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"CORE","type":"DATA_FRAME"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIEBwYXJhbSBhcnJheTxhcnJheS1rZXksIG1peGVkPiAkdmFsdWVzCiAqLw=="},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DSL\/functions.php","start_line_in_file":582,"slug":"rows","name":"rows","namespace":"Flow\\ETL\\DSL","parameters":[{"name":"schema","type":[{"name":"Schema","namespace":"Flow\\ETL","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"row","type":[{"name":"Row","namespace":"Flow\\ETL","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":true,"default_value":null}],"return_type":[{"name":"Rows","namespace":"Flow\\ETL","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"CORE","type":"DATA_FRAME"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DSL\/functions.php","start_line_in_file":591,"slug":"col","name":"col","namespace":"Flow\\ETL\\DSL","parameters":[{"name":"entry","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"UnresolvedReference","namespace":"Flow\\ETL\\Row","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"CORE","type":"DATA_FRAME"}}],"scalar_function_chain":true,"doc_comment":"LyoqCiAqIEFuIGFsaWFzIGZvciBgcmVmYC4KICov"},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DSL\/functions.php","start_line_in_file":601,"slug":"entry","name":"entry","namespace":"Flow\\ETL\\DSL","parameters":[{"name":"entry","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"UnresolvedReference","namespace":"Flow\\ETL\\Row","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"CORE","type":"SCALAR_FUNCTION"}},{"name":"DocumentationExample","namespace":"Flow\\Documentation\\Attribute","arguments":{"topic":"columns","example":"create"}}],"scalar_function_chain":true,"doc_comment":"LyoqCiAqIEFuIGFsaWFzIGZvciBgcmVmYC4KICov"},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DSL\/functions.php","start_line_in_file":608,"slug":"ref","name":"ref","namespace":"Flow\\ETL\\DSL","parameters":[{"name":"entry","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"UnresolvedReference","namespace":"Flow\\ETL\\Row","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"CORE","type":"SCALAR_FUNCTION"}},{"name":"DocumentationExample","namespace":"Flow\\Documentation\\Attribute","arguments":{"topic":"columns","example":"create"}}],"scalar_function_chain":true,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DSL\/functions.php","start_line_in_file":614,"slug":"structure-ref","name":"structure_ref","namespace":"Flow\\ETL\\DSL","parameters":[{"name":"entry","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"StructureFunctions","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"CORE","type":"SCALAR_FUNCTION"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DSL\/functions.php","start_line_in_file":626,"slug":"structure","name":"structure","namespace":"Flow\\ETL\\DSL","parameters":[{"name":"elements","type":[{"name":"array","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"Structure","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"CORE","type":"SCALAR_FUNCTION"}}],"scalar_function_chain":true,"doc_comment":"LyoqCiAqIEJ1aWxkcyBhIHN0cnVjdHVyZSBmcm9tIHNjYWxhciBmdW5jdGlvbnM6IG9uZSBlbGVtZW50IHBlciBrZXksIGluIGtleSBvcmRlci4KICogQW4gZWxlbWVudCBpcyBudWxsYWJsZSB3aGVuIGl0cyBmdW5jdGlvbiBpczsgdGhlIHN0cnVjdHVyZSBpdHNlbGYgbmV2ZXIgaXMuCiAqCiAqIEBwYXJhbSBhcnJheTxhcnJheS1rZXksIFNjYWxhckZ1bmN0aW9uPiAkZWxlbWVudHMKICov"},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DSL\/functions.php","start_line_in_file":632,"slug":"list-ref","name":"list_ref","namespace":"Flow\\ETL\\DSL","parameters":[{"name":"entry","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"ListFunctions","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"CORE","type":"SCALAR_FUNCTION"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DSL\/functions.php","start_line_in_file":638,"slug":"refs","name":"refs","namespace":"Flow\\ETL\\DSL","parameters":[{"name":"entries","type":[{"name":"Reference","namespace":"Flow\\ETL\\Row","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":true,"default_value":null}],"return_type":[{"name":"References","namespace":"Flow\\ETL\\Row","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"CORE","type":"SCALAR_FUNCTION"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DSL\/functions.php","start_line_in_file":644,"slug":"select","name":"select","namespace":"Flow\\ETL\\DSL","parameters":[{"name":"entries","type":[{"name":"Reference","namespace":"Flow\\ETL\\Row","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":true,"default_value":null}],"return_type":[{"name":"Select","namespace":"Flow\\ETL\\Transformation","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"CORE","type":"TRANSFORMER"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DSL\/functions.php","start_line_in_file":650,"slug":"drop","name":"drop","namespace":"Flow\\ETL\\DSL","parameters":[{"name":"entries","type":[{"name":"Reference","namespace":"Flow\\ETL\\Row","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":true,"default_value":null}],"return_type":[{"name":"Drop","namespace":"Flow\\ETL\\Transformation","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"CORE","type":"TRANSFORMER"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DSL\/functions.php","start_line_in_file":656,"slug":"add-row-index","name":"add_row_index","namespace":"Flow\\ETL\\DSL","parameters":[{"name":"column","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"'index'"},{"name":"startFrom","type":[{"name":"StartFrom","namespace":"Flow\\ETL\\Transformation\\AddRowIndex","is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"Flow\\ETL\\Transformation\\AddRowIndex\\StartFrom::..."}],"return_type":[{"name":"AddRowIndex","namespace":"Flow\\ETL\\Transformation","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"CORE","type":"TRANSFORMER"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DSL\/functions.php","start_line_in_file":665,"slug":"batch-size","name":"batch_size","namespace":"Flow\\ETL\\DSL","parameters":[{"name":"size","type":[{"name":"int","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"BatchSize","namespace":"Flow\\ETL\\Transformation","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"CORE","type":"TRANSFORMER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIEBwYXJhbSBpbnQ8MSwgbWF4PiAkc2l6ZQogKi8="},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DSL\/functions.php","start_line_in_file":671,"slug":"limit","name":"limit","namespace":"Flow\\ETL\\DSL","parameters":[{"name":"limit","type":[{"name":"int","namespace":null,"is_nullable":true,"is_variadic":false}],"has_default_value":false,"is_nullable":true,"is_variadic":false,"default_value":null}],"return_type":[{"name":"Limit","namespace":"Flow\\ETL\\Transformation","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"CORE","type":"TRANSFORMER"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DSL\/functions.php","start_line_in_file":680,"slug":"mask-columns","name":"mask_columns","namespace":"Flow\\ETL\\DSL","parameters":[{"name":"columns","type":[{"name":"array","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"[]"},{"name":"mask","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"'******'"}],"return_type":[{"name":"MaskColumns","namespace":"Flow\\ETL\\Transformation","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"CORE","type":"TRANSFORMER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIEBwYXJhbSBhcnJheTxpbnQsIHN0cmluZz4gJGNvbHVtbnMKICov"},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DSL\/functions.php","start_line_in_file":686,"slug":"optional","name":"optional","namespace":"Flow\\ETL\\DSL","parameters":[{"name":"function","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"Optional","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"CORE","type":"SCALAR_FUNCTION"}}],"scalar_function_chain":true,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DSL\/functions.php","start_line_in_file":693,"slug":"lit","name":"lit","namespace":"Flow\\ETL\\DSL","parameters":[{"name":"value","type":[{"name":"mixed","namespace":null,"is_nullable":true,"is_variadic":false}],"has_default_value":false,"is_nullable":true,"is_variadic":false,"default_value":null}],"return_type":[{"name":"Literal","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"CORE","type":"SCALAR_FUNCTION"}},{"name":"DocumentationExample","namespace":"Flow\\Documentation\\Attribute","arguments":{"topic":"columns","example":"create"}}],"scalar_function_chain":true,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DSL\/functions.php","start_line_in_file":699,"slug":"exists","name":"exists","namespace":"Flow\\ETL\\DSL","parameters":[{"name":"ref","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"Exists","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"CORE","type":"SCALAR_FUNCTION"}}],"scalar_function_chain":true,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DSL\/functions.php","start_line_in_file":705,"slug":"when","name":"when","namespace":"Flow\\ETL\\DSL","parameters":[{"name":"condition","type":[{"name":"mixed","namespace":null,"is_nullable":true,"is_variadic":false}],"has_default_value":false,"is_nullable":true,"is_variadic":false,"default_value":null},{"name":"then","type":[{"name":"mixed","namespace":null,"is_nullable":true,"is_variadic":false}],"has_default_value":false,"is_nullable":true,"is_variadic":false,"default_value":null},{"name":"else","type":[{"name":"mixed","namespace":null,"is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"null"}],"return_type":[{"name":"When","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"CORE","type":"SCALAR_FUNCTION"}}],"scalar_function_chain":true,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DSL\/functions.php","start_line_in_file":714,"slug":"structure-get","name":"structure_get","namespace":"Flow\\ETL\\DSL","parameters":[{"name":"ref","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"path","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"ArrayGet","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"CORE","type":"SCALAR_FUNCTION"}}],"scalar_function_chain":true,"doc_comment":"LyoqCiAqIEFsaWFzIGZvciBgYXJyYXlfZ2V0YC4KICov"},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DSL\/functions.php","start_line_in_file":720,"slug":"array-get","name":"array_get","namespace":"Flow\\ETL\\DSL","parameters":[{"name":"ref","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"path","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"ArrayGet","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"CORE","type":"SCALAR_FUNCTION"}}],"scalar_function_chain":true,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DSL\/functions.php","start_line_in_file":731,"slug":"structure-get-collection","name":"structure_get_collection","namespace":"Flow\\ETL\\DSL","parameters":[{"name":"ref","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"keys","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"array","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"ArrayGetCollection","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"CORE","type":"SCALAR_FUNCTION"}}],"scalar_function_chain":true,"doc_comment":"LyoqCiAqIEFsaWFzIGZvciBgYXJyYXlfZ2V0X2NvbGxlY3Rpb25gLgogKgogKiBAcGFyYW0gYXJyYXk8YXJyYXkta2V5LCBtaXhlZD58U2NhbGFyRnVuY3Rpb24gJGtleXMKICov"},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DSL\/functions.php","start_line_in_file":740,"slug":"array-get-collection","name":"array_get_collection","namespace":"Flow\\ETL\\DSL","parameters":[{"name":"ref","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"keys","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"array","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"ArrayGetCollection","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"CORE","type":"SCALAR_FUNCTION"}}],"scalar_function_chain":true,"doc_comment":"LyoqCiAqIEBwYXJhbSBhcnJheTxhcnJheS1rZXksIG1peGVkPnxTY2FsYXJGdW5jdGlvbiAka2V5cwogKi8="},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DSL\/functions.php","start_line_in_file":749,"slug":"structure-get-collection-first","name":"structure_get_collection_first","namespace":"Flow\\ETL\\DSL","parameters":[{"name":"ref","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"keys","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":true,"default_value":null}],"return_type":[{"name":"ArrayGetCollection","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"CORE","type":"SCALAR_FUNCTION"}}],"scalar_function_chain":true,"doc_comment":"LyoqCiAqIEFsaWFzIGZvciBgYXJyYXlfZ2V0X2NvbGxlY3Rpb25fZmlyc3RgLgogKi8="},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DSL\/functions.php","start_line_in_file":755,"slug":"array-get-collection-first","name":"array_get_collection_first","namespace":"Flow\\ETL\\DSL","parameters":[{"name":"ref","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"keys","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":true,"default_value":null}],"return_type":[{"name":"ArrayGetCollection","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"CORE","type":"SCALAR_FUNCTION"}}],"scalar_function_chain":true,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DSL\/functions.php","start_line_in_file":764,"slug":"array-exists","name":"array_exists","namespace":"Flow\\ETL\\DSL","parameters":[{"name":"ref","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"array","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"path","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"ArrayPathExists","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"CORE","type":"SCALAR_FUNCTION"}}],"scalar_function_chain":true,"doc_comment":"LyoqCiAqIEBwYXJhbSBhcnJheTxhcnJheS1rZXksIG1peGVkPnxTY2FsYXJGdW5jdGlvbiAkcmVmCiAqLw=="},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DSL\/functions.php","start_line_in_file":774,"slug":"array-merge","name":"array_merge","namespace":"Flow\\ETL\\DSL","parameters":[{"name":"left","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"array","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"right","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"array","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"ArrayMerge","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"CORE","type":"SCALAR_FUNCTION"}}],"scalar_function_chain":true,"doc_comment":"LyoqCiAqIEBwYXJhbSBhcnJheTxhcnJheS1rZXksIG1peGVkPnxTY2FsYXJGdW5jdGlvbiAkbGVmdAogKiBAcGFyYW0gYXJyYXk8YXJyYXkta2V5LCBtaXhlZD58U2NhbGFyRnVuY3Rpb24gJHJpZ2h0CiAqLw=="},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DSL\/functions.php","start_line_in_file":783,"slug":"array-merge-collection","name":"array_merge_collection","namespace":"Flow\\ETL\\DSL","parameters":[{"name":"array","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"array","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"ArrayMergeCollection","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"CORE","type":"SCALAR_FUNCTION"}}],"scalar_function_chain":true,"doc_comment":"LyoqCiAqIEBwYXJhbSBhcnJheTxhcnJheS1rZXksIG1peGVkPnxTY2FsYXJGdW5jdGlvbiAkYXJyYXkKICov"},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DSL\/functions.php","start_line_in_file":789,"slug":"array-key-rename","name":"array_key_rename","namespace":"Flow\\ETL\\DSL","parameters":[{"name":"ref","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"path","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"newName","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"ArrayKeyRename","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"CORE","type":"SCALAR_FUNCTION"}}],"scalar_function_chain":true,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DSL\/functions.php","start_line_in_file":795,"slug":"array-keys-style-convert","name":"array_keys_style_convert","namespace":"Flow\\ETL\\DSL","parameters":[{"name":"ref","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"style","type":[{"name":"StringStyles","namespace":"Flow\\ETL\\String","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"Flow\\ETL\\String\\StringStyles::..."}],"return_type":[{"name":"ArrayKeysStyleConvert","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"CORE","type":"SCALAR_FUNCTION"}}],"scalar_function_chain":true,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DSL\/functions.php","start_line_in_file":803,"slug":"array-sort","name":"array_sort","namespace":"Flow\\ETL\\DSL","parameters":[{"name":"function","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"sort_function","type":[{"name":"Sort","namespace":"Flow\\ETL\\Function\\ArraySort","is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"null"},{"name":"flags","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"int","namespace":null,"is_nullable":false,"is_variadic":false},{"name":"null","namespace":null,"is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"null"},{"name":"recursive","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"bool","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"true"}],"return_type":[{"name":"ArraySort","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"CORE","type":"SCALAR_FUNCTION"}}],"scalar_function_chain":true,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DSL\/functions.php","start_line_in_file":816,"slug":"array-reverse","name":"array_reverse","namespace":"Flow\\ETL\\DSL","parameters":[{"name":"function","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"array","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"preserveKeys","type":[{"name":"bool","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"false"}],"return_type":[{"name":"ArrayReverse","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"CORE","type":"SCALAR_FUNCTION"}}],"scalar_function_chain":true,"doc_comment":"LyoqCiAqIEBwYXJhbSBhcnJheTxhcnJheS1rZXksIG1peGVkPnxTY2FsYXJGdW5jdGlvbiAkZnVuY3Rpb24KICov"},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DSL\/functions.php","start_line_in_file":822,"slug":"now","name":"now","namespace":"Flow\\ETL\\DSL","parameters":[{"name":"time_zone","type":[{"name":"DateTimeZone","namespace":"","is_nullable":false,"is_variadic":false},{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"DateTimeZone::..."}],"return_type":[{"name":"Now","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"CORE","type":"SCALAR_FUNCTION"}}],"scalar_function_chain":true,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DSL\/functions.php","start_line_in_file":828,"slug":"between","name":"between","namespace":"Flow\\ETL\\DSL","parameters":[{"name":"value","type":[{"name":"mixed","namespace":null,"is_nullable":true,"is_variadic":false}],"has_default_value":false,"is_nullable":true,"is_variadic":false,"default_value":null},{"name":"lower_bound","type":[{"name":"mixed","namespace":null,"is_nullable":true,"is_variadic":false}],"has_default_value":false,"is_nullable":true,"is_variadic":false,"default_value":null},{"name":"upper_bound","type":[{"name":"mixed","namespace":null,"is_nullable":true,"is_variadic":false}],"has_default_value":false,"is_nullable":true,"is_variadic":false,"default_value":null},{"name":"boundary","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"Boundary","namespace":"Flow\\ETL\\Function\\Between","is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"Flow\\ETL\\Function\\Between\\Boundary::..."}],"return_type":[{"name":"Between","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"CORE","type":"SCALAR_FUNCTION"}}],"scalar_function_chain":true,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DSL\/functions.php","start_line_in_file":838,"slug":"to-date-time","name":"to_date_time","namespace":"Flow\\ETL\\DSL","parameters":[{"name":"ref","type":[{"name":"mixed","namespace":null,"is_nullable":true,"is_variadic":false}],"has_default_value":false,"is_nullable":true,"is_variadic":false,"default_value":null},{"name":"format","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"'Y-m-d H:i:s'"},{"name":"timeZone","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"DateTimeZone","namespace":"","is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"DateTimeZone::..."}],"return_type":[{"name":"ToDateTime","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"CORE","type":"SCALAR_FUNCTION"}}],"scalar_function_chain":true,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DSL\/functions.php","start_line_in_file":847,"slug":"to-date","name":"to_date","namespace":"Flow\\ETL\\DSL","parameters":[{"name":"ref","type":[{"name":"mixed","namespace":null,"is_nullable":true,"is_variadic":false}],"has_default_value":false,"is_nullable":true,"is_variadic":false,"default_value":null},{"name":"format","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"'Y-m-d'"},{"name":"timeZone","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"DateTimeZone","namespace":"","is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"DateTimeZone::..."}],"return_type":[{"name":"ToDate","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"CORE","type":"SCALAR_FUNCTION"}}],"scalar_function_chain":true,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DSL\/functions.php","start_line_in_file":856,"slug":"date-time-format","name":"date_time_format","namespace":"Flow\\ETL\\DSL","parameters":[{"name":"ref","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"format","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"DateTimeFormat","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"CORE","type":"SCALAR_FUNCTION"}}],"scalar_function_chain":true,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DSL\/functions.php","start_line_in_file":862,"slug":"split","name":"split","namespace":"Flow\\ETL\\DSL","parameters":[{"name":"value","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"separator","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"limit","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"int","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"9223372036854775807"}],"return_type":[{"name":"Split","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"CORE","type":"SCALAR_FUNCTION"}}],"scalar_function_chain":true,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DSL\/functions.php","start_line_in_file":875,"slug":"combine","name":"combine","namespace":"Flow\\ETL\\DSL","parameters":[{"name":"keys","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"array","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"values","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"array","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"Combine","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"CORE","type":"SCALAR_FUNCTION"}}],"scalar_function_chain":true,"doc_comment":"LyoqCiAqIEBwYXJhbSBhcnJheTxhcnJheS1rZXksIG1peGVkPnxTY2FsYXJGdW5jdGlvbiAka2V5cwogKiBAcGFyYW0gYXJyYXk8YXJyYXkta2V5LCBtaXhlZD58U2NhbGFyRnVuY3Rpb24gJHZhbHVlcwogKi8="},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DSL\/functions.php","start_line_in_file":884,"slug":"concat","name":"concat","namespace":"Flow\\ETL\\DSL","parameters":[{"name":"functions","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":true,"default_value":null}],"return_type":[{"name":"Concat","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"CORE","type":"SCALAR_FUNCTION"}}],"scalar_function_chain":true,"doc_comment":"LyoqCiAqIENvbmNhdCBhbGwgdmFsdWVzLiBJZiB5b3Ugd2FudCB0byBjb25jYXRlbmF0ZSB2YWx1ZXMgd2l0aCBzZXBhcmF0b3IgdXNlIGNvbmNhdF93cyBmdW5jdGlvbi4KICov"},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DSL\/functions.php","start_line_in_file":893,"slug":"concat-ws","name":"concat_ws","namespace":"Flow\\ETL\\DSL","parameters":[{"name":"separator","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"functions","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":true,"default_value":null}],"return_type":[{"name":"ConcatWithSeparator","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"CORE","type":"SCALAR_FUNCTION"}}],"scalar_function_chain":true,"doc_comment":"LyoqCiAqIENvbmNhdCBhbGwgdmFsdWVzIHdpdGggc2VwYXJhdG9yLgogKi8="},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DSL\/functions.php","start_line_in_file":899,"slug":"hash","name":"hash","namespace":"Flow\\ETL\\DSL","parameters":[{"name":"value","type":[{"name":"mixed","namespace":null,"is_nullable":true,"is_variadic":false}],"has_default_value":false,"is_nullable":true,"is_variadic":false,"default_value":null},{"name":"algorithm","type":[{"name":"Algorithm","namespace":"Flow\\ETL\\Hash","is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"Flow\\ETL\\Hash\\NativePHPHash::..."}],"return_type":[{"name":"Hash","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"CORE","type":"SCALAR_FUNCTION"}}],"scalar_function_chain":true,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DSL\/functions.php","start_line_in_file":908,"slug":"cast","name":"cast","namespace":"Flow\\ETL\\DSL","parameters":[{"name":"value","type":[{"name":"mixed","namespace":null,"is_nullable":true,"is_variadic":false}],"has_default_value":false,"is_nullable":true,"is_variadic":false,"default_value":null},{"name":"type","type":[{"name":"Type","namespace":"Flow\\Types","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"Cast","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"CORE","type":"SCALAR_FUNCTION"}}],"scalar_function_chain":true,"doc_comment":"LyoqCiAqIEBwYXJhbSBcRmxvd1xUeXBlc1xUeXBlPG1peGVkPnxzdHJpbmcgJHR5cGUKICov"},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DSL\/functions.php","start_line_in_file":914,"slug":"coalesce","name":"coalesce","namespace":"Flow\\ETL\\DSL","parameters":[{"name":"values","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":true,"default_value":null}],"return_type":[{"name":"Coalesce","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"CORE","type":"SCALAR_FUNCTION"}}],"scalar_function_chain":true,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DSL\/functions.php","start_line_in_file":920,"slug":"enum-name","name":"enum_name","namespace":"Flow\\ETL\\DSL","parameters":[{"name":"value","type":[{"name":"mixed","namespace":null,"is_nullable":true,"is_variadic":false}],"has_default_value":false,"is_nullable":true,"is_variadic":false,"default_value":null}],"return_type":[{"name":"EnumName","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"CORE","type":"SCALAR_FUNCTION"}}],"scalar_function_chain":true,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DSL\/functions.php","start_line_in_file":926,"slug":"enum-value","name":"enum_value","namespace":"Flow\\ETL\\DSL","parameters":[{"name":"value","type":[{"name":"mixed","namespace":null,"is_nullable":true,"is_variadic":false}],"has_default_value":false,"is_nullable":true,"is_variadic":false,"default_value":null}],"return_type":[{"name":"EnumValue","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"CORE","type":"SCALAR_FUNCTION"}}],"scalar_function_chain":true,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DSL\/functions.php","start_line_in_file":932,"slug":"count","name":"count","namespace":"Flow\\ETL\\DSL","parameters":[{"name":"function","type":[{"name":"Reference","namespace":"Flow\\ETL\\Row","is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"null"}],"return_type":[{"name":"Count","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"CORE","type":"AGGREGATING_FUNCTION"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DSL\/functions.php","start_line_in_file":944,"slug":"call","name":"call","namespace":"Flow\\ETL\\DSL","parameters":[{"name":"callable","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"return_type","type":[{"name":"Type","namespace":"Flow\\Types","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"parameters","type":[{"name":"array","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"[]"}],"return_type":[{"name":"CallUserFunc","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"CORE","type":"SCALAR_FUNCTION"}}],"scalar_function_chain":true,"doc_comment":"LyoqCiAqIENhbGxzIGEgdXNlci1kZWZpbmVkIGZ1bmN0aW9uIHdpdGggdGhlIGdpdmVuIHBhcmFtZXRlcnMuCiAqCiAqIEBwYXJhbSBUeXBlPG1peGVkPiAkcmV0dXJuX3R5cGUKICogQHBhcmFtIGFycmF5PG1peGVkPiAkcGFyYW1ldGVycwogKi8="},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DSL\/functions.php","start_line_in_file":971,"slug":"array-unpack","name":"array_unpack","namespace":"Flow\\ETL\\DSL","parameters":[{"name":"array","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"array","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"schema","type":[{"name":"Schema","namespace":"Flow\\ETL","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"ArrayUnpack","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"CORE","type":"SCALAR_FUNCTION"}}],"scalar_function_chain":true,"doc_comment":"LyoqCiAqIFVucGFja3MgZWFjaCBlbGVtZW50IG9mIGFuIGFycmF5IGludG8gYSBuZXcgZW50cnksIHVzaW5nIHRoZSBhcnJheSBrZXkgYXMgdGhlIGVudHJ5IG5hbWUuCiAqCiAqIEJlZm9yZToKICogKy0tKy0tLS0tLS0tLS0tLS0tLS0tLS0rCiAqIHxpZHwgICAgICAgICAgICAgIGFycmF5fAogKiArLS0rLS0tLS0tLS0tLS0tLS0tLS0tLSsKICogfCAxfHsiYSI6MSwiYiI6MiwiYyI6M318CiAqIHwgMnx7ImQiOjQsImUiOjUsImYiOjZ9fAogKiArLS0rLS0tLS0tLS0tLS0tLS0tLS0tLSsKICoKICogQWZ0ZXI6CiAqICstLSstLS0tLSstLS0tLSstLS0tLSstLS0tLSstLS0tLSsKICogfGlkfGFyci5ifGFyci5jfGFyci5kfGFyci5lfGFyci5mfAogKiArLS0rLS0tLS0rLS0tLS0rLS0tLS0rLS0tLS0rLS0tLS0rCiAqIHwgMXwgICAgMnwgICAgM3wgICAgIHwgICAgIHwgICAgIHwKICogfCAyfCAgICAgfCAgICAgfCAgICA0fCAgICA1fCAgICA2fAogKiArLS0rLS0tLS0rLS0tLS0rLS0tLS0rLS0tLS0rLS0tLS0rCiAqCiAqIEBwYXJhbSBhcnJheTxhcnJheS1rZXksIG1peGVkPnxTY2FsYXJGdW5jdGlvbiAkYXJyYXkKICov"},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DSL\/functions.php","start_line_in_file":1001,"slug":"array-expand","name":"array_expand","namespace":"Flow\\ETL\\DSL","parameters":[{"name":"function","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"expand","type":[{"name":"ArrayExpand","namespace":"Flow\\ETL\\Function\\ArrayExpand","is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"Flow\\ETL\\Function\\ArrayExpand\\ArrayExpand::..."}],"return_type":[{"name":"ArrayExpand","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"CORE","type":"SCALAR_FUNCTION"}}],"scalar_function_chain":true,"doc_comment":"LyoqCiAqIEV4cGFuZHMgZWFjaCB2YWx1ZSBpbnRvIGVudHJ5LCBpZiB0aGVyZSBhcmUgbW9yZSB0aGFuIG9uZSB2YWx1ZSwgbXVsdGlwbGUgcm93cyB3aWxsIGJlIGNyZWF0ZWQuCiAqIEFycmF5IGtleXMgYXJlIGlnbm9yZWQsIG9ubHkgdmFsdWVzIGFyZSB1c2VkIHRvIGNyZWF0ZSBuZXcgcm93cy4KICogTmVzdGVkIGluIGFub3RoZXIgZnVuY3Rpb24gKHN0cnVjdHVyZSgpLCBjb25jYXQoKSwgLi4uKSBpdCBzdGlsbCBnaXZlcyBvbmUgcm93IHBlciBlbGVtZW50LiBTZXZlcmFsCiAqIGV4cGFuZHMgaW4gb25lIGV4cHJlc3Npb24gYXJlIHppcHBlZCB0byB0aGUgbG9uZ2VzdCBsaXN0OyBhIHNob3J0ZXIgb25lIGdpdmVzIG51bGwsIHNvIGl0cyBlbGVtZW50CiAqIHR5cGUgYmVjb21lcyBudWxsYWJsZS4gSXQgaXMgcmVmdXNlZCBpbnNpZGUgYW5vdGhlciBhcnJheV9leHBhbmQoKSBhbmQgaW4gZmlsdGVyKCksIHVudGlsKCksCiAqIGR1cGxpY2F0ZVJvdygpLCBhZ2dyZWdhdGUoKSwgb3ZlcigpIGFuZCBvbkVhY2goKS4KICoKICogQmVmb3JlOgogKiAgICstLSstLS0tLS0tLS0tLS0tLS0tLS0tKwogKiAgIHxpZHwgICAgICAgICAgICAgIGFycmF5fAogKiAgICstLSstLS0tLS0tLS0tLS0tLS0tLS0tKwogKiAgIHwgMXx7ImEiOjEsImIiOjIsImMiOjN9fAogKiAgICstLSstLS0tLS0tLS0tLS0tLS0tLS0tKwogKgogKiBBZnRlcjoKICogICArLS0rLS0tLS0tLS0rCiAqICAgfGlkfGV4cGFuZGVkfAogKiAgICstLSstLS0tLS0tLSsKICogICB8IDF8ICAgICAgIDF8CiAqICAgfCAxfCAgICAgICAyfAogKiAgIHwgMXwgICAgICAgM3wKICogICArLS0rLS0tLS0tLS0rCiAqLw=="},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DSL\/functions.php","start_line_in_file":1007,"slug":"size","name":"size","namespace":"Flow\\ETL\\DSL","parameters":[{"name":"value","type":[{"name":"mixed","namespace":null,"is_nullable":true,"is_variadic":false}],"has_default_value":false,"is_nullable":true,"is_variadic":false,"default_value":null}],"return_type":[{"name":"Size","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"CORE","type":"SCALAR_FUNCTION"}}],"scalar_function_chain":true,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DSL\/functions.php","start_line_in_file":1013,"slug":"uuid-v4","name":"uuid_v4","namespace":"Flow\\ETL\\DSL","parameters":[],"return_type":[{"name":"Uuid","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"CORE","type":"SCALAR_FUNCTION"}}],"scalar_function_chain":true,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DSL\/functions.php","start_line_in_file":1019,"slug":"uuid-v7","name":"uuid_v7","namespace":"Flow\\ETL\\DSL","parameters":[{"name":"value","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"DateTimeInterface","namespace":"","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"Uuid","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"CORE","type":"SCALAR_FUNCTION"}}],"scalar_function_chain":true,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DSL\/functions.php","start_line_in_file":1025,"slug":"ulid","name":"ulid","namespace":"Flow\\ETL\\DSL","parameters":[{"name":"value","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false},{"name":"null","namespace":null,"is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"null"}],"return_type":[{"name":"Ulid","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"CORE","type":"SCALAR_FUNCTION"}}],"scalar_function_chain":true,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DSL\/functions.php","start_line_in_file":1031,"slug":"lower","name":"lower","namespace":"Flow\\ETL\\DSL","parameters":[{"name":"value","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"ToLower","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"CORE","type":"SCALAR_FUNCTION"}}],"scalar_function_chain":true,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DSL\/functions.php","start_line_in_file":1037,"slug":"capitalize","name":"capitalize","namespace":"Flow\\ETL\\DSL","parameters":[{"name":"value","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"Capitalize","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"CORE","type":"SCALAR_FUNCTION"}}],"scalar_function_chain":true,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DSL\/functions.php","start_line_in_file":1043,"slug":"upper","name":"upper","namespace":"Flow\\ETL\\DSL","parameters":[{"name":"value","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"ToUpper","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"CORE","type":"SCALAR_FUNCTION"}}],"scalar_function_chain":true,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DSL\/functions.php","start_line_in_file":1049,"slug":"all","name":"all","namespace":"Flow\\ETL\\DSL","parameters":[{"name":"functions","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":true,"default_value":null}],"return_type":[{"name":"All","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"CORE","type":"SCALAR_FUNCTION"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DSL\/functions.php","start_line_in_file":1055,"slug":"any","name":"any","namespace":"Flow\\ETL\\DSL","parameters":[{"name":"values","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":true,"default_value":null}],"return_type":[{"name":"Any","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"CORE","type":"SCALAR_FUNCTION"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DSL\/functions.php","start_line_in_file":1061,"slug":"not","name":"not","namespace":"Flow\\ETL\\DSL","parameters":[{"name":"value","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"Not","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"CORE","type":"SCALAR_FUNCTION"}}],"scalar_function_chain":true,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DSL\/functions.php","start_line_in_file":1067,"slug":"to-timezone","name":"to_timezone","namespace":"Flow\\ETL\\DSL","parameters":[{"name":"value","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"DateTimeInterface","namespace":"","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"timeZone","type":[{"name":"DateTimeZone","namespace":"","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"ToTimeZone","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"CORE","type":"SCALAR_FUNCTION"}}],"scalar_function_chain":true,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DSL\/functions.php","start_line_in_file":1073,"slug":"ignore-error-handler","name":"ignore_error_handler","namespace":"Flow\\ETL\\DSL","parameters":[],"return_type":[{"name":"IgnoreError","namespace":"Flow\\ETL\\ErrorHandler","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"CORE","type":"DATA_FRAME"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DSL\/functions.php","start_line_in_file":1079,"slug":"skip-rows-handler","name":"skip_rows_handler","namespace":"Flow\\ETL\\DSL","parameters":[],"return_type":[{"name":"SkipRows","namespace":"Flow\\ETL\\ErrorHandler","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"CORE","type":"DATA_FRAME"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DSL\/functions.php","start_line_in_file":1085,"slug":"throw-error-handler","name":"throw_error_handler","namespace":"Flow\\ETL\\DSL","parameters":[],"return_type":[{"name":"ThrowError","namespace":"Flow\\ETL\\ErrorHandler","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"CORE","type":"DATA_FRAME"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DSL\/functions.php","start_line_in_file":1091,"slug":"regex-replace","name":"regex_replace","namespace":"Flow\\ETL\\DSL","parameters":[{"name":"pattern","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"replacement","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"subject","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"limit","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"int","namespace":null,"is_nullable":false,"is_variadic":false},{"name":"null","namespace":null,"is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"null"}],"return_type":[{"name":"RegexReplace","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"CORE","type":"SCALAR_FUNCTION"}}],"scalar_function_chain":true,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DSL\/functions.php","start_line_in_file":1101,"slug":"regex-match-all","name":"regex_match_all","namespace":"Flow\\ETL\\DSL","parameters":[{"name":"pattern","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"subject","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"flags","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"int","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"0"},{"name":"offset","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"int","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"0"}],"return_type":[{"name":"RegexMatchAll","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"CORE","type":"SCALAR_FUNCTION"}}],"scalar_function_chain":true,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DSL\/functions.php","start_line_in_file":1111,"slug":"regex-match","name":"regex_match","namespace":"Flow\\ETL\\DSL","parameters":[{"name":"pattern","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"subject","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"flags","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"int","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"0"},{"name":"offset","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"int","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"0"}],"return_type":[{"name":"RegexMatch","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"CORE","type":"SCALAR_FUNCTION"}}],"scalar_function_chain":true,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DSL\/functions.php","start_line_in_file":1121,"slug":"regex","name":"regex","namespace":"Flow\\ETL\\DSL","parameters":[{"name":"pattern","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"subject","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"flags","type":[{"name":"int","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"0"},{"name":"offset","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"int","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"0"}],"return_type":[{"name":"Regex","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"CORE","type":"SCALAR_FUNCTION"}}],"scalar_function_chain":true,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DSL\/functions.php","start_line_in_file":1131,"slug":"regex-all","name":"regex_all","namespace":"Flow\\ETL\\DSL","parameters":[{"name":"pattern","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"subject","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"flags","type":[{"name":"int","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"0"},{"name":"offset","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"int","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"0"}],"return_type":[{"name":"RegexAll","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"CORE","type":"SCALAR_FUNCTION"}}],"scalar_function_chain":true,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DSL\/functions.php","start_line_in_file":1141,"slug":"sprintf","name":"sprintf","namespace":"Flow\\ETL\\DSL","parameters":[{"name":"format","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"args","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false},{"name":"int","namespace":null,"is_nullable":false,"is_variadic":false},{"name":"float","namespace":null,"is_nullable":false,"is_variadic":false},{"name":"null","namespace":null,"is_nullable":true,"is_variadic":false}],"has_default_value":false,"is_nullable":true,"is_variadic":true,"default_value":null}],"return_type":[{"name":"Sprintf","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"CORE","type":"SCALAR_FUNCTION"}}],"scalar_function_chain":true,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DSL\/functions.php","start_line_in_file":1147,"slug":"sanitize","name":"sanitize","namespace":"Flow\\ETL\\DSL","parameters":[{"name":"value","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"placeholder","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"'*'"},{"name":"skipCharacters","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"int","namespace":null,"is_nullable":false,"is_variadic":false},{"name":"null","namespace":null,"is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"null"}],"return_type":[{"name":"Sanitize","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"CORE","type":"SCALAR_FUNCTION"}}],"scalar_function_chain":true,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DSL\/functions.php","start_line_in_file":1156,"slug":"round","name":"round","namespace":"Flow\\ETL\\DSL","parameters":[{"name":"value","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"int","namespace":null,"is_nullable":false,"is_variadic":false},{"name":"float","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"precision","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"int","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"2"},{"name":"mode","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"int","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"1"}],"return_type":[{"name":"Round","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"CORE","type":"SCALAR_FUNCTION"}}],"scalar_function_chain":true,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DSL\/functions.php","start_line_in_file":1165,"slug":"number-format","name":"number_format","namespace":"Flow\\ETL\\DSL","parameters":[{"name":"value","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"int","namespace":null,"is_nullable":false,"is_variadic":false},{"name":"float","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"decimals","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"int","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"2"},{"name":"decimal_separator","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"'.'"},{"name":"thousands_separator","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"','"}],"return_type":[{"name":"NumberFormat","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"CORE","type":"SCALAR_FUNCTION"}}],"scalar_function_chain":true,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DSL\/functions.php","start_line_in_file":1179,"slug":"array-to-row","name":"array_to_row","namespace":"Flow\\ETL\\DSL","parameters":[{"name":"data","type":[{"name":"array","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"schema","type":[{"name":"Schema","namespace":"Flow\\ETL","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"hydrator","type":[{"name":"Hydrator","namespace":"Flow\\ETL\\Row","is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"Flow\\ETL\\Row\\AdaptiveRowHydrator::..."},{"name":"partitions","type":[{"name":"Partitions","namespace":"Flow\\Filesystem","is_nullable":false,"is_variadic":false},{"name":"array","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"[]"}],"return_type":[{"name":"Row","namespace":"Flow\\ETL","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"CORE","type":"DATA_FRAME"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIEBwYXJhbSBhcnJheTxhcnJheTxtaXhlZD4+fGFycmF5PG1peGVkfHN0cmluZz4gJGRhdGEKICogQHBhcmFtIGFycmF5PFBhcnRpdGlvbj58UGFydGl0aW9ucyAkcGFydGl0aW9ucwogKi8="},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DSL\/functions.php","start_line_in_file":1205,"slug":"array-to-rows","name":"array_to_rows","namespace":"Flow\\ETL\\DSL","parameters":[{"name":"data","type":[{"name":"array","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"schema","type":[{"name":"Schema","namespace":"Flow\\ETL","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"hydrator","type":[{"name":"Hydrator","namespace":"Flow\\ETL\\Row","is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"Flow\\ETL\\Row\\AdaptiveRowHydrator::..."}],"return_type":[{"name":"Rows","namespace":"Flow\\ETL","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"CORE","type":"DATA_FRAME"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIEBwYXJhbSBhcnJheTxhcnJheTxtaXhlZD4+fGFycmF5PG1peGVkfHN0cmluZz4gJGRhdGEKICov"},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DSL\/functions.php","start_line_in_file":1242,"slug":"rank","name":"rank","namespace":"Flow\\ETL\\DSL","parameters":[],"return_type":[{"name":"Rank","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"CORE","type":"WINDOW_FUNCTION"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DSL\/functions.php","start_line_in_file":1248,"slug":"dens-rank","name":"dens_rank","namespace":"Flow\\ETL\\DSL","parameters":[],"return_type":[{"name":"DenseRank","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"CORE","type":"WINDOW_FUNCTION"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DSL\/functions.php","start_line_in_file":1254,"slug":"dense-rank","name":"dense_rank","namespace":"Flow\\ETL\\DSL","parameters":[],"return_type":[{"name":"DenseRank","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"CORE","type":"WINDOW_FUNCTION"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DSL\/functions.php","start_line_in_file":1260,"slug":"average","name":"average","namespace":"Flow\\ETL\\DSL","parameters":[{"name":"ref","type":[{"name":"Reference","namespace":"Flow\\ETL\\Row","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"scale","type":[{"name":"int","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"2"},{"name":"rounding","type":[{"name":"Rounding","namespace":"Flow\\Calculator","is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"Flow\\Calculator\\Rounding::..."}],"return_type":[{"name":"Average","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"CORE","type":"AGGREGATING_FUNCTION"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DSL\/functions.php","start_line_in_file":1266,"slug":"greatest","name":"greatest","namespace":"Flow\\ETL\\DSL","parameters":[{"name":"values","type":[{"name":"mixed","namespace":null,"is_nullable":true,"is_variadic":false}],"has_default_value":false,"is_nullable":true,"is_variadic":true,"default_value":null}],"return_type":[{"name":"Greatest","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"CORE","type":"SCALAR_FUNCTION"}}],"scalar_function_chain":true,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DSL\/functions.php","start_line_in_file":1272,"slug":"least","name":"least","namespace":"Flow\\ETL\\DSL","parameters":[{"name":"values","type":[{"name":"mixed","namespace":null,"is_nullable":true,"is_variadic":false}],"has_default_value":false,"is_nullable":true,"is_variadic":true,"default_value":null}],"return_type":[{"name":"Least","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"CORE","type":"SCALAR_FUNCTION"}}],"scalar_function_chain":true,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DSL\/functions.php","start_line_in_file":1278,"slug":"collect","name":"collect","namespace":"Flow\\ETL\\DSL","parameters":[{"name":"ref","type":[{"name":"Reference","namespace":"Flow\\ETL\\Row","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"Collect","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"CORE","type":"AGGREGATING_FUNCTION"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DSL\/functions.php","start_line_in_file":1284,"slug":"string-agg","name":"string_agg","namespace":"Flow\\ETL\\DSL","parameters":[{"name":"ref","type":[{"name":"Reference","namespace":"Flow\\ETL\\Row","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"separator","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"', '"},{"name":"sort","type":[{"name":"SortOrder","namespace":"Flow\\ETL\\Row","is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"null"}],"return_type":[{"name":"StringAggregate","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"CORE","type":"AGGREGATING_FUNCTION"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DSL\/functions.php","start_line_in_file":1290,"slug":"collect-unique","name":"collect_unique","namespace":"Flow\\ETL\\DSL","parameters":[{"name":"ref","type":[{"name":"Reference","namespace":"Flow\\ETL\\Row","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"CollectUnique","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"CORE","type":"AGGREGATING_FUNCTION"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DSL\/functions.php","start_line_in_file":1296,"slug":"window","name":"window","namespace":"Flow\\ETL\\DSL","parameters":[],"return_type":[{"name":"Window","namespace":"Flow\\ETL","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"CORE","type":"DATA_FRAME"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DSL\/functions.php","start_line_in_file":1302,"slug":"unbounded-preceding","name":"unbounded_preceding","namespace":"Flow\\ETL\\DSL","parameters":[],"return_type":[{"name":"FrameBound","namespace":"Flow\\ETL\\Window","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"CORE","type":"DATA_FRAME"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DSL\/functions.php","start_line_in_file":1308,"slug":"preceding","name":"preceding","namespace":"Flow\\ETL\\DSL","parameters":[{"name":"offset","type":[{"name":"int","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"FrameBound","namespace":"Flow\\ETL\\Window","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"CORE","type":"DATA_FRAME"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DSL\/functions.php","start_line_in_file":1314,"slug":"current-row","name":"current_row","namespace":"Flow\\ETL\\DSL","parameters":[],"return_type":[{"name":"FrameBound","namespace":"Flow\\ETL\\Window","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"CORE","type":"DATA_FRAME"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DSL\/functions.php","start_line_in_file":1320,"slug":"following","name":"following","namespace":"Flow\\ETL\\DSL","parameters":[{"name":"offset","type":[{"name":"int","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"FrameBound","namespace":"Flow\\ETL\\Window","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"CORE","type":"DATA_FRAME"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DSL\/functions.php","start_line_in_file":1326,"slug":"unbounded-following","name":"unbounded_following","namespace":"Flow\\ETL\\DSL","parameters":[],"return_type":[{"name":"FrameBound","namespace":"Flow\\ETL\\Window","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"CORE","type":"DATA_FRAME"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DSL\/functions.php","start_line_in_file":1332,"slug":"sum","name":"sum","namespace":"Flow\\ETL\\DSL","parameters":[{"name":"ref","type":[{"name":"Reference","namespace":"Flow\\ETL\\Row","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"exact","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"bool","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"false"}],"return_type":[{"name":"Sum","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"CORE","type":"AGGREGATING_FUNCTION"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DSL\/functions.php","start_line_in_file":1338,"slug":"first","name":"first","namespace":"Flow\\ETL\\DSL","parameters":[{"name":"ref","type":[{"name":"Reference","namespace":"Flow\\ETL\\Row","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"First","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"CORE","type":"AGGREGATING_FUNCTION"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DSL\/functions.php","start_line_in_file":1344,"slug":"last","name":"last","namespace":"Flow\\ETL\\DSL","parameters":[{"name":"ref","type":[{"name":"Reference","namespace":"Flow\\ETL\\Row","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"Last","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"CORE","type":"AGGREGATING_FUNCTION"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DSL\/functions.php","start_line_in_file":1350,"slug":"max","name":"max","namespace":"Flow\\ETL\\DSL","parameters":[{"name":"ref","type":[{"name":"Reference","namespace":"Flow\\ETL\\Row","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"Max","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"CORE","type":"AGGREGATING_FUNCTION"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DSL\/functions.php","start_line_in_file":1356,"slug":"min","name":"min","namespace":"Flow\\ETL\\DSL","parameters":[{"name":"ref","type":[{"name":"Reference","namespace":"Flow\\ETL\\Row","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"Min","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"CORE","type":"AGGREGATING_FUNCTION"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DSL\/functions.php","start_line_in_file":1362,"slug":"row-number","name":"row_number","namespace":"Flow\\ETL\\DSL","parameters":[],"return_type":[{"name":"RowNumber","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"CORE","type":"DATA_FRAME"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DSL\/functions.php","start_line_in_file":1373,"slug":"schema","name":"schema","namespace":"Flow\\ETL\\DSL","parameters":[{"name":"definitions","type":[{"name":"Definition","namespace":"Flow\\ETL\\Schema","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":true,"default_value":null}],"return_type":[{"name":"Schema","namespace":"Flow\\ETL","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"CORE","type":"SCHEMA"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIEBwYXJhbSBEZWZpbml0aW9uPG1peGVkPiAuLi4kZGVmaW5pdGlvbnMKICoKICogQHJldHVybiBTY2hlbWEKICov"},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DSL\/functions.php","start_line_in_file":1382,"slug":"schema-to-json","name":"schema_to_json","namespace":"Flow\\ETL\\DSL","parameters":[{"name":"schema","type":[{"name":"Schema","namespace":"Flow\\ETL","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"pretty","type":[{"name":"bool","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"false"}],"return_type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"CORE","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIEBwYXJhbSBTY2hlbWEgJHNjaGVtYQogKi8="},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DSL\/functions.php","start_line_in_file":1391,"slug":"schema-to-php","name":"schema_to_php","namespace":"Flow\\ETL\\DSL","parameters":[{"name":"schema","type":[{"name":"Schema","namespace":"Flow\\ETL","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"valueFormatter","type":[{"name":"ValueFormatter","namespace":"Flow\\ETL\\Schema\\Formatter\\PHPFormatter","is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"Flow\\ETL\\Schema\\Formatter\\PHPFormatter\\ValueFormatter::..."},{"name":"typeFormatter","type":[{"name":"TypeFormatter","namespace":"Flow\\ETL\\Schema\\Formatter\\PHPFormatter","is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"Flow\\ETL\\Schema\\Formatter\\PHPFormatter\\TypeFormatter::..."}],"return_type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"CORE","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIEBwYXJhbSBTY2hlbWEgJHNjaGVtYQogKi8="},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DSL\/functions.php","start_line_in_file":1403,"slug":"schema-to-ascii","name":"schema_to_ascii","namespace":"Flow\\ETL\\DSL","parameters":[{"name":"schema","type":[{"name":"Schema","namespace":"Flow\\ETL","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"formatter","type":[{"name":"SchemaFormatter","namespace":"Flow\\ETL\\Schema","is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"null"}],"return_type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"CORE","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIEBwYXJhbSBTY2hlbWEgJHNjaGVtYQogKi8="},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DSL\/functions.php","start_line_in_file":1413,"slug":"schema-validate","name":"schema_validate","namespace":"Flow\\ETL\\DSL","parameters":[{"name":"expected","type":[{"name":"Schema","namespace":"Flow\\ETL","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"given","type":[{"name":"Schema","namespace":"Flow\\ETL","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"validator","type":[{"name":"SchemaValidator","namespace":"Flow\\ETL","is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"Flow\\ETL\\Schema\\Validator\\StrictValidator::..."}],"return_type":[{"name":"ValidationContext","namespace":"Flow\\ETL\\Schema\\Validator","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"CORE","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIEBwYXJhbSBTY2hlbWEgJGV4cGVjdGVkCiAqIEBwYXJhbSBTY2hlbWEgJGdpdmVuCiAqLw=="},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DSL\/functions.php","start_line_in_file":1422,"slug":"schema-evolving-validator","name":"schema_evolving_validator","namespace":"Flow\\ETL\\DSL","parameters":[],"return_type":[{"name":"EvolvingValidator","namespace":"Flow\\ETL\\Schema\\Validator","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"CORE","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DSL\/functions.php","start_line_in_file":1428,"slug":"schema-strict-validator","name":"schema_strict_validator","namespace":"Flow\\ETL\\DSL","parameters":[],"return_type":[{"name":"StrictValidator","namespace":"Flow\\ETL\\Schema\\Validator","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"CORE","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DSL\/functions.php","start_line_in_file":1434,"slug":"schema-selective-validator","name":"schema_selective_validator","namespace":"Flow\\ETL\\DSL","parameters":[],"return_type":[{"name":"SelectiveValidator","namespace":"Flow\\ETL\\Schema\\Validator","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"CORE","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DSL\/functions.php","start_line_in_file":1443,"slug":"schema-from-json","name":"schema_from_json","namespace":"Flow\\ETL\\DSL","parameters":[{"name":"schema","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"Schema","namespace":"Flow\\ETL","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"CORE","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIEByZXR1cm4gU2NoZW1hCiAqLw=="},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DSL\/functions.php","start_line_in_file":1456,"slug":"schema-metadata","name":"schema_metadata","namespace":"Flow\\ETL\\DSL","parameters":[{"name":"metadata","type":[{"name":"array","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"[]"}],"return_type":[{"name":"Metadata","namespace":"Flow\\ETL\\Schema","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"CORE","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIEBwYXJhbSBhcnJheTxzdHJpbmcsIGFycmF5PGJvb2x8ZmxvYXR8aW50fHN0cmluZz58Ym9vbHxmbG9hdHxpbnR8c3RyaW5nPiAkbWV0YWRhdGEKICov"},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DSL\/functions.php","start_line_in_file":1465,"slug":"int-schema","name":"int_schema","namespace":"Flow\\ETL\\DSL","parameters":[{"name":"name","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"nullable","type":[{"name":"bool","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"false"},{"name":"metadata","type":[{"name":"Metadata","namespace":"Flow\\ETL\\Schema","is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"null"}],"return_type":[{"name":"IntegerDefinition","namespace":"Flow\\ETL\\Schema\\Definition","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"CORE","type":"SCHEMA"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIEFsaWFzIGZvciBgaW50ZWdlcl9zY2hlbWFgLgogKi8="},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DSL\/functions.php","start_line_in_file":1471,"slug":"integer-schema","name":"integer_schema","namespace":"Flow\\ETL\\DSL","parameters":[{"name":"name","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"nullable","type":[{"name":"bool","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"false"},{"name":"metadata","type":[{"name":"Metadata","namespace":"Flow\\ETL\\Schema","is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"null"}],"return_type":[{"name":"IntegerDefinition","namespace":"Flow\\ETL\\Schema\\Definition","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"CORE","type":"SCHEMA"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DSL\/functions.php","start_line_in_file":1480,"slug":"str-schema","name":"str_schema","namespace":"Flow\\ETL\\DSL","parameters":[{"name":"name","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"nullable","type":[{"name":"bool","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"false"},{"name":"metadata","type":[{"name":"Metadata","namespace":"Flow\\ETL\\Schema","is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"null"}],"return_type":[{"name":"StringDefinition","namespace":"Flow\\ETL\\Schema\\Definition","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"CORE","type":"SCHEMA"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIEFsaWFzIGZvciBgc3RyaW5nX3NjaGVtYWAuCiAqLw=="},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DSL\/functions.php","start_line_in_file":1486,"slug":"string-schema","name":"string_schema","namespace":"Flow\\ETL\\DSL","parameters":[{"name":"name","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"nullable","type":[{"name":"bool","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"false"},{"name":"metadata","type":[{"name":"Metadata","namespace":"Flow\\ETL\\Schema","is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"null"}],"return_type":[{"name":"StringDefinition","namespace":"Flow\\ETL\\Schema\\Definition","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"CORE","type":"SCHEMA"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DSL\/functions.php","start_line_in_file":1492,"slug":"bool-schema","name":"bool_schema","namespace":"Flow\\ETL\\DSL","parameters":[{"name":"name","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"nullable","type":[{"name":"bool","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"false"},{"name":"metadata","type":[{"name":"Metadata","namespace":"Flow\\ETL\\Schema","is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"null"}],"return_type":[{"name":"BooleanDefinition","namespace":"Flow\\ETL\\Schema\\Definition","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"CORE","type":"SCHEMA"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DSL\/functions.php","start_line_in_file":1498,"slug":"float-schema","name":"float_schema","namespace":"Flow\\ETL\\DSL","parameters":[{"name":"name","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"nullable","type":[{"name":"bool","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"false"},{"name":"metadata","type":[{"name":"Metadata","namespace":"Flow\\ETL\\Schema","is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"null"}],"return_type":[{"name":"FloatDefinition","namespace":"Flow\\ETL\\Schema\\Definition","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"CORE","type":"SCHEMA"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DSL\/functions.php","start_line_in_file":1512,"slug":"map-schema","name":"map_schema","namespace":"Flow\\ETL\\DSL","parameters":[{"name":"name","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"type","type":[{"name":"MapType","namespace":"Flow\\Types\\Type\\Logical","is_nullable":false,"is_variadic":false},{"name":"Type","namespace":"Flow\\Types","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"nullable","type":[{"name":"bool","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"false"},{"name":"metadata","type":[{"name":"Metadata","namespace":"Flow\\ETL\\Schema","is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"null"}],"return_type":[{"name":"MapDefinition","namespace":"Flow\\ETL\\Schema\\Definition","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"CORE","type":"SCHEMA"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIEB0ZW1wbGF0ZSBUS2V5IG9mIGFycmF5LWtleQogKiBAdGVtcGxhdGUgVFZhbHVlCiAqCiAqIEBwYXJhbSBNYXBUeXBlPGFycmF5PFRLZXksIFRWYWx1ZT4+fFR5cGU8YXJyYXk8VEtleSwgVFZhbHVlPj4gJHR5cGUKICoKICogQHJldHVybiBNYXBEZWZpbml0aW9uPFRLZXksIFRWYWx1ZT4KICov"},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DSL\/functions.php","start_line_in_file":1526,"slug":"list-schema","name":"list_schema","namespace":"Flow\\ETL\\DSL","parameters":[{"name":"name","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"type","type":[{"name":"ListType","namespace":"Flow\\Types\\Type\\Logical","is_nullable":false,"is_variadic":false},{"name":"Type","namespace":"Flow\\Types","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"nullable","type":[{"name":"bool","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"false"},{"name":"metadata","type":[{"name":"Metadata","namespace":"Flow\\ETL\\Schema","is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"null"}],"return_type":[{"name":"ListDefinition","namespace":"Flow\\ETL\\Schema\\Definition","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"CORE","type":"SCHEMA"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIEB0ZW1wbGF0ZSBUCiAqCiAqIEBwYXJhbSBMaXN0VHlwZTxsaXN0PFQ+PnxUeXBlPGxpc3Q8VD4+ICR0eXBlCiAqCiAqIEByZXR1cm4gTGlzdERlZmluaXRpb248VD4KICov"},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DSL\/functions.php","start_line_in_file":1544,"slug":"enum-schema","name":"enum_schema","namespace":"Flow\\ETL\\DSL","parameters":[{"name":"name","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"type","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"nullable","type":[{"name":"bool","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"false"},{"name":"metadata","type":[{"name":"Metadata","namespace":"Flow\\ETL\\Schema","is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"null"}],"return_type":[{"name":"EnumDefinition","namespace":"Flow\\ETL\\Schema\\Definition","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"CORE","type":"SCHEMA"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIEB0ZW1wbGF0ZSBUIG9mIFxVbml0RW51bQogKgogKiBAcGFyYW0gY2xhc3Mtc3RyaW5nPFQ+ICR0eXBlCiAqCiAqIEByZXR1cm4gRW51bURlZmluaXRpb248VD4KICov"},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DSL\/functions.php","start_line_in_file":1550,"slug":"null-schema","name":"null_schema","namespace":"Flow\\ETL\\DSL","parameters":[{"name":"name","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"metadata","type":[{"name":"Metadata","namespace":"Flow\\ETL\\Schema","is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"null"}],"return_type":[{"name":"NullDefinition","namespace":"Flow\\ETL\\Schema\\Definition","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"CORE","type":"SCHEMA"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DSL\/functions.php","start_line_in_file":1556,"slug":"datetime-schema","name":"datetime_schema","namespace":"Flow\\ETL\\DSL","parameters":[{"name":"name","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"nullable","type":[{"name":"bool","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"false"},{"name":"metadata","type":[{"name":"Metadata","namespace":"Flow\\ETL\\Schema","is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"null"},{"name":"zone","type":[{"name":"DateTimeZone","namespace":"","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"'UTC'"}],"return_type":[{"name":"DateTimeDefinition","namespace":"Flow\\ETL\\Schema\\Definition","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"CORE","type":"SCHEMA"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DSL\/functions.php","start_line_in_file":1566,"slug":"time-schema","name":"time_schema","namespace":"Flow\\ETL\\DSL","parameters":[{"name":"name","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"nullable","type":[{"name":"bool","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"false"},{"name":"metadata","type":[{"name":"Metadata","namespace":"Flow\\ETL\\Schema","is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"null"}],"return_type":[{"name":"TimeDefinition","namespace":"Flow\\ETL\\Schema\\Definition","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"CORE","type":"SCHEMA"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DSL\/functions.php","start_line_in_file":1572,"slug":"date-schema","name":"date_schema","namespace":"Flow\\ETL\\DSL","parameters":[{"name":"name","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"nullable","type":[{"name":"bool","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"false"},{"name":"metadata","type":[{"name":"Metadata","namespace":"Flow\\ETL\\Schema","is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"null"}],"return_type":[{"name":"DateDefinition","namespace":"Flow\\ETL\\Schema\\Definition","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"CORE","type":"SCHEMA"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DSL\/functions.php","start_line_in_file":1578,"slug":"json-schema","name":"json_schema","namespace":"Flow\\ETL\\DSL","parameters":[{"name":"name","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"nullable","type":[{"name":"bool","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"false"},{"name":"metadata","type":[{"name":"Metadata","namespace":"Flow\\ETL\\Schema","is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"null"}],"return_type":[{"name":"JsonDefinition","namespace":"Flow\\ETL\\Schema\\Definition","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"CORE","type":"SCHEMA"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DSL\/functions.php","start_line_in_file":1584,"slug":"html-schema","name":"html_schema","namespace":"Flow\\ETL\\DSL","parameters":[{"name":"name","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"nullable","type":[{"name":"bool","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"false"},{"name":"metadata","type":[{"name":"Metadata","namespace":"Flow\\ETL\\Schema","is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"null"}],"return_type":[{"name":"HTMLDefinition","namespace":"Flow\\ETL\\Schema\\Definition","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"CORE","type":"SCHEMA"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DSL\/functions.php","start_line_in_file":1590,"slug":"html-element-schema","name":"html_element_schema","namespace":"Flow\\ETL\\DSL","parameters":[{"name":"name","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"nullable","type":[{"name":"bool","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"false"},{"name":"metadata","type":[{"name":"Metadata","namespace":"Flow\\ETL\\Schema","is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"null"}],"return_type":[{"name":"HTMLElementDefinition","namespace":"Flow\\ETL\\Schema\\Definition","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"CORE","type":"SCHEMA"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DSL\/functions.php","start_line_in_file":1596,"slug":"xml-schema","name":"xml_schema","namespace":"Flow\\ETL\\DSL","parameters":[{"name":"name","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"nullable","type":[{"name":"bool","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"false"},{"name":"metadata","type":[{"name":"Metadata","namespace":"Flow\\ETL\\Schema","is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"null"}],"return_type":[{"name":"XMLDefinition","namespace":"Flow\\ETL\\Schema\\Definition","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"CORE","type":"SCHEMA"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DSL\/functions.php","start_line_in_file":1602,"slug":"xml-element-schema","name":"xml_element_schema","namespace":"Flow\\ETL\\DSL","parameters":[{"name":"name","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"nullable","type":[{"name":"bool","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"false"},{"name":"metadata","type":[{"name":"Metadata","namespace":"Flow\\ETL\\Schema","is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"null"}],"return_type":[{"name":"XMLElementDefinition","namespace":"Flow\\ETL\\Schema\\Definition","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"CORE","type":"SCHEMA"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DSL\/functions.php","start_line_in_file":1615,"slug":"structure-schema","name":"structure_schema","namespace":"Flow\\ETL\\DSL","parameters":[{"name":"name","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"type","type":[{"name":"StructureType","namespace":"Flow\\Types\\Type\\Logical","is_nullable":false,"is_variadic":false},{"name":"Type","namespace":"Flow\\Types","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"nullable","type":[{"name":"bool","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"false"},{"name":"metadata","type":[{"name":"Metadata","namespace":"Flow\\ETL\\Schema","is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"null"}],"return_type":[{"name":"StructureDefinition","namespace":"Flow\\ETL\\Schema\\Definition","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"CORE","type":"SCHEMA"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIEB0ZW1wbGF0ZSBUCiAqCiAqIEBwYXJhbSBTdHJ1Y3R1cmVUeXBlPGFycmF5PGFycmF5LWtleSwgVD4+fFR5cGU8YXJyYXk8YXJyYXkta2V5LCBUPj4gJHR5cGUKICoKICogQHJldHVybiBTdHJ1Y3R1cmVEZWZpbml0aW9uPFQ+CiAqLw=="},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DSL\/functions.php","start_line_in_file":1626,"slug":"uuid-schema","name":"uuid_schema","namespace":"Flow\\ETL\\DSL","parameters":[{"name":"name","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"nullable","type":[{"name":"bool","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"false"},{"name":"metadata","type":[{"name":"Metadata","namespace":"Flow\\ETL\\Schema","is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"null"}],"return_type":[{"name":"UuidDefinition","namespace":"Flow\\ETL\\Schema\\Definition","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"CORE","type":"SCHEMA"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DSL\/functions.php","start_line_in_file":1632,"slug":"time-zone-schema","name":"time_zone_schema","namespace":"Flow\\ETL\\DSL","parameters":[{"name":"name","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"nullable","type":[{"name":"bool","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"false"},{"name":"metadata","type":[{"name":"Metadata","namespace":"Flow\\ETL\\Schema","is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"null"}],"return_type":[{"name":"TimeZoneDefinition","namespace":"Flow\\ETL\\Schema\\Definition","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"CORE","type":"SCHEMA"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DSL\/functions.php","start_line_in_file":1645,"slug":"definition-from-array","name":"definition_from_array","namespace":"Flow\\ETL\\DSL","parameters":[{"name":"definition","type":[{"name":"array","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"Definition","namespace":"Flow\\ETL\\Schema","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"CORE","type":"SCHEMA"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBhIERlZmluaXRpb24gZnJvbSBhbiBhcnJheSByZXByZXNlbnRhdGlvbi4KICoKICogQHBhcmFtIGFycmF5PGFycmF5LWtleSwgbWl4ZWQ+ICRkZWZpbml0aW9uCiAqCiAqIEByZXR1cm4gRGVmaW5pdGlvbjxtaXhlZD4KICov"},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DSL\/functions.php","start_line_in_file":1678,"slug":"definition-from-type","name":"definition_from_type","namespace":"Flow\\ETL\\DSL","parameters":[{"name":"ref","type":[{"name":"Reference","namespace":"Flow\\ETL\\Row","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"type","type":[{"name":"Type","namespace":"Flow\\Types","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"nullable","type":[{"name":"bool","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"false"},{"name":"metadata","type":[{"name":"Metadata","namespace":"Flow\\ETL\\Schema","is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"null"}],"return_type":[{"name":"Definition","namespace":"Flow\\ETL\\Schema","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"CORE","type":"SCHEMA"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBhIERlZmluaXRpb24gZnJvbSBhIFR5cGUuCiAqCiAqIEBwYXJhbSBUeXBlPG1peGVkPiAkdHlwZQogKgogKiBAcmV0dXJuIERlZmluaXRpb248bWl4ZWQ+CiAqLw=="},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DSL\/functions.php","start_line_in_file":1730,"slug":"infer-schema","name":"infer_schema","namespace":"Flow\\ETL\\DSL","parameters":[],"return_type":[{"name":"SchemaInferenceBuilder","namespace":"Flow\\ETL\\Schema\\Inference","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"CORE","type":"SCHEMA"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DSL\/functions.php","start_line_in_file":1736,"slug":"execution-context","name":"execution_context","namespace":"Flow\\ETL\\DSL","parameters":[{"name":"config","type":[{"name":"Config","namespace":"Flow\\ETL","is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"null"}],"return_type":[{"name":"FlowContext","namespace":"Flow\\ETL","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"CORE","type":"DATA_FRAME"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DSL\/functions.php","start_line_in_file":1742,"slug":"flow-context","name":"flow_context","namespace":"Flow\\ETL\\DSL","parameters":[{"name":"config","type":[{"name":"Config","namespace":"Flow\\ETL","is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"null"}],"return_type":[{"name":"FlowContext","namespace":"Flow\\ETL","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"CORE","type":"DATA_FRAME"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DSL\/functions.php","start_line_in_file":1748,"slug":"config","name":"config","namespace":"Flow\\ETL\\DSL","parameters":[],"return_type":[{"name":"Config","namespace":"Flow\\ETL","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"CORE","type":"DATA_FRAME"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DSL\/functions.php","start_line_in_file":1754,"slug":"config-builder","name":"config_builder","namespace":"Flow\\ETL\\DSL","parameters":[],"return_type":[{"name":"ConfigBuilder","namespace":"Flow\\ETL\\Config","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"CORE","type":"DATA_FRAME"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DSL\/functions.php","start_line_in_file":1760,"slug":"memory-sort","name":"memory_sort","namespace":"Flow\\ETL\\DSL","parameters":[],"return_type":[{"name":"MemorySortBuilder","namespace":"Flow\\ETL\\Config\\Sort","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"CORE","type":"DATA_FRAME"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DSL\/functions.php","start_line_in_file":1766,"slug":"external-sort","name":"external_sort","namespace":"Flow\\ETL\\DSL","parameters":[],"return_type":[{"name":"ExternalSortBuilder","namespace":"Flow\\ETL\\Config\\Sort","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"CORE","type":"DATA_FRAME"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DSL\/functions.php","start_line_in_file":1772,"slug":"hash-join","name":"hash_join","namespace":"Flow\\ETL\\DSL","parameters":[],"return_type":[{"name":"HashJoinBuilder","namespace":"Flow\\ETL\\Config\\Join","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"CORE","type":"DATA_FRAME"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DSL\/functions.php","start_line_in_file":1778,"slug":"hash-group-by","name":"hash_group_by","namespace":"Flow\\ETL\\DSL","parameters":[],"return_type":[{"name":"HashGroupByBuilder","namespace":"Flow\\ETL\\Config\\Grouping","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"CORE","type":"DATA_FRAME"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DSL\/functions.php","start_line_in_file":1784,"slug":"hash-repartition","name":"hash_repartition","namespace":"Flow\\ETL\\DSL","parameters":[],"return_type":[{"name":"HashRepartitionBuilder","namespace":"Flow\\ETL\\Config\\Repartition","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"CORE","type":"DATA_FRAME"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DSL\/functions.php","start_line_in_file":1793,"slug":"pivot-values","name":"pivot_values","namespace":"Flow\\ETL\\DSL","parameters":[{"name":"values","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false},{"name":"int","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":true,"default_value":null}],"return_type":[{"name":"DeclaredPivotValues","namespace":"Flow\\ETL\\GroupBy","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"CORE","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIERlY2xhcmVzIHRoZSBwaXZvdCBjb2x1bW5zIGEgZ3JvdXBCeSgpLT5waXZvdCgpIHByb2R1Y2VzLCBzbyB0aGUgcGxhbiBjYW4gbmFtZSB0aGVtIGJlZm9yZSBhIHJvdyBmbG93cy4KICov"},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DSL\/functions.php","start_line_in_file":1803,"slug":"discover-pivot-values","name":"discover_pivot_values","namespace":"Flow\\ETL\\DSL","parameters":[{"name":"maxValues","type":[{"name":"int","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"10000"}],"return_type":[{"name":"DiscoveredPivotValues","namespace":"Flow\\ETL\\GroupBy","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"CORE","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIFJlYWRzIHRoZSBwaXZvdCBjb2x1bW4gb25jZSBhdCBidWlsZCB0aW1lIGFuZCB0dXJucyB3aGF0IGl0IGZpbmRzIGludG8gZGVjbGFyZWQgdmFsdWVzLiBSZWZ1c2VzIGEKICogc291cmNlIHRoYXQgY2Fubm90IGJlIHJlYWQgdHdpY2UuCiAqLw=="},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DSL\/functions.php","start_line_in_file":1809,"slug":"partition-by","name":"partition_by","namespace":"Flow\\ETL\\DSL","parameters":[{"name":"entry","type":[{"name":"Reference","namespace":"Flow\\ETL\\Row","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"entries","type":[{"name":"Reference","namespace":"Flow\\ETL\\Row","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":true,"default_value":null}],"return_type":[{"name":"Partitioning","namespace":"Flow\\ETL\\Loader","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"CORE","type":"LOADER"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DSL\/functions.php","start_line_in_file":1818,"slug":"partition-types","name":"partition_types","namespace":"Flow\\ETL\\DSL","parameters":[{"name":"types","type":[{"name":"Type","namespace":"Flow\\Types","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":true,"default_value":null}],"return_type":[{"name":"PartitionTypes","namespace":"Flow\\ETL\\Extractor","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"CORE","type":"EXTRACTOR"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIEBwYXJhbSBUeXBlPG1peGVkPiAuLi4kdHlwZXMgcGFydGl0aW9uIGNvbHVtbiBuYW1lID0+IHR5cGUsIHBhc3NlZCBhcyBuYW1lZCBhcmd1bWVudHMKICov"},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DSL\/functions.php","start_line_in_file":1828,"slug":"overwrite","name":"overwrite","namespace":"Flow\\ETL\\DSL","parameters":[],"return_type":[{"name":"SaveMode","namespace":"Flow\\ETL\\Filesystem","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"CORE","type":"DATA_FRAME"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIEFsaWFzIGZvciBzYXZlX21vZGVfb3ZlcndyaXRlKCkuCiAqLw=="},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DSL\/functions.php","start_line_in_file":1834,"slug":"save-mode-overwrite","name":"save_mode_overwrite","namespace":"Flow\\ETL\\DSL","parameters":[],"return_type":[{"name":"SaveMode","namespace":"Flow\\ETL\\Filesystem","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"CORE","type":"DATA_FRAME"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DSL\/functions.php","start_line_in_file":1843,"slug":"ignore","name":"ignore","namespace":"Flow\\ETL\\DSL","parameters":[],"return_type":[{"name":"SaveMode","namespace":"Flow\\ETL\\Filesystem","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"CORE","type":"DATA_FRAME"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIEFsaWFzIGZvciBzYXZlX21vZGVfaWdub3JlKCkuCiAqLw=="},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DSL\/functions.php","start_line_in_file":1849,"slug":"save-mode-ignore","name":"save_mode_ignore","namespace":"Flow\\ETL\\DSL","parameters":[],"return_type":[{"name":"SaveMode","namespace":"Flow\\ETL\\Filesystem","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"CORE","type":"DATA_FRAME"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DSL\/functions.php","start_line_in_file":1858,"slug":"exception-if-exists","name":"exception_if_exists","namespace":"Flow\\ETL\\DSL","parameters":[],"return_type":[{"name":"SaveMode","namespace":"Flow\\ETL\\Filesystem","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"CORE","type":"DATA_FRAME"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIEFsaWFzIGZvciBzYXZlX21vZGVfZXhjZXB0aW9uX2lmX2V4aXN0cygpLgogKi8="},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DSL\/functions.php","start_line_in_file":1864,"slug":"save-mode-exception-if-exists","name":"save_mode_exception_if_exists","namespace":"Flow\\ETL\\DSL","parameters":[],"return_type":[{"name":"SaveMode","namespace":"Flow\\ETL\\Filesystem","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"CORE","type":"DATA_FRAME"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DSL\/functions.php","start_line_in_file":1873,"slug":"append","name":"append","namespace":"Flow\\ETL\\DSL","parameters":[],"return_type":[{"name":"SaveMode","namespace":"Flow\\ETL\\Filesystem","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"CORE","type":"DATA_FRAME"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIEFsaWFzIGZvciBzYXZlX21vZGVfYXBwZW5kKCkuCiAqLw=="},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DSL\/functions.php","start_line_in_file":1879,"slug":"save-mode-append","name":"save_mode_append","namespace":"Flow\\ETL\\DSL","parameters":[],"return_type":[{"name":"SaveMode","namespace":"Flow\\ETL\\Filesystem","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"CORE","type":"DATA_FRAME"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DSL\/functions.php","start_line_in_file":1885,"slug":"print-rows","name":"print_rows","namespace":"Flow\\ETL\\DSL","parameters":[{"name":"rows","type":[{"name":"Rows","namespace":"Flow\\ETL","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"truncate","type":[{"name":"int","namespace":null,"is_nullable":false,"is_variadic":false},{"name":"bool","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"false"},{"name":"formatter","type":[{"name":"Formatter","namespace":"Flow\\ETL","is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"null"}],"return_type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"CORE","type":"DATA_FRAME"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DSL\/functions.php","start_line_in_file":1891,"slug":"identical","name":"identical","namespace":"Flow\\ETL\\DSL","parameters":[{"name":"left","type":[{"name":"Reference","namespace":"Flow\\ETL\\Row","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"right","type":[{"name":"Reference","namespace":"Flow\\ETL\\Row","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"Identical","namespace":"Flow\\ETL\\Join\\Comparison","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"CORE","type":"COMPARISON"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DSL\/functions.php","start_line_in_file":1897,"slug":"equal","name":"equal","namespace":"Flow\\ETL\\DSL","parameters":[{"name":"left","type":[{"name":"Reference","namespace":"Flow\\ETL\\Row","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"right","type":[{"name":"Reference","namespace":"Flow\\ETL\\Row","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"Equal","namespace":"Flow\\ETL\\Join\\Comparison","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"CORE","type":"COMPARISON"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DSL\/functions.php","start_line_in_file":1903,"slug":"compare-all","name":"compare_all","namespace":"Flow\\ETL\\DSL","parameters":[{"name":"comparison","type":[{"name":"Comparison","namespace":"Flow\\ETL\\Join","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"comparisons","type":[{"name":"Comparison","namespace":"Flow\\ETL\\Join","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":true,"default_value":null}],"return_type":[{"name":"All","namespace":"Flow\\ETL\\Join\\Comparison","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"CORE","type":"COMPARISON"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DSL\/functions.php","start_line_in_file":1909,"slug":"compare-any","name":"compare_any","namespace":"Flow\\ETL\\DSL","parameters":[{"name":"comparison","type":[{"name":"Comparison","namespace":"Flow\\ETL\\Join","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"comparisons","type":[{"name":"Comparison","namespace":"Flow\\ETL\\Join","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":true,"default_value":null}],"return_type":[{"name":"Any","namespace":"Flow\\ETL\\Join\\Comparison","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"CORE","type":"COMPARISON"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DSL\/functions.php","start_line_in_file":1920,"slug":"join-on","name":"join_on","namespace":"Flow\\ETL\\DSL","parameters":[{"name":"comparisons","type":[{"name":"Comparison","namespace":"Flow\\ETL\\Join","is_nullable":false,"is_variadic":false},{"name":"array","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"join_prefix","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"''"}],"return_type":[{"name":"Expression","namespace":"Flow\\ETL\\Join","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"CORE","type":"DATA_FRAME"}},{"name":"DocumentationExample","namespace":"Flow\\Documentation\\Attribute","arguments":{"topic":"joins","example":"join"}},{"name":"DocumentationExample","namespace":"Flow\\Documentation\\Attribute","arguments":{"topic":"joins","example":"join_each"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIEBwYXJhbSBhcnJheTxDb21wYXJpc29ufHN0cmluZz58Q29tcGFyaXNvbiAkY29tcGFyaXNvbnMKICov"},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DSL\/functions.php","start_line_in_file":1926,"slug":"schema-sort-by-name","name":"schema_sort_by_name","namespace":"Flow\\ETL\\DSL","parameters":[{"name":"order","type":[{"name":"SortOrder","namespace":"Flow\\ETL\\Row","is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"Flow\\ETL\\Row\\SortOrder::..."}],"return_type":[{"name":"SortingStrategy","namespace":"Flow\\ETL\\Schema","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"CORE","type":"SCHEMA"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DSL\/functions.php","start_line_in_file":1935,"slug":"schema-sort-by-type","name":"schema_sort_by_type","namespace":"Flow\\ETL\\DSL","parameters":[{"name":"priorities","type":[{"name":"array","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"[...]"},{"name":"order","type":[{"name":"SortOrder","namespace":"Flow\\ETL\\Row","is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"Flow\\ETL\\Row\\SortOrder::..."}],"return_type":[{"name":"SortingStrategy","namespace":"Flow\\ETL\\Schema","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"CORE","type":"SCHEMA"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIEBwYXJhbSBhcnJheTxjbGFzcy1zdHJpbmc8VHlwZTxtaXhlZD4+LCBpbnQ+ICRwcmlvcml0aWVzCiAqLw=="},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DSL\/functions.php","start_line_in_file":1946,"slug":"schema-sort-by-type-and-name","name":"schema_sort_by_type_and_name","namespace":"Flow\\ETL\\DSL","parameters":[{"name":"priorities","type":[{"name":"array","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"[...]"},{"name":"order","type":[{"name":"SortOrder","namespace":"Flow\\ETL\\Row","is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"Flow\\ETL\\Row\\SortOrder::..."}],"return_type":[{"name":"SortingStrategy","namespace":"Flow\\ETL\\Schema","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"CORE","type":"SCHEMA"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIEBwYXJhbSBhcnJheTxjbGFzcy1zdHJpbmc8VHlwZTxtaXhlZD4+LCBpbnQ+ICRwcmlvcml0aWVzCiAqLw=="},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DSL\/functions.php","start_line_in_file":1957,"slug":"schema-sort-by-metadata","name":"schema_sort_by_metadata","namespace":"Flow\\ETL\\DSL","parameters":[{"name":"key","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"order","type":[{"name":"SortOrder","namespace":"Flow\\ETL\\Row","is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"Flow\\ETL\\Row\\SortOrder::..."}],"return_type":[{"name":"SortingStrategy","namespace":"Flow\\ETL\\Schema","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"CORE","type":"SCHEMA"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DSL\/functions.php","start_line_in_file":1967,"slug":"is-type","name":"is_type","namespace":"Flow\\ETL\\DSL","parameters":[{"name":"type","type":[{"name":"Type","namespace":"Flow\\Types","is_nullable":false,"is_variadic":false},{"name":"array","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"value","type":[{"name":"mixed","namespace":null,"is_nullable":true,"is_variadic":false}],"has_default_value":false,"is_nullable":true,"is_variadic":false,"default_value":null}],"return_type":[{"name":"bool","namespace":null,"is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"CORE","type":"DATA_FRAME"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIEBwYXJhbSBhcnJheTxzdHJpbmd8VHlwZTxtaXhlZD4+fFR5cGU8bWl4ZWQ+ICR0eXBlCiAqIEBwYXJhbSBtaXhlZCAkdmFsdWUKICov"},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DSL\/functions.php","start_line_in_file":2001,"slug":"generate-random-string","name":"generate_random_string","namespace":"Flow\\ETL\\DSL","parameters":[{"name":"length","type":[{"name":"int","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"32"},{"name":"generator","type":[{"name":"RandomValueGenerator","namespace":"Flow\\ETL","is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"Flow\\ETL\\NativePHPRandomValueGenerator::..."}],"return_type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"CORE","type":"DATA_FRAME"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DSL\/functions.php","start_line_in_file":2009,"slug":"generate-random-int","name":"generate_random_int","namespace":"Flow\\ETL\\DSL","parameters":[{"name":"start","type":[{"name":"int","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"-9223372036854775808"},{"name":"end","type":[{"name":"int","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"9223372036854775807"},{"name":"generator","type":[{"name":"RandomValueGenerator","namespace":"Flow\\ETL","is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"Flow\\ETL\\NativePHPRandomValueGenerator::..."}],"return_type":[{"name":"int","namespace":null,"is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"CORE","type":"DATA_FRAME"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DSL\/functions.php","start_line_in_file":2018,"slug":"random-string","name":"random_string","namespace":"Flow\\ETL\\DSL","parameters":[{"name":"length","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"int","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"generator","type":[{"name":"RandomValueGenerator","namespace":"Flow\\ETL","is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"Flow\\ETL\\NativePHPRandomValueGenerator::..."}],"return_type":[{"name":"RandomString","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"CORE","type":"DATA_FRAME"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DSL\/functions.php","start_line_in_file":2026,"slug":"date-interval-to-milliseconds","name":"date_interval_to_milliseconds","namespace":"Flow\\ETL\\DSL","parameters":[{"name":"interval","type":[{"name":"DateInterval","namespace":"","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"int","namespace":null,"is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"CORE","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DSL\/functions.php","start_line_in_file":2043,"slug":"date-interval-to-seconds","name":"date_interval_to_seconds","namespace":"Flow\\ETL\\DSL","parameters":[{"name":"interval","type":[{"name":"DateInterval","namespace":"","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"int","namespace":null,"is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"CORE","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DSL\/functions.php","start_line_in_file":2060,"slug":"date-interval-to-microseconds","name":"date_interval_to_microseconds","namespace":"Flow\\ETL\\DSL","parameters":[{"name":"interval","type":[{"name":"DateInterval","namespace":"","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"int","namespace":null,"is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"CORE","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DSL\/functions.php","start_line_in_file":2077,"slug":"with-entry","name":"with_entry","namespace":"Flow\\ETL\\DSL","parameters":[{"name":"name","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"function","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"WithEntry","namespace":"Flow\\ETL","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"CORE","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DSL\/functions.php","start_line_in_file":2083,"slug":"constraint-unique","name":"constraint_unique","namespace":"Flow\\ETL\\DSL","parameters":[{"name":"reference","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"references","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":true,"default_value":null}],"return_type":[{"name":"UniqueConstraint","namespace":"Flow\\ETL\\Constraint","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"CORE","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DSL\/functions.php","start_line_in_file":2089,"slug":"constraint-sorted-by","name":"constraint_sorted_by","namespace":"Flow\\ETL\\DSL","parameters":[{"name":"column","type":[{"name":"Reference","namespace":"Flow\\ETL\\Row","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"columns","type":[{"name":"Reference","namespace":"Flow\\ETL\\Row","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":true,"default_value":null}],"return_type":[{"name":"SortedByConstraint","namespace":"Flow\\ETL\\Constraint","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"CORE","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DSL\/functions.php","start_line_in_file":2099,"slug":"analyze","name":"analyze","namespace":"Flow\\ETL\\DSL","parameters":[],"return_type":[{"name":"Analyze","namespace":"Flow\\ETL","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"CORE","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DSL\/functions.php","start_line_in_file":2108,"slug":"match-cases","name":"match_cases","namespace":"Flow\\ETL\\DSL","parameters":[{"name":"cases","type":[{"name":"array","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"default","type":[{"name":"mixed","namespace":null,"is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"null"}],"return_type":[{"name":"MatchCases","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"CORE","type":"SCALAR_FUNCTION"}}],"scalar_function_chain":true,"doc_comment":"LyoqCiAqIEBwYXJhbSBhcnJheTxNYXRjaENvbmRpdGlvbj4gJGNhc2VzCiAqLw=="},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DSL\/functions.php","start_line_in_file":2114,"slug":"match-condition","name":"match_condition","namespace":"Flow\\ETL\\DSL","parameters":[{"name":"condition","type":[{"name":"mixed","namespace":null,"is_nullable":true,"is_variadic":false}],"has_default_value":false,"is_nullable":true,"is_variadic":false,"default_value":null},{"name":"then","type":[{"name":"mixed","namespace":null,"is_nullable":true,"is_variadic":false}],"has_default_value":false,"is_nullable":true,"is_variadic":false,"default_value":null}],"return_type":[{"name":"MatchCondition","namespace":"Flow\\ETL\\Function\\MatchCases","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"CORE","type":"SCALAR_FUNCTION"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DSL\/functions.php","start_line_in_file":2120,"slug":"clock","name":"clock","namespace":"Flow\\ETL\\DSL","parameters":[{"name":"time_zone","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"'UTC'"}],"return_type":[{"name":"ClockInterface","namespace":"Psr\\Clock","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"CORE","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/Floe\/DSL\/functions.php","start_line_in_file":31,"slug":"from-floe","name":"from_floe","namespace":"Flow\\Floe\\DSL","parameters":[{"name":"path","type":[{"name":"Path","namespace":"Flow\\Filesystem","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"codec","type":[{"name":"Codec","namespace":"Flow\\Floe","is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"Flow\\Floe\\Codec\\NoopCodec::..."},{"name":"chunk_size","type":[{"name":"int","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"65536"},{"name":"engine","type":[{"name":"FloeEngine","namespace":"Flow\\Floe","is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"Flow\\Floe\\FloeEngine::..."},{"name":"filesystem","type":[{"name":"Filesystem","namespace":"Flow\\Filesystem","is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"Flow\\Filesystem\\Local\\NativeLocalFilesystem::..."}],"return_type":[{"name":"FloeExtractor","namespace":"Flow\\Floe","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"FLOE","type":"EXTRACTOR"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIEBwYXJhbSBQYXRofHN0cmluZyAkcGF0aAogKi8="},{"repository_path":"src\/core\/etl\/src\/Flow\/Floe\/DSL\/functions.php","start_line_in_file":45,"slug":"to-floe","name":"to_floe","namespace":"Flow\\Floe\\DSL","parameters":[{"name":"path","type":[{"name":"Path","namespace":"Flow\\Filesystem","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"metadata","type":[{"name":"Metadata","namespace":"Flow\\ETL\\Schema","is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"null"},{"name":"options","type":[{"name":"Options","namespace":"Flow\\Floe","is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"Flow\\Floe\\Options::..."},{"name":"engine","type":[{"name":"FloeEngine","namespace":"Flow\\Floe","is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"Flow\\Floe\\FloeEngine::..."},{"name":"filesystem","type":[{"name":"Filesystem","namespace":"Flow\\Filesystem","is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"Flow\\Filesystem\\Local\\NativeLocalFilesystem::..."}],"return_type":[{"name":"FloeLoader","namespace":"Flow\\Floe","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"FLOE","type":"LOADER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIEBwYXJhbSBQYXRofHN0cmluZyAkcGF0aAogKi8="},{"repository_path":"src\/core\/etl\/src\/Flow\/Floe\/DSL\/functions.php","start_line_in_file":56,"slug":"floe-options","name":"floe_options","namespace":"Flow\\Floe\\DSL","parameters":[{"name":"buffer_size","type":[{"name":"int","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"65536"},{"name":"codec","type":[{"name":"Codec","namespace":"Flow\\Floe","is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"Flow\\Floe\\Codec\\NoopCodec::..."}],"return_type":[{"name":"Options","namespace":"Flow\\Floe","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"FLOE","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/Floe\/DSL\/functions.php","start_line_in_file":68,"slug":"merge-floe","name":"merge_floe","namespace":"Flow\\Floe\\DSL","parameters":[{"name":"sources","type":[{"name":"array","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"dest","type":[{"name":"Path","namespace":"Flow\\Filesystem","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"compact","type":[{"name":"bool","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"false"},{"name":"metadata","type":[{"name":"Metadata","namespace":"Flow\\ETL\\Schema","is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"null"},{"name":"filesystem","type":[{"name":"Filesystem","namespace":"Flow\\Filesystem","is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"Flow\\Filesystem\\Local\\NativeLocalFilesystem::..."}],"return_type":[{"name":"void","namespace":null,"is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"FLOE","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIE1lcmdlcyBzZXZlcmFsIEZsb2UgZmlsZXMgKHNhbWUgb3IgYXBwZW5kLWNvbXBhdGlibGUgZXZvbHZpbmcgc2NoZW1hKSBpbnRvIG9uZS4gQnl0ZS1zcGxpY2VzIGZyYW1lCiAqIHJlZ2lvbnMgYnkgZGVmYXVsdCAoTyhieXRlcyksIG5vIHJlLWVuY29kZSk7IGNvbXBhY3QgcmUtZW5jb2RlcyBhbGwgcm93cyBpbnRvIGZld2VyIHNlY3Rpb25zLgogKgogKiBAcGFyYW0gYXJyYXk8aW50LCBQYXRofHN0cmluZz4gJHNvdXJjZXMKICov"},{"repository_path":"src\/core\/etl\/src\/Flow\/Serializer\/DSL\/functions.php","start_line_in_file":18,"slug":"serialize-to-string","name":"serialize_to_string","namespace":"Flow\\Serializer\\DSL","parameters":[{"name":"serializer","type":[{"name":"Serializer","namespace":"Flow\\Serializer","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"rows","type":[{"name":"Rows","namespace":"Flow\\ETL","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"CORE","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/Serializer\/DSL\/functions.php","start_line_in_file":27,"slug":"unserialize-from-string","name":"unserialize_from_string","namespace":"Flow\\Serializer\\DSL","parameters":[{"name":"serializer","type":[{"name":"Serializer","namespace":"Flow\\Serializer","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"payload","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"Rows","namespace":"Flow\\ETL","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"CORE","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/adapter\/etl-adapter-avro\/src\/Flow\/ETL\/Adapter\/Avro\/functions.php","start_line_in_file":21,"slug":"from-avro","name":"from_avro","namespace":"Flow\\ETL\\DSL\\Adapter\\Avro","parameters":[{"name":"path","type":[{"name":"Path","namespace":"Flow\\Filesystem","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"filesystem","type":[{"name":"Filesystem","namespace":"Flow\\Filesystem","is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"Flow\\Filesystem\\Local\\NativeLocalFilesystem::..."}],"return_type":[{"name":"AvroExtractor","namespace":"Flow\\ETL\\Adapter\\Avro\\FlixTech","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"AVRO","type":"EXTRACTOR"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/adapter\/etl-adapter-avro\/src\/Flow\/ETL\/Adapter\/Avro\/functions.php","start_line_in_file":27,"slug":"to-avro","name":"to_avro","namespace":"Flow\\ETL\\DSL\\Adapter\\Avro","parameters":[{"name":"path","type":[{"name":"Path","namespace":"Flow\\Filesystem","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"schema","type":[{"name":"Schema","namespace":"Flow\\ETL","is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"null"},{"name":"filesystem","type":[{"name":"Filesystem","namespace":"Flow\\Filesystem","is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"Flow\\Filesystem\\Local\\NativeLocalFilesystem::..."}],"return_type":[{"name":"AvroLoader","namespace":"Flow\\ETL\\Adapter\\Avro\\FlixTech","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"AVRO","type":"LOADER"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/adapter\/etl-adapter-chartjs\/src\/Flow\/ETL\/Adapter\/ChartJS\/functions.php","start_line_in_file":23,"slug":"bar-chart","name":"bar_chart","namespace":"Flow\\ETL\\Adapter\\ChartJS","parameters":[{"name":"label","type":[{"name":"Reference","namespace":"Flow\\ETL\\Row","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"datasets","type":[{"name":"References","namespace":"Flow\\ETL\\Row","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"BarChart","namespace":"Flow\\ETL\\Adapter\\ChartJS\\Chart","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"CHART_JS","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/adapter\/etl-adapter-chartjs\/src\/Flow\/ETL\/Adapter\/ChartJS\/functions.php","start_line_in_file":29,"slug":"line-chart","name":"line_chart","namespace":"Flow\\ETL\\Adapter\\ChartJS","parameters":[{"name":"label","type":[{"name":"Reference","namespace":"Flow\\ETL\\Row","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"datasets","type":[{"name":"References","namespace":"Flow\\ETL\\Row","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"LineChart","namespace":"Flow\\ETL\\Adapter\\ChartJS\\Chart","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"CHART_JS","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/adapter\/etl-adapter-chartjs\/src\/Flow\/ETL\/Adapter\/ChartJS\/functions.php","start_line_in_file":35,"slug":"pie-chart","name":"pie_chart","namespace":"Flow\\ETL\\Adapter\\ChartJS","parameters":[{"name":"label","type":[{"name":"Reference","namespace":"Flow\\ETL\\Row","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"datasets","type":[{"name":"References","namespace":"Flow\\ETL\\Row","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"PieChart","namespace":"Flow\\ETL\\Adapter\\ChartJS\\Chart","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"CHART_JS","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/adapter\/etl-adapter-chartjs\/src\/Flow\/ETL\/Adapter\/ChartJS\/functions.php","start_line_in_file":41,"slug":"to-chartjs","name":"to_chartjs","namespace":"Flow\\ETL\\Adapter\\ChartJS","parameters":[{"name":"type","type":[{"name":"Chart","namespace":"Flow\\ETL\\Adapter\\ChartJS","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"ChartJSLoader","namespace":"Flow\\ETL\\Adapter\\ChartJS","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"CHART_JS","type":"LOADER"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/adapter\/etl-adapter-chartjs\/src\/Flow\/ETL\/Adapter\/ChartJS\/functions.php","start_line_in_file":52,"slug":"to-chartjs-file","name":"to_chartjs_file","namespace":"Flow\\ETL\\Adapter\\ChartJS","parameters":[{"name":"type","type":[{"name":"Chart","namespace":"Flow\\ETL\\Adapter\\ChartJS","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"output","type":[{"name":"Path","namespace":"Flow\\Filesystem","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false},{"name":"null","namespace":null,"is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"null"},{"name":"template","type":[{"name":"Path","namespace":"Flow\\Filesystem","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false},{"name":"null","namespace":null,"is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"null"},{"name":"filesystem","type":[{"name":"Filesystem","namespace":"Flow\\Filesystem","is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"Flow\\Filesystem\\Local\\NativeLocalFilesystem::..."}],"return_type":[{"name":"ChartJSLoader","namespace":"Flow\\ETL\\Adapter\\ChartJS","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"CHART_JS","type":"LOADER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIEBwYXJhbSBDaGFydCAkdHlwZQogKiBAcGFyYW0gbnVsbHxQYXRofHN0cmluZyAkb3V0cHV0IC0gQGRlcHJlY2F0ZWQgdXNlICRsb2FkZXItPndpdGhPdXRwdXRQYXRoKCkgaW5zdGVhZAogKiBAcGFyYW0gbnVsbHxQYXRofHN0cmluZyAkdGVtcGxhdGUgLSBAZGVwcmVjYXRlZCB1c2UgJGxvYWRlci0+d2l0aFRlbXBsYXRlKCkgaW5zdGVhZAogKi8="},{"repository_path":"src\/adapter\/etl-adapter-chartjs\/src\/Flow\/ETL\/Adapter\/ChartJS\/functions.php","start_line_in_file":84,"slug":"to-chartjs-var","name":"to_chartjs_var","namespace":"Flow\\ETL\\Adapter\\ChartJS","parameters":[{"name":"type","type":[{"name":"Chart","namespace":"Flow\\ETL\\Adapter\\ChartJS","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"output","type":[{"name":"array","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"ChartJSLoader","namespace":"Flow\\ETL\\Adapter\\ChartJS","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"CHART_JS","type":"LOADER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIEBwYXJhbSBDaGFydCAkdHlwZQogKiBAcGFyYW0gYXJyYXk8YXJyYXkta2V5LCBtaXhlZD4gJG91dHB1dCAtIEBkZXByZWNhdGVkIHVzZSAkbG9hZGVyLT53aXRoT3V0cHV0VmFyKCkgaW5zdGVhZAogKi8="},{"repository_path":"src\/adapter\/etl-adapter-csv\/src\/Flow\/ETL\/Adapter\/CSV\/functions.php","start_line_in_file":35,"slug":"from-csv","name":"from_csv","namespace":"Flow\\ETL\\Adapter\\CSV","parameters":[{"name":"path","type":[{"name":"Path","namespace":"Flow\\Filesystem","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"with_header","type":[{"name":"bool","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"true"},{"name":"empty_to_null","type":[{"name":"bool","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"true"},{"name":"separator","type":[{"name":"string","namespace":null,"is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"null"},{"name":"enclosure","type":[{"name":"string","namespace":null,"is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"null"},{"name":"escape","type":[{"name":"string","namespace":null,"is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"null"},{"name":"characters_read_in_line","type":[{"name":"int","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"10485760"},{"name":"schema","type":[{"name":"Schema","namespace":"Flow\\ETL","is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"null"},{"name":"filesystem","type":[{"name":"Filesystem","namespace":"Flow\\Filesystem","is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"Flow\\Filesystem\\Local\\NativeLocalFilesystem::..."}],"return_type":[{"name":"CSVExtractor","namespace":"Flow\\ETL\\Adapter\\CSV","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"CSV","type":"EXTRACTOR"}},{"name":"DocumentationExample","namespace":"Flow\\Documentation\\Attribute","arguments":{"topic":"reading","example":"csv"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIEBwYXJhbSBQYXRofHN0cmluZyAkcGF0aAogKiBAcGFyYW0gYm9vbCAkZW1wdHlfdG9fbnVsbCAtIEBkZXByZWNhdGVkIHVzZSAkbG9hZGVyLT53aXRoRW1wdHlUb051bGwoKSBpbnN0ZWFkCiAqIEBwYXJhbSBib29sICR3aXRoX2hlYWRlciAtIEBkZXByZWNhdGVkIHVzZSAkbG9hZGVyLT53aXRoSGVhZGVyKCkgaW5zdGVhZAogKiBAcGFyYW0gbnVsbHxzdHJpbmcgJHNlcGFyYXRvciAtIEBkZXByZWNhdGVkIHVzZSAkbG9hZGVyLT53aXRoU2VwYXJhdG9yKCkgaW5zdGVhZAogKiBAcGFyYW0gbnVsbHxzdHJpbmcgJGVuY2xvc3VyZSAtIEBkZXByZWNhdGVkIHVzZSAkbG9hZGVyLT53aXRoRW5jbG9zdXJlKCkgaW5zdGVhZAogKiBAcGFyYW0gbnVsbHxzdHJpbmcgJGVzY2FwZSAtIEBkZXByZWNhdGVkIHVzZSAkbG9hZGVyLT53aXRoRXNjYXBlKCkgaW5zdGVhZAogKiBAcGFyYW0gaW50PDEsIG1heD4gJGNoYXJhY3RlcnNfcmVhZF9pbl9saW5lIC0gQGRlcHJlY2F0ZWQgdXNlICRsb2FkZXItPndpdGhDaGFyYWN0ZXJzUmVhZEluTGluZSgpIGluc3RlYWQKICogQHBhcmFtIG51bGx8U2NoZW1hICRzY2hlbWEgLSBAZGVwcmVjYXRlZCB1c2UgJGxvYWRlci0+d2l0aFNjaGVtYSgpIGluc3RlYWQKICov"},{"repository_path":"src\/adapter\/etl-adapter-csv\/src\/Flow\/ETL\/Adapter\/CSV\/functions.php","start_line_in_file":80,"slug":"to-csv","name":"to_csv","namespace":"Flow\\ETL\\Adapter\\CSV","parameters":[{"name":"uri","type":[{"name":"Path","namespace":"Flow\\Filesystem","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"with_header","type":[{"name":"bool","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"true"},{"name":"separator","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"','"},{"name":"enclosure","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"'\\\"'"},{"name":"escape","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"'\\\\'"},{"name":"new_line_separator","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"'\\n'"},{"name":"datetime_format","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"'Y-m-d\\\\TH:i:sP'"},{"name":"filesystem","type":[{"name":"Filesystem","namespace":"Flow\\Filesystem","is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"Flow\\Filesystem\\Local\\NativeLocalFilesystem::..."}],"return_type":[{"name":"CSVLoader","namespace":"Flow\\ETL\\Adapter\\CSV","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"CSV","type":"LOADER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIEBwYXJhbSBQYXRofHN0cmluZyAkdXJpCiAqIEBwYXJhbSBib29sICR3aXRoX2hlYWRlciAtIEBkZXByZWNhdGVkIHVzZSAkbG9hZGVyLT53aXRoSGVhZGVyKCkgaW5zdGVhZAogKiBAcGFyYW0gc3RyaW5nICRzZXBhcmF0b3IgLSBAZGVwcmVjYXRlZCB1c2UgJGxvYWRlci0+d2l0aFNlcGFyYXRvcigpIGluc3RlYWQKICogQHBhcmFtIHN0cmluZyAkZW5jbG9zdXJlIC0gQGRlcHJlY2F0ZWQgdXNlICRsb2FkZXItPndpdGhFbmNsb3N1cmUoKSBpbnN0ZWFkCiAqIEBwYXJhbSBzdHJpbmcgJGVzY2FwZSAtIEBkZXByZWNhdGVkIHVzZSAkbG9hZGVyLT53aXRoRXNjYXBlKCkgaW5zdGVhZAogKiBAcGFyYW0gc3RyaW5nICRuZXdfbGluZV9zZXBhcmF0b3IgLSBAZGVwcmVjYXRlZCB1c2UgJGxvYWRlci0+d2l0aE5ld0xpbmVTZXBhcmF0b3IoKSBpbnN0ZWFkCiAqIEBwYXJhbSBzdHJpbmcgJGRhdGV0aW1lX2Zvcm1hdCAtIEBkZXByZWNhdGVkIHVzZSAkbG9hZGVyLT53aXRoRGF0ZVRpbWVGb3JtYXQoKSBpbnN0ZWFkCiAqLw=="},{"repository_path":"src\/adapter\/etl-adapter-csv\/src\/Flow\/ETL\/Adapter\/CSV\/functions.php","start_line_in_file":106,"slug":"csv-detect-separator","name":"csv_detect_separator","namespace":"Flow\\ETL\\Adapter\\CSV","parameters":[{"name":"stream","type":[{"name":"SourceStream","namespace":"Flow\\Filesystem","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"lines","type":[{"name":"int","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"5"},{"name":"fallback","type":[{"name":"Option","namespace":"Flow\\ETL\\Adapter\\CSV\\Detector","is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"Flow\\ETL\\Adapter\\CSV\\Detector\\Option::..."},{"name":"options","type":[{"name":"Options","namespace":"Flow\\ETL\\Adapter\\CSV\\Detector","is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"null"}],"return_type":[{"name":"Option","namespace":"Flow\\ETL\\Adapter\\CSV\\Detector","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"CSV","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIEBwYXJhbSBTb3VyY2VTdHJlYW0gJHN0cmVhbSAtIHZhbGlkIHJlc291cmNlIHRvIENTViBmaWxlCiAqIEBwYXJhbSBpbnQ8MSwgbWF4PiAkbGluZXMgLSBudW1iZXIgb2YgbGluZXMgdG8gcmVhZCBmcm9tIENTViBmaWxlLCBkZWZhdWx0IDUsIG1vcmUgbGluZXMgbWVhbnMgbW9yZSBhY2N1cmF0ZSBkZXRlY3Rpb24gYnV0IHNsb3dlciBkZXRlY3Rpb24KICogQHBhcmFtIG51bGx8T3B0aW9uICRmYWxsYmFjayAtIGZhbGxiYWNrIG9wdGlvbiB0byB1c2Ugd2hlbiBubyBiZXN0IG9wdGlvbiBjYW4gYmUgZGV0ZWN0ZWQsIGRlZmF1bHQgaXMgT3B0aW9uKCcsJywgJyInLCAnXFwnKQogKiBAcGFyYW0gbnVsbHxPcHRpb25zICRvcHRpb25zIC0gb3B0aW9ucyB0byB1c2UgZm9yIGRldGVjdGlvbiwgZGVmYXVsdCBpcyBPcHRpb25zOjphbGwoKQogKi8="},{"repository_path":"src\/adapter\/etl-adapter-doctrine\/src\/Flow\/ETL\/Adapter\/Doctrine\/functions.php","start_line_in_file":41,"slug":"dbal-dataframe-factory","name":"dbal_dataframe_factory","namespace":"Flow\\ETL\\Adapter\\Doctrine","parameters":[{"name":"connection","type":[{"name":"Connection","namespace":"Doctrine\\DBAL","is_nullable":false,"is_variadic":false},{"name":"array","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"query","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"parameters","type":[{"name":"QueryParameter","namespace":"Flow\\ETL\\Adapter\\Doctrine","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":true,"default_value":null}],"return_type":[{"name":"DbalDataFrameFactory","namespace":"Flow\\ETL\\Adapter\\Doctrine","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"DOCTRINE","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIEBwYXJhbSBhcnJheTxzdHJpbmcsIG1peGVkPnxDb25uZWN0aW9uICRjb25uZWN0aW9uCiAqIEBwYXJhbSBzdHJpbmcgJHF1ZXJ5CiAqIEBwYXJhbSBRdWVyeVBhcmFtZXRlciAuLi4kcGFyYW1ldGVycwogKi8="},{"repository_path":"src\/adapter\/etl-adapter-doctrine\/src\/Flow\/ETL\/Adapter\/Doctrine\/functions.php","start_line_in_file":61,"slug":"from-dbal-limit-offset","name":"from_dbal_limit_offset","namespace":"Flow\\ETL\\Adapter\\Doctrine","parameters":[{"name":"connection","type":[{"name":"Connection","namespace":"Doctrine\\DBAL","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"table","type":[{"name":"Table","namespace":"Flow\\ETL\\Adapter\\Doctrine","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"order_by","type":[{"name":"OrderBy","namespace":"Flow\\ETL\\Adapter\\Doctrine","is_nullable":false,"is_variadic":false},{"name":"array","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"page_size","type":[{"name":"int","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"1000"},{"name":"maximum","type":[{"name":"int","namespace":null,"is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"null"}],"return_type":[{"name":"DbalLimitOffsetExtractor","namespace":"Flow\\ETL\\Adapter\\Doctrine","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"DOCTRINE","type":"EXTRACTOR"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIEBwYXJhbSBDb25uZWN0aW9uICRjb25uZWN0aW9uCiAqIEBwYXJhbSBzdHJpbmd8VGFibGUgJHRhYmxlCiAqIEBwYXJhbSBhcnJheTxPcmRlckJ5PnxPcmRlckJ5ICRvcmRlcl9ieQogKiBAcGFyYW0gaW50ICRwYWdlX3NpemUgLSBiZWNvbWVzIHRoZSBleHRyYWN0b3IncyBiYXRjaCBzaXplOiByb3dzIHBlciBwYWdlCiAqIEBwYXJhbSBudWxsfGludCAkbWF4aW11bQogKgogKiBAdGhyb3dzIEludmFsaWRBcmd1bWVudEV4Y2VwdGlvbgogKi8="},{"repository_path":"src\/adapter\/etl-adapter-doctrine\/src\/Flow\/ETL\/Adapter\/Doctrine\/functions.php","start_line_in_file":88,"slug":"from-dbal-limit-offset-qb","name":"from_dbal_limit_offset_qb","namespace":"Flow\\ETL\\Adapter\\Doctrine","parameters":[{"name":"connection","type":[{"name":"Connection","namespace":"Doctrine\\DBAL","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"queryBuilder","type":[{"name":"QueryBuilder","namespace":"Doctrine\\DBAL\\Query","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"page_size","type":[{"name":"int","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"1000"},{"name":"maximum","type":[{"name":"int","namespace":null,"is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"null"},{"name":"offset","type":[{"name":"int","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"0"}],"return_type":[{"name":"DbalLimitOffsetExtractor","namespace":"Flow\\ETL\\Adapter\\Doctrine","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"DOCTRINE","type":"EXTRACTOR"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIEBwYXJhbSBDb25uZWN0aW9uICRjb25uZWN0aW9uCiAqIEBwYXJhbSBpbnQgJHBhZ2Vfc2l6ZSAtIGJlY29tZXMgdGhlIGV4dHJhY3RvcidzIGJhdGNoIHNpemU6IHJvd3MgcGVyIHBhZ2UKICogQHBhcmFtIG51bGx8aW50ICRtYXhpbXVtIC0gbWF4aW11bSBjYW4gYWxzbyBiZSB0YWtlbiBmcm9tIGEgcXVlcnkgYnVpbGRlciwgJG1heGltdW0gaG93ZXZlciBpcyB1c2VkIHJlZ2FyZGxlc3Mgb2YgdGhlIHF1ZXJ5IGJ1aWxkZXIgaWYgaXQncyBzZXQKICogQHBhcmFtIGludCAkb2Zmc2V0IC0gb2Zmc2V0IGNhbiBhbHNvIGJlIHRha2VuIGZyb20gYSBxdWVyeSBidWlsZGVyLCAkb2Zmc2V0IGhvd2V2ZXIgaXMgdXNlZCByZWdhcmRsZXNzIG9mIHRoZSBxdWVyeSBidWlsZGVyIGlmIGl0J3Mgc2V0IHRvIG5vbiAwIHZhbHVlCiAqLw=="},{"repository_path":"src\/adapter\/etl-adapter-doctrine\/src\/Flow\/ETL\/Adapter\/Doctrine\/functions.php","start_line_in_file":107,"slug":"from-dbal-key-set-qb","name":"from_dbal_key_set_qb","namespace":"Flow\\ETL\\Adapter\\Doctrine","parameters":[{"name":"connection","type":[{"name":"Connection","namespace":"Doctrine\\DBAL","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"queryBuilder","type":[{"name":"QueryBuilder","namespace":"Doctrine\\DBAL\\Query","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"key_set","type":[{"name":"KeySet","namespace":"Flow\\ETL\\Adapter\\Doctrine\\Pagination","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"DbalKeySetExtractor","namespace":"Flow\\ETL\\Adapter\\Doctrine","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"DOCTRINE","type":"EXTRACTOR"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/adapter\/etl-adapter-doctrine\/src\/Flow\/ETL\/Adapter\/Doctrine\/functions.php","start_line_in_file":117,"slug":"from-dbal-queries","name":"from_dbal_queries","namespace":"Flow\\ETL\\Adapter\\Doctrine","parameters":[{"name":"connection","type":[{"name":"Connection","namespace":"Doctrine\\DBAL","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"query","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"parameters_set","type":[{"name":"ParametersSet","namespace":"Flow\\ETL\\Adapter\\Doctrine","is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"null"},{"name":"types","type":[{"name":"array","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"[]"}],"return_type":[{"name":"DbalQueryExtractor","namespace":"Flow\\ETL\\Adapter\\Doctrine","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"DOCTRINE","type":"EXTRACTOR"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIEBwYXJhbSBudWxsfFBhcmFtZXRlcnNTZXQgJHBhcmFtZXRlcnNfc2V0IC0gZWFjaCBvbmUgcGFyYW1ldGVycyBhcnJheSB3aWxsIGJlIGV2YWx1YXRlZCBhcyBuZXcgcXVlcnkKICogQHBhcmFtIGFycmF5PGludDwwLCBtYXg+fHN0cmluZywgRGJhbEFycmF5VHlwZXxEYmFsUGFyYW1ldGVyVHlwZXxEYmFsVHlwZXxzdHJpbmc+ICR0eXBlcwogKi8="},{"repository_path":"src\/adapter\/etl-adapter-doctrine\/src\/Flow\/ETL\/Adapter\/Doctrine\/functions.php","start_line_in_file":143,"slug":"dbal-from-queries","name":"dbal_from_queries","namespace":"Flow\\ETL\\Adapter\\Doctrine","parameters":[{"name":"connection","type":[{"name":"Connection","namespace":"Doctrine\\DBAL","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"query","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"parameters_set","type":[{"name":"ParametersSet","namespace":"Flow\\ETL\\Adapter\\Doctrine","is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"null"},{"name":"types","type":[{"name":"array","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"[]"}],"return_type":[{"name":"DbalQueryExtractor","namespace":"Flow\\ETL\\Adapter\\Doctrine","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"DOCTRINE","type":"EXTRACTOR"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIEBkZXByZWNhdGVkIHVzZSBmcm9tX2RiYWxfcXVlcmllcygpIGluc3RlYWQKICoKICogQHBhcmFtIG51bGx8UGFyYW1ldGVyc1NldCAkcGFyYW1ldGVyc19zZXQgLSBlYWNoIG9uZSBwYXJhbWV0ZXJzIGFycmF5IHdpbGwgYmUgZXZhbHVhdGVkIGFzIG5ldyBxdWVyeQogKiBAcGFyYW0gYXJyYXk8aW50PDAsIG1heD58c3RyaW5nLCBEYmFsQXJyYXlUeXBlfERiYWxQYXJhbWV0ZXJUeXBlfERiYWxUeXBlfHN0cmluZz4gJHR5cGVzCiAqLw=="},{"repository_path":"src\/adapter\/etl-adapter-doctrine\/src\/Flow\/ETL\/Adapter\/Doctrine\/functions.php","start_line_in_file":157,"slug":"from-dbal-query","name":"from_dbal_query","namespace":"Flow\\ETL\\Adapter\\Doctrine","parameters":[{"name":"connection","type":[{"name":"Connection","namespace":"Doctrine\\DBAL","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"query","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"parameters","type":[{"name":"array","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"[]"},{"name":"types","type":[{"name":"array","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"[]"}],"return_type":[{"name":"DbalQueryExtractor","namespace":"Flow\\ETL\\Adapter\\Doctrine","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"DOCTRINE","type":"EXTRACTOR"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIEBwYXJhbSBhcnJheTxzdHJpbmcsIG1peGVkPnxsaXN0PG1peGVkPiAkcGFyYW1ldGVycyAtIEBkZXByZWNhdGVkIHVzZSBEYmFsUXVlcnlFeHRyYWN0b3I6OndpdGhQYXJhbWV0ZXJzKCkgaW5zdGVhZAogKiBAcGFyYW0gYXJyYXk8aW50PDAsIG1heD58c3RyaW5nLCBEYmFsQXJyYXlUeXBlfERiYWxQYXJhbWV0ZXJUeXBlfERiYWxUeXBlfHN0cmluZz4gJHR5cGVzIC0gQGRlcHJlY2F0ZWQgdXNlIERiYWxRdWVyeUV4dHJhY3Rvcjo6d2l0aFR5cGVzKCkgaW5zdGVhZAogKi8="},{"repository_path":"src\/adapter\/etl-adapter-doctrine\/src\/Flow\/ETL\/Adapter\/Doctrine\/functions.php","start_line_in_file":173,"slug":"dbal-from-query","name":"dbal_from_query","namespace":"Flow\\ETL\\Adapter\\Doctrine","parameters":[{"name":"connection","type":[{"name":"Connection","namespace":"Doctrine\\DBAL","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"query","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"parameters","type":[{"name":"array","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"[]"},{"name":"types","type":[{"name":"array","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"[]"}],"return_type":[{"name":"DbalQueryExtractor","namespace":"Flow\\ETL\\Adapter\\Doctrine","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"DOCTRINE","type":"EXTRACTOR"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIEBkZXByZWNhdGVkIHVzZSBmcm9tX2RiYWxfcXVlcnkoKSBpbnN0ZWFkCiAqCiAqIEBwYXJhbSBhcnJheTxzdHJpbmcsIG1peGVkPnxsaXN0PG1peGVkPiAkcGFyYW1ldGVycyAtIEBkZXByZWNhdGVkIHVzZSBEYmFsUXVlcnlFeHRyYWN0b3I6OndpdGhQYXJhbWV0ZXJzKCkgaW5zdGVhZAogKiBAcGFyYW0gYXJyYXk8aW50PDAsIG1heD58c3RyaW5nLCBEYmFsQXJyYXlUeXBlfERiYWxQYXJhbWV0ZXJUeXBlfERiYWxUeXBlfHN0cmluZz4gJHR5cGVzIC0gQGRlcHJlY2F0ZWQgdXNlIERiYWxRdWVyeUV4dHJhY3Rvcjo6d2l0aFR5cGVzKCkgaW5zdGVhZAogKi8="},{"repository_path":"src\/adapter\/etl-adapter-doctrine\/src\/Flow\/ETL\/Adapter\/Doctrine\/functions.php","start_line_in_file":199,"slug":"to-dbal-table-insert","name":"to_dbal_table_insert","namespace":"Flow\\ETL\\Adapter\\Doctrine","parameters":[{"name":"connection","type":[{"name":"Connection","namespace":"Doctrine\\DBAL","is_nullable":false,"is_variadic":false},{"name":"array","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"table","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"options","type":[{"name":"InsertOptions","namespace":"Flow\\Doctrine\\Bulk","is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"null"}],"return_type":[{"name":"DbalLoader","namespace":"Flow\\ETL\\Adapter\\Doctrine","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"DOCTRINE","type":"LOADER"}},{"name":"DocumentationExample","namespace":"Flow\\Documentation\\Attribute","arguments":{"topic":"writing","example":"dbal","option":"upsert"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIEluc2VydCBuZXcgcm93cyBpbnRvIGEgZGF0YWJhc2UgdGFibGUuCiAqIEluc2VydCBjYW4gYWxzbyBiZSB1c2VkIGFzIGFuIHVwc2VydCB3aXRoIHRoZSBoZWxwIG9mIEluc2VydE9wdGlvbnMuCiAqIEluc2VydE9wdGlvbnMgYXJlIHBsYXRmb3JtIHNwZWNpZmljLCBzbyBwbGVhc2UgY2hvb3NlIHRoZSByaWdodCBvbmUgZm9yIHlvdXIgZGF0YWJhc2UuCiAqCiAqICAtIE15U1FMSW5zZXJ0T3B0aW9ucwogKiAgLSBQb3N0Z3JlU1FMSW5zZXJ0T3B0aW9ucwogKiAgLSBTcWxpdGVJbnNlcnRPcHRpb25zCiAqCiAqIEluIG9yZGVyIHRvIGNvbnRyb2wgdGhlIHNpemUgb2YgdGhlIHNpbmdsZSBpbnNlcnQsIHVzZSBEYXRhRnJhbWU6OmNodW5rU2l6ZSgpIG1ldGhvZCBqdXN0IGJlZm9yZSBjYWxsaW5nIERhdGFGcmFtZTo6bG9hZCgpLgogKgogKiBAcGFyYW0gYXJyYXk8c3RyaW5nLCBtaXhlZD58Q29ubmVjdGlvbiAkY29ubmVjdGlvbgogKgogKiBAdGhyb3dzIEludmFsaWRBcmd1bWVudEV4Y2VwdGlvbgogKi8="},{"repository_path":"src\/adapter\/etl-adapter-doctrine\/src\/Flow\/ETL\/Adapter\/Doctrine\/functions.php","start_line_in_file":216,"slug":"to-dbal-table-update","name":"to_dbal_table_update","namespace":"Flow\\ETL\\Adapter\\Doctrine","parameters":[{"name":"connection","type":[{"name":"Connection","namespace":"Doctrine\\DBAL","is_nullable":false,"is_variadic":false},{"name":"array","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"table","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"options","type":[{"name":"UpdateOptions","namespace":"Flow\\Doctrine\\Bulk","is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"null"}],"return_type":[{"name":"DbalLoader","namespace":"Flow\\ETL\\Adapter\\Doctrine","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"DOCTRINE","type":"LOADER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqICBVcGRhdGUgZXhpc3Rpbmcgcm93cyBpbiBkYXRhYmFzZS4KICoKICogIEluIG9yZGVyIHRvIGNvbnRyb2wgdGhlIHNpemUgb2YgdGhlIHNpbmdsZSByZXF1ZXN0LCB1c2UgRGF0YUZyYW1lOjpjaHVua1NpemUoKSBtZXRob2QganVzdCBiZWZvcmUgY2FsbGluZyBEYXRhRnJhbWU6OmxvYWQoKS4KICoKICogQHBhcmFtIGFycmF5PHN0cmluZywgbWl4ZWQ+fENvbm5lY3Rpb24gJGNvbm5lY3Rpb24KICoKICogQHRocm93cyBJbnZhbGlkQXJndW1lbnRFeGNlcHRpb24KICov"},{"repository_path":"src\/adapter\/etl-adapter-doctrine\/src\/Flow\/ETL\/Adapter\/Doctrine\/functions.php","start_line_in_file":235,"slug":"to-dbal-table-delete","name":"to_dbal_table_delete","namespace":"Flow\\ETL\\Adapter\\Doctrine","parameters":[{"name":"connection","type":[{"name":"Connection","namespace":"Doctrine\\DBAL","is_nullable":false,"is_variadic":false},{"name":"array","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"table","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"DbalLoader","namespace":"Flow\\ETL\\Adapter\\Doctrine","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"DOCTRINE","type":"LOADER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIERlbGV0ZSByb3dzIGZyb20gZGF0YWJhc2UgdGFibGUgYmFzZWQgb24gdGhlIHByb3ZpZGVkIGRhdGEuCiAqCiAqIEluIG9yZGVyIHRvIGNvbnRyb2wgdGhlIHNpemUgb2YgdGhlIHNpbmdsZSByZXF1ZXN0LCB1c2UgRGF0YUZyYW1lOjpjaHVua1NpemUoKSBtZXRob2QganVzdCBiZWZvcmUgY2FsbGluZyBEYXRhRnJhbWU6OmxvYWQoKS4KICoKICogQHBhcmFtIGFycmF5PHN0cmluZywgbWl4ZWQ+fENvbm5lY3Rpb24gJGNvbm5lY3Rpb24KICoKICogQHRocm93cyBJbnZhbGlkQXJndW1lbnRFeGNlcHRpb24KICov"},{"repository_path":"src\/adapter\/etl-adapter-doctrine\/src\/Flow\/ETL\/Adapter\/Doctrine\/functions.php","start_line_in_file":250,"slug":"to-dbal-schema-table","name":"to_dbal_schema_table","namespace":"Flow\\ETL\\Adapter\\Doctrine","parameters":[{"name":"schema","type":[{"name":"Schema","namespace":"Flow\\ETL","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"table_name","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"table_options","type":[{"name":"array","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"[]"},{"name":"types_map","type":[{"name":"array","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"[]"}],"return_type":[{"name":"Table","namespace":"Doctrine\\DBAL\\Schema","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"DOCTRINE","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENvbnZlcnRzIGEgRmxvd1xFVExcU2NoZW1hIHRvIGEgRG9jdHJpbmVcREJBTFxTY2hlbWFcVGFibGUuCiAqCiAqIEBwYXJhbSBTY2hlbWEgJHNjaGVtYQogKiBAcGFyYW0gYXJyYXk8c3RyaW5nLCBtaXhlZD4gJHRhYmxlX29wdGlvbnMKICogQHBhcmFtIGFycmF5PGNsYXNzLXN0cmluZzxcRmxvd1xUeXBlc1xUeXBlPG1peGVkPj4sIGNsYXNzLXN0cmluZzxcRG9jdHJpbmVcREJBTFxUeXBlc1xUeXBlPj4gJHR5cGVzX21hcAogKi8="},{"repository_path":"src\/adapter\/etl-adapter-doctrine\/src\/Flow\/ETL\/Adapter\/Doctrine\/functions.php","start_line_in_file":267,"slug":"table-schema-to-flow-schema","name":"table_schema_to_flow_schema","namespace":"Flow\\ETL\\Adapter\\Doctrine","parameters":[{"name":"table","type":[{"name":"Table","namespace":"Doctrine\\DBAL\\Schema","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"types_map","type":[{"name":"array","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"[]"}],"return_type":[{"name":"Schema","namespace":"Flow\\ETL","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"DOCTRINE","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENvbnZlcnRzIGEgRG9jdHJpbmVcREJBTFxTY2hlbWFcVGFibGUgdG8gYSBGbG93XEVUTFxTY2hlbWEuCiAqCiAqIEBwYXJhbSBhcnJheTxjbGFzcy1zdHJpbmc8XEZsb3dcVHlwZXNcVHlwZTxtaXhlZD4+LCBjbGFzcy1zdHJpbmc8XERvY3RyaW5lXERCQUxcVHlwZXNcVHlwZT4+ICR0eXBlc19tYXAKICoKICogQHJldHVybiBTY2hlbWEKICov"},{"repository_path":"src\/adapter\/etl-adapter-doctrine\/src\/Flow\/ETL\/Adapter\/Doctrine\/functions.php","start_line_in_file":278,"slug":"postgresql-insert-options","name":"postgresql_insert_options","namespace":"Flow\\ETL\\Adapter\\Doctrine","parameters":[{"name":"skip_conflicts","type":[{"name":"bool","namespace":null,"is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"null"},{"name":"constraint","type":[{"name":"string","namespace":null,"is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"null"},{"name":"conflict_columns","type":[{"name":"array","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"[]"},{"name":"update_columns","type":[{"name":"array","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"[]"}],"return_type":[{"name":"PostgreSQLInsertOptions","namespace":"Flow\\Doctrine\\Bulk\\Dialect","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"DOCTRINE","type":"HELPER"}},{"name":"DocumentationExample","namespace":"Flow\\Documentation\\Attribute","arguments":{"topic":"writing","example":"dbal","option":"upsert"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIEBwYXJhbSBhcnJheTxzdHJpbmc+ICRjb25mbGljdF9jb2x1bW5zCiAqIEBwYXJhbSBhcnJheTxzdHJpbmc+ICR1cGRhdGVfY29sdW1ucwogKi8="},{"repository_path":"src\/adapter\/etl-adapter-doctrine\/src\/Flow\/ETL\/Adapter\/Doctrine\/functions.php","start_line_in_file":291,"slug":"mysql-insert-options","name":"mysql_insert_options","namespace":"Flow\\ETL\\Adapter\\Doctrine","parameters":[{"name":"skip_conflicts","type":[{"name":"bool","namespace":null,"is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"null"},{"name":"upsert","type":[{"name":"bool","namespace":null,"is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"null"},{"name":"update_columns","type":[{"name":"array","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"[]"}],"return_type":[{"name":"MySQLInsertOptions","namespace":"Flow\\Doctrine\\Bulk\\Dialect","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"DOCTRINE","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIEBwYXJhbSBhcnJheTxzdHJpbmc+ICR1cGRhdGVfY29sdW1ucwogKi8="},{"repository_path":"src\/adapter\/etl-adapter-doctrine\/src\/Flow\/ETL\/Adapter\/Doctrine\/functions.php","start_line_in_file":304,"slug":"sqlite-insert-options","name":"sqlite_insert_options","namespace":"Flow\\ETL\\Adapter\\Doctrine","parameters":[{"name":"skip_conflicts","type":[{"name":"bool","namespace":null,"is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"null"},{"name":"conflict_columns","type":[{"name":"array","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"[]"},{"name":"update_columns","type":[{"name":"array","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"[]"}],"return_type":[{"name":"SqliteInsertOptions","namespace":"Flow\\Doctrine\\Bulk\\Dialect","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"DOCTRINE","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIEBwYXJhbSBhcnJheTxzdHJpbmc+ICRjb25mbGljdF9jb2x1bW5zCiAqIEBwYXJhbSBhcnJheTxzdHJpbmc+ICR1cGRhdGVfY29sdW1ucwogKi8="},{"repository_path":"src\/adapter\/etl-adapter-doctrine\/src\/Flow\/ETL\/Adapter\/Doctrine\/functions.php","start_line_in_file":317,"slug":"postgresql-update-options","name":"postgresql_update_options","namespace":"Flow\\ETL\\Adapter\\Doctrine","parameters":[{"name":"primary_key_columns","type":[{"name":"array","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"[]"},{"name":"update_columns","type":[{"name":"array","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"[]"}],"return_type":[{"name":"PostgreSQLUpdateOptions","namespace":"Flow\\Doctrine\\Bulk\\Dialect","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"DOCTRINE","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIEBwYXJhbSBhcnJheTxzdHJpbmc+ICRwcmltYXJ5X2tleV9jb2x1bW5zCiAqIEBwYXJhbSBhcnJheTxzdHJpbmc+ICR1cGRhdGVfY29sdW1ucwogKi8="},{"repository_path":"src\/adapter\/etl-adapter-doctrine\/src\/Flow\/ETL\/Adapter\/Doctrine\/functions.php","start_line_in_file":338,"slug":"to-dbal-transaction","name":"to_dbal_transaction","namespace":"Flow\\ETL\\Adapter\\Doctrine","parameters":[{"name":"connection","type":[{"name":"Connection","namespace":"Doctrine\\DBAL","is_nullable":false,"is_variadic":false},{"name":"array","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"sinks","type":[{"name":"Loader","namespace":"Flow\\ETL","is_nullable":false,"is_variadic":false},{"name":"Sink","namespace":"Flow\\ETL","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":true,"default_value":null}],"return_type":[{"name":"Transactional","namespace":"Flow\\ETL\\Sink","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"DOCTRINE","type":"LOADER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIFdyaXRlIGV2ZXJ5IHNpbmsgd2l0aGluIGRhdGFiYXNlIHRyYW5zYWN0aW9ucy4KICogRWFjaCBiYXRjaCBvZiByb3dzIGlzIHdyaXR0ZW4gaW4gaXRzIG93biB0cmFuc2FjdGlvbjsgcm93cyBhIHNpbmsncyBUcmFuc2Zvcm1hdGlvbiBkZWxpdmVycyB3aGVuCiAqIHRoZSBydW4gZW5kcyAoYmxvY2tpbmcgb3BlcmF0aW9ucyBkcmFpbiB0aGVyZSkgYXJlIGNvbW1pdHRlZCBpbiBvbmUgZmluYWwgdHJhbnNhY3Rpb24uCiAqIElmIGFueSBzaW5rIGZhaWxzLCB0aGUgb3BlbiB0cmFuc2FjdGlvbiBpcyByb2xsZWQgYmFjay4KICogQSBwbGFpbiBMb2FkZXIgY2hpbGQgaXMgYSBiYXJlIHNpbmsgcm9vdDsgYSB0b190cmFuc2Zvcm1hdGlvbiguLi4pIGNoaWxkIGRlbGl2ZXJzIGluc2lkZSB0aGUgc2FtZQogKiB0cmFuc2FjdGlvbi4gRXZlcnkgY2hpbGQncyBsb2FkZXIgbXVzdCB1c2UgdGhlIHNhbWUgY29ubmVjdGlvbiBhcyB0aGUgdHJhbnNhY3Rpb246IHBhc3Mgb25lIGxpdmUKICogQ29ubmVjdGlvbiB0byBib3RoIC0gYSBsb2FkZXIgYnVpbHQgZnJvbSBhcnJheSBwYXJhbXMgb3BlbnMgaXRzIG93biBjb25uZWN0aW9uIGFuZCBlc2NhcGVzIHRoZQogKiB0cmFuc2FjdGlvbi4KICoKICogQHBhcmFtIGFycmF5PHN0cmluZywgbWl4ZWQ+fENvbm5lY3Rpb24gJGNvbm5lY3Rpb24KICogQHBhcmFtIExvYWRlcnxTaW5rIC4uLiRzaW5rcyAtIHNpbmtzIHdyaXR0ZW4gd2l0aGluIHRoZSB0cmFuc2FjdGlvbgogKgogKiBAdGhyb3dzIEludmFsaWRBcmd1bWVudEV4Y2VwdGlvbgogKi8="},{"repository_path":"src\/adapter\/etl-adapter-doctrine\/src\/Flow\/ETL\/Adapter\/Doctrine\/functions.php","start_line_in_file":347,"slug":"pagination-key-asc","name":"pagination_key_asc","namespace":"Flow\\ETL\\Adapter\\Doctrine","parameters":[{"name":"column","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"type","type":[{"name":"ParameterType","namespace":"Doctrine\\DBAL","is_nullable":false,"is_variadic":false},{"name":"Type","namespace":"Doctrine\\DBAL\\Types","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"Doctrine\\DBAL\\ParameterType::..."}],"return_type":[{"name":"Key","namespace":"Flow\\ETL\\Adapter\\Doctrine\\Pagination","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"DOCTRINE","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/adapter\/etl-adapter-doctrine\/src\/Flow\/ETL\/Adapter\/Doctrine\/functions.php","start_line_in_file":353,"slug":"pagination-key-desc","name":"pagination_key_desc","namespace":"Flow\\ETL\\Adapter\\Doctrine","parameters":[{"name":"column","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"type","type":[{"name":"ParameterType","namespace":"Doctrine\\DBAL","is_nullable":false,"is_variadic":false},{"name":"Type","namespace":"Doctrine\\DBAL\\Types","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"Doctrine\\DBAL\\ParameterType::..."}],"return_type":[{"name":"Key","namespace":"Flow\\ETL\\Adapter\\Doctrine\\Pagination","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"DOCTRINE","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/adapter\/etl-adapter-doctrine\/src\/Flow\/ETL\/Adapter\/Doctrine\/functions.php","start_line_in_file":359,"slug":"pagination-key-set","name":"pagination_key_set","namespace":"Flow\\ETL\\Adapter\\Doctrine","parameters":[{"name":"keys","type":[{"name":"Key","namespace":"Flow\\ETL\\Adapter\\Doctrine\\Pagination","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":true,"default_value":null}],"return_type":[{"name":"KeySet","namespace":"Flow\\ETL\\Adapter\\Doctrine\\Pagination","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"DOCTRINE","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/adapter\/etl-adapter-excel\/src\/Flow\/ETL\/Adapter\/Excel\/DSL\/functions.php","start_line_in_file":22,"slug":"from-excel","name":"from_excel","namespace":"Flow\\ETL\\Adapter\\Excel\\DSL","parameters":[{"name":"path","type":[{"name":"Path","namespace":"Flow\\Filesystem","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"filesystem","type":[{"name":"Filesystem","namespace":"Flow\\Filesystem","is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"Flow\\Filesystem\\Local\\NativeLocalFilesystem::..."}],"return_type":[{"name":"ExcelExtractor","namespace":"Flow\\ETL\\Adapter\\Excel","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"EXCEL","type":"EXTRACTOR"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/adapter\/etl-adapter-excel\/src\/Flow\/ETL\/Adapter\/Excel\/DSL\/functions.php","start_line_in_file":28,"slug":"to-excel","name":"to_excel","namespace":"Flow\\ETL\\Adapter\\Excel\\DSL","parameters":[{"name":"path","type":[{"name":"Path","namespace":"Flow\\Filesystem","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"filesystem","type":[{"name":"Filesystem","namespace":"Flow\\Filesystem","is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"Flow\\Filesystem\\Local\\NativeLocalFilesystem::..."}],"return_type":[{"name":"ExcelLoader","namespace":"Flow\\ETL\\Adapter\\Excel","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"EXCEL","type":"LOADER"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/adapter\/etl-adapter-excel\/src\/Flow\/ETL\/Adapter\/Excel\/DSL\/functions.php","start_line_in_file":34,"slug":"is-valid-excel-sheet-name","name":"is_valid_excel_sheet_name","namespace":"Flow\\ETL\\Adapter\\Excel\\DSL","parameters":[{"name":"sheet_name","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"IsValidExcelSheetName","namespace":"Flow\\ETL\\Adapter\\Excel\\Function","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"EXCEL","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/adapter\/etl-adapter-google-sheet\/src\/Flow\/ETL\/Adapter\/GoogleSheet\/functions.php","start_line_in_file":22,"slug":"from-google-sheet","name":"from_google_sheet","namespace":"Flow\\ETL\\Adapter\\GoogleSheet","parameters":[{"name":"auth_config","type":[{"name":"Sheets","namespace":"Google\\Service","is_nullable":false,"is_variadic":false},{"name":"array","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"spreadsheet_id","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"sheet_name","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"with_header","type":[{"name":"bool","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"true"},{"name":"rows_per_page","type":[{"name":"int","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"1000"},{"name":"options","type":[{"name":"array","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"[]"}],"return_type":[{"name":"GoogleSheetExtractor","namespace":"Flow\\ETL\\Adapter\\GoogleSheet","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"GOOGLE_SHEET","type":"EXTRACTOR"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIEBwYXJhbSBhcnJheXt0eXBlOiBzdHJpbmcsIHByb2plY3RfaWQ6IHN0cmluZywgcHJpdmF0ZV9rZXlfaWQ6IHN0cmluZywgcHJpdmF0ZV9rZXk6IHN0cmluZywgY2xpZW50X2VtYWlsOiBzdHJpbmcsIGNsaWVudF9pZDogc3RyaW5nLCBhdXRoX3VyaTogc3RyaW5nLCB0b2tlbl91cmk6IHN0cmluZywgYXV0aF9wcm92aWRlcl94NTA5X2NlcnRfdXJsOiBzdHJpbmcsIGNsaWVudF94NTA5X2NlcnRfdXJsOiBzdHJpbmd9fFNoZWV0cyAkYXV0aF9jb25maWcKICogQHBhcmFtIHN0cmluZyAkc3ByZWFkc2hlZXRfaWQKICogQHBhcmFtIHN0cmluZyAkc2hlZXRfbmFtZQogKiBAcGFyYW0gYm9vbCAkd2l0aF9oZWFkZXIgLSBAZGVwcmVjYXRlZCB1c2Ugd2l0aEhlYWRlciBtZXRob2QgaW5zdGVhZAogKiBAcGFyYW0gaW50ICRyb3dzX3Blcl9wYWdlIC0gaG93IG1hbnkgcm93cyBwZXIgcGFnZSB0byBmZXRjaCBmcm9tIEdvb2dsZSBTaGVldHMgQVBJIC0gQGRlcHJlY2F0ZWQgdXNlIHdpdGhSb3dzUGVyUGFnZSBtZXRob2QgaW5zdGVhZAogKiBAcGFyYW0gYXJyYXl7ZGF0ZVRpbWVSZW5kZXJPcHRpb24\/OiBzdHJpbmcsIG1ham9yRGltZW5zaW9uPzogc3RyaW5nLCB2YWx1ZVJlbmRlck9wdGlvbj86IHN0cmluZ30gJG9wdGlvbnMgLSBAZGVwcmVjYXRlZCB1c2Ugd2l0aE9wdGlvbnMgbWV0aG9kIGluc3RlYWQKICov"},{"repository_path":"src\/adapter\/etl-adapter-google-sheet\/src\/Flow\/ETL\/Adapter\/GoogleSheet\/functions.php","start_line_in_file":56,"slug":"from-google-sheet-columns","name":"from_google_sheet_columns","namespace":"Flow\\ETL\\Adapter\\GoogleSheet","parameters":[{"name":"auth_config","type":[{"name":"Sheets","namespace":"Google\\Service","is_nullable":false,"is_variadic":false},{"name":"array","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"spreadsheet_id","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"sheet_name","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"start_range_column","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"end_range_column","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"with_header","type":[{"name":"bool","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"true"},{"name":"rows_per_page","type":[{"name":"int","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"1000"},{"name":"options","type":[{"name":"array","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"[]"}],"return_type":[{"name":"GoogleSheetExtractor","namespace":"Flow\\ETL\\Adapter\\GoogleSheet","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"GOOGLE_SHEET","type":"EXTRACTOR"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIEBwYXJhbSBhcnJheXt0eXBlOiBzdHJpbmcsIHByb2plY3RfaWQ6IHN0cmluZywgcHJpdmF0ZV9rZXlfaWQ6IHN0cmluZywgcHJpdmF0ZV9rZXk6IHN0cmluZywgY2xpZW50X2VtYWlsOiBzdHJpbmcsIGNsaWVudF9pZDogc3RyaW5nLCBhdXRoX3VyaTogc3RyaW5nLCB0b2tlbl91cmk6IHN0cmluZywgYXV0aF9wcm92aWRlcl94NTA5X2NlcnRfdXJsOiBzdHJpbmcsIGNsaWVudF94NTA5X2NlcnRfdXJsOiBzdHJpbmd9fFNoZWV0cyAkYXV0aF9jb25maWcKICogQHBhcmFtIHN0cmluZyAkc3ByZWFkc2hlZXRfaWQKICogQHBhcmFtIHN0cmluZyAkc2hlZXRfbmFtZQogKiBAcGFyYW0gc3RyaW5nICRzdGFydF9yYW5nZV9jb2x1bW4KICogQHBhcmFtIHN0cmluZyAkZW5kX3JhbmdlX2NvbHVtbgogKiBAcGFyYW0gYm9vbCAkd2l0aF9oZWFkZXIgLSBAZGVwcmVjYXRlZCB1c2Ugd2l0aEhlYWRlciBtZXRob2QgaW5zdGVhZAogKiBAcGFyYW0gaW50ICRyb3dzX3Blcl9wYWdlIC0gaG93IG1hbnkgcm93cyBwZXIgcGFnZSB0byBmZXRjaCBmcm9tIEdvb2dsZSBTaGVldHMgQVBJLCBkZWZhdWx0IDEwMDAgLSBAZGVwcmVjYXRlZCB1c2Ugd2l0aFJvd3NQZXJQYWdlIG1ldGhvZCBpbnN0ZWFkCiAqIEBwYXJhbSBhcnJheXtkYXRlVGltZVJlbmRlck9wdGlvbj86IHN0cmluZywgbWFqb3JEaW1lbnNpb24\/OiBzdHJpbmcsIHZhbHVlUmVuZGVyT3B0aW9uPzogc3RyaW5nfSAkb3B0aW9ucyAtIEBkZXByZWNhdGVkIHVzZSB3aXRoT3B0aW9ucyBtZXRob2QgaW5zdGVhZAogKi8="},{"repository_path":"src\/adapter\/etl-adapter-http\/src\/Flow\/ETL\/Adapter\/Http\/DSL\/functions.php","start_line_in_file":35,"slug":"from-dynamic-http-requests","name":"from_dynamic_http_requests","namespace":"Flow\\ETL\\Adapter\\Http","parameters":[{"name":"client","type":[{"name":"ClientInterface","namespace":"Psr\\Http\\Client","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"requestFactory","type":[{"name":"NextRequestFactory","namespace":"Flow\\ETL\\Adapter\\Http\\DynamicExtractor","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"schema","type":[{"name":"Schema","namespace":"Flow\\ETL","is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"null"}],"return_type":[{"name":"PsrHttpClientDynamicExtractor","namespace":"Flow\\ETL\\Adapter\\Http","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"HTTP","type":"EXTRACTOR"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/adapter\/etl-adapter-http\/src\/Flow\/ETL\/Adapter\/Http\/DSL\/functions.php","start_line_in_file":53,"slug":"from-static-http-requests","name":"from_static_http_requests","namespace":"Flow\\ETL\\Adapter\\Http","parameters":[{"name":"client","type":[{"name":"ClientInterface","namespace":"Psr\\Http\\Client","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"requests","type":[{"name":"iterable","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"schema","type":[{"name":"Schema","namespace":"Flow\\ETL","is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"null"}],"return_type":[{"name":"PsrHttpClientStaticExtractor","namespace":"Flow\\ETL\\Adapter\\Http","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"HTTP","type":"EXTRACTOR"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIEBwYXJhbSBpdGVyYWJsZTxSZXF1ZXN0SW50ZXJmYWNlPiAkcmVxdWVzdHMKICov"},{"repository_path":"src\/adapter\/etl-adapter-http\/src\/Flow\/ETL\/Adapter\/Http\/DSL\/functions.php","start_line_in_file":68,"slug":"from-http-paginated","name":"from_http_paginated","namespace":"Flow\\ETL\\Adapter\\Http","parameters":[{"name":"client","type":[{"name":"ClientInterface","namespace":"Psr\\Http\\Client","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"request","type":[{"name":"RequestInterface","namespace":"Psr\\Http\\Message","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"paginator","type":[{"name":"Paginator","namespace":"Flow\\ETL\\Adapter\\Http\\Pagination","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"schema","type":[{"name":"Schema","namespace":"Flow\\ETL","is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"null"}],"return_type":[{"name":"PsrHttpClientPaginatedExtractor","namespace":"Flow\\ETL\\Adapter\\Http","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"HTTP","type":"EXTRACTOR"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/adapter\/etl-adapter-http\/src\/Flow\/ETL\/Adapter\/Http\/DSL\/functions.php","start_line_in_file":84,"slug":"http-pagination-page-number","name":"http_pagination_page_number","namespace":"Flow\\ETL\\Adapter\\Http","parameters":[{"name":"inject","type":[{"name":"RequestOption","namespace":"Flow\\ETL\\Adapter\\Http\\Pagination","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"start_page","type":[{"name":"int","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"1"},{"name":"page_size","type":[{"name":"int","namespace":null,"is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"null"},{"name":"size_option","type":[{"name":"RequestOption","namespace":"Flow\\ETL\\Adapter\\Http\\Pagination","is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"null"},{"name":"inject_on_first_request","type":[{"name":"bool","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"true"},{"name":"records_path","type":[{"name":"string","namespace":null,"is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"null"},{"name":"stop_on_client_error","type":[{"name":"bool","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"true"},{"name":"stop_when","type":[{"name":"StopWhen","namespace":"Flow\\ETL\\Adapter\\Http\\Pagination","is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"null"}],"return_type":[{"name":"Paginator","namespace":"Flow\\ETL\\Adapter\\Http\\Pagination","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"HTTP","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/adapter\/etl-adapter-http\/src\/Flow\/ETL\/Adapter\/Http\/DSL\/functions.php","start_line_in_file":104,"slug":"http-pagination-offset","name":"http_pagination_offset","namespace":"Flow\\ETL\\Adapter\\Http","parameters":[{"name":"offset_option","type":[{"name":"RequestOption","namespace":"Flow\\ETL\\Adapter\\Http\\Pagination","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"limit_option","type":[{"name":"RequestOption","namespace":"Flow\\ETL\\Adapter\\Http\\Pagination","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"limit","type":[{"name":"int","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"start_offset","type":[{"name":"int","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"0"},{"name":"total_path","type":[{"name":"string","namespace":null,"is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"null"},{"name":"inject_on_first_request","type":[{"name":"bool","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"true"},{"name":"stop_on_client_error","type":[{"name":"bool","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"true"},{"name":"stop_when","type":[{"name":"StopWhen","namespace":"Flow\\ETL\\Adapter\\Http\\Pagination","is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"null"}],"return_type":[{"name":"Paginator","namespace":"Flow\\ETL\\Adapter\\Http\\Pagination","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"HTTP","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/adapter\/etl-adapter-http\/src\/Flow\/ETL\/Adapter\/Http\/DSL\/functions.php","start_line_in_file":124,"slug":"http-pagination-cursor","name":"http_pagination_cursor","namespace":"Flow\\ETL\\Adapter\\Http","parameters":[{"name":"cursor_path","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"inject","type":[{"name":"RequestOption","namespace":"Flow\\ETL\\Adapter\\Http\\Pagination","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"stop_on_client_error","type":[{"name":"bool","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"true"},{"name":"stop_when","type":[{"name":"StopWhen","namespace":"Flow\\ETL\\Adapter\\Http\\Pagination","is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"null"}],"return_type":[{"name":"Paginator","namespace":"Flow\\ETL\\Adapter\\Http\\Pagination","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"HTTP","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/adapter\/etl-adapter-http\/src\/Flow\/ETL\/Adapter\/Http\/DSL\/functions.php","start_line_in_file":140,"slug":"http-pagination-next-url","name":"http_pagination_next_url","namespace":"Flow\\ETL\\Adapter\\Http","parameters":[{"name":"path","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"stop_on_client_error","type":[{"name":"bool","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"true"},{"name":"stop_when","type":[{"name":"StopWhen","namespace":"Flow\\ETL\\Adapter\\Http\\Pagination","is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"null"}],"return_type":[{"name":"Paginator","namespace":"Flow\\ETL\\Adapter\\Http\\Pagination","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"HTTP","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/adapter\/etl-adapter-http\/src\/Flow\/ETL\/Adapter\/Http\/DSL\/functions.php","start_line_in_file":155,"slug":"http-pagination-link-header","name":"http_pagination_link_header","namespace":"Flow\\ETL\\Adapter\\Http","parameters":[{"name":"rel","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"'next'"},{"name":"stop_on_client_error","type":[{"name":"bool","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"true"},{"name":"stop_when","type":[{"name":"StopWhen","namespace":"Flow\\ETL\\Adapter\\Http\\Pagination","is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"null"}],"return_type":[{"name":"Paginator","namespace":"Flow\\ETL\\Adapter\\Http\\Pagination","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"HTTP","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/adapter\/etl-adapter-http\/src\/Flow\/ETL\/Adapter\/Http\/DSL\/functions.php","start_line_in_file":164,"slug":"http-pagination-last-record-cursor","name":"http_pagination_last_record_cursor","namespace":"Flow\\ETL\\Adapter\\Http","parameters":[{"name":"record_path","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"inject","type":[{"name":"RequestOption","namespace":"Flow\\ETL\\Adapter\\Http\\Pagination","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"stop_on_client_error","type":[{"name":"bool","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"true"},{"name":"stop_when","type":[{"name":"StopWhen","namespace":"Flow\\ETL\\Adapter\\Http\\Pagination","is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"null"}],"return_type":[{"name":"Paginator","namespace":"Flow\\ETL\\Adapter\\Http\\Pagination","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"HTTP","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/adapter\/etl-adapter-http\/src\/Flow\/ETL\/Adapter\/Http\/DSL\/functions.php","start_line_in_file":180,"slug":"http-request-option-query","name":"http_request_option_query","namespace":"Flow\\ETL\\Adapter\\Http","parameters":[{"name":"name","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"RequestOption","namespace":"Flow\\ETL\\Adapter\\Http\\Pagination","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"HTTP","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/adapter\/etl-adapter-http\/src\/Flow\/ETL\/Adapter\/Http\/DSL\/functions.php","start_line_in_file":186,"slug":"http-request-option-header","name":"http_request_option_header","namespace":"Flow\\ETL\\Adapter\\Http","parameters":[{"name":"name","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"RequestOption","namespace":"Flow\\ETL\\Adapter\\Http\\Pagination","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"HTTP","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/adapter\/etl-adapter-http\/src\/Flow\/ETL\/Adapter\/Http\/DSL\/functions.php","start_line_in_file":192,"slug":"http-request-option-body","name":"http_request_option_body","namespace":"Flow\\ETL\\Adapter\\Http","parameters":[{"name":"path","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"stream_factory","type":[{"name":"StreamFactoryInterface","namespace":"Psr\\Http\\Message","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"RequestOption","namespace":"Flow\\ETL\\Adapter\\Http\\Pagination","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"HTTP","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/adapter\/etl-adapter-http\/src\/Flow\/ETL\/Adapter\/Http\/DSL\/functions.php","start_line_in_file":198,"slug":"http-request-option-uri","name":"http_request_option_uri","namespace":"Flow\\ETL\\Adapter\\Http","parameters":[],"return_type":[{"name":"RequestOption","namespace":"Flow\\ETL\\Adapter\\Http\\Pagination","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"HTTP","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/adapter\/etl-adapter-http\/src\/Flow\/ETL\/Adapter\/Http\/DSL\/functions.php","start_line_in_file":204,"slug":"http-stop-when-path-missing","name":"http_stop_when_path_missing","namespace":"Flow\\ETL\\Adapter\\Http","parameters":[{"name":"path","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"StopWhen","namespace":"Flow\\ETL\\Adapter\\Http\\Pagination","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"HTTP","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/adapter\/etl-adapter-http\/src\/Flow\/ETL\/Adapter\/Http\/DSL\/functions.php","start_line_in_file":210,"slug":"http-stop-when-empty-path","name":"http_stop_when_empty_path","namespace":"Flow\\ETL\\Adapter\\Http","parameters":[{"name":"path","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"StopWhen","namespace":"Flow\\ETL\\Adapter\\Http\\Pagination","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"HTTP","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/adapter\/etl-adapter-http\/src\/Flow\/ETL\/Adapter\/Http\/DSL\/functions.php","start_line_in_file":216,"slug":"http-stop-when-flag-false","name":"http_stop_when_flag_false","namespace":"Flow\\ETL\\Adapter\\Http","parameters":[{"name":"path","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"StopWhen","namespace":"Flow\\ETL\\Adapter\\Http\\Pagination","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"HTTP","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/adapter\/etl-adapter-http\/src\/Flow\/ETL\/Adapter\/Http\/DSL\/functions.php","start_line_in_file":222,"slug":"http-stop-when-flag-true","name":"http_stop_when_flag_true","namespace":"Flow\\ETL\\Adapter\\Http","parameters":[{"name":"path","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"StopWhen","namespace":"Flow\\ETL\\Adapter\\Http\\Pagination","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"HTTP","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/adapter\/etl-adapter-http\/src\/Flow\/ETL\/Adapter\/Http\/DSL\/functions.php","start_line_in_file":228,"slug":"http-stop-when-total-reached","name":"http_stop_when_total_reached","namespace":"Flow\\ETL\\Adapter\\Http","parameters":[{"name":"total_path","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"StopWhen","namespace":"Flow\\ETL\\Adapter\\Http\\Pagination","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"HTTP","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/adapter\/etl-adapter-http\/src\/Flow\/ETL\/Adapter\/Http\/DSL\/functions.php","start_line_in_file":234,"slug":"http-stop-when-max-pages","name":"http_stop_when_max_pages","namespace":"Flow\\ETL\\Adapter\\Http","parameters":[{"name":"pages","type":[{"name":"int","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"StopWhen","namespace":"Flow\\ETL\\Adapter\\Http\\Pagination","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"HTTP","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/adapter\/etl-adapter-http\/src\/Flow\/ETL\/Adapter\/Http\/DSL\/functions.php","start_line_in_file":240,"slug":"http-stop-when-max-results","name":"http_stop_when_max_results","namespace":"Flow\\ETL\\Adapter\\Http","parameters":[{"name":"count","type":[{"name":"int","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"StopWhen","namespace":"Flow\\ETL\\Adapter\\Http\\Pagination","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"HTTP","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/adapter\/etl-adapter-json\/src\/Flow\/ETL\/Adapter\/JSON\/functions.php","start_line_in_file":32,"slug":"from-json","name":"from_json","namespace":"Flow\\ETL\\Adapter\\JSON","parameters":[{"name":"path","type":[{"name":"Path","namespace":"Flow\\Filesystem","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"pointer","type":[{"name":"string","namespace":null,"is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"null"},{"name":"schema","type":[{"name":"Schema","namespace":"Flow\\ETL","is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"null"},{"name":"filesystem","type":[{"name":"Filesystem","namespace":"Flow\\Filesystem","is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"Flow\\Filesystem\\Local\\NativeLocalFilesystem::..."}],"return_type":[{"name":"JsonExtractor","namespace":"Flow\\ETL\\Adapter\\JSON\\JSONMachine","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"JSON","type":"EXTRACTOR"}},{"name":"DocumentationExample","namespace":"Flow\\Documentation\\Attribute","arguments":{"topic":"reading","example":"json"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIEBwYXJhbSBQYXRofHN0cmluZyAkcGF0aCAtIHN0cmluZyBpcyBpbnRlcm5hbGx5IHR1cm5lZCBpbnRvIHN0cmVhbQogKiBAcGFyYW0gP3N0cmluZyAkcG9pbnRlciAtIGlmIHlvdSB3YW50IHRvIGl0ZXJhdGUgb25seSByZXN1bHRzIG9mIGEgc3VidHJlZSwgdXNlIGEgcG9pbnRlciwgcmVhZCBtb3JlIGF0IGh0dHBzOi8vZ2l0aHViLmNvbS9oYWxheGEvanNvbi1tYWNoaW5lI3BhcnNpbmctYS1zdWJ0cmVlIC0gQGRlcHJlY2F0ZSB1c2Ugd2l0aFBvaW50ZXIgbWV0aG9kIGluc3RlYWQKICogQHBhcmFtIG51bGx8U2NoZW1hICRzY2hlbWEgLSBlbmZvcmNlIHNjaGVtYSBvbiB0aGUgZXh0cmFjdGVkIGRhdGEgLSBAZGVwcmVjYXRlIHVzZSB3aXRoU2NoZW1hIG1ldGhvZCBpbnN0ZWFkCiAqLw=="},{"repository_path":"src\/adapter\/etl-adapter-json\/src\/Flow\/ETL\/Adapter\/JSON\/functions.php","start_line_in_file":58,"slug":"from-json-lines","name":"from_json_lines","namespace":"Flow\\ETL\\Adapter\\JSON","parameters":[{"name":"path","type":[{"name":"Path","namespace":"Flow\\Filesystem","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"filesystem","type":[{"name":"Filesystem","namespace":"Flow\\Filesystem","is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"Flow\\Filesystem\\Local\\NativeLocalFilesystem::..."}],"return_type":[{"name":"JsonLinesExtractor","namespace":"Flow\\ETL\\Adapter\\JSON\\JSONMachine","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"JSON","type":"EXTRACTOR"}},{"name":"DocumentationExample","namespace":"Flow\\Documentation\\Attribute","arguments":{"topic":"reading","example":"jsonl"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIFVzZWQgdG8gcmVhZCBmcm9tIGEgSlNPTiBsaW5lcyBodHRwczovL2pzb25saW5lcy5vcmcvIGZvcm1hdHRlZCBmaWxlLgogKgogKiBAcGFyYW0gUGF0aHxzdHJpbmcgJHBhdGggLSBzdHJpbmcgaXMgaW50ZXJuYWxseSB0dXJuZWQgaW50byBzdHJlYW0KICov"},{"repository_path":"src\/adapter\/etl-adapter-json\/src\/Flow\/ETL\/Adapter\/JSON\/functions.php","start_line_in_file":72,"slug":"to-json","name":"to_json","namespace":"Flow\\ETL\\Adapter\\JSON","parameters":[{"name":"path","type":[{"name":"Path","namespace":"Flow\\Filesystem","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"flags","type":[{"name":"int","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"4194304"},{"name":"date_time_format","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"'Y-m-d\\\\TH:i:sP'"},{"name":"put_rows_in_new_lines","type":[{"name":"bool","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"false"},{"name":"filesystem","type":[{"name":"Filesystem","namespace":"Flow\\Filesystem","is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"Flow\\Filesystem\\Local\\NativeLocalFilesystem::..."}],"return_type":[{"name":"JsonLoader","namespace":"Flow\\ETL\\Adapter\\JSON","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"JSON","type":"LOADER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIEBwYXJhbSBQYXRofHN0cmluZyAkcGF0aAogKiBAcGFyYW0gaW50ICRmbGFncyAtIFBIUCBKU09OIEZsYWdzIC0gQGRlcHJlY2F0ZSB1c2Ugd2l0aEZsYWdzIG1ldGhvZCBpbnN0ZWFkCiAqIEBwYXJhbSBzdHJpbmcgJGRhdGVfdGltZV9mb3JtYXQgLSBmb3JtYXQgZm9yIERhdGVUaW1lSW50ZXJmYWNlOjpmb3JtYXQoKSAtIEBkZXByZWNhdGUgdXNlIHdpdGhEYXRlVGltZUZvcm1hdCBtZXRob2QgaW5zdGVhZAogKiBAcGFyYW0gYm9vbCAkcHV0X3Jvd3NfaW5fbmV3X2xpbmVzIC0gaWYgeW91IHdhbnQgdG8gcHV0IGVhY2ggcm93IGluIGEgbmV3IGxpbmUgLSBAZGVwcmVjYXRlIHVzZSB3aXRoUm93c0luTmV3TGluZXMgbWV0aG9kIGluc3RlYWQKICoKICogQHJldHVybiBKc29uTG9hZGVyCiAqLw=="},{"repository_path":"src\/adapter\/etl-adapter-json\/src\/Flow\/ETL\/Adapter\/JSON\/functions.php","start_line_in_file":93,"slug":"to-json-lines","name":"to_json_lines","namespace":"Flow\\ETL\\Adapter\\JSON","parameters":[{"name":"path","type":[{"name":"Path","namespace":"Flow\\Filesystem","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"filesystem","type":[{"name":"Filesystem","namespace":"Flow\\Filesystem","is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"Flow\\Filesystem\\Local\\NativeLocalFilesystem::..."}],"return_type":[{"name":"JsonLinesLoader","namespace":"Flow\\ETL\\Adapter\\JSON","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"JSON","type":"LOADER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIFVzZWQgdG8gd3JpdGUgdG8gYSBKU09OIGxpbmVzIGh0dHBzOi8vanNvbmxpbmVzLm9yZy8gZm9ybWF0dGVkIGZpbGUuCiAqCiAqIEBwYXJhbSBQYXRofHN0cmluZyAkcGF0aAogKgogKiBAcmV0dXJuIEpzb25MaW5lc0xvYWRlcgogKi8="},{"repository_path":"src\/adapter\/etl-adapter-json\/src\/Flow\/ETL\/Adapter\/JSON\/functions.php","start_line_in_file":107,"slug":"schema-from-json-schema","name":"schema_from_json_schema","namespace":"Flow\\ETL\\Adapter\\JSON","parameters":[{"name":"json_schema","type":[{"name":"Path","namespace":"Flow\\Filesystem","is_nullable":false,"is_variadic":false},{"name":"array","namespace":null,"is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"client","type":[{"name":"ClientInterface","namespace":"Psr\\Http\\Client","is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"null"},{"name":"request_factory","type":[{"name":"RequestFactoryInterface","namespace":"Psr\\Http\\Message","is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"null"},{"name":"filesystem","type":[{"name":"Filesystem","namespace":"Flow\\Filesystem","is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"Flow\\Filesystem\\Local\\NativeLocalFilesystem::..."}],"return_type":[{"name":"Schema","namespace":"Flow\\ETL","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"JSON","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENvbnZlcnQgYSBKU09OIFNjaGVtYSAoaHR0cHM6Ly9qc29uLXNjaGVtYS5vcmcpIGRvY3VtZW50IGludG8gYSBGbG93IFNjaGVtYS4KICoKICogQHBhcmFtIGFycmF5PHN0cmluZywgbWl4ZWQ+fFBhdGh8c3RyaW5nICRqc29uX3NjaGVtYSAtIGRlY29kZWQgZG9jdW1lbnQsIHJhdyBKU09OIGRvY3VtZW50IG9yIGEgcGF0aCB0byBhIHNjaGVtYSBmaWxlCiAqIEBwYXJhbSBudWxsfENsaWVudEludGVyZmFjZSAkY2xpZW50IC0gUFNSLTE4IGh0dHAgY2xpZW50LCByZXF1aXJlZCB0byByZXNvbHZlIHJlbW90ZSBodHRwKHMpIHJlZmVyZW5jZXMKICogQHBhcmFtIG51bGx8UmVxdWVzdEZhY3RvcnlJbnRlcmZhY2UgJHJlcXVlc3RfZmFjdG9yeSAtIFBTUi0xNyByZXF1ZXN0IGZhY3RvcnksIHJlcXVpcmVkIHRvIHJlc29sdmUgcmVtb3RlIGh0dHAocykgcmVmZXJlbmNlcwogKiBAcGFyYW0gRmlsZXN5c3RlbSAkZmlsZXN5c3RlbSAtIGZpbGVzeXN0ZW0gdXNlZCB0byByZWFkIGxvY2FsIHNjaGVtYSByZWZlcmVuY2VzCiAqLw=="},{"repository_path":"src\/adapter\/etl-adapter-json\/src\/Flow\/ETL\/Adapter\/JSON\/functions.php","start_line_in_file":122,"slug":"schema-to-json-schema","name":"schema_to_json_schema","namespace":"Flow\\ETL\\Adapter\\JSON","parameters":[{"name":"schema","type":[{"name":"Schema","namespace":"Flow\\ETL","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"array","namespace":null,"is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"JSON","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENvbnZlcnQgYSBGbG93IFNjaGVtYSBpbnRvIGEgSlNPTiBTY2hlbWEgKGh0dHBzOi8vanNvbi1zY2hlbWEub3JnLCBkcmFmdCAyMDIwLTEyKSBkb2N1bWVudC4KICoKICogQHJldHVybiBhcnJheTxzdHJpbmcsIG1peGVkPgogKi8="},{"repository_path":"src\/adapter\/etl-adapter-parquet\/src\/Flow\/ETL\/Adapter\/Parquet\/functions.php","start_line_in_file":36,"slug":"from-parquet","name":"from_parquet","namespace":"Flow\\ETL\\Adapter\\Parquet","parameters":[{"name":"path","type":[{"name":"Path","namespace":"Flow\\Filesystem","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"columns","type":[{"name":"array","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"[]"},{"name":"options","type":[{"name":"Options","namespace":"Flow\\Parquet","is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"Flow\\Parquet\\Options::..."},{"name":"byte_order","type":[{"name":"ByteOrder","namespace":"Flow\\Parquet\\Binary","is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"Flow\\Parquet\\Binary\\ByteOrder::..."},{"name":"offset","type":[{"name":"int","namespace":null,"is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"null"},{"name":"engine","type":[{"name":"ParquetEngine","namespace":"Flow\\Parquet","is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"null"},{"name":"filesystem","type":[{"name":"Filesystem","namespace":"Flow\\Filesystem","is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"Flow\\Filesystem\\Local\\NativeLocalFilesystem::..."}],"return_type":[{"name":"ParquetExtractor","namespace":"Flow\\ETL\\Adapter\\Parquet","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PARQUET","type":"EXTRACTOR"}},{"name":"DocumentationExample","namespace":"Flow\\Documentation\\Attribute","arguments":{"topic":"reading","example":"parquet"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIEBwYXJhbSBQYXRofHN0cmluZyAkcGF0aAogKiBAcGFyYW0gYXJyYXk8c3RyaW5nPiAkY29sdW1ucyAtIGxpc3Qgb2YgY29sdW1ucyB0byByZWFkIGZyb20gcGFycXVldCBmaWxlIC0gQGRlcHJlY2F0ZWQgdXNlIGB3aXRoQ29sdW1uc2AgbWV0aG9kIGluc3RlYWQKICogQHBhcmFtIE9wdGlvbnMgJG9wdGlvbnMgLSBAZGVwcmVjYXRlZCB1c2UgYHdpdGhPcHRpb25zYCBtZXRob2QgaW5zdGVhZAogKiBAcGFyYW0gQnl0ZU9yZGVyICRieXRlX29yZGVyIC0gQGRlcHJlY2F0ZWQgdXNlIGB3aXRoQnl0ZU9yZGVyYCBtZXRob2QgaW5zdGVhZAogKiBAcGFyYW0gbnVsbHxpbnQgJG9mZnNldCAtIEBkZXByZWNhdGVkIHVzZSBgd2l0aE9mZnNldGAgbWV0aG9kIGluc3RlYWQKICov"},{"repository_path":"src\/adapter\/etl-adapter-parquet\/src\/Flow\/ETL\/Adapter\/Parquet\/functions.php","start_line_in_file":69,"slug":"to-parquet","name":"to_parquet","namespace":"Flow\\ETL\\Adapter\\Parquet","parameters":[{"name":"path","type":[{"name":"Path","namespace":"Flow\\Filesystem","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"options","type":[{"name":"Options","namespace":"Flow\\Parquet","is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"null"},{"name":"compressions","type":[{"name":"Compressions","namespace":"Flow\\Parquet\\ParquetFile","is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"Flow\\Parquet\\ParquetFile\\Compressions::..."},{"name":"schema","type":[{"name":"Schema","namespace":"Flow\\ETL","is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"null"},{"name":"engine","type":[{"name":"ParquetEngine","namespace":"Flow\\Parquet","is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"null"},{"name":"filesystem","type":[{"name":"Filesystem","namespace":"Flow\\Filesystem","is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"Flow\\Filesystem\\Local\\NativeLocalFilesystem::..."}],"return_type":[{"name":"ParquetLoader","namespace":"Flow\\ETL\\Adapter\\Parquet","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PARQUET","type":"LOADER"}},{"name":"DocumentationExample","namespace":"Flow\\Documentation\\Attribute","arguments":{"topic":"writing","example":"parquet"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIEBwYXJhbSBQYXRofHN0cmluZyAkcGF0aAogKiBAcGFyYW0gbnVsbHxPcHRpb25zICRvcHRpb25zIC0gQGRlcHJlY2F0ZWQgdXNlIGB3aXRoT3B0aW9uc2AgbWV0aG9kIGluc3RlYWQKICogQHBhcmFtIENvbXByZXNzaW9ucyAkY29tcHJlc3Npb25zIC0gQGRlcHJlY2F0ZWQgdXNlIGB3aXRoQ29tcHJlc3Npb25zYCBtZXRob2QgaW5zdGVhZAogKiBAcGFyYW0gbnVsbHxTY2hlbWEgJHNjaGVtYSAtIEBkZXByZWNhdGVkIHVzZSBgd2l0aFNjaGVtYWAgbWV0aG9kIGluc3RlYWQKICov"},{"repository_path":"src\/adapter\/etl-adapter-parquet\/src\/Flow\/ETL\/Adapter\/Parquet\/functions.php","start_line_in_file":100,"slug":"array-to-generator","name":"array_to_generator","namespace":"Flow\\ETL\\Adapter\\Parquet","parameters":[{"name":"data","type":[{"name":"array","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"Generator","namespace":"","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PARQUET","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIEB0ZW1wbGF0ZSBUCiAqCiAqIEBwYXJhbSBhcnJheTxUPiAkZGF0YQogKgogKiBAcmV0dXJuIFxHZW5lcmF0b3I8VD4KICov"},{"repository_path":"src\/adapter\/etl-adapter-parquet\/src\/Flow\/ETL\/Adapter\/Parquet\/functions.php","start_line_in_file":111,"slug":"empty-generator","name":"empty_generator","namespace":"Flow\\ETL\\Adapter\\Parquet","parameters":[],"return_type":[{"name":"Generator","namespace":"","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PARQUET","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIEBkZXByZWNhdGVkIHVzZSBGbG93XFBhcnF1ZXRcZW1wdHlfZ2VuZXJhdG9yKCkgaW5zdGVhZAogKi8="},{"repository_path":"src\/adapter\/etl-adapter-parquet\/src\/Flow\/ETL\/Adapter\/Parquet\/functions.php","start_line_in_file":117,"slug":"schema-to-parquet","name":"schema_to_parquet","namespace":"Flow\\ETL\\Adapter\\Parquet","parameters":[{"name":"schema","type":[{"name":"Schema","namespace":"Flow\\ETL","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"Schema","namespace":"Flow\\Parquet\\ParquetFile","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PARQUET","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/adapter\/etl-adapter-parquet\/src\/Flow\/ETL\/Adapter\/Parquet\/functions.php","start_line_in_file":123,"slug":"schema-from-parquet","name":"schema_from_parquet","namespace":"Flow\\ETL\\Adapter\\Parquet","parameters":[{"name":"schema","type":[{"name":"Schema","namespace":"Flow\\Parquet\\ParquetFile","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"Schema","namespace":"Flow\\ETL","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PARQUET","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/adapter\/etl-adapter-postgresql\/src\/Flow\/ETL\/Adapter\/PostgreSql\/functions.php","start_line_in_file":39,"slug":"from-pgsql-cursor","name":"from_pgsql_cursor","namespace":"Flow\\ETL\\Adapter\\PostgreSql","parameters":[{"name":"client","type":[{"name":"Client","namespace":"Flow\\PostgreSql\\Client","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"query","type":[{"name":"Sql","namespace":"Flow\\PostgreSql\\QueryBuilder","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"parameters","type":[{"name":"array","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"[]"}],"return_type":[{"name":"PostgreSqlCursorExtractor","namespace":"Flow\\ETL\\Adapter\\PostgreSql","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"POSTGRESQL","type":"EXTRACTOR"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBhIFBvc3RncmVTUUwgY3Vyc29yIGV4dHJhY3RvciB1c2luZyBzZXJ2ZXItc2lkZSBjdXJzb3JzIGZvciBtZW1vcnktZWZmaWNpZW50IGV4dHJhY3Rpb24uCiAqCiAqIFVzZXMgREVDTEFSRSBDVVJTT1IgKyBGRVRDSCB0byBzdHJlYW0gZGF0YSB3aXRob3V0IGxvYWRpbmcgZW50aXJlIHJlc3VsdCBzZXQgaW50byBtZW1vcnkuCiAqIFRoaXMgaXMgdGhlIG9ubHkgd2F5IHRvIGFjaGlldmUgdHJ1ZSBsb3cgbWVtb3J5IGV4dHJhY3Rpb24gd2l0aCBQSFAncyBleHQtcGdzcWwuCiAqCiAqIE5vdGU6IFJlcXVpcmVzIGEgdHJhbnNhY3Rpb24gY29udGV4dCAoYXV0by1zdGFydGVkIGlmIG5vdCBpbiBvbmUpLgogKgogKiBAcGFyYW0gQ2xpZW50ICRjbGllbnQgUG9zdGdyZVNRTCBjbGllbnQKICogQHBhcmFtIFNxbHxzdHJpbmcgJHF1ZXJ5IFNRTCBxdWVyeSB0byBleGVjdXRlICh3cmFwcGVkIGluIERFQ0xBUkUgQ1VSU09SKQogKiBAcGFyYW0gbGlzdDxtaXhlZD4gJHBhcmFtZXRlcnMgVmFsdWVzIGJvdW5kIGJ5IHBvc2l0aW9uIHRvICQxLCAkMiwgLi4uIHBsYWNlaG9sZGVyczsgd3JhcCB3aXRoIHtAc2VlIFxGbG93XFBvc3RncmVTcWxcRFNMXHR5cGVkKCl9IHRvIGZvcmNlIGEgc3BlY2lmaWMgUG9zdGdyZVNRTCB0eXBlCiAqLw=="},{"repository_path":"src\/adapter\/etl-adapter-postgresql\/src\/Flow\/ETL\/Adapter\/PostgreSql\/functions.php","start_line_in_file":55,"slug":"from-pgsql-limit-offset","name":"from_pgsql_limit_offset","namespace":"Flow\\ETL\\Adapter\\PostgreSql","parameters":[{"name":"client","type":[{"name":"Client","namespace":"Flow\\PostgreSql\\Client","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"query","type":[{"name":"Sql","namespace":"Flow\\PostgreSql\\QueryBuilder","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"parameters","type":[{"name":"array","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"[]"}],"return_type":[{"name":"PostgreSqlLimitOffsetExtractor","namespace":"Flow\\ETL\\Adapter\\PostgreSql","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"POSTGRESQL","type":"EXTRACTOR"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBhIFBvc3RncmVTUUwgZXh0cmFjdG9yIHVzaW5nIExJTUlUL09GRlNFVCBwYWdpbmF0aW9uLgogKgogKiBTdWl0YWJsZSBmb3Igc21hbGxlciBkYXRhc2V0cy4gRm9yIGxhcmdlIGRhdGFzZXRzLCBjb25zaWRlciB1c2luZyBrZXlzZXQgcGFnaW5hdGlvbgogKiAoZnJvbV9wZ3NxbF9rZXlfc2V0KSB3aGljaCBpcyBtb3JlIGVmZmljaWVudC4KICoKICogQHBhcmFtIENsaWVudCAkY2xpZW50IFBvc3RncmVTUUwgY2xpZW50CiAqIEBwYXJhbSBTcWx8c3RyaW5nICRxdWVyeSBTUUwgcXVlcnkgdG8gZXhlY3V0ZSAobXVzdCBoYXZlIE9SREVSIEJZIGNsYXVzZSkKICogQHBhcmFtIGxpc3Q8bWl4ZWQ+ICRwYXJhbWV0ZXJzIFZhbHVlcyBib3VuZCBieSBwb3NpdGlvbiB0byAkMSwgJDIsIC4uLiBwbGFjZWhvbGRlcnM7IHdyYXAgd2l0aCB7QHNlZSBcRmxvd1xQb3N0Z3JlU3FsXERTTFx0eXBlZCgpfSB0byBmb3JjZSBhIHNwZWNpZmljIFBvc3RncmVTUUwgdHlwZQogKi8="},{"repository_path":"src\/adapter\/etl-adapter-postgresql\/src\/Flow\/ETL\/Adapter\/PostgreSql\/functions.php","start_line_in_file":75,"slug":"from-pgsql-key-set","name":"from_pgsql_key_set","namespace":"Flow\\ETL\\Adapter\\PostgreSql","parameters":[{"name":"client","type":[{"name":"Client","namespace":"Flow\\PostgreSql\\Client","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"query","type":[{"name":"Sql","namespace":"Flow\\PostgreSql\\QueryBuilder","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"keySet","type":[{"name":"KeySet","namespace":"Flow\\ETL\\Adapter\\PostgreSql\\Pagination","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"parameters","type":[{"name":"array","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"[]"}],"return_type":[{"name":"PostgreSqlKeySetExtractor","namespace":"Flow\\ETL\\Adapter\\PostgreSql","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"POSTGRESQL","type":"EXTRACTOR"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBhIFBvc3RncmVTUUwgZXh0cmFjdG9yIHVzaW5nIGtleXNldCAoY3Vyc29yLWJhc2VkKSBwYWdpbmF0aW9uLgogKgogKiBNb3JlIGVmZmljaWVudCB0aGFuIExJTUlUL09GRlNFVCBmb3IgbGFyZ2UgZGF0YXNldHMgLSB1c2VzIGluZGV4ZWQgV0hFUkUgY29uZGl0aW9ucwogKiBpbnN0ZWFkIG9mIHNraXBwaW5nIHJvd3MuCiAqCiAqIEBwYXJhbSBDbGllbnQgJGNsaWVudCBQb3N0Z3JlU1FMIGNsaWVudAogKiBAcGFyYW0gU3FsfHN0cmluZyAkcXVlcnkgU1FMIHF1ZXJ5IHRvIGV4ZWN1dGUgKG11c3QgaGF2ZSBPUkRFUiBCWSBtYXRjaGluZyBrZXlzZXQgY29sdW1ucykKICogQHBhcmFtIEtleVNldCAka2V5U2V0IENvbHVtbnMgdG8gdXNlIGZvciBrZXlzZXQgcGFnaW5hdGlvbgogKiBAcGFyYW0gbGlzdDxtaXhlZD4gJHBhcmFtZXRlcnMgVmFsdWVzIGJvdW5kIGJ5IHBvc2l0aW9uIHRvICQxLCAkMiwgLi4uIHBsYWNlaG9sZGVyczsgd3JhcCB3aXRoIHtAc2VlIFxGbG93XFBvc3RncmVTcWxcRFNMXHR5cGVkKCl9IHRvIGZvcmNlIGEgc3BlY2lmaWMgUG9zdGdyZVNRTCB0eXBlCiAqLw=="},{"repository_path":"src\/adapter\/etl-adapter-postgresql\/src\/Flow\/ETL\/Adapter\/PostgreSql\/functions.php","start_line_in_file":85,"slug":"pgsql-pagination-key-asc","name":"pgsql_pagination_key_asc","namespace":"Flow\\ETL\\Adapter\\PostgreSql","parameters":[{"name":"column","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"Key","namespace":"Flow\\ETL\\Adapter\\PostgreSql\\Pagination","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"POSTGRESQL","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/adapter\/etl-adapter-postgresql\/src\/Flow\/ETL\/Adapter\/PostgreSql\/functions.php","start_line_in_file":91,"slug":"pgsql-pagination-key-desc","name":"pgsql_pagination_key_desc","namespace":"Flow\\ETL\\Adapter\\PostgreSql","parameters":[{"name":"column","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"Key","namespace":"Flow\\ETL\\Adapter\\PostgreSql\\Pagination","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"POSTGRESQL","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/adapter\/etl-adapter-postgresql\/src\/Flow\/ETL\/Adapter\/PostgreSql\/functions.php","start_line_in_file":97,"slug":"pgsql-pagination-key-set","name":"pgsql_pagination_key_set","namespace":"Flow\\ETL\\Adapter\\PostgreSql","parameters":[{"name":"keys","type":[{"name":"Key","namespace":"Flow\\ETL\\Adapter\\PostgreSql\\Pagination","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":true,"default_value":null}],"return_type":[{"name":"KeySet","namespace":"Flow\\ETL\\Adapter\\PostgreSql\\Pagination","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"POSTGRESQL","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/adapter\/etl-adapter-postgresql\/src\/Flow\/ETL\/Adapter\/PostgreSql\/functions.php","start_line_in_file":103,"slug":"to-pgsql-table","name":"to_pgsql_table","namespace":"Flow\\ETL\\Adapter\\PostgreSql","parameters":[{"name":"client","type":[{"name":"Client","namespace":"Flow\\PostgreSql\\Client","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"table","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"PostgreSqlLoader","namespace":"Flow\\ETL\\Adapter\\PostgreSql","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"POSTGRESQL","type":"LOADER"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/adapter\/etl-adapter-postgresql\/src\/Flow\/ETL\/Adapter\/PostgreSql\/functions.php","start_line_in_file":112,"slug":"to-pgsql-transaction","name":"to_pgsql_transaction","namespace":"Flow\\ETL\\Adapter\\PostgreSql","parameters":[{"name":"client","type":[{"name":"Client","namespace":"Flow\\PostgreSql\\Client","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"sinks","type":[{"name":"Loader","namespace":"Flow\\ETL","is_nullable":false,"is_variadic":false},{"name":"Sink","namespace":"Flow\\ETL","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":true,"default_value":null}],"return_type":[{"name":"Transactional","namespace":"Flow\\ETL\\Sink","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"POSTGRESQL","type":"LOADER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIFdyaXRlIGV2ZXJ5IHNpbmsgd2l0aGluIFBvc3RncmVTUUwgdHJhbnNhY3Rpb25zLgogKi8="},{"repository_path":"src\/adapter\/etl-adapter-postgresql\/src\/Flow\/ETL\/Adapter\/PostgreSql\/functions.php","start_line_in_file":126,"slug":"pgsql-insert-options","name":"pgsql_insert_options","namespace":"Flow\\ETL\\Adapter\\PostgreSql","parameters":[{"name":"skipConflicts","type":[{"name":"bool","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"false"},{"name":"conflictColumns","type":[{"name":"array","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"[]"},{"name":"conflictConstraint","type":[{"name":"string","namespace":null,"is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"null"},{"name":"updateColumns","type":[{"name":"array","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"[]"}],"return_type":[{"name":"InsertOptions","namespace":"Flow\\ETL\\Adapter\\PostgreSql\\LoaderOptions","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"POSTGRESQL","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBpbnNlcnQgb3B0aW9ucyBmb3IgUG9zdGdyZVNRTCBsb2FkZXIuCiAqCiAqIEBwYXJhbSBib29sICRza2lwQ29uZmxpY3RzIElmIHRydWUsIHVzZSBPTiBDT05GTElDVCBETyBOT1RISU5HCiAqIEBwYXJhbSBsaXN0PHN0cmluZz4gJGNvbmZsaWN0Q29sdW1ucyBDb2x1bW4gbmFtZXMgZm9yIE9OIENPTkZMSUNUIChjb2x1bW5zKQogKiBAcGFyYW0gbnVsbHxzdHJpbmcgJGNvbmZsaWN0Q29uc3RyYWludCBDb25zdHJhaW50IG5hbWUgZm9yIE9OIENPTkZMSUNUIE9OIENPTlNUUkFJTlQKICogQHBhcmFtIGxpc3Q8c3RyaW5nPiAkdXBkYXRlQ29sdW1ucyBDb2x1bW5zIHRvIHVwZGF0ZSBvbiBjb25mbGljdCAoZW1wdHkgPSBhbGwgbm9uLWtleSBjb2x1bW5zKQogKi8="},{"repository_path":"src\/adapter\/etl-adapter-postgresql\/src\/Flow\/ETL\/Adapter\/PostgreSql\/functions.php","start_line_in_file":141,"slug":"pgsql-update-options","name":"pgsql_update_options","namespace":"Flow\\ETL\\Adapter\\PostgreSql","parameters":[{"name":"primaryKeys","type":[{"name":"array","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"UpdateOptions","namespace":"Flow\\ETL\\Adapter\\PostgreSql\\LoaderOptions","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"POSTGRESQL","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSB1cGRhdGUgb3B0aW9ucyBmb3IgUG9zdGdyZVNRTCBsb2FkZXIuCiAqCiAqIEBwYXJhbSBsaXN0PHN0cmluZz4gJHByaW1hcnlLZXlzIENvbHVtbnMgdG8gdXNlIGluIFdIRVJFIGNsYXVzZSBmb3IgbWF0Y2hpbmcgcm93cwogKi8="},{"repository_path":"src\/adapter\/etl-adapter-postgresql\/src\/Flow\/ETL\/Adapter\/PostgreSql\/functions.php","start_line_in_file":152,"slug":"pgsql-delete-options","name":"pgsql_delete_options","namespace":"Flow\\ETL\\Adapter\\PostgreSql","parameters":[{"name":"primaryKeys","type":[{"name":"array","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"DeleteOptions","namespace":"Flow\\ETL\\Adapter\\PostgreSql\\LoaderOptions","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"POSTGRESQL","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBkZWxldGUgb3B0aW9ucyBmb3IgUG9zdGdyZVNRTCBsb2FkZXIuCiAqCiAqIEBwYXJhbSBsaXN0PHN0cmluZz4gJHByaW1hcnlLZXlzIENvbHVtbnMgdG8gdXNlIGluIFdIRVJFIGNsYXVzZSBmb3IgbWF0Y2hpbmcgcm93cwogKi8="},{"repository_path":"src\/adapter\/etl-adapter-postgresql\/src\/Flow\/ETL\/Adapter\/PostgreSql\/functions.php","start_line_in_file":164,"slug":"to-pgsql-schema-table","name":"to_pgsql_schema_table","namespace":"Flow\\ETL\\Adapter\\PostgreSql","parameters":[{"name":"schema","type":[{"name":"Schema","namespace":"Flow\\ETL","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"tableName","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"databaseSchema","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"'public'"},{"name":"typesMap","type":[{"name":"EntryTypesMap","namespace":"Flow\\ETL\\Adapter\\PostgreSql","is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"null"},{"name":"options","type":[{"name":"TableOptions","namespace":"Flow\\PostgreSql\\Schema","is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"null"}],"return_type":[{"name":"Table","namespace":"Flow\\PostgreSql\\Schema","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"POSTGRESQL","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENvbnZlcnQgYSBGbG93IFNjaGVtYSBpbnRvIGEgUG9zdGdyZVNRTCB0YWJsZSBkZWZpbml0aW9uLgogKgogKiBAcGFyYW0gc3RyaW5nICRkYXRhYmFzZVNjaGVtYSBQb3N0Z3JlU1FMIHNjaGVtYSAobmFtZXNwYWNlKSB0aGUgdGFibGUgYmVsb25ncyB0bwogKiBAcGFyYW0gP1RhYmxlT3B0aW9ucyAkb3B0aW9ucyB0YWJsZS1sZXZlbCBvcHRpb25zIHRoZSBGbG93IFNjaGVtYSBjYW5ub3QgZXhwcmVzcyAoZS5nLiBVTkxPR0dFRCkKICov"},{"repository_path":"src\/adapter\/etl-adapter-postgresql\/src\/Flow\/ETL\/Adapter\/PostgreSql\/functions.php","start_line_in_file":178,"slug":"pgsql-table-to-flow-schema","name":"pgsql_table_to_flow_schema","namespace":"Flow\\ETL\\Adapter\\PostgreSql","parameters":[{"name":"table","type":[{"name":"Table","namespace":"Flow\\PostgreSql\\Schema","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"typesMap","type":[{"name":"EntryTypesMap","namespace":"Flow\\ETL\\Adapter\\PostgreSql","is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"null"}],"return_type":[{"name":"Schema","namespace":"Flow\\ETL","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"POSTGRESQL","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENvbnZlcnQgYSBQb3N0Z3JlU1FMIHRhYmxlIGRlZmluaXRpb24gaW50byBhIEZsb3cgU2NoZW1hLgogKi8="},{"repository_path":"src\/adapter\/etl-adapter-postgresql\/src\/Flow\/ETL\/Adapter\/PostgreSql\/functions.php","start_line_in_file":184,"slug":"pgsql-schema-sort-by-type","name":"pgsql_schema_sort_by_type","namespace":"Flow\\ETL\\Adapter\\PostgreSql","parameters":[{"name":"typesMap","type":[{"name":"EntryTypesMap","namespace":"Flow\\ETL\\Adapter\\PostgreSql","is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"null"}],"return_type":[{"name":"TypeStrategy","namespace":"Flow\\ETL\\Adapter\\PostgreSql\\Schema\\SortingStrategy","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"POSTGRESQL","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/adapter\/etl-adapter-seal\/src\/Flow\/ETL\/Adapter\/Seal\/functions.php","start_line_in_file":15,"slug":"to-seal-upsert","name":"to_seal_upsert","namespace":"Flow\\ETL\\Adapter\\Seal","parameters":[{"name":"engine","type":[{"name":"EngineInterface","namespace":"CmsIg\\Seal","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"index","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"SealLoader","namespace":"Flow\\ETL\\Adapter\\Seal","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"SEAL","type":"LOADER"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/adapter\/etl-adapter-seal\/src\/Flow\/ETL\/Adapter\/Seal\/functions.php","start_line_in_file":21,"slug":"to-seal-delete","name":"to_seal_delete","namespace":"Flow\\ETL\\Adapter\\Seal","parameters":[{"name":"engine","type":[{"name":"EngineInterface","namespace":"CmsIg\\Seal","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"index","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"SealLoader","namespace":"Flow\\ETL\\Adapter\\Seal","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"SEAL","type":"LOADER"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/adapter\/etl-adapter-seal\/src\/Flow\/ETL\/Adapter\/Seal\/functions.php","start_line_in_file":27,"slug":"to-seal-schema","name":"to_seal_schema","namespace":"Flow\\ETL\\Adapter\\Seal","parameters":[{"name":"schema","type":[{"name":"Schema","namespace":"Flow\\ETL","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"index_name","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"identifier","type":[{"name":"string","namespace":null,"is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"null"}],"return_type":[{"name":"Schema","namespace":"CmsIg\\Seal\\Schema","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"SEAL","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/adapter\/etl-adapter-seal\/src\/Flow\/ETL\/Adapter\/Seal\/functions.php","start_line_in_file":33,"slug":"seal-schema-to-flow","name":"seal_schema_to_flow","namespace":"Flow\\ETL\\Adapter\\Seal","parameters":[{"name":"schema","type":[{"name":"Schema","namespace":"CmsIg\\Seal\\Schema","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"Schema","namespace":"Flow\\ETL","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"SEAL","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/adapter\/etl-adapter-text\/src\/Flow\/ETL\/Adapter\/Text\/functions.php","start_line_in_file":21,"slug":"from-text","name":"from_text","namespace":"Flow\\ETL\\Adapter\\Text","parameters":[{"name":"path","type":[{"name":"Path","namespace":"Flow\\Filesystem","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"filesystem","type":[{"name":"Filesystem","namespace":"Flow\\Filesystem","is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"Flow\\Filesystem\\Local\\NativeLocalFilesystem::..."}],"return_type":[{"name":"TextExtractor","namespace":"Flow\\ETL\\Adapter\\Text","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"TEXT","type":"EXTRACTOR"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIEBwYXJhbSBQYXRofHN0cmluZyAkcGF0aAogKi8="},{"repository_path":"src\/adapter\/etl-adapter-text\/src\/Flow\/ETL\/Adapter\/Text\/functions.php","start_line_in_file":31,"slug":"to-text","name":"to_text","namespace":"Flow\\ETL\\Adapter\\Text","parameters":[{"name":"path","type":[{"name":"Path","namespace":"Flow\\Filesystem","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"new_line_separator","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"'\\n'"},{"name":"filesystem","type":[{"name":"Filesystem","namespace":"Flow\\Filesystem","is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"Flow\\Filesystem\\Local\\NativeLocalFilesystem::..."}],"return_type":[{"name":"TextLoader","namespace":"Flow\\ETL\\Adapter\\Text","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"TEXT","type":"LOADER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIEBwYXJhbSBQYXRofHN0cmluZyAkcGF0aAogKiBAcGFyYW0gc3RyaW5nICRuZXdfbGluZV9zZXBhcmF0b3IgLSBkZWZhdWx0IFBIUF9FT0wgLSBAZGVwcmVjYXRlZCB1c2Ugd2l0aE5ld0xpbmVTZXBhcmF0b3IgbWV0aG9kIGluc3RlYWQKICov"},{"repository_path":"src\/adapter\/etl-adapter-xml\/src\/Flow\/ETL\/Adapter\/XML\/functions.php","start_line_in_file":38,"slug":"from-xml","name":"from_xml","namespace":"Flow\\ETL\\Adapter\\XML","parameters":[{"name":"path","type":[{"name":"Path","namespace":"Flow\\Filesystem","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"xml_node_path","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"''"},{"name":"filesystem","type":[{"name":"Filesystem","namespace":"Flow\\Filesystem","is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"Flow\\Filesystem\\Local\\NativeLocalFilesystem::..."}],"return_type":[{"name":"XMLParserExtractor","namespace":"Flow\\ETL\\Adapter\\XML","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"XML","type":"EXTRACTOR"}},{"name":"DocumentationExample","namespace":"Flow\\Documentation\\Attribute","arguments":{"topic":"reading","example":"xml"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqICBJbiBvcmRlciB0byBpdGVyYXRlIG9ubHkgb3ZlciA8ZWxlbWVudD4gbm9kZXMgdXNlIGBmcm9tX3htbCgkZmlsZSktPndpdGhYTUxOb2RlUGF0aCgncm9vdC9lbGVtZW50cy9lbGVtZW50JylgLgogKgogKiAgPHJvb3Q+CiAqICAgIDxlbGVtZW50cz4KICogICAgICA8ZWxlbWVudD48L2VsZW1lbnQ+CiAqICAgICAgPGVsZW1lbnQ+PC9lbGVtZW50PgogKiAgICA8ZWxlbWVudHM+CiAqICA8L3Jvb3Q+CiAqCiAqICBYTUwgTm9kZSBQYXRoIGRvZXMgbm90IHN1cHBvcnQgYXR0cmlidXRlcyBhbmQgaXQncyBub3QgeHBhdGgsIGl0IGlzIGp1c3QgYSBzZXF1ZW5jZQogKiAgb2Ygbm9kZSBuYW1lcyBzZXBhcmF0ZWQgd2l0aCBzbGFzaC4KICoKICogQHBhcmFtIFBhdGh8c3RyaW5nICRwYXRoCiAqIEBwYXJhbSBzdHJpbmcgJHhtbF9ub2RlX3BhdGggLSBAZGVwcmVjYXRlZCB1c2UgYGZyb21feG1sKCRmaWxlKS0+d2l0aFhNTE5vZGVQYXRoKCR4bWxOb2RlUGF0aClgIG1ldGhvZCBpbnN0ZWFkCiAqLw=="},{"repository_path":"src\/adapter\/etl-adapter-xml\/src\/Flow\/ETL\/Adapter\/XML\/functions.php","start_line_in_file":57,"slug":"to-xml","name":"to_xml","namespace":"Flow\\ETL\\Adapter\\XML","parameters":[{"name":"path","type":[{"name":"Path","namespace":"Flow\\Filesystem","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"root_element_name","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"'rows'"},{"name":"row_element_name","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"'row'"},{"name":"attribute_prefix","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"'_'"},{"name":"date_time_format","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"'Y-m-d\\\\TH:i:s.uP'"},{"name":"xml_writer","type":[{"name":"XMLWriter","namespace":"Flow\\ETL\\Adapter\\XML","is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"Flow\\ETL\\Adapter\\XML\\XMLWriter\\StringXMLWriter::..."},{"name":"filesystem","type":[{"name":"Filesystem","namespace":"Flow\\Filesystem","is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"Flow\\Filesystem\\Local\\NativeLocalFilesystem::..."}],"return_type":[{"name":"XMLLoader","namespace":"Flow\\ETL\\Adapter\\XML\\Loader","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"XML","type":"LOADER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIEBwYXJhbSBQYXRofHN0cmluZyAkcGF0aAogKiBAcGFyYW0gc3RyaW5nICRyb290X2VsZW1lbnRfbmFtZSAtIEBkZXByZWNhdGVkIHVzZSBgd2l0aFJvb3RFbGVtZW50TmFtZSgpYCBtZXRob2QgaW5zdGVhZAogKiBAcGFyYW0gc3RyaW5nICRyb3dfZWxlbWVudF9uYW1lIC0gQGRlcHJlY2F0ZWQgdXNlIGB3aXRoUm93RWxlbWVudE5hbWUoKWAgbWV0aG9kIGluc3RlYWQKICogQHBhcmFtIHN0cmluZyAkYXR0cmlidXRlX3ByZWZpeCAtIEBkZXByZWNhdGVkIHVzZSBgd2l0aEF0dHJpYnV0ZVByZWZpeCgpYCBtZXRob2QgaW5zdGVhZAogKiBAcGFyYW0gc3RyaW5nICRkYXRlX3RpbWVfZm9ybWF0IC0gQGRlcHJlY2F0ZWQgdXNlIGB3aXRoRGF0ZVRpbWVGb3JtYXQoKWAgbWV0aG9kIGluc3RlYWQKICogQHBhcmFtIFhNTFdyaXRlciAkeG1sX3dyaXRlcgogKi8="},{"repository_path":"src\/lib\/filesystem\/src\/Flow\/Filesystem\/DSL\/functions.php","start_line_in_file":32,"slug":"mount","name":"mount","namespace":"Flow\\Filesystem\\DSL","parameters":[{"name":"protocol","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"Mount","namespace":"Flow\\Filesystem","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"FILESYSTEM","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/lib\/filesystem\/src\/Flow\/Filesystem\/DSL\/functions.php","start_line_in_file":38,"slug":"partition","name":"partition","namespace":"Flow\\Filesystem\\DSL","parameters":[{"name":"name","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"value","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"Partition","namespace":"Flow\\Filesystem","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"FILESYSTEM","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/lib\/filesystem\/src\/Flow\/Filesystem\/DSL\/functions.php","start_line_in_file":44,"slug":"partitions","name":"partitions","namespace":"Flow\\Filesystem\\DSL","parameters":[{"name":"partition","type":[{"name":"Partition","namespace":"Flow\\Filesystem","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":true,"default_value":null}],"return_type":[{"name":"Partitions","namespace":"Flow\\Filesystem","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"FILESYSTEM","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/lib\/filesystem\/src\/Flow\/Filesystem\/DSL\/functions.php","start_line_in_file":63,"slug":"path","name":"path","namespace":"Flow\\Filesystem\\DSL","parameters":[{"name":"path","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"options","type":[{"name":"Options","namespace":"Flow\\Filesystem\\Path","is_nullable":false,"is_variadic":false},{"name":"array","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"[]"}],"return_type":[{"name":"Path","namespace":"Flow\\Filesystem","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"FILESYSTEM","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIFBhdGggc3VwcG9ydHMgZ2xvYiBwYXR0ZXJucy4KICogRXhhbXBsZXM6CiAqICAtIHBhdGgoJyouY3N2JykgLSBhbnkgY3N2IGZpbGUgaW4gY3VycmVudCBkaXJlY3RvcnkKICogIC0gcGF0aCgnLyoqIC8gKi5jc3YnKSAtIGFueSBjc3YgZmlsZSBpbiBhbnkgc3ViZGlyZWN0b3J5IChyZW1vdmUgZW1wdHkgc3BhY2VzKQogKiAgLSBwYXRoKCcvZGlyL3BhcnRpdGlvbj0qIC8qLnBhcnF1ZXQnKSAtIGFueSBwYXJxdWV0IGZpbGUgaW4gZ2l2ZW4gcGFydGl0aW9uIGRpcmVjdG9yeS4KICoKICogR2xvYiBwYXR0ZXJuIGlzIGFsc28gc3VwcG9ydGVkIGJ5IHJlbW90ZSBmaWxlc3lzdGVtcyBsaWtlIEF6dXJlCiAqCiAqICAtIHBhdGgoJ2F6dXJlLWJsb2I6Ly9kaXJlY3RvcnkvKi5jc3YnKSAtIGFueSBjc3YgZmlsZSBpbiBnaXZlbiBkaXJlY3RvcnkKICoKICogQHBhcmFtIGFycmF5PHN0cmluZywgbnVsbHxib29sfGZsb2F0fGludHxzdHJpbmd8XFVuaXRFbnVtPnxQYXRoXE9wdGlvbnMgJG9wdGlvbnMKICov"},{"repository_path":"src\/lib\/filesystem\/src\/Flow\/Filesystem\/DSL\/functions.php","start_line_in_file":74,"slug":"path-real","name":"path_real","namespace":"Flow\\Filesystem\\DSL","parameters":[{"name":"path","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"options","type":[{"name":"array","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"[]"}],"return_type":[{"name":"Path","namespace":"Flow\\Filesystem","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"FILESYSTEM","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIFJlc29sdmUgcmVhbCBwYXRoIGZyb20gZ2l2ZW4gcGF0aC4KICoKICogQHBhcmFtIGFycmF5PHN0cmluZywgbnVsbHxib29sfGZsb2F0fGludHxzdHJpbmd8XFVuaXRFbnVtPiAkb3B0aW9ucwogKi8="},{"repository_path":"src\/lib\/filesystem\/src\/Flow\/Filesystem\/DSL\/functions.php","start_line_in_file":80,"slug":"native-local-filesystem","name":"native_local_filesystem","namespace":"Flow\\Filesystem\\DSL","parameters":[{"name":"protocol","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"'file'"}],"return_type":[{"name":"NativeLocalFilesystem","namespace":"Flow\\Filesystem\\Local","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"FILESYSTEM","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/lib\/filesystem\/src\/Flow\/Filesystem\/DSL\/functions.php","start_line_in_file":90,"slug":"stdout-filesystem","name":"stdout_filesystem","namespace":"Flow\\Filesystem\\DSL","parameters":[{"name":"protocol","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"'stdout'"}],"return_type":[{"name":"StdOutFilesystem","namespace":"Flow\\Filesystem\\Local","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"FILESYSTEM","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIFdyaXRlLW9ubHkgZmlsZXN5c3RlbSB1c2VmdWwgd2hlbiB3ZSBqdXN0IHdhbnQgdG8gd3JpdGUgdGhlIG91dHB1dCB0byBzdGRvdXQuCiAqIFRoZSBtYWluIHVzZSBjYXNlIGlzIGZvciBzdHJlYW1pbmcgZGF0YXNldHMgb3ZlciBodHRwLgogKi8="},{"repository_path":"src\/lib\/filesystem\/src\/Flow\/Filesystem\/DSL\/functions.php","start_line_in_file":99,"slug":"memory-filesystem","name":"memory_filesystem","namespace":"Flow\\Filesystem\\DSL","parameters":[{"name":"protocol","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"'memory'"}],"return_type":[{"name":"MemoryFilesystem","namespace":"Flow\\Filesystem\\Local","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"FILESYSTEM","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBhIG5ldyBtZW1vcnkgZmlsZXN5c3RlbSBhbmQgd3JpdGVzIGRhdGEgdG8gaXQgaW4gbWVtb3J5LgogKi8="},{"repository_path":"src\/lib\/filesystem\/src\/Flow\/Filesystem\/DSL\/functions.php","start_line_in_file":110,"slug":"fstab","name":"fstab","namespace":"Flow\\Filesystem\\DSL","parameters":[{"name":"filesystems","type":[{"name":"Filesystem","namespace":"Flow\\Filesystem","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":true,"default_value":null}],"return_type":[{"name":"FilesystemTable","namespace":"Flow\\Filesystem","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"FILESYSTEM","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBhIG5ldyBmaWxlc3lzdGVtIHRhYmxlIHdpdGggZ2l2ZW4gZmlsZXN5c3RlbXMuCiAqIEZpbGVzeXN0ZW1zIGNhbiBiZSBhbHNvIG1vdW50ZWQgbGF0ZXIuCiAqIElmIG5vIGZpbGVzeXN0ZW1zIGFyZSBwcm92aWRlZCwgbG9jYWwgZmlsZXN5c3RlbSBpcyBtb3VudGVkLgogKi8="},{"repository_path":"src\/lib\/filesystem\/src\/Flow\/Filesystem\/DSL\/functions.php","start_line_in_file":126,"slug":"traceable-filesystem","name":"traceable_filesystem","namespace":"Flow\\Filesystem\\DSL","parameters":[{"name":"filesystem","type":[{"name":"Filesystem","namespace":"Flow\\Filesystem","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"telemetryConfig","type":[{"name":"FilesystemTelemetryConfig","namespace":"Flow\\Filesystem\\Telemetry","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"TraceableFilesystem","namespace":"Flow\\Filesystem\\Telemetry","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"FILESYSTEM","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIFdyYXAgYSBmaWxlc3lzdGVtIHdpdGggdGVsZW1ldHJ5IHRyYWNpbmcgc3VwcG9ydC4KICogQWxsIGZpbGVzeXN0ZW0gYW5kIHN0cmVhbSBvcGVyYXRpb25zIHdpbGwgYmUgdHJhY2VkIGFjY29yZGluZyB0byB0aGUgY29uZmlndXJhdGlvbi4KICov"},{"repository_path":"src\/lib\/filesystem\/src\/Flow\/Filesystem\/DSL\/functions.php","start_line_in_file":135,"slug":"filesystem-telemetry-config","name":"filesystem_telemetry_config","namespace":"Flow\\Filesystem\\DSL","parameters":[{"name":"telemetry","type":[{"name":"Telemetry","namespace":"Flow\\Telemetry","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"clock","type":[{"name":"ClockInterface","namespace":"Psr\\Clock","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"options","type":[{"name":"FilesystemTelemetryOptions","namespace":"Flow\\Filesystem\\Telemetry","is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"null"}],"return_type":[{"name":"FilesystemTelemetryConfig","namespace":"Flow\\Filesystem\\Telemetry","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"FILESYSTEM","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBhIHRlbGVtZXRyeSBjb25maWd1cmF0aW9uIGZvciB0aGUgZmlsZXN5c3RlbS4KICov"},{"repository_path":"src\/lib\/filesystem\/src\/Flow\/Filesystem\/DSL\/functions.php","start_line_in_file":150,"slug":"filesystem-telemetry-options","name":"filesystem_telemetry_options","namespace":"Flow\\Filesystem\\DSL","parameters":[{"name":"trace_streams","type":[{"name":"bool","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"true"},{"name":"collect_metrics","type":[{"name":"bool","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"true"}],"return_type":[{"name":"FilesystemTelemetryOptions","namespace":"Flow\\Filesystem\\Telemetry","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"FILESYSTEM","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBvcHRpb25zIGZvciBmaWxlc3lzdGVtIHRlbGVtZXRyeS4KICoKICogQHBhcmFtIGJvb2wgJHRyYWNlX3N0cmVhbXMgQ3JlYXRlIGEgc2luZ2xlIHNwYW4gcGVyIHN0cmVhbSBsaWZlY3ljbGUgKGRlZmF1bHQ6IE9OKQogKiBAcGFyYW0gYm9vbCAkY29sbGVjdF9tZXRyaWNzIENvbGxlY3QgbWV0cmljcyBmb3IgYnl0ZXMvb3BlcmF0aW9uIGNvdW50cyAoZGVmYXVsdDogT04pCiAqLw=="},{"repository_path":"src\/lib\/filesystem\/src\/Flow\/Filesystem\/DSL\/functions.php","start_line_in_file":163,"slug":"file-copy","name":"file_copy","namespace":"Flow\\Filesystem\\DSL","parameters":[{"name":"table","type":[{"name":"FilesystemTable","namespace":"Flow\\Filesystem","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"options","type":[{"name":"OperationOptions","namespace":"Flow\\Filesystem\\Operations","is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"null"}],"return_type":[{"name":"Copy","namespace":"Flow\\Filesystem\\Operations","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"FILESYSTEM","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENvcHkgYSBmaWxlIGZyb20gb25lIHBhdGggdG8gYW5vdGhlciwgYWNyb3NzIGFueSBmaWxlc3lzdGVtcyBtb3VudGVkIGluIHRoZSB0YWJsZS4KICogQWx3YXlzIHN0cmVhbXMgYnl0ZXM7IHNhbWUtZmlsZXN5c3RlbSBjb3BpZXMgZG8gbm90IHVzZSBzZXJ2ZXItc2lkZSBvcHRpbWl6YXRpb25zCiAqIGJlY2F1c2UgYEZpbGVzeXN0ZW06Om12YCBpcyBhIG1vdmUsIG5vdCBhIGNvcHkuCiAqLw=="},{"repository_path":"src\/lib\/filesystem\/src\/Flow\/Filesystem\/DSL\/functions.php","start_line_in_file":174,"slug":"file-move","name":"file_move","namespace":"Flow\\Filesystem\\DSL","parameters":[{"name":"table","type":[{"name":"FilesystemTable","namespace":"Flow\\Filesystem","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"options","type":[{"name":"OperationOptions","namespace":"Flow\\Filesystem\\Operations","is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"null"}],"return_type":[{"name":"Move","namespace":"Flow\\Filesystem\\Operations","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"FILESYSTEM","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIE1vdmUgYSBmaWxlIGZyb20gb25lIHBhdGggdG8gYW5vdGhlciwgYWNyb3NzIGFueSBmaWxlc3lzdGVtcyBtb3VudGVkIGluIHRoZSB0YWJsZS4KICogSW50cmEtZmlsZXN5c3RlbSBtb3ZlcyBkZWxlZ2F0ZSB0byBgRmlsZXN5c3RlbTo6bXZgIGZvciBzZXJ2ZXItc2lkZSBvcHRpbWl6YXRpb25zOwogKiBjcm9zcy1maWxlc3lzdGVtIG1vdmVzIHN0cmVhbS1jb3B5IHRoZW4gcmVtb3ZlIHRoZSBzb3VyY2UgKG5vbi1hdG9taWMpLgogKi8="},{"repository_path":"src\/lib\/filesystem\/src\/Flow\/Filesystem\/DSL\/functions.php","start_line_in_file":185,"slug":"operation-options","name":"operation_options","namespace":"Flow\\Filesystem\\DSL","parameters":[{"name":"chunkSize","type":[{"name":"int","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"8192"}],"return_type":[{"name":"OperationOptions","namespace":"Flow\\Filesystem\\Operations","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"FILESYSTEM","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIE9wdGlvbnMgc2hhcmVkIGJ5IGZpbGVzeXN0ZW0gb3BlcmF0aW9ucy4KICoKICogQHBhcmFtIGludCAkY2h1bmtTaXplIE51bWJlciBvZiBieXRlcyByZWFkL3dyaXR0ZW4gcGVyIGl0ZXJhdGlvbiB3aGVuIHN0cmVhbWluZyBhY3Jvc3MgZmlsZXN5c3RlbXMgKGRlZmF1bHQ6IDgxOTIpCiAqLw=="},{"repository_path":"src\/lib\/types\/src\/Flow\/Types\/DSL\/functions.php","start_line_in_file":71,"slug":"type-structure","name":"type_structure","namespace":"Flow\\Types\\DSL","parameters":[{"name":"elements","type":[{"name":"array","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"[]"}],"return_type":[{"name":"StructureType","namespace":"Flow\\Types\\Type\\Logical","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"TYPES","type":"TYPE"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIEB0ZW1wbGF0ZSBUCiAqCiAqIEBwYXJhbSBhcnJheTxhcnJheS1rZXksIFN0cnVjdHVyZUVsZW1lbnQ8VD58VHlwZTxUPj4gJGVsZW1lbnRzCiAqCiAqIEByZXR1cm4gU3RydWN0dXJlVHlwZTxhcnJheTxhcnJheS1rZXksIFQ+PgogKi8="},{"repository_path":"src\/lib\/types\/src\/Flow\/Types\/DSL\/functions.php","start_line_in_file":86,"slug":"structure-element","name":"structure_element","namespace":"Flow\\Types\\DSL","parameters":[{"name":"name","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false},{"name":"int","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"type","type":[{"name":"Type","namespace":"Flow\\Types","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"optional","type":[{"name":"bool","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"false"}],"return_type":[{"name":"StructureElement","namespace":"Flow\\Types\\Type\\Logical","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"TYPES","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIEB0ZW1wbGF0ZSBUCiAqIEB0ZW1wbGF0ZSBUT3B0aW9uYWwgb2YgYm9vbAogKgogKiBAcGFyYW0gVHlwZTxUPiAkdHlwZQogKiBAcGFyYW0gVE9wdGlvbmFsICRvcHRpb25hbAogKgogKiBAcmV0dXJuIFN0cnVjdHVyZUVsZW1lbnQ8VCwgVE9wdGlvbmFsPgogKi8="},{"repository_path":"src\/lib\/types\/src\/Flow\/Types\/DSL\/functions.php","start_line_in_file":101,"slug":"type-union","name":"type_union","namespace":"Flow\\Types\\DSL","parameters":[{"name":"first","type":[{"name":"Type","namespace":"Flow\\Types","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"second","type":[{"name":"Type","namespace":"Flow\\Types","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"types","type":[{"name":"Type","namespace":"Flow\\Types","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":true,"default_value":null}],"return_type":[{"name":"UnionType","namespace":"Flow\\Types\\Type\\Native","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"TYPES","type":"TYPE"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIEB0ZW1wbGF0ZSBUCiAqCiAqIEBwYXJhbSBUeXBlPFQ+ICRmaXJzdAogKiBAcGFyYW0gVHlwZTxUPiAkc2Vjb25kCiAqIEBwYXJhbSBUeXBlPFQ+IC4uLiR0eXBlcwogKgogKiBAcmV0dXJuIFVuaW9uVHlwZTxULCBUPgogKi8="},{"repository_path":"src\/lib\/types\/src\/Flow\/Types\/DSL\/functions.php","start_line_in_file":122,"slug":"type-intersection","name":"type_intersection","namespace":"Flow\\Types\\DSL","parameters":[{"name":"first","type":[{"name":"Type","namespace":"Flow\\Types","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"second","type":[{"name":"Type","namespace":"Flow\\Types","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"types","type":[{"name":"Type","namespace":"Flow\\Types","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":true,"default_value":null}],"return_type":[{"name":"IntersectionType","namespace":"Flow\\Types\\Type\\Native","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"TYPES","type":"TYPE"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIEB0ZW1wbGF0ZSBUCiAqCiAqIEBwYXJhbSBUeXBlPFQ+ICRmaXJzdAogKiBAcGFyYW0gVHlwZTxUPiAkc2Vjb25kCiAqIEBwYXJhbSBUeXBlPFQ+IC4uLiR0eXBlcwogKgogKiBAcmV0dXJuIEludGVyc2VjdGlvblR5cGU8VCwgVD4KICov"},{"repository_path":"src\/lib\/types\/src\/Flow\/Types\/DSL\/functions.php","start_line_in_file":137,"slug":"type-numeric-string","name":"type_numeric_string","namespace":"Flow\\Types\\DSL","parameters":[],"return_type":[{"name":"NumericStringType","namespace":"Flow\\Types\\Type\\Logical","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"TYPES","type":"TYPE"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIEByZXR1cm4gTnVtZXJpY1N0cmluZ1R5cGU8bnVtZXJpYy1zdHJpbmc+CiAqLw=="},{"repository_path":"src\/lib\/types\/src\/Flow\/Types\/DSL\/functions.php","start_line_in_file":150,"slug":"type-optional","name":"type_optional","namespace":"Flow\\Types\\DSL","parameters":[{"name":"type","type":[{"name":"Type","namespace":"Flow\\Types","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"OptionalType","namespace":"Flow\\Types\\Type\\Logical","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"TYPES","type":"TYPE"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIEB0ZW1wbGF0ZSBUCiAqCiAqIEBwYXJhbSBUeXBlPFQ+ICR0eXBlCiAqCiAqIEByZXR1cm4gT3B0aW9uYWxUeXBlPFQ+CiAqLw=="},{"repository_path":"src\/lib\/types\/src\/Flow\/Types\/DSL\/functions.php","start_line_in_file":161,"slug":"type-from-array","name":"type_from_array","namespace":"Flow\\Types\\DSL","parameters":[{"name":"data","type":[{"name":"array","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"Type","namespace":"Flow\\Types","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"TYPES","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIEBwYXJhbSBhcnJheTxzdHJpbmcsIG1peGVkPiAkZGF0YQogKgogKiBAcmV0dXJuIFR5cGU8bWl4ZWQ+CiAqLw=="},{"repository_path":"src\/lib\/types\/src\/Flow\/Types\/DSL\/functions.php","start_line_in_file":172,"slug":"type-is-nullable","name":"type_is_nullable","namespace":"Flow\\Types\\DSL","parameters":[{"name":"type","type":[{"name":"Type","namespace":"Flow\\Types","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"bool","namespace":null,"is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"TYPES","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIEB0ZW1wbGF0ZSBUCiAqCiAqIEBwYXJhbSBUeXBlPFQ+ICR0eXBlCiAqLw=="},{"repository_path":"src\/lib\/types\/src\/Flow\/Types\/DSL\/functions.php","start_line_in_file":188,"slug":"type-bare","name":"type_bare","namespace":"Flow\\Types\\DSL","parameters":[{"name":"type","type":[{"name":"Type","namespace":"Flow\\Types","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"Type","namespace":"Flow\\Types","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"TYPES","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIFN0cmlwIGV4YWN0bHkgb25lIGxldmVsIG9mIG51bGxhYmlsaXR5LCB3aGljaGV2ZXIgb2YgdGhlIHR3byBzcGVsbGluZ3MgY2FycmllcyBpdAogKiAoT3B0aW9uYWxUeXBlLCBvciBhIFVuaW9uVHlwZSBjb250YWluaW5nIE51bGxUeXBlKS4gVG90YWw6IGEgTk9UIE5VTEwgdHlwZSBpcyByZXR1cm5lZCB1bmNoYW5nZWQuCiAqCiAqIEB0ZW1wbGF0ZSBUCiAqCiAqIEBwYXJhbSBUeXBlPFQ+ICR0eXBlCiAqCiAqIEByZXR1cm4gVHlwZTxtaXhlZD4KICov"},{"repository_path":"src\/lib\/types\/src\/Flow\/Types\/DSL\/functions.php","start_line_in_file":198,"slug":"type-equals","name":"type_equals","namespace":"Flow\\Types\\DSL","parameters":[{"name":"left","type":[{"name":"Type","namespace":"Flow\\Types","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"right","type":[{"name":"Type","namespace":"Flow\\Types","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"bool","namespace":null,"is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"TYPES","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIEBwYXJhbSBUeXBlPG1peGVkPiAkbGVmdAogKiBAcGFyYW0gVHlwZTxtaXhlZD4gJHJpZ2h0CiAqLw=="},{"repository_path":"src\/lib\/types\/src\/Flow\/Types\/DSL\/functions.php","start_line_in_file":211,"slug":"types","name":"types","namespace":"Flow\\Types\\DSL","parameters":[{"name":"types","type":[{"name":"Type","namespace":"Flow\\Types","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":true,"default_value":null}],"return_type":[{"name":"Types","namespace":"Flow\\Types\\Type","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"TYPES","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIEB0ZW1wbGF0ZSBUCiAqCiAqIEBwYXJhbSBUeXBlPFQ+IC4uLiR0eXBlcwogKgogKiBAcmV0dXJuIFR5cGVzPFQ+CiAqLw=="},{"repository_path":"src\/lib\/types\/src\/Flow\/Types\/DSL\/functions.php","start_line_in_file":224,"slug":"type-list","name":"type_list","namespace":"Flow\\Types\\DSL","parameters":[{"name":"element","type":[{"name":"Type","namespace":"Flow\\Types","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"ListType","namespace":"Flow\\Types\\Type\\Logical","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"TYPES","type":"TYPE"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIEB0ZW1wbGF0ZSBUCiAqCiAqIEBwYXJhbSBUeXBlPFQ+ICRlbGVtZW50CiAqCiAqIEByZXR1cm4gTGlzdFR5cGU8bGlzdDxUPj4KICov"},{"repository_path":"src\/lib\/types\/src\/Flow\/Types\/DSL\/functions.php","start_line_in_file":239,"slug":"type-map","name":"type_map","namespace":"Flow\\Types\\DSL","parameters":[{"name":"key_type","type":[{"name":"Type","namespace":"Flow\\Types","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"value_type","type":[{"name":"Type","namespace":"Flow\\Types","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"MapType","namespace":"Flow\\Types\\Type\\Logical","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"TYPES","type":"TYPE"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIEB0ZW1wbGF0ZSBUS2V5IG9mIGFycmF5LWtleQogKiBAdGVtcGxhdGUgVFZhbHVlCiAqCiAqIEBwYXJhbSBUeXBlPFRLZXk+ICRrZXlfdHlwZQogKiBAcGFyYW0gVHlwZTxUVmFsdWU+ICR2YWx1ZV90eXBlCiAqCiAqIEByZXR1cm4gTWFwVHlwZTxhcnJheTxUS2V5LCBUVmFsdWU+PgogKi8="},{"repository_path":"src\/lib\/types\/src\/Flow\/Types\/DSL\/functions.php","start_line_in_file":248,"slug":"type-json","name":"type_json","namespace":"Flow\\Types\\DSL","parameters":[],"return_type":[{"name":"JsonType","namespace":"Flow\\Types\\Type\\Logical","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"TYPES","type":"TYPE"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIEByZXR1cm4gSnNvblR5cGU8SnNvbj4KICov"},{"repository_path":"src\/lib\/types\/src\/Flow\/Types\/DSL\/functions.php","start_line_in_file":257,"slug":"type-datetime","name":"type_datetime","namespace":"Flow\\Types\\DSL","parameters":[{"name":"zone","type":[{"name":"DateTimeZone","namespace":"","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"'UTC'"}],"return_type":[{"name":"DateTimeType","namespace":"Flow\\Types\\Type\\Logical","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"TYPES","type":"TYPE"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIEByZXR1cm4gRGF0ZVRpbWVUeXBlPFxEYXRlVGltZUludGVyZmFjZT4KICov"},{"repository_path":"src\/lib\/types\/src\/Flow\/Types\/DSL\/functions.php","start_line_in_file":266,"slug":"type-date","name":"type_date","namespace":"Flow\\Types\\DSL","parameters":[],"return_type":[{"name":"DateType","namespace":"Flow\\Types\\Type\\Logical","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"TYPES","type":"TYPE"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIEByZXR1cm4gRGF0ZVR5cGU8XERhdGVUaW1lSW50ZXJmYWNlPgogKi8="},{"repository_path":"src\/lib\/types\/src\/Flow\/Types\/DSL\/functions.php","start_line_in_file":275,"slug":"type-time","name":"type_time","namespace":"Flow\\Types\\DSL","parameters":[],"return_type":[{"name":"TimeType","namespace":"Flow\\Types\\Type\\Logical","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"TYPES","type":"TYPE"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIEByZXR1cm4gVGltZVR5cGU8XERhdGVJbnRlcnZhbD4KICov"},{"repository_path":"src\/lib\/types\/src\/Flow\/Types\/DSL\/functions.php","start_line_in_file":284,"slug":"type-time-zone","name":"type_time_zone","namespace":"Flow\\Types\\DSL","parameters":[],"return_type":[{"name":"TimeZoneType","namespace":"Flow\\Types\\Type\\Logical","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"TYPES","type":"TYPE"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIEByZXR1cm4gVGltZVpvbmVUeXBlPFxEYXRlVGltZVpvbmU+CiAqLw=="},{"repository_path":"src\/lib\/types\/src\/Flow\/Types\/DSL\/functions.php","start_line_in_file":293,"slug":"type-xml","name":"type_xml","namespace":"Flow\\Types\\DSL","parameters":[],"return_type":[{"name":"XMLType","namespace":"Flow\\Types\\Type\\Logical","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"TYPES","type":"TYPE"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIEByZXR1cm4gWE1MVHlwZTxcRE9NRG9jdW1lbnR8WE1MRG9jdW1lbnQ+CiAqLw=="},{"repository_path":"src\/lib\/types\/src\/Flow\/Types\/DSL\/functions.php","start_line_in_file":302,"slug":"type-xml-element","name":"type_xml_element","namespace":"Flow\\Types\\DSL","parameters":[],"return_type":[{"name":"XMLElementType","namespace":"Flow\\Types\\Type\\Logical","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"TYPES","type":"TYPE"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIEByZXR1cm4gWE1MRWxlbWVudFR5cGU8XERPTUVsZW1lbnR8RWxlbWVudD4KICov"},{"repository_path":"src\/lib\/types\/src\/Flow\/Types\/DSL\/functions.php","start_line_in_file":311,"slug":"type-uuid","name":"type_uuid","namespace":"Flow\\Types\\DSL","parameters":[],"return_type":[{"name":"UuidType","namespace":"Flow\\Types\\Type\\Logical","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"TYPES","type":"TYPE"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIEByZXR1cm4gVXVpZFR5cGU8VXVpZD4KICov"},{"repository_path":"src\/lib\/types\/src\/Flow\/Types\/DSL\/functions.php","start_line_in_file":320,"slug":"type-integer","name":"type_integer","namespace":"Flow\\Types\\DSL","parameters":[],"return_type":[{"name":"IntegerType","namespace":"Flow\\Types\\Type\\Native","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"TYPES","type":"TYPE"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIEByZXR1cm4gSW50ZWdlclR5cGU8aW50PgogKi8="},{"repository_path":"src\/lib\/types\/src\/Flow\/Types\/DSL\/functions.php","start_line_in_file":329,"slug":"type-string","name":"type_string","namespace":"Flow\\Types\\DSL","parameters":[],"return_type":[{"name":"StringType","namespace":"Flow\\Types\\Type\\Native","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"TYPES","type":"TYPE"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIEByZXR1cm4gU3RyaW5nVHlwZTxzdHJpbmc+CiAqLw=="},{"repository_path":"src\/lib\/types\/src\/Flow\/Types\/DSL\/functions.php","start_line_in_file":338,"slug":"type-float","name":"type_float","namespace":"Flow\\Types\\DSL","parameters":[],"return_type":[{"name":"FloatType","namespace":"Flow\\Types\\Type\\Native","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"TYPES","type":"TYPE"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIEByZXR1cm4gRmxvYXRUeXBlPGZsb2F0PgogKi8="},{"repository_path":"src\/lib\/types\/src\/Flow\/Types\/DSL\/functions.php","start_line_in_file":347,"slug":"type-boolean","name":"type_boolean","namespace":"Flow\\Types\\DSL","parameters":[],"return_type":[{"name":"BooleanType","namespace":"Flow\\Types\\Type\\Native","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"TYPES","type":"TYPE"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIEByZXR1cm4gQm9vbGVhblR5cGU8Ym9vbD4KICov"},{"repository_path":"src\/lib\/types\/src\/Flow\/Types\/DSL\/functions.php","start_line_in_file":360,"slug":"type-instance-of","name":"type_instance_of","namespace":"Flow\\Types\\DSL","parameters":[{"name":"class","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"InstanceOfType","namespace":"Flow\\Types\\Type\\Logical","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"TYPES","type":"TYPE"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIEB0ZW1wbGF0ZSBUIG9mIG9iamVjdAogKgogKiBAcGFyYW0gY2xhc3Mtc3RyaW5nPFQ+ICRjbGFzcwogKgogKiBAcmV0dXJuIEluc3RhbmNlT2ZUeXBlPFQ+CiAqLw=="},{"repository_path":"src\/lib\/types\/src\/Flow\/Types\/DSL\/functions.php","start_line_in_file":369,"slug":"type-object","name":"type_object","namespace":"Flow\\Types\\DSL","parameters":[],"return_type":[{"name":"ObjectType","namespace":"Flow\\Types\\Type\\Native","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"TYPES","type":"TYPE"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIEByZXR1cm4gT2JqZWN0VHlwZTxvYmplY3Q+CiAqLw=="},{"repository_path":"src\/lib\/types\/src\/Flow\/Types\/DSL\/functions.php","start_line_in_file":378,"slug":"type-scalar","name":"type_scalar","namespace":"Flow\\Types\\DSL","parameters":[],"return_type":[{"name":"ScalarType","namespace":"Flow\\Types\\Type\\Logical","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"TYPES","type":"TYPE"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIEByZXR1cm4gU2NhbGFyVHlwZTxib29sfGZsb2F0fGludHxzdHJpbmc+CiAqLw=="},{"repository_path":"src\/lib\/types\/src\/Flow\/Types\/DSL\/functions.php","start_line_in_file":387,"slug":"type-resource","name":"type_resource","namespace":"Flow\\Types\\DSL","parameters":[],"return_type":[{"name":"ResourceType","namespace":"Flow\\Types\\Type\\Native","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"TYPES","type":"TYPE"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIEByZXR1cm4gUmVzb3VyY2VUeXBlPHJlc291cmNlPgogKi8="},{"repository_path":"src\/lib\/types\/src\/Flow\/Types\/DSL\/functions.php","start_line_in_file":396,"slug":"type-array","name":"type_array","namespace":"Flow\\Types\\DSL","parameters":[],"return_type":[{"name":"ArrayType","namespace":"Flow\\Types\\Type\\Native","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"TYPES","type":"TYPE"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIEByZXR1cm4gQXJyYXlUeXBlPGFycmF5PG1peGVkPj4KICov"},{"repository_path":"src\/lib\/types\/src\/Flow\/Types\/DSL\/functions.php","start_line_in_file":405,"slug":"type-callable","name":"type_callable","namespace":"Flow\\Types\\DSL","parameters":[],"return_type":[{"name":"CallableType","namespace":"Flow\\Types\\Type\\Native","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"TYPES","type":"TYPE"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIEByZXR1cm4gQ2FsbGFibGVUeXBlPGNhbGxhYmxlPgogKi8="},{"repository_path":"src\/lib\/types\/src\/Flow\/Types\/DSL\/functions.php","start_line_in_file":414,"slug":"type-null","name":"type_null","namespace":"Flow\\Types\\DSL","parameters":[],"return_type":[{"name":"NullType","namespace":"Flow\\Types\\Type\\Native","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"TYPES","type":"TYPE"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIEByZXR1cm4gTnVsbFR5cGU8bnVsbD4KICov"},{"repository_path":"src\/lib\/types\/src\/Flow\/Types\/DSL\/functions.php","start_line_in_file":423,"slug":"type-mixed","name":"type_mixed","namespace":"Flow\\Types\\DSL","parameters":[],"return_type":[{"name":"MixedType","namespace":"Flow\\Types\\Type\\Native","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"TYPES","type":"TYPE"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIEByZXR1cm4gTWl4ZWRUeXBlPG1peGVkPgogKi8="},{"repository_path":"src\/lib\/types\/src\/Flow\/Types\/DSL\/functions.php","start_line_in_file":432,"slug":"type-positive-integer","name":"type_positive_integer","namespace":"Flow\\Types\\DSL","parameters":[],"return_type":[{"name":"PositiveIntegerType","namespace":"Flow\\Types\\Type\\Logical","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"TYPES","type":"TYPE"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIEByZXR1cm4gUG9zaXRpdmVJbnRlZ2VyVHlwZTxpbnQ8MCwgbWF4Pj4KICov"},{"repository_path":"src\/lib\/types\/src\/Flow\/Types\/DSL\/functions.php","start_line_in_file":441,"slug":"type-non-empty-string","name":"type_non_empty_string","namespace":"Flow\\Types\\DSL","parameters":[],"return_type":[{"name":"NonEmptyStringType","namespace":"Flow\\Types\\Type\\Logical","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"TYPES","type":"TYPE"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIEByZXR1cm4gTm9uRW1wdHlTdHJpbmdUeXBlPG5vbi1lbXB0eS1zdHJpbmc+CiAqLw=="},{"repository_path":"src\/lib\/types\/src\/Flow\/Types\/DSL\/functions.php","start_line_in_file":450,"slug":"type-empty-array","name":"type_empty_array","namespace":"Flow\\Types\\DSL","parameters":[],"return_type":[{"name":"EmptyArrayType","namespace":"Flow\\Types\\Type\\Native","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"TYPES","type":"TYPE"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIEByZXR1cm4gRW1wdHlBcnJheVR5cGU8YXJyYXl7fT4KICov"},{"repository_path":"src\/lib\/types\/src\/Flow\/Types\/DSL\/functions.php","start_line_in_file":463,"slug":"type-enum","name":"type_enum","namespace":"Flow\\Types\\DSL","parameters":[{"name":"class","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"EnumType","namespace":"Flow\\Types\\Type\\Native","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"TYPES","type":"TYPE"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIEB0ZW1wbGF0ZSBUIG9mIFVuaXRFbnVtCiAqCiAqIEBwYXJhbSBjbGFzcy1zdHJpbmc8VD4gJGNsYXNzCiAqCiAqIEByZXR1cm4gRW51bVR5cGU8VD4KICov"},{"repository_path":"src\/lib\/types\/src\/Flow\/Types\/DSL\/functions.php","start_line_in_file":476,"slug":"type-literal","name":"type_literal","namespace":"Flow\\Types\\DSL","parameters":[{"name":"value","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false},{"name":"int","namespace":null,"is_nullable":false,"is_variadic":false},{"name":"float","namespace":null,"is_nullable":false,"is_variadic":false},{"name":"bool","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"LiteralType","namespace":"Flow\\Types\\Type\\Logical","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"TYPES","type":"TYPE"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIEB0ZW1wbGF0ZSBUIG9mIGJvb2x8ZmxvYXR8aW50fHN0cmluZwogKgogKiBAcGFyYW0gVCAkdmFsdWUKICoKICogQHJldHVybiBMaXRlcmFsVHlwZTxUPgogKi8="},{"repository_path":"src\/lib\/types\/src\/Flow\/Types\/DSL\/functions.php","start_line_in_file":485,"slug":"type-html","name":"type_html","namespace":"Flow\\Types\\DSL","parameters":[],"return_type":[{"name":"HTMLType","namespace":"Flow\\Types\\Type\\Logical","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"TYPES","type":"TYPE"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIEByZXR1cm4gSFRNTFR5cGU8SFRNTERvY3VtZW50PgogKi8="},{"repository_path":"src\/lib\/types\/src\/Flow\/Types\/DSL\/functions.php","start_line_in_file":494,"slug":"type-html-element","name":"type_html_element","namespace":"Flow\\Types\\DSL","parameters":[],"return_type":[{"name":"HTMLElementType","namespace":"Flow\\Types\\Type\\Logical","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"TYPES","type":"TYPE"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIEByZXR1cm4gSFRNTEVsZW1lbnRUeXBlPEhUTUxFbGVtZW50PgogKi8="},{"repository_path":"src\/lib\/types\/src\/Flow\/Types\/DSL\/functions.php","start_line_in_file":506,"slug":"type-is","name":"type_is","namespace":"Flow\\Types\\DSL","parameters":[{"name":"type","type":[{"name":"Type","namespace":"Flow\\Types","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"typeClass","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"bool","namespace":null,"is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"TYPES","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIEB0ZW1wbGF0ZSBUCiAqCiAqIEBwYXJhbSBUeXBlPFQ+ICR0eXBlCiAqIEBwYXJhbSBjbGFzcy1zdHJpbmc8VHlwZTxtaXhlZD4+ICR0eXBlQ2xhc3MKICov"},{"repository_path":"src\/lib\/types\/src\/Flow\/Types\/DSL\/functions.php","start_line_in_file":519,"slug":"type-is-any","name":"type_is_any","namespace":"Flow\\Types\\DSL","parameters":[{"name":"type","type":[{"name":"Type","namespace":"Flow\\Types","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"typeClass","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"typeClasses","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":true,"default_value":null}],"return_type":[{"name":"bool","namespace":null,"is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"TYPES","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIEB0ZW1wbGF0ZSBUCiAqCiAqIEBwYXJhbSBUeXBlPFQ+ICR0eXBlCiAqIEBwYXJhbSBjbGFzcy1zdHJpbmc8VHlwZTxtaXhlZD4+ICR0eXBlQ2xhc3MKICogQHBhcmFtIGNsYXNzLXN0cmluZzxUeXBlPG1peGVkPj4gLi4uJHR5cGVDbGFzc2VzCiAqLw=="},{"repository_path":"src\/lib\/types\/src\/Flow\/Types\/DSL\/functions.php","start_line_in_file":528,"slug":"get-type","name":"get_type","namespace":"Flow\\Types\\DSL","parameters":[{"name":"value","type":[{"name":"mixed","namespace":null,"is_nullable":true,"is_variadic":false}],"has_default_value":false,"is_nullable":true,"is_variadic":false,"default_value":null}],"return_type":[{"name":"Type","namespace":"Flow\\Types","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"TYPES","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIEByZXR1cm4gVHlwZTxtaXhlZD4KICov"},{"repository_path":"src\/lib\/types\/src\/Flow\/Types\/DSL\/functions.php","start_line_in_file":541,"slug":"type-class-string","name":"type_class_string","namespace":"Flow\\Types\\DSL","parameters":[{"name":"class","type":[{"name":"string","namespace":null,"is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"null"}],"return_type":[{"name":"ClassStringType","namespace":"Flow\\Types\\Type\\Logical","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"TYPES","type":"TYPE"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIEB0ZW1wbGF0ZSBUIG9mIG9iamVjdAogKgogKiBAcGFyYW0gbnVsbHxjbGFzcy1zdHJpbmc8VD4gJGNsYXNzCiAqCiAqIEByZXR1cm4gKCRjbGFzcyBpcyBudWxsID8gQ2xhc3NTdHJpbmdUeXBlPGNsYXNzLXN0cmluZz4gOiBDbGFzc1N0cmluZ1R5cGU8Y2xhc3Mtc3RyaW5nPFQ+PikKICov"},{"repository_path":"src\/lib\/types\/src\/Flow\/Types\/DSL\/functions.php","start_line_in_file":547,"slug":"dom-element-to-string","name":"dom_element_to_string","namespace":"Flow\\Types\\DSL","parameters":[{"name":"element","type":[{"name":"DOMElement","namespace":"","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"format_output","type":[{"name":"bool","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"false"},{"name":"preserver_white_space","type":[{"name":"bool","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"false"}],"return_type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false},{"name":"false","namespace":null,"is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"TYPES","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/schema.php","start_line_in_file":134,"slug":"column","name":"column","namespace":"Flow\\PostgreSql\\DSL","parameters":[{"name":"name","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"type","type":[{"name":"ColumnType","namespace":"Flow\\PostgreSql\\QueryBuilder\\Schema","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"ColumnDefinition","namespace":"Flow\\PostgreSql\\QueryBuilder\\Schema","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBhIGNvbHVtbiBkZWZpbml0aW9uIGZvciBDUkVBVEUgVEFCTEUuCiAqCiAqIEBwYXJhbSBzdHJpbmcgJG5hbWUgQ29sdW1uIG5hbWUKICogQHBhcmFtIENvbHVtblR5cGUgJHR5cGUgQ29sdW1uIGRhdGEgdHlwZQogKi8="},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/schema.php","start_line_in_file":143,"slug":"catalog","name":"catalog","namespace":"Flow\\PostgreSql\\DSL","parameters":[{"name":"schemas","type":[{"name":"array","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"Catalog","namespace":"Flow\\PostgreSql\\Schema","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIEBwYXJhbSBsaXN0PFNjaGVtYT4gJHNjaGVtYXMKICov"},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/schema.php","start_line_in_file":154,"slug":"primary-key","name":"primary_key","namespace":"Flow\\PostgreSql\\DSL","parameters":[{"name":"columns","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":true,"default_value":null}],"return_type":[{"name":"PrimaryKeyConstraint","namespace":"Flow\\PostgreSql\\QueryBuilder\\Schema\\Constraint","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBhIFBSSU1BUlkgS0VZIGNvbnN0cmFpbnQuCiAqCiAqIEBwYXJhbSBzdHJpbmcgLi4uJGNvbHVtbnMgQ29sdW1ucyB0aGF0IGZvcm0gdGhlIHByaW1hcnkga2V5CiAqLw=="},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/schema.php","start_line_in_file":165,"slug":"unique-constraint","name":"unique_constraint","namespace":"Flow\\PostgreSql\\DSL","parameters":[{"name":"columns","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":true,"default_value":null}],"return_type":[{"name":"UniqueConstraint","namespace":"Flow\\PostgreSql\\QueryBuilder\\Schema\\Constraint","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBhIFVOSVFVRSBjb25zdHJhaW50LgogKgogKiBAcGFyYW0gc3RyaW5nIC4uLiRjb2x1bW5zIENvbHVtbnMgdGhhdCBtdXN0IGJlIHVuaXF1ZSB0b2dldGhlcgogKi8="},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/schema.php","start_line_in_file":178,"slug":"foreign-key","name":"foreign_key","namespace":"Flow\\PostgreSql\\DSL","parameters":[{"name":"columns","type":[{"name":"array","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"referenceTable","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"referenceColumns","type":[{"name":"array","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"[]"}],"return_type":[{"name":"ForeignKeyConstraint","namespace":"Flow\\PostgreSql\\QueryBuilder\\Schema\\Constraint","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBhIEZPUkVJR04gS0VZIGNvbnN0cmFpbnQuCiAqCiAqIEBwYXJhbSBsaXN0PHN0cmluZz4gJGNvbHVtbnMgTG9jYWwgY29sdW1ucwogKiBAcGFyYW0gc3RyaW5nICRyZWZlcmVuY2VUYWJsZSBSZWZlcmVuY2VkIHRhYmxlCiAqIEBwYXJhbSBsaXN0PHN0cmluZz4gJHJlZmVyZW5jZUNvbHVtbnMgUmVmZXJlbmNlZCBjb2x1bW5zIChkZWZhdWx0cyB0byBzYW1lIGFzICRjb2x1bW5zIGlmIGVtcHR5KQogKi8="},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/schema.php","start_line_in_file":187,"slug":"check-constraint","name":"check_constraint","namespace":"Flow\\PostgreSql\\DSL","parameters":[{"name":"condition","type":[{"name":"Condition","namespace":"Flow\\PostgreSql\\QueryBuilder\\Condition","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"CheckConstraint","namespace":"Flow\\PostgreSql\\QueryBuilder\\Schema\\Constraint","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBhIENIRUNLIGNvbnN0cmFpbnQuCiAqLw=="},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/schema.php","start_line_in_file":218,"slug":"create","name":"create","namespace":"Flow\\PostgreSql\\DSL","parameters":[],"return_type":[{"name":"CreateFactory","namespace":"Flow\\PostgreSql\\QueryBuilder\\Factory","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"SCHEMA"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBhIGZhY3RvcnkgZm9yIGJ1aWxkaW5nIENSRUFURSBzdGF0ZW1lbnRzLgogKgogKiBQcm92aWRlcyBhIHVuaWZpZWQgZW50cnkgcG9pbnQgZm9yIGFsbCBDUkVBVEUgb3BlcmF0aW9uczoKICogLSBjcmVhdGUoKS0+dGFibGUoKSAtIENSRUFURSBUQUJMRQogKiAtIGNyZWF0ZSgpLT50YWJsZUFzKCkgLSBDUkVBVEUgVEFCTEUgQVMKICogLSBjcmVhdGUoKS0+aW5kZXgoKSAtIENSRUFURSBJTkRFWAogKiAtIGNyZWF0ZSgpLT52aWV3KCkgLSBDUkVBVEUgVklFVwogKiAtIGNyZWF0ZSgpLT5tYXRlcmlhbGl6ZWRWaWV3KCkgLSBDUkVBVEUgTUFURVJJQUxJWkVEIFZJRVcKICogLSBjcmVhdGUoKS0+c2VxdWVuY2UoKSAtIENSRUFURSBTRVFVRU5DRQogKiAtIGNyZWF0ZSgpLT5zY2hlbWEoKSAtIENSRUFURSBTQ0hFTUEKICogLSBjcmVhdGUoKS0+cm9sZSgpIC0gQ1JFQVRFIFJPTEUKICogLSBjcmVhdGUoKS0+ZnVuY3Rpb24oKSAtIENSRUFURSBGVU5DVElPTgogKiAtIGNyZWF0ZSgpLT5wcm9jZWR1cmUoKSAtIENSRUFURSBQUk9DRURVUkUKICogLSBjcmVhdGUoKS0+dHJpZ2dlcigpIC0gQ1JFQVRFIFRSSUdHRVIKICogLSBjcmVhdGUoKS0+cnVsZSgpIC0gQ1JFQVRFIFJVTEUKICogLSBjcmVhdGUoKS0+ZXh0ZW5zaW9uKCkgLSBDUkVBVEUgRVhURU5TSU9OCiAqIC0gY3JlYXRlKCktPmNvbXBvc2l0ZVR5cGUoKSAtIENSRUFURSBUWVBFIChjb21wb3NpdGUpCiAqIC0gY3JlYXRlKCktPmVudW1UeXBlKCkgLSBDUkVBVEUgVFlQRSAoZW51bSkKICogLSBjcmVhdGUoKS0+cmFuZ2VUeXBlKCkgLSBDUkVBVEUgVFlQRSAocmFuZ2UpCiAqIC0gY3JlYXRlKCktPmRvbWFpbigpIC0gQ1JFQVRFIERPTUFJTgogKgogKiBFeGFtcGxlOiBjcmVhdGUoKS0+dGFibGUoJ3VzZXJzJyktPmNvbHVtbnMoY29sX2RlZignaWQnLCBjb2x1bW5fdHlwZV9zZXJpYWwoKSkpCiAqIEV4YW1wbGU6IGNyZWF0ZSgpLT5pbmRleCgnaWR4X2VtYWlsJyktPm9uKCd1c2VycycpLT5jb2x1bW5zKCdlbWFpbCcpCiAqLw=="},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/schema.php","start_line_in_file":247,"slug":"drop","name":"drop","namespace":"Flow\\PostgreSql\\DSL","parameters":[],"return_type":[{"name":"DropFactory","namespace":"Flow\\PostgreSql\\QueryBuilder\\Factory","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"SCHEMA"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBhIGZhY3RvcnkgZm9yIGJ1aWxkaW5nIERST1Agc3RhdGVtZW50cy4KICoKICogUHJvdmlkZXMgYSB1bmlmaWVkIGVudHJ5IHBvaW50IGZvciBhbGwgRFJPUCBvcGVyYXRpb25zOgogKiAtIGRyb3AoKS0+dGFibGUoKSAtIERST1AgVEFCTEUKICogLSBkcm9wKCktPmluZGV4KCkgLSBEUk9QIElOREVYCiAqIC0gZHJvcCgpLT52aWV3KCkgLSBEUk9QIFZJRVcKICogLSBkcm9wKCktPm1hdGVyaWFsaXplZFZpZXcoKSAtIERST1AgTUFURVJJQUxJWkVEIFZJRVcKICogLSBkcm9wKCktPnNlcXVlbmNlKCkgLSBEUk9QIFNFUVVFTkNFCiAqIC0gZHJvcCgpLT5zY2hlbWEoKSAtIERST1AgU0NIRU1BCiAqIC0gZHJvcCgpLT5yb2xlKCkgLSBEUk9QIFJPTEUKICogLSBkcm9wKCktPmZ1bmN0aW9uKCkgLSBEUk9QIEZVTkNUSU9OCiAqIC0gZHJvcCgpLT5wcm9jZWR1cmUoKSAtIERST1AgUFJPQ0VEVVJFCiAqIC0gZHJvcCgpLT50cmlnZ2VyKCkgLSBEUk9QIFRSSUdHRVIKICogLSBkcm9wKCktPnJ1bGUoKSAtIERST1AgUlVMRQogKiAtIGRyb3AoKS0+ZXh0ZW5zaW9uKCkgLSBEUk9QIEVYVEVOU0lPTgogKiAtIGRyb3AoKS0+dHlwZSgpIC0gRFJPUCBUWVBFCiAqIC0gZHJvcCgpLT5kb21haW4oKSAtIERST1AgRE9NQUlOCiAqIC0gZHJvcCgpLT5vd25lZCgpIC0gRFJPUCBPV05FRAogKgogKiBFeGFtcGxlOiBkcm9wKCktPnRhYmxlKCd1c2VycycsICdvcmRlcnMnKS0+aWZFeGlzdHMoKS0+Y2FzY2FkZSgpCiAqIEV4YW1wbGU6IGRyb3AoKS0+aW5kZXgoJ2lkeF9lbWFpbCcpLT5pZkV4aXN0cygpCiAqLw=="},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/schema.php","start_line_in_file":281,"slug":"alter","name":"alter","namespace":"Flow\\PostgreSql\\DSL","parameters":[],"return_type":[{"name":"AlterFactory","namespace":"Flow\\PostgreSql\\QueryBuilder\\Factory","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"SCHEMA"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBhIGZhY3RvcnkgZm9yIGJ1aWxkaW5nIEFMVEVSIHN0YXRlbWVudHMuCiAqCiAqIFByb3ZpZGVzIGEgdW5pZmllZCBlbnRyeSBwb2ludCBmb3IgYWxsIEFMVEVSIG9wZXJhdGlvbnM6CiAqIC0gYWx0ZXIoKS0+dGFibGUoKSAtIEFMVEVSIFRBQkxFCiAqIC0gYWx0ZXIoKS0+aW5kZXgoKSAtIEFMVEVSIElOREVYCiAqIC0gYWx0ZXIoKS0+dmlldygpIC0gQUxURVIgVklFVwogKiAtIGFsdGVyKCktPm1hdGVyaWFsaXplZFZpZXcoKSAtIEFMVEVSIE1BVEVSSUFMSVpFRCBWSUVXCiAqIC0gYWx0ZXIoKS0+c2VxdWVuY2UoKSAtIEFMVEVSIFNFUVVFTkNFCiAqIC0gYWx0ZXIoKS0+c2NoZW1hKCkgLSBBTFRFUiBTQ0hFTUEKICogLSBhbHRlcigpLT5yb2xlKCkgLSBBTFRFUiBST0xFCiAqIC0gYWx0ZXIoKS0+ZnVuY3Rpb24oKSAtIEFMVEVSIEZVTkNUSU9OCiAqIC0gYWx0ZXIoKS0+cHJvY2VkdXJlKCkgLSBBTFRFUiBQUk9DRURVUkUKICogLSBhbHRlcigpLT50cmlnZ2VyKCkgLSBBTFRFUiBUUklHR0VSCiAqIC0gYWx0ZXIoKS0+ZXh0ZW5zaW9uKCkgLSBBTFRFUiBFWFRFTlNJT04KICogLSBhbHRlcigpLT5lbnVtVHlwZSgpIC0gQUxURVIgVFlQRSAoZW51bSkKICogLSBhbHRlcigpLT5kb21haW4oKSAtIEFMVEVSIERPTUFJTgogKgogKiBSZW5hbWUgb3BlcmF0aW9ucyBhcmUgYWxzbyB1bmRlciBhbHRlcigpOgogKiAtIGFsdGVyKCktPmluZGV4KCdvbGQnKS0+cmVuYW1lVG8oJ25ldycpCiAqIC0gYWx0ZXIoKS0+dmlldygnb2xkJyktPnJlbmFtZVRvKCduZXcnKQogKiAtIGFsdGVyKCktPnNjaGVtYSgnb2xkJyktPnJlbmFtZVRvKCduZXcnKQogKiAtIGFsdGVyKCktPnJvbGUoJ29sZCcpLT5yZW5hbWVUbygnbmV3JykKICogLSBhbHRlcigpLT50cmlnZ2VyKCdvbGQnKS0+b24oJ3RhYmxlJyktPnJlbmFtZVRvKCduZXcnKQogKgogKiBFeGFtcGxlOiBhbHRlcigpLT50YWJsZSgndXNlcnMnKS0+YWRkQ29sdW1uKGNvbF9kZWYoJ2VtYWlsJywgY29sdW1uX3R5cGVfdGV4dCgpKSkKICogRXhhbXBsZTogYWx0ZXIoKS0+c2VxdWVuY2UoJ3VzZXJfaWRfc2VxJyktPnJlc3RhcnQoMTAwMCkKICov"},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/schema.php","start_line_in_file":292,"slug":"truncate-table","name":"truncate_table","namespace":"Flow\\PostgreSql\\DSL","parameters":[{"name":"tables","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":true,"default_value":null}],"return_type":[{"name":"TruncateFinalStep","namespace":"Flow\\PostgreSql\\QueryBuilder\\Schema\\Truncate","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBhIFRSVU5DQVRFIFRBQkxFIGJ1aWxkZXIuCiAqCiAqIEBwYXJhbSBzdHJpbmcgLi4uJHRhYmxlcyBUYWJsZSBuYW1lcyB0byB0cnVuY2F0ZQogKi8="},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/schema.php","start_line_in_file":310,"slug":"refresh-materialized-view","name":"refresh_materialized_view","namespace":"Flow\\PostgreSql\\DSL","parameters":[{"name":"name","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"schema","type":[{"name":"string","namespace":null,"is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"null"}],"return_type":[{"name":"RefreshMatViewOptionsStep","namespace":"Flow\\PostgreSql\\QueryBuilder\\Schema\\View\\RefreshMaterializedView","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"SCHEMA"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBhIFJFRlJFU0ggTUFURVJJQUxJWkVEIFZJRVcgYnVpbGRlci4KICoKICogRXhhbXBsZTogcmVmcmVzaF9tYXRlcmlhbGl6ZWRfdmlldygndXNlcl9zdGF0cycpCiAqIFByb2R1Y2VzOiBSRUZSRVNIIE1BVEVSSUFMSVpFRCBWSUVXIHVzZXJfc3RhdHMKICoKICogRXhhbXBsZTogcmVmcmVzaF9tYXRlcmlhbGl6ZWRfdmlldygndXNlcl9zdGF0cycpLT5jb25jdXJyZW50bHkoKS0+d2l0aERhdGEoKQogKiBQcm9kdWNlczogUkVGUkVTSCBNQVRFUklBTElaRUQgVklFVyBDT05DVVJSRU5UTFkgdXNlcl9zdGF0cyBXSVRIIERBVEEKICoKICogQHBhcmFtIHN0cmluZyAkbmFtZSBWaWV3IG5hbWUgKG1heSBpbmNsdWRlIHNjaGVtYSBhcyAic2NoZW1hLnZpZXciKQogKiBAcGFyYW0gbnVsbHxzdHJpbmcgJHNjaGVtYSBTY2hlbWEgbmFtZSAob3B0aW9uYWwsIG92ZXJyaWRlcyBwYXJzZWQgc2NoZW1hKQogKi8="},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/schema.php","start_line_in_file":319,"slug":"ref-action-cascade","name":"ref_action_cascade","namespace":"Flow\\PostgreSql\\DSL","parameters":[],"return_type":[{"name":"ReferentialAction","namespace":"Flow\\PostgreSql\\QueryBuilder\\Schema","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIEdldCBhIENBU0NBREUgcmVmZXJlbnRpYWwgYWN0aW9uLgogKi8="},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/schema.php","start_line_in_file":328,"slug":"ref-action-restrict","name":"ref_action_restrict","namespace":"Flow\\PostgreSql\\DSL","parameters":[],"return_type":[{"name":"ReferentialAction","namespace":"Flow\\PostgreSql\\QueryBuilder\\Schema","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIEdldCBhIFJFU1RSSUNUIHJlZmVyZW50aWFsIGFjdGlvbi4KICov"},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/schema.php","start_line_in_file":337,"slug":"ref-action-set-null","name":"ref_action_set_null","namespace":"Flow\\PostgreSql\\DSL","parameters":[],"return_type":[{"name":"ReferentialAction","namespace":"Flow\\PostgreSql\\QueryBuilder\\Schema","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIEdldCBhIFNFVCBOVUxMIHJlZmVyZW50aWFsIGFjdGlvbi4KICov"},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/schema.php","start_line_in_file":346,"slug":"ref-action-set-default","name":"ref_action_set_default","namespace":"Flow\\PostgreSql\\DSL","parameters":[],"return_type":[{"name":"ReferentialAction","namespace":"Flow\\PostgreSql\\QueryBuilder\\Schema","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIEdldCBhIFNFVCBERUZBVUxUIHJlZmVyZW50aWFsIGFjdGlvbi4KICov"},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/schema.php","start_line_in_file":355,"slug":"ref-action-no-action","name":"ref_action_no_action","namespace":"Flow\\PostgreSql\\DSL","parameters":[],"return_type":[{"name":"ReferentialAction","namespace":"Flow\\PostgreSql\\QueryBuilder\\Schema","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIEdldCBhIE5PIEFDVElPTiByZWZlcmVudGlhbCBhY3Rpb24uCiAqLw=="},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/schema.php","start_line_in_file":370,"slug":"reindex-index","name":"reindex_index","namespace":"Flow\\PostgreSql\\DSL","parameters":[{"name":"name","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"ReindexFinalStep","namespace":"Flow\\PostgreSql\\QueryBuilder\\Schema\\Index\\Reindex","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"SCHEMA"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIFN0YXJ0IGJ1aWxkaW5nIGEgUkVJTkRFWCBJTkRFWCBzdGF0ZW1lbnQuCiAqCiAqIFVzZSBjaGFpbmFibGUgbWV0aG9kczogLT5jb25jdXJyZW50bHkoKSwgLT52ZXJib3NlKCksIC0+dGFibGVzcGFjZSgpCiAqCiAqIEV4YW1wbGU6IHJlaW5kZXhfaW5kZXgoJ2lkeF91c2Vyc19lbWFpbCcpLT5jb25jdXJyZW50bHkoKQogKgogKiBAcGFyYW0gc3RyaW5nICRuYW1lIFRoZSBpbmRleCBuYW1lIChtYXkgaW5jbHVkZSBzY2hlbWE6IHNjaGVtYS5pbmRleCkKICov"},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/schema.php","start_line_in_file":385,"slug":"reindex-table","name":"reindex_table","namespace":"Flow\\PostgreSql\\DSL","parameters":[{"name":"name","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"ReindexFinalStep","namespace":"Flow\\PostgreSql\\QueryBuilder\\Schema\\Index\\Reindex","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"SCHEMA"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIFN0YXJ0IGJ1aWxkaW5nIGEgUkVJTkRFWCBUQUJMRSBzdGF0ZW1lbnQuCiAqCiAqIFVzZSBjaGFpbmFibGUgbWV0aG9kczogLT5jb25jdXJyZW50bHkoKSwgLT52ZXJib3NlKCksIC0+dGFibGVzcGFjZSgpCiAqCiAqIEV4YW1wbGU6IHJlaW5kZXhfdGFibGUoJ3VzZXJzJyktPmNvbmN1cnJlbnRseSgpCiAqCiAqIEBwYXJhbSBzdHJpbmcgJG5hbWUgVGhlIHRhYmxlIG5hbWUgKG1heSBpbmNsdWRlIHNjaGVtYTogc2NoZW1hLnRhYmxlKQogKi8="},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/schema.php","start_line_in_file":400,"slug":"reindex-schema","name":"reindex_schema","namespace":"Flow\\PostgreSql\\DSL","parameters":[{"name":"name","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"ReindexFinalStep","namespace":"Flow\\PostgreSql\\QueryBuilder\\Schema\\Index\\Reindex","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"SCHEMA"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIFN0YXJ0IGJ1aWxkaW5nIGEgUkVJTkRFWCBTQ0hFTUEgc3RhdGVtZW50LgogKgogKiBVc2UgY2hhaW5hYmxlIG1ldGhvZHM6IC0+Y29uY3VycmVudGx5KCksIC0+dmVyYm9zZSgpLCAtPnRhYmxlc3BhY2UoKQogKgogKiBFeGFtcGxlOiByZWluZGV4X3NjaGVtYSgncHVibGljJyktPmNvbmN1cnJlbnRseSgpCiAqCiAqIEBwYXJhbSBzdHJpbmcgJG5hbWUgVGhlIHNjaGVtYSBuYW1lCiAqLw=="},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/schema.php","start_line_in_file":415,"slug":"reindex-database","name":"reindex_database","namespace":"Flow\\PostgreSql\\DSL","parameters":[{"name":"name","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"ReindexFinalStep","namespace":"Flow\\PostgreSql\\QueryBuilder\\Schema\\Index\\Reindex","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"SCHEMA"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIFN0YXJ0IGJ1aWxkaW5nIGEgUkVJTkRFWCBEQVRBQkFTRSBzdGF0ZW1lbnQuCiAqCiAqIFVzZSBjaGFpbmFibGUgbWV0aG9kczogLT5jb25jdXJyZW50bHkoKSwgLT52ZXJib3NlKCksIC0+dGFibGVzcGFjZSgpCiAqCiAqIEV4YW1wbGU6IHJlaW5kZXhfZGF0YWJhc2UoJ215ZGInKS0+Y29uY3VycmVudGx5KCkKICoKICogQHBhcmFtIHN0cmluZyAkbmFtZSBUaGUgZGF0YWJhc2UgbmFtZQogKi8="},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/schema.php","start_line_in_file":430,"slug":"index-col","name":"index_col","namespace":"Flow\\PostgreSql\\DSL","parameters":[{"name":"name","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"IndexColumn","namespace":"Flow\\PostgreSql\\QueryBuilder\\Schema\\Index","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"SCHEMA"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBhbiBpbmRleCBjb2x1bW4gc3BlY2lmaWNhdGlvbi4KICoKICogVXNlIGNoYWluYWJsZSBtZXRob2RzOiAtPmFzYygpLCAtPmRlc2MoKSwgLT5udWxsc0ZpcnN0KCksIC0+bnVsbHNMYXN0KCksIC0+b3BjbGFzcygpLCAtPmNvbGxhdGUoKQogKgogKiBFeGFtcGxlOiBpbmRleF9jb2woJ2VtYWlsJyktPmRlc2MoKS0+bnVsbHNMYXN0KCkKICoKICogQHBhcmFtIHN0cmluZyAkbmFtZSBUaGUgY29sdW1uIG5hbWUKICov"},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/schema.php","start_line_in_file":445,"slug":"index-expr","name":"index_expr","namespace":"Flow\\PostgreSql\\DSL","parameters":[{"name":"expression","type":[{"name":"Expression","namespace":"Flow\\PostgreSql\\QueryBuilder\\Expression","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"IndexColumn","namespace":"Flow\\PostgreSql\\QueryBuilder\\Schema\\Index","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"SCHEMA"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBhbiBpbmRleCBjb2x1bW4gc3BlY2lmaWNhdGlvbiBmcm9tIGFuIGV4cHJlc3Npb24uCiAqCiAqIFVzZSBjaGFpbmFibGUgbWV0aG9kczogLT5hc2MoKSwgLT5kZXNjKCksIC0+bnVsbHNGaXJzdCgpLCAtPm51bGxzTGFzdCgpLCAtPm9wY2xhc3MoKSwgLT5jb2xsYXRlKCkKICoKICogRXhhbXBsZTogaW5kZXhfZXhwcihmbl9jYWxsKCdsb3dlcicsIGNvbCgnZW1haWwnKSkpLT5kZXNjKCkKICoKICogQHBhcmFtIEV4cHJlc3Npb24gJGV4cHJlc3Npb24gVGhlIGV4cHJlc3Npb24gdG8gaW5kZXgKICov"},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/schema.php","start_line_in_file":454,"slug":"index-method-btree","name":"index_method_btree","namespace":"Flow\\PostgreSql\\DSL","parameters":[],"return_type":[{"name":"IndexMethod","namespace":"Flow\\PostgreSql\\QueryBuilder\\Schema\\Index","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIEdldCB0aGUgQlRSRUUgaW5kZXggbWV0aG9kLgogKi8="},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/schema.php","start_line_in_file":463,"slug":"index-method-hash","name":"index_method_hash","namespace":"Flow\\PostgreSql\\DSL","parameters":[],"return_type":[{"name":"IndexMethod","namespace":"Flow\\PostgreSql\\QueryBuilder\\Schema\\Index","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIEdldCB0aGUgSEFTSCBpbmRleCBtZXRob2QuCiAqLw=="},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/schema.php","start_line_in_file":472,"slug":"index-method-gist","name":"index_method_gist","namespace":"Flow\\PostgreSql\\DSL","parameters":[],"return_type":[{"name":"IndexMethod","namespace":"Flow\\PostgreSql\\QueryBuilder\\Schema\\Index","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIEdldCB0aGUgR0lTVCBpbmRleCBtZXRob2QuCiAqLw=="},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/schema.php","start_line_in_file":481,"slug":"index-method-spgist","name":"index_method_spgist","namespace":"Flow\\PostgreSql\\DSL","parameters":[],"return_type":[{"name":"IndexMethod","namespace":"Flow\\PostgreSql\\QueryBuilder\\Schema\\Index","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIEdldCB0aGUgU1BHSVNUIGluZGV4IG1ldGhvZC4KICov"},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/schema.php","start_line_in_file":490,"slug":"index-method-gin","name":"index_method_gin","namespace":"Flow\\PostgreSql\\DSL","parameters":[],"return_type":[{"name":"IndexMethod","namespace":"Flow\\PostgreSql\\QueryBuilder\\Schema\\Index","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIEdldCB0aGUgR0lOIGluZGV4IG1ldGhvZC4KICov"},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/schema.php","start_line_in_file":499,"slug":"index-method-brin","name":"index_method_brin","namespace":"Flow\\PostgreSql\\DSL","parameters":[],"return_type":[{"name":"IndexMethod","namespace":"Flow\\PostgreSql\\QueryBuilder\\Schema\\Index","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIEdldCB0aGUgQlJJTiBpbmRleCBtZXRob2QuCiAqLw=="},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/schema.php","start_line_in_file":511,"slug":"vacuum","name":"vacuum","namespace":"Flow\\PostgreSql\\DSL","parameters":[],"return_type":[{"name":"VacuumFinalStep","namespace":"Flow\\PostgreSql\\QueryBuilder\\Utility","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBhIFZBQ1VVTSBidWlsZGVyLgogKgogKiBFeGFtcGxlOiB2YWN1dW0oKS0+dGFibGUoJ3VzZXJzJykKICogUHJvZHVjZXM6IFZBQ1VVTSB1c2VycwogKi8="},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/schema.php","start_line_in_file":523,"slug":"analyze","name":"analyze","namespace":"Flow\\PostgreSql\\DSL","parameters":[],"return_type":[{"name":"AnalyzeFinalStep","namespace":"Flow\\PostgreSql\\QueryBuilder\\Utility","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBhbiBBTkFMWVpFIGJ1aWxkZXIuCiAqCiAqIEV4YW1wbGU6IGFuYWx5emUoKS0+dGFibGUoJ3VzZXJzJykKICogUHJvZHVjZXM6IEFOQUxZWkUgdXNlcnMKICov"},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/schema.php","start_line_in_file":537,"slug":"explain","name":"explain","namespace":"Flow\\PostgreSql\\DSL","parameters":[{"name":"query","type":[{"name":"SelectFinalStep","namespace":"Flow\\PostgreSql\\QueryBuilder\\Select","is_nullable":false,"is_variadic":false},{"name":"InsertBuilder","namespace":"Flow\\PostgreSql\\QueryBuilder\\Insert","is_nullable":false,"is_variadic":false},{"name":"UpdateBuilder","namespace":"Flow\\PostgreSql\\QueryBuilder\\Update","is_nullable":false,"is_variadic":false},{"name":"DeleteBuilder","namespace":"Flow\\PostgreSql\\QueryBuilder\\Delete","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"ExplainFinalStep","namespace":"Flow\\PostgreSql\\QueryBuilder\\Utility","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBhbiBFWFBMQUlOIGJ1aWxkZXIgZm9yIGEgcXVlcnkuCiAqCiAqIEV4YW1wbGU6IGV4cGxhaW4oc2VsZWN0KCktPmZyb20oJ3VzZXJzJykpCiAqIFByb2R1Y2VzOiBFWFBMQUlOIFNFTEVDVCAqIEZST00gdXNlcnMKICoKICogQHBhcmFtIERlbGV0ZUJ1aWxkZXJ8SW5zZXJ0QnVpbGRlcnxTZWxlY3RGaW5hbFN0ZXB8VXBkYXRlQnVpbGRlciAkcXVlcnkgUXVlcnkgdG8gZXhwbGFpbgogKi8="},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/schema.php","start_line_in_file":549,"slug":"lock-table","name":"lock_table","namespace":"Flow\\PostgreSql\\DSL","parameters":[{"name":"tables","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":true,"default_value":null}],"return_type":[{"name":"LockFinalStep","namespace":"Flow\\PostgreSql\\QueryBuilder\\Utility","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBhIExPQ0sgVEFCTEUgYnVpbGRlci4KICoKICogRXhhbXBsZTogbG9ja190YWJsZSgndXNlcnMnLCAnb3JkZXJzJyktPmFjY2Vzc0V4Y2x1c2l2ZSgpCiAqIFByb2R1Y2VzOiBMT0NLIFRBQkxFIHVzZXJzLCBvcmRlcnMgSU4gQUNDRVNTIEVYQ0xVU0lWRSBNT0RFCiAqLw=="},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/schema.php","start_line_in_file":564,"slug":"comment","name":"comment","namespace":"Flow\\PostgreSql\\DSL","parameters":[{"name":"target","type":[{"name":"CommentTarget","namespace":"Flow\\PostgreSql\\QueryBuilder\\Utility","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"name","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"CommentFinalStep","namespace":"Flow\\PostgreSql\\QueryBuilder\\Utility","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBhIENPTU1FTlQgT04gYnVpbGRlci4KICoKICogRXhhbXBsZTogY29tbWVudChDb21tZW50VGFyZ2V0OjpUQUJMRSwgJ3VzZXJzJyktPmlzKCdVc2VyIGFjY291bnRzIHRhYmxlJykKICogUHJvZHVjZXM6IENPTU1FTlQgT04gVEFCTEUgdXNlcnMgSVMgJ1VzZXIgYWNjb3VudHMgdGFibGUnCiAqCiAqIEBwYXJhbSBDb21tZW50VGFyZ2V0ICR0YXJnZXQgVGFyZ2V0IHR5cGUgKFRBQkxFLCBDT0xVTU4sIElOREVYLCBldGMuKQogKiBAcGFyYW0gc3RyaW5nICRuYW1lIFRhcmdldCBuYW1lICh1c2UgJ3RhYmxlLmNvbHVtbicgZm9yIENPTFVNTiB0YXJnZXRzKQogKi8="},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/schema.php","start_line_in_file":576,"slug":"cluster","name":"cluster","namespace":"Flow\\PostgreSql\\DSL","parameters":[],"return_type":[{"name":"ClusterFinalStep","namespace":"Flow\\PostgreSql\\QueryBuilder\\Utility","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBhIENMVVNURVIgYnVpbGRlci4KICoKICogRXhhbXBsZTogY2x1c3RlcigpLT50YWJsZSgndXNlcnMnKS0+dXNpbmcoJ2lkeF91c2Vyc19wa2V5JykKICogUHJvZHVjZXM6IENMVVNURVIgdXNlcnMgVVNJTkcgaWR4X3VzZXJzX3BrZXkKICov"},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/schema.php","start_line_in_file":590,"slug":"discard","name":"discard","namespace":"Flow\\PostgreSql\\DSL","parameters":[{"name":"type","type":[{"name":"DiscardType","namespace":"Flow\\PostgreSql\\QueryBuilder\\Utility","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"DiscardFinalStep","namespace":"Flow\\PostgreSql\\QueryBuilder\\Utility","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBhIERJU0NBUkQgYnVpbGRlci4KICoKICogRXhhbXBsZTogZGlzY2FyZChEaXNjYXJkVHlwZTo6QUxMKQogKiBQcm9kdWNlczogRElTQ0FSRCBBTEwKICoKICogQHBhcmFtIERpc2NhcmRUeXBlICR0eXBlIFR5cGUgb2YgcmVzb3VyY2VzIHRvIGRpc2NhcmQgKEFMTCwgUExBTlMsIFNFUVVFTkNFUywgVEVNUCkKICov"},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/schema.php","start_line_in_file":609,"slug":"grant","name":"grant","namespace":"Flow\\PostgreSql\\DSL","parameters":[{"name":"privileges","type":[{"name":"TablePrivilege","namespace":"Flow\\PostgreSql\\QueryBuilder\\Schema\\Grant","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":true,"default_value":null}],"return_type":[{"name":"GrantOnStep","namespace":"Flow\\PostgreSql\\QueryBuilder\\Schema\\Grant","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBhIEdSQU5UIHByaXZpbGVnZXMgYnVpbGRlci4KICoKICogRXhhbXBsZTogZ3JhbnQoVGFibGVQcml2aWxlZ2U6OlNFTEVDVCktPm9uVGFibGUoJ3VzZXJzJyktPnRvKCdhcHBfdXNlcicpCiAqIFByb2R1Y2VzOiBHUkFOVCBTRUxFQ1QgT04gdXNlcnMgVE8gYXBwX3VzZXIKICoKICogRXhhbXBsZTogZ3JhbnQoVGFibGVQcml2aWxlZ2U6OkFMTCktPm9uQWxsVGFibGVzSW5TY2hlbWEoJ3B1YmxpYycpLT50bygnYWRtaW4nKQogKiBQcm9kdWNlczogR1JBTlQgQUxMIE9OIEFMTCBUQUJMRVMgSU4gU0NIRU1BIHB1YmxpYyBUTyBhZG1pbgogKgogKiBAcGFyYW0gc3RyaW5nfFRhYmxlUHJpdmlsZWdlIC4uLiRwcml2aWxlZ2VzIFRoZSBwcml2aWxlZ2VzIHRvIGdyYW50CiAqCiAqIEByZXR1cm4gR3JhbnRPblN0ZXAgQnVpbGRlciBmb3IgZ3JhbnQgb3B0aW9ucwogKi8="},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/schema.php","start_line_in_file":628,"slug":"grant-role","name":"grant_role","namespace":"Flow\\PostgreSql\\DSL","parameters":[{"name":"roles","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":true,"default_value":null}],"return_type":[{"name":"GrantRoleToStep","namespace":"Flow\\PostgreSql\\QueryBuilder\\Schema\\Grant","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBhIEdSQU5UIHJvbGUgYnVpbGRlci4KICoKICogRXhhbXBsZTogZ3JhbnRfcm9sZSgnYWRtaW4nKS0+dG8oJ3VzZXIxJykKICogUHJvZHVjZXM6IEdSQU5UIGFkbWluIFRPIHVzZXIxCiAqCiAqIEV4YW1wbGU6IGdyYW50X3JvbGUoJ2FkbWluJywgJ2RldmVsb3BlcicpLT50bygndXNlcjEnKS0+d2l0aEFkbWluT3B0aW9uKCkKICogUHJvZHVjZXM6IEdSQU5UIGFkbWluLCBkZXZlbG9wZXIgVE8gdXNlcjEgV0lUSCBBRE1JTiBPUFRJT04KICoKICogQHBhcmFtIHN0cmluZyAuLi4kcm9sZXMgVGhlIHJvbGVzIHRvIGdyYW50CiAqCiAqIEByZXR1cm4gR3JhbnRSb2xlVG9TdGVwIEJ1aWxkZXIgZm9yIGdyYW50IHJvbGUgb3B0aW9ucwogKi8="},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/schema.php","start_line_in_file":647,"slug":"revoke","name":"revoke","namespace":"Flow\\PostgreSql\\DSL","parameters":[{"name":"privileges","type":[{"name":"TablePrivilege","namespace":"Flow\\PostgreSql\\QueryBuilder\\Schema\\Grant","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":true,"default_value":null}],"return_type":[{"name":"RevokeOnStep","namespace":"Flow\\PostgreSql\\QueryBuilder\\Schema\\Grant","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBhIFJFVk9LRSBwcml2aWxlZ2VzIGJ1aWxkZXIuCiAqCiAqIEV4YW1wbGU6IHJldm9rZShUYWJsZVByaXZpbGVnZTo6U0VMRUNUKS0+b25UYWJsZSgndXNlcnMnKS0+ZnJvbSgnYXBwX3VzZXInKQogKiBQcm9kdWNlczogUkVWT0tFIFNFTEVDVCBPTiB1c2VycyBGUk9NIGFwcF91c2VyCiAqCiAqIEV4YW1wbGU6IHJldm9rZShUYWJsZVByaXZpbGVnZTo6QUxMKS0+b25UYWJsZSgndXNlcnMnKS0+ZnJvbSgnYXBwX3VzZXInKS0+Y2FzY2FkZSgpCiAqIFByb2R1Y2VzOiBSRVZPS0UgQUxMIE9OIHVzZXJzIEZST00gYXBwX3VzZXIgQ0FTQ0FERQogKgogKiBAcGFyYW0gc3RyaW5nfFRhYmxlUHJpdmlsZWdlIC4uLiRwcml2aWxlZ2VzIFRoZSBwcml2aWxlZ2VzIHRvIHJldm9rZQogKgogKiBAcmV0dXJuIFJldm9rZU9uU3RlcCBCdWlsZGVyIGZvciByZXZva2Ugb3B0aW9ucwogKi8="},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/schema.php","start_line_in_file":666,"slug":"revoke-role","name":"revoke_role","namespace":"Flow\\PostgreSql\\DSL","parameters":[{"name":"roles","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":true,"default_value":null}],"return_type":[{"name":"RevokeRoleFromStep","namespace":"Flow\\PostgreSql\\QueryBuilder\\Schema\\Grant","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBhIFJFVk9LRSByb2xlIGJ1aWxkZXIuCiAqCiAqIEV4YW1wbGU6IHJldm9rZV9yb2xlKCdhZG1pbicpLT5mcm9tKCd1c2VyMScpCiAqIFByb2R1Y2VzOiBSRVZPS0UgYWRtaW4gRlJPTSB1c2VyMQogKgogKiBFeGFtcGxlOiByZXZva2Vfcm9sZSgnYWRtaW4nKS0+ZnJvbSgndXNlcjEnKS0+Y2FzY2FkZSgpCiAqIFByb2R1Y2VzOiBSRVZPS0UgYWRtaW4gRlJPTSB1c2VyMSBDQVNDQURFCiAqCiAqIEBwYXJhbSBzdHJpbmcgLi4uJHJvbGVzIFRoZSByb2xlcyB0byByZXZva2UKICoKICogQHJldHVybiBSZXZva2VSb2xlRnJvbVN0ZXAgQnVpbGRlciBmb3IgcmV2b2tlIHJvbGUgb3B0aW9ucwogKi8="},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/schema.php","start_line_in_file":682,"slug":"set-role","name":"set_role","namespace":"Flow\\PostgreSql\\DSL","parameters":[{"name":"role","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"SetRoleFinalStep","namespace":"Flow\\PostgreSql\\QueryBuilder\\Schema\\Session","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBhIFNFVCBST0xFIGJ1aWxkZXIuCiAqCiAqIEV4YW1wbGU6IHNldF9yb2xlKCdhZG1pbicpCiAqIFByb2R1Y2VzOiBTRVQgUk9MRSBhZG1pbgogKgogKiBAcGFyYW0gc3RyaW5nICRyb2xlIFRoZSByb2xlIHRvIHNldAogKgogKiBAcmV0dXJuIFNldFJvbGVGaW5hbFN0ZXAgQnVpbGRlciBmb3Igc2V0IHJvbGUKICov"},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/schema.php","start_line_in_file":696,"slug":"reset-role","name":"reset_role","namespace":"Flow\\PostgreSql\\DSL","parameters":[],"return_type":[{"name":"ResetRoleFinalStep","namespace":"Flow\\PostgreSql\\QueryBuilder\\Schema\\Session","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBhIFJFU0VUIFJPTEUgYnVpbGRlci4KICoKICogRXhhbXBsZTogcmVzZXRfcm9sZSgpCiAqIFByb2R1Y2VzOiBSRVNFVCBST0xFCiAqCiAqIEByZXR1cm4gUmVzZXRSb2xlRmluYWxTdGVwIEJ1aWxkZXIgZm9yIHJlc2V0IHJvbGUKICov"},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/schema.php","start_line_in_file":712,"slug":"reassign-owned","name":"reassign_owned","namespace":"Flow\\PostgreSql\\DSL","parameters":[{"name":"roles","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":true,"default_value":null}],"return_type":[{"name":"ReassignOwnedToStep","namespace":"Flow\\PostgreSql\\QueryBuilder\\Schema\\Ownership","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBhIFJFQVNTSUdOIE9XTkVEIGJ1aWxkZXIuCiAqCiAqIEV4YW1wbGU6IHJlYXNzaWduX293bmVkKCdvbGRfcm9sZScpLT50bygnbmV3X3JvbGUnKQogKiBQcm9kdWNlczogUkVBU1NJR04gT1dORUQgQlkgb2xkX3JvbGUgVE8gbmV3X3JvbGUKICoKICogQHBhcmFtIHN0cmluZyAuLi4kcm9sZXMgVGhlIHJvbGVzIHdob3NlIG93bmVkIG9iamVjdHMgc2hvdWxkIGJlIHJlYXNzaWduZWQKICoKICogQHJldHVybiBSZWFzc2lnbk93bmVkVG9TdGVwIEJ1aWxkZXIgZm9yIHJlYXNzaWduIG93bmVkIG9wdGlvbnMKICov"},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/schema.php","start_line_in_file":731,"slug":"drop-owned","name":"drop_owned","namespace":"Flow\\PostgreSql\\DSL","parameters":[{"name":"roles","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":true,"default_value":null}],"return_type":[{"name":"DropOwnedFinalStep","namespace":"Flow\\PostgreSql\\QueryBuilder\\Schema\\Ownership","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBhIERST1AgT1dORUQgYnVpbGRlci4KICoKICogRXhhbXBsZTogZHJvcF9vd25lZCgncm9sZTEnKQogKiBQcm9kdWNlczogRFJPUCBPV05FRCBCWSByb2xlMQogKgogKiBFeGFtcGxlOiBkcm9wX293bmVkKCdyb2xlMScsICdyb2xlMicpLT5jYXNjYWRlKCkKICogUHJvZHVjZXM6IERST1AgT1dORUQgQlkgcm9sZTEsIHJvbGUyIENBU0NBREUKICoKICogQHBhcmFtIHN0cmluZyAuLi4kcm9sZXMgVGhlIHJvbGVzIHdob3NlIG93bmVkIG9iamVjdHMgc2hvdWxkIGJlIGRyb3BwZWQKICoKICogQHJldHVybiBEcm9wT3duZWRGaW5hbFN0ZXAgQnVpbGRlciBmb3IgZHJvcCBvd25lZCBvcHRpb25zCiAqLw=="},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/schema.php","start_line_in_file":749,"slug":"func-arg","name":"func_arg","namespace":"Flow\\PostgreSql\\DSL","parameters":[{"name":"type","type":[{"name":"ColumnType","namespace":"Flow\\PostgreSql\\QueryBuilder\\Schema","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"FunctionArgument","namespace":"Flow\\PostgreSql\\QueryBuilder\\Schema\\Function","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZXMgYSBuZXcgZnVuY3Rpb24gYXJndW1lbnQgZm9yIHVzZSBpbiBmdW5jdGlvbi9wcm9jZWR1cmUgZGVmaW5pdGlvbnMuCiAqCiAqIEV4YW1wbGU6IGZ1bmNfYXJnKGNvbHVtbl90eXBlX2ludGVnZXIoKSkKICogRXhhbXBsZTogZnVuY19hcmcoY29sdW1uX3R5cGVfdGV4dCgpKS0+bmFtZWQoJ3VzZXJuYW1lJykKICogRXhhbXBsZTogZnVuY19hcmcoY29sdW1uX3R5cGVfaW50ZWdlcigpKS0+bmFtZWQoJ2NvdW50JyktPmRlZmF1bHQoJzAnKQogKiBFeGFtcGxlOiBmdW5jX2FyZyhjb2x1bW5fdHlwZV90ZXh0KCkpLT5vdXQoKQogKgogKiBAcGFyYW0gQ29sdW1uVHlwZSAkdHlwZSBUaGUgUG9zdGdyZVNRTCBkYXRhIHR5cGUgZm9yIHRoZSBhcmd1bWVudAogKgogKiBAcmV0dXJuIEZ1bmN0aW9uQXJndW1lbnQgQnVpbGRlciBmb3IgZnVuY3Rpb24gYXJndW1lbnQgb3B0aW9ucwogKi8="},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/schema.php","start_line_in_file":768,"slug":"call","name":"call","namespace":"Flow\\PostgreSql\\DSL","parameters":[{"name":"procedure","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"CallFinalStep","namespace":"Flow\\PostgreSql\\QueryBuilder\\Schema\\Function","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZXMgYSBDQUxMIHN0YXRlbWVudCBidWlsZGVyIGZvciBpbnZva2luZyBhIHByb2NlZHVyZS4KICoKICogRXhhbXBsZTogY2FsbCgndXBkYXRlX3N0YXRzJyktPndpdGgoMTIzKQogKiBQcm9kdWNlczogQ0FMTCB1cGRhdGVfc3RhdHMoMTIzKQogKgogKiBFeGFtcGxlOiBjYWxsKCdwcm9jZXNzX2RhdGEnKS0+d2l0aCgndGVzdCcsIDQyLCB0cnVlKQogKiBQcm9kdWNlczogQ0FMTCBwcm9jZXNzX2RhdGEoJ3Rlc3QnLCA0MiwgdHJ1ZSkKICoKICogQHBhcmFtIHN0cmluZyAkcHJvY2VkdXJlIFRoZSBuYW1lIG9mIHRoZSBwcm9jZWR1cmUgdG8gY2FsbAogKgogKiBAcmV0dXJuIENhbGxGaW5hbFN0ZXAgQnVpbGRlciBmb3IgY2FsbCBzdGF0ZW1lbnQgb3B0aW9ucwogKi8="},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/schema.php","start_line_in_file":787,"slug":"do-block","name":"do_block","namespace":"Flow\\PostgreSql\\DSL","parameters":[{"name":"code","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"DoFinalStep","namespace":"Flow\\PostgreSql\\QueryBuilder\\Schema\\Function","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZXMgYSBETyBzdGF0ZW1lbnQgYnVpbGRlciBmb3IgZXhlY3V0aW5nIGFuIGFub255bW91cyBjb2RlIGJsb2NrLgogKgogKiBFeGFtcGxlOiBkb19ibG9jaygnQkVHSU4gUkFJU0UgTk9USUNFICQkSGVsbG8gV29ybGQkJDsgRU5EOycpCiAqIFByb2R1Y2VzOiBETyAkJCBCRUdJTiBSQUlTRSBOT1RJQ0UgJCRIZWxsbyBXb3JsZCQkOyBFTkQ7ICQkIExBTkdVQUdFIHBscGdzcWwKICoKICogRXhhbXBsZTogZG9fYmxvY2soJ1NFTEVDVCAxJyktPmxhbmd1YWdlKCdzcWwnKQogKiBQcm9kdWNlczogRE8gJCQgU0VMRUNUIDEgJCQgTEFOR1VBR0Ugc3FsCiAqCiAqIEBwYXJhbSBzdHJpbmcgJGNvZGUgVGhlIGFub255bW91cyBjb2RlIGJsb2NrIHRvIGV4ZWN1dGUKICoKICogQHJldHVybiBEb0ZpbmFsU3RlcCBCdWlsZGVyIGZvciBETyBzdGF0ZW1lbnQgb3B0aW9ucwogKi8="},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/schema.php","start_line_in_file":807,"slug":"type-attr","name":"type_attr","namespace":"Flow\\PostgreSql\\DSL","parameters":[{"name":"name","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"type","type":[{"name":"ColumnType","namespace":"Flow\\PostgreSql\\QueryBuilder\\Schema","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"TypeAttribute","namespace":"Flow\\PostgreSql\\QueryBuilder\\Schema\\Type","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZXMgYSB0eXBlIGF0dHJpYnV0ZSBmb3IgY29tcG9zaXRlIHR5cGVzLgogKgogKiBFeGFtcGxlOiB0eXBlX2F0dHIoJ25hbWUnLCBjb2x1bW5fdHlwZV90ZXh0KCkpCiAqIFByb2R1Y2VzOiBuYW1lIHRleHQKICoKICogRXhhbXBsZTogdHlwZV9hdHRyKCdkZXNjcmlwdGlvbicsIGNvbHVtbl90eXBlX3RleHQoKSktPmNvbGxhdGUoJ2VuX1VTJykKICogUHJvZHVjZXM6IGRlc2NyaXB0aW9uIHRleHQgQ09MTEFURSAiZW5fVVMiCiAqCiAqIEBwYXJhbSBzdHJpbmcgJG5hbWUgVGhlIGF0dHJpYnV0ZSBuYW1lCiAqIEBwYXJhbSBDb2x1bW5UeXBlICR0eXBlIFRoZSBhdHRyaWJ1dGUgdHlwZQogKgogKiBAcmV0dXJuIFR5cGVBdHRyaWJ1dGUgVHlwZSBhdHRyaWJ1dGUgdmFsdWUgb2JqZWN0CiAqLw=="},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/schema.php","start_line_in_file":816,"slug":"column-type-integer","name":"column_type_integer","namespace":"Flow\\PostgreSql\\DSL","parameters":[],"return_type":[{"name":"ColumnType","namespace":"Flow\\PostgreSql\\QueryBuilder\\Schema","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBhbiBpbnRlZ2VyIGRhdGEgdHlwZSAoUG9zdGdyZVNRTCBpbnQ0KS4KICov"},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/schema.php","start_line_in_file":825,"slug":"column-type-smallint","name":"column_type_smallint","namespace":"Flow\\PostgreSql\\DSL","parameters":[],"return_type":[{"name":"ColumnType","namespace":"Flow\\PostgreSql\\QueryBuilder\\Schema","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBhIHNtYWxsaW50IGRhdGEgdHlwZSAoUG9zdGdyZVNRTCBpbnQyKS4KICov"},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/schema.php","start_line_in_file":834,"slug":"column-type-bigint","name":"column_type_bigint","namespace":"Flow\\PostgreSql\\DSL","parameters":[],"return_type":[{"name":"ColumnType","namespace":"Flow\\PostgreSql\\QueryBuilder\\Schema","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBhIGJpZ2ludCBkYXRhIHR5cGUgKFBvc3RncmVTUUwgaW50OCkuCiAqLw=="},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/schema.php","start_line_in_file":843,"slug":"column-type-boolean","name":"column_type_boolean","namespace":"Flow\\PostgreSql\\DSL","parameters":[],"return_type":[{"name":"ColumnType","namespace":"Flow\\PostgreSql\\QueryBuilder\\Schema","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBhIGJvb2xlYW4gZGF0YSB0eXBlLgogKi8="},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/schema.php","start_line_in_file":852,"slug":"column-type-text","name":"column_type_text","namespace":"Flow\\PostgreSql\\DSL","parameters":[],"return_type":[{"name":"ColumnType","namespace":"Flow\\PostgreSql\\QueryBuilder\\Schema","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBhIHRleHQgZGF0YSB0eXBlLgogKi8="},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/schema.php","start_line_in_file":861,"slug":"column-type-varchar","name":"column_type_varchar","namespace":"Flow\\PostgreSql\\DSL","parameters":[{"name":"length","type":[{"name":"int","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"ColumnType","namespace":"Flow\\PostgreSql\\QueryBuilder\\Schema","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBhIHZhcmNoYXIgZGF0YSB0eXBlIHdpdGggbGVuZ3RoIGNvbnN0cmFpbnQuCiAqLw=="},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/schema.php","start_line_in_file":870,"slug":"column-type-char","name":"column_type_char","namespace":"Flow\\PostgreSql\\DSL","parameters":[{"name":"length","type":[{"name":"int","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"ColumnType","namespace":"Flow\\PostgreSql\\QueryBuilder\\Schema","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBhIGNoYXIgZGF0YSB0eXBlIHdpdGggbGVuZ3RoIGNvbnN0cmFpbnQuCiAqLw=="},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/schema.php","start_line_in_file":879,"slug":"column-type-numeric","name":"column_type_numeric","namespace":"Flow\\PostgreSql\\DSL","parameters":[{"name":"precision","type":[{"name":"int","namespace":null,"is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"null"},{"name":"scale","type":[{"name":"int","namespace":null,"is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"null"}],"return_type":[{"name":"ColumnType","namespace":"Flow\\PostgreSql\\QueryBuilder\\Schema","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBhIG51bWVyaWMgZGF0YSB0eXBlIHdpdGggb3B0aW9uYWwgcHJlY2lzaW9uIGFuZCBzY2FsZS4KICov"},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/schema.php","start_line_in_file":888,"slug":"column-type-decimal","name":"column_type_decimal","namespace":"Flow\\PostgreSql\\DSL","parameters":[{"name":"precision","type":[{"name":"int","namespace":null,"is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"null"},{"name":"scale","type":[{"name":"int","namespace":null,"is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"null"}],"return_type":[{"name":"ColumnType","namespace":"Flow\\PostgreSql\\QueryBuilder\\Schema","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBhIGRlY2ltYWwgZGF0YSB0eXBlIHdpdGggb3B0aW9uYWwgcHJlY2lzaW9uIGFuZCBzY2FsZS4KICov"},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/schema.php","start_line_in_file":897,"slug":"column-type-real","name":"column_type_real","namespace":"Flow\\PostgreSql\\DSL","parameters":[],"return_type":[{"name":"ColumnType","namespace":"Flow\\PostgreSql\\QueryBuilder\\Schema","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBhIHJlYWwgZGF0YSB0eXBlIChQb3N0Z3JlU1FMIGZsb2F0NCkuCiAqLw=="},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/schema.php","start_line_in_file":906,"slug":"column-type-double-precision","name":"column_type_double_precision","namespace":"Flow\\PostgreSql\\DSL","parameters":[],"return_type":[{"name":"ColumnType","namespace":"Flow\\PostgreSql\\QueryBuilder\\Schema","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBhIGRvdWJsZSBwcmVjaXNpb24gZGF0YSB0eXBlIChQb3N0Z3JlU1FMIGZsb2F0OCkuCiAqLw=="},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/schema.php","start_line_in_file":915,"slug":"column-type-date","name":"column_type_date","namespace":"Flow\\PostgreSql\\DSL","parameters":[],"return_type":[{"name":"ColumnType","namespace":"Flow\\PostgreSql\\QueryBuilder\\Schema","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBhIGRhdGUgZGF0YSB0eXBlLgogKi8="},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/schema.php","start_line_in_file":924,"slug":"column-type-time","name":"column_type_time","namespace":"Flow\\PostgreSql\\DSL","parameters":[{"name":"precision","type":[{"name":"int","namespace":null,"is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"null"}],"return_type":[{"name":"ColumnType","namespace":"Flow\\PostgreSql\\QueryBuilder\\Schema","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBhIHRpbWUgZGF0YSB0eXBlIHdpdGggb3B0aW9uYWwgcHJlY2lzaW9uLgogKi8="},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/schema.php","start_line_in_file":933,"slug":"column-type-timestamp","name":"column_type_timestamp","namespace":"Flow\\PostgreSql\\DSL","parameters":[{"name":"precision","type":[{"name":"int","namespace":null,"is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"null"}],"return_type":[{"name":"ColumnType","namespace":"Flow\\PostgreSql\\QueryBuilder\\Schema","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBhIHRpbWVzdGFtcCBkYXRhIHR5cGUgd2l0aCBvcHRpb25hbCBwcmVjaXNpb24uCiAqLw=="},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/schema.php","start_line_in_file":942,"slug":"column-type-timestamptz","name":"column_type_timestamptz","namespace":"Flow\\PostgreSql\\DSL","parameters":[{"name":"precision","type":[{"name":"int","namespace":null,"is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"null"}],"return_type":[{"name":"ColumnType","namespace":"Flow\\PostgreSql\\QueryBuilder\\Schema","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBhIHRpbWVzdGFtcCB3aXRoIHRpbWUgem9uZSBkYXRhIHR5cGUgd2l0aCBvcHRpb25hbCBwcmVjaXNpb24uCiAqLw=="},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/schema.php","start_line_in_file":951,"slug":"column-type-interval","name":"column_type_interval","namespace":"Flow\\PostgreSql\\DSL","parameters":[],"return_type":[{"name":"ColumnType","namespace":"Flow\\PostgreSql\\QueryBuilder\\Schema","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBhbiBpbnRlcnZhbCBkYXRhIHR5cGUuCiAqLw=="},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/schema.php","start_line_in_file":960,"slug":"column-type-uuid","name":"column_type_uuid","namespace":"Flow\\PostgreSql\\DSL","parameters":[],"return_type":[{"name":"ColumnType","namespace":"Flow\\PostgreSql\\QueryBuilder\\Schema","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBhIFVVSUQgZGF0YSB0eXBlLgogKi8="},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/schema.php","start_line_in_file":969,"slug":"column-type-json","name":"column_type_json","namespace":"Flow\\PostgreSql\\DSL","parameters":[],"return_type":[{"name":"ColumnType","namespace":"Flow\\PostgreSql\\QueryBuilder\\Schema","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBhIEpTT04gZGF0YSB0eXBlLgogKi8="},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/schema.php","start_line_in_file":978,"slug":"column-type-jsonb","name":"column_type_jsonb","namespace":"Flow\\PostgreSql\\DSL","parameters":[],"return_type":[{"name":"ColumnType","namespace":"Flow\\PostgreSql\\QueryBuilder\\Schema","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBhIEpTT05CIGRhdGEgdHlwZS4KICov"},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/schema.php","start_line_in_file":987,"slug":"column-type-bytea","name":"column_type_bytea","namespace":"Flow\\PostgreSql\\DSL","parameters":[],"return_type":[{"name":"ColumnType","namespace":"Flow\\PostgreSql\\QueryBuilder\\Schema","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBhIGJ5dGVhIGRhdGEgdHlwZS4KICov"},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/schema.php","start_line_in_file":996,"slug":"column-type-xml","name":"column_type_xml","namespace":"Flow\\PostgreSql\\DSL","parameters":[],"return_type":[{"name":"ColumnType","namespace":"Flow\\PostgreSql\\QueryBuilder\\Schema","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBhbiBYTUwgZGF0YSB0eXBlLgogKi8="},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/schema.php","start_line_in_file":1005,"slug":"column-type-inet","name":"column_type_inet","namespace":"Flow\\PostgreSql\\DSL","parameters":[],"return_type":[{"name":"ColumnType","namespace":"Flow\\PostgreSql\\QueryBuilder\\Schema","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBhbiBpbmV0IGRhdGEgdHlwZS4KICov"},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/schema.php","start_line_in_file":1014,"slug":"column-type-cidr","name":"column_type_cidr","namespace":"Flow\\PostgreSql\\DSL","parameters":[],"return_type":[{"name":"ColumnType","namespace":"Flow\\PostgreSql\\QueryBuilder\\Schema","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBhIGNpZHIgZGF0YSB0eXBlLgogKi8="},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/schema.php","start_line_in_file":1023,"slug":"column-type-macaddr","name":"column_type_macaddr","namespace":"Flow\\PostgreSql\\DSL","parameters":[],"return_type":[{"name":"ColumnType","namespace":"Flow\\PostgreSql\\QueryBuilder\\Schema","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBhIG1hY2FkZHIgZGF0YSB0eXBlLgogKi8="},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/schema.php","start_line_in_file":1032,"slug":"column-type-serial","name":"column_type_serial","namespace":"Flow\\PostgreSql\\DSL","parameters":[],"return_type":[{"name":"ColumnType","namespace":"Flow\\PostgreSql\\QueryBuilder\\Schema","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBhIHNlcmlhbCBkYXRhIHR5cGUuCiAqLw=="},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/schema.php","start_line_in_file":1041,"slug":"column-type-smallserial","name":"column_type_smallserial","namespace":"Flow\\PostgreSql\\DSL","parameters":[],"return_type":[{"name":"ColumnType","namespace":"Flow\\PostgreSql\\QueryBuilder\\Schema","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBhIHNtYWxsc2VyaWFsIGRhdGEgdHlwZS4KICov"},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/schema.php","start_line_in_file":1050,"slug":"column-type-bigserial","name":"column_type_bigserial","namespace":"Flow\\PostgreSql\\DSL","parameters":[],"return_type":[{"name":"ColumnType","namespace":"Flow\\PostgreSql\\QueryBuilder\\Schema","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBhIGJpZ3NlcmlhbCBkYXRhIHR5cGUuCiAqLw=="},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/schema.php","start_line_in_file":1059,"slug":"column-type-array","name":"column_type_array","namespace":"Flow\\PostgreSql\\DSL","parameters":[{"name":"elementType","type":[{"name":"ColumnType","namespace":"Flow\\PostgreSql\\QueryBuilder\\Schema","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"ColumnType","namespace":"Flow\\PostgreSql\\QueryBuilder\\Schema","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBhbiBhcnJheSBkYXRhIHR5cGUgZnJvbSBhbiBlbGVtZW50IHR5cGUuCiAqLw=="},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/schema.php","start_line_in_file":1071,"slug":"column-type-custom","name":"column_type_custom","namespace":"Flow\\PostgreSql\\DSL","parameters":[{"name":"typeName","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"schema","type":[{"name":"string","namespace":null,"is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"null"}],"return_type":[{"name":"ColumnType","namespace":"Flow\\PostgreSql\\QueryBuilder\\Schema","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBhIGN1c3RvbSBkYXRhIHR5cGUuCiAqCiAqIEBwYXJhbSBzdHJpbmcgJHR5cGVOYW1lIFR5cGUgbmFtZQogKiBAcGFyYW0gbnVsbHxzdHJpbmcgJHNjaGVtYSBPcHRpb25hbCBzY2hlbWEgbmFtZQogKi8="},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/schema.php","start_line_in_file":1084,"slug":"column-type-from-string","name":"column_type_from_string","namespace":"Flow\\PostgreSql\\DSL","parameters":[{"name":"typeName","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"ColumnType","namespace":"Flow\\PostgreSql\\QueryBuilder\\Schema","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIFBhcnNlIGEgUG9zdGdyZVNRTCB0eXBlIHN0cmluZyBpbnRvIGEgQ29sdW1uVHlwZS4KICoKICogSGFuZGxlcyBhbGwgUG9zdGdyZVNRTCB0eXBlIHN5bnRheCBpbmNsdWRpbmcgcHJlY2lzaW9uLCBhcnJheXMsIGFuZCBzY2hlbWEtcXVhbGlmaWVkIHR5cGVzLgogKgogKiBAcGFyYW0gc3RyaW5nICR0eXBlTmFtZSBQb3N0Z3JlU1FMIHR5cGUgc3RyaW5nIChlLmcuLCAnaW50ZWdlcicsICdjaGFyYWN0ZXIgdmFyeWluZygyNTUpJywgJ3RleHRbXScpCiAqLw=="},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/schema.php","start_line_in_file":1092,"slug":"value-type-text","name":"value_type_text","namespace":"Flow\\PostgreSql\\DSL","parameters":[],"return_type":[{"name":"ValueType","namespace":"Flow\\PostgreSql\\Client\\Types","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/schema.php","start_line_in_file":1098,"slug":"value-type-varchar","name":"value_type_varchar","namespace":"Flow\\PostgreSql\\DSL","parameters":[],"return_type":[{"name":"ValueType","namespace":"Flow\\PostgreSql\\Client\\Types","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/schema.php","start_line_in_file":1104,"slug":"value-type-char","name":"value_type_char","namespace":"Flow\\PostgreSql\\DSL","parameters":[],"return_type":[{"name":"ValueType","namespace":"Flow\\PostgreSql\\Client\\Types","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/schema.php","start_line_in_file":1110,"slug":"value-type-bpchar","name":"value_type_bpchar","namespace":"Flow\\PostgreSql\\DSL","parameters":[],"return_type":[{"name":"ValueType","namespace":"Flow\\PostgreSql\\Client\\Types","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/schema.php","start_line_in_file":1116,"slug":"value-type-int2","name":"value_type_int2","namespace":"Flow\\PostgreSql\\DSL","parameters":[],"return_type":[{"name":"ValueType","namespace":"Flow\\PostgreSql\\Client\\Types","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/schema.php","start_line_in_file":1122,"slug":"value-type-smallint","name":"value_type_smallint","namespace":"Flow\\PostgreSql\\DSL","parameters":[],"return_type":[{"name":"ValueType","namespace":"Flow\\PostgreSql\\Client\\Types","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/schema.php","start_line_in_file":1128,"slug":"value-type-int4","name":"value_type_int4","namespace":"Flow\\PostgreSql\\DSL","parameters":[],"return_type":[{"name":"ValueType","namespace":"Flow\\PostgreSql\\Client\\Types","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/schema.php","start_line_in_file":1134,"slug":"value-type-integer","name":"value_type_integer","namespace":"Flow\\PostgreSql\\DSL","parameters":[],"return_type":[{"name":"ValueType","namespace":"Flow\\PostgreSql\\Client\\Types","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/schema.php","start_line_in_file":1140,"slug":"value-type-int8","name":"value_type_int8","namespace":"Flow\\PostgreSql\\DSL","parameters":[],"return_type":[{"name":"ValueType","namespace":"Flow\\PostgreSql\\Client\\Types","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/schema.php","start_line_in_file":1146,"slug":"value-type-bigint","name":"value_type_bigint","namespace":"Flow\\PostgreSql\\DSL","parameters":[],"return_type":[{"name":"ValueType","namespace":"Flow\\PostgreSql\\Client\\Types","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/schema.php","start_line_in_file":1152,"slug":"value-type-float4","name":"value_type_float4","namespace":"Flow\\PostgreSql\\DSL","parameters":[],"return_type":[{"name":"ValueType","namespace":"Flow\\PostgreSql\\Client\\Types","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/schema.php","start_line_in_file":1158,"slug":"value-type-real","name":"value_type_real","namespace":"Flow\\PostgreSql\\DSL","parameters":[],"return_type":[{"name":"ValueType","namespace":"Flow\\PostgreSql\\Client\\Types","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/schema.php","start_line_in_file":1164,"slug":"value-type-float8","name":"value_type_float8","namespace":"Flow\\PostgreSql\\DSL","parameters":[],"return_type":[{"name":"ValueType","namespace":"Flow\\PostgreSql\\Client\\Types","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/schema.php","start_line_in_file":1170,"slug":"value-type-double","name":"value_type_double","namespace":"Flow\\PostgreSql\\DSL","parameters":[],"return_type":[{"name":"ValueType","namespace":"Flow\\PostgreSql\\Client\\Types","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/schema.php","start_line_in_file":1176,"slug":"value-type-numeric","name":"value_type_numeric","namespace":"Flow\\PostgreSql\\DSL","parameters":[],"return_type":[{"name":"ValueType","namespace":"Flow\\PostgreSql\\Client\\Types","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/schema.php","start_line_in_file":1182,"slug":"value-type-money","name":"value_type_money","namespace":"Flow\\PostgreSql\\DSL","parameters":[],"return_type":[{"name":"ValueType","namespace":"Flow\\PostgreSql\\Client\\Types","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/schema.php","start_line_in_file":1188,"slug":"value-type-bool","name":"value_type_bool","namespace":"Flow\\PostgreSql\\DSL","parameters":[],"return_type":[{"name":"ValueType","namespace":"Flow\\PostgreSql\\Client\\Types","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/schema.php","start_line_in_file":1194,"slug":"value-type-boolean","name":"value_type_boolean","namespace":"Flow\\PostgreSql\\DSL","parameters":[],"return_type":[{"name":"ValueType","namespace":"Flow\\PostgreSql\\Client\\Types","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/schema.php","start_line_in_file":1200,"slug":"value-type-bytea","name":"value_type_bytea","namespace":"Flow\\PostgreSql\\DSL","parameters":[],"return_type":[{"name":"ValueType","namespace":"Flow\\PostgreSql\\Client\\Types","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/schema.php","start_line_in_file":1206,"slug":"value-type-bit","name":"value_type_bit","namespace":"Flow\\PostgreSql\\DSL","parameters":[],"return_type":[{"name":"ValueType","namespace":"Flow\\PostgreSql\\Client\\Types","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/schema.php","start_line_in_file":1212,"slug":"value-type-varbit","name":"value_type_varbit","namespace":"Flow\\PostgreSql\\DSL","parameters":[],"return_type":[{"name":"ValueType","namespace":"Flow\\PostgreSql\\Client\\Types","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/schema.php","start_line_in_file":1218,"slug":"value-type-date","name":"value_type_date","namespace":"Flow\\PostgreSql\\DSL","parameters":[],"return_type":[{"name":"ValueType","namespace":"Flow\\PostgreSql\\Client\\Types","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/schema.php","start_line_in_file":1224,"slug":"value-type-time","name":"value_type_time","namespace":"Flow\\PostgreSql\\DSL","parameters":[],"return_type":[{"name":"ValueType","namespace":"Flow\\PostgreSql\\Client\\Types","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/schema.php","start_line_in_file":1230,"slug":"value-type-timetz","name":"value_type_timetz","namespace":"Flow\\PostgreSql\\DSL","parameters":[],"return_type":[{"name":"ValueType","namespace":"Flow\\PostgreSql\\Client\\Types","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/schema.php","start_line_in_file":1236,"slug":"value-type-timestamp","name":"value_type_timestamp","namespace":"Flow\\PostgreSql\\DSL","parameters":[],"return_type":[{"name":"ValueType","namespace":"Flow\\PostgreSql\\Client\\Types","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/schema.php","start_line_in_file":1242,"slug":"value-type-timestamptz","name":"value_type_timestamptz","namespace":"Flow\\PostgreSql\\DSL","parameters":[],"return_type":[{"name":"ValueType","namespace":"Flow\\PostgreSql\\Client\\Types","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/schema.php","start_line_in_file":1248,"slug":"value-type-interval","name":"value_type_interval","namespace":"Flow\\PostgreSql\\DSL","parameters":[],"return_type":[{"name":"ValueType","namespace":"Flow\\PostgreSql\\Client\\Types","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/schema.php","start_line_in_file":1254,"slug":"value-type-json","name":"value_type_json","namespace":"Flow\\PostgreSql\\DSL","parameters":[],"return_type":[{"name":"ValueType","namespace":"Flow\\PostgreSql\\Client\\Types","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/schema.php","start_line_in_file":1260,"slug":"value-type-jsonb","name":"value_type_jsonb","namespace":"Flow\\PostgreSql\\DSL","parameters":[],"return_type":[{"name":"ValueType","namespace":"Flow\\PostgreSql\\Client\\Types","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/schema.php","start_line_in_file":1266,"slug":"value-type-uuid","name":"value_type_uuid","namespace":"Flow\\PostgreSql\\DSL","parameters":[],"return_type":[{"name":"ValueType","namespace":"Flow\\PostgreSql\\Client\\Types","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/schema.php","start_line_in_file":1272,"slug":"value-type-inet","name":"value_type_inet","namespace":"Flow\\PostgreSql\\DSL","parameters":[],"return_type":[{"name":"ValueType","namespace":"Flow\\PostgreSql\\Client\\Types","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/schema.php","start_line_in_file":1278,"slug":"value-type-cidr","name":"value_type_cidr","namespace":"Flow\\PostgreSql\\DSL","parameters":[],"return_type":[{"name":"ValueType","namespace":"Flow\\PostgreSql\\Client\\Types","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/schema.php","start_line_in_file":1284,"slug":"value-type-macaddr","name":"value_type_macaddr","namespace":"Flow\\PostgreSql\\DSL","parameters":[],"return_type":[{"name":"ValueType","namespace":"Flow\\PostgreSql\\Client\\Types","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/schema.php","start_line_in_file":1290,"slug":"value-type-macaddr8","name":"value_type_macaddr8","namespace":"Flow\\PostgreSql\\DSL","parameters":[],"return_type":[{"name":"ValueType","namespace":"Flow\\PostgreSql\\Client\\Types","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/schema.php","start_line_in_file":1296,"slug":"value-type-xml","name":"value_type_xml","namespace":"Flow\\PostgreSql\\DSL","parameters":[],"return_type":[{"name":"ValueType","namespace":"Flow\\PostgreSql\\Client\\Types","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/schema.php","start_line_in_file":1302,"slug":"value-type-oid","name":"value_type_oid","namespace":"Flow\\PostgreSql\\DSL","parameters":[],"return_type":[{"name":"ValueType","namespace":"Flow\\PostgreSql\\Client\\Types","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/schema.php","start_line_in_file":1308,"slug":"value-type-text-array","name":"value_type_text_array","namespace":"Flow\\PostgreSql\\DSL","parameters":[],"return_type":[{"name":"ValueType","namespace":"Flow\\PostgreSql\\Client\\Types","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/schema.php","start_line_in_file":1314,"slug":"value-type-varchar-array","name":"value_type_varchar_array","namespace":"Flow\\PostgreSql\\DSL","parameters":[],"return_type":[{"name":"ValueType","namespace":"Flow\\PostgreSql\\Client\\Types","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/schema.php","start_line_in_file":1320,"slug":"value-type-int2-array","name":"value_type_int2_array","namespace":"Flow\\PostgreSql\\DSL","parameters":[],"return_type":[{"name":"ValueType","namespace":"Flow\\PostgreSql\\Client\\Types","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/schema.php","start_line_in_file":1326,"slug":"value-type-int4-array","name":"value_type_int4_array","namespace":"Flow\\PostgreSql\\DSL","parameters":[],"return_type":[{"name":"ValueType","namespace":"Flow\\PostgreSql\\Client\\Types","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/schema.php","start_line_in_file":1332,"slug":"value-type-int8-array","name":"value_type_int8_array","namespace":"Flow\\PostgreSql\\DSL","parameters":[],"return_type":[{"name":"ValueType","namespace":"Flow\\PostgreSql\\Client\\Types","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/schema.php","start_line_in_file":1338,"slug":"value-type-float4-array","name":"value_type_float4_array","namespace":"Flow\\PostgreSql\\DSL","parameters":[],"return_type":[{"name":"ValueType","namespace":"Flow\\PostgreSql\\Client\\Types","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/schema.php","start_line_in_file":1344,"slug":"value-type-float8-array","name":"value_type_float8_array","namespace":"Flow\\PostgreSql\\DSL","parameters":[],"return_type":[{"name":"ValueType","namespace":"Flow\\PostgreSql\\Client\\Types","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/schema.php","start_line_in_file":1350,"slug":"value-type-bool-array","name":"value_type_bool_array","namespace":"Flow\\PostgreSql\\DSL","parameters":[],"return_type":[{"name":"ValueType","namespace":"Flow\\PostgreSql\\Client\\Types","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/schema.php","start_line_in_file":1356,"slug":"value-type-uuid-array","name":"value_type_uuid_array","namespace":"Flow\\PostgreSql\\DSL","parameters":[],"return_type":[{"name":"ValueType","namespace":"Flow\\PostgreSql\\Client\\Types","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/schema.php","start_line_in_file":1362,"slug":"value-type-json-array","name":"value_type_json_array","namespace":"Flow\\PostgreSql\\DSL","parameters":[],"return_type":[{"name":"ValueType","namespace":"Flow\\PostgreSql\\Client\\Types","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/schema.php","start_line_in_file":1368,"slug":"value-type-jsonb-array","name":"value_type_jsonb_array","namespace":"Flow\\PostgreSql\\DSL","parameters":[],"return_type":[{"name":"ValueType","namespace":"Flow\\PostgreSql\\Client\\Types","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/schema.php","start_line_in_file":1384,"slug":"schema","name":"schema","namespace":"Flow\\PostgreSql\\DSL","parameters":[{"name":"name","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"tables","type":[{"name":"array","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"[]"},{"name":"sequences","type":[{"name":"array","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"[]"},{"name":"views","type":[{"name":"array","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"[]"},{"name":"materializedViews","type":[{"name":"array","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"[]"},{"name":"functions","type":[{"name":"array","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"[]"},{"name":"procedures","type":[{"name":"array","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"[]"},{"name":"domains","type":[{"name":"array","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"[]"},{"name":"extensions","type":[{"name":"array","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"[]"}],"return_type":[{"name":"Schema","namespace":"Flow\\PostgreSql\\Schema","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIEBwYXJhbSBsaXN0PFNjaGVtYVRhYmxlPiAkdGFibGVzCiAqIEBwYXJhbSBsaXN0PFNjaGVtYVNlcXVlbmNlPiAkc2VxdWVuY2VzCiAqIEBwYXJhbSBsaXN0PFNjaGVtYVZpZXc+ICR2aWV3cwogKiBAcGFyYW0gbGlzdDxTY2hlbWFNYXRlcmlhbGl6ZWRWaWV3PiAkbWF0ZXJpYWxpemVkVmlld3MKICogQHBhcmFtIGxpc3Q8U2NoZW1hRnVuY3Rpb24+ICRmdW5jdGlvbnMKICogQHBhcmFtIGxpc3Q8U2NoZW1hUHJvY2VkdXJlPiAkcHJvY2VkdXJlcwogKiBAcGFyYW0gbGlzdDxTY2hlbWFEb21haW4+ICRkb21haW5zCiAqIEBwYXJhbSBsaXN0PFNjaGVtYUV4dGVuc2lvbj4gJGV4dGVuc2lvbnMKICov"},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/schema.php","start_line_in_file":1420,"slug":"schema-table","name":"schema_table","namespace":"Flow\\PostgreSql\\DSL","parameters":[{"name":"name","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"columns","type":[{"name":"array","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"primaryKey","type":[{"name":"PrimaryKey","namespace":"Flow\\PostgreSql\\Schema\\Constraint","is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"null"},{"name":"indexes","type":[{"name":"array","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"[]"},{"name":"foreignKeys","type":[{"name":"array","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"[]"},{"name":"uniqueConstraints","type":[{"name":"array","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"[]"},{"name":"checkConstraints","type":[{"name":"array","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"[]"},{"name":"excludeConstraints","type":[{"name":"array","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"[]"},{"name":"triggers","type":[{"name":"array","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"[]"},{"name":"schema","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"'public'"},{"name":"unlogged","type":[{"name":"bool","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"false"},{"name":"partitionStrategy","type":[{"name":"PartitionStrategy","namespace":"Flow\\PostgreSql\\Schema","is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"null"},{"name":"partitionColumns","type":[{"name":"array","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"[]"},{"name":"inherits","type":[{"name":"array","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"[]"},{"name":"tablespace","type":[{"name":"string","namespace":null,"is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"null"}],"return_type":[{"name":"Table","namespace":"Flow\\PostgreSql\\Schema","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIEBwYXJhbSBub24tZW1wdHktbGlzdDxTY2hlbWFDb2x1bW4+ICRjb2x1bW5zCiAqIEBwYXJhbSBsaXN0PFNjaGVtYUluZGV4PiAkaW5kZXhlcwogKiBAcGFyYW0gbGlzdDxTY2hlbWFGb3JlaWduS2V5PiAkZm9yZWlnbktleXMKICogQHBhcmFtIGxpc3Q8U2NoZW1hVW5pcXVlQ29uc3RyYWludD4gJHVuaXF1ZUNvbnN0cmFpbnRzCiAqIEBwYXJhbSBsaXN0PFNjaGVtYUNoZWNrQ29uc3RyYWludD4gJGNoZWNrQ29uc3RyYWludHMKICogQHBhcmFtIGxpc3Q8U2NoZW1hRXhjbHVkZUNvbnN0cmFpbnQ+ICRleGNsdWRlQ29uc3RyYWludHMKICogQHBhcmFtIGxpc3Q8U2NoZW1hVHJpZ2dlcj4gJHRyaWdnZXJzCiAqIEBwYXJhbSBsaXN0PHN0cmluZz4gJHBhcnRpdGlvbkNvbHVtbnMKICogQHBhcmFtIGxpc3Q8c3RyaW5nPiAkaW5oZXJpdHMKICov"},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/schema.php","start_line_in_file":1465,"slug":"schema-table-options","name":"schema_table_options","namespace":"Flow\\PostgreSql\\DSL","parameters":[{"name":"foreignKeys","type":[{"name":"array","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"[]"},{"name":"checkConstraints","type":[{"name":"array","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"[]"},{"name":"excludeConstraints","type":[{"name":"array","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"[]"},{"name":"triggers","type":[{"name":"array","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"[]"},{"name":"unlogged","type":[{"name":"bool","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"false"},{"name":"partitionStrategy","type":[{"name":"PartitionStrategy","namespace":"Flow\\PostgreSql\\Schema","is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"null"},{"name":"partitionColumns","type":[{"name":"array","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"[]"},{"name":"inherits","type":[{"name":"array","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"[]"},{"name":"tablespace","type":[{"name":"string","namespace":null,"is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"null"}],"return_type":[{"name":"TableOptions","namespace":"Flow\\PostgreSql\\Schema","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIEBwYXJhbSBsaXN0PFNjaGVtYUZvcmVpZ25LZXk+ICRmb3JlaWduS2V5cwogKiBAcGFyYW0gbGlzdDxTY2hlbWFDaGVja0NvbnN0cmFpbnQ+ICRjaGVja0NvbnN0cmFpbnRzCiAqIEBwYXJhbSBsaXN0PFNjaGVtYUV4Y2x1ZGVDb25zdHJhaW50PiAkZXhjbHVkZUNvbnN0cmFpbnRzCiAqIEBwYXJhbSBsaXN0PFNjaGVtYVRyaWdnZXI+ICR0cmlnZ2VycwogKiBAcGFyYW0gbGlzdDxzdHJpbmc+ICRwYXJ0aXRpb25Db2x1bW5zCiAqIEBwYXJhbSBsaXN0PHN0cmluZz4gJGluaGVyaXRzCiAqLw=="},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/schema.php","start_line_in_file":1490,"slug":"schema-column","name":"schema_column","namespace":"Flow\\PostgreSql\\DSL","parameters":[{"name":"name","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"type","type":[{"name":"ColumnType","namespace":"Flow\\PostgreSql\\QueryBuilder\\Schema","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"nullable","type":[{"name":"bool","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"true"},{"name":"default","type":[{"name":"Expression","namespace":"Flow\\PostgreSql\\QueryBuilder\\Expression","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false},{"name":"int","namespace":null,"is_nullable":false,"is_variadic":false},{"name":"float","namespace":null,"is_nullable":false,"is_variadic":false},{"name":"bool","namespace":null,"is_nullable":false,"is_variadic":false},{"name":"null","namespace":null,"is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"null"},{"name":"isIdentity","type":[{"name":"bool","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"false"},{"name":"identityGeneration","type":[{"name":"IdentityGeneration","namespace":"Flow\\PostgreSql\\Schema","is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"null"},{"name":"isGenerated","type":[{"name":"bool","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"false"},{"name":"generationExpression","type":[{"name":"string","namespace":null,"is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"null"},{"name":"ordinalPosition","type":[{"name":"int","namespace":null,"is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"null"}],"return_type":[{"name":"Column","namespace":"Flow\\PostgreSql\\Schema","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/schema.php","start_line_in_file":1515,"slug":"schema-column-integer","name":"schema_column_integer","namespace":"Flow\\PostgreSql\\DSL","parameters":[{"name":"name","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"nullable","type":[{"name":"bool","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"true"},{"name":"default","type":[{"name":"Expression","namespace":"Flow\\PostgreSql\\QueryBuilder\\Expression","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false},{"name":"int","namespace":null,"is_nullable":false,"is_variadic":false},{"name":"float","namespace":null,"is_nullable":false,"is_variadic":false},{"name":"bool","namespace":null,"is_nullable":false,"is_variadic":false},{"name":"null","namespace":null,"is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"null"}],"return_type":[{"name":"Column","namespace":"Flow\\PostgreSql\\Schema","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/schema.php","start_line_in_file":1524,"slug":"schema-column-smallint","name":"schema_column_smallint","namespace":"Flow\\PostgreSql\\DSL","parameters":[{"name":"name","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"nullable","type":[{"name":"bool","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"true"},{"name":"default","type":[{"name":"Expression","namespace":"Flow\\PostgreSql\\QueryBuilder\\Expression","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false},{"name":"int","namespace":null,"is_nullable":false,"is_variadic":false},{"name":"float","namespace":null,"is_nullable":false,"is_variadic":false},{"name":"bool","namespace":null,"is_nullable":false,"is_variadic":false},{"name":"null","namespace":null,"is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"null"}],"return_type":[{"name":"Column","namespace":"Flow\\PostgreSql\\Schema","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/schema.php","start_line_in_file":1533,"slug":"schema-column-bigint","name":"schema_column_bigint","namespace":"Flow\\PostgreSql\\DSL","parameters":[{"name":"name","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"nullable","type":[{"name":"bool","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"true"},{"name":"default","type":[{"name":"Expression","namespace":"Flow\\PostgreSql\\QueryBuilder\\Expression","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false},{"name":"int","namespace":null,"is_nullable":false,"is_variadic":false},{"name":"float","namespace":null,"is_nullable":false,"is_variadic":false},{"name":"bool","namespace":null,"is_nullable":false,"is_variadic":false},{"name":"null","namespace":null,"is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"null"}],"return_type":[{"name":"Column","namespace":"Flow\\PostgreSql\\Schema","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/schema.php","start_line_in_file":1542,"slug":"schema-column-serial","name":"schema_column_serial","namespace":"Flow\\PostgreSql\\DSL","parameters":[{"name":"name","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"Column","namespace":"Flow\\PostgreSql\\Schema","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/schema.php","start_line_in_file":1548,"slug":"schema-column-small-serial","name":"schema_column_small_serial","namespace":"Flow\\PostgreSql\\DSL","parameters":[{"name":"name","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"Column","namespace":"Flow\\PostgreSql\\Schema","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/schema.php","start_line_in_file":1554,"slug":"schema-column-big-serial","name":"schema_column_big_serial","namespace":"Flow\\PostgreSql\\DSL","parameters":[{"name":"name","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"Column","namespace":"Flow\\PostgreSql\\Schema","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/schema.php","start_line_in_file":1560,"slug":"schema-column-boolean","name":"schema_column_boolean","namespace":"Flow\\PostgreSql\\DSL","parameters":[{"name":"name","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"nullable","type":[{"name":"bool","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"true"},{"name":"default","type":[{"name":"Expression","namespace":"Flow\\PostgreSql\\QueryBuilder\\Expression","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false},{"name":"int","namespace":null,"is_nullable":false,"is_variadic":false},{"name":"float","namespace":null,"is_nullable":false,"is_variadic":false},{"name":"bool","namespace":null,"is_nullable":false,"is_variadic":false},{"name":"null","namespace":null,"is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"null"}],"return_type":[{"name":"Column","namespace":"Flow\\PostgreSql\\Schema","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/schema.php","start_line_in_file":1569,"slug":"schema-column-text","name":"schema_column_text","namespace":"Flow\\PostgreSql\\DSL","parameters":[{"name":"name","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"nullable","type":[{"name":"bool","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"true"},{"name":"default","type":[{"name":"Expression","namespace":"Flow\\PostgreSql\\QueryBuilder\\Expression","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false},{"name":"int","namespace":null,"is_nullable":false,"is_variadic":false},{"name":"float","namespace":null,"is_nullable":false,"is_variadic":false},{"name":"bool","namespace":null,"is_nullable":false,"is_variadic":false},{"name":"null","namespace":null,"is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"null"}],"return_type":[{"name":"Column","namespace":"Flow\\PostgreSql\\Schema","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/schema.php","start_line_in_file":1578,"slug":"schema-column-varchar","name":"schema_column_varchar","namespace":"Flow\\PostgreSql\\DSL","parameters":[{"name":"name","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"length","type":[{"name":"int","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"nullable","type":[{"name":"bool","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"true"},{"name":"default","type":[{"name":"Expression","namespace":"Flow\\PostgreSql\\QueryBuilder\\Expression","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false},{"name":"int","namespace":null,"is_nullable":false,"is_variadic":false},{"name":"float","namespace":null,"is_nullable":false,"is_variadic":false},{"name":"bool","namespace":null,"is_nullable":false,"is_variadic":false},{"name":"null","namespace":null,"is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"null"}],"return_type":[{"name":"Column","namespace":"Flow\\PostgreSql\\Schema","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/schema.php","start_line_in_file":1588,"slug":"schema-column-char","name":"schema_column_char","namespace":"Flow\\PostgreSql\\DSL","parameters":[{"name":"name","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"length","type":[{"name":"int","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"nullable","type":[{"name":"bool","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"true"},{"name":"default","type":[{"name":"Expression","namespace":"Flow\\PostgreSql\\QueryBuilder\\Expression","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false},{"name":"int","namespace":null,"is_nullable":false,"is_variadic":false},{"name":"float","namespace":null,"is_nullable":false,"is_variadic":false},{"name":"bool","namespace":null,"is_nullable":false,"is_variadic":false},{"name":"null","namespace":null,"is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"null"}],"return_type":[{"name":"Column","namespace":"Flow\\PostgreSql\\Schema","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/schema.php","start_line_in_file":1598,"slug":"schema-column-numeric","name":"schema_column_numeric","namespace":"Flow\\PostgreSql\\DSL","parameters":[{"name":"name","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"precision","type":[{"name":"int","namespace":null,"is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"null"},{"name":"scale","type":[{"name":"int","namespace":null,"is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"null"},{"name":"nullable","type":[{"name":"bool","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"true"},{"name":"default","type":[{"name":"Expression","namespace":"Flow\\PostgreSql\\QueryBuilder\\Expression","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false},{"name":"int","namespace":null,"is_nullable":false,"is_variadic":false},{"name":"float","namespace":null,"is_nullable":false,"is_variadic":false},{"name":"bool","namespace":null,"is_nullable":false,"is_variadic":false},{"name":"null","namespace":null,"is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"null"}],"return_type":[{"name":"Column","namespace":"Flow\\PostgreSql\\Schema","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/schema.php","start_line_in_file":1609,"slug":"schema-column-real","name":"schema_column_real","namespace":"Flow\\PostgreSql\\DSL","parameters":[{"name":"name","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"nullable","type":[{"name":"bool","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"true"},{"name":"default","type":[{"name":"Expression","namespace":"Flow\\PostgreSql\\QueryBuilder\\Expression","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false},{"name":"int","namespace":null,"is_nullable":false,"is_variadic":false},{"name":"float","namespace":null,"is_nullable":false,"is_variadic":false},{"name":"bool","namespace":null,"is_nullable":false,"is_variadic":false},{"name":"null","namespace":null,"is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"null"}],"return_type":[{"name":"Column","namespace":"Flow\\PostgreSql\\Schema","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/schema.php","start_line_in_file":1618,"slug":"schema-column-double-precision","name":"schema_column_double_precision","namespace":"Flow\\PostgreSql\\DSL","parameters":[{"name":"name","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"nullable","type":[{"name":"bool","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"true"},{"name":"default","type":[{"name":"Expression","namespace":"Flow\\PostgreSql\\QueryBuilder\\Expression","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false},{"name":"int","namespace":null,"is_nullable":false,"is_variadic":false},{"name":"float","namespace":null,"is_nullable":false,"is_variadic":false},{"name":"bool","namespace":null,"is_nullable":false,"is_variadic":false},{"name":"null","namespace":null,"is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"null"}],"return_type":[{"name":"Column","namespace":"Flow\\PostgreSql\\Schema","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/schema.php","start_line_in_file":1627,"slug":"schema-column-date","name":"schema_column_date","namespace":"Flow\\PostgreSql\\DSL","parameters":[{"name":"name","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"nullable","type":[{"name":"bool","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"true"},{"name":"default","type":[{"name":"Expression","namespace":"Flow\\PostgreSql\\QueryBuilder\\Expression","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false},{"name":"int","namespace":null,"is_nullable":false,"is_variadic":false},{"name":"float","namespace":null,"is_nullable":false,"is_variadic":false},{"name":"bool","namespace":null,"is_nullable":false,"is_variadic":false},{"name":"null","namespace":null,"is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"null"}],"return_type":[{"name":"Column","namespace":"Flow\\PostgreSql\\Schema","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/schema.php","start_line_in_file":1636,"slug":"schema-column-time","name":"schema_column_time","namespace":"Flow\\PostgreSql\\DSL","parameters":[{"name":"name","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"precision","type":[{"name":"int","namespace":null,"is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"null"},{"name":"nullable","type":[{"name":"bool","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"true"},{"name":"default","type":[{"name":"Expression","namespace":"Flow\\PostgreSql\\QueryBuilder\\Expression","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false},{"name":"int","namespace":null,"is_nullable":false,"is_variadic":false},{"name":"float","namespace":null,"is_nullable":false,"is_variadic":false},{"name":"bool","namespace":null,"is_nullable":false,"is_variadic":false},{"name":"null","namespace":null,"is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"null"}],"return_type":[{"name":"Column","namespace":"Flow\\PostgreSql\\Schema","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/schema.php","start_line_in_file":1646,"slug":"schema-column-timestamp","name":"schema_column_timestamp","namespace":"Flow\\PostgreSql\\DSL","parameters":[{"name":"name","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"precision","type":[{"name":"int","namespace":null,"is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"null"},{"name":"nullable","type":[{"name":"bool","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"true"},{"name":"default","type":[{"name":"Expression","namespace":"Flow\\PostgreSql\\QueryBuilder\\Expression","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false},{"name":"int","namespace":null,"is_nullable":false,"is_variadic":false},{"name":"float","namespace":null,"is_nullable":false,"is_variadic":false},{"name":"bool","namespace":null,"is_nullable":false,"is_variadic":false},{"name":"null","namespace":null,"is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"null"}],"return_type":[{"name":"Column","namespace":"Flow\\PostgreSql\\Schema","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/schema.php","start_line_in_file":1656,"slug":"schema-column-timestamp-tz","name":"schema_column_timestamp_tz","namespace":"Flow\\PostgreSql\\DSL","parameters":[{"name":"name","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"precision","type":[{"name":"int","namespace":null,"is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"null"},{"name":"nullable","type":[{"name":"bool","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"true"},{"name":"default","type":[{"name":"Expression","namespace":"Flow\\PostgreSql\\QueryBuilder\\Expression","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false},{"name":"int","namespace":null,"is_nullable":false,"is_variadic":false},{"name":"float","namespace":null,"is_nullable":false,"is_variadic":false},{"name":"bool","namespace":null,"is_nullable":false,"is_variadic":false},{"name":"null","namespace":null,"is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"null"}],"return_type":[{"name":"Column","namespace":"Flow\\PostgreSql\\Schema","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/schema.php","start_line_in_file":1666,"slug":"schema-column-interval","name":"schema_column_interval","namespace":"Flow\\PostgreSql\\DSL","parameters":[{"name":"name","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"nullable","type":[{"name":"bool","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"true"},{"name":"default","type":[{"name":"Expression","namespace":"Flow\\PostgreSql\\QueryBuilder\\Expression","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false},{"name":"int","namespace":null,"is_nullable":false,"is_variadic":false},{"name":"float","namespace":null,"is_nullable":false,"is_variadic":false},{"name":"bool","namespace":null,"is_nullable":false,"is_variadic":false},{"name":"null","namespace":null,"is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"null"}],"return_type":[{"name":"Column","namespace":"Flow\\PostgreSql\\Schema","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/schema.php","start_line_in_file":1675,"slug":"schema-column-uuid","name":"schema_column_uuid","namespace":"Flow\\PostgreSql\\DSL","parameters":[{"name":"name","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"nullable","type":[{"name":"bool","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"true"},{"name":"default","type":[{"name":"Expression","namespace":"Flow\\PostgreSql\\QueryBuilder\\Expression","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false},{"name":"int","namespace":null,"is_nullable":false,"is_variadic":false},{"name":"float","namespace":null,"is_nullable":false,"is_variadic":false},{"name":"bool","namespace":null,"is_nullable":false,"is_variadic":false},{"name":"null","namespace":null,"is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"null"}],"return_type":[{"name":"Column","namespace":"Flow\\PostgreSql\\Schema","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/schema.php","start_line_in_file":1684,"slug":"schema-column-json","name":"schema_column_json","namespace":"Flow\\PostgreSql\\DSL","parameters":[{"name":"name","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"nullable","type":[{"name":"bool","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"true"},{"name":"default","type":[{"name":"Expression","namespace":"Flow\\PostgreSql\\QueryBuilder\\Expression","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false},{"name":"int","namespace":null,"is_nullable":false,"is_variadic":false},{"name":"float","namespace":null,"is_nullable":false,"is_variadic":false},{"name":"bool","namespace":null,"is_nullable":false,"is_variadic":false},{"name":"null","namespace":null,"is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"null"}],"return_type":[{"name":"Column","namespace":"Flow\\PostgreSql\\Schema","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/schema.php","start_line_in_file":1693,"slug":"schema-column-jsonb","name":"schema_column_jsonb","namespace":"Flow\\PostgreSql\\DSL","parameters":[{"name":"name","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"nullable","type":[{"name":"bool","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"true"},{"name":"default","type":[{"name":"Expression","namespace":"Flow\\PostgreSql\\QueryBuilder\\Expression","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false},{"name":"int","namespace":null,"is_nullable":false,"is_variadic":false},{"name":"float","namespace":null,"is_nullable":false,"is_variadic":false},{"name":"bool","namespace":null,"is_nullable":false,"is_variadic":false},{"name":"null","namespace":null,"is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"null"}],"return_type":[{"name":"Column","namespace":"Flow\\PostgreSql\\Schema","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/schema.php","start_line_in_file":1702,"slug":"schema-column-bytea","name":"schema_column_bytea","namespace":"Flow\\PostgreSql\\DSL","parameters":[{"name":"name","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"nullable","type":[{"name":"bool","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"true"},{"name":"default","type":[{"name":"Expression","namespace":"Flow\\PostgreSql\\QueryBuilder\\Expression","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false},{"name":"int","namespace":null,"is_nullable":false,"is_variadic":false},{"name":"float","namespace":null,"is_nullable":false,"is_variadic":false},{"name":"bool","namespace":null,"is_nullable":false,"is_variadic":false},{"name":"null","namespace":null,"is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"null"}],"return_type":[{"name":"Column","namespace":"Flow\\PostgreSql\\Schema","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/schema.php","start_line_in_file":1711,"slug":"schema-column-inet","name":"schema_column_inet","namespace":"Flow\\PostgreSql\\DSL","parameters":[{"name":"name","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"nullable","type":[{"name":"bool","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"true"},{"name":"default","type":[{"name":"Expression","namespace":"Flow\\PostgreSql\\QueryBuilder\\Expression","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false},{"name":"int","namespace":null,"is_nullable":false,"is_variadic":false},{"name":"float","namespace":null,"is_nullable":false,"is_variadic":false},{"name":"bool","namespace":null,"is_nullable":false,"is_variadic":false},{"name":"null","namespace":null,"is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"null"}],"return_type":[{"name":"Column","namespace":"Flow\\PostgreSql\\Schema","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/schema.php","start_line_in_file":1720,"slug":"schema-column-cidr","name":"schema_column_cidr","namespace":"Flow\\PostgreSql\\DSL","parameters":[{"name":"name","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"nullable","type":[{"name":"bool","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"true"},{"name":"default","type":[{"name":"Expression","namespace":"Flow\\PostgreSql\\QueryBuilder\\Expression","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false},{"name":"int","namespace":null,"is_nullable":false,"is_variadic":false},{"name":"float","namespace":null,"is_nullable":false,"is_variadic":false},{"name":"bool","namespace":null,"is_nullable":false,"is_variadic":false},{"name":"null","namespace":null,"is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"null"}],"return_type":[{"name":"Column","namespace":"Flow\\PostgreSql\\Schema","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/schema.php","start_line_in_file":1729,"slug":"schema-column-macaddr","name":"schema_column_macaddr","namespace":"Flow\\PostgreSql\\DSL","parameters":[{"name":"name","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"nullable","type":[{"name":"bool","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"true"},{"name":"default","type":[{"name":"Expression","namespace":"Flow\\PostgreSql\\QueryBuilder\\Expression","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false},{"name":"int","namespace":null,"is_nullable":false,"is_variadic":false},{"name":"float","namespace":null,"is_nullable":false,"is_variadic":false},{"name":"bool","namespace":null,"is_nullable":false,"is_variadic":false},{"name":"null","namespace":null,"is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"null"}],"return_type":[{"name":"Column","namespace":"Flow\\PostgreSql\\Schema","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/schema.php","start_line_in_file":1741,"slug":"schema-primary-key","name":"schema_primary_key","namespace":"Flow\\PostgreSql\\DSL","parameters":[{"name":"columns","type":[{"name":"array","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"name","type":[{"name":"string","namespace":null,"is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"null"}],"return_type":[{"name":"PrimaryKey","namespace":"Flow\\PostgreSql\\Schema\\Constraint","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIEBwYXJhbSBub24tZW1wdHktbGlzdDxzdHJpbmc+ICRjb2x1bW5zCiAqLw=="},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/schema.php","start_line_in_file":1751,"slug":"schema-foreign-key","name":"schema_foreign_key","namespace":"Flow\\PostgreSql\\DSL","parameters":[{"name":"columns","type":[{"name":"array","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"referenceTable","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"referenceColumns","type":[{"name":"array","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"name","type":[{"name":"string","namespace":null,"is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"null"},{"name":"referenceSchema","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"'public'"},{"name":"onUpdate","type":[{"name":"ReferentialAction","namespace":"Flow\\PostgreSql\\QueryBuilder\\Schema","is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"Flow\\PostgreSql\\QueryBuilder\\Schema\\ReferentialAction::..."},{"name":"onDelete","type":[{"name":"ReferentialAction","namespace":"Flow\\PostgreSql\\QueryBuilder\\Schema","is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"Flow\\PostgreSql\\QueryBuilder\\Schema\\ReferentialAction::..."},{"name":"deferrable","type":[{"name":"bool","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"false"},{"name":"initiallyDeferred","type":[{"name":"bool","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"false"}],"return_type":[{"name":"ForeignKey","namespace":"Flow\\PostgreSql\\Schema\\Constraint","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIEBwYXJhbSBub24tZW1wdHktbGlzdDxzdHJpbmc+ICRjb2x1bW5zCiAqIEBwYXJhbSBub24tZW1wdHktbGlzdDxzdHJpbmc+ICRyZWZlcmVuY2VDb2x1bW5zCiAqLw=="},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/schema.php","start_line_in_file":1779,"slug":"schema-unique","name":"schema_unique","namespace":"Flow\\PostgreSql\\DSL","parameters":[{"name":"columns","type":[{"name":"array","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"name","type":[{"name":"string","namespace":null,"is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"null"},{"name":"nullsNotDistinct","type":[{"name":"bool","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"false"}],"return_type":[{"name":"UniqueConstraint","namespace":"Flow\\PostgreSql\\Schema\\Constraint","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIEBwYXJhbSBub24tZW1wdHktbGlzdDxzdHJpbmc+ICRjb2x1bW5zCiAqLw=="},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/schema.php","start_line_in_file":1785,"slug":"schema-check","name":"schema_check","namespace":"Flow\\PostgreSql\\DSL","parameters":[{"name":"expression","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"name","type":[{"name":"string","namespace":null,"is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"null"},{"name":"noInherit","type":[{"name":"bool","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"false"}],"return_type":[{"name":"CheckConstraint","namespace":"Flow\\PostgreSql\\Schema\\Constraint","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/schema.php","start_line_in_file":1791,"slug":"schema-exclude","name":"schema_exclude","namespace":"Flow\\PostgreSql\\DSL","parameters":[{"name":"definition","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"name","type":[{"name":"string","namespace":null,"is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"null"}],"return_type":[{"name":"ExcludeConstraint","namespace":"Flow\\PostgreSql\\Schema\\Constraint","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/schema.php","start_line_in_file":1800,"slug":"schema-index","name":"schema_index","namespace":"Flow\\PostgreSql\\DSL","parameters":[{"name":"name","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"columns","type":[{"name":"array","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"unique","type":[{"name":"bool","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"false"},{"name":"method","type":[{"name":"IndexMethod","namespace":"Flow\\PostgreSql\\Schema","is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"Flow\\PostgreSql\\Schema\\IndexMethod::..."},{"name":"primary","type":[{"name":"bool","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"false"},{"name":"predicate","type":[{"name":"string","namespace":null,"is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"null"}],"return_type":[{"name":"Index","namespace":"Flow\\PostgreSql\\Schema","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIEBwYXJhbSBub24tZW1wdHktbGlzdDxzdHJpbmc+ICRjb2x1bW5zCiAqLw=="},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/schema.php","start_line_in_file":1812,"slug":"schema-sequence","name":"schema_sequence","namespace":"Flow\\PostgreSql\\DSL","parameters":[{"name":"name","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"dataType","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"'bigint'"},{"name":"startValue","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false},{"name":"int","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"1"},{"name":"minValue","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false},{"name":"int","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"1"},{"name":"maxValue","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false},{"name":"int","namespace":null,"is_nullable":false,"is_variadic":false},{"name":"null","namespace":null,"is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"null"},{"name":"incrementBy","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false},{"name":"int","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"1"},{"name":"cycle","type":[{"name":"bool","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"false"},{"name":"cacheValue","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false},{"name":"int","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"1"},{"name":"ownedByTable","type":[{"name":"string","namespace":null,"is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"null"},{"name":"ownedByColumn","type":[{"name":"string","namespace":null,"is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"null"}],"return_type":[{"name":"Sequence","namespace":"Flow\\PostgreSql\\Schema","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/schema.php","start_line_in_file":1839,"slug":"schema-view","name":"schema_view","namespace":"Flow\\PostgreSql\\DSL","parameters":[{"name":"name","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"definition","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"isUpdatable","type":[{"name":"bool","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"false"}],"return_type":[{"name":"View","namespace":"Flow\\PostgreSql\\Schema","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/schema.php","start_line_in_file":1848,"slug":"schema-materialized-view","name":"schema_materialized_view","namespace":"Flow\\PostgreSql\\DSL","parameters":[{"name":"name","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"definition","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"indexes","type":[{"name":"array","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"[]"}],"return_type":[{"name":"MaterializedView","namespace":"Flow\\PostgreSql\\Schema","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIEBwYXJhbSBsaXN0PFNjaGVtYUluZGV4PiAkaW5kZXhlcwogKi8="},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/schema.php","start_line_in_file":1857,"slug":"schema-function","name":"schema_function","namespace":"Flow\\PostgreSql\\DSL","parameters":[{"name":"name","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"returnType","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"argumentTypes","type":[{"name":"array","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"[]"},{"name":"language","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"'sql'"},{"name":"definition","type":[{"name":"string","namespace":null,"is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"null"},{"name":"isStrict","type":[{"name":"bool","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"false"},{"name":"volatility","type":[{"name":"FunctionVolatility","namespace":"Flow\\PostgreSql\\Schema","is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"null"}],"return_type":[{"name":"Func","namespace":"Flow\\PostgreSql\\Schema","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIEBwYXJhbSBsaXN0PHN0cmluZz4gJGFyZ3VtZW50VHlwZXMKICov"},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/schema.php","start_line_in_file":1873,"slug":"schema-procedure","name":"schema_procedure","namespace":"Flow\\PostgreSql\\DSL","parameters":[{"name":"name","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"argumentTypes","type":[{"name":"array","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"[]"},{"name":"language","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"'sql'"},{"name":"definition","type":[{"name":"string","namespace":null,"is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"null"}],"return_type":[{"name":"Procedure","namespace":"Flow\\PostgreSql\\Schema","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIEBwYXJhbSBsaXN0PHN0cmluZz4gJGFyZ3VtZW50VHlwZXMKICov"},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/schema.php","start_line_in_file":1886,"slug":"schema-trigger","name":"schema_trigger","namespace":"Flow\\PostgreSql\\DSL","parameters":[{"name":"name","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"tableName","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"timing","type":[{"name":"TriggerTiming","namespace":"Flow\\PostgreSql\\Schema","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"events","type":[{"name":"array","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"functionName","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"forEachRow","type":[{"name":"bool","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"false"},{"name":"whenCondition","type":[{"name":"string","namespace":null,"is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"null"}],"return_type":[{"name":"Trigger","namespace":"Flow\\PostgreSql\\Schema","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIEBwYXJhbSBub24tZW1wdHktbGlzdDxUcmlnZ2VyRXZlbnQ+ICRldmVudHMKICov"},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/schema.php","start_line_in_file":1902,"slug":"schema-domain","name":"schema_domain","namespace":"Flow\\PostgreSql\\DSL","parameters":[{"name":"name","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"baseType","type":[{"name":"ColumnType","namespace":"Flow\\PostgreSql\\QueryBuilder\\Schema","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"nullable","type":[{"name":"bool","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"true"},{"name":"default","type":[{"name":"Expression","namespace":"Flow\\PostgreSql\\QueryBuilder\\Expression","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false},{"name":"int","namespace":null,"is_nullable":false,"is_variadic":false},{"name":"float","namespace":null,"is_nullable":false,"is_variadic":false},{"name":"bool","namespace":null,"is_nullable":false,"is_variadic":false},{"name":"null","namespace":null,"is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"null"},{"name":"checkConstraints","type":[{"name":"array","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"[]"}],"return_type":[{"name":"Domain","namespace":"Flow\\PostgreSql\\Schema","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIEBwYXJhbSBsaXN0PFNjaGVtYUNoZWNrQ29uc3RyYWludD4gJGNoZWNrQ29uc3RyYWludHMKICov"},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/schema.php","start_line_in_file":1913,"slug":"schema-extension","name":"schema_extension","namespace":"Flow\\PostgreSql\\DSL","parameters":[{"name":"name","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"version","type":[{"name":"string","namespace":null,"is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"null"}],"return_type":[{"name":"Extension","namespace":"Flow\\PostgreSql\\Schema","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/schema.php","start_line_in_file":1922,"slug":"client-catalog-provider","name":"client_catalog_provider","namespace":"Flow\\PostgreSql\\DSL","parameters":[{"name":"client","type":[{"name":"Client","namespace":"Flow\\PostgreSql\\Client","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"schemaNames","type":[{"name":"array","namespace":null,"is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"null"},{"name":"exclusionPolicy","type":[{"name":"ExclusionPolicy","namespace":"Flow\\PostgreSql\\Schema\\Exclusion","is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"null"}],"return_type":[{"name":"CatalogProvider","namespace":"Flow\\PostgreSql\\Schema","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIEBwYXJhbSA\/bGlzdDxzdHJpbmc+ICRzY2hlbWFOYW1lcwogKi8="},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/schema.php","start_line_in_file":1931,"slug":"exclude-any","name":"exclude_any","namespace":"Flow\\PostgreSql\\DSL","parameters":[{"name":"policies","type":[{"name":"ExclusionPolicy","namespace":"Flow\\PostgreSql\\Schema\\Exclusion","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":true,"default_value":null}],"return_type":[{"name":"ExclusionPolicy","namespace":"Flow\\PostgreSql\\Schema\\Exclusion","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/schema.php","start_line_in_file":1937,"slug":"exclude-exact","name":"exclude_exact","namespace":"Flow\\PostgreSql\\DSL","parameters":[{"name":"name","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"ExclusionPolicy","namespace":"Flow\\PostgreSql\\Schema\\Exclusion","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/schema.php","start_line_in_file":1943,"slug":"exclude-starts-with","name":"exclude_starts_with","namespace":"Flow\\PostgreSql\\DSL","parameters":[{"name":"prefix","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"ExclusionPolicy","namespace":"Flow\\PostgreSql\\Schema\\Exclusion","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/schema.php","start_line_in_file":1949,"slug":"exclude-ends-with","name":"exclude_ends_with","namespace":"Flow\\PostgreSql\\DSL","parameters":[{"name":"suffix","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"ExclusionPolicy","namespace":"Flow\\PostgreSql\\Schema\\Exclusion","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/schema.php","start_line_in_file":1955,"slug":"exclude-pattern","name":"exclude_pattern","namespace":"Flow\\PostgreSql\\DSL","parameters":[{"name":"pattern","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"ExclusionPolicy","namespace":"Flow\\PostgreSql\\Schema\\Exclusion","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/schema.php","start_line_in_file":1961,"slug":"exclude-schema","name":"exclude_schema","namespace":"Flow\\PostgreSql\\DSL","parameters":[{"name":"schema","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"ExclusionPolicy","namespace":"Flow\\PostgreSql\\Schema\\Exclusion","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/schema.php","start_line_in_file":1967,"slug":"exclude-scoped","name":"exclude_scoped","namespace":"Flow\\PostgreSql\\DSL","parameters":[{"name":"policy","type":[{"name":"ExclusionPolicy","namespace":"Flow\\PostgreSql\\Schema\\Exclusion","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"type","type":[{"name":"SchemaObjectType","namespace":"Flow\\PostgreSql\\Schema\\Exclusion","is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"null"},{"name":"schema","type":[{"name":"string","namespace":null,"is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"null"}],"return_type":[{"name":"ExclusionPolicy","namespace":"Flow\\PostgreSql\\Schema\\Exclusion","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/schema.php","start_line_in_file":1976,"slug":"manual-catalog-provider","name":"manual_catalog_provider","namespace":"Flow\\PostgreSql\\DSL","parameters":[{"name":"catalog","type":[{"name":"Catalog","namespace":"Flow\\PostgreSql\\Schema","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"CatalogProvider","namespace":"Flow\\PostgreSql\\Schema","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/schema.php","start_line_in_file":1982,"slug":"chain-catalog-provider","name":"chain_catalog_provider","namespace":"Flow\\PostgreSql\\DSL","parameters":[{"name":"providers","type":[{"name":"CatalogProvider","namespace":"Flow\\PostgreSql\\Schema","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":true,"default_value":null}],"return_type":[{"name":"ChainCatalogProvider","namespace":"Flow\\PostgreSql\\Schema","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/schema.php","start_line_in_file":1991,"slug":"catalog-comparator","name":"catalog_comparator","namespace":"Flow\\PostgreSql\\DSL","parameters":[{"name":"renameStrategy","type":[{"name":"RenameStrategy","namespace":"Flow\\PostgreSql\\Schema\\Diff","is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"null"},{"name":"viewDependencyResolver","type":[{"name":"ViewDependencyResolver","namespace":"Flow\\PostgreSql\\Schema\\Diff","is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"null"},{"name":"tableOrderStrategy","type":[{"name":"ExecutionOrderStrategy","namespace":"Flow\\PostgreSql\\Schema","is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"null"},{"name":"dropIfExists","type":[{"name":"bool","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"false"}],"return_type":[{"name":"CatalogComparator","namespace":"Flow\\PostgreSql\\Schema\\Diff","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIEBwYXJhbSBudWxsfEV4ZWN1dGlvbk9yZGVyU3RyYXRlZ3k8XEZsb3dcUG9zdGdyZVNxbFxTY2hlbWFcVGFibGU+ICR0YWJsZU9yZGVyU3RyYXRlZ3kKICov"},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/schema.php","start_line_in_file":2006,"slug":"ast-view-dependency-resolver","name":"ast_view_dependency_resolver","namespace":"Flow\\PostgreSql\\DSL","parameters":[],"return_type":[{"name":"AstViewDependencyResolver","namespace":"Flow\\PostgreSql\\Schema\\Diff","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/schema.php","start_line_in_file":2012,"slug":"noop-view-dependency-resolver","name":"noop_view_dependency_resolver","namespace":"Flow\\PostgreSql\\DSL","parameters":[],"return_type":[{"name":"NoopViewDependencyResolver","namespace":"Flow\\PostgreSql\\Schema\\Diff","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/schema.php","start_line_in_file":2018,"slug":"foreign-key-dependency-order","name":"foreign_key_dependency_order","namespace":"Flow\\PostgreSql\\DSL","parameters":[],"return_type":[{"name":"ForeignKeyDependencyOrder","namespace":"Flow\\PostgreSql\\Schema","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/schema.php","start_line_in_file":2027,"slug":"no-execution-order","name":"no_execution_order","namespace":"Flow\\PostgreSql\\DSL","parameters":[],"return_type":[{"name":"NoExecutionOrder","namespace":"Flow\\PostgreSql\\Schema","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIEByZXR1cm4gTm9FeGVjdXRpb25PcmRlcjxtaXhlZD4KICov"},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/schema.php","start_line_in_file":2033,"slug":"view-dependency-order","name":"view_dependency_order","namespace":"Flow\\PostgreSql\\DSL","parameters":[],"return_type":[{"name":"ViewDependencyOrder","namespace":"Flow\\PostgreSql\\Schema","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/schema.php","start_line_in_file":2039,"slug":"materialized-view-dependency-order","name":"materialized_view_dependency_order","namespace":"Flow\\PostgreSql\\DSL","parameters":[],"return_type":[{"name":"MaterializedViewDependencyOrder","namespace":"Flow\\PostgreSql\\Schema","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/query.php","start_line_in_file":117,"slug":"select","name":"select","namespace":"Flow\\PostgreSql\\DSL","parameters":[{"name":"expressions","type":[{"name":"Expression","namespace":"Flow\\PostgreSql\\QueryBuilder\\Expression","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":true,"default_value":null}],"return_type":[{"name":"SelectBuilder","namespace":"Flow\\PostgreSql\\QueryBuilder\\Select","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBhIG5ldyBTRUxFQ1QgcXVlcnkgYnVpbGRlci4KICoKICogQHBhcmFtIEV4cHJlc3Npb258c3RyaW5nIC4uLiRleHByZXNzaW9ucyBDb2x1bW5zIHRvIHNlbGVjdC4gSWYgZW1wdHksIHJldHVybnMgU2VsZWN0U2VsZWN0U3RlcC4KICov"},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/query.php","start_line_in_file":134,"slug":"parsed-select","name":"parsed_select","namespace":"Flow\\PostgreSql\\DSL","parameters":[{"name":"sql","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"ParsedSelect","namespace":"Flow\\PostgreSql\\QueryBuilder\\Select","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBhIFNlbGVjdEZpbmFsU3RlcCBmcm9tIGEgcmF3IFNRTCBTRUxFQ1Qgc3RyaW5nLgogKi8="},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/query.php","start_line_in_file":146,"slug":"with","name":"with","namespace":"Flow\\PostgreSql\\DSL","parameters":[{"name":"ctes","type":[{"name":"CTE","namespace":"Flow\\PostgreSql\\QueryBuilder\\Clause","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":true,"default_value":null}],"return_type":[{"name":"WithBuilder","namespace":"Flow\\PostgreSql\\QueryBuilder\\With","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBhIFdJVEggY2xhdXNlIGJ1aWxkZXIgZm9yIENURXMuCiAqCiAqIEV4YW1wbGU6IHdpdGgoY3RlKCd1c2VycycsICRzdWJxdWVyeSkpLT5zZWxlY3Qoc3RhcigpKS0+ZnJvbSh0YWJsZSgndXNlcnMnKSkKICogRXhhbXBsZTogd2l0aChjdGUoJ2EnLCAkcTEpLCBjdGUoJ2InLCAkcTIpKS0+cmVjdXJzaXZlKCktPnNlbGVjdCguLi4pLT5mcm9tKHRhYmxlKCdhJykpCiAqLw=="},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/query.php","start_line_in_file":159,"slug":"insert","name":"insert","namespace":"Flow\\PostgreSql\\DSL","parameters":[],"return_type":[{"name":"InsertIntoStep","namespace":"Flow\\PostgreSql\\QueryBuilder\\Insert","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBhIG5ldyBJTlNFUlQgcXVlcnkgYnVpbGRlci4KICov"},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/query.php","start_line_in_file":175,"slug":"bulk-insert","name":"bulk_insert","namespace":"Flow\\PostgreSql\\DSL","parameters":[{"name":"table","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"columns","type":[{"name":"array","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"rowCount","type":[{"name":"int","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"BulkInsert","namespace":"Flow\\PostgreSql\\QueryBuilder\\Insert","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBhbiBvcHRpbWl6ZWQgYnVsayBJTlNFUlQgcXVlcnkgZm9yIGhpZ2gtcGVyZm9ybWFuY2UgbXVsdGktcm93IGluc2VydHMuCiAqCiAqIFVubGlrZSBpbnNlcnQoKSB3aGljaCB1c2VzIGltbXV0YWJsZSBidWlsZGVyIHBhdHRlcm5zIChPKG7CsikgZm9yIG4gcm93cyksCiAqIHRoaXMgZnVuY3Rpb24gZ2VuZXJhdGVzIFNRTCBkaXJlY3RseSB1c2luZyBzdHJpbmcgb3BlcmF0aW9ucyAoTyhuKSBjb21wbGV4aXR5KS4KICoKICogQHBhcmFtIHN0cmluZyAkdGFibGUgVGFibGUgbmFtZQogKiBAcGFyYW0gbGlzdDxzdHJpbmc+ICRjb2x1bW5zIENvbHVtbiBuYW1lcwogKiBAcGFyYW0gaW50ICRyb3dDb3VudCBOdW1iZXIgb2Ygcm93cyB0byBpbnNlcnQKICov"},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/query.php","start_line_in_file":184,"slug":"update","name":"update","namespace":"Flow\\PostgreSql\\DSL","parameters":[],"return_type":[{"name":"UpdateTableStep","namespace":"Flow\\PostgreSql\\QueryBuilder\\Update","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBhIG5ldyBVUERBVEUgcXVlcnkgYnVpbGRlci4KICov"},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/query.php","start_line_in_file":193,"slug":"delete","name":"delete","namespace":"Flow\\PostgreSql\\DSL","parameters":[],"return_type":[{"name":"DeleteFromStep","namespace":"Flow\\PostgreSql\\QueryBuilder\\Delete","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBhIG5ldyBERUxFVEUgcXVlcnkgYnVpbGRlci4KICov"},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/query.php","start_line_in_file":205,"slug":"merge","name":"merge","namespace":"Flow\\PostgreSql\\DSL","parameters":[{"name":"table","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"alias","type":[{"name":"string","namespace":null,"is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"null"}],"return_type":[{"name":"MergeUsingStep","namespace":"Flow\\PostgreSql\\QueryBuilder\\Merge","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBhIG5ldyBNRVJHRSBxdWVyeSBidWlsZGVyLgogKgogKiBAcGFyYW0gc3RyaW5nICR0YWJsZSBUYXJnZXQgdGFibGUgbmFtZQogKiBAcGFyYW0gbnVsbHxzdHJpbmcgJGFsaWFzIE9wdGlvbmFsIHRhYmxlIGFsaWFzCiAqLw=="},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/query.php","start_line_in_file":219,"slug":"copy","name":"copy","namespace":"Flow\\PostgreSql\\DSL","parameters":[],"return_type":[{"name":"CopyFactory","namespace":"Flow\\PostgreSql\\QueryBuilder\\Factory","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBhIG5ldyBDT1BZIHF1ZXJ5IGJ1aWxkZXIgZm9yIGRhdGEgaW1wb3J0L2V4cG9ydC4KICoKICogVXNhZ2U6CiAqICAgY29weSgpLT5mcm9tKCd1c2VycycpLT5maWxlKCcvdG1wL3VzZXJzLmNzdicpLT5mb3JtYXQoQ29weUZvcm1hdDo6Q1NWKQogKiAgIGNvcHkoKS0+dG8oJ3VzZXJzJyktPmZpbGUoJy90bXAvdXNlcnMuY3N2JyktPmZvcm1hdChDb3B5Rm9ybWF0OjpDU1YpCiAqICAgY29weSgpLT50b1F1ZXJ5KHNlbGVjdCguLi4pKS0+ZmlsZSgnL3RtcC9kYXRhLmNzdicpCiAqLw=="},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/query.php","start_line_in_file":231,"slug":"listen","name":"listen","namespace":"Flow\\PostgreSql\\DSL","parameters":[{"name":"channel","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"ListenFinalStep","namespace":"Flow\\PostgreSql\\QueryBuilder\\Listen","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBhIExJU1RFTiBzdGF0ZW1lbnQgdG8gc3Vic2NyaWJlIHRoZSBjdXJyZW50IHNlc3Npb24gdG8gYSBub3RpZmljYXRpb24gY2hhbm5lbC4KICoKICogVXNhZ2U6CiAqICAgbGlzdGVuKCdteV9jaGFubmVsJyktPnRvU3FsKCkgIC8vIExJU1RFTiBteV9jaGFubmVsCiAqLw=="},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/query.php","start_line_in_file":243,"slug":"unlisten","name":"unlisten","namespace":"Flow\\PostgreSql\\DSL","parameters":[{"name":"channel","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"UnlistenFinalStep","namespace":"Flow\\PostgreSql\\QueryBuilder\\Unlisten","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBhbiBVTkxJU1RFTiBzdGF0ZW1lbnQgdG8gdW5zdWJzY3JpYmUgdGhlIGN1cnJlbnQgc2Vzc2lvbiBmcm9tIGEgbm90aWZpY2F0aW9uIGNoYW5uZWwuCiAqCiAqIFVzYWdlOgogKiAgIHVubGlzdGVuKCdteV9jaGFubmVsJyktPnRvU3FsKCkgIC8vIFVOTElTVEVOIG15X2NoYW5uZWwKICov"},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/query.php","start_line_in_file":256,"slug":"notify","name":"notify","namespace":"Flow\\PostgreSql\\DSL","parameters":[{"name":"channel","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"NotifyFinalStep","namespace":"Flow\\PostgreSql\\QueryBuilder\\Notify","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBhIE5PVElGWSBzdGF0ZW1lbnQgdG8gc2VuZCBhIG5vdGlmaWNhdGlvbiBvbiBhIGNoYW5uZWwsIG9wdGlvbmFsbHkgd2l0aCBhIHBheWxvYWQuCiAqCiAqIFVzYWdlOgogKiAgIG5vdGlmeSgnbXlfY2hhbm5lbCcpLT50b1NxbCgpICAgICAgICAgICAgICAgICAgICAgICAgICAgLy8gTk9USUZZIG15X2NoYW5uZWwKICogICBub3RpZnkoJ215X2NoYW5uZWwnKS0+d2l0aFBheWxvYWQoJ2hlbGxvJyktPnRvU3FsKCkgICAgIC8vIE5PVElGWSBteV9jaGFubmVsLCAnaGVsbG8nCiAqLw=="},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/query.php","start_line_in_file":277,"slug":"col","name":"col","namespace":"Flow\\PostgreSql\\DSL","parameters":[{"name":"column","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"table","type":[{"name":"string","namespace":null,"is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"null"},{"name":"schema","type":[{"name":"string","namespace":null,"is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"null"}],"return_type":[{"name":"Column","namespace":"Flow\\PostgreSql\\QueryBuilder\\Expression","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBhIGNvbHVtbiByZWZlcmVuY2UgZXhwcmVzc2lvbi4KICoKICogQ2FuIGJlIHVzZWQgaW4gdHdvIG1vZGVzOgogKiAtIFBhcnNlIG1vZGU6IGNvbCgndXNlcnMuaWQnKSBvciBjb2woJ3NjaGVtYS50YWJsZS5jb2x1bW4nKSAtIHBhcnNlcyBkb3Qtc2VwYXJhdGVkIHN0cmluZwogKiAtIEV4cGxpY2l0IG1vZGU6IGNvbCgnaWQnLCAndXNlcnMnKSBvciBjb2woJ2lkJywgJ3VzZXJzJywgJ3NjaGVtYScpIC0gc2VwYXJhdGUgYXJndW1lbnRzCiAqCiAqIFdoZW4gJHRhYmxlIG9yICRzY2hlbWEgaXMgcHJvdmlkZWQsICRjb2x1bW4gbXVzdCBiZSBhIHBsYWluIGNvbHVtbiBuYW1lIChubyBkb3RzKS4KICoKICogQHBhcmFtIHN0cmluZyAkY29sdW1uIENvbHVtbiBuYW1lLCBvciBkb3Qtc2VwYXJhdGVkIHBhdGggbGlrZSAidGFibGUuY29sdW1uIiBvciAic2NoZW1hLnRhYmxlLmNvbHVtbiIKICogQHBhcmFtIG51bGx8c3RyaW5nICR0YWJsZSBUYWJsZSBuYW1lIChvcHRpb25hbCwgdHJpZ2dlcnMgZXhwbGljaXQgbW9kZSkKICogQHBhcmFtIG51bGx8c3RyaW5nICRzY2hlbWEgU2NoZW1hIG5hbWUgKG9wdGlvbmFsLCByZXF1aXJlcyAkdGFibGUpCiAqCiAqIEB0aHJvd3MgSW52YWxpZEV4cHJlc3Npb25FeGNlcHRpb24gd2hlbiAkc2NoZW1hIGlzIHByb3ZpZGVkIHdpdGhvdXQgJHRhYmxlLCBvciB3aGVuICRjb2x1bW4gY29udGFpbnMgZG90cyBpbiBleHBsaWNpdCBtb2RlCiAqLw=="},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/query.php","start_line_in_file":304,"slug":"star","name":"star","namespace":"Flow\\PostgreSql\\DSL","parameters":[{"name":"table","type":[{"name":"string","namespace":null,"is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"null"}],"return_type":[{"name":"Star","namespace":"Flow\\PostgreSql\\QueryBuilder\\Expression","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBhIFNFTEVDVCAqIGV4cHJlc3Npb24uCiAqLw=="},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/query.php","start_line_in_file":320,"slug":"literal","name":"literal","namespace":"Flow\\PostgreSql\\DSL","parameters":[{"name":"value","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false},{"name":"int","namespace":null,"is_nullable":false,"is_variadic":false},{"name":"float","namespace":null,"is_nullable":false,"is_variadic":false},{"name":"bool","namespace":null,"is_nullable":false,"is_variadic":false},{"name":"null","namespace":null,"is_nullable":true,"is_variadic":false}],"has_default_value":false,"is_nullable":true,"is_variadic":false,"default_value":null}],"return_type":[{"name":"Literal","namespace":"Flow\\PostgreSql\\QueryBuilder\\Expression","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBhIGxpdGVyYWwgdmFsdWUgZm9yIHVzZSBpbiBxdWVyaWVzLgogKgogKiBBdXRvbWF0aWNhbGx5IGRldGVjdHMgdGhlIHR5cGUgYW5kIGNyZWF0ZXMgdGhlIGFwcHJvcHJpYXRlIGxpdGVyYWw6CiAqIC0gbGl0ZXJhbCgnaGVsbG8nKSBjcmVhdGVzIGEgc3RyaW5nIGxpdGVyYWwKICogLSBsaXRlcmFsKDQyKSBjcmVhdGVzIGFuIGludGVnZXIgbGl0ZXJhbAogKiAtIGxpdGVyYWwoMy4xNCkgY3JlYXRlcyBhIGZsb2F0IGxpdGVyYWwKICogLSBsaXRlcmFsKHRydWUpIGNyZWF0ZXMgYSBib29sZWFuIGxpdGVyYWwKICogLSBsaXRlcmFsKG51bGwpIGNyZWF0ZXMgYSBOVUxMIGxpdGVyYWwKICov"},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/query.php","start_line_in_file":335,"slug":"param","name":"param","namespace":"Flow\\PostgreSql\\DSL","parameters":[{"name":"position","type":[{"name":"int","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"Parameter","namespace":"Flow\\PostgreSql\\QueryBuilder\\Expression","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBhIHBvc2l0aW9uYWwgcGFyYW1ldGVyICgkMSwgJDIsIGV0Yy4pLgogKi8="},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/query.php","start_line_in_file":344,"slug":"parameters","name":"parameters","namespace":"Flow\\PostgreSql\\DSL","parameters":[{"name":"count","type":[{"name":"int","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"startAt","type":[{"name":"int","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"1"}],"return_type":[{"name":"array","namespace":null,"is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIEByZXR1cm4gbGlzdDxQYXJhbWV0ZXI+CiAqLw=="},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/query.php","start_line_in_file":370,"slug":"func","name":"func","namespace":"Flow\\PostgreSql\\DSL","parameters":[{"name":"name","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"args","type":[{"name":"array","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"[]"}],"return_type":[{"name":"FunctionCall","namespace":"Flow\\PostgreSql\\QueryBuilder\\Expression","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBhIGZ1bmN0aW9uIGNhbGwgZXhwcmVzc2lvbi4KICoKICogQHBhcmFtIHN0cmluZyAkbmFtZSBGdW5jdGlvbiBuYW1lIChjYW4gaW5jbHVkZSBzY2hlbWEgbGlrZSAicGdfY2F0YWxvZy5ub3ciKQogKiBAcGFyYW0gbGlzdDxFeHByZXNzaW9ufHN0cmluZz4gJGFyZ3MgRnVuY3Rpb24gYXJndW1lbnRzCiAqLw=="},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/query.php","start_line_in_file":404,"slug":"agg","name":"agg","namespace":"Flow\\PostgreSql\\DSL","parameters":[{"name":"name","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"args","type":[{"name":"array","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"[]"},{"name":"distinct","type":[{"name":"bool","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"false"}],"return_type":[{"name":"AggregateCall","namespace":"Flow\\PostgreSql\\QueryBuilder\\Expression","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBhbiBhZ2dyZWdhdGUgZnVuY3Rpb24gY2FsbCAoQ09VTlQsIFNVTSwgQVZHLCBldGMuKS4KICoKICogQHBhcmFtIHN0cmluZyAkbmFtZSBBZ2dyZWdhdGUgZnVuY3Rpb24gbmFtZQogKiBAcGFyYW0gbGlzdDxFeHByZXNzaW9ufHN0cmluZz4gJGFyZ3MgRnVuY3Rpb24gYXJndW1lbnRzCiAqIEBwYXJhbSBib29sICRkaXN0aW5jdCBVc2UgRElTVElOQ1QgbW9kaWZpZXIKICov"},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/query.php","start_line_in_file":418,"slug":"agg-count","name":"agg_count","namespace":"Flow\\PostgreSql\\DSL","parameters":[{"name":"expr","type":[{"name":"Expression","namespace":"Flow\\PostgreSql\\QueryBuilder\\Expression","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false},{"name":"null","namespace":null,"is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"null"},{"name":"distinct","type":[{"name":"bool","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"false"}],"return_type":[{"name":"AggregateCall","namespace":"Flow\\PostgreSql\\QueryBuilder\\Expression","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBDT1VOVCgqKSBhZ2dyZWdhdGUuCiAqLw=="},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/query.php","start_line_in_file":431,"slug":"count-all","name":"count_all","namespace":"Flow\\PostgreSql\\DSL","parameters":[],"return_type":[{"name":"AggregateCall","namespace":"Flow\\PostgreSql\\QueryBuilder\\Expression","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBDT1VOVCgqKSBhZ2dyZWdhdGUuCiAqLw=="},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/query.php","start_line_in_file":440,"slug":"agg-sum","name":"agg_sum","namespace":"Flow\\PostgreSql\\DSL","parameters":[{"name":"expr","type":[{"name":"Expression","namespace":"Flow\\PostgreSql\\QueryBuilder\\Expression","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"distinct","type":[{"name":"bool","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"false"}],"return_type":[{"name":"AggregateCall","namespace":"Flow\\PostgreSql\\QueryBuilder\\Expression","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBTVU0gYWdncmVnYXRlLgogKi8="},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/query.php","start_line_in_file":449,"slug":"agg-avg","name":"agg_avg","namespace":"Flow\\PostgreSql\\DSL","parameters":[{"name":"expr","type":[{"name":"Expression","namespace":"Flow\\PostgreSql\\QueryBuilder\\Expression","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"distinct","type":[{"name":"bool","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"false"}],"return_type":[{"name":"AggregateCall","namespace":"Flow\\PostgreSql\\QueryBuilder\\Expression","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBBVkcgYWdncmVnYXRlLgogKi8="},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/query.php","start_line_in_file":458,"slug":"agg-min","name":"agg_min","namespace":"Flow\\PostgreSql\\DSL","parameters":[{"name":"expr","type":[{"name":"Expression","namespace":"Flow\\PostgreSql\\QueryBuilder\\Expression","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"AggregateCall","namespace":"Flow\\PostgreSql\\QueryBuilder\\Expression","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBNSU4gYWdncmVnYXRlLgogKi8="},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/query.php","start_line_in_file":467,"slug":"agg-max","name":"agg_max","namespace":"Flow\\PostgreSql\\DSL","parameters":[{"name":"expr","type":[{"name":"Expression","namespace":"Flow\\PostgreSql\\QueryBuilder\\Expression","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"AggregateCall","namespace":"Flow\\PostgreSql\\QueryBuilder\\Expression","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBNQVggYWdncmVnYXRlLgogKi8="},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/query.php","start_line_in_file":478,"slug":"coalesce","name":"coalesce","namespace":"Flow\\PostgreSql\\DSL","parameters":[{"name":"expressions","type":[{"name":"Expression","namespace":"Flow\\PostgreSql\\QueryBuilder\\Expression","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":true,"default_value":null}],"return_type":[{"name":"Coalesce","namespace":"Flow\\PostgreSql\\QueryBuilder\\Expression","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBhIENPQUxFU0NFIGV4cHJlc3Npb24uCiAqCiAqIEBwYXJhbSBFeHByZXNzaW9ufHN0cmluZyAuLi4kZXhwcmVzc2lvbnMgRXhwcmVzc2lvbnMgdG8gY29hbGVzY2UKICov"},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/query.php","start_line_in_file":489,"slug":"nullif","name":"nullif","namespace":"Flow\\PostgreSql\\DSL","parameters":[{"name":"expr1","type":[{"name":"Expression","namespace":"Flow\\PostgreSql\\QueryBuilder\\Expression","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"expr2","type":[{"name":"Expression","namespace":"Flow\\PostgreSql\\QueryBuilder\\Expression","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"NullIf","namespace":"Flow\\PostgreSql\\QueryBuilder\\Expression","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBhIE5VTExJRiBleHByZXNzaW9uLgogKi8="},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/query.php","start_line_in_file":503,"slug":"greatest","name":"greatest","namespace":"Flow\\PostgreSql\\DSL","parameters":[{"name":"expressions","type":[{"name":"Expression","namespace":"Flow\\PostgreSql\\QueryBuilder\\Expression","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":true,"default_value":null}],"return_type":[{"name":"Greatest","namespace":"Flow\\PostgreSql\\QueryBuilder\\Expression","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBhIEdSRUFURVNUIGV4cHJlc3Npb24uCiAqCiAqIEBwYXJhbSBFeHByZXNzaW9ufHN0cmluZyAuLi4kZXhwcmVzc2lvbnMgRXhwcmVzc2lvbnMgdG8gY29tcGFyZQogKi8="},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/query.php","start_line_in_file":516,"slug":"least","name":"least","namespace":"Flow\\PostgreSql\\DSL","parameters":[{"name":"expressions","type":[{"name":"Expression","namespace":"Flow\\PostgreSql\\QueryBuilder\\Expression","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":true,"default_value":null}],"return_type":[{"name":"Least","namespace":"Flow\\PostgreSql\\QueryBuilder\\Expression","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBhIExFQVNUIGV4cHJlc3Npb24uCiAqCiAqIEBwYXJhbSBFeHByZXNzaW9ufHN0cmluZyAuLi4kZXhwcmVzc2lvbnMgRXhwcmVzc2lvbnMgdG8gY29tcGFyZQogKi8="},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/query.php","start_line_in_file":530,"slug":"cast","name":"cast","namespace":"Flow\\PostgreSql\\DSL","parameters":[{"name":"expr","type":[{"name":"Expression","namespace":"Flow\\PostgreSql\\QueryBuilder\\Expression","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"dataType","type":[{"name":"ColumnType","namespace":"Flow\\PostgreSql\\QueryBuilder\\Schema","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"TypeCast","namespace":"Flow\\PostgreSql\\QueryBuilder\\Expression","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBhIHR5cGUgY2FzdCBleHByZXNzaW9uLgogKgogKiBAcGFyYW0gRXhwcmVzc2lvbnxzdHJpbmcgJGV4cHIgRXhwcmVzc2lvbiB0byBjYXN0CiAqIEBwYXJhbSBDb2x1bW5UeXBlICRkYXRhVHlwZSBUYXJnZXQgZGF0YSB0eXBlICh1c2UgY29sdW1uX3R5cGVfKiBmdW5jdGlvbnMpCiAqLw=="},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/query.php","start_line_in_file":542,"slug":"collate","name":"collate","namespace":"Flow\\PostgreSql\\DSL","parameters":[{"name":"expr","type":[{"name":"Expression","namespace":"Flow\\PostgreSql\\QueryBuilder\\Expression","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"collation","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"Collate","namespace":"Flow\\PostgreSql\\QueryBuilder\\Expression","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBhIENPTExBVEUgZXhwcmVzc2lvbi4KICoKICogQHBhcmFtIEV4cHJlc3Npb258c3RyaW5nICRleHByIEV4cHJlc3Npb24gdG8gY29sbGF0ZQogKiBAcGFyYW0gc3RyaW5nICRjb2xsYXRpb24gQ29sbGF0aW9uIG5hbWUgYXMgd3JpdHRlbiBpbiBTUUw6ICdDJywgJyJkZV9ERSInLCAncGdfY2F0YWxvZy4iQyInCiAqLw=="},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/query.php","start_line_in_file":557,"slug":"current-timestamp","name":"current_timestamp","namespace":"Flow\\PostgreSql\\DSL","parameters":[],"return_type":[{"name":"SQLValueFunctionExpression","namespace":"Flow\\PostgreSql\\QueryBuilder\\Expression","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIFNRTCBzdGFuZGFyZCBDVVJSRU5UX1RJTUVTVEFNUCBmdW5jdGlvbi4KICoKICogUmV0dXJucyB0aGUgY3VycmVudCBkYXRlIGFuZCB0aW1lIChhdCB0aGUgc3RhcnQgb2YgdGhlIHRyYW5zYWN0aW9uKS4KICogVXNlZnVsIGFzIGEgY29sdW1uIGRlZmF1bHQgdmFsdWUgb3IgaW4gU0VMRUNUIHF1ZXJpZXMuCiAqCiAqIEV4YW1wbGU6IGNvbHVtbignY3JlYXRlZF9hdCcsIGNvbHVtbl90eXBlX3RpbWVzdGFtcCgpKS0+ZGVmYXVsdChjdXJyZW50X3RpbWVzdGFtcCgpKQogKiBFeGFtcGxlOiBzZWxlY3QoKS0+c2VsZWN0KGN1cnJlbnRfdGltZXN0YW1wKCktPmFzKCdub3cnKSkKICov"},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/query.php","start_line_in_file":572,"slug":"current-date","name":"current_date","namespace":"Flow\\PostgreSql\\DSL","parameters":[],"return_type":[{"name":"SQLValueFunctionExpression","namespace":"Flow\\PostgreSql\\QueryBuilder\\Expression","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIFNRTCBzdGFuZGFyZCBDVVJSRU5UX0RBVEUgZnVuY3Rpb24uCiAqCiAqIFJldHVybnMgdGhlIGN1cnJlbnQgZGF0ZSAoYXQgdGhlIHN0YXJ0IG9mIHRoZSB0cmFuc2FjdGlvbikuCiAqIFVzZWZ1bCBhcyBhIGNvbHVtbiBkZWZhdWx0IHZhbHVlIG9yIGluIFNFTEVDVCBxdWVyaWVzLgogKgogKiBFeGFtcGxlOiBjb2x1bW4oJ2JpcnRoX2RhdGUnLCBjb2x1bW5fdHlwZV9kYXRlKCkpLT5kZWZhdWx0KGN1cnJlbnRfZGF0ZSgpKQogKiBFeGFtcGxlOiBzZWxlY3QoKS0+c2VsZWN0KGN1cnJlbnRfZGF0ZSgpLT5hcygndG9kYXknKSkKICov"},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/query.php","start_line_in_file":587,"slug":"current-time","name":"current_time","namespace":"Flow\\PostgreSql\\DSL","parameters":[],"return_type":[{"name":"SQLValueFunctionExpression","namespace":"Flow\\PostgreSql\\QueryBuilder\\Expression","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIFNRTCBzdGFuZGFyZCBDVVJSRU5UX1RJTUUgZnVuY3Rpb24uCiAqCiAqIFJldHVybnMgdGhlIGN1cnJlbnQgdGltZSAoYXQgdGhlIHN0YXJ0IG9mIHRoZSB0cmFuc2FjdGlvbikuCiAqIFVzZWZ1bCBhcyBhIGNvbHVtbiBkZWZhdWx0IHZhbHVlIG9yIGluIFNFTEVDVCBxdWVyaWVzLgogKgogKiBFeGFtcGxlOiBjb2x1bW4oJ3N0YXJ0X3RpbWUnLCBjb2x1bW5fdHlwZV90aW1lKCkpLT5kZWZhdWx0KGN1cnJlbnRfdGltZSgpKQogKiBFeGFtcGxlOiBzZWxlY3QoKS0+c2VsZWN0KGN1cnJlbnRfdGltZSgpLT5hcygnbm93X3RpbWUnKSkKICov"},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/query.php","start_line_in_file":600,"slug":"case-when","name":"case_when","namespace":"Flow\\PostgreSql\\DSL","parameters":[{"name":"whenClauses","type":[{"name":"array","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"elseResult","type":[{"name":"Expression","namespace":"Flow\\PostgreSql\\QueryBuilder\\Expression","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false},{"name":"null","namespace":null,"is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"null"},{"name":"operand","type":[{"name":"Expression","namespace":"Flow\\PostgreSql\\QueryBuilder\\Expression","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false},{"name":"null","namespace":null,"is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"null"}],"return_type":[{"name":"CaseExpression","namespace":"Flow\\PostgreSql\\QueryBuilder\\Expression","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBhIENBU0UgZXhwcmVzc2lvbi4KICoKICogQHBhcmFtIG5vbi1lbXB0eS1saXN0PFdoZW5DbGF1c2U+ICR3aGVuQ2xhdXNlcyBXSEVOIGNsYXVzZXMKICogQHBhcmFtIG51bGx8RXhwcmVzc2lvbnxzdHJpbmcgJGVsc2VSZXN1bHQgRUxTRSByZXN1bHQgKG9wdGlvbmFsKQogKiBAcGFyYW0gbnVsbHxFeHByZXNzaW9ufHN0cmluZyAkb3BlcmFuZCBDQVNFIG9wZXJhbmQgZm9yIHNpbXBsZSBDQVNFIChvcHRpb25hbCkKICov"},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/query.php","start_line_in_file":616,"slug":"when","name":"when","namespace":"Flow\\PostgreSql\\DSL","parameters":[{"name":"condition","type":[{"name":"Expression","namespace":"Flow\\PostgreSql\\QueryBuilder\\Expression","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"result","type":[{"name":"Expression","namespace":"Flow\\PostgreSql\\QueryBuilder\\Expression","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"WhenClause","namespace":"Flow\\PostgreSql\\QueryBuilder\\Expression","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBhIFdIRU4gY2xhdXNlIGZvciBDQVNFIGV4cHJlc3Npb24uCiAqLw=="},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/query.php","start_line_in_file":628,"slug":"sub-select","name":"sub_select","namespace":"Flow\\PostgreSql\\DSL","parameters":[{"name":"query","type":[{"name":"SelectFinalStep","namespace":"Flow\\PostgreSql\\QueryBuilder\\Select","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"Subquery","namespace":"Flow\\PostgreSql\\QueryBuilder\\Expression","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBhIHN1YnF1ZXJ5IGV4cHJlc3Npb24uCiAqLw=="},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/query.php","start_line_in_file":642,"slug":"array-expr","name":"array_expr","namespace":"Flow\\PostgreSql\\DSL","parameters":[{"name":"elements","type":[{"name":"array","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"ArrayExpression","namespace":"Flow\\PostgreSql\\QueryBuilder\\Expression","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBhbiBhcnJheSBleHByZXNzaW9uLgogKgogKiBAcGFyYW0gbGlzdDxFeHByZXNzaW9ufHN0cmluZz4gJGVsZW1lbnRzIEFycmF5IGVsZW1lbnRzCiAqLw=="},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/query.php","start_line_in_file":655,"slug":"row-expr","name":"row_expr","namespace":"Flow\\PostgreSql\\DSL","parameters":[{"name":"elements","type":[{"name":"array","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"RowExpression","namespace":"Flow\\PostgreSql\\QueryBuilder\\Expression","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBhIHJvdyBleHByZXNzaW9uLgogKgogKiBAcGFyYW0gbGlzdDxFeHByZXNzaW9ufHN0cmluZz4gJGVsZW1lbnRzIFJvdyBlbGVtZW50cwogKi8="},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/query.php","start_line_in_file":666,"slug":"binary-expr","name":"binary_expr","namespace":"Flow\\PostgreSql\\DSL","parameters":[{"name":"left","type":[{"name":"Expression","namespace":"Flow\\PostgreSql\\QueryBuilder\\Expression","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"operator","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"right","type":[{"name":"Expression","namespace":"Flow\\PostgreSql\\QueryBuilder\\Expression","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"BinaryExpression","namespace":"Flow\\PostgreSql\\QueryBuilder\\Expression","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBhIGJpbmFyeSBleHByZXNzaW9uIChsZWZ0IG9wIHJpZ2h0KS4KICov"},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/query.php","start_line_in_file":684,"slug":"window-func","name":"window_func","namespace":"Flow\\PostgreSql\\DSL","parameters":[{"name":"name","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"args","type":[{"name":"array","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"[]"},{"name":"partitionBy","type":[{"name":"array","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"[]"},{"name":"orderBy","type":[{"name":"array","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"[]"}],"return_type":[{"name":"WindowFunction","namespace":"Flow\\PostgreSql\\QueryBuilder\\Expression","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBhIHdpbmRvdyBmdW5jdGlvbi4KICoKICogQHBhcmFtIHN0cmluZyAkbmFtZSBGdW5jdGlvbiBuYW1lCiAqIEBwYXJhbSBsaXN0PEV4cHJlc3Npb258c3RyaW5nPiAkYXJncyBGdW5jdGlvbiBhcmd1bWVudHMKICogQHBhcmFtIGxpc3Q8RXhwcmVzc2lvbnxzdHJpbmc+ICRwYXJ0aXRpb25CeSBQQVJUSVRJT04gQlkgZXhwcmVzc2lvbnMKICogQHBhcmFtIGxpc3Q8T3JkZXJCeT4gJG9yZGVyQnkgT1JERVIgQlkgaXRlbXMKICov"},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/query.php","start_line_in_file":705,"slug":"concat","name":"concat","namespace":"Flow\\PostgreSql\\DSL","parameters":[{"name":"expressions","type":[{"name":"Expression","namespace":"Flow\\PostgreSql\\QueryBuilder\\Expression","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":true,"default_value":null}],"return_type":[{"name":"BinaryExpression","namespace":"Flow\\PostgreSql\\QueryBuilder\\Expression","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENvbmNhdGVuYXRlIGV4cHJlc3Npb25zIHdpdGggdGhlIHx8IG9wZXJhdG9yLgogKgogKiBFeGFtcGxlOiBjb25jYXQoY29sKCdzY2hlbWEnKSwgbGl0ZXJhbCgnLicpLCBjb2woJ3RhYmxlJykpCiAqIFByb2R1Y2VzOiBzY2hlbWEgfHwgJy4nIHx8IHRhYmxlCiAqCiAqIEBwYXJhbSBFeHByZXNzaW9ufHN0cmluZyAuLi4kZXhwcmVzc2lvbnMgQXQgbGVhc3QgMiBleHByZXNzaW9ucyB0byBjb25jYXRlbmF0ZQogKi8="},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/query.php","start_line_in_file":735,"slug":"table","name":"table","namespace":"Flow\\PostgreSql\\DSL","parameters":[{"name":"name","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"schema","type":[{"name":"string","namespace":null,"is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"null"}],"return_type":[{"name":"Table","namespace":"Flow\\PostgreSql\\QueryBuilder\\Table","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBhIHRhYmxlIHJlZmVyZW5jZS4KICoKICogU3VwcG9ydHMgZG90IG5vdGF0aW9uIGZvciBzY2hlbWEtcXVhbGlmaWVkIG5hbWVzOiAicHVibGljLnVzZXJzIiBvciBleHBsaWNpdCBzY2hlbWEgcGFyYW1ldGVyLgogKiBEb3VibGUtcXVvdGVkIGlkZW50aWZpZXJzIHByZXNlcnZlIGRvdHM6ICcibXkudGFibGUiJyBjcmVhdGVzIGEgc2luZ2xlIGlkZW50aWZpZXIuCiAqCiAqIEBwYXJhbSBzdHJpbmcgJG5hbWUgVGFibGUgbmFtZSAobWF5IGluY2x1ZGUgc2NoZW1hIGFzICJzY2hlbWEudGFibGUiKQogKiBAcGFyYW0gbnVsbHxzdHJpbmcgJHNjaGVtYSBTY2hlbWEgbmFtZSAob3B0aW9uYWwsIG92ZXJyaWRlcyBwYXJzZWQgc2NoZW1hKQogKi8="},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/query.php","start_line_in_file":750,"slug":"derived","name":"derived","namespace":"Flow\\PostgreSql\\DSL","parameters":[{"name":"query","type":[{"name":"SelectFinalStep","namespace":"Flow\\PostgreSql\\QueryBuilder\\Select","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"alias","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"DerivedTable","namespace":"Flow\\PostgreSql\\QueryBuilder\\Table","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBhIGRlcml2ZWQgdGFibGUgKHN1YnF1ZXJ5IGluIEZST00gY2xhdXNlKS4KICov"},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/query.php","start_line_in_file":764,"slug":"lateral","name":"lateral","namespace":"Flow\\PostgreSql\\DSL","parameters":[{"name":"reference","type":[{"name":"TableReference","namespace":"Flow\\PostgreSql\\QueryBuilder\\Table","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"Lateral","namespace":"Flow\\PostgreSql\\QueryBuilder\\Table","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBhIExBVEVSQUwgc3VicXVlcnkuCiAqCiAqIEBwYXJhbSBUYWJsZVJlZmVyZW5jZSAkcmVmZXJlbmNlIFRoZSBzdWJxdWVyeSBvciB0YWJsZSBmdW5jdGlvbiByZWZlcmVuY2UKICov"},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/query.php","start_line_in_file":776,"slug":"table-func","name":"table_func","namespace":"Flow\\PostgreSql\\DSL","parameters":[{"name":"function","type":[{"name":"FunctionCall","namespace":"Flow\\PostgreSql\\QueryBuilder\\Expression","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"withOrdinality","type":[{"name":"bool","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"false"}],"return_type":[{"name":"TableFunction","namespace":"Flow\\PostgreSql\\QueryBuilder\\Table","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBhIHRhYmxlIGZ1bmN0aW9uIHJlZmVyZW5jZS4KICoKICogQHBhcmFtIEZ1bmN0aW9uQ2FsbCAkZnVuY3Rpb24gVGhlIHRhYmxlLXZhbHVlZCBmdW5jdGlvbgogKiBAcGFyYW0gYm9vbCAkd2l0aE9yZGluYWxpdHkgV2hldGhlciB0byBhZGQgV0lUSCBPUkRJTkFMSVRZCiAqLw=="},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/query.php","start_line_in_file":795,"slug":"values-table","name":"values_table","namespace":"Flow\\PostgreSql\\DSL","parameters":[{"name":"rows","type":[{"name":"RowExpression","namespace":"Flow\\PostgreSql\\QueryBuilder\\Expression","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":true,"default_value":null}],"return_type":[{"name":"ValuesTable","namespace":"Flow\\PostgreSql\\QueryBuilder\\Table","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBhIFZBTFVFUyBjbGF1c2UgYXMgYSB0YWJsZSByZWZlcmVuY2UuCiAqCiAqIFVzYWdlOgogKiAgIHNlbGVjdCgpLT5mcm9tKAogKiAgICAgICB2YWx1ZXNfdGFibGUoCiAqICAgICAgICAgICByb3dfZXhwcihbbGl0ZXJhbCgxKSwgbGl0ZXJhbCgnQWxpY2UnKV0pLAogKiAgICAgICAgICAgcm93X2V4cHIoW2xpdGVyYWwoMiksIGxpdGVyYWwoJ0JvYicpXSkKICogICAgICAgKS0+YXMoJ3QnLCBbJ2lkJywgJ25hbWUnXSkKICogICApCiAqCiAqIEdlbmVyYXRlczogU0VMRUNUICogRlJPTSAoVkFMVUVTICgxLCAnQWxpY2UnKSwgKDIsICdCb2InKSkgQVMgdChpZCwgbmFtZSkKICov"},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/query.php","start_line_in_file":804,"slug":"order-by","name":"order_by","namespace":"Flow\\PostgreSql\\DSL","parameters":[{"name":"expr","type":[{"name":"Expression","namespace":"Flow\\PostgreSql\\QueryBuilder\\Expression","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"direction","type":[{"name":"SortDirection","namespace":"Flow\\PostgreSql\\QueryBuilder\\Clause","is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"Flow\\PostgreSql\\QueryBuilder\\Clause\\SortDirection::..."},{"name":"nulls","type":[{"name":"NullsPosition","namespace":"Flow\\PostgreSql\\QueryBuilder\\Clause","is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"Flow\\PostgreSql\\QueryBuilder\\Clause\\NullsPosition::..."}],"return_type":[{"name":"OrderBy","namespace":"Flow\\PostgreSql\\QueryBuilder\\Clause","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBhbiBPUkRFUiBCWSBpdGVtLgogKi8="},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/query.php","start_line_in_file":816,"slug":"asc","name":"asc","namespace":"Flow\\PostgreSql\\DSL","parameters":[{"name":"expr","type":[{"name":"Expression","namespace":"Flow\\PostgreSql\\QueryBuilder\\Expression","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"nulls","type":[{"name":"NullsPosition","namespace":"Flow\\PostgreSql\\QueryBuilder\\Clause","is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"Flow\\PostgreSql\\QueryBuilder\\Clause\\NullsPosition::..."}],"return_type":[{"name":"OrderBy","namespace":"Flow\\PostgreSql\\QueryBuilder\\Clause","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBhbiBPUkRFUiBCWSBpdGVtIHdpdGggQVNDIGRpcmVjdGlvbi4KICov"},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/query.php","start_line_in_file":825,"slug":"desc","name":"desc","namespace":"Flow\\PostgreSql\\DSL","parameters":[{"name":"expr","type":[{"name":"Expression","namespace":"Flow\\PostgreSql\\QueryBuilder\\Expression","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"nulls","type":[{"name":"NullsPosition","namespace":"Flow\\PostgreSql\\QueryBuilder\\Clause","is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"Flow\\PostgreSql\\QueryBuilder\\Clause\\NullsPosition::..."}],"return_type":[{"name":"OrderBy","namespace":"Flow\\PostgreSql\\QueryBuilder\\Clause","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBhbiBPUkRFUiBCWSBpdGVtIHdpdGggREVTQyBkaXJlY3Rpb24uCiAqLw=="},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/query.php","start_line_in_file":839,"slug":"cte","name":"cte","namespace":"Flow\\PostgreSql\\DSL","parameters":[{"name":"name","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"query","type":[{"name":"SelectFinalStep","namespace":"Flow\\PostgreSql\\QueryBuilder\\Select","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"columnNames","type":[{"name":"array","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"[]"},{"name":"materialization","type":[{"name":"CTEMaterialization","namespace":"Flow\\PostgreSql\\QueryBuilder\\Clause","is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"Flow\\PostgreSql\\QueryBuilder\\Clause\\CTEMaterialization::..."},{"name":"recursive","type":[{"name":"bool","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"false"}],"return_type":[{"name":"CTE","namespace":"Flow\\PostgreSql\\QueryBuilder\\Clause","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBhIENURSAoQ29tbW9uIFRhYmxlIEV4cHJlc3Npb24pLgogKgogKiBAcGFyYW0gc3RyaW5nICRuYW1lIENURSBuYW1lCiAqIEBwYXJhbSBTZWxlY3RGaW5hbFN0ZXAgJHF1ZXJ5IENURSBxdWVyeQogKiBAcGFyYW0gYXJyYXk8c3RyaW5nPiAkY29sdW1uTmFtZXMgQ29sdW1uIGFsaWFzZXMgKG9wdGlvbmFsKQogKiBAcGFyYW0gQ1RFTWF0ZXJpYWxpemF0aW9uICRtYXRlcmlhbGl6YXRpb24gTWF0ZXJpYWxpemF0aW9uIGhpbnQKICov"},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/query.php","start_line_in_file":861,"slug":"window-def","name":"window_def","namespace":"Flow\\PostgreSql\\DSL","parameters":[{"name":"name","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"partitionBy","type":[{"name":"array","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"[]"},{"name":"orderBy","type":[{"name":"array","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"[]"},{"name":"frame","type":[{"name":"WindowFrame","namespace":"Flow\\PostgreSql\\QueryBuilder\\Clause","is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"null"}],"return_type":[{"name":"WindowDefinition","namespace":"Flow\\PostgreSql\\QueryBuilder\\Clause","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBhIHdpbmRvdyBkZWZpbml0aW9uIGZvciBXSU5ET1cgY2xhdXNlLgogKgogKiBAcGFyYW0gc3RyaW5nICRuYW1lIFdpbmRvdyBuYW1lCiAqIEBwYXJhbSBsaXN0PEV4cHJlc3Npb258c3RyaW5nPiAkcGFydGl0aW9uQnkgUEFSVElUSU9OIEJZIGV4cHJlc3Npb25zCiAqIEBwYXJhbSBsaXN0PE9yZGVyQnk+ICRvcmRlckJ5IE9SREVSIEJZIGl0ZW1zCiAqIEBwYXJhbSBudWxsfFdpbmRvd0ZyYW1lICRmcmFtZSBXaW5kb3cgZnJhbWUgc3BlY2lmaWNhdGlvbgogKi8="},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/query.php","start_line_in_file":881,"slug":"window-frame","name":"window_frame","namespace":"Flow\\PostgreSql\\DSL","parameters":[{"name":"mode","type":[{"name":"FrameMode","namespace":"Flow\\PostgreSql\\QueryBuilder\\Clause","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"start","type":[{"name":"FrameBound","namespace":"Flow\\PostgreSql\\QueryBuilder\\Clause","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"end","type":[{"name":"FrameBound","namespace":"Flow\\PostgreSql\\QueryBuilder\\Clause","is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"null"},{"name":"exclusion","type":[{"name":"FrameExclusion","namespace":"Flow\\PostgreSql\\QueryBuilder\\Clause","is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"Flow\\PostgreSql\\QueryBuilder\\Clause\\FrameExclusion::..."}],"return_type":[{"name":"WindowFrame","namespace":"Flow\\PostgreSql\\QueryBuilder\\Clause","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBhIHdpbmRvdyBmcmFtZSBzcGVjaWZpY2F0aW9uLgogKi8="},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/query.php","start_line_in_file":894,"slug":"frame-current-row","name":"frame_current_row","namespace":"Flow\\PostgreSql\\DSL","parameters":[],"return_type":[{"name":"FrameBound","namespace":"Flow\\PostgreSql\\QueryBuilder\\Clause","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBhIGZyYW1lIGJvdW5kIGZvciBDVVJSRU5UIFJPVy4KICov"},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/query.php","start_line_in_file":903,"slug":"frame-unbounded-preceding","name":"frame_unbounded_preceding","namespace":"Flow\\PostgreSql\\DSL","parameters":[],"return_type":[{"name":"FrameBound","namespace":"Flow\\PostgreSql\\QueryBuilder\\Clause","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBhIGZyYW1lIGJvdW5kIGZvciBVTkJPVU5ERUQgUFJFQ0VESU5HLgogKi8="},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/query.php","start_line_in_file":912,"slug":"frame-unbounded-following","name":"frame_unbounded_following","namespace":"Flow\\PostgreSql\\DSL","parameters":[],"return_type":[{"name":"FrameBound","namespace":"Flow\\PostgreSql\\QueryBuilder\\Clause","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBhIGZyYW1lIGJvdW5kIGZvciBVTkJPVU5ERUQgRk9MTE9XSU5HLgogKi8="},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/query.php","start_line_in_file":921,"slug":"frame-preceding","name":"frame_preceding","namespace":"Flow\\PostgreSql\\DSL","parameters":[{"name":"offset","type":[{"name":"Expression","namespace":"Flow\\PostgreSql\\QueryBuilder\\Expression","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"FrameBound","namespace":"Flow\\PostgreSql\\QueryBuilder\\Clause","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBhIGZyYW1lIGJvdW5kIGZvciBOIFBSRUNFRElORy4KICov"},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/query.php","start_line_in_file":930,"slug":"frame-following","name":"frame_following","namespace":"Flow\\PostgreSql\\DSL","parameters":[{"name":"offset","type":[{"name":"Expression","namespace":"Flow\\PostgreSql\\QueryBuilder\\Expression","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"FrameBound","namespace":"Flow\\PostgreSql\\QueryBuilder\\Clause","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBhIGZyYW1lIGJvdW5kIGZvciBOIEZPTExPV0lORy4KICov"},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/query.php","start_line_in_file":943,"slug":"lock-for","name":"lock_for","namespace":"Flow\\PostgreSql\\DSL","parameters":[{"name":"strength","type":[{"name":"LockStrength","namespace":"Flow\\PostgreSql\\QueryBuilder\\Clause","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"tables","type":[{"name":"array","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"[]"},{"name":"waitPolicy","type":[{"name":"LockWaitPolicy","namespace":"Flow\\PostgreSql\\QueryBuilder\\Clause","is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"Flow\\PostgreSql\\QueryBuilder\\Clause\\LockWaitPolicy::..."}],"return_type":[{"name":"LockingClause","namespace":"Flow\\PostgreSql\\QueryBuilder\\Clause","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBhIGxvY2tpbmcgY2xhdXNlIChGT1IgVVBEQVRFLCBGT1IgU0hBUkUsIGV0Yy4pLgogKgogKiBAcGFyYW0gTG9ja1N0cmVuZ3RoICRzdHJlbmd0aCBMb2NrIHN0cmVuZ3RoCiAqIEBwYXJhbSBsaXN0PHN0cmluZz4gJHRhYmxlcyBUYWJsZXMgdG8gbG9jayAoZW1wdHkgZm9yIGFsbCkKICogQHBhcmFtIExvY2tXYWl0UG9saWN5ICR3YWl0UG9saWN5IFdhaXQgcG9saWN5CiAqLw=="},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/query.php","start_line_in_file":957,"slug":"for-update","name":"for_update","namespace":"Flow\\PostgreSql\\DSL","parameters":[{"name":"tables","type":[{"name":"array","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"[]"}],"return_type":[{"name":"LockingClause","namespace":"Flow\\PostgreSql\\QueryBuilder\\Clause","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBhIEZPUiBVUERBVEUgbG9ja2luZyBjbGF1c2UuCiAqCiAqIEBwYXJhbSBsaXN0PHN0cmluZz4gJHRhYmxlcyBUYWJsZXMgdG8gbG9jayAoZW1wdHkgZm9yIGFsbCkKICov"},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/query.php","start_line_in_file":968,"slug":"for-share","name":"for_share","namespace":"Flow\\PostgreSql\\DSL","parameters":[{"name":"tables","type":[{"name":"array","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"[]"}],"return_type":[{"name":"LockingClause","namespace":"Flow\\PostgreSql\\QueryBuilder\\Clause","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBhIEZPUiBTSEFSRSBsb2NraW5nIGNsYXVzZS4KICoKICogQHBhcmFtIGxpc3Q8c3RyaW5nPiAkdGFibGVzIFRhYmxlcyB0byBsb2NrIChlbXB0eSBmb3IgYWxsKQogKi8="},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/query.php","start_line_in_file":977,"slug":"on-conflict-nothing","name":"on_conflict_nothing","namespace":"Flow\\PostgreSql\\DSL","parameters":[{"name":"target","type":[{"name":"ConflictTarget","namespace":"Flow\\PostgreSql\\QueryBuilder\\Clause","is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"null"}],"return_type":[{"name":"OnConflictClause","namespace":"Flow\\PostgreSql\\QueryBuilder\\Clause","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBhbiBPTiBDT05GTElDVCBETyBOT1RISU5HIGNsYXVzZS4KICov"},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/query.php","start_line_in_file":989,"slug":"on-conflict-update","name":"on_conflict_update","namespace":"Flow\\PostgreSql\\DSL","parameters":[{"name":"target","type":[{"name":"ConflictTarget","namespace":"Flow\\PostgreSql\\QueryBuilder\\Clause","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"updates","type":[{"name":"array","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"OnConflictClause","namespace":"Flow\\PostgreSql\\QueryBuilder\\Clause","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBhbiBPTiBDT05GTElDVCBETyBVUERBVEUgY2xhdXNlLgogKgogKiBAcGFyYW0gQ29uZmxpY3RUYXJnZXQgJHRhcmdldCBDb25mbGljdCB0YXJnZXQgKGNvbHVtbnMgb3IgY29uc3RyYWludCkKICogQHBhcmFtIGFycmF5PHN0cmluZywgRXhwcmVzc2lvbnxzdHJpbmc+ICR1cGRhdGVzIENvbHVtbiB1cGRhdGVzCiAqLw=="},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/query.php","start_line_in_file":1003,"slug":"conflict-columns","name":"conflict_columns","namespace":"Flow\\PostgreSql\\DSL","parameters":[{"name":"columns","type":[{"name":"array","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"ConflictTarget","namespace":"Flow\\PostgreSql\\QueryBuilder\\Clause","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBhIGNvbmZsaWN0IHRhcmdldCBmb3IgT04gQ09ORkxJQ1QgKGNvbHVtbnMpLgogKgogKiBAcGFyYW0gbGlzdDxzdHJpbmc+ICRjb2x1bW5zIENvbHVtbnMgdGhhdCBkZWZpbmUgdW5pcXVlbmVzcwogKi8="},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/query.php","start_line_in_file":1012,"slug":"conflict-constraint","name":"conflict_constraint","namespace":"Flow\\PostgreSql\\DSL","parameters":[{"name":"name","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"ConflictTarget","namespace":"Flow\\PostgreSql\\QueryBuilder\\Clause","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBhIGNvbmZsaWN0IHRhcmdldCBmb3IgT04gQ09ORkxJQ1QgT04gQ09OU1RSQUlOVC4KICov"},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/query.php","start_line_in_file":1023,"slug":"returning","name":"returning","namespace":"Flow\\PostgreSql\\DSL","parameters":[{"name":"expressions","type":[{"name":"Expression","namespace":"Flow\\PostgreSql\\QueryBuilder\\Expression","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":true,"default_value":null}],"return_type":[{"name":"ReturningClause","namespace":"Flow\\PostgreSql\\QueryBuilder\\Clause","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBhIFJFVFVSTklORyBjbGF1c2UuCiAqCiAqIEBwYXJhbSBFeHByZXNzaW9ufHN0cmluZyAuLi4kZXhwcmVzc2lvbnMgRXhwcmVzc2lvbnMgdG8gcmV0dXJuCiAqLw=="},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/query.php","start_line_in_file":1034,"slug":"returning-all","name":"returning_all","namespace":"Flow\\PostgreSql\\DSL","parameters":[],"return_type":[{"name":"ReturningClause","namespace":"Flow\\PostgreSql\\QueryBuilder\\Clause","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBhIFJFVFVSTklORyAqIGNsYXVzZS4KICov"},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/query.php","start_line_in_file":1046,"slug":"begin","name":"begin","namespace":"Flow\\PostgreSql\\DSL","parameters":[],"return_type":[{"name":"BeginOptionsStep","namespace":"Flow\\PostgreSql\\QueryBuilder\\Transaction","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBhIEJFR0lOIHRyYW5zYWN0aW9uIGJ1aWxkZXIuCiAqCiAqIEV4YW1wbGU6IGJlZ2luKCktPmlzb2xhdGlvbkxldmVsKElzb2xhdGlvbkxldmVsOjpTRVJJQUxJWkFCTEUpLT5yZWFkT25seSgpCiAqIFByb2R1Y2VzOiBCRUdJTiBJU09MQVRJT04gTEVWRUwgU0VSSUFMSVpBQkxFIFJFQUQgT05MWQogKi8="},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/query.php","start_line_in_file":1058,"slug":"commit","name":"commit","namespace":"Flow\\PostgreSql\\DSL","parameters":[],"return_type":[{"name":"CommitOptionsStep","namespace":"Flow\\PostgreSql\\QueryBuilder\\Transaction","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBhIENPTU1JVCB0cmFuc2FjdGlvbiBidWlsZGVyLgogKgogKiBFeGFtcGxlOiBjb21taXQoKS0+YW5kQ2hhaW4oKQogKiBQcm9kdWNlczogQ09NTUlUIEFORCBDSEFJTgogKi8="},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/query.php","start_line_in_file":1070,"slug":"rollback","name":"rollback","namespace":"Flow\\PostgreSql\\DSL","parameters":[],"return_type":[{"name":"RollbackOptionsStep","namespace":"Flow\\PostgreSql\\QueryBuilder\\Transaction","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBhIFJPTExCQUNLIHRyYW5zYWN0aW9uIGJ1aWxkZXIuCiAqCiAqIEV4YW1wbGU6IHJvbGxiYWNrKCktPnRvU2F2ZXBvaW50KCdteV9zYXZlcG9pbnQnKQogKiBQcm9kdWNlczogUk9MTEJBQ0sgVE8gU0FWRVBPSU5UIG15X3NhdmVwb2ludAogKi8="},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/query.php","start_line_in_file":1082,"slug":"savepoint","name":"savepoint","namespace":"Flow\\PostgreSql\\DSL","parameters":[{"name":"name","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"SavepointFinalStep","namespace":"Flow\\PostgreSql\\QueryBuilder\\Transaction","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBhIFNBVkVQT0lOVC4KICoKICogRXhhbXBsZTogc2F2ZXBvaW50KCdteV9zYXZlcG9pbnQnKQogKiBQcm9kdWNlczogU0FWRVBPSU5UIG15X3NhdmVwb2ludAogKi8="},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/query.php","start_line_in_file":1094,"slug":"release-savepoint","name":"release_savepoint","namespace":"Flow\\PostgreSql\\DSL","parameters":[{"name":"name","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"SavepointFinalStep","namespace":"Flow\\PostgreSql\\QueryBuilder\\Transaction","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIFJlbGVhc2UgYSBTQVZFUE9JTlQuCiAqCiAqIEV4YW1wbGU6IHJlbGVhc2Vfc2F2ZXBvaW50KCdteV9zYXZlcG9pbnQnKQogKiBQcm9kdWNlczogUkVMRUFTRSBteV9zYXZlcG9pbnQKICov"},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/query.php","start_line_in_file":1106,"slug":"set-transaction","name":"set_transaction","namespace":"Flow\\PostgreSql\\DSL","parameters":[],"return_type":[{"name":"SetTransactionOptionsStep","namespace":"Flow\\PostgreSql\\QueryBuilder\\Transaction","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBhIFNFVCBUUkFOU0FDVElPTiBidWlsZGVyLgogKgogKiBFeGFtcGxlOiBzZXRfdHJhbnNhY3Rpb24oKS0+aXNvbGF0aW9uTGV2ZWwoSXNvbGF0aW9uTGV2ZWw6OlNFUklBTElaQUJMRSktPnJlYWRPbmx5KCkKICogUHJvZHVjZXM6IFNFVCBUUkFOU0FDVElPTiBJU09MQVRJT04gTEVWRUwgU0VSSUFMSVpBQkxFLCBSRUFEIE9OTFkKICov"},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/query.php","start_line_in_file":1118,"slug":"set-session-transaction","name":"set_session_transaction","namespace":"Flow\\PostgreSql\\DSL","parameters":[],"return_type":[{"name":"SetTransactionOptionsStep","namespace":"Flow\\PostgreSql\\QueryBuilder\\Transaction","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBhIFNFVCBTRVNTSU9OIENIQVJBQ1RFUklTVElDUyBBUyBUUkFOU0FDVElPTiBidWlsZGVyLgogKgogKiBFeGFtcGxlOiBzZXRfc2Vzc2lvbl90cmFuc2FjdGlvbigpLT5pc29sYXRpb25MZXZlbChJc29sYXRpb25MZXZlbDo6U0VSSUFMSVpBQkxFKQogKiBQcm9kdWNlczogU0VUIFNFU1NJT04gQ0hBUkFDVEVSSVNUSUNTIEFTIFRSQU5TQUNUSU9OIElTT0xBVElPTiBMRVZFTCBTRVJJQUxJWkFCTEUKICov"},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/query.php","start_line_in_file":1130,"slug":"transaction-snapshot","name":"transaction_snapshot","namespace":"Flow\\PostgreSql\\DSL","parameters":[{"name":"snapshotId","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"SetTransactionFinalStep","namespace":"Flow\\PostgreSql\\QueryBuilder\\Transaction","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBhIFNFVCBUUkFOU0FDVElPTiBTTkFQU0hPVCBidWlsZGVyLgogKgogKiBFeGFtcGxlOiB0cmFuc2FjdGlvbl9zbmFwc2hvdCgnMDAwMDAwMDMtMDAwMDAwMUEtMScpCiAqIFByb2R1Y2VzOiBTRVQgVFJBTlNBQ1RJT04gU05BUFNIT1QgJzAwMDAwMDAzLTAwMDAwMDFBLTEnCiAqLw=="},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/query.php","start_line_in_file":1142,"slug":"prepare-transaction","name":"prepare_transaction","namespace":"Flow\\PostgreSql\\DSL","parameters":[{"name":"transactionId","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"PreparedTransactionFinalStep","namespace":"Flow\\PostgreSql\\QueryBuilder\\Transaction","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBhIFBSRVBBUkUgVFJBTlNBQ1RJT04gYnVpbGRlci4KICoKICogRXhhbXBsZTogcHJlcGFyZV90cmFuc2FjdGlvbignbXlfdHJhbnNhY3Rpb24nKQogKiBQcm9kdWNlczogUFJFUEFSRSBUUkFOU0FDVElPTiAnbXlfdHJhbnNhY3Rpb24nCiAqLw=="},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/query.php","start_line_in_file":1154,"slug":"commit-prepared","name":"commit_prepared","namespace":"Flow\\PostgreSql\\DSL","parameters":[{"name":"transactionId","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"PreparedTransactionFinalStep","namespace":"Flow\\PostgreSql\\QueryBuilder\\Transaction","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBhIENPTU1JVCBQUkVQQVJFRCBidWlsZGVyLgogKgogKiBFeGFtcGxlOiBjb21taXRfcHJlcGFyZWQoJ215X3RyYW5zYWN0aW9uJykKICogUHJvZHVjZXM6IENPTU1JVCBQUkVQQVJFRCAnbXlfdHJhbnNhY3Rpb24nCiAqLw=="},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/query.php","start_line_in_file":1166,"slug":"rollback-prepared","name":"rollback_prepared","namespace":"Flow\\PostgreSql\\DSL","parameters":[{"name":"transactionId","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"PreparedTransactionFinalStep","namespace":"Flow\\PostgreSql\\QueryBuilder\\Transaction","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBhIFJPTExCQUNLIFBSRVBBUkVEIGJ1aWxkZXIuCiAqCiAqIEV4YW1wbGU6IHJvbGxiYWNrX3ByZXBhcmVkKCdteV90cmFuc2FjdGlvbicpCiAqIFByb2R1Y2VzOiBST0xMQkFDSyBQUkVQQVJFRCAnbXlfdHJhbnNhY3Rpb24nCiAqLw=="},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/query.php","start_line_in_file":1189,"slug":"declare-cursor","name":"declare_cursor","namespace":"Flow\\PostgreSql\\DSL","parameters":[{"name":"cursorName","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"query","type":[{"name":"SelectFinalStep","namespace":"Flow\\PostgreSql\\QueryBuilder\\Select","is_nullable":false,"is_variadic":false},{"name":"Sql","namespace":"Flow\\PostgreSql\\QueryBuilder","is_nullable":false,"is_variadic":false},{"name":"ParsedQuery","namespace":"Flow\\PostgreSql","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"DeclareCursorOptionsStep","namespace":"Flow\\PostgreSql\\QueryBuilder\\Cursor","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIERlY2xhcmUgYSBzZXJ2ZXItc2lkZSBjdXJzb3IgZm9yIGEgcXVlcnkuCiAqCiAqIEN1cnNvcnMgbXVzdCBiZSBkZWNsYXJlZCB3aXRoaW4gYSB0cmFuc2FjdGlvbiBhbmQgcHJvdmlkZSBtZW1vcnktZWZmaWNpZW50CiAqIGl0ZXJhdGlvbiBvdmVyIGxhcmdlIHJlc3VsdCBzZXRzIHZpYSBGRVRDSCBjb21tYW5kcy4KICoKICogRXhhbXBsZSB3aXRoIHF1ZXJ5IGJ1aWxkZXI6CiAqICAgZGVjbGFyZV9jdXJzb3IoJ215X2N1cnNvcicsIHNlbGVjdChzdGFyKCkpLT5mcm9tKHRhYmxlKCd1c2VycycpKSktPm5vU2Nyb2xsKCkKICogICBQcm9kdWNlczogREVDTEFSRSBteV9jdXJzb3IgTk8gU0NST0xMIENVUlNPUiBGT1IgU0VMRUNUICogRlJPTSB1c2VycwogKgogKiBFeGFtcGxlIHdpdGggcmF3IFNRTDoKICogICBkZWNsYXJlX2N1cnNvcignbXlfY3Vyc29yJywgJ1NFTEVDVCAqIEZST00gdXNlcnMgV0hFUkUgYWN0aXZlID0gdHJ1ZScpLT53aXRoSG9sZCgpCiAqICAgUHJvZHVjZXM6IERFQ0xBUkUgbXlfY3Vyc29yIE5PIFNDUk9MTCBDVVJTT1IgV0lUSCBIT0xEIEZPUiBTRUxFQ1QgKiBGUk9NIHVzZXJzIFdIRVJFIGFjdGl2ZSA9IHRydWUKICoKICogQHBhcmFtIHN0cmluZyAkY3Vyc29yTmFtZSBVbmlxdWUgY3Vyc29yIG5hbWUKICogQHBhcmFtIFBhcnNlZFF1ZXJ5fFNlbGVjdEZpbmFsU3RlcHxTcWx8c3RyaW5nICRxdWVyeSBRdWVyeSB0byBpdGVyYXRlIG92ZXIKICov"},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/query.php","start_line_in_file":1214,"slug":"fetch","name":"fetch","namespace":"Flow\\PostgreSql\\DSL","parameters":[{"name":"cursorName","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"FetchCursorBuilder","namespace":"Flow\\PostgreSql\\QueryBuilder\\Cursor","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIEZldGNoIHJvd3MgZnJvbSBhIGN1cnNvci4KICoKICogRXhhbXBsZTogZmV0Y2goJ215X2N1cnNvcicpLT5mb3J3YXJkKDEwMCkKICogUHJvZHVjZXM6IEZFVENIIEZPUldBUkQgMTAwIG15X2N1cnNvcgogKgogKiBFeGFtcGxlOiBmZXRjaCgnbXlfY3Vyc29yJyktPmFsbCgpCiAqIFByb2R1Y2VzOiBGRVRDSCBBTEwgbXlfY3Vyc29yCiAqCiAqIEBwYXJhbSBzdHJpbmcgJGN1cnNvck5hbWUgQ3Vyc29yIHRvIGZldGNoIGZyb20KICov"},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/query.php","start_line_in_file":1231,"slug":"close-cursor","name":"close_cursor","namespace":"Flow\\PostgreSql\\DSL","parameters":[{"name":"cursorName","type":[{"name":"string","namespace":null,"is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"null"}],"return_type":[{"name":"CloseCursorFinalStep","namespace":"Flow\\PostgreSql\\QueryBuilder\\Cursor","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENsb3NlIGEgY3Vyc29yLgogKgogKiBFeGFtcGxlOiBjbG9zZV9jdXJzb3IoJ215X2N1cnNvcicpCiAqIFByb2R1Y2VzOiBDTE9TRSBteV9jdXJzb3IKICoKICogRXhhbXBsZTogY2xvc2VfY3Vyc29yKCkgLSBjbG9zZXMgYWxsIGN1cnNvcnMKICogUHJvZHVjZXM6IENMT1NFIEFMTAogKgogKiBAcGFyYW0gbnVsbHxzdHJpbmcgJGN1cnNvck5hbWUgQ3Vyc29yIHRvIGNsb3NlLCBvciBudWxsIHRvIGNsb3NlIGFsbAogKi8="},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/condition.php","start_line_in_file":42,"slug":"eq","name":"eq","namespace":"Flow\\PostgreSql\\DSL","parameters":[{"name":"left","type":[{"name":"Expression","namespace":"Flow\\PostgreSql\\QueryBuilder\\Expression","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"right","type":[{"name":"Expression","namespace":"Flow\\PostgreSql\\QueryBuilder\\Expression","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"Comparison","namespace":"Flow\\PostgreSql\\QueryBuilder\\Condition","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBhbiBlcXVhbGl0eSBjb21wYXJpc29uIChjb2x1bW4gPSB2YWx1ZSkuCiAqLw=="},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/condition.php","start_line_in_file":55,"slug":"ne","name":"ne","namespace":"Flow\\PostgreSql\\DSL","parameters":[{"name":"left","type":[{"name":"Expression","namespace":"Flow\\PostgreSql\\QueryBuilder\\Expression","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"right","type":[{"name":"Expression","namespace":"Flow\\PostgreSql\\QueryBuilder\\Expression","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"Comparison","namespace":"Flow\\PostgreSql\\QueryBuilder\\Condition","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBhIG5vdC1lcXVhbCBjb21wYXJpc29uIChjb2x1bW4gIT0gdmFsdWUpLgogKi8="},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/condition.php","start_line_in_file":68,"slug":"lt","name":"lt","namespace":"Flow\\PostgreSql\\DSL","parameters":[{"name":"left","type":[{"name":"Expression","namespace":"Flow\\PostgreSql\\QueryBuilder\\Expression","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"right","type":[{"name":"Expression","namespace":"Flow\\PostgreSql\\QueryBuilder\\Expression","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"Comparison","namespace":"Flow\\PostgreSql\\QueryBuilder\\Condition","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBhIGxlc3MtdGhhbiBjb21wYXJpc29uIChjb2x1bW4gPCB2YWx1ZSkuCiAqLw=="},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/condition.php","start_line_in_file":81,"slug":"le","name":"le","namespace":"Flow\\PostgreSql\\DSL","parameters":[{"name":"left","type":[{"name":"Expression","namespace":"Flow\\PostgreSql\\QueryBuilder\\Expression","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"right","type":[{"name":"Expression","namespace":"Flow\\PostgreSql\\QueryBuilder\\Expression","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"Comparison","namespace":"Flow\\PostgreSql\\QueryBuilder\\Condition","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBhIGxlc3MtdGhhbi1vci1lcXVhbCBjb21wYXJpc29uIChjb2x1bW4gPD0gdmFsdWUpLgogKi8="},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/condition.php","start_line_in_file":94,"slug":"gt","name":"gt","namespace":"Flow\\PostgreSql\\DSL","parameters":[{"name":"left","type":[{"name":"Expression","namespace":"Flow\\PostgreSql\\QueryBuilder\\Expression","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"right","type":[{"name":"Expression","namespace":"Flow\\PostgreSql\\QueryBuilder\\Expression","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"Comparison","namespace":"Flow\\PostgreSql\\QueryBuilder\\Condition","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBhIGdyZWF0ZXItdGhhbiBjb21wYXJpc29uIChjb2x1bW4gPiB2YWx1ZSkuCiAqLw=="},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/condition.php","start_line_in_file":107,"slug":"ge","name":"ge","namespace":"Flow\\PostgreSql\\DSL","parameters":[{"name":"left","type":[{"name":"Expression","namespace":"Flow\\PostgreSql\\QueryBuilder\\Expression","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"right","type":[{"name":"Expression","namespace":"Flow\\PostgreSql\\QueryBuilder\\Expression","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"Comparison","namespace":"Flow\\PostgreSql\\QueryBuilder\\Condition","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBhIGdyZWF0ZXItdGhhbi1vci1lcXVhbCBjb21wYXJpc29uIChjb2x1bW4gPj0gdmFsdWUpLgogKi8="},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/condition.php","start_line_in_file":120,"slug":"between","name":"between","namespace":"Flow\\PostgreSql\\DSL","parameters":[{"name":"expr","type":[{"name":"Expression","namespace":"Flow\\PostgreSql\\QueryBuilder\\Expression","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"low","type":[{"name":"Expression","namespace":"Flow\\PostgreSql\\QueryBuilder\\Expression","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"high","type":[{"name":"Expression","namespace":"Flow\\PostgreSql\\QueryBuilder\\Expression","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"not","type":[{"name":"bool","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"false"}],"return_type":[{"name":"Between","namespace":"Flow\\PostgreSql\\QueryBuilder\\Condition","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBhIEJFVFdFRU4gY29uZGl0aW9uLgogKi8="},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/condition.php","start_line_in_file":139,"slug":"in","name":"in_","namespace":"Flow\\PostgreSql\\DSL","parameters":[{"name":"expr","type":[{"name":"Expression","namespace":"Flow\\PostgreSql\\QueryBuilder\\Expression","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"values","type":[{"name":"array","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"In","namespace":"Flow\\PostgreSql\\QueryBuilder\\Condition","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBhbiBJTiBjb25kaXRpb24uCiAqCiAqIEBwYXJhbSBFeHByZXNzaW9ufHN0cmluZyAkZXhwciBFeHByZXNzaW9uIHRvIGNoZWNrCiAqIEBwYXJhbSBsaXN0PEV4cHJlc3Npb24+ICR2YWx1ZXMgTGlzdCBvZiB2YWx1ZXMgKG11c3QgYmUgbm9uLWVtcHR5KQogKgogKiBAdGhyb3dzIFxJbnZhbGlkQXJndW1lbnRFeGNlcHRpb24gd2hlbiB2YWx1ZXMgYXJyYXkgaXMgZW1wdHkKICov"},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/condition.php","start_line_in_file":156,"slug":"is-null","name":"is_null","namespace":"Flow\\PostgreSql\\DSL","parameters":[{"name":"expr","type":[{"name":"Expression","namespace":"Flow\\PostgreSql\\QueryBuilder\\Expression","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"not","type":[{"name":"bool","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"false"}],"return_type":[{"name":"IsNull","namespace":"Flow\\PostgreSql\\QueryBuilder\\Condition","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBhbiBJUyBOVUxMIGNvbmRpdGlvbi4KICov"},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/condition.php","start_line_in_file":165,"slug":"like","name":"like","namespace":"Flow\\PostgreSql\\DSL","parameters":[{"name":"expr","type":[{"name":"Expression","namespace":"Flow\\PostgreSql\\QueryBuilder\\Expression","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"pattern","type":[{"name":"Expression","namespace":"Flow\\PostgreSql\\QueryBuilder\\Expression","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"caseInsensitive","type":[{"name":"bool","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"false"},{"name":"negated","type":[{"name":"bool","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"false"}],"return_type":[{"name":"Like","namespace":"Flow\\PostgreSql\\QueryBuilder\\Condition","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBhIExJS0UgY29uZGl0aW9uLgogKi8="},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/condition.php","start_line_in_file":183,"slug":"similar-to","name":"similar_to","namespace":"Flow\\PostgreSql\\DSL","parameters":[{"name":"expr","type":[{"name":"Expression","namespace":"Flow\\PostgreSql\\QueryBuilder\\Expression","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"pattern","type":[{"name":"Expression","namespace":"Flow\\PostgreSql\\QueryBuilder\\Expression","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"SimilarTo","namespace":"Flow\\PostgreSql\\QueryBuilder\\Condition","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBhIFNJTUlMQVIgVE8gY29uZGl0aW9uLgogKi8="},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/condition.php","start_line_in_file":195,"slug":"distinct-from","name":"distinct_from","namespace":"Flow\\PostgreSql\\DSL","parameters":[{"name":"left","type":[{"name":"Expression","namespace":"Flow\\PostgreSql\\QueryBuilder\\Expression","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"right","type":[{"name":"Expression","namespace":"Flow\\PostgreSql\\QueryBuilder\\Expression","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"not","type":[{"name":"bool","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"false"}],"return_type":[{"name":"IsDistinctFrom","namespace":"Flow\\PostgreSql\\QueryBuilder\\Condition","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBhbiBJUyBESVNUSU5DVCBGUk9NIGNvbmRpdGlvbi4KICov"},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/condition.php","start_line_in_file":208,"slug":"exists","name":"exists","namespace":"Flow\\PostgreSql\\DSL","parameters":[{"name":"subquery","type":[{"name":"SelectFinalStep","namespace":"Flow\\PostgreSql\\QueryBuilder\\Select","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"Exists","namespace":"Flow\\PostgreSql\\QueryBuilder\\Condition","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBhbiBFWElTVFMgY29uZGl0aW9uLgogKi8="},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/condition.php","start_line_in_file":223,"slug":"any","name":"any_","namespace":"Flow\\PostgreSql\\DSL","parameters":[{"name":"left","type":[{"name":"Expression","namespace":"Flow\\PostgreSql\\QueryBuilder\\Expression","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"operator","type":[{"name":"ComparisonOperator","namespace":"Flow\\PostgreSql\\QueryBuilder\\Condition","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"arrayOrSubquery","type":[{"name":"Expression","namespace":"Flow\\PostgreSql\\QueryBuilder\\Expression","is_nullable":false,"is_variadic":false},{"name":"SelectFinalStep","namespace":"Flow\\PostgreSql\\QueryBuilder\\Select","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"Any","namespace":"Flow\\PostgreSql\\QueryBuilder\\Condition","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBhbiBBTlkgY29uZGl0aW9uIHdpdGggYSBzdWJxdWVyeSBvciBhcnJheSBleHByZXNzaW9uLgogKgogKiBFeGFtcGxlOiBhbnlfKGNvbCgnaWQnKSwgQ29tcGFyaXNvbk9wZXJhdG9yOjpFUSwgc2VsZWN0KGNvbCgndXNlcl9pZCcpKS0+ZnJvbSh0YWJsZSgnb3JkZXJzJykpKQogKiBFeGFtcGxlOiBhbnlfKGNvbCgnYXR0bnVtJywgJ2EnKSwgQ29tcGFyaXNvbk9wZXJhdG9yOjpFUSwgY29sKCdjb25rZXknLCAnY29uJykpCiAqLw=="},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/condition.php","start_line_in_file":243,"slug":"all","name":"all_","namespace":"Flow\\PostgreSql\\DSL","parameters":[{"name":"left","type":[{"name":"Expression","namespace":"Flow\\PostgreSql\\QueryBuilder\\Expression","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"operator","type":[{"name":"ComparisonOperator","namespace":"Flow\\PostgreSql\\QueryBuilder\\Condition","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"arrayOrSubquery","type":[{"name":"Expression","namespace":"Flow\\PostgreSql\\QueryBuilder\\Expression","is_nullable":false,"is_variadic":false},{"name":"SelectFinalStep","namespace":"Flow\\PostgreSql\\QueryBuilder\\Select","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"All","namespace":"Flow\\PostgreSql\\QueryBuilder\\Condition","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBhbiBBTEwgY29uZGl0aW9uIHdpdGggYSBzdWJxdWVyeSBvciBhcnJheSBleHByZXNzaW9uLgogKgogKiBFeGFtcGxlOiBhbGxfKGNvbCgnaWQnKSwgQ29tcGFyaXNvbk9wZXJhdG9yOjpFUSwgc2VsZWN0KGNvbCgndXNlcl9pZCcpKS0+ZnJvbSh0YWJsZSgnb3JkZXJzJykpKQogKiBFeGFtcGxlOiBhbGxfKGNvbCgndmFsdWUnKSwgQ29tcGFyaXNvbk9wZXJhdG9yOjpHVCwgY29sKCd0aHJlc2hvbGRzJykpCiAqLw=="},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/condition.php","start_line_in_file":262,"slug":"is-true","name":"is_true","namespace":"Flow\\PostgreSql\\DSL","parameters":[{"name":"expr","type":[{"name":"Expression","namespace":"Flow\\PostgreSql\\QueryBuilder\\Expression","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"BooleanCondition","namespace":"Flow\\PostgreSql\\QueryBuilder\\Condition","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIFdyYXAgYW4gZXhwcmVzc2lvbiBhcyBhIGJvb2xlYW4gY29uZGl0aW9uIGZvciB1c2UgaW4gV0hFUkUvSEFWSU5HL0pPSU4gT04uCiAqCiAqIEV4YW1wbGU6IGlzX3RydWUoY29sKCdpc19hY3RpdmUnKSkg4oCUIHVzZXMgYSBib29sZWFuIGNvbHVtbiBpbiBXSEVSRSBjbGF1c2UuCiAqLw=="},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/condition.php","start_line_in_file":274,"slug":"not-like","name":"not_like","namespace":"Flow\\PostgreSql\\DSL","parameters":[{"name":"expr","type":[{"name":"Expression","namespace":"Flow\\PostgreSql\\QueryBuilder\\Expression","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"pattern","type":[{"name":"Expression","namespace":"Flow\\PostgreSql\\QueryBuilder\\Expression","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"caseInsensitive","type":[{"name":"bool","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"false"}],"return_type":[{"name":"Like","namespace":"Flow\\PostgreSql\\QueryBuilder\\Condition","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBhIE5PVCBMSUtFIGNvbmRpdGlvbi4KICoKICogRXhhbXBsZTogbm90X2xpa2UoY29sKCduYW1lJyksIGxpdGVyYWwoJ3BnXyUnKSkKICogUHJvZHVjZXM6IG5hbWUgTk9UIExJS0UgJ3BnXyUnCiAqLw=="},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/condition.php","start_line_in_file":302,"slug":"conditions","name":"conditions","namespace":"Flow\\PostgreSql\\DSL","parameters":[],"return_type":[{"name":"ConditionBuilder","namespace":"Flow\\PostgreSql\\QueryBuilder\\Condition","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBhIGNvbmRpdGlvbiBidWlsZGVyIGZvciBmbHVlbnQgY29uZGl0aW9uIGNvbXBvc2l0aW9uLgogKgogKiBUaGlzIGJ1aWxkZXIgYWxsb3dzIGluY3JlbWVudGFsIGNvbmRpdGlvbiBidWlsZGluZyB3aXRoIGEgZmx1ZW50IEFQSToKICoKICogYGBgcGhwCiAqICRidWlsZGVyID0gY29uZGl0aW9ucygpOwogKgogKiBpZiAoJGhhc0ZpbHRlcikgewogKiAgICAgJGJ1aWxkZXIgPSAkYnVpbGRlci0+YW5kKGVxKGNvbCgnc3RhdHVzJyksIGxpdGVyYWwoJ2FjdGl2ZScpKSk7CiAqIH0KICoKICogaWYgKCEkYnVpbGRlci0+aXNFbXB0eSgpKSB7CiAqICAgICAkcXVlcnkgPSBzZWxlY3QoKS0+ZnJvbSh0YWJsZSgndXNlcnMnKSktPndoZXJlKCRidWlsZGVyKTsKICogfQogKiBgYGAKICov"},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/condition.php","start_line_in_file":313,"slug":"and","name":"and_","namespace":"Flow\\PostgreSql\\DSL","parameters":[{"name":"conditions","type":[{"name":"Condition","namespace":"Flow\\PostgreSql\\QueryBuilder\\Condition","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":true,"default_value":null}],"return_type":[{"name":"AndCondition","namespace":"Flow\\PostgreSql\\QueryBuilder\\Condition","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENvbWJpbmUgY29uZGl0aW9ucyB3aXRoIEFORC4KICoKICogQHBhcmFtIENvbmRpdGlvbiAuLi4kY29uZGl0aW9ucyBDb25kaXRpb25zIHRvIGNvbWJpbmUKICov"},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/condition.php","start_line_in_file":324,"slug":"or","name":"or_","namespace":"Flow\\PostgreSql\\DSL","parameters":[{"name":"conditions","type":[{"name":"Condition","namespace":"Flow\\PostgreSql\\QueryBuilder\\Condition","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":true,"default_value":null}],"return_type":[{"name":"OrCondition","namespace":"Flow\\PostgreSql\\QueryBuilder\\Condition","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENvbWJpbmUgY29uZGl0aW9ucyB3aXRoIE9SLgogKgogKiBAcGFyYW0gQ29uZGl0aW9uIC4uLiRjb25kaXRpb25zIENvbmRpdGlvbnMgdG8gY29tYmluZQogKi8="},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/condition.php","start_line_in_file":336,"slug":"not","name":"not_","namespace":"Flow\\PostgreSql\\DSL","parameters":[{"name":"expression","type":[{"name":"Expression","namespace":"Flow\\PostgreSql\\QueryBuilder\\Expression","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"NotCondition","namespace":"Flow\\PostgreSql\\QueryBuilder\\Condition","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIE5lZ2F0ZSBhIGNvbmRpdGlvbiBvciBleHByZXNzaW9uIHdpdGggTk9ULgogKgogKiBBY2NlcHRzIGJvdGggQ29uZGl0aW9uIGFuZCBFeHByZXNzaW9uIOKAlCBOT1QgYWx3YXlzIHByb2R1Y2VzIGEgYm9vbGVhbiByZXN1bHQuCiAqIENhbiBiZSB1c2VkIGluIFdIRVJFIGNsYXVzZXMgYW5kIFNFTEVDVCBsaXN0cyAodmlhIC0+YXMoJ2FsaWFzJykpLgogKi8="},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/condition.php","start_line_in_file":348,"slug":"json-contains","name":"json_contains","namespace":"Flow\\PostgreSql\\DSL","parameters":[{"name":"left","type":[{"name":"Expression","namespace":"Flow\\PostgreSql\\QueryBuilder\\Expression","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"right","type":[{"name":"Expression","namespace":"Flow\\PostgreSql\\QueryBuilder\\Expression","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"OperatorCondition","namespace":"Flow\\PostgreSql\\QueryBuilder\\Condition","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBhIEpTT05CIGNvbnRhaW5zIGNvbmRpdGlvbiAoQD4pLgogKgogKiBFeGFtcGxlOiBqc29uX2NvbnRhaW5zKGNvbCgnbWV0YWRhdGEnKSwgbGl0ZXJhbF9qc29uKCd7ImNhdGVnb3J5IjogImVsZWN0cm9uaWNzIn0nKSkKICogUHJvZHVjZXM6IG1ldGFkYXRhIEA+ICd7ImNhdGVnb3J5IjogImVsZWN0cm9uaWNzIn0nCiAqLw=="},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/condition.php","start_line_in_file":364,"slug":"json-contained-by","name":"json_contained_by","namespace":"Flow\\PostgreSql\\DSL","parameters":[{"name":"left","type":[{"name":"Expression","namespace":"Flow\\PostgreSql\\QueryBuilder\\Expression","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"right","type":[{"name":"Expression","namespace":"Flow\\PostgreSql\\QueryBuilder\\Expression","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"OperatorCondition","namespace":"Flow\\PostgreSql\\QueryBuilder\\Condition","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBhIEpTT05CIGlzIGNvbnRhaW5lZCBieSBjb25kaXRpb24gKDxAKS4KICoKICogRXhhbXBsZToganNvbl9jb250YWluZWRfYnkoY29sKCdtZXRhZGF0YScpLCBsaXRlcmFsX2pzb24oJ3siY2F0ZWdvcnkiOiAiZWxlY3Ryb25pY3MiLCAicHJpY2UiOiAxMDB9JykpCiAqIFByb2R1Y2VzOiBtZXRhZGF0YSA8QCAneyJjYXRlZ29yeSI6ICJlbGVjdHJvbmljcyIsICJwcmljZSI6IDEwMH0nCiAqLw=="},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/condition.php","start_line_in_file":381,"slug":"json-get","name":"json_get","namespace":"Flow\\PostgreSql\\DSL","parameters":[{"name":"expr","type":[{"name":"Expression","namespace":"Flow\\PostgreSql\\QueryBuilder\\Expression","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"key","type":[{"name":"Expression","namespace":"Flow\\PostgreSql\\QueryBuilder\\Expression","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"BinaryExpression","namespace":"Flow\\PostgreSql\\QueryBuilder\\Expression","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBhIEpTT04gZmllbGQgYWNjZXNzIGV4cHJlc3Npb24gKC0+KS4KICogUmV0dXJucyBKU09OLgogKgogKiBFeGFtcGxlOiBqc29uX2dldChjb2woJ21ldGFkYXRhJyksIGxpdGVyYWxfc3RyaW5nKCdjYXRlZ29yeScpKQogKiBQcm9kdWNlczogbWV0YWRhdGEgLT4gJ2NhdGVnb3J5JwogKi8="},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/condition.php","start_line_in_file":398,"slug":"json-get-text","name":"json_get_text","namespace":"Flow\\PostgreSql\\DSL","parameters":[{"name":"expr","type":[{"name":"Expression","namespace":"Flow\\PostgreSql\\QueryBuilder\\Expression","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"key","type":[{"name":"Expression","namespace":"Flow\\PostgreSql\\QueryBuilder\\Expression","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"BinaryExpression","namespace":"Flow\\PostgreSql\\QueryBuilder\\Expression","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBhIEpTT04gZmllbGQgYWNjZXNzIGV4cHJlc3Npb24gKC0+PikuCiAqIFJldHVybnMgdGV4dC4KICoKICogRXhhbXBsZToganNvbl9nZXRfdGV4dChjb2woJ21ldGFkYXRhJyksIGxpdGVyYWxfc3RyaW5nKCduYW1lJykpCiAqIFByb2R1Y2VzOiBtZXRhZGF0YSAtPj4gJ25hbWUnCiAqLw=="},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/condition.php","start_line_in_file":415,"slug":"json-path","name":"json_path","namespace":"Flow\\PostgreSql\\DSL","parameters":[{"name":"expr","type":[{"name":"Expression","namespace":"Flow\\PostgreSql\\QueryBuilder\\Expression","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"path","type":[{"name":"Expression","namespace":"Flow\\PostgreSql\\QueryBuilder\\Expression","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"BinaryExpression","namespace":"Flow\\PostgreSql\\QueryBuilder\\Expression","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBhIEpTT04gcGF0aCBhY2Nlc3MgZXhwcmVzc2lvbiAoIz4pLgogKiBSZXR1cm5zIEpTT04uCiAqCiAqIEV4YW1wbGU6IGpzb25fcGF0aChjb2woJ21ldGFkYXRhJyksIGxpdGVyYWxfc3RyaW5nKCd7Y2F0ZWdvcnksbmFtZX0nKSkKICogUHJvZHVjZXM6IG1ldGFkYXRhICM+ICd7Y2F0ZWdvcnksbmFtZX0nCiAqLw=="},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/condition.php","start_line_in_file":432,"slug":"json-path-text","name":"json_path_text","namespace":"Flow\\PostgreSql\\DSL","parameters":[{"name":"expr","type":[{"name":"Expression","namespace":"Flow\\PostgreSql\\QueryBuilder\\Expression","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"path","type":[{"name":"Expression","namespace":"Flow\\PostgreSql\\QueryBuilder\\Expression","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"BinaryExpression","namespace":"Flow\\PostgreSql\\QueryBuilder\\Expression","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBhIEpTT04gcGF0aCBhY2Nlc3MgZXhwcmVzc2lvbiAoIz4+KS4KICogUmV0dXJucyB0ZXh0LgogKgogKiBFeGFtcGxlOiBqc29uX3BhdGhfdGV4dChjb2woJ21ldGFkYXRhJyksIGxpdGVyYWxfc3RyaW5nKCd7Y2F0ZWdvcnksbmFtZX0nKSkKICogUHJvZHVjZXM6IG1ldGFkYXRhICM+PiAne2NhdGVnb3J5LG5hbWV9JwogKi8="},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/condition.php","start_line_in_file":448,"slug":"json-exists","name":"json_exists","namespace":"Flow\\PostgreSql\\DSL","parameters":[{"name":"expr","type":[{"name":"Expression","namespace":"Flow\\PostgreSql\\QueryBuilder\\Expression","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"key","type":[{"name":"Expression","namespace":"Flow\\PostgreSql\\QueryBuilder\\Expression","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"OperatorCondition","namespace":"Flow\\PostgreSql\\QueryBuilder\\Condition","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBhIEpTT05CIGtleSBleGlzdHMgY29uZGl0aW9uICg\/KS4KICoKICogRXhhbXBsZToganNvbl9leGlzdHMoY29sKCdtZXRhZGF0YScpLCBsaXRlcmFsX3N0cmluZygnY2F0ZWdvcnknKSkKICogUHJvZHVjZXM6IG1ldGFkYXRhID8gJ2NhdGVnb3J5JwogKi8="},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/condition.php","start_line_in_file":464,"slug":"json-exists-any","name":"json_exists_any","namespace":"Flow\\PostgreSql\\DSL","parameters":[{"name":"expr","type":[{"name":"Expression","namespace":"Flow\\PostgreSql\\QueryBuilder\\Expression","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"keys","type":[{"name":"Expression","namespace":"Flow\\PostgreSql\\QueryBuilder\\Expression","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"OperatorCondition","namespace":"Flow\\PostgreSql\\QueryBuilder\\Condition","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBhIEpTT05CIGFueSBrZXkgZXhpc3RzIGNvbmRpdGlvbiAoP3wpLgogKgogKiBFeGFtcGxlOiBqc29uX2V4aXN0c19hbnkoY29sKCdtZXRhZGF0YScpLCBhcnJheV9leHByKFtsaXRlcmFsKCdjYXRlZ29yeScpLCBsaXRlcmFsKCduYW1lJyldKSkKICogUHJvZHVjZXM6IG1ldGFkYXRhID98IGFycmF5WydjYXRlZ29yeScsICduYW1lJ10KICov"},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/condition.php","start_line_in_file":480,"slug":"json-exists-all","name":"json_exists_all","namespace":"Flow\\PostgreSql\\DSL","parameters":[{"name":"expr","type":[{"name":"Expression","namespace":"Flow\\PostgreSql\\QueryBuilder\\Expression","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"keys","type":[{"name":"Expression","namespace":"Flow\\PostgreSql\\QueryBuilder\\Expression","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"OperatorCondition","namespace":"Flow\\PostgreSql\\QueryBuilder\\Condition","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBhIEpTT05CIGFsbCBrZXlzIGV4aXN0IGNvbmRpdGlvbiAoPyYpLgogKgogKiBFeGFtcGxlOiBqc29uX2V4aXN0c19hbGwoY29sKCdtZXRhZGF0YScpLCBhcnJheV9leHByKFtsaXRlcmFsKCdjYXRlZ29yeScpLCBsaXRlcmFsKCduYW1lJyldKSkKICogUHJvZHVjZXM6IG1ldGFkYXRhID8mIGFycmF5WydjYXRlZ29yeScsICduYW1lJ10KICov"},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/condition.php","start_line_in_file":496,"slug":"array-contains","name":"array_contains","namespace":"Flow\\PostgreSql\\DSL","parameters":[{"name":"left","type":[{"name":"Expression","namespace":"Flow\\PostgreSql\\QueryBuilder\\Expression","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"right","type":[{"name":"Expression","namespace":"Flow\\PostgreSql\\QueryBuilder\\Expression","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"OperatorCondition","namespace":"Flow\\PostgreSql\\QueryBuilder\\Condition","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBhbiBhcnJheSBjb250YWlucyBjb25kaXRpb24gKEA+KS4KICoKICogRXhhbXBsZTogYXJyYXlfY29udGFpbnMoY29sKCd0YWdzJyksIGFycmF5X2V4cHIoW2xpdGVyYWwoJ3NhbGUnKV0pKQogKiBQcm9kdWNlczogdGFncyBAPiBBUlJBWVsnc2FsZSddCiAqLw=="},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/condition.php","start_line_in_file":512,"slug":"array-contained-by","name":"array_contained_by","namespace":"Flow\\PostgreSql\\DSL","parameters":[{"name":"left","type":[{"name":"Expression","namespace":"Flow\\PostgreSql\\QueryBuilder\\Expression","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"right","type":[{"name":"Expression","namespace":"Flow\\PostgreSql\\QueryBuilder\\Expression","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"OperatorCondition","namespace":"Flow\\PostgreSql\\QueryBuilder\\Condition","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBhbiBhcnJheSBpcyBjb250YWluZWQgYnkgY29uZGl0aW9uICg8QCkuCiAqCiAqIEV4YW1wbGU6IGFycmF5X2NvbnRhaW5lZF9ieShjb2woJ3RhZ3MnKSwgYXJyYXlfZXhwcihbbGl0ZXJhbCgnc2FsZScpLCBsaXRlcmFsKCdmZWF0dXJlZCcpLCBsaXRlcmFsKCduZXcnKV0pKQogKiBQcm9kdWNlczogdGFncyA8QCBBUlJBWVsnc2FsZScsICdmZWF0dXJlZCcsICduZXcnXQogKi8="},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/condition.php","start_line_in_file":528,"slug":"array-overlap","name":"array_overlap","namespace":"Flow\\PostgreSql\\DSL","parameters":[{"name":"left","type":[{"name":"Expression","namespace":"Flow\\PostgreSql\\QueryBuilder\\Expression","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"right","type":[{"name":"Expression","namespace":"Flow\\PostgreSql\\QueryBuilder\\Expression","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"OperatorCondition","namespace":"Flow\\PostgreSql\\QueryBuilder\\Condition","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBhbiBhcnJheSBvdmVybGFwIGNvbmRpdGlvbiAoJiYpLgogKgogKiBFeGFtcGxlOiBhcnJheV9vdmVybGFwKGNvbCgndGFncycpLCBhcnJheV9leHByKFtsaXRlcmFsKCdzYWxlJyksIGxpdGVyYWwoJ2ZlYXR1cmVkJyldKSkKICogUHJvZHVjZXM6IHRhZ3MgJiYgQVJSQVlbJ3NhbGUnLCAnZmVhdHVyZWQnXQogKi8="},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/condition.php","start_line_in_file":546,"slug":"regex-match","name":"regex_match","namespace":"Flow\\PostgreSql\\DSL","parameters":[{"name":"expr","type":[{"name":"Expression","namespace":"Flow\\PostgreSql\\QueryBuilder\\Expression","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"pattern","type":[{"name":"Expression","namespace":"Flow\\PostgreSql\\QueryBuilder\\Expression","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"OperatorCondition","namespace":"Flow\\PostgreSql\\QueryBuilder\\Condition","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBhIFBPU0lYIHJlZ2V4IG1hdGNoIGNvbmRpdGlvbiAofikuCiAqIENhc2Utc2Vuc2l0aXZlLgogKgogKiBFeGFtcGxlOiByZWdleF9tYXRjaChjb2woJ2VtYWlsJyksIGxpdGVyYWxfc3RyaW5nKCcuKkBnbWFpbFxcLmNvbScpKQogKgogKiBQcm9kdWNlczogZW1haWwgfiAnLipAZ21haWxcLmNvbScKICov"},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/condition.php","start_line_in_file":564,"slug":"regex-imatch","name":"regex_imatch","namespace":"Flow\\PostgreSql\\DSL","parameters":[{"name":"expr","type":[{"name":"Expression","namespace":"Flow\\PostgreSql\\QueryBuilder\\Expression","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"pattern","type":[{"name":"Expression","namespace":"Flow\\PostgreSql\\QueryBuilder\\Expression","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"OperatorCondition","namespace":"Flow\\PostgreSql\\QueryBuilder\\Condition","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBhIFBPU0lYIHJlZ2V4IG1hdGNoIGNvbmRpdGlvbiAofiopLgogKiBDYXNlLWluc2Vuc2l0aXZlLgogKgogKiBFeGFtcGxlOiByZWdleF9pbWF0Y2goY29sKCdlbWFpbCcpLCBsaXRlcmFsX3N0cmluZygnLipAZ21haWxcXC5jb20nKSkKICoKICogUHJvZHVjZXM6IGVtYWlsIH4qICcuKkBnbWFpbFwuY29tJwogKi8="},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/condition.php","start_line_in_file":582,"slug":"not-regex-match","name":"not_regex_match","namespace":"Flow\\PostgreSql\\DSL","parameters":[{"name":"expr","type":[{"name":"Expression","namespace":"Flow\\PostgreSql\\QueryBuilder\\Expression","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"pattern","type":[{"name":"Expression","namespace":"Flow\\PostgreSql\\QueryBuilder\\Expression","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"OperatorCondition","namespace":"Flow\\PostgreSql\\QueryBuilder\\Condition","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBhIFBPU0lYIHJlZ2V4IG5vdCBtYXRjaCBjb25kaXRpb24gKCF+KS4KICogQ2FzZS1zZW5zaXRpdmUuCiAqCiAqIEV4YW1wbGU6IG5vdF9yZWdleF9tYXRjaChjb2woJ2VtYWlsJyksIGxpdGVyYWxfc3RyaW5nKCcuKkBzcGFtXFwuY29tJykpCiAqCiAqIFByb2R1Y2VzOiBlbWFpbCAhfiAnLipAc3BhbVwuY29tJwogKi8="},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/condition.php","start_line_in_file":600,"slug":"not-regex-imatch","name":"not_regex_imatch","namespace":"Flow\\PostgreSql\\DSL","parameters":[{"name":"expr","type":[{"name":"Expression","namespace":"Flow\\PostgreSql\\QueryBuilder\\Expression","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"pattern","type":[{"name":"Expression","namespace":"Flow\\PostgreSql\\QueryBuilder\\Expression","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"OperatorCondition","namespace":"Flow\\PostgreSql\\QueryBuilder\\Condition","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBhIFBPU0lYIHJlZ2V4IG5vdCBtYXRjaCBjb25kaXRpb24gKCF+KikuCiAqIENhc2UtaW5zZW5zaXRpdmUuCiAqCiAqIEV4YW1wbGU6IG5vdF9yZWdleF9pbWF0Y2goY29sKCdlbWFpbCcpLCBsaXRlcmFsX3N0cmluZygnLipAc3BhbVxcLmNvbScpKQogKgogKiBQcm9kdWNlczogZW1haWwgIX4qICcuKkBzcGFtXC5jb20nCiAqLw=="},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/condition.php","start_line_in_file":616,"slug":"text-search-match","name":"text_search_match","namespace":"Flow\\PostgreSql\\DSL","parameters":[{"name":"document","type":[{"name":"Expression","namespace":"Flow\\PostgreSql\\QueryBuilder\\Expression","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"query","type":[{"name":"Expression","namespace":"Flow\\PostgreSql\\QueryBuilder\\Expression","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"OperatorCondition","namespace":"Flow\\PostgreSql\\QueryBuilder\\Condition","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBhIGZ1bGwtdGV4dCBzZWFyY2ggbWF0Y2ggY29uZGl0aW9uIChAQCkuCiAqCiAqIEV4YW1wbGU6IHRleHRfc2VhcmNoX21hdGNoKGNvbCgnZG9jdW1lbnQnKSwgZnVuYygndG9fdHNxdWVyeScsIFtsaXRlcmFsKCdlbmdsaXNoJyksIGxpdGVyYWwoJ2hlbGxvICYgd29ybGQnKV0pKQogKiBQcm9kdWNlczogZG9jdW1lbnQgQEAgdG9fdHNxdWVyeSgnZW5nbGlzaCcsICdoZWxsbyAmIHdvcmxkJykKICov"},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/parser.php","start_line_in_file":33,"slug":"sql-parser","name":"sql_parser","namespace":"Flow\\PostgreSql\\DSL","parameters":[],"return_type":[{"name":"Parser","namespace":"Flow\\PostgreSql","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/parser.php","start_line_in_file":39,"slug":"sql-parse","name":"sql_parse","namespace":"Flow\\PostgreSql\\DSL","parameters":[{"name":"sql","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"ParsedQuery","namespace":"Flow\\PostgreSql","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/parser.php","start_line_in_file":49,"slug":"sql-fingerprint","name":"sql_fingerprint","namespace":"Flow\\PostgreSql\\DSL","parameters":[{"name":"sql","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"string","namespace":null,"is_nullable":true,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIFJldHVybnMgYSBmaW5nZXJwcmludCBvZiB0aGUgZ2l2ZW4gU1FMIHF1ZXJ5LgogKiBMaXRlcmFsIHZhbHVlcyBhcmUgbm9ybWFsaXplZCBzbyB0aGV5IHdvbid0IGFmZmVjdCB0aGUgZmluZ2VycHJpbnQuCiAqLw=="},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/parser.php","start_line_in_file":60,"slug":"sql-normalize","name":"sql_normalize","namespace":"Flow\\PostgreSql\\DSL","parameters":[{"name":"sql","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"string","namespace":null,"is_nullable":true,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIE5vcm1hbGl6ZSBTUUwgcXVlcnkgYnkgcmVwbGFjaW5nIGxpdGVyYWwgdmFsdWVzIGFuZCBuYW1lZCBwYXJhbWV0ZXJzIHdpdGggcG9zaXRpb25hbCBwYXJhbWV0ZXJzLgogKiBXSEVSRSBpZCA9IDppZCB3aWxsIGJlIGNoYW5nZWQgaW50byBXSEVSRSBpZCA9ICQxCiAqIFdIRVJFIGlkID0gMSB3aWxsIGJlIGNoYW5nZWQgaW50byBXSEVSRSBpZCA9ICQxLgogKi8="},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/parser.php","start_line_in_file":70,"slug":"sql-normalize-utility","name":"sql_normalize_utility","namespace":"Flow\\PostgreSql\\DSL","parameters":[{"name":"sql","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"string","namespace":null,"is_nullable":true,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIE5vcm1hbGl6ZSB1dGlsaXR5IFNRTCBzdGF0ZW1lbnRzIChEREwgbGlrZSBDUkVBVEUsIEFMVEVSLCBEUk9QKS4KICogVGhpcyBoYW5kbGVzIERETCBzdGF0ZW1lbnRzIGRpZmZlcmVudGx5IGZyb20gcGdfbm9ybWFsaXplKCkgd2hpY2ggaXMgb3B0aW1pemVkIGZvciBETUwuCiAqLw=="},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/parser.php","start_line_in_file":81,"slug":"sql-split","name":"sql_split","namespace":"Flow\\PostgreSql\\DSL","parameters":[{"name":"sql","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"array","namespace":null,"is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIFNwbGl0IHN0cmluZyB3aXRoIG11bHRpcGxlIFNRTCBzdGF0ZW1lbnRzIGludG8gYXJyYXkgb2YgaW5kaXZpZHVhbCBzdGF0ZW1lbnRzLgogKgogKiBAcmV0dXJuIGFycmF5PHN0cmluZz4KICov"},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/parser.php","start_line_in_file":90,"slug":"sql-deparse-options","name":"sql_deparse_options","namespace":"Flow\\PostgreSql\\DSL","parameters":[],"return_type":[{"name":"DeparseOptions","namespace":"Flow\\PostgreSql","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBEZXBhcnNlT3B0aW9ucyBmb3IgY29uZmlndXJpbmcgU1FMIGZvcm1hdHRpbmcuCiAqLw=="},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/parser.php","start_line_in_file":104,"slug":"sql-deparse","name":"sql_deparse","namespace":"Flow\\PostgreSql\\DSL","parameters":[{"name":"query","type":[{"name":"ParsedQuery","namespace":"Flow\\PostgreSql","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"options","type":[{"name":"DeparseOptions","namespace":"Flow\\PostgreSql","is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"null"}],"return_type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENvbnZlcnQgYSBQYXJzZWRRdWVyeSBBU1QgYmFjayB0byBTUUwgc3RyaW5nLgogKgogKiBXaGVuIGNhbGxlZCB3aXRob3V0IG9wdGlvbnMsIHJldHVybnMgdGhlIFNRTCBhcyBhIHNpbXBsZSBzdHJpbmcuCiAqIFdoZW4gY2FsbGVkIHdpdGggRGVwYXJzZU9wdGlvbnMsIGFwcGxpZXMgZm9ybWF0dGluZyAocHJldHR5LXByaW50aW5nLCBpbmRlbnRhdGlvbiwgZXRjLikuCiAqCiAqIEB0aHJvd3MgXFJ1bnRpbWVFeGNlcHRpb24gaWYgZGVwYXJzaW5nIGZhaWxzCiAqLw=="},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/parser.php","start_line_in_file":120,"slug":"sql-format","name":"sql_format","namespace":"Flow\\PostgreSql\\DSL","parameters":[{"name":"sql","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"options","type":[{"name":"DeparseOptions","namespace":"Flow\\PostgreSql","is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"null"}],"return_type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIFBhcnNlIGFuZCBmb3JtYXQgU1FMIHF1ZXJ5IHdpdGggcHJldHR5IHByaW50aW5nLgogKgogKiBUaGlzIGlzIGEgY29udmVuaWVuY2UgZnVuY3Rpb24gdGhhdCBwYXJzZXMgU1FMIGFuZCByZXR1cm5zIGl0IGZvcm1hdHRlZC4KICoKICogQHBhcmFtIHN0cmluZyAkc3FsIFRoZSBTUUwgcXVlcnkgdG8gZm9ybWF0CiAqIEBwYXJhbSBudWxsfERlcGFyc2VPcHRpb25zICRvcHRpb25zIEZvcm1hdHRpbmcgb3B0aW9ucyAoZGVmYXVsdHMgdG8gcHJldHR5LXByaW50IGVuYWJsZWQpCiAqCiAqIEB0aHJvd3MgXFJ1bnRpbWVFeGNlcHRpb24gaWYgcGFyc2luZyBvciBkZXBhcnNpbmcgZmFpbHMKICov"},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/parser.php","start_line_in_file":132,"slug":"sql-summary","name":"sql_summary","namespace":"Flow\\PostgreSql\\DSL","parameters":[{"name":"sql","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"options","type":[{"name":"int","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"0"},{"name":"truncateLimit","type":[{"name":"int","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"0"}],"return_type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIEdlbmVyYXRlIGEgc3VtbWFyeSBvZiBwYXJzZWQgcXVlcmllcyBpbiBwcm90b2J1ZiBmb3JtYXQuCiAqIFVzZWZ1bCBmb3IgcXVlcnkgbW9uaXRvcmluZyBhbmQgbG9nZ2luZyB3aXRob3V0IGZ1bGwgQVNUIG92ZXJoZWFkLgogKi8="},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/parser.php","start_line_in_file":147,"slug":"sql-to-paginated-query","name":"sql_to_paginated_query","namespace":"Flow\\PostgreSql\\DSL","parameters":[{"name":"sql","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"limit","type":[{"name":"int","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"offset","type":[{"name":"int","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"0"}],"return_type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIFRyYW5zZm9ybSBhIFNRTCBxdWVyeSBpbnRvIGEgcGFnaW5hdGVkIHF1ZXJ5IHdpdGggTElNSVQgYW5kIE9GRlNFVC4KICoKICogQHBhcmFtIHN0cmluZyAkc3FsIFRoZSBTUUwgcXVlcnkgdG8gcGFnaW5hdGUKICogQHBhcmFtIGludCAkbGltaXQgTWF4aW11bSBudW1iZXIgb2Ygcm93cyB0byByZXR1cm4KICogQHBhcmFtIGludCAkb2Zmc2V0IE51bWJlciBvZiByb3dzIHRvIHNraXAgKHJlcXVpcmVzIE9SREVSIEJZIGluIHF1ZXJ5KQogKgogKiBAcmV0dXJuIHN0cmluZyBUaGUgcGFnaW5hdGVkIFNRTCBxdWVyeQogKi8="},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/parser.php","start_line_in_file":164,"slug":"sql-to-limited-query","name":"sql_to_limited_query","namespace":"Flow\\PostgreSql\\DSL","parameters":[{"name":"sql","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"limit","type":[{"name":"int","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIFRyYW5zZm9ybSBhIFNRTCBxdWVyeSB0byBsaW1pdCByZXN1bHRzIHRvIGEgc3BlY2lmaWMgbnVtYmVyIG9mIHJvd3MuCiAqCiAqIEBwYXJhbSBzdHJpbmcgJHNxbCBUaGUgU1FMIHF1ZXJ5IHRvIGxpbWl0CiAqIEBwYXJhbSBpbnQgJGxpbWl0IE1heGltdW0gbnVtYmVyIG9mIHJvd3MgdG8gcmV0dXJuCiAqCiAqIEByZXR1cm4gc3RyaW5nIFRoZSBsaW1pdGVkIFNRTCBxdWVyeQogKi8="},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/parser.php","start_line_in_file":183,"slug":"sql-to-count-query","name":"sql_to_count_query","namespace":"Flow\\PostgreSql\\DSL","parameters":[{"name":"sql","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIFRyYW5zZm9ybSBhIFNRTCBxdWVyeSBpbnRvIGEgQ09VTlQgcXVlcnkgZm9yIHBhZ2luYXRpb24uCiAqCiAqIFdyYXBzIHRoZSBxdWVyeSBpbjogU0VMRUNUIENPVU5UKCopIEZST00gKC4uLikgQVMgX2NvdW50X3N1YnEKICogUmVtb3ZlcyBPUkRFUiBCWSBhbmQgTElNSVQvT0ZGU0VUIGZyb20gdGhlIGlubmVyIHF1ZXJ5LgogKgogKiBAcGFyYW0gc3RyaW5nICRzcWwgVGhlIFNRTCBxdWVyeSB0byB0cmFuc2Zvcm0KICoKICogQHJldHVybiBzdHJpbmcgVGhlIENPVU5UIHF1ZXJ5CiAqLw=="},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/parser.php","start_line_in_file":205,"slug":"sql-to-keyset-query","name":"sql_to_keyset_query","namespace":"Flow\\PostgreSql\\DSL","parameters":[{"name":"sql","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"limit","type":[{"name":"int","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"columns","type":[{"name":"array","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"cursor","type":[{"name":"array","namespace":null,"is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"null"}],"return_type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIFRyYW5zZm9ybSBhIFNRTCBxdWVyeSBpbnRvIGEga2V5c2V0IChjdXJzb3ItYmFzZWQpIHBhZ2luYXRlZCBxdWVyeS4KICoKICogTW9yZSBlZmZpY2llbnQgdGhhbiBPRkZTRVQgZm9yIGxhcmdlIGRhdGFzZXRzIC0gdXNlcyBpbmRleGVkIFdIRVJFIGNvbmRpdGlvbnMuCiAqIEF1dG9tYXRpY2FsbHkgZGV0ZWN0cyBleGlzdGluZyBxdWVyeSBwYXJhbWV0ZXJzIGFuZCBhcHBlbmRzIGtleXNldCBwbGFjZWhvbGRlcnMgYXQgdGhlIGVuZC4KICoKICogQHBhcmFtIHN0cmluZyAkc3FsIFRoZSBTUUwgcXVlcnkgdG8gcGFnaW5hdGUgKG11c3QgaGF2ZSBPUkRFUiBCWSkKICogQHBhcmFtIGludCAkbGltaXQgTWF4aW11bSBudW1iZXIgb2Ygcm93cyB0byByZXR1cm4KICogQHBhcmFtIGxpc3Q8S2V5c2V0Q29sdW1uPiAkY29sdW1ucyBDb2x1bW5zIGZvciBrZXlzZXQgcGFnaW5hdGlvbiAobXVzdCBtYXRjaCBPUkRFUiBCWSkKICogQHBhcmFtIG51bGx8bGlzdDxudWxsfGJvb2x8ZmxvYXR8aW50fHN0cmluZz4gJGN1cnNvciBWYWx1ZXMgZnJvbSBsYXN0IHJvdyBvZiBwcmV2aW91cyBwYWdlIChudWxsIGZvciBmaXJzdCBwYWdlKQogKgogKiBAcmV0dXJuIHN0cmluZyBUaGUgcGFnaW5hdGVkIFNRTCBxdWVyeQogKi8="},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/parser.php","start_line_in_file":220,"slug":"sql-keyset-column","name":"sql_keyset_column","namespace":"Flow\\PostgreSql\\DSL","parameters":[{"name":"column","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"order","type":[{"name":"SortOrder","namespace":"Flow\\PostgreSql\\AST\\Transformers","is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"Flow\\PostgreSql\\AST\\Transformers\\SortOrder::..."}],"return_type":[{"name":"KeysetColumn","namespace":"Flow\\PostgreSql\\AST\\Transformers","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBhIEtleXNldENvbHVtbiBmb3Iga2V5c2V0IHBhZ2luYXRpb24uCiAqCiAqIEBwYXJhbSBzdHJpbmcgJGNvbHVtbiBDb2x1bW4gbmFtZSAoY2FuIGluY2x1ZGUgdGFibGUgYWxpYXMgbGlrZSAidS5pZCIpCiAqIEBwYXJhbSBTb3J0T3JkZXIgJG9yZGVyIFNvcnQgb3JkZXIgKEFTQyBvciBERVNDKQogKi8="},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/parser.php","start_line_in_file":229,"slug":"sql-query-columns","name":"sql_query_columns","namespace":"Flow\\PostgreSql\\DSL","parameters":[{"name":"query","type":[{"name":"ParsedQuery","namespace":"Flow\\PostgreSql","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"Columns","namespace":"Flow\\PostgreSql\\Extractors","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIEV4dHJhY3QgY29sdW1ucyBmcm9tIGEgcGFyc2VkIFNRTCBxdWVyeS4KICov"},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/parser.php","start_line_in_file":238,"slug":"sql-query-tables","name":"sql_query_tables","namespace":"Flow\\PostgreSql\\DSL","parameters":[{"name":"query","type":[{"name":"ParsedQuery","namespace":"Flow\\PostgreSql","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"Tables","namespace":"Flow\\PostgreSql\\Extractors","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIEV4dHJhY3QgdGFibGVzIGZyb20gYSBwYXJzZWQgU1FMIHF1ZXJ5LgogKi8="},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/parser.php","start_line_in_file":247,"slug":"sql-query-functions","name":"sql_query_functions","namespace":"Flow\\PostgreSql\\DSL","parameters":[{"name":"query","type":[{"name":"ParsedQuery","namespace":"Flow\\PostgreSql","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"Functions","namespace":"Flow\\PostgreSql\\Extractors","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIEV4dHJhY3QgZnVuY3Rpb25zIGZyb20gYSBwYXJzZWQgU1FMIHF1ZXJ5LgogKi8="},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/parser.php","start_line_in_file":256,"slug":"sql-query-order-by","name":"sql_query_order_by","namespace":"Flow\\PostgreSql\\DSL","parameters":[{"name":"query","type":[{"name":"ParsedQuery","namespace":"Flow\\PostgreSql","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"OrderBy","namespace":"Flow\\PostgreSql\\Extractors","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIEV4dHJhY3QgT1JERVIgQlkgY2xhdXNlcyBmcm9tIGEgcGFyc2VkIFNRTCBxdWVyeS4KICov"},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/parser.php","start_line_in_file":270,"slug":"sql-query-depth","name":"sql_query_depth","namespace":"Flow\\PostgreSql\\DSL","parameters":[{"name":"sql","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"int","namespace":null,"is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIEdldCB0aGUgbWF4aW11bSBuZXN0aW5nIGRlcHRoIG9mIGEgU1FMIHF1ZXJ5LgogKgogKiBFeGFtcGxlOgogKiAtICJTRUxFQ1QgKiBGUk9NIHQiID0+IDEKICogLSAiU0VMRUNUICogRlJPTSAoU0VMRUNUICogRlJPTSB0KSIgPT4gMgogKiAtICJTRUxFQ1QgKiBGUk9NIChTRUxFQ1QgKiBGUk9NIChTRUxFQ1QgKiBGUk9NIHQpKSIgPT4gMwogKi8="},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/parser.php","start_line_in_file":287,"slug":"sql-to-explain","name":"sql_to_explain","namespace":"Flow\\PostgreSql\\DSL","parameters":[{"name":"sql","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"config","type":[{"name":"ExplainConfig","namespace":"Flow\\PostgreSql\\AST\\Transformers","is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"null"}],"return_type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIFRyYW5zZm9ybSBhIFNRTCBxdWVyeSBpbnRvIGFuIEVYUExBSU4gcXVlcnkuCiAqCiAqIFJldHVybnMgdGhlIG1vZGlmaWVkIFNRTCB3aXRoIEVYUExBSU4gd3JhcHBlZCBhcm91bmQgaXQuCiAqIERlZmF1bHRzIHRvIEVYUExBSU4gQU5BTFlaRSB3aXRoIEpTT04gZm9ybWF0IGZvciBlYXN5IHBhcnNpbmcuCiAqCiAqIEBwYXJhbSBzdHJpbmcgJHNxbCBUaGUgU1FMIHF1ZXJ5IHRvIGV4cGxhaW4KICogQHBhcmFtIG51bGx8RXhwbGFpbkNvbmZpZyAkY29uZmlnIEVYUExBSU4gY29uZmlndXJhdGlvbiAoZGVmYXVsdHMgdG8gZm9yQW5hbHlzaXMoKSkKICoKICogQHJldHVybiBzdHJpbmcgVGhlIEVYUExBSU4gcXVlcnkKICov"},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/parser.php","start_line_in_file":307,"slug":"sql-explain-config","name":"sql_explain_config","namespace":"Flow\\PostgreSql\\DSL","parameters":[{"name":"analyze","type":[{"name":"bool","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"true"},{"name":"verbose","type":[{"name":"bool","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"false"},{"name":"costs","type":[{"name":"bool","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"true"},{"name":"buffers","type":[{"name":"bool","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"true"},{"name":"timing","type":[{"name":"bool","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"true"},{"name":"format","type":[{"name":"ExplainFormat","namespace":"Flow\\PostgreSql\\QueryBuilder\\Utility","is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"Flow\\PostgreSql\\QueryBuilder\\Utility\\ExplainFormat::..."}],"return_type":[{"name":"ExplainConfig","namespace":"Flow\\PostgreSql\\AST\\Transformers","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBhbiBFeHBsYWluQ29uZmlnIGZvciBjdXN0b21pemluZyBFWFBMQUlOIG9wdGlvbnMuCiAqCiAqIEBwYXJhbSBib29sICRhbmFseXplIFdoZXRoZXIgdG8gYWN0dWFsbHkgZXhlY3V0ZSB0aGUgcXVlcnkgKEFOQUxZWkUpCiAqIEBwYXJhbSBib29sICR2ZXJib3NlIEluY2x1ZGUgdmVyYm9zZSBvdXRwdXQKICogQHBhcmFtIGJvb2wgJGNvc3RzIEluY2x1ZGUgY29zdCBlc3RpbWF0ZXMgKGRlZmF1bHQgdHJ1ZSkKICogQHBhcmFtIGJvb2wgJGJ1ZmZlcnMgSW5jbHVkZSBidWZmZXIgdXNhZ2Ugc3RhdGlzdGljcyAocmVxdWlyZXMgYW5hbHl6ZSkKICogQHBhcmFtIGJvb2wgJHRpbWluZyBJbmNsdWRlIHRpbWluZyBpbmZvcm1hdGlvbiAocmVxdWlyZXMgYW5hbHl6ZSkKICogQHBhcmFtIEV4cGxhaW5Gb3JtYXQgJGZvcm1hdCBPdXRwdXQgZm9ybWF0IChKU09OIHJlY29tbWVuZGVkIGZvciBwYXJzaW5nKQogKi8="},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/parser.php","start_line_in_file":329,"slug":"sql-explain-modifier","name":"sql_explain_modifier","namespace":"Flow\\PostgreSql\\DSL","parameters":[{"name":"config","type":[{"name":"ExplainConfig","namespace":"Flow\\PostgreSql\\AST\\Transformers","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"ExplainModifier","namespace":"Flow\\PostgreSql\\AST\\Transformers","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBhbiBFeHBsYWluTW9kaWZpZXIgZm9yIHRyYW5zZm9ybWluZyBxdWVyaWVzIGludG8gRVhQTEFJTiBxdWVyaWVzLgogKi8="},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/parser.php","start_line_in_file":342,"slug":"sql-explain-parse","name":"sql_explain_parse","namespace":"Flow\\PostgreSql\\DSL","parameters":[{"name":"jsonOutput","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"Plan","namespace":"Flow\\PostgreSql\\Explain\\Plan","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIFBhcnNlIEVYUExBSU4gSlNPTiBvdXRwdXQgaW50byBhIFBsYW4gb2JqZWN0LgogKgogKiBAcGFyYW0gc3RyaW5nICRqc29uT3V0cHV0IFRoZSBKU09OIG91dHB1dCBmcm9tIEVYUExBSU4gKEZPUk1BVCBKU09OKQogKgogKiBAcmV0dXJuIFBsYW4gVGhlIHBhcnNlZCBleGVjdXRpb24gcGxhbgogKi8="},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/parser.php","start_line_in_file":355,"slug":"sql-analyze","name":"sql_analyze","namespace":"Flow\\PostgreSql\\DSL","parameters":[{"name":"plan","type":[{"name":"Plan","namespace":"Flow\\PostgreSql\\Explain\\Plan","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"PlanAnalyzer","namespace":"Flow\\PostgreSql\\Explain\\Analyzer","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBhIHBsYW4gYW5hbHl6ZXIgZm9yIGFuYWx5emluZyBFWFBMQUlOIHBsYW5zLgogKgogKiBAcGFyYW0gUGxhbiAkcGxhbiBUaGUgZXhlY3V0aW9uIHBsYW4gdG8gYW5hbHl6ZQogKgogKiBAcmV0dXJuIFBsYW5BbmFseXplciBUaGUgYW5hbHl6ZXIgZm9yIGV4dHJhY3RpbmcgaW5zaWdodHMKICov"},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/client.php","start_line_in_file":46,"slug":"pgsql-connection","name":"pgsql_connection","namespace":"Flow\\PostgreSql\\DSL","parameters":[{"name":"connectionString","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"ConnectionParameters","namespace":"Flow\\PostgreSql\\Client","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBjb25uZWN0aW9uIHBhcmFtZXRlcnMgZnJvbSBhIGNvbm5lY3Rpb24gc3RyaW5nLgogKgogKiBBY2NlcHRzIGxpYnBxLXN0eWxlIGNvbm5lY3Rpb24gc3RyaW5nczoKICogLSBLZXktdmFsdWUgZm9ybWF0OiAiaG9zdD1sb2NhbGhvc3QgcG9ydD01NDMyIGRibmFtZT1teWRiIHVzZXI9bXl1c2VyIHBhc3N3b3JkPXNlY3JldCIKICogLSBVUkkgZm9ybWF0OiAicG9zdGdyZXNxbDovL3VzZXI6cGFzc3dvcmRAbG9jYWxob3N0OjU0MzIvZGJuYW1lIgogKgogKiBAZXhhbXBsZQogKiAkcGFyYW1zID0gcGdzcWxfY29ubmVjdGlvbignaG9zdD1sb2NhbGhvc3QgZGJuYW1lPW15ZGInKTsKICogJHBhcmFtcyA9IHBnc3FsX2Nvbm5lY3Rpb24oJ3Bvc3RncmVzcWw6Ly91c2VyOnBhc3NAbG9jYWxob3N0L215ZGInKTsKICov"},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/client.php","start_line_in_file":68,"slug":"pgsql-connection-dsn","name":"pgsql_connection_dsn","namespace":"Flow\\PostgreSql\\DSL","parameters":[{"name":"dsn","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"ConnectionParameters","namespace":"Flow\\PostgreSql\\Client","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBjb25uZWN0aW9uIHBhcmFtZXRlcnMgZnJvbSBhIERTTiBzdHJpbmcuCiAqCiAqIFBhcnNlcyBzdGFuZGFyZCBQb3N0Z3JlU1FMIERTTiBmb3JtYXQgY29tbW9ubHkgdXNlZCBpbiBlbnZpcm9ubWVudCB2YXJpYWJsZXMKICogKGUuZy4sIERBVEFCQVNFX1VSTCkuIFN1cHBvcnRzIHBvc3RncmVzOi8vLCBwb3N0Z3Jlc3FsOi8vLCBhbmQgcGdzcWw6Ly8gc2NoZW1lcy4KICoKICogQHBhcmFtIHN0cmluZyAkZHNuIERTTiBzdHJpbmcgaW4gZm9ybWF0OiBwb3N0Z3JlczovL3VzZXI6cGFzc3dvcmRAaG9zdDpwb3J0L2RhdGFiYXNlP29wdGlvbnMKICoKICogQHRocm93cyBDbGllbnRcRHNuUGFyc2VyRXhjZXB0aW9uIElmIHRoZSBEU04gY2Fubm90IGJlIHBhcnNlZAogKgogKiBAZXhhbXBsZQogKiAkcGFyYW1zID0gcGdzcWxfY29ubmVjdGlvbl9kc24oJ3Bvc3RncmVzOi8vbXl1c2VyOnNlY3JldEBsb2NhbGhvc3Q6NTQzMi9teWRiJyk7CiAqICRwYXJhbXMgPSBwZ3NxbF9jb25uZWN0aW9uX2RzbigncG9zdGdyZXNxbDovL3VzZXI6cGFzc0BkYi5leGFtcGxlLmNvbS9hcHA\/c3NsbW9kZT1yZXF1aXJlJyk7CiAqICRwYXJhbXMgPSBwZ3NxbF9jb25uZWN0aW9uX2RzbigncGdzcWw6Ly91c2VyOnBhc3NAbG9jYWxob3N0L215ZGInKTsgLy8gU3ltZm9ueS9Eb2N0cmluZSBmb3JtYXQKICogJHBhcmFtcyA9IHBnc3FsX2Nvbm5lY3Rpb25fZHNuKGdldGVudignREFUQUJBU0VfVVJMJykpOwogKi8="},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/client.php","start_line_in_file":95,"slug":"pgsql-connection-params","name":"pgsql_connection_params","namespace":"Flow\\PostgreSql\\DSL","parameters":[{"name":"database","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"host","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"'localhost'"},{"name":"port","type":[{"name":"int","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"5432"},{"name":"user","type":[{"name":"string","namespace":null,"is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"null"},{"name":"password","type":[{"name":"string","namespace":null,"is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"null"},{"name":"options","type":[{"name":"array","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"[]"}],"return_type":[{"name":"ConnectionParameters","namespace":"Flow\\PostgreSql\\Client","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBjb25uZWN0aW9uIHBhcmFtZXRlcnMgZnJvbSBpbmRpdmlkdWFsIHZhbHVlcy4KICoKICogQWxsb3dzIHNwZWNpZnlpbmcgY29ubmVjdGlvbiBwYXJhbWV0ZXJzIGluZGl2aWR1YWxseSBmb3IgYmV0dGVyIHR5cGUgc2FmZXR5CiAqIGFuZCBJREUgc3VwcG9ydC4KICoKICogQHBhcmFtIHN0cmluZyAkZGF0YWJhc2UgRGF0YWJhc2UgbmFtZSAocmVxdWlyZWQpCiAqIEBwYXJhbSBzdHJpbmcgJGhvc3QgSG9zdG5hbWUgKGRlZmF1bHQ6IGxvY2FsaG9zdCkKICogQHBhcmFtIGludCAkcG9ydCBQb3J0IG51bWJlciAoZGVmYXVsdDogNTQzMikKICogQHBhcmFtIG51bGx8c3RyaW5nICR1c2VyIFVzZXJuYW1lIChvcHRpb25hbCkKICogQHBhcmFtIG51bGx8c3RyaW5nICRwYXNzd29yZCBQYXNzd29yZCAob3B0aW9uYWwpCiAqIEBwYXJhbSBhcnJheTxzdHJpbmcsIHN0cmluZz4gJG9wdGlvbnMgQWRkaXRpb25hbCBsaWJwcSBvcHRpb25zCiAqCiAqIEBleGFtcGxlCiAqICRwYXJhbXMgPSBwZ3NxbF9jb25uZWN0aW9uX3BhcmFtcygKICogICAgIGRhdGFiYXNlOiAnbXlkYicsCiAqICAgICBob3N0OiAnbG9jYWxob3N0JywKICogICAgIHVzZXI6ICdteXVzZXInLAogKiAgICAgcGFzc3dvcmQ6ICdzZWNyZXQnLAogKiApOwogKi8="},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/client.php","start_line_in_file":126,"slug":"pgsql-client","name":"pgsql_client","namespace":"Flow\\PostgreSql\\DSL","parameters":[{"name":"params","type":[{"name":"ConnectionParameters","namespace":"Flow\\PostgreSql\\Client","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"valueConverters","type":[{"name":"ValueConverters","namespace":"Flow\\PostgreSql\\Client\\Types","is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"null"},{"name":"context","type":[{"name":"Context","namespace":"Flow\\PostgreSql\\Client","is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"null"}],"return_type":[{"name":"Client","namespace":"Flow\\PostgreSql\\Client","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBhIFBvc3RncmVTUUwgY2xpZW50IHVzaW5nIGV4dC1wZ3NxbC4KICoKICogVGhlIGNsaWVudCBjb25uZWN0cyBpbW1lZGlhdGVseSBhbmQgaXMgcmVhZHkgdG8gZXhlY3V0ZSBxdWVyaWVzLgogKgogKiBAcGFyYW0gQ2xpZW50XENvbm5lY3Rpb25QYXJhbWV0ZXJzICRwYXJhbXMgQ29ubmVjdGlvbiBwYXJhbWV0ZXJzCiAqIEBwYXJhbSBudWxsfFZhbHVlQ29udmVydGVycyAkdmFsdWVDb252ZXJ0ZXJzIEN1c3RvbSB0eXBlIGNvbnZlcnRlcnMgKG9wdGlvbmFsKQogKiBAcGFyYW0gbnVsbHxDb250ZXh0ICRjb250ZXh0IEJhc2UgbWFwcGVyIENvbnRleHQg4oCUIHRoZSBDbGllbnQgZW5yaWNoZXMgaXQgd2l0aCBzcWwvcGFyYW1ldGVycy9zZWxmIHBlciBxdWVyeSBiZWZvcmUgaGFuZGluZyBpdCB0byBSb3dNYXBwZXI6Om1hcCgpCiAqCiAqIEB0aHJvd3MgQ29ubmVjdGlvbkV4Y2VwdGlvbiBJZiBjb25uZWN0aW9uIGZhaWxzCiAqLw=="},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/client.php","start_line_in_file":143,"slug":"postgresql-context","name":"postgresql_context","namespace":"Flow\\PostgreSql\\DSL","parameters":[{"name":"data","type":[{"name":"array","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"[]"},{"name":"catalog","type":[{"name":"Catalog","namespace":"Flow\\PostgreSql\\Schema","is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"null"}],"return_type":[{"name":"Context","namespace":"Flow\\PostgreSql\\Client","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBhIFJvd01hcHBlciBDb250ZXh0IHNlZWRlZCB3aXRoIHVzZXItc3VwcGxpZWQga2V5L3ZhbHVlIGRhdGEgYW5kIGFuIG9wdGlvbmFsIENhdGFsb2cuCiAqCiAqIFRoZSBDb250ZXh0IGlzIGxhdGVyIGVucmljaGVkIHdpdGggYSBRdWVyeSAoc3FsICsgcGFyYW1ldGVycykgYW5kIHRoZSBleGVjdXRpbmcgQ2xpZW50IGJ5IHRoZQogKiBQb3N0Z3JlU1FMIENsaWVudCBiZWZvcmUgYmVpbmcgaGFuZGVkIHRvIFJvd01hcHBlcjo6bWFwKCkuCiAqCiAqIEBwYXJhbSBhcnJheTxzdHJpbmcsIG1peGVkPiAkZGF0YSBVc2VyLXN1cHBsaWVkIGtleS92YWx1ZSBwYWlycwogKi8="},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/client.php","start_line_in_file":176,"slug":"postgresql-telemetry-options","name":"postgresql_telemetry_options","namespace":"Flow\\PostgreSql\\DSL","parameters":[{"name":"traceQueries","type":[{"name":"bool","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"true"},{"name":"transactionSpans","type":[{"name":"TransactionSpanMode","namespace":"Flow\\PostgreSql\\Client\\Telemetry","is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"Flow\\PostgreSql\\Client\\Telemetry\\TransactionSpanMode::..."},{"name":"collectMetrics","type":[{"name":"bool","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"true"},{"name":"logQueries","type":[{"name":"bool","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"false"},{"name":"maxQueryLength","type":[{"name":"int","namespace":null,"is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"1000"},{"name":"includeParameters","type":[{"name":"bool","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"false"},{"name":"maxParameters","type":[{"name":"int","namespace":null,"is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"10"},{"name":"maxParameterLength","type":[{"name":"int","namespace":null,"is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"100"}],"return_type":[{"name":"PostgreSqlTelemetryOptions","namespace":"Flow\\PostgreSql\\Client\\Telemetry","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSB0ZWxlbWV0cnkgb3B0aW9ucyBmb3IgUG9zdGdyZVNRTCBjbGllbnQgaW5zdHJ1bWVudGF0aW9uLgogKgogKiBDb250cm9scyB3aGljaCB0ZWxlbWV0cnkgc2lnbmFscyAodHJhY2VzLCBtZXRyaWNzLCBsb2dzKSBhcmUgZW5hYmxlZAogKiBhbmQgaG93IHF1ZXJ5IGluZm9ybWF0aW9uIGlzIGNhcHR1cmVkLgogKgogKiBAcGFyYW0gYm9vbCAkdHJhY2VRdWVyaWVzIENyZWF0ZSBzcGFucyBmb3IgcXVlcnkgZXhlY3V0aW9uIChkZWZhdWx0OiB0cnVlKQogKiBAcGFyYW0gVHJhbnNhY3Rpb25TcGFuTW9kZSAkdHJhbnNhY3Rpb25TcGFucyBIb3cgdHJhbnNhY3Rpb25zIGFyZSB0cmFjZWQ6IEdST1VQRUQgKGRlZmF1bHQpLCBQRVJfT1BFUkFUSU9OIG9yIE9GRgogKiBAcGFyYW0gYm9vbCAkY29sbGVjdE1ldHJpY3MgQ29sbGVjdCBkdXJhdGlvbiBhbmQgcm93IGNvdW50IG1ldHJpY3MgKGRlZmF1bHQ6IHRydWUpCiAqIEBwYXJhbSBib29sICRsb2dRdWVyaWVzIExvZyBleGVjdXRlZCBxdWVyaWVzIChkZWZhdWx0OiBmYWxzZSkKICogQHBhcmFtIG51bGx8aW50ICRtYXhRdWVyeUxlbmd0aCBNYXhpbXVtIHF1ZXJ5IHRleHQgbGVuZ3RoIGluIHRlbGVtZXRyeSAoZGVmYXVsdDogMTAwMCwgbnVsbCA9IHVubGltaXRlZCkKICogQHBhcmFtIGJvb2wgJGluY2x1ZGVQYXJhbWV0ZXJzIEluY2x1ZGUgcXVlcnkgcGFyYW1ldGVycyBpbiB0ZWxlbWV0cnkgKGRlZmF1bHQ6IGZhbHNlLCBzZWN1cml0eSBjb25zaWRlcmF0aW9uKQogKgogKiBAZXhhbXBsZQogKiAvLyBEZWZhdWx0IG9wdGlvbnMgKHRyYWNlcyBhbmQgbWV0cmljcyBlbmFibGVkKQogKiAkb3B0aW9ucyA9IHBvc3RncmVzcWxfdGVsZW1ldHJ5X29wdGlvbnMoKTsKICoKICogLy8gRW5hYmxlIHF1ZXJ5IGxvZ2dpbmcKICogJG9wdGlvbnMgPSBwb3N0Z3Jlc3FsX3RlbGVtZXRyeV9vcHRpb25zKGxvZ1F1ZXJpZXM6IHRydWUpOwogKgogKiAvLyBNZXRyaWNzIG9ubHksIG5vIHNwYW5zCiAqICRvcHRpb25zID0gcG9zdGdyZXNxbF90ZWxlbWV0cnlfb3B0aW9ucygKICogICAgIHRyYWNlUXVlcmllczogZmFsc2UsCiAqICAgICB0cmFuc2FjdGlvblNwYW5zOiBUcmFuc2FjdGlvblNwYW5Nb2RlOjpPRkYsCiAqICAgICBjb2xsZWN0TWV0cmljczogdHJ1ZSwKICogKTsKICov"},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/client.php","start_line_in_file":214,"slug":"postgresql-telemetry-config","name":"postgresql_telemetry_config","namespace":"Flow\\PostgreSql\\DSL","parameters":[{"name":"telemetry","type":[{"name":"Telemetry","namespace":"Flow\\Telemetry","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"clock","type":[{"name":"ClockInterface","namespace":"Psr\\Clock","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"options","type":[{"name":"PostgreSqlTelemetryOptions","namespace":"Flow\\PostgreSql\\Client\\Telemetry","is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"null"}],"return_type":[{"name":"PostgreSqlTelemetryConfig","namespace":"Flow\\PostgreSql\\Client\\Telemetry","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSB0ZWxlbWV0cnkgY29uZmlndXJhdGlvbiBmb3IgUG9zdGdyZVNRTCBjbGllbnQuCiAqCiAqIEJ1bmRsZXMgdGVsZW1ldHJ5IGluc3RhbmNlLCBjbG9jaywgYW5kIG9wdGlvbnMgbmVlZGVkIHRvIGluc3RydW1lbnQgYSBQb3N0Z3JlU1FMIGNsaWVudC4KICoKICogQHBhcmFtIFRlbGVtZXRyeSAkdGVsZW1ldHJ5IFRoZSB0ZWxlbWV0cnkgaW5zdGFuY2UKICogQHBhcmFtIENsb2NrSW50ZXJmYWNlICRjbG9jayBDbG9jayBmb3IgdGltZXN0YW1wcwogKiBAcGFyYW0gbnVsbHxQb3N0Z3JlU3FsVGVsZW1ldHJ5T3B0aW9ucyAkb3B0aW9ucyBUZWxlbWV0cnkgb3B0aW9ucyAoZGVmYXVsdDogYWxsIGVuYWJsZWQpCiAqCiAqIEBleGFtcGxlCiAqICRjb25maWcgPSBwb3N0Z3Jlc3FsX3RlbGVtZXRyeV9jb25maWcoCiAqICAgICB0ZWxlbWV0cnkocmVzb3VyY2UoWydzZXJ2aWNlLm5hbWUnID0+ICdteS1hcHAnXSkpLAogKiAgICAgbmV3IFN5c3RlbUNsb2NrKCksCiAqICk7CiAqLw=="},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/client.php","start_line_in_file":260,"slug":"traceable-postgresql-client","name":"traceable_postgresql_client","namespace":"Flow\\PostgreSql\\DSL","parameters":[{"name":"client","type":[{"name":"Client","namespace":"Flow\\PostgreSql\\Client","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"telemetryConfig","type":[{"name":"PostgreSqlTelemetryConfig","namespace":"Flow\\PostgreSql\\Client\\Telemetry","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"TraceableClient","namespace":"Flow\\PostgreSql\\Client\\Telemetry","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIFdyYXAgYSBQb3N0Z3JlU1FMIGNsaWVudCB3aXRoIHRlbGVtZXRyeSBpbnN0cnVtZW50YXRpb24uCiAqCiAqIFJldHVybnMgYSBkZWNvcmF0b3IgdGhhdCBhZGRzIHNwYW5zLCBtZXRyaWNzLCBhbmQgbG9ncyB0byBhbGwKICogcXVlcnkgYW5kIHRyYW5zYWN0aW9uIG9wZXJhdGlvbnMgZm9sbG93aW5nIE9wZW5UZWxlbWV0cnkgY29udmVudGlvbnMuCiAqCiAqIEBwYXJhbSBDbGllbnRcQ2xpZW50ICRjbGllbnQgVGhlIFBvc3RncmVTUUwgY2xpZW50IHRvIGluc3RydW1lbnQKICogQHBhcmFtIFBvc3RncmVTcWxUZWxlbWV0cnlDb25maWcgJHRlbGVtZXRyeUNvbmZpZyBUZWxlbWV0cnkgY29uZmlndXJhdGlvbgogKgogKiBAZXhhbXBsZQogKiAkY2xpZW50ID0gcGdzcWxfY2xpZW50KHBnc3FsX2Nvbm5lY3Rpb24oJ2hvc3Q9bG9jYWxob3N0IGRibmFtZT1teWRiJykpOwogKgogKiAkdHJhY2VhYmxlQ2xpZW50ID0gdHJhY2VhYmxlX3Bvc3RncmVzcWxfY2xpZW50KAogKiAgICAgJGNsaWVudCwKICogICAgIHBvc3RncmVzcWxfdGVsZW1ldHJ5X2NvbmZpZygKICogICAgICAgICB0ZWxlbWV0cnkocmVzb3VyY2UoWydzZXJ2aWNlLm5hbWUnID0+ICdteS1hcHAnXSkpLAogKiAgICAgICAgIG5ldyBTeXN0ZW1DbG9jaygpLAogKiAgICAgICAgIHBvc3RncmVzcWxfdGVsZW1ldHJ5X29wdGlvbnMoCiAqICAgICAgICAgICAgIHRyYWNlUXVlcmllczogdHJ1ZSwKICogICAgICAgICAgICAgdHJhbnNhY3Rpb25TcGFuczogVHJhbnNhY3Rpb25TcGFuTW9kZTo6R1JPVVBFRCwKICogICAgICAgICAgICAgY29sbGVjdE1ldHJpY3M6IHRydWUsCiAqICAgICAgICAgICAgIGxvZ1F1ZXJpZXM6IHRydWUsCiAqICAgICAgICAgICAgIG1heFF1ZXJ5TGVuZ3RoOiA1MDAsCiAqICAgICAgICAgKSwKICogICAgICksCiAqICk7CiAqCiAqIC8vIEFsbCBvcGVyYXRpb25zIG5vdyB0cmFjZWQKICogJHRyYWNlYWJsZUNsaWVudC0+dHJhbnNhY3Rpb24oZnVuY3Rpb24gKENsaWVudCAkY2xpZW50KSB7CiAqICAgICAkdXNlciA9ICRjbGllbnQtPmZldGNoU2luZ2xlKCdTRUxFQ1QgKiBGUk9NIHVzZXJzIFdIRVJFIGlkID0gJDEnLCBbMTIzXSk7CiAqICAgICAkY2xpZW50LT5leGVjdXRlKCdVUERBVEUgdXNlcnMgU0VUIGxhc3RfbG9naW4gPSBOT1coKSBXSEVSRSBpZCA9ICQxJywgWzEyM10pOwogKiB9KTsKICov"},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/client.php","start_line_in_file":300,"slug":"constructor-mapper","name":"constructor_mapper","namespace":"Flow\\PostgreSql\\DSL","parameters":[{"name":"class","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"ConstructorMapper","namespace":"Flow\\PostgreSql\\Client\\RowMapper","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIEB0ZW1wbGF0ZSBUIG9mIG9iamVjdAogKgogKiBAcGFyYW0gY2xhc3Mtc3RyaW5nPFQ+ICRjbGFzcwogKgogKiBAcmV0dXJuIENvbnN0cnVjdG9yTWFwcGVyPFQ+CiAqLw=="},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/client.php","start_line_in_file":315,"slug":"type-mapper","name":"type_mapper","namespace":"Flow\\PostgreSql\\DSL","parameters":[{"name":"type","type":[{"name":"Type","namespace":"Flow\\Types","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"next","type":[{"name":"RowMapper","namespace":"Flow\\PostgreSql\\Client","is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"null"}],"return_type":[{"name":"TypeMapper","namespace":"Flow\\PostgreSql\\Client\\RowMapper","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIEB0ZW1wbGF0ZSBUVHlwZQogKiBAdGVtcGxhdGUgVE91dAogKgogKiBAcGFyYW0gRmxvd1R5cGU8VFR5cGU+ICR0eXBlCiAqIEBwYXJhbSBudWxsfFJvd01hcHBlcjxUT3V0PiAkbmV4dAogKgogKiBAcmV0dXJuICgkbmV4dCBpcyBudWxsID8gVHlwZU1hcHBlcjxUVHlwZSwgVFR5cGU+IDogVHlwZU1hcHBlcjxUVHlwZSwgVE91dD4pCiAqLw=="},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/client.php","start_line_in_file":343,"slug":"static-factory-mapper","name":"static_factory_mapper","namespace":"Flow\\PostgreSql\\DSL","parameters":[{"name":"class","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"method","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"StaticFactoryMapper","namespace":"Flow\\PostgreSql\\Client\\RowMapper","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBhIHJvdyBtYXBwZXIgYmFja2VkIGJ5IGEgcHVibGljIHN0YXRpYyBmYWN0b3J5IG1ldGhvZC4KICoKICogVGhlIGZhY3RvcnkgbWV0aG9kIG11c3QgYWNjZXB0IGEgc2luZ2xlIGFycmF5PHN0cmluZywgbWl4ZWQ+ICRyb3cgYW5kIHJldHVybgogKiBhbiBpbnN0YW5jZSBvZiB0aGUgdGFyZ2V0IGNsYXNzLiBJZiB5b3VyIGZhY3RvcnkgbmVlZHMgYWNjZXNzIHRvIHRoZSBtYXBwaW5nCiAqIENvbnRleHQgKHNxbC9wYXJhbWV0ZXJzL2NsaWVudC9jYXRhbG9nL3VzZXItZGF0YSksIGltcGxlbWVudCBSb3dNYXBwZXIgZGlyZWN0bHkuCiAqCiAqIEB0ZW1wbGF0ZSBUIG9mIG9iamVjdAogKgogKiBAcGFyYW0gY2xhc3Mtc3RyaW5nPFQ+ICRjbGFzcwogKiBAcGFyYW0gbm9uLWVtcHR5LXN0cmluZyAkbWV0aG9kCiAqCiAqIEByZXR1cm4gU3RhdGljRmFjdG9yeU1hcHBlcjxUPgogKi8="},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/client.php","start_line_in_file":372,"slug":"typed","name":"typed","namespace":"Flow\\PostgreSql\\DSL","parameters":[{"name":"value","type":[{"name":"mixed","namespace":null,"is_nullable":true,"is_variadic":false}],"has_default_value":false,"is_nullable":true,"is_variadic":false,"default_value":null},{"name":"targetType","type":[{"name":"ValueType","namespace":"Flow\\PostgreSql\\Client\\Types","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"TypedValue","namespace":"Flow\\PostgreSql\\Client","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIFdyYXAgYSB2YWx1ZSB3aXRoIGV4cGxpY2l0IFBvc3RncmVTUUwgdHlwZSBpbmZvcm1hdGlvbiBmb3IgcGFyYW1ldGVyIGJpbmRpbmcuCiAqCiAqIFVzZSB3aGVuIGF1dG8tZGV0ZWN0aW9uIGlzbid0IHN1ZmZpY2llbnQgb3Igd2hlbiB5b3UgbmVlZCB0byBzcGVjaWZ5CiAqIHRoZSBleGFjdCBQb3N0Z3JlU1FMIHR5cGUgKHNpbmNlIG9uZSBQSFAgdHlwZSBjYW4gbWFwIHRvIG11bHRpcGxlIFBvc3RncmVTUUwgdHlwZXMpOgogKiAtIGludCBjb3VsZCBiZSBJTlQyLCBJTlQ0LCBvciBJTlQ4CiAqIC0gc3RyaW5nIGNvdWxkIGJlIFRFWFQsIFZBUkNIQVIsIG9yIENIQVIKICogLSBhcnJheSBtdXN0IGFsd2F5cyB1c2UgdHlwZWQoKSBzaW5jZSBhdXRvLWRldGVjdGlvbiBjYW5ub3QgZGV0ZXJtaW5lIGVsZW1lbnQgdHlwZQogKiAtIERhdGVUaW1lSW50ZXJmYWNlIGNvdWxkIGJlIFRJTUVTVEFNUCBvciBUSU1FU1RBTVBUWgogKiAtIEpzb24gY291bGQgYmUgSlNPTiBvciBKU09OQgogKgogKiBAcGFyYW0gbWl4ZWQgJHZhbHVlIFRoZSB2YWx1ZSB0byBiaW5kCiAqIEBwYXJhbSBWYWx1ZVR5cGUgJHRhcmdldFR5cGUgVGhlIFBvc3RncmVTUUwgdHlwZSB0byBjb252ZXJ0IHRoZSB2YWx1ZSB0bwogKgogKiBAZXhhbXBsZQogKiAkY2xpZW50LT5mZXRjaCgKICogICAgICdTRUxFQ1QgKiBGUk9NIHVzZXJzIFdIRVJFIGlkID0gJDEgQU5EIHRhZ3MgPSAkMicsCiAqICAgICBbCiAqICAgICAgICAgdHlwZWQoJzU1MGU4NDAwLWUyOWItNDFkNC1hNzE2LTQ0NjY1NTQ0MDAwMCcsIFZhbHVlVHlwZTo6VVVJRCksCiAqICAgICAgICAgdHlwZWQoWyd0YWcxJywgJ3RhZzInXSwgVmFsdWVUeXBlOjpURVhUX0FSUkFZKSwKICogICAgIF0KICogKTsKICov"},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/client.php","start_line_in_file":386,"slug":"converted-parameters","name":"converted_parameters","namespace":"Flow\\PostgreSql\\DSL","parameters":[{"name":"values","type":[{"name":"array","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"ConvertedParameters","namespace":"Flow\\PostgreSql\\Client","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIFBhcmFtZXRlcnMgYWxyZWFkeSBpbiBQb3N0Z3JlU1FMJ3MgdGV4dCBmb3JtLCB3aGljaCBDbGllbnQ6OmV4ZWN1dGUoKSBzZW5kcyB3aXRob3V0IHJ1bm5pbmcgYSBjb252ZXJ0ZXIuCiAqCiAqIEBwYXJhbSBsaXN0PG51bGx8c3RyaW5nPiAkdmFsdWVzCiAqCiAqIEBleGFtcGxlCiAqICRjbGllbnQtPmV4ZWN1dGUoJ1VQREFURSB1c2VycyBTRVQgYWN0aXZlID0gJDEgV0hFUkUgaWQgPSAkMicsIGNvbnZlcnRlZF9wYXJhbWV0ZXJzKFsnZicsICcxJ10pKTsKICov"},{"repository_path":"src\/lib\/telemetry\/src\/Flow\/Telemetry\/DSL\/functions.php","start_line_in_file":126,"slug":"trace-id","name":"trace_id","namespace":"Flow\\Telemetry\\DSL","parameters":[{"name":"hex","type":[{"name":"string","namespace":null,"is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"null"}],"return_type":[{"name":"TraceId","namespace":"Flow\\Telemetry\\Context","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"TELEMETRY","type":"TYPE"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBhIFRyYWNlSWQuCiAqCiAqIElmIGEgaGV4IHN0cmluZyBpcyBwcm92aWRlZCwgY3JlYXRlcyBhIFRyYWNlSWQgZnJvbSBpdC4KICogT3RoZXJ3aXNlLCBnZW5lcmF0ZXMgYSBuZXcgcmFuZG9tIFRyYWNlSWQuCiAqCiAqIEBwYXJhbSBudWxsfHN0cmluZyAkaGV4IE9wdGlvbmFsIDMyLWNoYXJhY3RlciBoZXhhZGVjaW1hbCBzdHJpbmcKICoKICogQHRocm93cyBcSW52YWxpZEFyZ3VtZW50RXhjZXB0aW9uIGlmIHRoZSBoZXggc3RyaW5nIGlzIGludmFsaWQKICov"},{"repository_path":"src\/lib\/telemetry\/src\/Flow\/Telemetry\/DSL\/functions.php","start_line_in_file":146,"slug":"span-id","name":"span_id","namespace":"Flow\\Telemetry\\DSL","parameters":[{"name":"hex","type":[{"name":"string","namespace":null,"is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"null"}],"return_type":[{"name":"SpanId","namespace":"Flow\\Telemetry\\Context","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"TELEMETRY","type":"TYPE"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBhIFNwYW5JZC4KICoKICogSWYgYSBoZXggc3RyaW5nIGlzIHByb3ZpZGVkLCBjcmVhdGVzIGEgU3BhbklkIGZyb20gaXQuCiAqIE90aGVyd2lzZSwgZ2VuZXJhdGVzIGEgbmV3IHJhbmRvbSBTcGFuSWQuCiAqCiAqIEBwYXJhbSBudWxsfHN0cmluZyAkaGV4IE9wdGlvbmFsIDE2LWNoYXJhY3RlciBoZXhhZGVjaW1hbCBzdHJpbmcKICoKICogQHRocm93cyBcSW52YWxpZEFyZ3VtZW50RXhjZXB0aW9uIGlmIHRoZSBoZXggc3RyaW5nIGlzIGludmFsaWQKICov"},{"repository_path":"src\/lib\/telemetry\/src\/Flow\/Telemetry\/DSL\/functions.php","start_line_in_file":161,"slug":"baggage","name":"baggage","namespace":"Flow\\Telemetry\\DSL","parameters":[{"name":"entries","type":[{"name":"array","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"[]"}],"return_type":[{"name":"Baggage","namespace":"Flow\\Telemetry\\Context","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"TELEMETRY","type":"TYPE"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBhIEJhZ2dhZ2UuCiAqCiAqIEBwYXJhbSBhcnJheTxzdHJpbmcsIHN0cmluZz4gJGVudHJpZXMgSW5pdGlhbCBrZXktdmFsdWUgZW50cmllcwogKi8="},{"repository_path":"src\/lib\/telemetry\/src\/Flow\/Telemetry\/DSL\/functions.php","start_line_in_file":175,"slug":"context","name":"context","namespace":"Flow\\Telemetry\\DSL","parameters":[{"name":"baggage","type":[{"name":"Baggage","namespace":"Flow\\Telemetry\\Context","is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"null"}],"return_type":[{"name":"Context","namespace":"Flow\\Telemetry\\Context","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"TELEMETRY","type":"TYPE"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBhIHJvb3QgQ29udGV4dCAobm8gYWN0aXZlIHNwYW4pLgogKgogKiBBIHNwYW4gY3JlYXRlZCBpbiB0aGlzIGNvbnRleHQgYmVjb21lcyBhIG5ldyB0cmFjZSByb290LiBBdHRhY2ggYW4gYWN0aXZlIHNwYW4gd2l0aAogKiBDb250ZXh0Ojp3aXRoQWN0aXZlU3BhbigpIHRvIG1ha2Ugc3Vic2VxdWVudCBzcGFucyBpdHMgY2hpbGRyZW4uCiAqCiAqIEBwYXJhbSBudWxsfEJhZ2dhZ2UgJGJhZ2dhZ2UgT3B0aW9uYWwgQmFnZ2FnZSB0byB1c2UKICov"},{"repository_path":"src\/lib\/telemetry\/src\/Flow\/Telemetry\/DSL\/functions.php","start_line_in_file":189,"slug":"memory-context-storage","name":"memory_context_storage","namespace":"Flow\\Telemetry\\DSL","parameters":[{"name":"context","type":[{"name":"Context","namespace":"Flow\\Telemetry\\Context","is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"null"}],"return_type":[{"name":"MemoryContextStorage","namespace":"Flow\\Telemetry\\Context","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"TELEMETRY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBhIE1lbW9yeUNvbnRleHRTdG9yYWdlLgogKgogKiBJbi1tZW1vcnkgY29udGV4dCBzdG9yYWdlIGZvciBzdG9yaW5nIGFuZCByZXRyaWV2aW5nIHRoZSBjdXJyZW50IGNvbnRleHQuCiAqIEEgc2luZ2xlIGluc3RhbmNlIHNob3VsZCBiZSBzaGFyZWQgYWNyb3NzIGFsbCBwcm92aWRlcnMgd2l0aGluIGEgcmVxdWVzdCBsaWZlY3ljbGUuCiAqCiAqIEBwYXJhbSBudWxsfENvbnRleHQgJGNvbnRleHQgT3B0aW9uYWwgaW5pdGlhbCBjb250ZXh0CiAqLw=="},{"repository_path":"src\/lib\/telemetry\/src\/Flow\/Telemetry\/DSL\/functions.php","start_line_in_file":200,"slug":"resource","name":"resource","namespace":"Flow\\Telemetry\\DSL","parameters":[{"name":"attributes","type":[{"name":"Attributes","namespace":"Flow\\Telemetry","is_nullable":false,"is_variadic":false},{"name":"array","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"[]"}],"return_type":[{"name":"Resource","namespace":"Flow\\Telemetry","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"TELEMETRY","type":"TYPE"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBhIFJlc291cmNlLgogKgogKiBAcGFyYW0gYXJyYXk8c3RyaW5nLCBhcnJheTxhcnJheS1rZXksIG1peGVkPnxib29sfFxEYXRlVGltZUludGVyZmFjZXxmbG9hdHxpbnR8c3RyaW5nfFxUaHJvd2FibGU+fEF0dHJpYnV0ZXMgJGF0dHJpYnV0ZXMgUmVzb3VyY2UgYXR0cmlidXRlcwogKi8="},{"repository_path":"src\/lib\/telemetry\/src\/Flow\/Telemetry\/DSL\/functions.php","start_line_in_file":213,"slug":"span-context","name":"span_context","namespace":"Flow\\Telemetry\\DSL","parameters":[{"name":"traceId","type":[{"name":"TraceId","namespace":"Flow\\Telemetry\\Context","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"spanId","type":[{"name":"SpanId","namespace":"Flow\\Telemetry\\Context","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"parentSpanId","type":[{"name":"SpanId","namespace":"Flow\\Telemetry\\Context","is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"null"}],"return_type":[{"name":"SpanContext","namespace":"Flow\\Telemetry\\Tracer","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"TELEMETRY","type":"TYPE"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBhIFNwYW5Db250ZXh0LgogKgogKiBAcGFyYW0gVHJhY2VJZCAkdHJhY2VJZCBUaGUgdHJhY2UgSUQKICogQHBhcmFtIFNwYW5JZCAkc3BhbklkIFRoZSBzcGFuIElECiAqIEBwYXJhbSBudWxsfFNwYW5JZCAkcGFyZW50U3BhbklkIE9wdGlvbmFsIHBhcmVudCBzcGFuIElECiAqLw=="},{"repository_path":"src\/lib\/telemetry\/src\/Flow\/Telemetry\/DSL\/functions.php","start_line_in_file":226,"slug":"span-event","name":"span_event","namespace":"Flow\\Telemetry\\DSL","parameters":[{"name":"name","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"timestamp","type":[{"name":"DateTimeImmutable","namespace":"","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"attributes","type":[{"name":"Attributes","namespace":"Flow\\Telemetry","is_nullable":false,"is_variadic":false},{"name":"array","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"[]"}],"return_type":[{"name":"GenericEvent","namespace":"Flow\\Telemetry\\Tracer","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"TELEMETRY","type":"TYPE"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBhIFNwYW5FdmVudCAoR2VuZXJpY0V2ZW50KSB3aXRoIGFuIGV4cGxpY2l0IHRpbWVzdGFtcC4KICoKICogQHBhcmFtIHN0cmluZyAkbmFtZSBFdmVudCBuYW1lCiAqIEBwYXJhbSBcRGF0ZVRpbWVJbW11dGFibGUgJHRpbWVzdGFtcCBFdmVudCB0aW1lc3RhbXAKICogQHBhcmFtIGFycmF5PHN0cmluZywgYXJyYXk8YXJyYXkta2V5LCBtaXhlZD58Ym9vbHxcRGF0ZVRpbWVJbnRlcmZhY2V8ZmxvYXR8aW50fHN0cmluZ3xcVGhyb3dhYmxlPnxBdHRyaWJ1dGVzICRhdHRyaWJ1dGVzIEV2ZW50IGF0dHJpYnV0ZXMKICov"},{"repository_path":"src\/lib\/telemetry\/src\/Flow\/Telemetry\/DSL\/functions.php","start_line_in_file":238,"slug":"span-link","name":"span_link","namespace":"Flow\\Telemetry\\DSL","parameters":[{"name":"context","type":[{"name":"SpanContext","namespace":"Flow\\Telemetry\\Tracer","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"attributes","type":[{"name":"Attributes","namespace":"Flow\\Telemetry","is_nullable":false,"is_variadic":false},{"name":"array","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"[]"}],"return_type":[{"name":"SpanLink","namespace":"Flow\\Telemetry\\Tracer","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"TELEMETRY","type":"TYPE"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBhIFNwYW5MaW5rLgogKgogKiBAcGFyYW0gU3BhbkNvbnRleHQgJGNvbnRleHQgVGhlIGxpbmtlZCBzcGFuIGNvbnRleHQKICogQHBhcmFtIGFycmF5PHN0cmluZywgYXJyYXk8YXJyYXkta2V5LCBtaXhlZD58Ym9vbHxcRGF0ZVRpbWVJbnRlcmZhY2V8ZmxvYXR8aW50fHN0cmluZ3xcVGhyb3dhYmxlPnxBdHRyaWJ1dGVzICRhdHRyaWJ1dGVzIExpbmsgYXR0cmlidXRlcwogKi8="},{"repository_path":"src\/lib\/telemetry\/src\/Flow\/Telemetry\/DSL\/functions.php","start_line_in_file":254,"slug":"span-limits","name":"span_limits","namespace":"Flow\\Telemetry\\DSL","parameters":[{"name":"attributeCountLimit","type":[{"name":"int","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"128"},{"name":"eventCountLimit","type":[{"name":"int","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"128"},{"name":"linkCountLimit","type":[{"name":"int","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"128"},{"name":"attributePerEventCountLimit","type":[{"name":"int","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"128"},{"name":"attributePerLinkCountLimit","type":[{"name":"int","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"128"},{"name":"attributeValueLengthLimit","type":[{"name":"int","namespace":null,"is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"null"}],"return_type":[{"name":"SpanLimits","namespace":"Flow\\Telemetry\\Tracer","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"TELEMETRY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBTcGFuTGltaXRzIGNvbmZpZ3VyYXRpb24uCiAqCiAqIEBwYXJhbSBpbnQgJGF0dHJpYnV0ZUNvdW50TGltaXQgTWF4aW11bSBudW1iZXIgb2YgYXR0cmlidXRlcyBwZXIgc3BhbgogKiBAcGFyYW0gaW50ICRldmVudENvdW50TGltaXQgTWF4aW11bSBudW1iZXIgb2YgZXZlbnRzIHBlciBzcGFuCiAqIEBwYXJhbSBpbnQgJGxpbmtDb3VudExpbWl0IE1heGltdW0gbnVtYmVyIG9mIGxpbmtzIHBlciBzcGFuCiAqIEBwYXJhbSBpbnQgJGF0dHJpYnV0ZVBlckV2ZW50Q291bnRMaW1pdCBNYXhpbXVtIG51bWJlciBvZiBhdHRyaWJ1dGVzIHBlciBldmVudAogKiBAcGFyYW0gaW50ICRhdHRyaWJ1dGVQZXJMaW5rQ291bnRMaW1pdCBNYXhpbXVtIG51bWJlciBvZiBhdHRyaWJ1dGVzIHBlciBsaW5rCiAqIEBwYXJhbSBudWxsfGludCAkYXR0cmlidXRlVmFsdWVMZW5ndGhMaW1pdCBNYXhpbXVtIGxlbmd0aCBmb3Igc3RyaW5nIGF0dHJpYnV0ZSB2YWx1ZXMgKG51bGwgPSB1bmxpbWl0ZWQpCiAqLw=="},{"repository_path":"src\/lib\/telemetry\/src\/Flow\/Telemetry\/DSL\/functions.php","start_line_in_file":279,"slug":"log-record-limits","name":"log_record_limits","namespace":"Flow\\Telemetry\\DSL","parameters":[{"name":"attributeCountLimit","type":[{"name":"int","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"128"},{"name":"attributeValueLengthLimit","type":[{"name":"int","namespace":null,"is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"null"}],"return_type":[{"name":"LogRecordLimits","namespace":"Flow\\Telemetry\\Logger","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"TELEMETRY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBMb2dSZWNvcmRMaW1pdHMgY29uZmlndXJhdGlvbi4KICoKICogQHBhcmFtIGludCAkYXR0cmlidXRlQ291bnRMaW1pdCBNYXhpbXVtIG51bWJlciBvZiBhdHRyaWJ1dGVzIHBlciBsb2cgcmVjb3JkCiAqIEBwYXJhbSBudWxsfGludCAkYXR0cmlidXRlVmFsdWVMZW5ndGhMaW1pdCBNYXhpbXVtIGxlbmd0aCBmb3Igc3RyaW5nIGF0dHJpYnV0ZSB2YWx1ZXMgKG51bGwgPSB1bmxpbWl0ZWQpCiAqLw=="},{"repository_path":"src\/lib\/telemetry\/src\/Flow\/Telemetry\/DSL\/functions.php","start_line_in_file":290,"slug":"metric-limits","name":"metric_limits","namespace":"Flow\\Telemetry\\DSL","parameters":[{"name":"cardinalityLimit","type":[{"name":"int","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"2000"}],"return_type":[{"name":"MetricLimits","namespace":"Flow\\Telemetry\\Meter","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"TELEMETRY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBNZXRyaWNMaW1pdHMgY29uZmlndXJhdGlvbi4KICoKICogQHBhcmFtIGludCAkY2FyZGluYWxpdHlMaW1pdCBNYXhpbXVtIG51bWJlciBvZiB1bmlxdWUgYXR0cmlidXRlIGNvbWJpbmF0aW9ucyBwZXIgaW5zdHJ1bWVudAogKi8="},{"repository_path":"src\/lib\/telemetry\/src\/Flow\/Telemetry\/DSL\/functions.php","start_line_in_file":301,"slug":"void-span-processor","name":"void_span_processor","namespace":"Flow\\Telemetry\\DSL","parameters":[],"return_type":[{"name":"VoidSpanProcessor","namespace":"Flow\\Telemetry\\Provider\\Void","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"TELEMETRY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBhIFZvaWRTcGFuUHJvY2Vzc29yLgogKgogKiBOby1vcCBzcGFuIHByb2Nlc3NvciB0aGF0IGRpc2NhcmRzIGFsbCBkYXRhLgogKi8="},{"repository_path":"src\/lib\/telemetry\/src\/Flow\/Telemetry\/DSL\/functions.php","start_line_in_file":312,"slug":"void-metric-processor","name":"void_metric_processor","namespace":"Flow\\Telemetry\\DSL","parameters":[],"return_type":[{"name":"VoidMetricProcessor","namespace":"Flow\\Telemetry\\Provider\\Void","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"TELEMETRY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBhIFZvaWRNZXRyaWNQcm9jZXNzb3IuCiAqCiAqIE5vLW9wIG1ldHJpYyBwcm9jZXNzb3IgdGhhdCBkaXNjYXJkcyBhbGwgZGF0YS4KICov"},{"repository_path":"src\/lib\/telemetry\/src\/Flow\/Telemetry\/DSL\/functions.php","start_line_in_file":323,"slug":"void-log-processor","name":"void_log_processor","namespace":"Flow\\Telemetry\\DSL","parameters":[],"return_type":[{"name":"VoidLogProcessor","namespace":"Flow\\Telemetry\\Provider\\Void","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"TELEMETRY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBhIFZvaWRMb2dQcm9jZXNzb3IuCiAqCiAqIE5vLW9wIGxvZyBwcm9jZXNzb3IgdGhhdCBkaXNjYXJkcyBhbGwgZGF0YS4KICov"},{"repository_path":"src\/lib\/telemetry\/src\/Flow\/Telemetry\/DSL\/functions.php","start_line_in_file":334,"slug":"void-exporter","name":"void_exporter","namespace":"Flow\\Telemetry\\DSL","parameters":[],"return_type":[{"name":"VoidExporter","namespace":"Flow\\Telemetry\\Provider\\Void","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"TELEMETRY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBhIFZvaWRFeHBvcnRlci4KICoKICogTm8tb3AgdW5pZmllZCBleHBvcnRlciB0aGF0IGRpc2NhcmRzIGxvZ3MsIG1ldHJpY3MsIGFuZCBzcGFucy4KICov"},{"repository_path":"src\/lib\/telemetry\/src\/Flow\/Telemetry\/DSL\/functions.php","start_line_in_file":348,"slug":"memory-exporter","name":"memory_exporter","namespace":"Flow\\Telemetry\\DSL","parameters":[{"name":"maxEntriesPerSignal","type":[{"name":"int","namespace":null,"is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"null"}],"return_type":[{"name":"MemoryExporter","namespace":"Flow\\Telemetry\\Provider\\Memory","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"TELEMETRY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBhIE1lbW9yeUV4cG9ydGVyLgogKgogKiBVbmlmaWVkIGV4cG9ydGVyIHRoYXQgc3RvcmVzIGxvZ3MsIG1ldHJpY3MsIGFuZCBzcGFucyBpbiBtZW1vcnkgZm9yIGRpcmVjdCBhY2Nlc3MuCiAqIFVzZWZ1bCBmb3IgdGVzdGluZyBhbmQgaW5zcGVjdGlvbiB3aXRob3V0IHNlcmlhbGl6YXRpb24uCiAqCiAqIEBwYXJhbSBudWxsfGludCAkbWF4RW50cmllc1BlclNpZ25hbCBtYXhpbXVtIGVudHJpZXMgcmV0YWluZWQgcGVyIHNpZ25hbCB0eXBlOyBudWxsIGtlZXBzIGV2ZXJ5dGhpbmcKICov"},{"repository_path":"src\/lib\/telemetry\/src\/Flow\/Telemetry\/DSL\/functions.php","start_line_in_file":360,"slug":"memory-span-processor","name":"memory_span_processor","namespace":"Flow\\Telemetry\\DSL","parameters":[{"name":"exporter","type":[{"name":"Exporter","namespace":"Flow\\Telemetry\\Exporter","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"errorHandler","type":[{"name":"ErrorHandler","namespace":"Flow\\Telemetry\\ErrorHandler","is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"Flow\\Telemetry\\ErrorHandler\\ErrorLogHandler::..."}],"return_type":[{"name":"MemorySpanProcessor","namespace":"Flow\\Telemetry\\Provider\\Memory","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"TELEMETRY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBhIE1lbW9yeVNwYW5Qcm9jZXNzb3IuCiAqCiAqIEBwYXJhbSBFeHBvcnRlciAkZXhwb3J0ZXIgVGhlIGV4cG9ydGVyIHRvIHNlbmQgc3BhbnMgdG8KICogQHBhcmFtIEVycm9ySGFuZGxlciAkZXJyb3JIYW5kbGVyIEhhbmRsZXIgZm9yIFRocm93YWJsZXMgcmFpc2VkIGJ5IHRoZSBleHBvcnRlcgogKi8="},{"repository_path":"src\/lib\/telemetry\/src\/Flow\/Telemetry\/DSL\/functions.php","start_line_in_file":374,"slug":"memory-metric-processor","name":"memory_metric_processor","namespace":"Flow\\Telemetry\\DSL","parameters":[{"name":"exporter","type":[{"name":"Exporter","namespace":"Flow\\Telemetry\\Exporter","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"errorHandler","type":[{"name":"ErrorHandler","namespace":"Flow\\Telemetry\\ErrorHandler","is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"Flow\\Telemetry\\ErrorHandler\\ErrorLogHandler::..."}],"return_type":[{"name":"MemoryMetricProcessor","namespace":"Flow\\Telemetry\\Provider\\Memory","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"TELEMETRY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBhIE1lbW9yeU1ldHJpY1Byb2Nlc3Nvci4KICoKICogQHBhcmFtIEV4cG9ydGVyICRleHBvcnRlciBUaGUgZXhwb3J0ZXIgdG8gc2VuZCBtZXRyaWNzIHRvCiAqIEBwYXJhbSBFcnJvckhhbmRsZXIgJGVycm9ySGFuZGxlciBIYW5kbGVyIGZvciBUaHJvd2FibGVzIHJhaXNlZCBieSB0aGUgZXhwb3J0ZXIKICov"},{"repository_path":"src\/lib\/telemetry\/src\/Flow\/Telemetry\/DSL\/functions.php","start_line_in_file":388,"slug":"memory-log-processor","name":"memory_log_processor","namespace":"Flow\\Telemetry\\DSL","parameters":[{"name":"exporter","type":[{"name":"Exporter","namespace":"Flow\\Telemetry\\Exporter","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"errorHandler","type":[{"name":"ErrorHandler","namespace":"Flow\\Telemetry\\ErrorHandler","is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"Flow\\Telemetry\\ErrorHandler\\ErrorLogHandler::..."}],"return_type":[{"name":"MemoryLogProcessor","namespace":"Flow\\Telemetry\\Provider\\Memory","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"TELEMETRY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBhIE1lbW9yeUxvZ1Byb2Nlc3Nvci4KICoKICogQHBhcmFtIEV4cG9ydGVyICRleHBvcnRlciBUaGUgZXhwb3J0ZXIgdG8gc2VuZCBsb2dzIHRvCiAqIEBwYXJhbSBFcnJvckhhbmRsZXIgJGVycm9ySGFuZGxlciBIYW5kbGVyIGZvciBUaHJvd2FibGVzIHJhaXNlZCBieSB0aGUgZXhwb3J0ZXIKICov"},{"repository_path":"src\/lib\/telemetry\/src\/Flow\/Telemetry\/DSL\/functions.php","start_line_in_file":406,"slug":"tracer-provider","name":"tracer_provider","namespace":"Flow\\Telemetry\\DSL","parameters":[{"name":"processor","type":[{"name":"SpanProcessor","namespace":"Flow\\Telemetry\\Tracer","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"clock","type":[{"name":"ClockInterface","namespace":"Psr\\Clock","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"contextStorage","type":[{"name":"ContextStorage","namespace":"Flow\\Telemetry\\Context","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"sampler","type":[{"name":"Sampler","namespace":"Flow\\Telemetry\\Tracer\\Sampler","is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"Flow\\Telemetry\\Tracer\\Sampler\\ParentBasedSampler::..."},{"name":"limits","type":[{"name":"SpanLimits","namespace":"Flow\\Telemetry\\Tracer","is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"Flow\\Telemetry\\Tracer\\SpanLimits::..."},{"name":"errorHandler","type":[{"name":"ErrorHandler","namespace":"Flow\\Telemetry\\ErrorHandler","is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"Flow\\Telemetry\\ErrorHandler\\ErrorLogHandler::..."}],"return_type":[{"name":"TracerProvider","namespace":"Flow\\Telemetry\\Tracer","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"TELEMETRY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBhIFRyYWNlclByb3ZpZGVyLgogKgogKiBAcGFyYW0gU3BhblByb2Nlc3NvciAkcHJvY2Vzc29yIFRoZSBwcm9jZXNzb3IgZm9yIHNwYW5zCiAqIEBwYXJhbSBDbG9ja0ludGVyZmFjZSAkY2xvY2sgVGhlIGNsb2NrIGZvciB0aW1lc3RhbXBzCiAqIEBwYXJhbSBDb250ZXh0U3RvcmFnZSAkY29udGV4dFN0b3JhZ2UgU3RvcmFnZSBmb3IgY29udGV4dCBwcm9wYWdhdGlvbgogKiBAcGFyYW0gU2FtcGxlciAkc2FtcGxlciBTYW1wbGluZyBzdHJhdGVneSBmb3Igc3BhbnMKICogQHBhcmFtIFNwYW5MaW1pdHMgJGxpbWl0cyBMaW1pdHMgZm9yIHNwYW4gYXR0cmlidXRlcywgZXZlbnRzLCBhbmQgbGlua3MKICogQHBhcmFtIEVycm9ySGFuZGxlciAkZXJyb3JIYW5kbGVyIEhhbmRsZXIgZm9yIHJ1bnRpbWUgVGhyb3dhYmxlcyByYWlzZWQgYnkgdGhlIHByb2Nlc3NvcgogKi8="},{"repository_path":"src\/lib\/telemetry\/src\/Flow\/Telemetry\/DSL\/functions.php","start_line_in_file":427,"slug":"logger-provider","name":"logger_provider","namespace":"Flow\\Telemetry\\DSL","parameters":[{"name":"processor","type":[{"name":"LogProcessor","namespace":"Flow\\Telemetry\\Logger","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"clock","type":[{"name":"ClockInterface","namespace":"Psr\\Clock","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"contextStorage","type":[{"name":"ContextStorage","namespace":"Flow\\Telemetry\\Context","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"limits","type":[{"name":"LogRecordLimits","namespace":"Flow\\Telemetry\\Logger","is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"Flow\\Telemetry\\Logger\\LogRecordLimits::..."},{"name":"errorHandler","type":[{"name":"ErrorHandler","namespace":"Flow\\Telemetry\\ErrorHandler","is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"Flow\\Telemetry\\ErrorHandler\\ErrorLogHandler::..."}],"return_type":[{"name":"LoggerProvider","namespace":"Flow\\Telemetry\\Logger","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"TELEMETRY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBhIExvZ2dlclByb3ZpZGVyLgogKgogKiBAcGFyYW0gTG9nUHJvY2Vzc29yICRwcm9jZXNzb3IgVGhlIHByb2Nlc3NvciBmb3IgbG9ncwogKiBAcGFyYW0gQ2xvY2tJbnRlcmZhY2UgJGNsb2NrIFRoZSBjbG9jayBmb3IgdGltZXN0YW1wcwogKiBAcGFyYW0gQ29udGV4dFN0b3JhZ2UgJGNvbnRleHRTdG9yYWdlIFN0b3JhZ2UgZm9yIHNwYW4gY29ycmVsYXRpb24KICogQHBhcmFtIExvZ1JlY29yZExpbWl0cyAkbGltaXRzIExpbWl0cyBmb3IgbG9nIHJlY29yZCBhdHRyaWJ1dGVzCiAqIEBwYXJhbSBFcnJvckhhbmRsZXIgJGVycm9ySGFuZGxlciBIYW5kbGVyIGZvciBydW50aW1lIFRocm93YWJsZXMgcmFpc2VkIGJ5IHRoZSBwcm9jZXNzb3IKICov"},{"repository_path":"src\/lib\/telemetry\/src\/Flow\/Telemetry\/DSL\/functions.php","start_line_in_file":448,"slug":"meter-provider","name":"meter_provider","namespace":"Flow\\Telemetry\\DSL","parameters":[{"name":"processor","type":[{"name":"MetricProcessor","namespace":"Flow\\Telemetry\\Meter","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"clock","type":[{"name":"ClockInterface","namespace":"Psr\\Clock","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"temporality","type":[{"name":"AggregationTemporality","namespace":"Flow\\Telemetry\\Meter","is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"Flow\\Telemetry\\Meter\\AggregationTemporality::..."},{"name":"exemplarFilter","type":[{"name":"ExemplarFilter","namespace":"Flow\\Telemetry\\Meter\\Exemplar","is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"Flow\\Telemetry\\Meter\\Exemplar\\TraceBasedExemplarFilter::..."},{"name":"limits","type":[{"name":"MetricLimits","namespace":"Flow\\Telemetry\\Meter","is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"Flow\\Telemetry\\Meter\\MetricLimits::..."},{"name":"errorHandler","type":[{"name":"ErrorHandler","namespace":"Flow\\Telemetry\\ErrorHandler","is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"Flow\\Telemetry\\ErrorHandler\\ErrorLogHandler::..."}],"return_type":[{"name":"MeterProvider","namespace":"Flow\\Telemetry\\Meter","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"TELEMETRY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBhIE1ldGVyUHJvdmlkZXIuCiAqCiAqIEBwYXJhbSBNZXRyaWNQcm9jZXNzb3IgJHByb2Nlc3NvciBUaGUgcHJvY2Vzc29yIGZvciBtZXRyaWNzCiAqIEBwYXJhbSBDbG9ja0ludGVyZmFjZSAkY2xvY2sgVGhlIGNsb2NrIGZvciB0aW1lc3RhbXBzCiAqIEBwYXJhbSBBZ2dyZWdhdGlvblRlbXBvcmFsaXR5ICR0ZW1wb3JhbGl0eSBBZ2dyZWdhdGlvbiB0ZW1wb3JhbGl0eSBmb3IgbWV0cmljcwogKiBAcGFyYW0gRXhlbXBsYXJGaWx0ZXIgJGV4ZW1wbGFyRmlsdGVyIEZpbHRlciBmb3IgZXhlbXBsYXIgc2FtcGxpbmcgKGRlZmF1bHQ6IFRyYWNlQmFzZWRFeGVtcGxhckZpbHRlcikKICogQHBhcmFtIE1ldHJpY0xpbWl0cyAkbGltaXRzIENhcmRpbmFsaXR5IGxpbWl0cyBmb3IgbWV0cmljIGluc3RydW1lbnRzCiAqIEBwYXJhbSBFcnJvckhhbmRsZXIgJGVycm9ySGFuZGxlciBIYW5kbGVyIGZvciBydW50aW1lIFRocm93YWJsZXMgcmFpc2VkIGJ5IHRoZSBwcm9jZXNzb3IKICov"},{"repository_path":"src\/lib\/telemetry\/src\/Flow\/Telemetry\/DSL\/functions.php","start_line_in_file":471,"slug":"telemetry","name":"telemetry","namespace":"Flow\\Telemetry\\DSL","parameters":[{"name":"resource","type":[{"name":"Resource","namespace":"Flow\\Telemetry","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"tracerProvider","type":[{"name":"TracerProvider","namespace":"Flow\\Telemetry\\Tracer","is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"null"},{"name":"meterProvider","type":[{"name":"MeterProvider","namespace":"Flow\\Telemetry\\Meter","is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"null"},{"name":"loggerProvider","type":[{"name":"LoggerProvider","namespace":"Flow\\Telemetry\\Logger","is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"null"},{"name":"errorHandler","type":[{"name":"ErrorHandler","namespace":"Flow\\Telemetry\\ErrorHandler","is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"Flow\\Telemetry\\ErrorHandler\\ErrorLogHandler::..."}],"return_type":[{"name":"Telemetry","namespace":"Flow\\Telemetry","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"TELEMETRY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBhIG5ldyBUZWxlbWV0cnkgaW5zdGFuY2Ugd2l0aCB0aGUgZ2l2ZW4gcHJvdmlkZXJzLgogKgogKiBJZiBwcm92aWRlcnMgYXJlIG5vdCBzcGVjaWZpZWQsIHZvaWQgcHJvdmlkZXJzIChuby1vcCkgYXJlIHVzZWQuCiAqCiAqIEBwYXJhbSBcRmxvd1xUZWxlbWV0cnlcUmVzb3VyY2UgJHJlc291cmNlIFRoZSByZXNvdXJjZSBkZXNjcmliaW5nIHRoZSBlbnRpdHkgcHJvZHVjaW5nIHRlbGVtZXRyeQogKiBAcGFyYW0gbnVsbHxUcmFjZXJQcm92aWRlciAkdHJhY2VyUHJvdmlkZXIgVGhlIHRyYWNlciBwcm92aWRlciAobnVsbCBmb3Igdm9pZC9kaXNhYmxlZCkKICogQHBhcmFtIG51bGx8TWV0ZXJQcm92aWRlciAkbWV0ZXJQcm92aWRlciBUaGUgbWV0ZXIgcHJvdmlkZXIgKG51bGwgZm9yIHZvaWQvZGlzYWJsZWQpCiAqIEBwYXJhbSBudWxsfExvZ2dlclByb3ZpZGVyICRsb2dnZXJQcm92aWRlciBUaGUgbG9nZ2VyIHByb3ZpZGVyIChudWxsIGZvciB2b2lkL2Rpc2FibGVkKQogKiBAcGFyYW0gRXJyb3JIYW5kbGVyICRlcnJvckhhbmRsZXIgSGFuZGxlciBwcm9wYWdhdGVkIHRvIGRlZmF1bHQgdm9pZCBwcm92aWRlcnMgd2hlbiBleHBsaWNpdCBvbmVzIGFyZSBub3Qgc3VwcGxpZWQKICov"},{"repository_path":"src\/lib\/telemetry\/src\/Flow\/Telemetry\/DSL\/functions.php","start_line_in_file":507,"slug":"instrumentation-scope","name":"instrumentation_scope","namespace":"Flow\\Telemetry\\DSL","parameters":[{"name":"name","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"version","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"'unknown'"},{"name":"schemaUrl","type":[{"name":"string","namespace":null,"is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"null"},{"name":"attributes","type":[{"name":"Attributes","namespace":"Flow\\Telemetry","is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"Flow\\Telemetry\\Attributes::..."}],"return_type":[{"name":"InstrumentationScope","namespace":"Flow\\Telemetry","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"TELEMETRY","type":"TYPE"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBhbiBJbnN0cnVtZW50YXRpb25TY29wZS4KICoKICogQHBhcmFtIHN0cmluZyAkbmFtZSBUaGUgaW5zdHJ1bWVudGF0aW9uIHNjb3BlIG5hbWUKICogQHBhcmFtIHN0cmluZyAkdmVyc2lvbiBUaGUgaW5zdHJ1bWVudGF0aW9uIHNjb3BlIHZlcnNpb24KICogQHBhcmFtIG51bGx8c3RyaW5nICRzY2hlbWFVcmwgT3B0aW9uYWwgc2NoZW1hIFVSTAogKi8="},{"repository_path":"src\/lib\/telemetry\/src\/Flow\/Telemetry\/DSL\/functions.php","start_line_in_file":524,"slug":"batching-span-processor","name":"batching_span_processor","namespace":"Flow\\Telemetry\\DSL","parameters":[{"name":"exporter","type":[{"name":"Exporter","namespace":"Flow\\Telemetry\\Exporter","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"batchSize","type":[{"name":"int","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"512"},{"name":"errorHandler","type":[{"name":"ErrorHandler","namespace":"Flow\\Telemetry\\ErrorHandler","is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"Flow\\Telemetry\\ErrorHandler\\ErrorLogHandler::..."}],"return_type":[{"name":"BatchingSpanProcessor","namespace":"Flow\\Telemetry\\Tracer\\Processor","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"TELEMETRY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBhIEJhdGNoaW5nU3BhblByb2Nlc3Nvci4KICoKICogQHBhcmFtIEV4cG9ydGVyICRleHBvcnRlciBUaGUgZXhwb3J0ZXIgdG8gc2VuZCBzcGFucyB0bwogKiBAcGFyYW0gaW50ICRiYXRjaFNpemUgTnVtYmVyIG9mIHNwYW5zIHRvIGNvbGxlY3QgYmVmb3JlIGV4cG9ydGluZyAoZGVmYXVsdCA1MTIpCiAqIEBwYXJhbSBFcnJvckhhbmRsZXIgJGVycm9ySGFuZGxlciBIYW5kbGVyIGZvciBUaHJvd2FibGVzIHJhaXNlZCBieSB0aGUgZXhwb3J0ZXIKICov"},{"repository_path":"src\/lib\/telemetry\/src\/Flow\/Telemetry\/DSL\/functions.php","start_line_in_file":539,"slug":"pass-through-span-processor","name":"pass_through_span_processor","namespace":"Flow\\Telemetry\\DSL","parameters":[{"name":"exporter","type":[{"name":"Exporter","namespace":"Flow\\Telemetry\\Exporter","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"errorHandler","type":[{"name":"ErrorHandler","namespace":"Flow\\Telemetry\\ErrorHandler","is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"Flow\\Telemetry\\ErrorHandler\\ErrorLogHandler::..."}],"return_type":[{"name":"PassThroughSpanProcessor","namespace":"Flow\\Telemetry\\Tracer\\Processor","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"TELEMETRY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBhIFBhc3NUaHJvdWdoU3BhblByb2Nlc3Nvci4KICoKICogQHBhcmFtIEV4cG9ydGVyICRleHBvcnRlciBUaGUgZXhwb3J0ZXIgdG8gc2VuZCBzcGFucyB0bwogKiBAcGFyYW0gRXJyb3JIYW5kbGVyICRlcnJvckhhbmRsZXIgSGFuZGxlciBmb3IgVGhyb3dhYmxlcyByYWlzZWQgYnkgdGhlIGV4cG9ydGVyCiAqLw=="},{"repository_path":"src\/lib\/telemetry\/src\/Flow\/Telemetry\/DSL\/functions.php","start_line_in_file":554,"slug":"batching-metric-processor","name":"batching_metric_processor","namespace":"Flow\\Telemetry\\DSL","parameters":[{"name":"exporter","type":[{"name":"Exporter","namespace":"Flow\\Telemetry\\Exporter","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"batchSize","type":[{"name":"int","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"512"},{"name":"errorHandler","type":[{"name":"ErrorHandler","namespace":"Flow\\Telemetry\\ErrorHandler","is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"Flow\\Telemetry\\ErrorHandler\\ErrorLogHandler::..."}],"return_type":[{"name":"BatchingMetricProcessor","namespace":"Flow\\Telemetry\\Meter\\Processor","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"TELEMETRY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBhIEJhdGNoaW5nTWV0cmljUHJvY2Vzc29yLgogKgogKiBAcGFyYW0gRXhwb3J0ZXIgJGV4cG9ydGVyIFRoZSBleHBvcnRlciB0byBzZW5kIG1ldHJpY3MgdG8KICogQHBhcmFtIGludCAkYmF0Y2hTaXplIE51bWJlciBvZiBtZXRyaWNzIHRvIGNvbGxlY3QgYmVmb3JlIGV4cG9ydGluZyAoZGVmYXVsdCA1MTIpCiAqIEBwYXJhbSBFcnJvckhhbmRsZXIgJGVycm9ySGFuZGxlciBIYW5kbGVyIGZvciBUaHJvd2FibGVzIHJhaXNlZCBieSB0aGUgZXhwb3J0ZXIKICov"},{"repository_path":"src\/lib\/telemetry\/src\/Flow\/Telemetry\/DSL\/functions.php","start_line_in_file":569,"slug":"pass-through-metric-processor","name":"pass_through_metric_processor","namespace":"Flow\\Telemetry\\DSL","parameters":[{"name":"exporter","type":[{"name":"Exporter","namespace":"Flow\\Telemetry\\Exporter","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"errorHandler","type":[{"name":"ErrorHandler","namespace":"Flow\\Telemetry\\ErrorHandler","is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"Flow\\Telemetry\\ErrorHandler\\ErrorLogHandler::..."}],"return_type":[{"name":"PassThroughMetricProcessor","namespace":"Flow\\Telemetry\\Meter\\Processor","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"TELEMETRY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBhIFBhc3NUaHJvdWdoTWV0cmljUHJvY2Vzc29yLgogKgogKiBAcGFyYW0gRXhwb3J0ZXIgJGV4cG9ydGVyIFRoZSBleHBvcnRlciB0byBzZW5kIG1ldHJpY3MgdG8KICogQHBhcmFtIEVycm9ySGFuZGxlciAkZXJyb3JIYW5kbGVyIEhhbmRsZXIgZm9yIFRocm93YWJsZXMgcmFpc2VkIGJ5IHRoZSBleHBvcnRlcgogKi8="},{"repository_path":"src\/lib\/telemetry\/src\/Flow\/Telemetry\/DSL\/functions.php","start_line_in_file":584,"slug":"batching-log-processor","name":"batching_log_processor","namespace":"Flow\\Telemetry\\DSL","parameters":[{"name":"exporter","type":[{"name":"Exporter","namespace":"Flow\\Telemetry\\Exporter","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"batchSize","type":[{"name":"int","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"512"},{"name":"errorHandler","type":[{"name":"ErrorHandler","namespace":"Flow\\Telemetry\\ErrorHandler","is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"Flow\\Telemetry\\ErrorHandler\\ErrorLogHandler::..."}],"return_type":[{"name":"BatchingLogProcessor","namespace":"Flow\\Telemetry\\Logger\\Processor","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"TELEMETRY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBhIEJhdGNoaW5nTG9nUHJvY2Vzc29yLgogKgogKiBAcGFyYW0gRXhwb3J0ZXIgJGV4cG9ydGVyIFRoZSBleHBvcnRlciB0byBzZW5kIGxvZ3MgdG8KICogQHBhcmFtIGludCAkYmF0Y2hTaXplIE51bWJlciBvZiBsb2dzIHRvIGNvbGxlY3QgYmVmb3JlIGV4cG9ydGluZyAoZGVmYXVsdCA1MTIpCiAqIEBwYXJhbSBFcnJvckhhbmRsZXIgJGVycm9ySGFuZGxlciBIYW5kbGVyIGZvciBUaHJvd2FibGVzIHJhaXNlZCBieSB0aGUgZXhwb3J0ZXIKICov"},{"repository_path":"src\/lib\/telemetry\/src\/Flow\/Telemetry\/DSL\/functions.php","start_line_in_file":599,"slug":"pass-through-log-processor","name":"pass_through_log_processor","namespace":"Flow\\Telemetry\\DSL","parameters":[{"name":"exporter","type":[{"name":"Exporter","namespace":"Flow\\Telemetry\\Exporter","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"errorHandler","type":[{"name":"ErrorHandler","namespace":"Flow\\Telemetry\\ErrorHandler","is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"Flow\\Telemetry\\ErrorHandler\\ErrorLogHandler::..."}],"return_type":[{"name":"PassThroughLogProcessor","namespace":"Flow\\Telemetry\\Logger\\Processor","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"TELEMETRY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBhIFBhc3NUaHJvdWdoTG9nUHJvY2Vzc29yLgogKgogKiBAcGFyYW0gRXhwb3J0ZXIgJGV4cG9ydGVyIFRoZSBleHBvcnRlciB0byBzZW5kIGxvZ3MgdG8KICogQHBhcmFtIEVycm9ySGFuZGxlciAkZXJyb3JIYW5kbGVyIEhhbmRsZXIgZm9yIFRocm93YWJsZXMgcmFpc2VkIGJ5IHRoZSBleHBvcnRlcgogKi8="},{"repository_path":"src\/lib\/telemetry\/src\/Flow\/Telemetry\/DSL\/functions.php","start_line_in_file":614,"slug":"pipeline-log-processor","name":"pipeline_log_processor","namespace":"Flow\\Telemetry\\DSL","parameters":[{"name":"middleware","type":[{"name":"array","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"sink","type":[{"name":"LogSink","namespace":"Flow\\Telemetry\\Logger","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"PipelineLogProcessor","namespace":"Flow\\Telemetry\\Logger\\Processor","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"TELEMETRY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBhIFBpcGVsaW5lTG9nUHJvY2Vzc29yOiBydW4gZWFjaCBsb2cgZW50cnkgdGhyb3VnaCBhbiBvcmRlcmVkIGNoYWluIG9mCiAqIG1pZGRsZXdhcmUsIHRoZW4gZm9yd2FyZCB0aGUgc3Vydml2b3JzIHRvIGEgc2luZ2xlIHNpbmsuCiAqCiAqIEBwYXJhbSBsaXN0PExvZ01pZGRsZXdhcmU+ICRtaWRkbGV3YXJlIHJ1biBpbiBvcmRlcjsgdGhlIGZpcnN0IHRvIGRyb3AgYW4gZW50cnkgc2hvcnQtY2lyY3VpdHMgdGhlIHJlc3QKICogQHBhcmFtIExvZ1NpbmsgJHNpbmsgdGhlIHRlcm1pbmFsIHByb2Nlc3NvciB0aGF0IGV4cG9ydHMgc3Vydml2aW5nIGVudHJpZXMKICov"},{"repository_path":"src\/lib\/telemetry\/src\/Flow\/Telemetry\/DSL\/functions.php","start_line_in_file":626,"slug":"enriching-log-middleware","name":"enriching_log_middleware","namespace":"Flow\\Telemetry\\DSL","parameters":[{"name":"attributes","type":[{"name":"Attributes","namespace":"Flow\\Telemetry","is_nullable":false,"is_variadic":false},{"name":"array","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"EnrichingLogMiddleware","namespace":"Flow\\Telemetry\\Logger\\Middleware","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"TELEMETRY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBhbiBFbnJpY2hpbmdMb2dNaWRkbGV3YXJlIHRoYXQgbWVyZ2VzIGRlZmF1bHQgYXR0cmlidXRlcyBpbnRvIGV2ZXJ5IGxvZwogKiBlbnRyeS4gQXR0cmlidXRlcyBzZXQgYXQgdGhlIGNhbGwgc2l0ZSB3aW4gb3ZlciB0aGVzZSBkZWZhdWx0cy4KICoKICogQHBhcmFtIGFycmF5PHN0cmluZywgYXJyYXk8YXJyYXkta2V5LCBtaXhlZD58Ym9vbHxcRGF0ZVRpbWVJbnRlcmZhY2V8ZmxvYXR8aW50fHN0cmluZ3xcVGhyb3dhYmxlPnxBdHRyaWJ1dGVzICRhdHRyaWJ1dGVzCiAqLw=="},{"repository_path":"src\/lib\/telemetry\/src\/Flow\/Telemetry\/DSL\/functions.php","start_line_in_file":637,"slug":"attribute-filtering-log-middleware","name":"attribute_filtering_log_middleware","namespace":"Flow\\Telemetry\\DSL","parameters":[{"name":"filter","type":[{"name":"AttributeFilter","namespace":"Flow\\Telemetry\\Filter","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"AttributeFilteringLogMiddleware","namespace":"Flow\\Telemetry\\Logger\\Middleware","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"TELEMETRY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBhbiBBdHRyaWJ1dGVGaWx0ZXJpbmdMb2dNaWRkbGV3YXJlIHRoYXQgZHJvcHMgbG9nIGVudHJpZXMgbWF0Y2hpbmcgdGhlIGZpbHRlci4KICoKICogQHBhcmFtIEF0dHJpYnV0ZUZpbHRlciAkZmlsdGVyIFRoZSBhdHRyaWJ1dGUgZmlsdGVyIHRvIGFwcGx5CiAqLw=="},{"repository_path":"src\/lib\/telemetry\/src\/Flow\/Telemetry\/DSL\/functions.php","start_line_in_file":648,"slug":"severity-filtering-log-middleware","name":"severity_filtering_log_middleware","namespace":"Flow\\Telemetry\\DSL","parameters":[{"name":"minimumSeverity","type":[{"name":"Severity","namespace":"Flow\\Telemetry\\Logger","is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"Flow\\Telemetry\\Logger\\Severity::..."}],"return_type":[{"name":"SeverityFilteringLogMiddleware","namespace":"Flow\\Telemetry\\Logger\\Middleware","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"TELEMETRY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBhIFNldmVyaXR5RmlsdGVyaW5nTG9nTWlkZGxld2FyZSB0aGF0IGRyb3BzIGxvZyBlbnRyaWVzIGJlbG93IGEgbWluaW11bSBzZXZlcml0eS4KICoKICogQHBhcmFtIFNldmVyaXR5ICRtaW5pbXVtU2V2ZXJpdHkgTWluaW11bSBzZXZlcml0eSBsZXZlbCAoZGVmYXVsdDogSU5GTykKICov"},{"repository_path":"src\/lib\/telemetry\/src\/Flow\/Telemetry\/DSL\/functions.php","start_line_in_file":662,"slug":"attribute-rule","name":"attribute_rule","namespace":"Flow\\Telemetry\\DSL","parameters":[{"name":"path","type":[{"name":"array","namespace":null,"is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"mode","type":[{"name":"MatchMode","namespace":"Flow\\Telemetry\\Filter","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"expected","type":[{"name":"DateTimeInterface","namespace":"","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false},{"name":"int","namespace":null,"is_nullable":false,"is_variadic":false},{"name":"float","namespace":null,"is_nullable":false,"is_variadic":false},{"name":"bool","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"caseSensitive","type":[{"name":"bool","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"true"}],"return_type":[{"name":"AttributeRule","namespace":"Flow\\Telemetry\\Filter","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"TELEMETRY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBhIHNpbmdsZSBhdHRyaWJ1dGUtbWF0Y2hpbmcgcnVsZSBmb3IgYW4gQXR0cmlidXRlRmlsdGVyLgogKgogKiBAcGFyYW0gYXJyYXk8c3RyaW5nPnxzdHJpbmcgJHBhdGggYXR0cmlidXRlIHBhdGg6IGEgdG9wLWxldmVsIGtleSwgb3Igc2VnbWVudHMgZGVzY2VuZGluZyBpbnRvIG5lc3RlZCBhcnJheSB2YWx1ZXMKICogQHBhcmFtIE1hdGNoTW9kZSAkbW9kZSBjb21wYXJpc29uIGFwcGxpZWQgYmV0d2VlbiB0aGUgdmFsdWUgYXQgdGhlIHBhdGggYW5kIHRoZSBleHBlY3RlZCB2YWx1ZQogKiBAcGFyYW0gYm9vbHxEYXRlVGltZUludGVyZmFjZXxmbG9hdHxpbnR8c3RyaW5nICRleHBlY3RlZCBleHBlY3RlZCB2YWx1ZSAobXVzdCBiZSBhIHN0cmluZyBmb3IgdGhlIHBhdHRlcm4gbW9kZXMpCiAqIEBwYXJhbSBib29sICRjYXNlU2Vuc2l0aXZlIGFwcGxpZXMgdG8gdGhlIHN1YnN0cmluZyBtb2RlcyBvbmx5IChTVEFSVFNfV0lUSCwgRU5EU19XSVRILCBDT05UQUlOUykKICov"},{"repository_path":"src\/lib\/telemetry\/src\/Flow\/Telemetry\/DSL\/functions.php","start_line_in_file":675,"slug":"all","name":"all","namespace":"Flow\\Telemetry\\DSL","parameters":[{"name":"matchers","type":[{"name":"Matcher","namespace":"Flow\\Telemetry\\Filter","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":true,"default_value":null}],"return_type":[{"name":"All","namespace":"Flow\\Telemetry\\Filter","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"TELEMETRY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENvbWJpbmUgbWF0Y2hlcnMgc28gdGhhdCBldmVyeSBvbmUgbXVzdCBtYXRjaCAobG9naWNhbCBBTkQpLgogKi8="},{"repository_path":"src\/lib\/telemetry\/src\/Flow\/Telemetry\/DSL\/functions.php","start_line_in_file":684,"slug":"any","name":"any","namespace":"Flow\\Telemetry\\DSL","parameters":[{"name":"matchers","type":[{"name":"Matcher","namespace":"Flow\\Telemetry\\Filter","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":true,"default_value":null}],"return_type":[{"name":"Any","namespace":"Flow\\Telemetry\\Filter","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"TELEMETRY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENvbWJpbmUgbWF0Y2hlcnMgc28gdGhhdCBhdCBsZWFzdCBvbmUgbXVzdCBtYXRjaCAobG9naWNhbCBPUikuCiAqLw=="},{"repository_path":"src\/lib\/telemetry\/src\/Flow\/Telemetry\/DSL\/functions.php","start_line_in_file":693,"slug":"not","name":"not","namespace":"Flow\\Telemetry\\DSL","parameters":[{"name":"matcher","type":[{"name":"Matcher","namespace":"Flow\\Telemetry\\Filter","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"Not","namespace":"Flow\\Telemetry\\Filter","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"TELEMETRY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIE5lZ2F0ZSBhIG1hdGNoZXIgKGxvZ2ljYWwgTk9UKS4KICov"},{"repository_path":"src\/lib\/telemetry\/src\/Flow\/Telemetry\/DSL\/functions.php","start_line_in_file":712,"slug":"attribute-filter","name":"attribute_filter","namespace":"Flow\\Telemetry\\DSL","parameters":[{"name":"matcher","type":[{"name":"Matcher","namespace":"Flow\\Telemetry\\Filter","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"exclude","type":[{"name":"bool","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"true"},{"name":"sources","type":[{"name":"array","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"[...]"},{"name":"cacheDir","type":[{"name":"string","namespace":null,"is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"null"},{"name":"cacheDirPermissions","type":[{"name":"int","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"448"}],"return_type":[{"name":"AttributeFilter","namespace":"Flow\\Telemetry\\Filter","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"TELEMETRY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBhbiBBdHRyaWJ1dGVGaWx0ZXIgZnJvbSBhIG1hdGNoZXIuCiAqCiAqIEBwYXJhbSBNYXRjaGVyICRtYXRjaGVyIHRoZSBtYXRjaGVyIHRvIGV2YWx1YXRlIGFnYWluc3QgYSBzaWduYWwncyBhdHRyaWJ1dGVzIChjb21wb3NlIHdpdGggYWxsKCksIGFueSgpLCBub3QoKSkKICogQHBhcmFtIGJvb2wgJGV4Y2x1ZGUgd2hlbiB0cnVlIChkZWZhdWx0KSBhIG1hdGNoIGRyb3BzIHRoZSBzaWduYWw7IHdoZW4gZmFsc2Ugb25seSBtYXRjaGluZyBzaWduYWxzIGFyZSBrZXB0CiAqIEBwYXJhbSBsaXN0PEF0dHJpYnV0ZVNvdXJjZT4gJHNvdXJjZXMgd2hpY2ggYXR0cmlidXRlIHNldHMgdG8gaW5zcGVjdCAoc2lnbmFsLCByZXNvdXJjZSBhbmQvb3Igc2NvcGUpOyB0aGUgbWF0Y2hlciBpcwogKiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIE9SLWNvbWJpbmVkIGFjcm9zcyB0aGVtLCBkZWZhdWx0aW5nIHRvIHRoZSBzaWduYWwncyBvd24gYXR0cmlidXRlcwogKiBAcGFyYW0gbnVsbHxzdHJpbmcgJGNhY2hlRGlyIGRpcmVjdG9yeSBmb3IgdGhlIGdlbmVyYXRlZCBtYXRjaGVyIGZpbGUgKGRlZmF1bHRzIHRvIHRoZSBzeXN0ZW0gdGVtcCBkaXJlY3RvcnkpLiBJdCBpcwogKiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIGByZXF1aXJlYGQsIHNvIGl0IE1VU1QgYmUgdHJ1c3RlZCAtIG5vdCB3cml0YWJsZSBieSB1bnRydXN0ZWQgdXNlcnMuIFByZWZlciBhbgogKiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIGFwcGxpY2F0aW9uLXByaXZhdGUgZGlyZWN0b3J5IG92ZXIgdGhlIHNoYXJlZCBzeXN0ZW0gdGVtcCBpbiBtdWx0aS10ZW5hbnQgZW52aXJvbm1lbnRzLgogKiBAcGFyYW0gaW50ICRjYWNoZURpclBlcm1pc3Npb25zIG1vZGUgYXBwbGllZCB3aGVuIHRoZSBjYWNoZSBkaXJlY3RvcnkgaXMgY3JlYXRlZCAob2N0YWwsIHN1YmplY3QgdG8gdW1hc2s7IGRlZmF1bHRzCiAqICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgdG8gMDcwMCAtIG93bmVyIG9ubHksIHNpbmNlIHRoZSBkaXJlY3RvcnkgaG9sZHMgYHJlcXVpcmVgZCBQSFApCiAqLw=="},{"repository_path":"src\/lib\/telemetry\/src\/Flow\/Telemetry\/DSL\/functions.php","start_line_in_file":729,"slug":"attribute-filtering-metric-processor","name":"attribute_filtering_metric_processor","namespace":"Flow\\Telemetry\\DSL","parameters":[{"name":"processor","type":[{"name":"MetricProcessor","namespace":"Flow\\Telemetry\\Meter","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"filter","type":[{"name":"AttributeFilter","namespace":"Flow\\Telemetry\\Filter","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"AttributeFilteringMetricProcessor","namespace":"Flow\\Telemetry\\Meter\\Processor","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"TELEMETRY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBhbiBBdHRyaWJ1dGVGaWx0ZXJpbmdNZXRyaWNQcm9jZXNzb3IuCiAqCiAqIEBwYXJhbSBNZXRyaWNQcm9jZXNzb3IgJHByb2Nlc3NvciBUaGUgcHJvY2Vzc29yIHRvIHdyYXAKICogQHBhcmFtIEF0dHJpYnV0ZUZpbHRlciAkZmlsdGVyIFRoZSBhdHRyaWJ1dGUgZmlsdGVyIHRvIGFwcGx5CiAqLw=="},{"repository_path":"src\/lib\/telemetry\/src\/Flow\/Telemetry\/DSL\/functions.php","start_line_in_file":743,"slug":"attribute-filtering-span-processor","name":"attribute_filtering_span_processor","namespace":"Flow\\Telemetry\\DSL","parameters":[{"name":"processor","type":[{"name":"SpanProcessor","namespace":"Flow\\Telemetry\\Tracer","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"filter","type":[{"name":"AttributeFilter","namespace":"Flow\\Telemetry\\Filter","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"AttributeFilteringSpanProcessor","namespace":"Flow\\Telemetry\\Tracer\\Processor","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"TELEMETRY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBhbiBBdHRyaWJ1dGVGaWx0ZXJpbmdTcGFuUHJvY2Vzc29yLgogKgogKiBAcGFyYW0gU3BhblByb2Nlc3NvciAkcHJvY2Vzc29yIFRoZSBwcm9jZXNzb3IgdG8gd3JhcAogKiBAcGFyYW0gQXR0cmlidXRlRmlsdGVyICRmaWx0ZXIgVGhlIGF0dHJpYnV0ZSBmaWx0ZXIgdG8gYXBwbHkKICov"},{"repository_path":"src\/lib\/telemetry\/src\/Flow\/Telemetry\/DSL\/functions.php","start_line_in_file":762,"slug":"console-exporter","name":"console_exporter","namespace":"Flow\\Telemetry\\DSL","parameters":[{"name":"colors","type":[{"name":"bool","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"true"},{"name":"maxLogBodyLength","type":[{"name":"int","namespace":null,"is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"100"},{"name":"logOptions","type":[{"name":"ConsoleLogOptions","namespace":"Flow\\Telemetry\\Provider\\Console","is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"Flow\\Telemetry\\Provider\\Console\\ConsoleLogOptions::..."},{"name":"metricOptions","type":[{"name":"ConsoleMetricOptions","namespace":"Flow\\Telemetry\\Provider\\Console","is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"Flow\\Telemetry\\Provider\\Console\\ConsoleMetricOptions::..."},{"name":"spanOptions","type":[{"name":"ConsoleSpanOptions","namespace":"Flow\\Telemetry\\Provider\\Console","is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"Flow\\Telemetry\\Provider\\Console\\ConsoleSpanOptions::..."}],"return_type":[{"name":"ConsoleExporter","namespace":"Flow\\Telemetry\\Provider\\Console","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"TELEMETRY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBhIHVuaWZpZWQgQ29uc29sZUV4cG9ydGVyIGZvciBsb2dzLCBtZXRyaWNzLCBhbmQgc3BhbnMuCiAqCiAqIE91dHB1dHMgdGVsZW1ldHJ5IHRvIHRoZSBjb25zb2xlIHdpdGggQVNDSUkgdGFibGUgZm9ybWF0dGluZyBhbmQgb3B0aW9uYWwgQU5TSSBjb2xvcnMuCiAqCiAqIEBwYXJhbSBib29sICRjb2xvcnMgV2hldGhlciB0byB1c2UgQU5TSSBjb2xvcnMgKGRlZmF1bHQ6IHRydWUpCiAqIEBwYXJhbSBudWxsfGludCAkbWF4TG9nQm9keUxlbmd0aCBNYXhpbXVtIGxlbmd0aCBmb3IgbG9nIGJvZHkrYXR0cmlidXRlcyBjb2x1bW4gKG51bGwgPSBubyBsaW1pdCkKICogQHBhcmFtIENvbnNvbGVMb2dPcHRpb25zICRsb2dPcHRpb25zIERpc3BsYXkgb3B0aW9ucyBmb3IgbG9nIHJlY29yZHMKICogQHBhcmFtIENvbnNvbGVNZXRyaWNPcHRpb25zICRtZXRyaWNPcHRpb25zIERpc3BsYXkgb3B0aW9ucyBmb3IgbWV0cmljcwogKiBAcGFyYW0gQ29uc29sZVNwYW5PcHRpb25zICRzcGFuT3B0aW9ucyBEaXNwbGF5IG9wdGlvbnMgZm9yIHNwYW5zCiAqLw=="},{"repository_path":"src\/lib\/telemetry\/src\/Flow\/Telemetry\/DSL\/functions.php","start_line_in_file":776,"slug":"console-span-options","name":"console_span_options","namespace":"Flow\\Telemetry\\DSL","parameters":[],"return_type":[{"name":"ConsoleSpanOptions","namespace":"Flow\\Telemetry\\Provider\\Console","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"TELEMETRY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBDb25zb2xlU3Bhbk9wdGlvbnMgd2l0aCBhbGwgZGlzcGxheSBvcHRpb25zIGVuYWJsZWQgKGRlZmF1bHQgYmVoYXZpb3IpLgogKi8="},{"repository_path":"src\/lib\/telemetry\/src\/Flow\/Telemetry\/DSL\/functions.php","start_line_in_file":785,"slug":"console-span-options-minimal","name":"console_span_options_minimal","namespace":"Flow\\Telemetry\\DSL","parameters":[],"return_type":[{"name":"ConsoleSpanOptions","namespace":"Flow\\Telemetry\\Provider\\Console","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"TELEMETRY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBDb25zb2xlU3Bhbk9wdGlvbnMgd2l0aCBtaW5pbWFsIGRpc3BsYXkgKGxlZ2FjeSBjb21wYWN0IGZvcm1hdCkuCiAqLw=="},{"repository_path":"src\/lib\/telemetry\/src\/Flow\/Telemetry\/DSL\/functions.php","start_line_in_file":794,"slug":"console-log-options","name":"console_log_options","namespace":"Flow\\Telemetry\\DSL","parameters":[],"return_type":[{"name":"ConsoleLogOptions","namespace":"Flow\\Telemetry\\Provider\\Console","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"TELEMETRY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBDb25zb2xlTG9nT3B0aW9ucyB3aXRoIGFsbCBkaXNwbGF5IG9wdGlvbnMgZW5hYmxlZCAoZGVmYXVsdCBiZWhhdmlvcikuCiAqLw=="},{"repository_path":"src\/lib\/telemetry\/src\/Flow\/Telemetry\/DSL\/functions.php","start_line_in_file":803,"slug":"console-log-options-minimal","name":"console_log_options_minimal","namespace":"Flow\\Telemetry\\DSL","parameters":[],"return_type":[{"name":"ConsoleLogOptions","namespace":"Flow\\Telemetry\\Provider\\Console","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"TELEMETRY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBDb25zb2xlTG9nT3B0aW9ucyB3aXRoIG1pbmltYWwgZGlzcGxheSAobGVnYWN5IGNvbXBhY3QgZm9ybWF0KS4KICov"},{"repository_path":"src\/lib\/telemetry\/src\/Flow\/Telemetry\/DSL\/functions.php","start_line_in_file":812,"slug":"console-metric-options","name":"console_metric_options","namespace":"Flow\\Telemetry\\DSL","parameters":[],"return_type":[{"name":"ConsoleMetricOptions","namespace":"Flow\\Telemetry\\Provider\\Console","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"TELEMETRY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBDb25zb2xlTWV0cmljT3B0aW9ucyB3aXRoIGFsbCBkaXNwbGF5IG9wdGlvbnMgZW5hYmxlZCAoZGVmYXVsdCBiZWhhdmlvcikuCiAqLw=="},{"repository_path":"src\/lib\/telemetry\/src\/Flow\/Telemetry\/DSL\/functions.php","start_line_in_file":821,"slug":"console-metric-options-minimal","name":"console_metric_options_minimal","namespace":"Flow\\Telemetry\\DSL","parameters":[],"return_type":[{"name":"ConsoleMetricOptions","namespace":"Flow\\Telemetry\\Provider\\Console","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"TELEMETRY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBDb25zb2xlTWV0cmljT3B0aW9ucyB3aXRoIG1pbmltYWwgZGlzcGxheSAobGVnYWN5IGNvbXBhY3QgZm9ybWF0KS4KICov"},{"repository_path":"src\/lib\/telemetry\/src\/Flow\/Telemetry\/DSL\/functions.php","start_line_in_file":830,"slug":"always-on-exemplar-filter","name":"always_on_exemplar_filter","namespace":"Flow\\Telemetry\\DSL","parameters":[],"return_type":[{"name":"AlwaysOnExemplarFilter","namespace":"Flow\\Telemetry\\Meter\\Exemplar","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"TELEMETRY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBhbiBBbHdheXNPbkV4ZW1wbGFyRmlsdGVyLgogKi8="},{"repository_path":"src\/lib\/telemetry\/src\/Flow\/Telemetry\/DSL\/functions.php","start_line_in_file":839,"slug":"always-off-exemplar-filter","name":"always_off_exemplar_filter","namespace":"Flow\\Telemetry\\DSL","parameters":[],"return_type":[{"name":"AlwaysOffExemplarFilter","namespace":"Flow\\Telemetry\\Meter\\Exemplar","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"TELEMETRY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBhbiBBbHdheXNPZmZFeGVtcGxhckZpbHRlci4KICov"},{"repository_path":"src\/lib\/telemetry\/src\/Flow\/Telemetry\/DSL\/functions.php","start_line_in_file":848,"slug":"trace-based-exemplar-filter","name":"trace_based_exemplar_filter","namespace":"Flow\\Telemetry\\DSL","parameters":[],"return_type":[{"name":"TraceBasedExemplarFilter","namespace":"Flow\\Telemetry\\Meter\\Exemplar","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"TELEMETRY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBhIFRyYWNlQmFzZWRFeGVtcGxhckZpbHRlci4KICov"},{"repository_path":"src\/lib\/telemetry\/src\/Flow\/Telemetry\/DSL\/functions.php","start_line_in_file":857,"slug":"always-on-sampler","name":"always_on_sampler","namespace":"Flow\\Telemetry\\DSL","parameters":[],"return_type":[{"name":"AlwaysOnSampler","namespace":"Flow\\Telemetry\\Tracer\\Sampler","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"TELEMETRY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBhbiBBbHdheXNPblNhbXBsZXIuIFJlY29yZHMgYW5kIHNhbXBsZXMgZXZlcnkgc3Bhbi4KICov"},{"repository_path":"src\/lib\/telemetry\/src\/Flow\/Telemetry\/DSL\/functions.php","start_line_in_file":866,"slug":"always-off-sampler","name":"always_off_sampler","namespace":"Flow\\Telemetry\\DSL","parameters":[],"return_type":[{"name":"AlwaysOffSampler","namespace":"Flow\\Telemetry\\Tracer\\Sampler","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"TELEMETRY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBhbiBBbHdheXNPZmZTYW1wbGVyLiBEcm9wcyBldmVyeSBzcGFuLgogKi8="},{"repository_path":"src\/lib\/telemetry\/src\/Flow\/Telemetry\/DSL\/functions.php","start_line_in_file":877,"slug":"trace-id-ratio-based-sampler","name":"trace_id_ratio_based_sampler","namespace":"Flow\\Telemetry\\DSL","parameters":[{"name":"ratio","type":[{"name":"float","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"TraceIdRatioBasedSampler","namespace":"Flow\\Telemetry\\Tracer\\Sampler","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"TELEMETRY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBhIFRyYWNlSWRSYXRpb0Jhc2VkU2FtcGxlci4gU2FtcGxlcyBhIGRldGVybWluaXN0aWMgZnJhY3Rpb24gb2YgdHJhY2VzLgogKgogKiBAcGFyYW0gZmxvYXQgJHJhdGlvIFNhbXBsaW5nIHByb2JhYmlsaXR5IGJldHdlZW4gMC4wIGFuZCAxLjAKICov"},{"repository_path":"src\/lib\/telemetry\/src\/Flow\/Telemetry\/DSL\/functions.php","start_line_in_file":889,"slug":"parent-based-sampler","name":"parent_based_sampler","namespace":"Flow\\Telemetry\\DSL","parameters":[{"name":"root","type":[{"name":"Sampler","namespace":"Flow\\Telemetry\\Tracer\\Sampler","is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"Flow\\Telemetry\\Tracer\\Sampler\\AlwaysOnSampler::..."}],"return_type":[{"name":"ParentBasedSampler","namespace":"Flow\\Telemetry\\Tracer\\Sampler","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"TELEMETRY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBhIFBhcmVudEJhc2VkU2FtcGxlci4gSG9ub3JzIHRoZSBwYXJlbnQgc3BhbidzIHNhbXBsaW5nIGRlY2lzaW9uLCBmYWxsaW5nCiAqIGJhY2sgdG8gdGhlIHJvb3Qgc2FtcGxlciBmb3Igc3BhbnMgd2l0aG91dCBhIHBhcmVudC4KICoKICogQHBhcmFtIFNhbXBsZXIgJHJvb3QgU2FtcGxlciB1c2VkIGZvciByb290IHNwYW5zIChubyBwYXJlbnQpCiAqLw=="},{"repository_path":"src\/lib\/telemetry\/src\/Flow\/Telemetry\/DSL\/functions.php","start_line_in_file":905,"slug":"attribute-matching-sampler","name":"attribute_matching_sampler","namespace":"Flow\\Telemetry\\DSL","parameters":[{"name":"filter","type":[{"name":"AttributeFilter","namespace":"Flow\\Telemetry\\Filter","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"delegate","type":[{"name":"Sampler","namespace":"Flow\\Telemetry\\Tracer\\Sampler","is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"Flow\\Telemetry\\Tracer\\Sampler\\AlwaysOnSampler::..."}],"return_type":[{"name":"AttributeMatchingSampler","namespace":"Flow\\Telemetry\\Tracer\\Sampler","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"TELEMETRY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBhbiBBdHRyaWJ1dGVNYXRjaGluZ1NhbXBsZXIuIERyb3BzIHNwYW5zIHdob3NlIHN0YXJ0LXRpbWUgYXR0cmlidXRlcyBtYXRjaAogKiB0aGUgZmlsdGVyIChvciBrZWVwcyBPTkxZIG1hdGNoaW5nIHNwYW5zIHdoZW4gdGhlIGZpbHRlcidzIGV4Y2x1ZGUgaXMgZmFsc2UpLCBhbmQKICogZGVmZXJzIGFsbCBvdGhlciBzcGFucyB0byB0aGUgZGVsZWdhdGUgc2FtcGxlci4KICoKICogT25seSBhdHRyaWJ1dGVzIGF2YWlsYWJsZSBhdCBzcGFuIHN0YXJ0IGFyZSB2aXNpYmxlOyBhdHRyaWJ1dGVzIGFkZGVkIGxhdGVyIGFyZSBub3QuCiAqCiAqIEBwYXJhbSBBdHRyaWJ1dGVGaWx0ZXIgJGZpbHRlciBUaGUgYXR0cmlidXRlIGZpbHRlciBldmFsdWF0ZWQgYWdhaW5zdCB0aGUgc3BhbidzIHN0YXJ0IGF0dHJpYnV0ZXMKICogQHBhcmFtIFNhbXBsZXIgJGRlbGVnYXRlIFNhbXBsZXIgdGhhdCBkZWNpZGVzIHNwYW5zIHdoaWNoIGRvIG5vdCBtYXRjaCAoZGVmYXVsdDogQWx3YXlzT25TYW1wbGVyKQogKi8="},{"repository_path":"src\/lib\/telemetry\/src\/Flow\/Telemetry\/DSL\/functions.php","start_line_in_file":919,"slug":"propagation-context","name":"propagation_context","namespace":"Flow\\Telemetry\\DSL","parameters":[{"name":"spanContext","type":[{"name":"SpanContext","namespace":"Flow\\Telemetry\\Tracer","is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"null"},{"name":"baggage","type":[{"name":"Baggage","namespace":"Flow\\Telemetry\\Context","is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"null"}],"return_type":[{"name":"PropagationContext","namespace":"Flow\\Telemetry\\Propagation","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"TELEMETRY","type":"TYPE"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBhIFByb3BhZ2F0aW9uQ29udGV4dC4KICoKICogQHBhcmFtIG51bGx8U3BhbkNvbnRleHQgJHNwYW5Db250ZXh0IE9wdGlvbmFsIHNwYW4gY29udGV4dAogKiBAcGFyYW0gbnVsbHxCYWdnYWdlICRiYWdnYWdlIE9wdGlvbmFsIGJhZ2dhZ2UKICov"},{"repository_path":"src\/lib\/telemetry\/src\/Flow\/Telemetry\/DSL\/functions.php","start_line_in_file":930,"slug":"array-carrier","name":"array_carrier","namespace":"Flow\\Telemetry\\DSL","parameters":[{"name":"data","type":[{"name":"array","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"[]"}],"return_type":[{"name":"ArrayCarrier","namespace":"Flow\\Telemetry\\Propagation","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"TELEMETRY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBhbiBBcnJheUNhcnJpZXIuCiAqCiAqIEBwYXJhbSBhcnJheTxzdHJpbmcsIHN0cmluZz4gJGRhdGEgSW5pdGlhbCBjYXJyaWVyIGRhdGEKICov"},{"repository_path":"src\/lib\/telemetry\/src\/Flow\/Telemetry\/DSL\/functions.php","start_line_in_file":939,"slug":"superglobal-carrier","name":"superglobal_carrier","namespace":"Flow\\Telemetry\\DSL","parameters":[],"return_type":[{"name":"SuperglobalCarrier","namespace":"Flow\\Telemetry\\Propagation","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"TELEMETRY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBhIFN1cGVyZ2xvYmFsQ2Fycmllci4KICov"},{"repository_path":"src\/lib\/telemetry\/src\/Flow\/Telemetry\/DSL\/functions.php","start_line_in_file":948,"slug":"w3c-trace-context","name":"w3c_trace_context","namespace":"Flow\\Telemetry\\DSL","parameters":[],"return_type":[{"name":"W3CTraceContext","namespace":"Flow\\Telemetry\\Propagation","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"TELEMETRY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBhIFczQ1RyYWNlQ29udGV4dCBwcm9wYWdhdG9yLgogKi8="},{"repository_path":"src\/lib\/telemetry\/src\/Flow\/Telemetry\/DSL\/functions.php","start_line_in_file":957,"slug":"w3c-baggage","name":"w3c_baggage","namespace":"Flow\\Telemetry\\DSL","parameters":[],"return_type":[{"name":"W3CBaggage","namespace":"Flow\\Telemetry\\Propagation","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"TELEMETRY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBhIFczQ0JhZ2dhZ2UgcHJvcGFnYXRvci4KICov"},{"repository_path":"src\/lib\/telemetry\/src\/Flow\/Telemetry\/DSL\/functions.php","start_line_in_file":968,"slug":"composite-propagator","name":"composite_propagator","namespace":"Flow\\Telemetry\\DSL","parameters":[{"name":"propagators","type":[{"name":"Propagator","namespace":"Flow\\Telemetry\\Propagation","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":true,"default_value":null}],"return_type":[{"name":"CompositePropagator","namespace":"Flow\\Telemetry\\Propagation","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"TELEMETRY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBhIENvbXBvc2l0ZVByb3BhZ2F0b3IuCiAqCiAqIEBwYXJhbSBQcm9wYWdhdG9yIC4uLiRwcm9wYWdhdG9ycyBUaGUgcHJvcGFnYXRvcnMgdG8gY29tYmluZQogKi8="},{"repository_path":"src\/lib\/telemetry\/src\/Flow\/Telemetry\/DSL\/functions.php","start_line_in_file":979,"slug":"chain-detector","name":"chain_detector","namespace":"Flow\\Telemetry\\DSL","parameters":[{"name":"detectors","type":[{"name":"ResourceDetector","namespace":"Flow\\Telemetry\\Resource","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":true,"default_value":null}],"return_type":[{"name":"ChainDetector","namespace":"Flow\\Telemetry\\Resource\\Detector","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"TELEMETRY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBhIENoYWluRGV0ZWN0b3IuCiAqCiAqIEBwYXJhbSBSZXNvdXJjZURldGVjdG9yIC4uLiRkZXRlY3RvcnMgVGhlIGRldGVjdG9ycyB0byBjaGFpbgogKi8="},{"repository_path":"src\/lib\/telemetry\/src\/Flow\/Telemetry\/DSL\/functions.php","start_line_in_file":988,"slug":"os-detector","name":"os_detector","namespace":"Flow\\Telemetry\\DSL","parameters":[],"return_type":[{"name":"OsDetector","namespace":"Flow\\Telemetry\\Resource\\Detector","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"TELEMETRY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBhbiBPc0RldGVjdG9yLgogKi8="},{"repository_path":"src\/lib\/telemetry\/src\/Flow\/Telemetry\/DSL\/functions.php","start_line_in_file":997,"slug":"host-detector","name":"host_detector","namespace":"Flow\\Telemetry\\DSL","parameters":[],"return_type":[{"name":"HostDetector","namespace":"Flow\\Telemetry\\Resource\\Detector","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"TELEMETRY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBhIEhvc3REZXRlY3Rvci4KICov"},{"repository_path":"src\/lib\/telemetry\/src\/Flow\/Telemetry\/DSL\/functions.php","start_line_in_file":1006,"slug":"process-detector","name":"process_detector","namespace":"Flow\\Telemetry\\DSL","parameters":[],"return_type":[{"name":"ProcessDetector","namespace":"Flow\\Telemetry\\Resource\\Detector","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"TELEMETRY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBhIFByb2Nlc3NEZXRlY3Rvci4KICov"},{"repository_path":"src\/lib\/telemetry\/src\/Flow\/Telemetry\/DSL\/functions.php","start_line_in_file":1015,"slug":"environment-detector","name":"environment_detector","namespace":"Flow\\Telemetry\\DSL","parameters":[],"return_type":[{"name":"EnvironmentDetector","namespace":"Flow\\Telemetry\\Resource\\Detector","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"TELEMETRY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBhbiBFbnZpcm9ubWVudERldGVjdG9yLgogKi8="},{"repository_path":"src\/lib\/telemetry\/src\/Flow\/Telemetry\/DSL\/functions.php","start_line_in_file":1024,"slug":"composer-detector","name":"composer_detector","namespace":"Flow\\Telemetry\\DSL","parameters":[],"return_type":[{"name":"ComposerDetector","namespace":"Flow\\Telemetry\\Resource\\Detector","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"TELEMETRY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBhIENvbXBvc2VyRGV0ZWN0b3IuCiAqLw=="},{"repository_path":"src\/lib\/telemetry\/src\/Flow\/Telemetry\/DSL\/functions.php","start_line_in_file":1036,"slug":"git-detector","name":"git_detector","namespace":"Flow\\Telemetry\\DSL","parameters":[{"name":"workingDirectory","type":[{"name":"string","namespace":null,"is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"null"},{"name":"gitBinary","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"'git'"}],"return_type":[{"name":"GitDetector","namespace":"Flow\\Telemetry\\Resource\\Detector","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"TELEMETRY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBhIEdpdERldGVjdG9yLgogKgogKiBAcGFyYW0gbnVsbHxzdHJpbmcgJHdvcmtpbmdEaXJlY3RvcnkgRGlyZWN0b3J5IHRvIHJ1biBnaXQgaW4gKGRlZmF1bHQ6IGN1cnJlbnQgd29ya2luZyBkaXJlY3RvcnkpCiAqIEBwYXJhbSBzdHJpbmcgJGdpdEJpbmFyeSBQYXRoIHRvIHRoZSBnaXQgYmluYXJ5IChkZWZhdWx0OiAiZ2l0IiwgcmVzb2x2ZWQgZnJvbSAkUEFUSCkKICov"},{"repository_path":"src\/lib\/telemetry\/src\/Flow\/Telemetry\/DSL\/functions.php","start_line_in_file":1047,"slug":"manual-detector","name":"manual_detector","namespace":"Flow\\Telemetry\\DSL","parameters":[{"name":"attributes","type":[{"name":"array","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"ManualDetector","namespace":"Flow\\Telemetry\\Resource\\Detector","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"TELEMETRY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBhIE1hbnVhbERldGVjdG9yLgogKgogKiBAcGFyYW0gYXJyYXk8c3RyaW5nLCBhcnJheTxhcnJheS1rZXksIG1peGVkPnxib29sfFxEYXRlVGltZUludGVyZmFjZXxmbG9hdHxpbnR8c3RyaW5nfFxUaHJvd2FibGU+ICRhdHRyaWJ1dGVzIFJlc291cmNlIGF0dHJpYnV0ZXMKICov"},{"repository_path":"src\/lib\/telemetry\/src\/Flow\/Telemetry\/DSL\/functions.php","start_line_in_file":1059,"slug":"caching-detector","name":"caching_detector","namespace":"Flow\\Telemetry\\DSL","parameters":[{"name":"detector","type":[{"name":"ResourceDetector","namespace":"Flow\\Telemetry\\Resource","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"cachePath","type":[{"name":"string","namespace":null,"is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"null"}],"return_type":[{"name":"CachingDetector","namespace":"Flow\\Telemetry\\Resource\\Detector","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"TELEMETRY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBhIENhY2hpbmdEZXRlY3Rvci4KICoKICogQHBhcmFtIFJlc291cmNlRGV0ZWN0b3IgJGRldGVjdG9yIFRoZSBkZXRlY3RvciB0byB3cmFwCiAqIEBwYXJhbSBudWxsfHN0cmluZyAkY2FjaGVQYXRoIENhY2hlIGZpbGUgcGF0aCAoZGVmYXVsdDogc3lzX2dldF90ZW1wX2RpcigpL2Zsb3dfdGVsZW1ldHJ5X3Jlc291cmNlLmNhY2hlKQogKi8="},{"repository_path":"src\/lib\/telemetry\/src\/Flow\/Telemetry\/DSL\/functions.php","start_line_in_file":1070,"slug":"resource-detector","name":"resource_detector","namespace":"Flow\\Telemetry\\DSL","parameters":[{"name":"detectors","type":[{"name":"array","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"[]"}],"return_type":[{"name":"ChainDetector","namespace":"Flow\\Telemetry\\Resource\\Detector","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"TELEMETRY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBhIHJlc291cmNlIGRldGVjdG9yIGNoYWluLgogKgogKiBAcGFyYW0gYXJyYXk8UmVzb3VyY2VEZXRlY3Rvcj4gJGRldGVjdG9ycyBPcHRpb25hbCBjdXN0b20gZGV0ZWN0b3JzIChlbXB0eSA9IHVzZSBkZWZhdWx0cykKICov"},{"repository_path":"src\/lib\/telemetry\/src\/Flow\/Telemetry\/DSL\/functions.php","start_line_in_file":1089,"slug":"error-log-handler","name":"error_log_handler","namespace":"Flow\\Telemetry\\DSL","parameters":[{"name":"messageType","type":[{"name":"ErrorLogMessageType","namespace":"Flow\\Telemetry\\ErrorHandler","is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"Flow\\Telemetry\\ErrorHandler\\ErrorLogMessageType::..."},{"name":"expandNewlines","type":[{"name":"bool","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"false"},{"name":"messagePrefix","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"'[flow-telemetry]'"}],"return_type":[{"name":"ErrorLogHandler","namespace":"Flow\\Telemetry\\ErrorHandler","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"TELEMETRY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSB0aGUgZGVmYXVsdCBFcnJvckxvZ0hhbmRsZXIuIFdyaXRlcyB2aWEgUEhQJ3MgZXJyb3JfbG9nKCkuCiAqLw=="},{"repository_path":"src\/lib\/telemetry\/src\/Flow\/Telemetry\/DSL\/functions.php","start_line_in_file":1102,"slug":"stream-error-handler","name":"stream_error_handler","namespace":"Flow\\Telemetry\\DSL","parameters":[{"name":"destination","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"filePermissions","type":[{"name":"int","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"420"},{"name":"createDirectories","type":[{"name":"bool","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"true"},{"name":"messagePrefix","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"'[flow-telemetry]'"}],"return_type":[{"name":"StreamHandler","namespace":"Flow\\Telemetry\\ErrorHandler","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"TELEMETRY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBhIFN0cmVhbUhhbmRsZXIuIEFwcGVuZHMgZm9ybWF0dGVkIFRocm93YWJsZXMgKG9uZSBwZXIgbGluZSkgdG8gYSBmaWxlCiAqIHBhdGggb3IgcGhwOi8vIHN0cmVhbSB3cmFwcGVyLgogKi8="},{"repository_path":"src\/lib\/telemetry\/src\/Flow\/Telemetry\/DSL\/functions.php","start_line_in_file":1115,"slug":"syslog-error-handler","name":"syslog_error_handler","namespace":"Flow\\Telemetry\\DSL","parameters":[{"name":"ident","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"'flow-telemetry'"},{"name":"facility","type":[{"name":"SyslogFacility","namespace":"Flow\\Telemetry\\ErrorHandler","is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"Flow\\Telemetry\\ErrorHandler\\SyslogFacility::..."},{"name":"logOpts","type":[{"name":"int","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"1"},{"name":"severity","type":[{"name":"SyslogSeverity","namespace":"Flow\\Telemetry\\ErrorHandler","is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"Flow\\Telemetry\\ErrorHandler\\SyslogSeverity::..."}],"return_type":[{"name":"SyslogHandler","namespace":"Flow\\Telemetry\\ErrorHandler","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"TELEMETRY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBhIFN5c2xvZ0hhbmRsZXIuIFdyaXRlcyB2aWEgb3BlbmxvZy9zeXNsb2cvY2xvc2Vsb2cuCiAqLw=="},{"repository_path":"src\/lib\/telemetry\/src\/Flow\/Telemetry\/DSL\/functions.php","start_line_in_file":1128,"slug":"udp-syslog-error-handler","name":"udp_syslog_error_handler","namespace":"Flow\\Telemetry\\DSL","parameters":[{"name":"host","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"port","type":[{"name":"int","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"514"},{"name":"ident","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"'flow-telemetry'"},{"name":"facility","type":[{"name":"SyslogFacility","namespace":"Flow\\Telemetry\\ErrorHandler","is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"Flow\\Telemetry\\ErrorHandler\\SyslogFacility::..."},{"name":"severity","type":[{"name":"SyslogSeverity","namespace":"Flow\\Telemetry\\ErrorHandler","is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"Flow\\Telemetry\\ErrorHandler\\SyslogSeverity::..."}],"return_type":[{"name":"UdpSyslogHandler","namespace":"Flow\\Telemetry\\ErrorHandler","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"TELEMETRY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBhIFVkcFN5c2xvZ0hhbmRsZXIuIFNlbmRzIFJGQyA1NDI0LXN0eWxlIHN5c2xvZyBmcmFtZXMgb3ZlciBVRFAuCiAqLw=="},{"repository_path":"src\/lib\/telemetry\/src\/Flow\/Telemetry\/DSL\/functions.php","start_line_in_file":1142,"slug":"composite-error-handler","name":"composite_error_handler","namespace":"Flow\\Telemetry\\DSL","parameters":[{"name":"handlers","type":[{"name":"ErrorHandler","namespace":"Flow\\Telemetry\\ErrorHandler","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":true,"default_value":null}],"return_type":[{"name":"CompositeErrorHandler","namespace":"Flow\\Telemetry\\ErrorHandler","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"TELEMETRY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIEZhbiBlcnJvcnMgb3V0IHRvIG11bHRpcGxlIGhhbmRsZXJzLgogKi8="},{"repository_path":"src\/lib\/telemetry\/src\/Flow\/Telemetry\/DSL\/functions.php","start_line_in_file":1151,"slug":"null-error-handler","name":"null_error_handler","namespace":"Flow\\Telemetry\\DSL","parameters":[],"return_type":[{"name":"NullErrorHandler","namespace":"Flow\\Telemetry\\ErrorHandler","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"TELEMETRY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIERpc2NhcmQgZXZlcnkgZXJyb3IuIFVzZSBvbmx5IGluIHRlc3RzIG9yIGZvciBleHBsaWNpdCBzaWxlbmNlLgogKi8="},{"repository_path":"src\/lib\/azure-sdk\/src\/Flow\/Azure\/SDK\/DSL\/functions.php","start_line_in_file":29,"slug":"azurite-url-factory","name":"azurite_url_factory","namespace":"Flow\\Azure\\SDK\\DSL","parameters":[{"name":"host","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"'localhost'"},{"name":"port","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"'10000'"},{"name":"secure","type":[{"name":"bool","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"false"}],"return_type":[{"name":"AzuriteURLFactory","namespace":"Flow\\Azure\\SDK\\BlobService\\URLFactory","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"AZURE_SDK","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/lib\/azure-sdk\/src\/Flow\/Azure\/SDK\/DSL\/functions.php","start_line_in_file":38,"slug":"azure-shared-key-authorization-factory","name":"azure_shared_key_authorization_factory","namespace":"Flow\\Azure\\SDK\\DSL","parameters":[{"name":"account","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"key","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"SharedKeyFactory","namespace":"Flow\\Azure\\SDK\\AuthorizationFactory","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"AZURE_SDK","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/lib\/azure-sdk\/src\/Flow\/Azure\/SDK\/DSL\/functions.php","start_line_in_file":48,"slug":"azure-blob-service-config","name":"azure_blob_service_config","namespace":"Flow\\Azure\\SDK\\DSL","parameters":[{"name":"account","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"container","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"Configuration","namespace":"Flow\\Azure\\SDK\\BlobService","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"AZURE_SDK","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/lib\/azure-sdk\/src\/Flow\/Azure\/SDK\/DSL\/functions.php","start_line_in_file":54,"slug":"azure-url-factory","name":"azure_url_factory","namespace":"Flow\\Azure\\SDK\\DSL","parameters":[{"name":"host","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"'blob.core.windows.net'"}],"return_type":[{"name":"AzureURLFactory","namespace":"Flow\\Azure\\SDK\\BlobService\\URLFactory","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"AZURE_SDK","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/lib\/azure-sdk\/src\/Flow\/Azure\/SDK\/DSL\/functions.php","start_line_in_file":60,"slug":"azure-http-factory","name":"azure_http_factory","namespace":"Flow\\Azure\\SDK\\DSL","parameters":[{"name":"request_factory","type":[{"name":"RequestFactoryInterface","namespace":"Psr\\Http\\Message","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"stream_factory","type":[{"name":"StreamFactoryInterface","namespace":"Psr\\Http\\Message","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"HttpFactory","namespace":"Flow\\Azure\\SDK","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"AZURE_SDK","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/lib\/azure-sdk\/src\/Flow\/Azure\/SDK\/DSL\/functions.php","start_line_in_file":68,"slug":"azure-blob-service","name":"azure_blob_service","namespace":"Flow\\Azure\\SDK\\DSL","parameters":[{"name":"configuration","type":[{"name":"Configuration","namespace":"Flow\\Azure\\SDK\\BlobService","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"azure_authorization_factory","type":[{"name":"AuthorizationFactory","namespace":"Flow\\Azure\\SDK","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"client","type":[{"name":"ClientInterface","namespace":"Psr\\Http\\Client","is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"null"},{"name":"azure_http_factory","type":[{"name":"HttpFactory","namespace":"Flow\\Azure\\SDK","is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"null"},{"name":"azure_url_factory","type":[{"name":"URLFactory","namespace":"Flow\\Azure\\SDK","is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"null"},{"name":"logger","type":[{"name":"LoggerInterface","namespace":"Psr\\Log","is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"null"}],"return_type":[{"name":"BlobServiceInterface","namespace":"Flow\\Azure\\SDK","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"AZURE_SDK","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/bridge\/filesystem\/azure\/src\/Flow\/Filesystem\/Bridge\/Azure\/DSL\/functions.php","start_line_in_file":16,"slug":"azure-filesystem-options","name":"azure_filesystem_options","namespace":"Flow\\Filesystem\\Bridge\\Azure\\DSL","parameters":[],"return_type":[{"name":"Options","namespace":"Flow\\Filesystem\\Bridge\\Azure","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"AZURE_FILESYSTEM","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/bridge\/filesystem\/azure\/src\/Flow\/Filesystem\/Bridge\/Azure\/DSL\/functions.php","start_line_in_file":22,"slug":"azure-filesystem","name":"azure_filesystem","namespace":"Flow\\Filesystem\\Bridge\\Azure\\DSL","parameters":[{"name":"blob_service","type":[{"name":"BlobServiceInterface","namespace":"Flow\\Azure\\SDK","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"options","type":[{"name":"Options","namespace":"Flow\\Filesystem\\Bridge\\Azure","is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"Flow\\Filesystem\\Bridge\\Azure\\Options::..."},{"name":"protocol","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"'azure-blob'"}],"return_type":[{"name":"AzureBlobFilesystem","namespace":"Flow\\Filesystem\\Bridge\\Azure","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"AZURE_FILESYSTEM","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/bridge\/filesystem\/async-aws\/src\/Flow\/Filesystem\/Bridge\/AsyncAWS\/DSL\/functions.php","start_line_in_file":20,"slug":"aws-s3-client","name":"aws_s3_client","namespace":"Flow\\Filesystem\\Bridge\\AsyncAWS\\DSL","parameters":[{"name":"configuration","type":[{"name":"array","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"S3Client","namespace":"AsyncAws\\S3","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"S3_FILESYSTEM","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIEBwYXJhbSBhcnJheTxDb25maWd1cmF0aW9uOjpPUFRJT05fKiwgbnVsbHxzdHJpbmc+ICRjb25maWd1cmF0aW9uIC0gZm9yIGRldGFpbHMgcGxlYXNlIHNlZSBodHRwczovL2FzeW5jLWF3cy5jb20vY2xpZW50cy9zMy5odG1sCiAqLw=="},{"repository_path":"src\/bridge\/filesystem\/async-aws\/src\/Flow\/Filesystem\/Bridge\/AsyncAWS\/DSL\/functions.php","start_line_in_file":26,"slug":"aws-s3-filesystem","name":"aws_s3_filesystem","namespace":"Flow\\Filesystem\\Bridge\\AsyncAWS\\DSL","parameters":[{"name":"bucket","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"s3Client","type":[{"name":"S3Client","namespace":"AsyncAws\\S3","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"options","type":[{"name":"Options","namespace":"Flow\\Filesystem\\Bridge\\AsyncAWS","is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"Flow\\Filesystem\\Bridge\\AsyncAWS\\Options::..."},{"name":"protocol","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"'aws-s3'"}],"return_type":[{"name":"AsyncAWSS3Filesystem","namespace":"Flow\\Filesystem\\Bridge\\AsyncAWS","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"S3_FILESYSTEM","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/bridge\/filesystem\/sftp\/src\/Flow\/Filesystem\/Bridge\/SFTP\/DSL\/functions.php","start_line_in_file":24,"slug":"sftp-client","name":"sftp_client","namespace":"Flow\\Filesystem\\Bridge\\SFTP\\DSL","parameters":[{"name":"host","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"user","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"credential","type":[{"name":"PrivateKey","namespace":"phpseclib4\\Crypt\\Common","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"port","type":[{"name":"int","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"22"}],"return_type":[{"name":"SFTP","namespace":"phpseclib4\\Net","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"SFTP_FILESYSTEM","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIEB0aHJvd3MgUnVudGltZUV4Y2VwdGlvbgogKi8="},{"repository_path":"src\/bridge\/filesystem\/sftp\/src\/Flow\/Filesystem\/Bridge\/SFTP\/DSL\/functions.php","start_line_in_file":46,"slug":"sftp-filesystem","name":"sftp_filesystem","namespace":"Flow\\Filesystem\\Bridge\\SFTP\\DSL","parameters":[{"name":"sftp","type":[{"name":"SFTP","namespace":"phpseclib4\\Net","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"options","type":[{"name":"Options","namespace":"Flow\\Filesystem\\Bridge\\SFTP","is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"Flow\\Filesystem\\Bridge\\SFTP\\Options::..."},{"name":"protocol","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"'sftp'"}],"return_type":[{"name":"SFTPFilesystem","namespace":"Flow\\Filesystem\\Bridge\\SFTP","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"SFTP_FILESYSTEM","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/bridge\/filesystem\/sftp\/src\/Flow\/Filesystem\/Bridge\/SFTP\/DSL\/functions.php","start_line_in_file":52,"slug":"sftp-filesystem-options","name":"sftp_filesystem_options","namespace":"Flow\\Filesystem\\Bridge\\SFTP\\DSL","parameters":[],"return_type":[{"name":"Options","namespace":"Flow\\Filesystem\\Bridge\\SFTP","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"SFTP_FILESYSTEM","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/bridge\/monolog\/telemetry\/src\/Flow\/Bridge\/Monolog\/Telemetry\/DSL\/functions.php","start_line_in_file":40,"slug":"value-normalizer","name":"value_normalizer","namespace":"Flow\\Bridge\\Monolog\\Telemetry\\DSL","parameters":[],"return_type":[{"name":"ValueNormalizer","namespace":"Flow\\Bridge\\Monolog\\Telemetry","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"MONOLOG_TELEMETRY_BRIDGE","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBhIFZhbHVlTm9ybWFsaXplciBmb3IgY29udmVydGluZyBhcmJpdHJhcnkgUEhQIHZhbHVlcyB0byBUZWxlbWV0cnkgYXR0cmlidXRlIHR5cGVzLgogKgogKiBUaGUgbm9ybWFsaXplciBoYW5kbGVzOgogKiAtIG51bGwg4oaSICdudWxsJyBzdHJpbmcKICogLSBzY2FsYXJzIChzdHJpbmcsIGludCwgZmxvYXQsIGJvb2wpIOKGkiB1bmNoYW5nZWQKICogLSBEYXRlVGltZUludGVyZmFjZSDihpIgdW5jaGFuZ2VkCiAqIC0gVGhyb3dhYmxlIOKGkiB1bmNoYW5nZWQKICogLSBhcnJheXMg4oaSIHJlY3Vyc2l2ZWx5IG5vcm1hbGl6ZWQKICogLSBvYmplY3RzIHdpdGggX190b1N0cmluZygpIOKGkiBzdHJpbmcgY2FzdAogKiAtIG9iamVjdHMgd2l0aG91dCBfX3RvU3RyaW5nKCkg4oaSIGNsYXNzIG5hbWUKICogLSBvdGhlciB0eXBlcyDihpIgZ2V0X2RlYnVnX3R5cGUoKSByZXN1bHQKICoKICogRXhhbXBsZSB1c2FnZToKICogYGBgcGhwCiAqICRub3JtYWxpemVyID0gdmFsdWVfbm9ybWFsaXplcigpOwogKiAkbm9ybWFsaXplZCA9ICRub3JtYWxpemVyLT5ub3JtYWxpemUoJHZhbHVlKTsKICogYGBgCiAqLw=="},{"repository_path":"src\/bridge\/monolog\/telemetry\/src\/Flow\/Bridge\/Monolog\/Telemetry\/DSL\/functions.php","start_line_in_file":73,"slug":"severity-mapper","name":"severity_mapper","namespace":"Flow\\Bridge\\Monolog\\Telemetry\\DSL","parameters":[{"name":"customMapping","type":[{"name":"array","namespace":null,"is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"null"}],"return_type":[{"name":"SeverityMapper","namespace":"Flow\\Bridge\\Monolog\\Telemetry","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"MONOLOG_TELEMETRY_BRIDGE","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBhIFNldmVyaXR5TWFwcGVyIGZvciBtYXBwaW5nIE1vbm9sb2cgbGV2ZWxzIHRvIFRlbGVtZXRyeSBzZXZlcml0aWVzLgogKgogKiBAcGFyYW0gbnVsbHxhcnJheTxpbnQsIFNldmVyaXR5PiAkY3VzdG9tTWFwcGluZyBPcHRpb25hbCBjdXN0b20gbWFwcGluZyAoTW9ub2xvZyBMZXZlbCB2YWx1ZSA9PiBUZWxlbWV0cnkgU2V2ZXJpdHkpCiAqCiAqIEV4YW1wbGUgd2l0aCBkZWZhdWx0IG1hcHBpbmc6CiAqIGBgYHBocAogKiAkbWFwcGVyID0gc2V2ZXJpdHlfbWFwcGVyKCk7CiAqIGBgYAogKgogKiBFeGFtcGxlIHdpdGggY3VzdG9tIG1hcHBpbmc6CiAqIGBgYHBocAogKiB1c2UgTW9ub2xvZ1xMZXZlbDsKICogdXNlIEZsb3dcVGVsZW1ldHJ5XExvZ2dlclxTZXZlcml0eTsKICoKICogJG1hcHBlciA9IHNldmVyaXR5X21hcHBlcihbCiAqICAgICBMZXZlbDo6RGVidWctPnZhbHVlID0+IFNldmVyaXR5OjpERUJVRywKICogICAgIExldmVsOjpJbmZvLT52YWx1ZSA9PiBTZXZlcml0eTo6SU5GTywKICogICAgIExldmVsOjpOb3RpY2UtPnZhbHVlID0+IFNldmVyaXR5OjpXQVJOLCAgLy8gQ3VzdG9tOiBOT1RJQ0Ug4oaSIFdBUk4gaW5zdGVhZCBvZiBJTkZPCiAqICAgICBMZXZlbDo6V2FybmluZy0+dmFsdWUgPT4gU2V2ZXJpdHk6OldBUk4sCiAqICAgICBMZXZlbDo6RXJyb3ItPnZhbHVlID0+IFNldmVyaXR5OjpFUlJPUiwKICogICAgIExldmVsOjpDcml0aWNhbC0+dmFsdWUgPT4gU2V2ZXJpdHk6OkZBVEFMLAogKiAgICAgTGV2ZWw6OkFsZXJ0LT52YWx1ZSA9PiBTZXZlcml0eTo6RkFUQUwsCiAqICAgICBMZXZlbDo6RW1lcmdlbmN5LT52YWx1ZSA9PiBTZXZlcml0eTo6RkFUQUwsCiAqIF0pOwogKiBgYGAKICov"},{"repository_path":"src\/bridge\/monolog\/telemetry\/src\/Flow\/Bridge\/Monolog\/Telemetry\/DSL\/functions.php","start_line_in_file":107,"slug":"log-record-converter","name":"log_record_converter","namespace":"Flow\\Bridge\\Monolog\\Telemetry\\DSL","parameters":[{"name":"severityMapper","type":[{"name":"SeverityMapper","namespace":"Flow\\Bridge\\Monolog\\Telemetry","is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"null"},{"name":"valueNormalizer","type":[{"name":"ValueNormalizer","namespace":"Flow\\Bridge\\Monolog\\Telemetry","is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"null"}],"return_type":[{"name":"LogRecordConverter","namespace":"Flow\\Bridge\\Monolog\\Telemetry","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"MONOLOG_TELEMETRY_BRIDGE","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBhIExvZ1JlY29yZENvbnZlcnRlciBmb3IgY29udmVydGluZyBNb25vbG9nIExvZ1JlY29yZCB0byBUZWxlbWV0cnkgTG9nUmVjb3JkLgogKgogKiBUaGUgY29udmVydGVyIGhhbmRsZXM6CiAqIC0gU2V2ZXJpdHkgbWFwcGluZyBmcm9tIE1vbm9sb2cgTGV2ZWwgdG8gVGVsZW1ldHJ5IFNldmVyaXR5CiAqIC0gTWVzc2FnZSBib2R5IGNvbnZlcnNpb24KICogLSBDaGFubmVsIGFuZCBsZXZlbCBuYW1lIGFzIG1vbm9sb2cuKiBhdHRyaWJ1dGVzCiAqIC0gQ29udGV4dCB2YWx1ZXMgYXMgY29udGV4dC4qIGF0dHJpYnV0ZXMgKFRocm93YWJsZXMgdXNlIHNldEV4Y2VwdGlvbigpKQogKiAtIEV4dHJhIHZhbHVlcyBhcyBleHRyYS4qIGF0dHJpYnV0ZXMKICoKICogQHBhcmFtIG51bGx8U2V2ZXJpdHlNYXBwZXIgJHNldmVyaXR5TWFwcGVyIEN1c3RvbSBzZXZlcml0eSBtYXBwZXIgKGRlZmF1bHRzIHRvIHN0YW5kYXJkIG1hcHBpbmcpCiAqIEBwYXJhbSBudWxsfFZhbHVlTm9ybWFsaXplciAkdmFsdWVOb3JtYWxpemVyIEN1c3RvbSB2YWx1ZSBub3JtYWxpemVyIChkZWZhdWx0cyB0byBzdGFuZGFyZCBub3JtYWxpemVyKQogKgogKiBFeGFtcGxlIHVzYWdlOgogKiBgYGBwaHAKICogJGNvbnZlcnRlciA9IGxvZ19yZWNvcmRfY29udmVydGVyKCk7CiAqICR0ZWxlbWV0cnlSZWNvcmQgPSAkY29udmVydGVyLT5jb252ZXJ0KCRtb25vbG9nUmVjb3JkKTsKICogYGBgCiAqCiAqIEV4YW1wbGUgd2l0aCBjdXN0b20gbWFwcGVyOgogKiBgYGBwaHAKICogJGNvbnZlcnRlciA9IGxvZ19yZWNvcmRfY29udmVydGVyKAogKiAgICAgc2V2ZXJpdHlNYXBwZXI6IHNldmVyaXR5X21hcHBlcihbCiAqICAgICAgICAgTGV2ZWw6OkRlYnVnLT52YWx1ZSA9PiBTZXZlcml0eTo6VFJBQ0UsCiAqICAgICBdKQogKiApOwogKiBgYGAKICov"},{"repository_path":"src\/bridge\/monolog\/telemetry\/src\/Flow\/Bridge\/Monolog\/Telemetry\/DSL\/functions.php","start_line_in_file":149,"slug":"telemetry-handler","name":"telemetry_handler","namespace":"Flow\\Bridge\\Monolog\\Telemetry\\DSL","parameters":[{"name":"logger","type":[{"name":"Logger","namespace":"Flow\\Telemetry\\Logger","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"converter","type":[{"name":"LogRecordConverter","namespace":"Flow\\Bridge\\Monolog\\Telemetry","is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"Flow\\Bridge\\Monolog\\Telemetry\\LogRecordConverter::..."},{"name":"level","type":[{"name":"Level","namespace":"Monolog","is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"Monolog\\Level::..."},{"name":"bubble","type":[{"name":"bool","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"true"},{"name":"errorHandler","type":[{"name":"ErrorHandler","namespace":"Flow\\Telemetry\\ErrorHandler","is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"Flow\\Telemetry\\ErrorHandler\\ErrorLogHandler::..."}],"return_type":[{"name":"TelemetryHandler","namespace":"Flow\\Bridge\\Monolog\\Telemetry","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"MONOLOG_TELEMETRY_BRIDGE","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBhIFRlbGVtZXRyeUhhbmRsZXIgZm9yIGZvcndhcmRpbmcgTW9ub2xvZyBsb2dzIHRvIEZsb3cgVGVsZW1ldHJ5LgogKgogKiBAcGFyYW0gTG9nZ2VyICRsb2dnZXIgVGhlIEZsb3cgVGVsZW1ldHJ5IGxvZ2dlciB0byBmb3J3YXJkIGxvZ3MgdG8KICogQHBhcmFtIExvZ1JlY29yZENvbnZlcnRlciAkY29udmVydGVyIENvbnZlcnRlciB0byB0cmFuc2Zvcm0gTW9ub2xvZyBMb2dSZWNvcmQgdG8gVGVsZW1ldHJ5IExvZ1JlY29yZAogKiBAcGFyYW0gTGV2ZWwgJGxldmVsIFRoZSBtaW5pbXVtIGxvZ2dpbmcgbGV2ZWwgYXQgd2hpY2ggdGhpcyBoYW5kbGVyIHdpbGwgYmUgdHJpZ2dlcmVkCiAqIEBwYXJhbSBib29sICRidWJibGUgV2hldGhlciBtZXNzYWdlcyBoYW5kbGVkIGJ5IHRoaXMgaGFuZGxlciBzaG91bGQgYnViYmxlIHVwIHRvIG90aGVyIGhhbmRsZXJzCiAqCiAqIEV4YW1wbGUgdXNhZ2U6CiAqIGBgYHBocAogKiB1c2UgTW9ub2xvZ1xMb2dnZXIgYXMgTW9ub2xvZ0xvZ2dlcjsKICogdXNlIGZ1bmN0aW9uIEZsb3dcQnJpZGdlXE1vbm9sb2dcVGVsZW1ldHJ5XERTTFx0ZWxlbWV0cnlfaGFuZGxlcjsKICogdXNlIGZ1bmN0aW9uIEZsb3dcVGVsZW1ldHJ5XERTTFx0ZWxlbWV0cnk7CiAqCiAqICR0ZWxlbWV0cnkgPSB0ZWxlbWV0cnkoKTsKICogJGxvZ2dlciA9ICR0ZWxlbWV0cnktPmxvZ2dlcignbXktYXBwJyk7CiAqCiAqICRtb25vbG9nID0gbmV3IE1vbm9sb2dMb2dnZXIoJ2NoYW5uZWwnKTsKICogJG1vbm9sb2ctPnB1c2hIYW5kbGVyKHRlbGVtZXRyeV9oYW5kbGVyKCRsb2dnZXIpKTsKICoKICogJG1vbm9sb2ctPmluZm8oJ1VzZXIgbG9nZ2VkIGluJywgWyd1c2VyX2lkJyA9PiAxMjNdKTsKICogLy8g4oaSIEZvcndhcmRlZCB0byBGbG93IFRlbGVtZXRyeSB3aXRoIElORk8gc2V2ZXJpdHkKICogYGBgCiAqCiAqIEV4YW1wbGUgd2l0aCBjdXN0b20gY29udmVydGVyOgogKiBgYGBwaHAKICogJGNvbnZlcnRlciA9IGxvZ19yZWNvcmRfY29udmVydGVyKAogKiAgICAgc2V2ZXJpdHlNYXBwZXI6IHNldmVyaXR5X21hcHBlcihbCiAqICAgICAgICAgTGV2ZWw6OkRlYnVnLT52YWx1ZSA9PiBTZXZlcml0eTo6VFJBQ0UsCiAqICAgICBdKQogKiApOwogKiAkbW9ub2xvZy0+cHVzaEhhbmRsZXIodGVsZW1ldHJ5X2hhbmRsZXIoJGxvZ2dlciwgJGNvbnZlcnRlcikpOwogKiBgYGAKICov"},{"repository_path":"src\/bridge\/symfony\/http-foundation-telemetry\/src\/Flow\/Bridge\/Symfony\/HttpFoundationTelemetry\/DSL\/functions.php","start_line_in_file":16,"slug":"symfony-request-carrier","name":"symfony_request_carrier","namespace":"Flow\\Bridge\\Symfony\\HttpFoundationTelemetry\\DSL","parameters":[{"name":"request","type":[{"name":"Request","namespace":"Symfony\\Component\\HttpFoundation","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"RequestCarrier","namespace":"Flow\\Bridge\\Symfony\\HttpFoundationTelemetry","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"SYMFONY_HTTP_FOUNDATION_TELEMETRY_BRIDGE","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/bridge\/symfony\/http-foundation-telemetry\/src\/Flow\/Bridge\/Symfony\/HttpFoundationTelemetry\/DSL\/functions.php","start_line_in_file":22,"slug":"symfony-response-carrier","name":"symfony_response_carrier","namespace":"Flow\\Bridge\\Symfony\\HttpFoundationTelemetry\\DSL","parameters":[{"name":"response","type":[{"name":"Response","namespace":"Symfony\\Component\\HttpFoundation","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"ResponseCarrier","namespace":"Flow\\Bridge\\Symfony\\HttpFoundationTelemetry","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"SYMFONY_HTTP_FOUNDATION_TELEMETRY_BRIDGE","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/bridge\/psr7\/telemetry\/src\/Flow\/Bridge\/Psr7\/Telemetry\/DSL\/functions.php","start_line_in_file":16,"slug":"psr7-request-carrier","name":"psr7_request_carrier","namespace":"Flow\\Bridge\\Psr7\\Telemetry\\DSL","parameters":[{"name":"request","type":[{"name":"ServerRequestInterface","namespace":"Psr\\Http\\Message","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"RequestCarrier","namespace":"Flow\\Bridge\\Psr7\\Telemetry","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PSR7_TELEMETRY_BRIDGE","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/bridge\/psr7\/telemetry\/src\/Flow\/Bridge\/Psr7\/Telemetry\/DSL\/functions.php","start_line_in_file":22,"slug":"psr7-response-carrier","name":"psr7_response_carrier","namespace":"Flow\\Bridge\\Psr7\\Telemetry\\DSL","parameters":[{"name":"response","type":[{"name":"ResponseInterface","namespace":"Psr\\Http\\Message","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"ResponseCarrier","namespace":"Flow\\Bridge\\Psr7\\Telemetry","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PSR7_TELEMETRY_BRIDGE","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/bridge\/psr18\/telemetry\/src\/Flow\/Bridge\/Psr18\/Telemetry\/DSL\/functions.php","start_line_in_file":15,"slug":"psr18-traceable-client","name":"psr18_traceable_client","namespace":"Flow\\Bridge\\Psr18\\Telemetry\\DSL","parameters":[{"name":"client","type":[{"name":"ClientInterface","namespace":"Psr\\Http\\Client","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"telemetry","type":[{"name":"Telemetry","namespace":"Flow\\Telemetry","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"PSR18TraceableClient","namespace":"Flow\\Bridge\\Psr18\\Telemetry","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PSR18_TELEMETRY_BRIDGE","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/bridge\/telemetry\/otlp\/src\/Flow\/Bridge\/Telemetry\/OTLP\/DSL\/functions.php","start_line_in_file":49,"slug":"otlp-json-serializer","name":"otlp_json_serializer","namespace":"Flow\\Bridge\\Telemetry\\OTLP\\DSL","parameters":[],"return_type":[{"name":"JsonSerializer","namespace":"Flow\\Bridge\\Telemetry\\OTLP\\Serializer","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"TELEMETRY_OTLP","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBhIEpTT04gc2VyaWFsaXplciBmb3IgT1RMUC4KICoKICogUmV0dXJucyBhIEpzb25TZXJpYWxpemVyIHRoYXQgY29udmVydHMgdGVsZW1ldHJ5IGRhdGEgdG8gT1RMUCBKU09OIHdpcmUgZm9ybWF0LgogKiBVc2UgdGhpcyB3aXRoIEN1cmxUcmFuc3BvcnQgZm9yIEpTT04gb3ZlciBIVFRQLgogKgogKiBFeGFtcGxlIHVzYWdlOgogKiBgYGBwaHAKICogJHNlcmlhbGl6ZXIgPSBvdGxwX2pzb25fc2VyaWFsaXplcigpOwogKiAkdHJhbnNwb3J0ID0gb3RscF9jdXJsX3RyYW5zcG9ydCgkZW5kcG9pbnQsICRzZXJpYWxpemVyKTsKICogYGBgCiAqLw=="},{"repository_path":"src\/bridge\/telemetry\/otlp\/src\/Flow\/Bridge\/Telemetry\/OTLP\/DSL\/functions.php","start_line_in_file":70,"slug":"otlp-protobuf-serializer","name":"otlp_protobuf_serializer","namespace":"Flow\\Bridge\\Telemetry\\OTLP\\DSL","parameters":[],"return_type":[{"name":"ProtobufSerializer","namespace":"Flow\\Bridge\\Telemetry\\OTLP\\Serializer","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"TELEMETRY_OTLP","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBhIFByb3RvYnVmIHNlcmlhbGl6ZXIgZm9yIE9UTFAuCiAqCiAqIFJldHVybnMgYSBQcm90b2J1ZlNlcmlhbGl6ZXIgdGhhdCBjb252ZXJ0cyB0ZWxlbWV0cnkgZGF0YSB0byBPVExQIFByb3RvYnVmIGJpbmFyeSBmb3JtYXQuCiAqIFVzZSB0aGlzIHdpdGggQ3VybFRyYW5zcG9ydCBmb3IgUHJvdG9idWYgb3ZlciBIVFRQLCBvciB3aXRoIEdycGNUcmFuc3BvcnQuCiAqCiAqIFJlcXVpcmVzOgogKiAtIGdvb2dsZS9wcm90b2J1ZiBwYWNrYWdlCiAqCiAqIEV4YW1wbGUgdXNhZ2U6CiAqIGBgYHBocAogKiAkc2VyaWFsaXplciA9IG90bHBfcHJvdG9idWZfc2VyaWFsaXplcigpOwogKiAkdHJhbnNwb3J0ID0gb3RscF9jdXJsX3RyYW5zcG9ydCgkZW5kcG9pbnQsICRzZXJpYWxpemVyKTsKICogYGBgCiAqLw=="},{"repository_path":"src\/bridge\/telemetry\/otlp\/src\/Flow\/Bridge\/Telemetry\/OTLP\/DSL\/functions.php","start_line_in_file":94,"slug":"otlp-grpc-transport","name":"otlp_grpc_transport","namespace":"Flow\\Bridge\\Telemetry\\OTLP\\DSL","parameters":[{"name":"endpoint","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"headers","type":[{"name":"array","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"[]"},{"name":"insecure","type":[{"name":"bool","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"true"},{"name":"timeoutMs","type":[{"name":"int","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"250"},{"name":"shutdownTimeoutMs","type":[{"name":"int","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"5000"},{"name":"failover","type":[{"name":"Transport","namespace":"Flow\\Bridge\\Telemetry\\OTLP\\Transport","is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"null"}],"return_type":[{"name":"Transport","namespace":"Flow\\Bridge\\Telemetry\\OTLP\\Transport","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"TELEMETRY_OTLP","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBhIGdSUEMgdHJhbnNwb3J0IGZvciBPVExQIGVuZHBvaW50cy4KICoKICogQ3JlYXRlcyBhIEdycGNUcmFuc3BvcnQgY29uZmlndXJlZCB0byBzZW5kIHRlbGVtZXRyeSBkYXRhIHRvIGFuIE9UTFAtY29tcGF0aWJsZQogKiBlbmRwb2ludCB1c2luZyBnUlBDIHByb3RvY29sIHdpdGggUHJvdG9idWYgc2VyaWFsaXphdGlvbi4gT1RMUC9nUlBDIG1hbmRhdGVzCiAqIFByb3RvYnVmLCBzbyB0aGUgc2VyaWFsaXplciBpcyBidWlsdCBpbnRlcm5hbGx5IGFuZCBub3QgY29uZmlndXJhYmxlLgogKgogKiBSZXF1aXJlczoKICogLSBleHQtZ3JwYyBQSFAgZXh0ZW5zaW9uCiAqIC0gZ29vZ2xlL3Byb3RvYnVmIHBhY2thZ2UKICoKICogQHBhcmFtIHN0cmluZyAkZW5kcG9pbnQgZ1JQQyBlbmRwb2ludCAoZS5nLiwgJ2xvY2FsaG9zdDo0MzE3JykKICogQHBhcmFtIGFycmF5PHN0cmluZywgc3RyaW5nPiAkaGVhZGVycyBBZGRpdGlvbmFsIGhlYWRlcnMgKG1ldGFkYXRhKSB0byBpbmNsdWRlIGluIHJlcXVlc3RzCiAqIEBwYXJhbSBib29sICRpbnNlY3VyZSBXaGV0aGVyIHRvIHVzZSBpbnNlY3VyZSBjaGFubmVsIGNyZWRlbnRpYWxzIChkZWZhdWx0IHRydWUgZm9yIGxvY2FsIGRldikKICogQHBhcmFtIGludCAkdGltZW91dE1zIFBlci1jYWxsIGRlYWRsaW5lIGluIG1pbGxpc2Vjb25kcyAoY292ZXJzIGNvbm5lY3QgKyBzZW5kICsgcmVjZWl2ZSkKICogQHBhcmFtIGludCAkc2h1dGRvd25UaW1lb3V0TXMgV2FsbC1jbG9jayBidWRnZXQgZm9yIGRyYWluaW5nIHBlbmRpbmcgY2FsbHMgYXQgc2h1dGRvd24KICogQHBhcmFtID9UcmFuc3BvcnQgJGZhaWxvdmVyIE9wdGlvbmFsIGZhaWxvdmVyIHRyYW5zcG9ydCByZWNlaXZpbmcgcHJpb3IgYmF0Y2hlcyB3aGVuIHByaW1hcnkgZmFpbHMKICov"},{"repository_path":"src\/bridge\/telemetry\/otlp\/src\/Flow\/Bridge\/Telemetry\/OTLP\/DSL\/functions.php","start_line_in_file":109,"slug":"otlp-curl-options","name":"otlp_curl_options","namespace":"Flow\\Bridge\\Telemetry\\OTLP\\DSL","parameters":[],"return_type":[{"name":"CurlTransportOptions","namespace":"Flow\\Bridge\\Telemetry\\OTLP\\Transport","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"TELEMETRY_OTLP","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBjdXJsIHRyYW5zcG9ydCBvcHRpb25zIGZvciBPVExQLgogKi8="},{"repository_path":"src\/bridge\/telemetry\/otlp\/src\/Flow\/Bridge\/Telemetry\/OTLP\/DSL\/functions.php","start_line_in_file":130,"slug":"otlp-curl-transport","name":"otlp_curl_transport","namespace":"Flow\\Bridge\\Telemetry\\OTLP\\DSL","parameters":[{"name":"endpoint","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"serializer","type":[{"name":"JsonSerializer","namespace":"Flow\\Bridge\\Telemetry\\OTLP\\Serializer","is_nullable":false,"is_variadic":false},{"name":"ProtobufSerializer","namespace":"Flow\\Bridge\\Telemetry\\OTLP\\Serializer","is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"Flow\\Bridge\\Telemetry\\OTLP\\Serializer\\JsonSerializer::..."},{"name":"options","type":[{"name":"CurlTransportOptions","namespace":"Flow\\Bridge\\Telemetry\\OTLP\\Transport","is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"Flow\\Bridge\\Telemetry\\OTLP\\Transport\\CurlTransportOptions::..."},{"name":"failover","type":[{"name":"Transport","namespace":"Flow\\Bridge\\Telemetry\\OTLP\\Transport","is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"null"}],"return_type":[{"name":"Transport","namespace":"Flow\\Bridge\\Telemetry\\OTLP\\Transport","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"TELEMETRY_OTLP","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBhIHN5bmNocm9ub3VzIGN1cmwgdHJhbnNwb3J0IGZvciBPVExQIGVuZHBvaW50cy4KICoKICogQ3JlYXRlcyBhIEN1cmxUcmFuc3BvcnQgdGhhdCBkcml2ZXMgZWFjaCByZXF1ZXN0IHRvIGNvbXBsZXRpb24gYW5kIHJlcG9ydHMgdGhlCiAqIG91dGNvbWUgaW1tZWRpYXRlbHkgKHJldHVybnMgb24gc3VjY2VzcywgdGhyb3dzIG9uIGZhaWx1cmUpLiBPVExQL0hUVFAgYWxsb3dzCiAqIEpTT04gb3IgUHJvdG9idWYgZW5jb2Rpbmc7IGRlZmF1bHRzIHRvIEpTT04uIEtlZXBpbmcgZXhwb3J0IG9mZiB0aGUgYXBwbGljYXRpb24KICogaG90IHBhdGggaXMgdGhlIGpvYiBvZiB0aGUgYmF0Y2hpbmcgcHJvY2Vzc29yIGluIGZyb250IG9mIHRoZSBleHBvcnRlci4KICoKICogUmVxdWlyZXM6IGV4dC1jdXJsIFBIUCBleHRlbnNpb24KICoKICogQHBhcmFtIHN0cmluZyAkZW5kcG9pbnQgT1RMUCBlbmRwb2ludCBVUkwgKGUuZy4sICdodHRwOi8vbG9jYWxob3N0OjQzMTgnKQogKiBAcGFyYW0gSnNvblNlcmlhbGl6ZXJ8UHJvdG9idWZTZXJpYWxpemVyICRzZXJpYWxpemVyIFNlcmlhbGl6ZXIgZm9yIGVuY29kaW5nIHRlbGVtZXRyeSBkYXRhIChKU09OIG9yIFByb3RvYnVmKQogKiBAcGFyYW0gQ3VybFRyYW5zcG9ydE9wdGlvbnMgJG9wdGlvbnMgVHJhbnNwb3J0IGNvbmZpZ3VyYXRpb24gb3B0aW9ucwogKiBAcGFyYW0gP1RyYW5zcG9ydCAkZmFpbG92ZXIgT3B0aW9uYWwgZmFpbG92ZXIgdHJhbnNwb3J0IHJlY2VpdmluZyB0aGUgYmF0Y2ggd2hlbiB0aGUgcHJpbWFyeSBmYWlscwogKi8="},{"repository_path":"src\/bridge\/telemetry\/otlp\/src\/Flow\/Bridge\/Telemetry\/OTLP\/DSL\/functions.php","start_line_in_file":143,"slug":"otlp-async-curl-options","name":"otlp_async_curl_options","namespace":"Flow\\Bridge\\Telemetry\\OTLP\\DSL","parameters":[],"return_type":[{"name":"AsyncCurlTransportOptions","namespace":"Flow\\Bridge\\Telemetry\\OTLP\\Transport","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"TELEMETRY_OTLP","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBhc3luYyBjdXJsIHRyYW5zcG9ydCBvcHRpb25zIGZvciBPVExQLgogKi8="},{"repository_path":"src\/bridge\/telemetry\/otlp\/src\/Flow\/Bridge\/Telemetry\/OTLP\/DSL\/functions.php","start_line_in_file":158,"slug":"otlp-async-curl-transport","name":"otlp_async_curl_transport","namespace":"Flow\\Bridge\\Telemetry\\OTLP\\DSL","parameters":[{"name":"endpoint","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"serializer","type":[{"name":"JsonSerializer","namespace":"Flow\\Bridge\\Telemetry\\OTLP\\Serializer","is_nullable":false,"is_variadic":false},{"name":"ProtobufSerializer","namespace":"Flow\\Bridge\\Telemetry\\OTLP\\Serializer","is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"Flow\\Bridge\\Telemetry\\OTLP\\Serializer\\JsonSerializer::..."},{"name":"options","type":[{"name":"AsyncCurlTransportOptions","namespace":"Flow\\Bridge\\Telemetry\\OTLP\\Transport","is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"Flow\\Bridge\\Telemetry\\OTLP\\Transport\\AsyncCurlTransportOptions::..."},{"name":"failover","type":[{"name":"Transport","namespace":"Flow\\Bridge\\Telemetry\\OTLP\\Transport","is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"null"},{"name":"error_handler","type":[{"name":"ErrorHandler","namespace":"Flow\\Telemetry\\ErrorHandler","is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"Flow\\Telemetry\\ErrorHandler\\ErrorLogHandler::..."}],"return_type":[{"name":"AsyncCurlTransport","namespace":"Flow\\Bridge\\Telemetry\\OTLP\\Transport","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"TELEMETRY_OTLP","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBhbiBhc3luY2hyb25vdXMgY3VybCB0cmFuc3BvcnQgZm9yIE9UTFAgZW5kcG9pbnRzLgogKgogKiBAcGFyYW0gc3RyaW5nICRlbmRwb2ludCBPVExQIGVuZHBvaW50IFVSTCAoZS5nLiwgJ2h0dHA6Ly9sb2NhbGhvc3Q6NDMxOCcpCiAqIEBwYXJhbSBKc29uU2VyaWFsaXplcnxQcm90b2J1ZlNlcmlhbGl6ZXIgJHNlcmlhbGl6ZXIgU2VyaWFsaXplciBmb3IgZW5jb2RpbmcgdGVsZW1ldHJ5IGRhdGEgKEpTT04gb3IgUHJvdG9idWYpCiAqIEBwYXJhbSBBc3luY0N1cmxUcmFuc3BvcnRPcHRpb25zICRvcHRpb25zIFRyYW5zcG9ydCBjb25maWd1cmF0aW9uIG9wdGlvbnMKICogQHBhcmFtID9UcmFuc3BvcnQgJGZhaWxvdmVyIE9wdGlvbmFsIGZhaWxvdmVyIHRyYW5zcG9ydCByZWNlaXZpbmcgcHJpb3IgYmF0Y2hlcyB3aGVuIHByaW1hcnkgZmFpbHMKICogQHBhcmFtIEVycm9ySGFuZGxlciAkZXJyb3JfaGFuZGxlciBIYW5kbGVyIGZvciBmYWlsdXJlcyByZWFwZWQgb24gc2VuZCgpL3RpY2soKS9zaHV0ZG93bigpIChubyBmYWlsb3ZlcikKICov"},{"repository_path":"src\/bridge\/telemetry\/otlp\/src\/Flow\/Bridge\/Telemetry\/OTLP\/DSL\/functions.php","start_line_in_file":180,"slug":"otlp-stream-transport","name":"otlp_stream_transport","namespace":"Flow\\Bridge\\Telemetry\\OTLP\\DSL","parameters":[{"name":"destination","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"filePermissions","type":[{"name":"int","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"420"},{"name":"createDirectories","type":[{"name":"bool","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"true"}],"return_type":[{"name":"Transport","namespace":"Flow\\Bridge\\Telemetry\\OTLP\\Transport","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"TELEMETRY_OTLP","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBhIHN0cmVhbSB0cmFuc3BvcnQgZm9yIE9UTFAgdGhhdCB3cml0ZXMgSlNPTkwgdG8gYSBzaW5nbGUgZGVzdGluYXRpb24uCiAqCiAqIEFjY2VwdHMgYW4gYWJzb2x1dGUgZmlsZSBwYXRoIG9yIGEgcGhwOi8vIHN0cmVhbSB3cmFwcGVyIHN1Y2ggYXMKICogJ3BocDovL3N0ZG91dCcsICdwaHA6Ly9zdGRlcnInLCAncGhwOi8vbWVtb3J5Jywgb3IgJ3BocDovL3RlbXAnLiBFYWNoCiAqIGV4cG9ydCgpIGNhbGwgYXBwZW5kcyBvbmUgSlNPTiBMaW5lIHVuZGVyIExPQ0tfRVggc28gY29uY3VycmVudCB3cml0ZXJzCiAqIGludGVybGVhdmUgYXQgbGluZSBib3VuZGFyaWVzLiBUaGUgJGZpbGVQZXJtaXNzaW9ucyBhbmQgJGNyZWF0ZURpcmVjdG9yaWVzCiAqIHBhcmFtZXRlcnMgYXBwbHkgb25seSB3aGVuIHRoZSBkZXN0aW5hdGlvbiBpcyBhIGZpbGUgcGF0aC4KICoKICogUGVyIHRoZSBPVExQIEZpbGUgRXhwb3J0ZXIgc3BlYyBvbmx5IEpTT04gZW5jb2RpbmcgaXMgc3VwcG9ydGVkLgogKi8="},{"repository_path":"src\/bridge\/telemetry\/otlp\/src\/Flow\/Bridge\/Telemetry\/OTLP\/DSL\/functions.php","start_line_in_file":203,"slug":"otlp-exporter","name":"otlp_exporter","namespace":"Flow\\Bridge\\Telemetry\\OTLP\\DSL","parameters":[{"name":"transport","type":[{"name":"Transport","namespace":"Flow\\Bridge\\Telemetry\\OTLP\\Transport","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"errorHandler","type":[{"name":"ErrorHandler","namespace":"Flow\\Telemetry\\ErrorHandler","is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"Flow\\Telemetry\\ErrorHandler\\ErrorLogHandler::..."}],"return_type":[{"name":"OTLPExporter","namespace":"Flow\\Bridge\\Telemetry\\OTLP\\Exporter","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"TELEMETRY_OTLP","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBhbiBPVExQIGV4cG9ydGVyIHRoYXQgZGlzcGF0Y2hlcyBsb2dzLCBtZXRyaWNzLCBhbmQgc3BhbnMgdGhyb3VnaCBhIHNpbmdsZSB0cmFuc3BvcnQuCiAqCiAqIEV4YW1wbGUgdXNhZ2U6CiAqIGBgYHBocAogKiAkZXhwb3J0ZXIgPSBvdGxwX2V4cG9ydGVyKCR0cmFuc3BvcnQpOwogKiAkc3BhblByb2Nlc3NvciA9IGJhdGNoaW5nX3NwYW5fcHJvY2Vzc29yKCRleHBvcnRlcik7CiAqICRtZXRyaWNQcm9jZXNzb3IgPSBiYXRjaGluZ19tZXRyaWNfcHJvY2Vzc29yKCRleHBvcnRlcik7CiAqICRsb2dQcm9jZXNzb3IgPSBiYXRjaGluZ19sb2dfcHJvY2Vzc29yKCRleHBvcnRlcik7CiAqIGBgYAogKgogKiBAcGFyYW0gVHJhbnNwb3J0ICR0cmFuc3BvcnQgVGhlIHRyYW5zcG9ydCBmb3Igc2VuZGluZyB0ZWxlbWV0cnkgZGF0YQogKiBAcGFyYW0gRXJyb3JIYW5kbGVyICRlcnJvckhhbmRsZXIgSGFuZGxlciBmb3IgVGhyb3dhYmxlcyByYWlzZWQgYnkgdGhlIHRyYW5zcG9ydAogKi8="},{"repository_path":"src\/bridge\/telemetry\/otlp\/src\/Flow\/Bridge\/Telemetry\/OTLP\/DSL\/functions.php","start_line_in_file":217,"slug":"otlp-tracer-provider","name":"otlp_tracer_provider","namespace":"Flow\\Bridge\\Telemetry\\OTLP\\DSL","parameters":[{"name":"processor","type":[{"name":"SpanProcessor","namespace":"Flow\\Telemetry\\Tracer","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"clock","type":[{"name":"ClockInterface","namespace":"Psr\\Clock","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"sampler","type":[{"name":"Sampler","namespace":"Flow\\Telemetry\\Tracer\\Sampler","is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"Flow\\Telemetry\\Tracer\\Sampler\\ParentBasedSampler::..."},{"name":"contextStorage","type":[{"name":"ContextStorage","namespace":"Flow\\Telemetry\\Context","is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"Flow\\Telemetry\\Context\\MemoryContextStorage::..."}],"return_type":[{"name":"TracerProvider","namespace":"Flow\\Telemetry\\Tracer","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"TELEMETRY_OTLP","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBhIHRyYWNlciBwcm92aWRlciBjb25maWd1cmVkIGZvciBPVExQIGV4cG9ydC4KICoKICogQHBhcmFtIFNwYW5Qcm9jZXNzb3IgJHByb2Nlc3NvciBUaGUgcHJvY2Vzc29yIGZvciBoYW5kbGluZyBzcGFucwogKiBAcGFyYW0gQ2xvY2tJbnRlcmZhY2UgJGNsb2NrIFRoZSBjbG9jayBmb3IgdGltZXN0YW1wcwogKiBAcGFyYW0gU2FtcGxlciAkc2FtcGxlciBUaGUgc2FtcGxlciBmb3IgZGVjaWRpbmcgd2hldGhlciB0byByZWNvcmQgc3BhbnMKICogQHBhcmFtIENvbnRleHRTdG9yYWdlICRjb250ZXh0U3RvcmFnZSBUaGUgY29udGV4dCBzdG9yYWdlIGZvciBwcm9wYWdhdGluZyB0cmFjZSBjb250ZXh0CiAqLw=="},{"repository_path":"src\/bridge\/telemetry\/otlp\/src\/Flow\/Bridge\/Telemetry\/OTLP\/DSL\/functions.php","start_line_in_file":234,"slug":"otlp-meter-provider","name":"otlp_meter_provider","namespace":"Flow\\Bridge\\Telemetry\\OTLP\\DSL","parameters":[{"name":"processor","type":[{"name":"MetricProcessor","namespace":"Flow\\Telemetry\\Meter","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"clock","type":[{"name":"ClockInterface","namespace":"Psr\\Clock","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"temporality","type":[{"name":"AggregationTemporality","namespace":"Flow\\Telemetry\\Meter","is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"Flow\\Telemetry\\Meter\\AggregationTemporality::..."}],"return_type":[{"name":"MeterProvider","namespace":"Flow\\Telemetry\\Meter","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"TELEMETRY_OTLP","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBhIG1ldGVyIHByb3ZpZGVyIGNvbmZpZ3VyZWQgZm9yIE9UTFAgZXhwb3J0LgogKgogKiBAcGFyYW0gTWV0cmljUHJvY2Vzc29yICRwcm9jZXNzb3IgVGhlIHByb2Nlc3NvciBmb3IgaGFuZGxpbmcgbWV0cmljcwogKiBAcGFyYW0gQ2xvY2tJbnRlcmZhY2UgJGNsb2NrIFRoZSBjbG9jayBmb3IgdGltZXN0YW1wcwogKiBAcGFyYW0gQWdncmVnYXRpb25UZW1wb3JhbGl0eSAkdGVtcG9yYWxpdHkgVGhlIGFnZ3JlZ2F0aW9uIHRlbXBvcmFsaXR5IGZvciBtZXRyaWNzCiAqLw=="},{"repository_path":"src\/bridge\/telemetry\/otlp\/src\/Flow\/Bridge\/Telemetry\/OTLP\/DSL\/functions.php","start_line_in_file":250,"slug":"otlp-logger-provider","name":"otlp_logger_provider","namespace":"Flow\\Bridge\\Telemetry\\OTLP\\DSL","parameters":[{"name":"processor","type":[{"name":"LogProcessor","namespace":"Flow\\Telemetry\\Logger","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"clock","type":[{"name":"ClockInterface","namespace":"Psr\\Clock","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"contextStorage","type":[{"name":"ContextStorage","namespace":"Flow\\Telemetry\\Context","is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"Flow\\Telemetry\\Context\\MemoryContextStorage::..."}],"return_type":[{"name":"LoggerProvider","namespace":"Flow\\Telemetry\\Logger","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"TELEMETRY_OTLP","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBhIGxvZ2dlciBwcm92aWRlciBjb25maWd1cmVkIGZvciBPVExQIGV4cG9ydC4KICoKICogQHBhcmFtIExvZ1Byb2Nlc3NvciAkcHJvY2Vzc29yIFRoZSBwcm9jZXNzb3IgZm9yIGhhbmRsaW5nIGxvZyByZWNvcmRzCiAqIEBwYXJhbSBDbG9ja0ludGVyZmFjZSAkY2xvY2sgVGhlIGNsb2NrIGZvciB0aW1lc3RhbXBzCiAqIEBwYXJhbSBDb250ZXh0U3RvcmFnZSAkY29udGV4dFN0b3JhZ2UgVGhlIGNvbnRleHQgc3RvcmFnZSBmb3IgcHJvcGFnYXRpbmcgY29udGV4dAogKi8="}] \ No newline at end of file +[{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DSL\/functions.php","start_line_in_file":275,"slug":"df","name":"df","namespace":"Flow\\ETL\\DSL","parameters":[{"name":"config","type":[{"name":"Config","namespace":"Flow\\ETL","is_nullable":false,"is_variadic":false},{"name":"ConfigBuilder","namespace":"Flow\\ETL\\Config","is_nullable":false,"is_variadic":false},{"name":"null","namespace":null,"is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"null"}],"return_type":[{"name":"Flow","namespace":"Flow\\ETL","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"CORE","type":"DATA_FRAME"}},{"name":"DocumentationExample","namespace":"Flow\\Documentation\\Attribute","arguments":{"topic":"reading","example":"data_frame"}},{"name":"DocumentationExample","namespace":"Flow\\Documentation\\Attribute","arguments":{"topic":"writing","example":"overwrite"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIEFsaWFzIGZvciBkYXRhX2ZyYW1lKCkgOiBGbG93LgogKi8="},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DSL\/functions.php","start_line_in_file":283,"slug":"data-frame","name":"data_frame","namespace":"Flow\\ETL\\DSL","parameters":[{"name":"config","type":[{"name":"Config","namespace":"Flow\\ETL","is_nullable":false,"is_variadic":false},{"name":"ConfigBuilder","namespace":"Flow\\ETL\\Config","is_nullable":false,"is_variadic":false},{"name":"null","namespace":null,"is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"null"}],"return_type":[{"name":"Flow","namespace":"Flow\\ETL","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"CORE","type":"DATA_FRAME"}},{"name":"DocumentationExample","namespace":"Flow\\Documentation\\Attribute","arguments":{"topic":"reading","example":"data_frame"}},{"name":"DocumentationExample","namespace":"Flow\\Documentation\\Attribute","arguments":{"topic":"writing","example":"overwrite"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DSL\/functions.php","start_line_in_file":289,"slug":"telemetry-options","name":"telemetry_options","namespace":"Flow\\ETL\\DSL","parameters":[{"name":"trace_loading","type":[{"name":"bool","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"false"},{"name":"trace_transformations","type":[{"name":"bool","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"false"},{"name":"trace_cache","type":[{"name":"bool","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"false"},{"name":"collect_metrics","type":[{"name":"bool","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"false"}],"return_type":[{"name":"TelemetryOptions","namespace":"Flow\\ETL\\Config\\Telemetry","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"CORE","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DSL\/functions.php","start_line_in_file":301,"slug":"from-rows","name":"from_rows","namespace":"Flow\\ETL\\DSL","parameters":[{"name":"rows","type":[{"name":"Rows","namespace":"Flow\\ETL","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":true,"default_value":null}],"return_type":[{"name":"RowsExtractor","namespace":"Flow\\ETL\\Extractor","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"CORE","type":"EXTRACTOR"}},{"name":"DocumentationExample","namespace":"Flow\\Documentation\\Attribute","arguments":{"topic":"reading","example":"data_frame"}},{"name":"DocumentationExample","namespace":"Flow\\Documentation\\Attribute","arguments":{"topic":"writing","example":"overwrite"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DSL\/functions.php","start_line_in_file":308,"slug":"from-path-partitions","name":"from_path_partitions","namespace":"Flow\\ETL\\DSL","parameters":[{"name":"path","type":[{"name":"Path","namespace":"Flow\\Filesystem","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"filesystem","type":[{"name":"Filesystem","namespace":"Flow\\Filesystem","is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"Flow\\Filesystem\\Local\\NativeLocalFilesystem::..."}],"return_type":[{"name":"PathPartitionsExtractor","namespace":"Flow\\ETL\\Extractor","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"CORE","type":"EXTRACTOR"}},{"name":"DocumentationExample","namespace":"Flow\\Documentation\\Attribute","arguments":{"topic":"partitioning","example":"path_partitions"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DSL\/functions.php","start_line_in_file":324,"slug":"from-array","name":"from_array","namespace":"Flow\\ETL\\DSL","parameters":[{"name":"array","type":[{"name":"iterable","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"schema","type":[{"name":"Schema","namespace":"Flow\\ETL","is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"null"},{"name":"filesystem","type":[{"name":"Filesystem","namespace":"Flow\\Filesystem","is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"Flow\\Filesystem\\Local\\NativeLocalFilesystem::..."},{"name":"spillRoot","type":[{"name":"Path","namespace":"Flow\\Filesystem","is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"null"}],"return_type":[{"name":"ArrayExtractor","namespace":"Flow\\ETL\\Extractor","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"CORE","type":"EXTRACTOR"}},{"name":"DocumentationExample","namespace":"Flow\\Documentation\\Attribute","arguments":{"topic":"reading","example":"array"}},{"name":"DocumentationExample","namespace":"Flow\\Documentation\\Attribute","arguments":{"topic":"reading","example":"data_frame"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIEBwYXJhbSBpdGVyYWJsZTxhcnJheTxtaXhlZD4+ICRhcnJheQogKiBAcGFyYW0gbnVsbHxTY2hlbWEgJHNjaGVtYSAtIEBkZXByZWNhdGVkIHVzZSB3aXRoU2NoZW1hKCkgbWV0aG9kIGluc3RlYWQKICogQHBhcmFtIG51bGx8UGF0aCAkc3BpbGxSb290IC0gd2hlcmUgYSBub24tYXJyYXkgJGFycmF5IGlzIHNwaWxsZWQgd2hpbGUgaXQgaXMgZGVzY3JpYmVkOyBudWxsIHJlc29sdmVzIHRvCiAqICAgICAgICAgICAgICAgICAgICAgICAgICAkZmlsZXN5c3RlbS0+Z2V0U3lzdGVtVG1wRGlyKCkgYW5kIG9ubHkgb24gdGhhdCBwYXRoCiAqLw=="},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DSL\/functions.php","start_line_in_file":345,"slug":"from-cache","name":"from_cache","namespace":"Flow\\ETL\\DSL","parameters":[{"name":"id","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"fallback_extractor","type":[{"name":"Extractor","namespace":"Flow\\ETL","is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"null"},{"name":"clear","type":[{"name":"bool","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"false"},{"name":"cache","type":[{"name":"Cache","namespace":"Flow\\ETL","is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"null"}],"return_type":[{"name":"CacheExtractor","namespace":"Flow\\ETL\\Extractor","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"CORE","type":"EXTRACTOR"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIEBwYXJhbSBzdHJpbmcgJGlkIC0gY2FjaGUgaWQgZnJvbSB3aGljaCBkYXRhIHdpbGwgYmUgZXh0cmFjdGVkCiAqIEBwYXJhbSBudWxsfEV4dHJhY3RvciAkZmFsbGJhY2tfZXh0cmFjdG9yIC0gZXh0cmFjdG9yIHRoYXQgd2lsbCBiZSB1c2VkIHdoZW4gY2FjaGUgaXMgZW1wdHkgLSBAZGVwcmVjYXRlZCB1c2Ugd2l0aEZhbGxiYWNrRXh0cmFjdG9yKCkgbWV0aG9kIGluc3RlYWQKICogQHBhcmFtIGJvb2wgJGNsZWFyIC0gY2xlYXIgY2FjaGUgYWZ0ZXIgZXh0cmFjdGlvbiAtIEBkZXByZWNhdGVkIHVzZSB3aXRoQ2xlYXJPbkZpbmlzaCgpIG1ldGhvZCBpbnN0ZWFkCiAqLw=="},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DSL\/functions.php","start_line_in_file":365,"slug":"from-all","name":"from_all","namespace":"Flow\\ETL\\DSL","parameters":[{"name":"extractors","type":[{"name":"Extractor","namespace":"Flow\\ETL","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":true,"default_value":null}],"return_type":[{"name":"ChainExtractor","namespace":"Flow\\ETL\\Extractor","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"CORE","type":"EXTRACTOR"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DSL\/functions.php","start_line_in_file":371,"slug":"from-memory","name":"from_memory","namespace":"Flow\\ETL\\DSL","parameters":[{"name":"memory","type":[{"name":"Memory","namespace":"Flow\\ETL\\Memory","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"MemoryExtractor","namespace":"Flow\\ETL\\Extractor","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"CORE","type":"EXTRACTOR"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DSL\/functions.php","start_line_in_file":377,"slug":"files","name":"files","namespace":"Flow\\ETL\\DSL","parameters":[{"name":"directory","type":[{"name":"Path","namespace":"Flow\\Filesystem","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"filesystem","type":[{"name":"Filesystem","namespace":"Flow\\Filesystem","is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"Flow\\Filesystem\\Local\\NativeLocalFilesystem::..."}],"return_type":[{"name":"FilesExtractor","namespace":"Flow\\ETL\\Extractor","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"CORE","type":"EXTRACTOR"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DSL\/functions.php","start_line_in_file":383,"slug":"filesystem-cache","name":"filesystem_cache","namespace":"Flow\\ETL\\DSL","parameters":[{"name":"serializer","type":[{"name":"Serializer","namespace":"Flow\\Serializer","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"cache_dir","type":[{"name":"Path","namespace":"Flow\\Filesystem","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false},{"name":"null","namespace":null,"is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"null"},{"name":"filesystem","type":[{"name":"Filesystem","namespace":"Flow\\Filesystem","is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"Flow\\Filesystem\\Local\\NativeLocalFilesystem::..."}],"return_type":[{"name":"FilesystemCache","namespace":"Flow\\ETL\\Cache\\Implementation","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"CORE","type":"DATA_FRAME"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DSL\/functions.php","start_line_in_file":395,"slug":"batched-by","name":"batched_by","namespace":"Flow\\ETL\\DSL","parameters":[{"name":"extractor","type":[{"name":"Extractor","namespace":"Flow\\ETL","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"column","type":[{"name":"Reference","namespace":"Flow\\ETL\\Row","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"min_size","type":[{"name":"int","namespace":null,"is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"null"}],"return_type":[{"name":"BatchByExtractor","namespace":"Flow\\ETL\\Extractor","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"CORE","type":"EXTRACTOR"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIEBwYXJhbSBudWxsfGludDwxLCBtYXg+ICRtaW5fc2l6ZQogKi8="},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DSL\/functions.php","start_line_in_file":409,"slug":"batches","name":"batches","namespace":"Flow\\ETL\\DSL","parameters":[{"name":"extractor","type":[{"name":"Extractor","namespace":"Flow\\ETL","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"size","type":[{"name":"int","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"BatchExtractor","namespace":"Flow\\ETL\\Extractor","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"CORE","type":"EXTRACTOR"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIEBwYXJhbSBpbnQ8MSwgbWF4PiAkc2l6ZQogKi8="},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DSL\/functions.php","start_line_in_file":415,"slug":"from-data-frame","name":"from_data_frame","namespace":"Flow\\ETL\\DSL","parameters":[{"name":"data_frame","type":[{"name":"DataFrame","namespace":"Flow\\ETL","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"DataFrameExtractor","namespace":"Flow\\ETL\\Extractor","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"CORE","type":"EXTRACTOR"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DSL\/functions.php","start_line_in_file":421,"slug":"from-sequence-date-period","name":"from_sequence_date_period","namespace":"Flow\\ETL\\DSL","parameters":[{"name":"entry_name","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"start","type":[{"name":"DateTimeInterface","namespace":"","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"interval","type":[{"name":"DateInterval","namespace":"","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"end","type":[{"name":"DateTimeInterface","namespace":"","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"options","type":[{"name":"int","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"0"}],"return_type":[{"name":"SequenceExtractor","namespace":"Flow\\ETL\\Extractor","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"CORE","type":"EXTRACTOR"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DSL\/functions.php","start_line_in_file":435,"slug":"from-sequence-date-period-recurrences","name":"from_sequence_date_period_recurrences","namespace":"Flow\\ETL\\DSL","parameters":[{"name":"entry_name","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"start","type":[{"name":"DateTimeInterface","namespace":"","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"interval","type":[{"name":"DateInterval","namespace":"","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"recurrences","type":[{"name":"int","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"options","type":[{"name":"int","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"0"}],"return_type":[{"name":"SequenceExtractor","namespace":"Flow\\ETL\\Extractor","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"CORE","type":"EXTRACTOR"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DSL\/functions.php","start_line_in_file":449,"slug":"from-sequence-number","name":"from_sequence_number","namespace":"Flow\\ETL\\DSL","parameters":[{"name":"entry_name","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"start","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false},{"name":"int","namespace":null,"is_nullable":false,"is_variadic":false},{"name":"float","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"end","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false},{"name":"int","namespace":null,"is_nullable":false,"is_variadic":false},{"name":"float","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"step","type":[{"name":"int","namespace":null,"is_nullable":false,"is_variadic":false},{"name":"float","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"1"}],"return_type":[{"name":"SequenceExtractor","namespace":"Flow\\ETL\\Extractor","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"CORE","type":"EXTRACTOR"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DSL\/functions.php","start_line_in_file":459,"slug":"to-memory","name":"to_memory","namespace":"Flow\\ETL\\DSL","parameters":[{"name":"memory","type":[{"name":"Memory","namespace":"Flow\\ETL\\Memory","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"MemoryLoader","namespace":"Flow\\ETL\\Loader","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"CORE","type":"LOADER"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DSL\/functions.php","start_line_in_file":473,"slug":"to-array","name":"to_array","namespace":"Flow\\ETL\\DSL","parameters":[{"name":"array","type":[{"name":"array","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"ArrayLoader","namespace":"Flow\\ETL\\Loader","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"CORE","type":"LOADER"}},{"name":"DocumentationExample","namespace":"Flow\\Documentation\\Attribute","arguments":{"topic":"writing","example":"array"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENvbnZlcnQgcm93cyB0byBhbiBhcnJheSBhbmQgc3RvcmUgdGhlbSBpbiBwYXNzZWQgYXJyYXkgdmFyaWFibGUuCiAqCiAqIEBwYXJhbSBhcnJheTxhcnJheS1rZXksIG1peGVkPiAkYXJyYXkKICoKICogQHBhcmFtLW91dCBhcnJheTxhcnJheTxtaXhlZD4+ICRhcnJheQogKi8="},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DSL\/functions.php","start_line_in_file":481,"slug":"to-output","name":"to_output","namespace":"Flow\\ETL\\DSL","parameters":[{"name":"truncate","type":[{"name":"int","namespace":null,"is_nullable":false,"is_variadic":false},{"name":"bool","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"20"},{"name":"output","type":[{"name":"Output","namespace":"Flow\\ETL\\Loader\\StreamLoader","is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"Flow\\ETL\\Loader\\StreamLoader\\Output::..."},{"name":"formatter","type":[{"name":"Formatter","namespace":"Flow\\ETL","is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"Flow\\ETL\\Formatter\\AsciiTableFormatter::..."},{"name":"schemaFormatter","type":[{"name":"SchemaFormatter","namespace":"Flow\\ETL\\Schema","is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"Flow\\ETL\\Row\\Formatter\\ASCIISchemaFormatter::..."}],"return_type":[{"name":"StreamLoader","namespace":"Flow\\ETL\\Loader","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"CORE","type":"LOADER"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DSL\/functions.php","start_line_in_file":491,"slug":"to-stderr","name":"to_stderr","namespace":"Flow\\ETL\\DSL","parameters":[{"name":"truncate","type":[{"name":"int","namespace":null,"is_nullable":false,"is_variadic":false},{"name":"bool","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"20"},{"name":"output","type":[{"name":"Output","namespace":"Flow\\ETL\\Loader\\StreamLoader","is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"Flow\\ETL\\Loader\\StreamLoader\\Output::..."},{"name":"formatter","type":[{"name":"Formatter","namespace":"Flow\\ETL","is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"Flow\\ETL\\Formatter\\AsciiTableFormatter::..."},{"name":"schemaFormatter","type":[{"name":"SchemaFormatter","namespace":"Flow\\ETL\\Schema","is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"Flow\\ETL\\Row\\Formatter\\ASCIISchemaFormatter::..."}],"return_type":[{"name":"StreamLoader","namespace":"Flow\\ETL\\Loader","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"CORE","type":"LOADER"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DSL\/functions.php","start_line_in_file":501,"slug":"to-stdout","name":"to_stdout","namespace":"Flow\\ETL\\DSL","parameters":[{"name":"truncate","type":[{"name":"int","namespace":null,"is_nullable":false,"is_variadic":false},{"name":"bool","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"20"},{"name":"output","type":[{"name":"Output","namespace":"Flow\\ETL\\Loader\\StreamLoader","is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"Flow\\ETL\\Loader\\StreamLoader\\Output::..."},{"name":"formatter","type":[{"name":"Formatter","namespace":"Flow\\ETL","is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"Flow\\ETL\\Formatter\\AsciiTableFormatter::..."},{"name":"schemaFormatter","type":[{"name":"SchemaFormatter","namespace":"Flow\\ETL\\Schema","is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"Flow\\ETL\\Row\\Formatter\\ASCIISchemaFormatter::..."}],"return_type":[{"name":"StreamLoader","namespace":"Flow\\ETL\\Loader","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"CORE","type":"LOADER"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DSL\/functions.php","start_line_in_file":511,"slug":"to-stream","name":"to_stream","namespace":"Flow\\ETL\\DSL","parameters":[{"name":"uri","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"truncate","type":[{"name":"int","namespace":null,"is_nullable":false,"is_variadic":false},{"name":"bool","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"20"},{"name":"output","type":[{"name":"Output","namespace":"Flow\\ETL\\Loader\\StreamLoader","is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"Flow\\ETL\\Loader\\StreamLoader\\Output::..."},{"name":"mode","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"'w'"},{"name":"formatter","type":[{"name":"Formatter","namespace":"Flow\\ETL","is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"Flow\\ETL\\Formatter\\AsciiTableFormatter::..."},{"name":"schemaFormatter","type":[{"name":"SchemaFormatter","namespace":"Flow\\ETL\\Schema","is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"Flow\\ETL\\Row\\Formatter\\ASCIISchemaFormatter::..."}],"return_type":[{"name":"StreamLoader","namespace":"Flow\\ETL\\Loader","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"CORE","type":"LOADER"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DSL\/functions.php","start_line_in_file":531,"slug":"to-transformation","name":"to_transformation","namespace":"Flow\\ETL\\DSL","parameters":[{"name":"transformer","type":[{"name":"Transformer","namespace":"Flow\\ETL","is_nullable":false,"is_variadic":false},{"name":"Transformation","namespace":"Flow\\ETL","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"sink","type":[{"name":"Loader","namespace":"Flow\\ETL","is_nullable":false,"is_variadic":false},{"name":"Sink","namespace":"Flow\\ETL","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"Transformed","namespace":"Flow\\ETL\\Sink","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"CORE","type":"LOADER"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DSL\/functions.php","start_line_in_file":537,"slug":"to-branch","name":"to_branch","namespace":"Flow\\ETL\\DSL","parameters":[{"name":"condition","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"sink","type":[{"name":"Loader","namespace":"Flow\\ETL","is_nullable":false,"is_variadic":false},{"name":"Sink","namespace":"Flow\\ETL","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"Branched","namespace":"Flow\\ETL\\Sink","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"CORE","type":"LOADER"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DSL\/functions.php","start_line_in_file":543,"slug":"rename-style","name":"rename_style","namespace":"Flow\\ETL\\DSL","parameters":[{"name":"style","type":[{"name":"StringStyles","namespace":"Flow\\ETL\\String","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"RenameCaseEntryStrategy","namespace":"Flow\\ETL\\Transformer\\Rename","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"CORE","type":"TRANSFORMER"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DSL\/functions.php","start_line_in_file":553,"slug":"rename-replace","name":"rename_replace","namespace":"Flow\\ETL\\DSL","parameters":[{"name":"search","type":[{"name":"array","namespace":null,"is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"replace","type":[{"name":"array","namespace":null,"is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"RenameReplaceEntryStrategy","namespace":"Flow\\ETL\\Transformer\\Rename","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"CORE","type":"TRANSFORMER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIEBwYXJhbSBhcnJheTxzdHJpbmc+fHN0cmluZyAkc2VhcmNoCiAqIEBwYXJhbSBhcnJheTxzdHJpbmc+fHN0cmluZyAkcmVwbGFjZQogKi8="},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DSL\/functions.php","start_line_in_file":562,"slug":"rename-map","name":"rename_map","namespace":"Flow\\ETL\\DSL","parameters":[{"name":"renames","type":[{"name":"array","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"RenameMapEntryStrategy","namespace":"Flow\\ETL\\Transformer\\Rename","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"CORE","type":"TRANSFORMER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIEBwYXJhbSBhcnJheTxhcnJheS1rZXksIHN0cmluZz4gJHJlbmFtZXMgTWFwIG9mIG9sZF9uYW1lID0+IG5ld19uYW1lCiAqLw=="},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DSL\/functions.php","start_line_in_file":568,"slug":"rows","name":"rows","namespace":"Flow\\ETL\\DSL","parameters":[{"name":"schema","type":[{"name":"Schema","namespace":"Flow\\ETL","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"backend","type":[{"name":"Backend","namespace":"Flow\\ETL\\Column","is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"Flow\\ETL\\Column\\AdaptiveBackend::..."}],"return_type":[{"name":"Rows","namespace":"Flow\\ETL","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"CORE","type":"DATA_FRAME"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DSL\/functions.php","start_line_in_file":577,"slug":"col","name":"col","namespace":"Flow\\ETL\\DSL","parameters":[{"name":"entry","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"UnresolvedReference","namespace":"Flow\\ETL\\Row","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"CORE","type":"DATA_FRAME"}}],"scalar_function_chain":true,"doc_comment":"LyoqCiAqIEFuIGFsaWFzIGZvciBgcmVmYC4KICov"},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DSL\/functions.php","start_line_in_file":587,"slug":"entry","name":"entry","namespace":"Flow\\ETL\\DSL","parameters":[{"name":"entry","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"UnresolvedReference","namespace":"Flow\\ETL\\Row","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"CORE","type":"SCALAR_FUNCTION"}},{"name":"DocumentationExample","namespace":"Flow\\Documentation\\Attribute","arguments":{"topic":"columns","example":"create"}}],"scalar_function_chain":true,"doc_comment":"LyoqCiAqIEFuIGFsaWFzIGZvciBgcmVmYC4KICov"},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DSL\/functions.php","start_line_in_file":594,"slug":"ref","name":"ref","namespace":"Flow\\ETL\\DSL","parameters":[{"name":"entry","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"UnresolvedReference","namespace":"Flow\\ETL\\Row","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"CORE","type":"SCALAR_FUNCTION"}},{"name":"DocumentationExample","namespace":"Flow\\Documentation\\Attribute","arguments":{"topic":"columns","example":"create"}}],"scalar_function_chain":true,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DSL\/functions.php","start_line_in_file":600,"slug":"structure-ref","name":"structure_ref","namespace":"Flow\\ETL\\DSL","parameters":[{"name":"entry","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"StructureFunctions","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"CORE","type":"SCALAR_FUNCTION"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DSL\/functions.php","start_line_in_file":612,"slug":"structure","name":"structure","namespace":"Flow\\ETL\\DSL","parameters":[{"name":"elements","type":[{"name":"array","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"Structure","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"CORE","type":"SCALAR_FUNCTION"}}],"scalar_function_chain":true,"doc_comment":"LyoqCiAqIEJ1aWxkcyBhIHN0cnVjdHVyZSBmcm9tIHNjYWxhciBmdW5jdGlvbnM6IG9uZSBlbGVtZW50IHBlciBrZXksIGluIGtleSBvcmRlci4KICogQW4gZWxlbWVudCBpcyBudWxsYWJsZSB3aGVuIGl0cyBmdW5jdGlvbiBpczsgdGhlIHN0cnVjdHVyZSBpdHNlbGYgbmV2ZXIgaXMuCiAqCiAqIEBwYXJhbSBhcnJheTxhcnJheS1rZXksIFNjYWxhckZ1bmN0aW9uPiAkZWxlbWVudHMKICov"},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DSL\/functions.php","start_line_in_file":618,"slug":"list-ref","name":"list_ref","namespace":"Flow\\ETL\\DSL","parameters":[{"name":"entry","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"ListFunctions","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"CORE","type":"SCALAR_FUNCTION"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DSL\/functions.php","start_line_in_file":624,"slug":"refs","name":"refs","namespace":"Flow\\ETL\\DSL","parameters":[{"name":"entries","type":[{"name":"Reference","namespace":"Flow\\ETL\\Row","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":true,"default_value":null}],"return_type":[{"name":"References","namespace":"Flow\\ETL\\Row","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"CORE","type":"SCALAR_FUNCTION"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DSL\/functions.php","start_line_in_file":630,"slug":"select","name":"select","namespace":"Flow\\ETL\\DSL","parameters":[{"name":"entries","type":[{"name":"Reference","namespace":"Flow\\ETL\\Row","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":true,"default_value":null}],"return_type":[{"name":"Select","namespace":"Flow\\ETL\\Transformation","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"CORE","type":"TRANSFORMER"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DSL\/functions.php","start_line_in_file":636,"slug":"drop","name":"drop","namespace":"Flow\\ETL\\DSL","parameters":[{"name":"entries","type":[{"name":"Reference","namespace":"Flow\\ETL\\Row","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":true,"default_value":null}],"return_type":[{"name":"Drop","namespace":"Flow\\ETL\\Transformation","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"CORE","type":"TRANSFORMER"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DSL\/functions.php","start_line_in_file":642,"slug":"add-row-index","name":"add_row_index","namespace":"Flow\\ETL\\DSL","parameters":[{"name":"column","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"'index'"},{"name":"startFrom","type":[{"name":"StartFrom","namespace":"Flow\\ETL\\Transformation\\AddRowIndex","is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"Flow\\ETL\\Transformation\\AddRowIndex\\StartFrom::..."}],"return_type":[{"name":"AddRowIndex","namespace":"Flow\\ETL\\Transformation","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"CORE","type":"TRANSFORMER"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DSL\/functions.php","start_line_in_file":651,"slug":"batch-size","name":"batch_size","namespace":"Flow\\ETL\\DSL","parameters":[{"name":"size","type":[{"name":"int","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"BatchSize","namespace":"Flow\\ETL\\Transformation","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"CORE","type":"TRANSFORMER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIEBwYXJhbSBpbnQ8MSwgbWF4PiAkc2l6ZQogKi8="},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DSL\/functions.php","start_line_in_file":657,"slug":"limit","name":"limit","namespace":"Flow\\ETL\\DSL","parameters":[{"name":"limit","type":[{"name":"int","namespace":null,"is_nullable":true,"is_variadic":false}],"has_default_value":false,"is_nullable":true,"is_variadic":false,"default_value":null}],"return_type":[{"name":"Limit","namespace":"Flow\\ETL\\Transformation","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"CORE","type":"TRANSFORMER"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DSL\/functions.php","start_line_in_file":666,"slug":"mask-columns","name":"mask_columns","namespace":"Flow\\ETL\\DSL","parameters":[{"name":"columns","type":[{"name":"array","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"[]"},{"name":"mask","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"'******'"}],"return_type":[{"name":"MaskColumns","namespace":"Flow\\ETL\\Transformation","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"CORE","type":"TRANSFORMER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIEBwYXJhbSBhcnJheTxpbnQsIHN0cmluZz4gJGNvbHVtbnMKICov"},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DSL\/functions.php","start_line_in_file":672,"slug":"optional","name":"optional","namespace":"Flow\\ETL\\DSL","parameters":[{"name":"function","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"Optional","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"CORE","type":"SCALAR_FUNCTION"}}],"scalar_function_chain":true,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DSL\/functions.php","start_line_in_file":679,"slug":"lit","name":"lit","namespace":"Flow\\ETL\\DSL","parameters":[{"name":"value","type":[{"name":"mixed","namespace":null,"is_nullable":true,"is_variadic":false}],"has_default_value":false,"is_nullable":true,"is_variadic":false,"default_value":null}],"return_type":[{"name":"Literal","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"CORE","type":"SCALAR_FUNCTION"}},{"name":"DocumentationExample","namespace":"Flow\\Documentation\\Attribute","arguments":{"topic":"columns","example":"create"}}],"scalar_function_chain":true,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DSL\/functions.php","start_line_in_file":685,"slug":"exists","name":"exists","namespace":"Flow\\ETL\\DSL","parameters":[{"name":"ref","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"Exists","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"CORE","type":"SCALAR_FUNCTION"}}],"scalar_function_chain":true,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DSL\/functions.php","start_line_in_file":691,"slug":"when","name":"when","namespace":"Flow\\ETL\\DSL","parameters":[{"name":"condition","type":[{"name":"mixed","namespace":null,"is_nullable":true,"is_variadic":false}],"has_default_value":false,"is_nullable":true,"is_variadic":false,"default_value":null},{"name":"then","type":[{"name":"mixed","namespace":null,"is_nullable":true,"is_variadic":false}],"has_default_value":false,"is_nullable":true,"is_variadic":false,"default_value":null},{"name":"else","type":[{"name":"mixed","namespace":null,"is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"null"}],"return_type":[{"name":"When","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"CORE","type":"SCALAR_FUNCTION"}}],"scalar_function_chain":true,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DSL\/functions.php","start_line_in_file":700,"slug":"structure-get","name":"structure_get","namespace":"Flow\\ETL\\DSL","parameters":[{"name":"ref","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"path","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"ArrayGet","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"CORE","type":"SCALAR_FUNCTION"}}],"scalar_function_chain":true,"doc_comment":"LyoqCiAqIEFsaWFzIGZvciBgYXJyYXlfZ2V0YC4KICov"},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DSL\/functions.php","start_line_in_file":706,"slug":"array-get","name":"array_get","namespace":"Flow\\ETL\\DSL","parameters":[{"name":"ref","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"path","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"ArrayGet","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"CORE","type":"SCALAR_FUNCTION"}}],"scalar_function_chain":true,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DSL\/functions.php","start_line_in_file":717,"slug":"structure-get-collection","name":"structure_get_collection","namespace":"Flow\\ETL\\DSL","parameters":[{"name":"ref","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"keys","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"array","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"ArrayGetCollection","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"CORE","type":"SCALAR_FUNCTION"}}],"scalar_function_chain":true,"doc_comment":"LyoqCiAqIEFsaWFzIGZvciBgYXJyYXlfZ2V0X2NvbGxlY3Rpb25gLgogKgogKiBAcGFyYW0gYXJyYXk8YXJyYXkta2V5LCBtaXhlZD58U2NhbGFyRnVuY3Rpb24gJGtleXMKICov"},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DSL\/functions.php","start_line_in_file":726,"slug":"array-get-collection","name":"array_get_collection","namespace":"Flow\\ETL\\DSL","parameters":[{"name":"ref","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"keys","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"array","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"ArrayGetCollection","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"CORE","type":"SCALAR_FUNCTION"}}],"scalar_function_chain":true,"doc_comment":"LyoqCiAqIEBwYXJhbSBhcnJheTxhcnJheS1rZXksIG1peGVkPnxTY2FsYXJGdW5jdGlvbiAka2V5cwogKi8="},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DSL\/functions.php","start_line_in_file":735,"slug":"structure-get-collection-first","name":"structure_get_collection_first","namespace":"Flow\\ETL\\DSL","parameters":[{"name":"ref","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"keys","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":true,"default_value":null}],"return_type":[{"name":"ArrayGetCollection","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"CORE","type":"SCALAR_FUNCTION"}}],"scalar_function_chain":true,"doc_comment":"LyoqCiAqIEFsaWFzIGZvciBgYXJyYXlfZ2V0X2NvbGxlY3Rpb25fZmlyc3RgLgogKi8="},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DSL\/functions.php","start_line_in_file":741,"slug":"array-get-collection-first","name":"array_get_collection_first","namespace":"Flow\\ETL\\DSL","parameters":[{"name":"ref","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"keys","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":true,"default_value":null}],"return_type":[{"name":"ArrayGetCollection","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"CORE","type":"SCALAR_FUNCTION"}}],"scalar_function_chain":true,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DSL\/functions.php","start_line_in_file":750,"slug":"array-exists","name":"array_exists","namespace":"Flow\\ETL\\DSL","parameters":[{"name":"ref","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"array","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"path","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"ArrayPathExists","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"CORE","type":"SCALAR_FUNCTION"}}],"scalar_function_chain":true,"doc_comment":"LyoqCiAqIEBwYXJhbSBhcnJheTxhcnJheS1rZXksIG1peGVkPnxTY2FsYXJGdW5jdGlvbiAkcmVmCiAqLw=="},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DSL\/functions.php","start_line_in_file":760,"slug":"array-merge","name":"array_merge","namespace":"Flow\\ETL\\DSL","parameters":[{"name":"left","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"array","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"right","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"array","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"ArrayMerge","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"CORE","type":"SCALAR_FUNCTION"}}],"scalar_function_chain":true,"doc_comment":"LyoqCiAqIEBwYXJhbSBhcnJheTxhcnJheS1rZXksIG1peGVkPnxTY2FsYXJGdW5jdGlvbiAkbGVmdAogKiBAcGFyYW0gYXJyYXk8YXJyYXkta2V5LCBtaXhlZD58U2NhbGFyRnVuY3Rpb24gJHJpZ2h0CiAqLw=="},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DSL\/functions.php","start_line_in_file":769,"slug":"array-merge-collection","name":"array_merge_collection","namespace":"Flow\\ETL\\DSL","parameters":[{"name":"array","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"array","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"ArrayMergeCollection","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"CORE","type":"SCALAR_FUNCTION"}}],"scalar_function_chain":true,"doc_comment":"LyoqCiAqIEBwYXJhbSBhcnJheTxhcnJheS1rZXksIG1peGVkPnxTY2FsYXJGdW5jdGlvbiAkYXJyYXkKICov"},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DSL\/functions.php","start_line_in_file":775,"slug":"array-key-rename","name":"array_key_rename","namespace":"Flow\\ETL\\DSL","parameters":[{"name":"ref","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"path","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"newName","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"ArrayKeyRename","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"CORE","type":"SCALAR_FUNCTION"}}],"scalar_function_chain":true,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DSL\/functions.php","start_line_in_file":781,"slug":"array-keys-style-convert","name":"array_keys_style_convert","namespace":"Flow\\ETL\\DSL","parameters":[{"name":"ref","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"style","type":[{"name":"StringStyles","namespace":"Flow\\ETL\\String","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"Flow\\ETL\\String\\StringStyles::..."}],"return_type":[{"name":"ArrayKeysStyleConvert","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"CORE","type":"SCALAR_FUNCTION"}}],"scalar_function_chain":true,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DSL\/functions.php","start_line_in_file":789,"slug":"array-sort","name":"array_sort","namespace":"Flow\\ETL\\DSL","parameters":[{"name":"function","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"sort_function","type":[{"name":"Sort","namespace":"Flow\\ETL\\Function\\ArraySort","is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"null"},{"name":"flags","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"int","namespace":null,"is_nullable":false,"is_variadic":false},{"name":"null","namespace":null,"is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"null"},{"name":"recursive","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"bool","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"true"}],"return_type":[{"name":"ArraySort","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"CORE","type":"SCALAR_FUNCTION"}}],"scalar_function_chain":true,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DSL\/functions.php","start_line_in_file":802,"slug":"array-reverse","name":"array_reverse","namespace":"Flow\\ETL\\DSL","parameters":[{"name":"function","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"array","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"preserveKeys","type":[{"name":"bool","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"false"}],"return_type":[{"name":"ArrayReverse","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"CORE","type":"SCALAR_FUNCTION"}}],"scalar_function_chain":true,"doc_comment":"LyoqCiAqIEBwYXJhbSBhcnJheTxhcnJheS1rZXksIG1peGVkPnxTY2FsYXJGdW5jdGlvbiAkZnVuY3Rpb24KICov"},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DSL\/functions.php","start_line_in_file":808,"slug":"now","name":"now","namespace":"Flow\\ETL\\DSL","parameters":[{"name":"time_zone","type":[{"name":"DateTimeZone","namespace":"","is_nullable":false,"is_variadic":false},{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"DateTimeZone::..."}],"return_type":[{"name":"Now","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"CORE","type":"SCALAR_FUNCTION"}}],"scalar_function_chain":true,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DSL\/functions.php","start_line_in_file":814,"slug":"between","name":"between","namespace":"Flow\\ETL\\DSL","parameters":[{"name":"value","type":[{"name":"mixed","namespace":null,"is_nullable":true,"is_variadic":false}],"has_default_value":false,"is_nullable":true,"is_variadic":false,"default_value":null},{"name":"lower_bound","type":[{"name":"mixed","namespace":null,"is_nullable":true,"is_variadic":false}],"has_default_value":false,"is_nullable":true,"is_variadic":false,"default_value":null},{"name":"upper_bound","type":[{"name":"mixed","namespace":null,"is_nullable":true,"is_variadic":false}],"has_default_value":false,"is_nullable":true,"is_variadic":false,"default_value":null},{"name":"boundary","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"Boundary","namespace":"Flow\\ETL\\Function\\Between","is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"Flow\\ETL\\Function\\Between\\Boundary::..."}],"return_type":[{"name":"Between","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"CORE","type":"SCALAR_FUNCTION"}}],"scalar_function_chain":true,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DSL\/functions.php","start_line_in_file":824,"slug":"to-date-time","name":"to_date_time","namespace":"Flow\\ETL\\DSL","parameters":[{"name":"ref","type":[{"name":"mixed","namespace":null,"is_nullable":true,"is_variadic":false}],"has_default_value":false,"is_nullable":true,"is_variadic":false,"default_value":null},{"name":"format","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"'Y-m-d H:i:s'"},{"name":"timeZone","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"DateTimeZone","namespace":"","is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"DateTimeZone::..."}],"return_type":[{"name":"ToDateTime","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"CORE","type":"SCALAR_FUNCTION"}}],"scalar_function_chain":true,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DSL\/functions.php","start_line_in_file":833,"slug":"to-date","name":"to_date","namespace":"Flow\\ETL\\DSL","parameters":[{"name":"ref","type":[{"name":"mixed","namespace":null,"is_nullable":true,"is_variadic":false}],"has_default_value":false,"is_nullable":true,"is_variadic":false,"default_value":null},{"name":"format","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"'Y-m-d'"},{"name":"timeZone","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"DateTimeZone","namespace":"","is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"DateTimeZone::..."}],"return_type":[{"name":"ToDate","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"CORE","type":"SCALAR_FUNCTION"}}],"scalar_function_chain":true,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DSL\/functions.php","start_line_in_file":842,"slug":"date-time-format","name":"date_time_format","namespace":"Flow\\ETL\\DSL","parameters":[{"name":"ref","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"format","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"DateTimeFormat","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"CORE","type":"SCALAR_FUNCTION"}}],"scalar_function_chain":true,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DSL\/functions.php","start_line_in_file":848,"slug":"split","name":"split","namespace":"Flow\\ETL\\DSL","parameters":[{"name":"value","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"separator","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"limit","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"int","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"9223372036854775807"}],"return_type":[{"name":"Split","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"CORE","type":"SCALAR_FUNCTION"}}],"scalar_function_chain":true,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DSL\/functions.php","start_line_in_file":861,"slug":"combine","name":"combine","namespace":"Flow\\ETL\\DSL","parameters":[{"name":"keys","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"array","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"values","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"array","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"Combine","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"CORE","type":"SCALAR_FUNCTION"}}],"scalar_function_chain":true,"doc_comment":"LyoqCiAqIEBwYXJhbSBhcnJheTxhcnJheS1rZXksIG1peGVkPnxTY2FsYXJGdW5jdGlvbiAka2V5cwogKiBAcGFyYW0gYXJyYXk8YXJyYXkta2V5LCBtaXhlZD58U2NhbGFyRnVuY3Rpb24gJHZhbHVlcwogKi8="},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DSL\/functions.php","start_line_in_file":870,"slug":"concat","name":"concat","namespace":"Flow\\ETL\\DSL","parameters":[{"name":"functions","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":true,"default_value":null}],"return_type":[{"name":"Concat","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"CORE","type":"SCALAR_FUNCTION"}}],"scalar_function_chain":true,"doc_comment":"LyoqCiAqIENvbmNhdCBhbGwgdmFsdWVzLiBJZiB5b3Ugd2FudCB0byBjb25jYXRlbmF0ZSB2YWx1ZXMgd2l0aCBzZXBhcmF0b3IgdXNlIGNvbmNhdF93cyBmdW5jdGlvbi4KICov"},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DSL\/functions.php","start_line_in_file":879,"slug":"concat-ws","name":"concat_ws","namespace":"Flow\\ETL\\DSL","parameters":[{"name":"separator","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"functions","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":true,"default_value":null}],"return_type":[{"name":"ConcatWithSeparator","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"CORE","type":"SCALAR_FUNCTION"}}],"scalar_function_chain":true,"doc_comment":"LyoqCiAqIENvbmNhdCBhbGwgdmFsdWVzIHdpdGggc2VwYXJhdG9yLgogKi8="},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DSL\/functions.php","start_line_in_file":885,"slug":"hash","name":"hash","namespace":"Flow\\ETL\\DSL","parameters":[{"name":"value","type":[{"name":"mixed","namespace":null,"is_nullable":true,"is_variadic":false}],"has_default_value":false,"is_nullable":true,"is_variadic":false,"default_value":null},{"name":"algorithm","type":[{"name":"Algorithm","namespace":"Flow\\ETL\\Hash","is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"Flow\\ETL\\Hash\\NativePHPHash::..."}],"return_type":[{"name":"Hash","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"CORE","type":"SCALAR_FUNCTION"}}],"scalar_function_chain":true,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DSL\/functions.php","start_line_in_file":894,"slug":"cast","name":"cast","namespace":"Flow\\ETL\\DSL","parameters":[{"name":"value","type":[{"name":"mixed","namespace":null,"is_nullable":true,"is_variadic":false}],"has_default_value":false,"is_nullable":true,"is_variadic":false,"default_value":null},{"name":"type","type":[{"name":"Type","namespace":"Flow\\Types","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"Cast","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"CORE","type":"SCALAR_FUNCTION"}}],"scalar_function_chain":true,"doc_comment":"LyoqCiAqIEBwYXJhbSBcRmxvd1xUeXBlc1xUeXBlPG1peGVkPnxzdHJpbmcgJHR5cGUKICov"},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DSL\/functions.php","start_line_in_file":900,"slug":"coalesce","name":"coalesce","namespace":"Flow\\ETL\\DSL","parameters":[{"name":"values","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":true,"default_value":null}],"return_type":[{"name":"Coalesce","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"CORE","type":"SCALAR_FUNCTION"}}],"scalar_function_chain":true,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DSL\/functions.php","start_line_in_file":906,"slug":"enum-name","name":"enum_name","namespace":"Flow\\ETL\\DSL","parameters":[{"name":"value","type":[{"name":"mixed","namespace":null,"is_nullable":true,"is_variadic":false}],"has_default_value":false,"is_nullable":true,"is_variadic":false,"default_value":null}],"return_type":[{"name":"EnumName","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"CORE","type":"SCALAR_FUNCTION"}}],"scalar_function_chain":true,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DSL\/functions.php","start_line_in_file":912,"slug":"enum-value","name":"enum_value","namespace":"Flow\\ETL\\DSL","parameters":[{"name":"value","type":[{"name":"mixed","namespace":null,"is_nullable":true,"is_variadic":false}],"has_default_value":false,"is_nullable":true,"is_variadic":false,"default_value":null}],"return_type":[{"name":"EnumValue","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"CORE","type":"SCALAR_FUNCTION"}}],"scalar_function_chain":true,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DSL\/functions.php","start_line_in_file":918,"slug":"count","name":"count","namespace":"Flow\\ETL\\DSL","parameters":[{"name":"function","type":[{"name":"Reference","namespace":"Flow\\ETL\\Row","is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"null"}],"return_type":[{"name":"Count","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"CORE","type":"AGGREGATING_FUNCTION"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DSL\/functions.php","start_line_in_file":930,"slug":"call","name":"call","namespace":"Flow\\ETL\\DSL","parameters":[{"name":"callable","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"return_type","type":[{"name":"Type","namespace":"Flow\\Types","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"parameters","type":[{"name":"array","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"[]"}],"return_type":[{"name":"CallUserFunc","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"CORE","type":"SCALAR_FUNCTION"}}],"scalar_function_chain":true,"doc_comment":"LyoqCiAqIENhbGxzIGEgdXNlci1kZWZpbmVkIGZ1bmN0aW9uIHdpdGggdGhlIGdpdmVuIHBhcmFtZXRlcnMuCiAqCiAqIEBwYXJhbSBUeXBlPG1peGVkPiAkcmV0dXJuX3R5cGUKICogQHBhcmFtIGFycmF5PG1peGVkPiAkcGFyYW1ldGVycwogKi8="},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DSL\/functions.php","start_line_in_file":957,"slug":"array-unpack","name":"array_unpack","namespace":"Flow\\ETL\\DSL","parameters":[{"name":"array","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"array","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"schema","type":[{"name":"Schema","namespace":"Flow\\ETL","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"ArrayUnpack","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"CORE","type":"SCALAR_FUNCTION"}}],"scalar_function_chain":true,"doc_comment":"LyoqCiAqIFVucGFja3MgZWFjaCBlbGVtZW50IG9mIGFuIGFycmF5IGludG8gYSBuZXcgZW50cnksIHVzaW5nIHRoZSBhcnJheSBrZXkgYXMgdGhlIGVudHJ5IG5hbWUuCiAqCiAqIEJlZm9yZToKICogKy0tKy0tLS0tLS0tLS0tLS0tLS0tLS0rCiAqIHxpZHwgICAgICAgICAgICAgIGFycmF5fAogKiArLS0rLS0tLS0tLS0tLS0tLS0tLS0tLSsKICogfCAxfHsiYSI6MSwiYiI6MiwiYyI6M318CiAqIHwgMnx7ImQiOjQsImUiOjUsImYiOjZ9fAogKiArLS0rLS0tLS0tLS0tLS0tLS0tLS0tLSsKICoKICogQWZ0ZXI6CiAqICstLSstLS0tLSstLS0tLSstLS0tLSstLS0tLSstLS0tLSsKICogfGlkfGFyci5ifGFyci5jfGFyci5kfGFyci5lfGFyci5mfAogKiArLS0rLS0tLS0rLS0tLS0rLS0tLS0rLS0tLS0rLS0tLS0rCiAqIHwgMXwgICAgMnwgICAgM3wgICAgIHwgICAgIHwgICAgIHwKICogfCAyfCAgICAgfCAgICAgfCAgICA0fCAgICA1fCAgICA2fAogKiArLS0rLS0tLS0rLS0tLS0rLS0tLS0rLS0tLS0rLS0tLS0rCiAqCiAqIEBwYXJhbSBhcnJheTxhcnJheS1rZXksIG1peGVkPnxTY2FsYXJGdW5jdGlvbiAkYXJyYXkKICov"},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DSL\/functions.php","start_line_in_file":987,"slug":"array-expand","name":"array_expand","namespace":"Flow\\ETL\\DSL","parameters":[{"name":"function","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"expand","type":[{"name":"ArrayExpand","namespace":"Flow\\ETL\\Function\\ArrayExpand","is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"Flow\\ETL\\Function\\ArrayExpand\\ArrayExpand::..."}],"return_type":[{"name":"ArrayExpand","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"CORE","type":"SCALAR_FUNCTION"}}],"scalar_function_chain":true,"doc_comment":"LyoqCiAqIEV4cGFuZHMgZWFjaCB2YWx1ZSBpbnRvIGVudHJ5LCBpZiB0aGVyZSBhcmUgbW9yZSB0aGFuIG9uZSB2YWx1ZSwgbXVsdGlwbGUgcm93cyB3aWxsIGJlIGNyZWF0ZWQuCiAqIEFycmF5IGtleXMgYXJlIGlnbm9yZWQsIG9ubHkgdmFsdWVzIGFyZSB1c2VkIHRvIGNyZWF0ZSBuZXcgcm93cy4KICogTmVzdGVkIGluIGFub3RoZXIgZnVuY3Rpb24gKHN0cnVjdHVyZSgpLCBjb25jYXQoKSwgLi4uKSBpdCBzdGlsbCBnaXZlcyBvbmUgcm93IHBlciBlbGVtZW50LiBTZXZlcmFsCiAqIGV4cGFuZHMgaW4gb25lIGV4cHJlc3Npb24gYXJlIHppcHBlZCB0byB0aGUgbG9uZ2VzdCBsaXN0OyBhIHNob3J0ZXIgb25lIGdpdmVzIG51bGwsIHNvIGl0cyBlbGVtZW50CiAqIHR5cGUgYmVjb21lcyBudWxsYWJsZS4gSXQgaXMgcmVmdXNlZCBpbnNpZGUgYW5vdGhlciBhcnJheV9leHBhbmQoKSBhbmQgaW4gZmlsdGVyKCksIHVudGlsKCksCiAqIGR1cGxpY2F0ZVJvdygpLCBhZ2dyZWdhdGUoKSwgb3ZlcigpIGFuZCBvbkVhY2goKS4KICoKICogQmVmb3JlOgogKiAgICstLSstLS0tLS0tLS0tLS0tLS0tLS0tKwogKiAgIHxpZHwgICAgICAgICAgICAgIGFycmF5fAogKiAgICstLSstLS0tLS0tLS0tLS0tLS0tLS0tKwogKiAgIHwgMXx7ImEiOjEsImIiOjIsImMiOjN9fAogKiAgICstLSstLS0tLS0tLS0tLS0tLS0tLS0tKwogKgogKiBBZnRlcjoKICogICArLS0rLS0tLS0tLS0rCiAqICAgfGlkfGV4cGFuZGVkfAogKiAgICstLSstLS0tLS0tLSsKICogICB8IDF8ICAgICAgIDF8CiAqICAgfCAxfCAgICAgICAyfAogKiAgIHwgMXwgICAgICAgM3wKICogICArLS0rLS0tLS0tLS0rCiAqLw=="},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DSL\/functions.php","start_line_in_file":993,"slug":"size","name":"size","namespace":"Flow\\ETL\\DSL","parameters":[{"name":"value","type":[{"name":"mixed","namespace":null,"is_nullable":true,"is_variadic":false}],"has_default_value":false,"is_nullable":true,"is_variadic":false,"default_value":null}],"return_type":[{"name":"Size","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"CORE","type":"SCALAR_FUNCTION"}}],"scalar_function_chain":true,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DSL\/functions.php","start_line_in_file":999,"slug":"uuid-v4","name":"uuid_v4","namespace":"Flow\\ETL\\DSL","parameters":[],"return_type":[{"name":"Uuid","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"CORE","type":"SCALAR_FUNCTION"}}],"scalar_function_chain":true,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DSL\/functions.php","start_line_in_file":1005,"slug":"uuid-v7","name":"uuid_v7","namespace":"Flow\\ETL\\DSL","parameters":[{"name":"value","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"DateTimeInterface","namespace":"","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"Uuid","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"CORE","type":"SCALAR_FUNCTION"}}],"scalar_function_chain":true,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DSL\/functions.php","start_line_in_file":1011,"slug":"ulid","name":"ulid","namespace":"Flow\\ETL\\DSL","parameters":[{"name":"value","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false},{"name":"null","namespace":null,"is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"null"}],"return_type":[{"name":"Ulid","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"CORE","type":"SCALAR_FUNCTION"}}],"scalar_function_chain":true,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DSL\/functions.php","start_line_in_file":1017,"slug":"lower","name":"lower","namespace":"Flow\\ETL\\DSL","parameters":[{"name":"value","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"ToLower","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"CORE","type":"SCALAR_FUNCTION"}}],"scalar_function_chain":true,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DSL\/functions.php","start_line_in_file":1023,"slug":"capitalize","name":"capitalize","namespace":"Flow\\ETL\\DSL","parameters":[{"name":"value","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"Capitalize","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"CORE","type":"SCALAR_FUNCTION"}}],"scalar_function_chain":true,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DSL\/functions.php","start_line_in_file":1029,"slug":"upper","name":"upper","namespace":"Flow\\ETL\\DSL","parameters":[{"name":"value","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"ToUpper","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"CORE","type":"SCALAR_FUNCTION"}}],"scalar_function_chain":true,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DSL\/functions.php","start_line_in_file":1035,"slug":"all","name":"all","namespace":"Flow\\ETL\\DSL","parameters":[{"name":"functions","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":true,"default_value":null}],"return_type":[{"name":"All","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"CORE","type":"SCALAR_FUNCTION"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DSL\/functions.php","start_line_in_file":1041,"slug":"any","name":"any","namespace":"Flow\\ETL\\DSL","parameters":[{"name":"values","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":true,"default_value":null}],"return_type":[{"name":"Any","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"CORE","type":"SCALAR_FUNCTION"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DSL\/functions.php","start_line_in_file":1047,"slug":"not","name":"not","namespace":"Flow\\ETL\\DSL","parameters":[{"name":"value","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"Not","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"CORE","type":"SCALAR_FUNCTION"}}],"scalar_function_chain":true,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DSL\/functions.php","start_line_in_file":1053,"slug":"to-timezone","name":"to_timezone","namespace":"Flow\\ETL\\DSL","parameters":[{"name":"value","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"DateTimeInterface","namespace":"","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"timeZone","type":[{"name":"DateTimeZone","namespace":"","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"ToTimeZone","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"CORE","type":"SCALAR_FUNCTION"}}],"scalar_function_chain":true,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DSL\/functions.php","start_line_in_file":1059,"slug":"ignore-error-handler","name":"ignore_error_handler","namespace":"Flow\\ETL\\DSL","parameters":[],"return_type":[{"name":"IgnoreError","namespace":"Flow\\ETL\\ErrorHandler","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"CORE","type":"DATA_FRAME"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DSL\/functions.php","start_line_in_file":1065,"slug":"skip-rows-handler","name":"skip_rows_handler","namespace":"Flow\\ETL\\DSL","parameters":[],"return_type":[{"name":"SkipRows","namespace":"Flow\\ETL\\ErrorHandler","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"CORE","type":"DATA_FRAME"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DSL\/functions.php","start_line_in_file":1071,"slug":"throw-error-handler","name":"throw_error_handler","namespace":"Flow\\ETL\\DSL","parameters":[],"return_type":[{"name":"ThrowError","namespace":"Flow\\ETL\\ErrorHandler","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"CORE","type":"DATA_FRAME"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DSL\/functions.php","start_line_in_file":1077,"slug":"regex-replace","name":"regex_replace","namespace":"Flow\\ETL\\DSL","parameters":[{"name":"pattern","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"replacement","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"subject","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"limit","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"int","namespace":null,"is_nullable":false,"is_variadic":false},{"name":"null","namespace":null,"is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"null"}],"return_type":[{"name":"RegexReplace","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"CORE","type":"SCALAR_FUNCTION"}}],"scalar_function_chain":true,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DSL\/functions.php","start_line_in_file":1087,"slug":"regex-match-all","name":"regex_match_all","namespace":"Flow\\ETL\\DSL","parameters":[{"name":"pattern","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"subject","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"flags","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"int","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"0"},{"name":"offset","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"int","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"0"}],"return_type":[{"name":"RegexMatchAll","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"CORE","type":"SCALAR_FUNCTION"}}],"scalar_function_chain":true,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DSL\/functions.php","start_line_in_file":1097,"slug":"regex-match","name":"regex_match","namespace":"Flow\\ETL\\DSL","parameters":[{"name":"pattern","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"subject","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"flags","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"int","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"0"},{"name":"offset","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"int","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"0"}],"return_type":[{"name":"RegexMatch","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"CORE","type":"SCALAR_FUNCTION"}}],"scalar_function_chain":true,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DSL\/functions.php","start_line_in_file":1107,"slug":"regex","name":"regex","namespace":"Flow\\ETL\\DSL","parameters":[{"name":"pattern","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"subject","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"flags","type":[{"name":"int","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"0"},{"name":"offset","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"int","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"0"}],"return_type":[{"name":"Regex","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"CORE","type":"SCALAR_FUNCTION"}}],"scalar_function_chain":true,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DSL\/functions.php","start_line_in_file":1117,"slug":"regex-all","name":"regex_all","namespace":"Flow\\ETL\\DSL","parameters":[{"name":"pattern","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"subject","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"flags","type":[{"name":"int","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"0"},{"name":"offset","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"int","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"0"}],"return_type":[{"name":"RegexAll","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"CORE","type":"SCALAR_FUNCTION"}}],"scalar_function_chain":true,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DSL\/functions.php","start_line_in_file":1127,"slug":"sprintf","name":"sprintf","namespace":"Flow\\ETL\\DSL","parameters":[{"name":"format","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"args","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false},{"name":"int","namespace":null,"is_nullable":false,"is_variadic":false},{"name":"float","namespace":null,"is_nullable":false,"is_variadic":false},{"name":"null","namespace":null,"is_nullable":true,"is_variadic":false}],"has_default_value":false,"is_nullable":true,"is_variadic":true,"default_value":null}],"return_type":[{"name":"Sprintf","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"CORE","type":"SCALAR_FUNCTION"}}],"scalar_function_chain":true,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DSL\/functions.php","start_line_in_file":1133,"slug":"sanitize","name":"sanitize","namespace":"Flow\\ETL\\DSL","parameters":[{"name":"value","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"placeholder","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"'*'"},{"name":"skipCharacters","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"int","namespace":null,"is_nullable":false,"is_variadic":false},{"name":"null","namespace":null,"is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"null"}],"return_type":[{"name":"Sanitize","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"CORE","type":"SCALAR_FUNCTION"}}],"scalar_function_chain":true,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DSL\/functions.php","start_line_in_file":1142,"slug":"round","name":"round","namespace":"Flow\\ETL\\DSL","parameters":[{"name":"value","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"int","namespace":null,"is_nullable":false,"is_variadic":false},{"name":"float","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"precision","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"int","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"2"},{"name":"mode","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"int","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"1"}],"return_type":[{"name":"Round","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"CORE","type":"SCALAR_FUNCTION"}}],"scalar_function_chain":true,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DSL\/functions.php","start_line_in_file":1151,"slug":"number-format","name":"number_format","namespace":"Flow\\ETL\\DSL","parameters":[{"name":"value","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"int","namespace":null,"is_nullable":false,"is_variadic":false},{"name":"float","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"decimals","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"int","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"2"},{"name":"decimal_separator","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"'.'"},{"name":"thousands_separator","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"','"}],"return_type":[{"name":"NumberFormat","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"CORE","type":"SCALAR_FUNCTION"}}],"scalar_function_chain":true,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DSL\/functions.php","start_line_in_file":1164,"slug":"array-to-rows","name":"array_to_rows","namespace":"Flow\\ETL\\DSL","parameters":[{"name":"data","type":[{"name":"array","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"schema","type":[{"name":"Schema","namespace":"Flow\\ETL","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"backend","type":[{"name":"Backend","namespace":"Flow\\ETL\\Column","is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"Flow\\ETL\\Column\\AdaptiveBackend::..."}],"return_type":[{"name":"Rows","namespace":"Flow\\ETL","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"CORE","type":"DATA_FRAME"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIEBwYXJhbSBhcnJheTxhcnJheTxtaXhlZD4+fGFycmF5PG1peGVkfHN0cmluZz4gJGRhdGEKICov"},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DSL\/functions.php","start_line_in_file":1172,"slug":"rank","name":"rank","namespace":"Flow\\ETL\\DSL","parameters":[],"return_type":[{"name":"Rank","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"CORE","type":"WINDOW_FUNCTION"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DSL\/functions.php","start_line_in_file":1178,"slug":"dens-rank","name":"dens_rank","namespace":"Flow\\ETL\\DSL","parameters":[],"return_type":[{"name":"DenseRank","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"CORE","type":"WINDOW_FUNCTION"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DSL\/functions.php","start_line_in_file":1184,"slug":"dense-rank","name":"dense_rank","namespace":"Flow\\ETL\\DSL","parameters":[],"return_type":[{"name":"DenseRank","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"CORE","type":"WINDOW_FUNCTION"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DSL\/functions.php","start_line_in_file":1190,"slug":"average","name":"average","namespace":"Flow\\ETL\\DSL","parameters":[{"name":"ref","type":[{"name":"Reference","namespace":"Flow\\ETL\\Row","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"scale","type":[{"name":"int","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"2"},{"name":"rounding","type":[{"name":"Rounding","namespace":"Flow\\Calculator","is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"Flow\\Calculator\\Rounding::..."},{"name":"exact","type":[{"name":"bool","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"false"}],"return_type":[{"name":"Average","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"CORE","type":"AGGREGATING_FUNCTION"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DSL\/functions.php","start_line_in_file":1200,"slug":"greatest","name":"greatest","namespace":"Flow\\ETL\\DSL","parameters":[{"name":"values","type":[{"name":"mixed","namespace":null,"is_nullable":true,"is_variadic":false}],"has_default_value":false,"is_nullable":true,"is_variadic":true,"default_value":null}],"return_type":[{"name":"Greatest","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"CORE","type":"SCALAR_FUNCTION"}}],"scalar_function_chain":true,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DSL\/functions.php","start_line_in_file":1206,"slug":"least","name":"least","namespace":"Flow\\ETL\\DSL","parameters":[{"name":"values","type":[{"name":"mixed","namespace":null,"is_nullable":true,"is_variadic":false}],"has_default_value":false,"is_nullable":true,"is_variadic":true,"default_value":null}],"return_type":[{"name":"Least","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"CORE","type":"SCALAR_FUNCTION"}}],"scalar_function_chain":true,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DSL\/functions.php","start_line_in_file":1212,"slug":"collect","name":"collect","namespace":"Flow\\ETL\\DSL","parameters":[{"name":"ref","type":[{"name":"Reference","namespace":"Flow\\ETL\\Row","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"Collect","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"CORE","type":"AGGREGATING_FUNCTION"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DSL\/functions.php","start_line_in_file":1218,"slug":"string-agg","name":"string_agg","namespace":"Flow\\ETL\\DSL","parameters":[{"name":"ref","type":[{"name":"Reference","namespace":"Flow\\ETL\\Row","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"separator","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"', '"},{"name":"sort","type":[{"name":"SortOrder","namespace":"Flow\\ETL\\Row","is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"null"}],"return_type":[{"name":"StringAggregate","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"CORE","type":"AGGREGATING_FUNCTION"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DSL\/functions.php","start_line_in_file":1224,"slug":"collect-unique","name":"collect_unique","namespace":"Flow\\ETL\\DSL","parameters":[{"name":"ref","type":[{"name":"Reference","namespace":"Flow\\ETL\\Row","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"CollectUnique","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"CORE","type":"AGGREGATING_FUNCTION"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DSL\/functions.php","start_line_in_file":1230,"slug":"window","name":"window","namespace":"Flow\\ETL\\DSL","parameters":[],"return_type":[{"name":"Window","namespace":"Flow\\ETL","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"CORE","type":"DATA_FRAME"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DSL\/functions.php","start_line_in_file":1236,"slug":"unbounded-preceding","name":"unbounded_preceding","namespace":"Flow\\ETL\\DSL","parameters":[],"return_type":[{"name":"FrameBound","namespace":"Flow\\ETL\\Window","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"CORE","type":"DATA_FRAME"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DSL\/functions.php","start_line_in_file":1242,"slug":"preceding","name":"preceding","namespace":"Flow\\ETL\\DSL","parameters":[{"name":"offset","type":[{"name":"int","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"FrameBound","namespace":"Flow\\ETL\\Window","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"CORE","type":"DATA_FRAME"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DSL\/functions.php","start_line_in_file":1248,"slug":"current-row","name":"current_row","namespace":"Flow\\ETL\\DSL","parameters":[],"return_type":[{"name":"FrameBound","namespace":"Flow\\ETL\\Window","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"CORE","type":"DATA_FRAME"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DSL\/functions.php","start_line_in_file":1254,"slug":"following","name":"following","namespace":"Flow\\ETL\\DSL","parameters":[{"name":"offset","type":[{"name":"int","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"FrameBound","namespace":"Flow\\ETL\\Window","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"CORE","type":"DATA_FRAME"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DSL\/functions.php","start_line_in_file":1260,"slug":"unbounded-following","name":"unbounded_following","namespace":"Flow\\ETL\\DSL","parameters":[],"return_type":[{"name":"FrameBound","namespace":"Flow\\ETL\\Window","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"CORE","type":"DATA_FRAME"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DSL\/functions.php","start_line_in_file":1266,"slug":"sum","name":"sum","namespace":"Flow\\ETL\\DSL","parameters":[{"name":"ref","type":[{"name":"Reference","namespace":"Flow\\ETL\\Row","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"exact","type":[{"name":"bool","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"false"}],"return_type":[{"name":"Sum","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"CORE","type":"AGGREGATING_FUNCTION"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DSL\/functions.php","start_line_in_file":1272,"slug":"first","name":"first","namespace":"Flow\\ETL\\DSL","parameters":[{"name":"ref","type":[{"name":"Reference","namespace":"Flow\\ETL\\Row","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"First","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"CORE","type":"AGGREGATING_FUNCTION"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DSL\/functions.php","start_line_in_file":1278,"slug":"last","name":"last","namespace":"Flow\\ETL\\DSL","parameters":[{"name":"ref","type":[{"name":"Reference","namespace":"Flow\\ETL\\Row","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"Last","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"CORE","type":"AGGREGATING_FUNCTION"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DSL\/functions.php","start_line_in_file":1284,"slug":"max","name":"max","namespace":"Flow\\ETL\\DSL","parameters":[{"name":"ref","type":[{"name":"Reference","namespace":"Flow\\ETL\\Row","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"Max","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"CORE","type":"AGGREGATING_FUNCTION"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DSL\/functions.php","start_line_in_file":1290,"slug":"min","name":"min","namespace":"Flow\\ETL\\DSL","parameters":[{"name":"ref","type":[{"name":"Reference","namespace":"Flow\\ETL\\Row","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"Min","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"CORE","type":"AGGREGATING_FUNCTION"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DSL\/functions.php","start_line_in_file":1296,"slug":"row-number","name":"row_number","namespace":"Flow\\ETL\\DSL","parameters":[],"return_type":[{"name":"RowNumber","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"CORE","type":"DATA_FRAME"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DSL\/functions.php","start_line_in_file":1307,"slug":"schema","name":"schema","namespace":"Flow\\ETL\\DSL","parameters":[{"name":"definitions","type":[{"name":"Definition","namespace":"Flow\\ETL\\Schema","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":true,"default_value":null}],"return_type":[{"name":"Schema","namespace":"Flow\\ETL","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"CORE","type":"SCHEMA"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIEBwYXJhbSBEZWZpbml0aW9uPG1peGVkPiAuLi4kZGVmaW5pdGlvbnMKICoKICogQHJldHVybiBTY2hlbWEKICov"},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DSL\/functions.php","start_line_in_file":1316,"slug":"schema-to-json","name":"schema_to_json","namespace":"Flow\\ETL\\DSL","parameters":[{"name":"schema","type":[{"name":"Schema","namespace":"Flow\\ETL","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"pretty","type":[{"name":"bool","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"false"}],"return_type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"CORE","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIEBwYXJhbSBTY2hlbWEgJHNjaGVtYQogKi8="},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DSL\/functions.php","start_line_in_file":1325,"slug":"schema-to-php","name":"schema_to_php","namespace":"Flow\\ETL\\DSL","parameters":[{"name":"schema","type":[{"name":"Schema","namespace":"Flow\\ETL","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"valueFormatter","type":[{"name":"ValueFormatter","namespace":"Flow\\ETL\\Schema\\Formatter\\PHPFormatter","is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"Flow\\ETL\\Schema\\Formatter\\PHPFormatter\\ValueFormatter::..."},{"name":"typeFormatter","type":[{"name":"TypeFormatter","namespace":"Flow\\ETL\\Schema\\Formatter\\PHPFormatter","is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"Flow\\ETL\\Schema\\Formatter\\PHPFormatter\\TypeFormatter::..."}],"return_type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"CORE","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIEBwYXJhbSBTY2hlbWEgJHNjaGVtYQogKi8="},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DSL\/functions.php","start_line_in_file":1337,"slug":"schema-to-ascii","name":"schema_to_ascii","namespace":"Flow\\ETL\\DSL","parameters":[{"name":"schema","type":[{"name":"Schema","namespace":"Flow\\ETL","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"formatter","type":[{"name":"SchemaFormatter","namespace":"Flow\\ETL\\Schema","is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"null"}],"return_type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"CORE","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIEBwYXJhbSBTY2hlbWEgJHNjaGVtYQogKi8="},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DSL\/functions.php","start_line_in_file":1347,"slug":"schema-validate","name":"schema_validate","namespace":"Flow\\ETL\\DSL","parameters":[{"name":"expected","type":[{"name":"Schema","namespace":"Flow\\ETL","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"given","type":[{"name":"Schema","namespace":"Flow\\ETL","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"validator","type":[{"name":"SchemaValidator","namespace":"Flow\\ETL","is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"Flow\\ETL\\Schema\\Validator\\StrictValidator::..."}],"return_type":[{"name":"ValidationContext","namespace":"Flow\\ETL\\Schema\\Validator","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"CORE","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIEBwYXJhbSBTY2hlbWEgJGV4cGVjdGVkCiAqIEBwYXJhbSBTY2hlbWEgJGdpdmVuCiAqLw=="},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DSL\/functions.php","start_line_in_file":1356,"slug":"schema-evolving-validator","name":"schema_evolving_validator","namespace":"Flow\\ETL\\DSL","parameters":[],"return_type":[{"name":"EvolvingValidator","namespace":"Flow\\ETL\\Schema\\Validator","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"CORE","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DSL\/functions.php","start_line_in_file":1362,"slug":"schema-strict-validator","name":"schema_strict_validator","namespace":"Flow\\ETL\\DSL","parameters":[],"return_type":[{"name":"StrictValidator","namespace":"Flow\\ETL\\Schema\\Validator","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"CORE","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DSL\/functions.php","start_line_in_file":1368,"slug":"schema-selective-validator","name":"schema_selective_validator","namespace":"Flow\\ETL\\DSL","parameters":[],"return_type":[{"name":"SelectiveValidator","namespace":"Flow\\ETL\\Schema\\Validator","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"CORE","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DSL\/functions.php","start_line_in_file":1377,"slug":"schema-from-json","name":"schema_from_json","namespace":"Flow\\ETL\\DSL","parameters":[{"name":"schema","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"Schema","namespace":"Flow\\ETL","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"CORE","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIEByZXR1cm4gU2NoZW1hCiAqLw=="},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DSL\/functions.php","start_line_in_file":1390,"slug":"schema-metadata","name":"schema_metadata","namespace":"Flow\\ETL\\DSL","parameters":[{"name":"metadata","type":[{"name":"array","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"[]"}],"return_type":[{"name":"Metadata","namespace":"Flow\\ETL\\Schema","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"CORE","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIEBwYXJhbSBhcnJheTxzdHJpbmcsIGFycmF5PGJvb2x8ZmxvYXR8aW50fHN0cmluZz58Ym9vbHxmbG9hdHxpbnR8c3RyaW5nPiAkbWV0YWRhdGEKICov"},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DSL\/functions.php","start_line_in_file":1399,"slug":"int-schema","name":"int_schema","namespace":"Flow\\ETL\\DSL","parameters":[{"name":"name","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"nullable","type":[{"name":"bool","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"false"},{"name":"metadata","type":[{"name":"Metadata","namespace":"Flow\\ETL\\Schema","is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"null"}],"return_type":[{"name":"IntegerDefinition","namespace":"Flow\\ETL\\Schema\\Definition","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"CORE","type":"SCHEMA"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIEFsaWFzIGZvciBgaW50ZWdlcl9zY2hlbWFgLgogKi8="},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DSL\/functions.php","start_line_in_file":1405,"slug":"integer-schema","name":"integer_schema","namespace":"Flow\\ETL\\DSL","parameters":[{"name":"name","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"nullable","type":[{"name":"bool","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"false"},{"name":"metadata","type":[{"name":"Metadata","namespace":"Flow\\ETL\\Schema","is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"null"}],"return_type":[{"name":"IntegerDefinition","namespace":"Flow\\ETL\\Schema\\Definition","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"CORE","type":"SCHEMA"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DSL\/functions.php","start_line_in_file":1414,"slug":"str-schema","name":"str_schema","namespace":"Flow\\ETL\\DSL","parameters":[{"name":"name","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"nullable","type":[{"name":"bool","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"false"},{"name":"metadata","type":[{"name":"Metadata","namespace":"Flow\\ETL\\Schema","is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"null"}],"return_type":[{"name":"StringDefinition","namespace":"Flow\\ETL\\Schema\\Definition","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"CORE","type":"SCHEMA"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIEFsaWFzIGZvciBgc3RyaW5nX3NjaGVtYWAuCiAqLw=="},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DSL\/functions.php","start_line_in_file":1420,"slug":"string-schema","name":"string_schema","namespace":"Flow\\ETL\\DSL","parameters":[{"name":"name","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"nullable","type":[{"name":"bool","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"false"},{"name":"metadata","type":[{"name":"Metadata","namespace":"Flow\\ETL\\Schema","is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"null"}],"return_type":[{"name":"StringDefinition","namespace":"Flow\\ETL\\Schema\\Definition","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"CORE","type":"SCHEMA"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DSL\/functions.php","start_line_in_file":1426,"slug":"bool-schema","name":"bool_schema","namespace":"Flow\\ETL\\DSL","parameters":[{"name":"name","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"nullable","type":[{"name":"bool","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"false"},{"name":"metadata","type":[{"name":"Metadata","namespace":"Flow\\ETL\\Schema","is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"null"}],"return_type":[{"name":"BooleanDefinition","namespace":"Flow\\ETL\\Schema\\Definition","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"CORE","type":"SCHEMA"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DSL\/functions.php","start_line_in_file":1432,"slug":"float-schema","name":"float_schema","namespace":"Flow\\ETL\\DSL","parameters":[{"name":"name","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"nullable","type":[{"name":"bool","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"false"},{"name":"metadata","type":[{"name":"Metadata","namespace":"Flow\\ETL\\Schema","is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"null"}],"return_type":[{"name":"FloatDefinition","namespace":"Flow\\ETL\\Schema\\Definition","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"CORE","type":"SCHEMA"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DSL\/functions.php","start_line_in_file":1446,"slug":"map-schema","name":"map_schema","namespace":"Flow\\ETL\\DSL","parameters":[{"name":"name","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"type","type":[{"name":"MapType","namespace":"Flow\\Types\\Type\\Logical","is_nullable":false,"is_variadic":false},{"name":"Type","namespace":"Flow\\Types","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"nullable","type":[{"name":"bool","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"false"},{"name":"metadata","type":[{"name":"Metadata","namespace":"Flow\\ETL\\Schema","is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"null"}],"return_type":[{"name":"MapDefinition","namespace":"Flow\\ETL\\Schema\\Definition","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"CORE","type":"SCHEMA"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIEB0ZW1wbGF0ZSBUS2V5IG9mIGFycmF5LWtleQogKiBAdGVtcGxhdGUgVFZhbHVlCiAqCiAqIEBwYXJhbSBNYXBUeXBlPGFycmF5PFRLZXksIFRWYWx1ZT4+fFR5cGU8YXJyYXk8VEtleSwgVFZhbHVlPj4gJHR5cGUKICoKICogQHJldHVybiBNYXBEZWZpbml0aW9uPFRLZXksIFRWYWx1ZT4KICov"},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DSL\/functions.php","start_line_in_file":1460,"slug":"list-schema","name":"list_schema","namespace":"Flow\\ETL\\DSL","parameters":[{"name":"name","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"type","type":[{"name":"ListType","namespace":"Flow\\Types\\Type\\Logical","is_nullable":false,"is_variadic":false},{"name":"Type","namespace":"Flow\\Types","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"nullable","type":[{"name":"bool","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"false"},{"name":"metadata","type":[{"name":"Metadata","namespace":"Flow\\ETL\\Schema","is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"null"}],"return_type":[{"name":"ListDefinition","namespace":"Flow\\ETL\\Schema\\Definition","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"CORE","type":"SCHEMA"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIEB0ZW1wbGF0ZSBUCiAqCiAqIEBwYXJhbSBMaXN0VHlwZTxsaXN0PFQ+PnxUeXBlPGxpc3Q8VD4+ICR0eXBlCiAqCiAqIEByZXR1cm4gTGlzdERlZmluaXRpb248VD4KICov"},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DSL\/functions.php","start_line_in_file":1478,"slug":"enum-schema","name":"enum_schema","namespace":"Flow\\ETL\\DSL","parameters":[{"name":"name","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"type","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"nullable","type":[{"name":"bool","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"false"},{"name":"metadata","type":[{"name":"Metadata","namespace":"Flow\\ETL\\Schema","is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"null"}],"return_type":[{"name":"EnumDefinition","namespace":"Flow\\ETL\\Schema\\Definition","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"CORE","type":"SCHEMA"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIEB0ZW1wbGF0ZSBUIG9mIFxVbml0RW51bQogKgogKiBAcGFyYW0gY2xhc3Mtc3RyaW5nPFQ+ICR0eXBlCiAqCiAqIEByZXR1cm4gRW51bURlZmluaXRpb248VD4KICov"},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DSL\/functions.php","start_line_in_file":1484,"slug":"null-schema","name":"null_schema","namespace":"Flow\\ETL\\DSL","parameters":[{"name":"name","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"metadata","type":[{"name":"Metadata","namespace":"Flow\\ETL\\Schema","is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"null"}],"return_type":[{"name":"NullDefinition","namespace":"Flow\\ETL\\Schema\\Definition","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"CORE","type":"SCHEMA"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DSL\/functions.php","start_line_in_file":1490,"slug":"datetime-schema","name":"datetime_schema","namespace":"Flow\\ETL\\DSL","parameters":[{"name":"name","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"nullable","type":[{"name":"bool","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"false"},{"name":"metadata","type":[{"name":"Metadata","namespace":"Flow\\ETL\\Schema","is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"null"},{"name":"zone","type":[{"name":"DateTimeZone","namespace":"","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"'UTC'"}],"return_type":[{"name":"DateTimeDefinition","namespace":"Flow\\ETL\\Schema\\Definition","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"CORE","type":"SCHEMA"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DSL\/functions.php","start_line_in_file":1500,"slug":"time-schema","name":"time_schema","namespace":"Flow\\ETL\\DSL","parameters":[{"name":"name","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"nullable","type":[{"name":"bool","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"false"},{"name":"metadata","type":[{"name":"Metadata","namespace":"Flow\\ETL\\Schema","is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"null"}],"return_type":[{"name":"TimeDefinition","namespace":"Flow\\ETL\\Schema\\Definition","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"CORE","type":"SCHEMA"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DSL\/functions.php","start_line_in_file":1506,"slug":"date-schema","name":"date_schema","namespace":"Flow\\ETL\\DSL","parameters":[{"name":"name","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"nullable","type":[{"name":"bool","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"false"},{"name":"metadata","type":[{"name":"Metadata","namespace":"Flow\\ETL\\Schema","is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"null"}],"return_type":[{"name":"DateDefinition","namespace":"Flow\\ETL\\Schema\\Definition","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"CORE","type":"SCHEMA"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DSL\/functions.php","start_line_in_file":1512,"slug":"json-schema","name":"json_schema","namespace":"Flow\\ETL\\DSL","parameters":[{"name":"name","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"nullable","type":[{"name":"bool","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"false"},{"name":"metadata","type":[{"name":"Metadata","namespace":"Flow\\ETL\\Schema","is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"null"}],"return_type":[{"name":"JsonDefinition","namespace":"Flow\\ETL\\Schema\\Definition","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"CORE","type":"SCHEMA"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DSL\/functions.php","start_line_in_file":1518,"slug":"html-schema","name":"html_schema","namespace":"Flow\\ETL\\DSL","parameters":[{"name":"name","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"nullable","type":[{"name":"bool","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"false"},{"name":"metadata","type":[{"name":"Metadata","namespace":"Flow\\ETL\\Schema","is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"null"}],"return_type":[{"name":"HTMLDefinition","namespace":"Flow\\ETL\\Schema\\Definition","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"CORE","type":"SCHEMA"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DSL\/functions.php","start_line_in_file":1524,"slug":"html-element-schema","name":"html_element_schema","namespace":"Flow\\ETL\\DSL","parameters":[{"name":"name","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"nullable","type":[{"name":"bool","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"false"},{"name":"metadata","type":[{"name":"Metadata","namespace":"Flow\\ETL\\Schema","is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"null"}],"return_type":[{"name":"HTMLElementDefinition","namespace":"Flow\\ETL\\Schema\\Definition","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"CORE","type":"SCHEMA"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DSL\/functions.php","start_line_in_file":1530,"slug":"xml-schema","name":"xml_schema","namespace":"Flow\\ETL\\DSL","parameters":[{"name":"name","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"nullable","type":[{"name":"bool","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"false"},{"name":"metadata","type":[{"name":"Metadata","namespace":"Flow\\ETL\\Schema","is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"null"}],"return_type":[{"name":"XMLDefinition","namespace":"Flow\\ETL\\Schema\\Definition","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"CORE","type":"SCHEMA"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DSL\/functions.php","start_line_in_file":1536,"slug":"xml-element-schema","name":"xml_element_schema","namespace":"Flow\\ETL\\DSL","parameters":[{"name":"name","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"nullable","type":[{"name":"bool","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"false"},{"name":"metadata","type":[{"name":"Metadata","namespace":"Flow\\ETL\\Schema","is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"null"}],"return_type":[{"name":"XMLElementDefinition","namespace":"Flow\\ETL\\Schema\\Definition","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"CORE","type":"SCHEMA"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DSL\/functions.php","start_line_in_file":1549,"slug":"structure-schema","name":"structure_schema","namespace":"Flow\\ETL\\DSL","parameters":[{"name":"name","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"type","type":[{"name":"StructureType","namespace":"Flow\\Types\\Type\\Logical","is_nullable":false,"is_variadic":false},{"name":"Type","namespace":"Flow\\Types","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"nullable","type":[{"name":"bool","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"false"},{"name":"metadata","type":[{"name":"Metadata","namespace":"Flow\\ETL\\Schema","is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"null"}],"return_type":[{"name":"StructureDefinition","namespace":"Flow\\ETL\\Schema\\Definition","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"CORE","type":"SCHEMA"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIEB0ZW1wbGF0ZSBUCiAqCiAqIEBwYXJhbSBTdHJ1Y3R1cmVUeXBlPGFycmF5PGFycmF5LWtleSwgVD4+fFR5cGU8YXJyYXk8YXJyYXkta2V5LCBUPj4gJHR5cGUKICoKICogQHJldHVybiBTdHJ1Y3R1cmVEZWZpbml0aW9uPFQ+CiAqLw=="},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DSL\/functions.php","start_line_in_file":1560,"slug":"uuid-schema","name":"uuid_schema","namespace":"Flow\\ETL\\DSL","parameters":[{"name":"name","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"nullable","type":[{"name":"bool","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"false"},{"name":"metadata","type":[{"name":"Metadata","namespace":"Flow\\ETL\\Schema","is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"null"}],"return_type":[{"name":"UuidDefinition","namespace":"Flow\\ETL\\Schema\\Definition","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"CORE","type":"SCHEMA"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DSL\/functions.php","start_line_in_file":1566,"slug":"time-zone-schema","name":"time_zone_schema","namespace":"Flow\\ETL\\DSL","parameters":[{"name":"name","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"nullable","type":[{"name":"bool","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"false"},{"name":"metadata","type":[{"name":"Metadata","namespace":"Flow\\ETL\\Schema","is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"null"}],"return_type":[{"name":"TimeZoneDefinition","namespace":"Flow\\ETL\\Schema\\Definition","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"CORE","type":"SCHEMA"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DSL\/functions.php","start_line_in_file":1579,"slug":"definition-from-array","name":"definition_from_array","namespace":"Flow\\ETL\\DSL","parameters":[{"name":"definition","type":[{"name":"array","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"Definition","namespace":"Flow\\ETL\\Schema","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"CORE","type":"SCHEMA"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBhIERlZmluaXRpb24gZnJvbSBhbiBhcnJheSByZXByZXNlbnRhdGlvbi4KICoKICogQHBhcmFtIGFycmF5PGFycmF5LWtleSwgbWl4ZWQ+ICRkZWZpbml0aW9uCiAqCiAqIEByZXR1cm4gRGVmaW5pdGlvbjxtaXhlZD4KICov"},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DSL\/functions.php","start_line_in_file":1612,"slug":"definition-from-type","name":"definition_from_type","namespace":"Flow\\ETL\\DSL","parameters":[{"name":"ref","type":[{"name":"Reference","namespace":"Flow\\ETL\\Row","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"type","type":[{"name":"Type","namespace":"Flow\\Types","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"nullable","type":[{"name":"bool","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"false"},{"name":"metadata","type":[{"name":"Metadata","namespace":"Flow\\ETL\\Schema","is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"null"}],"return_type":[{"name":"Definition","namespace":"Flow\\ETL\\Schema","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"CORE","type":"SCHEMA"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBhIERlZmluaXRpb24gZnJvbSBhIFR5cGUuCiAqCiAqIEBwYXJhbSBUeXBlPG1peGVkPiAkdHlwZQogKgogKiBAcmV0dXJuIERlZmluaXRpb248bWl4ZWQ+CiAqLw=="},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DSL\/functions.php","start_line_in_file":1664,"slug":"infer-schema","name":"infer_schema","namespace":"Flow\\ETL\\DSL","parameters":[],"return_type":[{"name":"SchemaInferenceBuilder","namespace":"Flow\\ETL\\Schema\\Inference","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"CORE","type":"SCHEMA"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DSL\/functions.php","start_line_in_file":1670,"slug":"execution-context","name":"execution_context","namespace":"Flow\\ETL\\DSL","parameters":[{"name":"config","type":[{"name":"Config","namespace":"Flow\\ETL","is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"null"}],"return_type":[{"name":"FlowContext","namespace":"Flow\\ETL","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"CORE","type":"DATA_FRAME"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DSL\/functions.php","start_line_in_file":1676,"slug":"flow-context","name":"flow_context","namespace":"Flow\\ETL\\DSL","parameters":[{"name":"config","type":[{"name":"Config","namespace":"Flow\\ETL","is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"null"}],"return_type":[{"name":"FlowContext","namespace":"Flow\\ETL","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"CORE","type":"DATA_FRAME"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DSL\/functions.php","start_line_in_file":1682,"slug":"config","name":"config","namespace":"Flow\\ETL\\DSL","parameters":[],"return_type":[{"name":"Config","namespace":"Flow\\ETL","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"CORE","type":"DATA_FRAME"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DSL\/functions.php","start_line_in_file":1688,"slug":"config-builder","name":"config_builder","namespace":"Flow\\ETL\\DSL","parameters":[],"return_type":[{"name":"ConfigBuilder","namespace":"Flow\\ETL\\Config","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"CORE","type":"DATA_FRAME"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DSL\/functions.php","start_line_in_file":1694,"slug":"memory-sort","name":"memory_sort","namespace":"Flow\\ETL\\DSL","parameters":[],"return_type":[{"name":"MemorySortBuilder","namespace":"Flow\\ETL\\Config\\Sort","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"CORE","type":"DATA_FRAME"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DSL\/functions.php","start_line_in_file":1700,"slug":"external-sort","name":"external_sort","namespace":"Flow\\ETL\\DSL","parameters":[],"return_type":[{"name":"ExternalSortBuilder","namespace":"Flow\\ETL\\Config\\Sort","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"CORE","type":"DATA_FRAME"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DSL\/functions.php","start_line_in_file":1706,"slug":"hash-join","name":"hash_join","namespace":"Flow\\ETL\\DSL","parameters":[],"return_type":[{"name":"HashJoinBuilder","namespace":"Flow\\ETL\\Config\\Join","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"CORE","type":"DATA_FRAME"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DSL\/functions.php","start_line_in_file":1712,"slug":"hash-group-by","name":"hash_group_by","namespace":"Flow\\ETL\\DSL","parameters":[],"return_type":[{"name":"HashGroupByBuilder","namespace":"Flow\\ETL\\Config\\Grouping","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"CORE","type":"DATA_FRAME"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DSL\/functions.php","start_line_in_file":1718,"slug":"hash-repartition","name":"hash_repartition","namespace":"Flow\\ETL\\DSL","parameters":[],"return_type":[{"name":"HashRepartitionBuilder","namespace":"Flow\\ETL\\Config\\Repartition","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"CORE","type":"DATA_FRAME"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DSL\/functions.php","start_line_in_file":1727,"slug":"pivot-values","name":"pivot_values","namespace":"Flow\\ETL\\DSL","parameters":[{"name":"values","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false},{"name":"int","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":true,"default_value":null}],"return_type":[{"name":"DeclaredPivotValues","namespace":"Flow\\ETL\\GroupBy","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"CORE","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIERlY2xhcmVzIHRoZSBwaXZvdCBjb2x1bW5zIGEgZ3JvdXBCeSgpLT5waXZvdCgpIHByb2R1Y2VzLCBzbyB0aGUgcGxhbiBjYW4gbmFtZSB0aGVtIGJlZm9yZSBhIHJvdyBmbG93cy4KICov"},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DSL\/functions.php","start_line_in_file":1737,"slug":"discover-pivot-values","name":"discover_pivot_values","namespace":"Flow\\ETL\\DSL","parameters":[{"name":"maxValues","type":[{"name":"int","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"10000"}],"return_type":[{"name":"DiscoveredPivotValues","namespace":"Flow\\ETL\\GroupBy","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"CORE","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIFJlYWRzIHRoZSBwaXZvdCBjb2x1bW4gb25jZSBhdCBidWlsZCB0aW1lIGFuZCB0dXJucyB3aGF0IGl0IGZpbmRzIGludG8gZGVjbGFyZWQgdmFsdWVzLiBSZWZ1c2VzIGEKICogc291cmNlIHRoYXQgY2Fubm90IGJlIHJlYWQgdHdpY2UuCiAqLw=="},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DSL\/functions.php","start_line_in_file":1743,"slug":"partition-by","name":"partition_by","namespace":"Flow\\ETL\\DSL","parameters":[{"name":"entry","type":[{"name":"Reference","namespace":"Flow\\ETL\\Row","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"entries","type":[{"name":"Reference","namespace":"Flow\\ETL\\Row","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":true,"default_value":null}],"return_type":[{"name":"Partitioning","namespace":"Flow\\ETL\\Loader","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"CORE","type":"LOADER"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DSL\/functions.php","start_line_in_file":1752,"slug":"partition-types","name":"partition_types","namespace":"Flow\\ETL\\DSL","parameters":[{"name":"types","type":[{"name":"Type","namespace":"Flow\\Types","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":true,"default_value":null}],"return_type":[{"name":"PartitionTypes","namespace":"Flow\\ETL\\Extractor\\File","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"CORE","type":"EXTRACTOR"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIEBwYXJhbSBUeXBlPG1peGVkPiAuLi4kdHlwZXMgcGFydGl0aW9uIGNvbHVtbiBuYW1lID0+IHR5cGUsIHBhc3NlZCBhcyBuYW1lZCBhcmd1bWVudHMKICov"},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DSL\/functions.php","start_line_in_file":1762,"slug":"overwrite","name":"overwrite","namespace":"Flow\\ETL\\DSL","parameters":[],"return_type":[{"name":"SaveMode","namespace":"Flow\\ETL\\Filesystem","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"CORE","type":"DATA_FRAME"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIEFsaWFzIGZvciBzYXZlX21vZGVfb3ZlcndyaXRlKCkuCiAqLw=="},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DSL\/functions.php","start_line_in_file":1768,"slug":"save-mode-overwrite","name":"save_mode_overwrite","namespace":"Flow\\ETL\\DSL","parameters":[],"return_type":[{"name":"SaveMode","namespace":"Flow\\ETL\\Filesystem","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"CORE","type":"DATA_FRAME"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DSL\/functions.php","start_line_in_file":1777,"slug":"ignore","name":"ignore","namespace":"Flow\\ETL\\DSL","parameters":[],"return_type":[{"name":"SaveMode","namespace":"Flow\\ETL\\Filesystem","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"CORE","type":"DATA_FRAME"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIEFsaWFzIGZvciBzYXZlX21vZGVfaWdub3JlKCkuCiAqLw=="},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DSL\/functions.php","start_line_in_file":1783,"slug":"save-mode-ignore","name":"save_mode_ignore","namespace":"Flow\\ETL\\DSL","parameters":[],"return_type":[{"name":"SaveMode","namespace":"Flow\\ETL\\Filesystem","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"CORE","type":"DATA_FRAME"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DSL\/functions.php","start_line_in_file":1792,"slug":"exception-if-exists","name":"exception_if_exists","namespace":"Flow\\ETL\\DSL","parameters":[],"return_type":[{"name":"SaveMode","namespace":"Flow\\ETL\\Filesystem","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"CORE","type":"DATA_FRAME"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIEFsaWFzIGZvciBzYXZlX21vZGVfZXhjZXB0aW9uX2lmX2V4aXN0cygpLgogKi8="},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DSL\/functions.php","start_line_in_file":1798,"slug":"save-mode-exception-if-exists","name":"save_mode_exception_if_exists","namespace":"Flow\\ETL\\DSL","parameters":[],"return_type":[{"name":"SaveMode","namespace":"Flow\\ETL\\Filesystem","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"CORE","type":"DATA_FRAME"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DSL\/functions.php","start_line_in_file":1807,"slug":"append","name":"append","namespace":"Flow\\ETL\\DSL","parameters":[],"return_type":[{"name":"SaveMode","namespace":"Flow\\ETL\\Filesystem","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"CORE","type":"DATA_FRAME"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIEFsaWFzIGZvciBzYXZlX21vZGVfYXBwZW5kKCkuCiAqLw=="},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DSL\/functions.php","start_line_in_file":1813,"slug":"save-mode-append","name":"save_mode_append","namespace":"Flow\\ETL\\DSL","parameters":[],"return_type":[{"name":"SaveMode","namespace":"Flow\\ETL\\Filesystem","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"CORE","type":"DATA_FRAME"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DSL\/functions.php","start_line_in_file":1819,"slug":"print-rows","name":"print_rows","namespace":"Flow\\ETL\\DSL","parameters":[{"name":"rows","type":[{"name":"Rows","namespace":"Flow\\ETL","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"truncate","type":[{"name":"int","namespace":null,"is_nullable":false,"is_variadic":false},{"name":"bool","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"false"},{"name":"formatter","type":[{"name":"Formatter","namespace":"Flow\\ETL","is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"null"}],"return_type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"CORE","type":"DATA_FRAME"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DSL\/functions.php","start_line_in_file":1825,"slug":"identical","name":"identical","namespace":"Flow\\ETL\\DSL","parameters":[{"name":"left","type":[{"name":"Reference","namespace":"Flow\\ETL\\Row","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"right","type":[{"name":"Reference","namespace":"Flow\\ETL\\Row","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"Identical","namespace":"Flow\\ETL\\Join\\Comparison","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"CORE","type":"COMPARISON"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DSL\/functions.php","start_line_in_file":1831,"slug":"equal","name":"equal","namespace":"Flow\\ETL\\DSL","parameters":[{"name":"left","type":[{"name":"Reference","namespace":"Flow\\ETL\\Row","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"right","type":[{"name":"Reference","namespace":"Flow\\ETL\\Row","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"Equal","namespace":"Flow\\ETL\\Join\\Comparison","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"CORE","type":"COMPARISON"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DSL\/functions.php","start_line_in_file":1837,"slug":"compare-all","name":"compare_all","namespace":"Flow\\ETL\\DSL","parameters":[{"name":"comparison","type":[{"name":"Comparison","namespace":"Flow\\ETL\\Join","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"comparisons","type":[{"name":"Comparison","namespace":"Flow\\ETL\\Join","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":true,"default_value":null}],"return_type":[{"name":"All","namespace":"Flow\\ETL\\Join\\Comparison","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"CORE","type":"COMPARISON"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DSL\/functions.php","start_line_in_file":1843,"slug":"compare-any","name":"compare_any","namespace":"Flow\\ETL\\DSL","parameters":[{"name":"comparison","type":[{"name":"Comparison","namespace":"Flow\\ETL\\Join","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"comparisons","type":[{"name":"Comparison","namespace":"Flow\\ETL\\Join","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":true,"default_value":null}],"return_type":[{"name":"Any","namespace":"Flow\\ETL\\Join\\Comparison","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"CORE","type":"COMPARISON"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DSL\/functions.php","start_line_in_file":1854,"slug":"join-on","name":"join_on","namespace":"Flow\\ETL\\DSL","parameters":[{"name":"comparisons","type":[{"name":"Comparison","namespace":"Flow\\ETL\\Join","is_nullable":false,"is_variadic":false},{"name":"array","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"join_prefix","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"''"}],"return_type":[{"name":"Expression","namespace":"Flow\\ETL\\Join","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"CORE","type":"DATA_FRAME"}},{"name":"DocumentationExample","namespace":"Flow\\Documentation\\Attribute","arguments":{"topic":"joins","example":"join"}},{"name":"DocumentationExample","namespace":"Flow\\Documentation\\Attribute","arguments":{"topic":"joins","example":"join_each"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIEBwYXJhbSBhcnJheTxDb21wYXJpc29ufHN0cmluZz58Q29tcGFyaXNvbiAkY29tcGFyaXNvbnMKICov"},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DSL\/functions.php","start_line_in_file":1860,"slug":"schema-sort-by-name","name":"schema_sort_by_name","namespace":"Flow\\ETL\\DSL","parameters":[{"name":"order","type":[{"name":"SortOrder","namespace":"Flow\\ETL\\Row","is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"Flow\\ETL\\Row\\SortOrder::..."}],"return_type":[{"name":"SortingStrategy","namespace":"Flow\\ETL\\Schema","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"CORE","type":"SCHEMA"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DSL\/functions.php","start_line_in_file":1869,"slug":"schema-sort-by-type","name":"schema_sort_by_type","namespace":"Flow\\ETL\\DSL","parameters":[{"name":"priorities","type":[{"name":"array","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"[...]"},{"name":"order","type":[{"name":"SortOrder","namespace":"Flow\\ETL\\Row","is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"Flow\\ETL\\Row\\SortOrder::..."}],"return_type":[{"name":"SortingStrategy","namespace":"Flow\\ETL\\Schema","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"CORE","type":"SCHEMA"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIEBwYXJhbSBhcnJheTxjbGFzcy1zdHJpbmc8VHlwZTxtaXhlZD4+LCBpbnQ+ICRwcmlvcml0aWVzCiAqLw=="},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DSL\/functions.php","start_line_in_file":1880,"slug":"schema-sort-by-type-and-name","name":"schema_sort_by_type_and_name","namespace":"Flow\\ETL\\DSL","parameters":[{"name":"priorities","type":[{"name":"array","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"[...]"},{"name":"order","type":[{"name":"SortOrder","namespace":"Flow\\ETL\\Row","is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"Flow\\ETL\\Row\\SortOrder::..."}],"return_type":[{"name":"SortingStrategy","namespace":"Flow\\ETL\\Schema","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"CORE","type":"SCHEMA"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIEBwYXJhbSBhcnJheTxjbGFzcy1zdHJpbmc8VHlwZTxtaXhlZD4+LCBpbnQ+ICRwcmlvcml0aWVzCiAqLw=="},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DSL\/functions.php","start_line_in_file":1891,"slug":"schema-sort-by-metadata","name":"schema_sort_by_metadata","namespace":"Flow\\ETL\\DSL","parameters":[{"name":"key","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"order","type":[{"name":"SortOrder","namespace":"Flow\\ETL\\Row","is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"Flow\\ETL\\Row\\SortOrder::..."}],"return_type":[{"name":"SortingStrategy","namespace":"Flow\\ETL\\Schema","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"CORE","type":"SCHEMA"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DSL\/functions.php","start_line_in_file":1901,"slug":"is-type","name":"is_type","namespace":"Flow\\ETL\\DSL","parameters":[{"name":"type","type":[{"name":"Type","namespace":"Flow\\Types","is_nullable":false,"is_variadic":false},{"name":"array","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"value","type":[{"name":"mixed","namespace":null,"is_nullable":true,"is_variadic":false}],"has_default_value":false,"is_nullable":true,"is_variadic":false,"default_value":null}],"return_type":[{"name":"bool","namespace":null,"is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"CORE","type":"DATA_FRAME"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIEBwYXJhbSBhcnJheTxzdHJpbmd8VHlwZTxtaXhlZD4+fFR5cGU8bWl4ZWQ+ICR0eXBlCiAqIEBwYXJhbSBtaXhlZCAkdmFsdWUKICov"},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DSL\/functions.php","start_line_in_file":1935,"slug":"generate-random-string","name":"generate_random_string","namespace":"Flow\\ETL\\DSL","parameters":[{"name":"length","type":[{"name":"int","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"32"},{"name":"generator","type":[{"name":"RandomValueGenerator","namespace":"Flow\\ETL","is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"Flow\\ETL\\NativePHPRandomValueGenerator::..."}],"return_type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"CORE","type":"DATA_FRAME"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DSL\/functions.php","start_line_in_file":1943,"slug":"generate-random-int","name":"generate_random_int","namespace":"Flow\\ETL\\DSL","parameters":[{"name":"start","type":[{"name":"int","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"-9223372036854775808"},{"name":"end","type":[{"name":"int","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"9223372036854775807"},{"name":"generator","type":[{"name":"RandomValueGenerator","namespace":"Flow\\ETL","is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"Flow\\ETL\\NativePHPRandomValueGenerator::..."}],"return_type":[{"name":"int","namespace":null,"is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"CORE","type":"DATA_FRAME"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DSL\/functions.php","start_line_in_file":1952,"slug":"random-string","name":"random_string","namespace":"Flow\\ETL\\DSL","parameters":[{"name":"length","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"int","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"generator","type":[{"name":"RandomValueGenerator","namespace":"Flow\\ETL","is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"Flow\\ETL\\NativePHPRandomValueGenerator::..."}],"return_type":[{"name":"RandomString","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"CORE","type":"DATA_FRAME"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DSL\/functions.php","start_line_in_file":1960,"slug":"date-interval-to-milliseconds","name":"date_interval_to_milliseconds","namespace":"Flow\\ETL\\DSL","parameters":[{"name":"interval","type":[{"name":"DateInterval","namespace":"","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"int","namespace":null,"is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"CORE","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DSL\/functions.php","start_line_in_file":1977,"slug":"date-interval-to-seconds","name":"date_interval_to_seconds","namespace":"Flow\\ETL\\DSL","parameters":[{"name":"interval","type":[{"name":"DateInterval","namespace":"","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"int","namespace":null,"is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"CORE","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DSL\/functions.php","start_line_in_file":1994,"slug":"date-interval-to-microseconds","name":"date_interval_to_microseconds","namespace":"Flow\\ETL\\DSL","parameters":[{"name":"interval","type":[{"name":"DateInterval","namespace":"","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"int","namespace":null,"is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"CORE","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DSL\/functions.php","start_line_in_file":2011,"slug":"with-entry","name":"with_entry","namespace":"Flow\\ETL\\DSL","parameters":[{"name":"name","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"function","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"WithEntry","namespace":"Flow\\ETL","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"CORE","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DSL\/functions.php","start_line_in_file":2017,"slug":"constraint-unique","name":"constraint_unique","namespace":"Flow\\ETL\\DSL","parameters":[{"name":"reference","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"references","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":true,"default_value":null}],"return_type":[{"name":"UniqueConstraint","namespace":"Flow\\ETL\\Constraint","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"CORE","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DSL\/functions.php","start_line_in_file":2023,"slug":"constraint-sorted-by","name":"constraint_sorted_by","namespace":"Flow\\ETL\\DSL","parameters":[{"name":"column","type":[{"name":"Reference","namespace":"Flow\\ETL\\Row","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"columns","type":[{"name":"Reference","namespace":"Flow\\ETL\\Row","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":true,"default_value":null}],"return_type":[{"name":"SortedByConstraint","namespace":"Flow\\ETL\\Constraint","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"CORE","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DSL\/functions.php","start_line_in_file":2033,"slug":"analyze","name":"analyze","namespace":"Flow\\ETL\\DSL","parameters":[],"return_type":[{"name":"Analyze","namespace":"Flow\\ETL","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"CORE","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DSL\/functions.php","start_line_in_file":2042,"slug":"match-cases","name":"match_cases","namespace":"Flow\\ETL\\DSL","parameters":[{"name":"cases","type":[{"name":"array","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"default","type":[{"name":"mixed","namespace":null,"is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"null"}],"return_type":[{"name":"MatchCases","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"CORE","type":"SCALAR_FUNCTION"}}],"scalar_function_chain":true,"doc_comment":"LyoqCiAqIEBwYXJhbSBhcnJheTxNYXRjaENvbmRpdGlvbj4gJGNhc2VzCiAqLw=="},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DSL\/functions.php","start_line_in_file":2048,"slug":"match-condition","name":"match_condition","namespace":"Flow\\ETL\\DSL","parameters":[{"name":"condition","type":[{"name":"mixed","namespace":null,"is_nullable":true,"is_variadic":false}],"has_default_value":false,"is_nullable":true,"is_variadic":false,"default_value":null},{"name":"then","type":[{"name":"mixed","namespace":null,"is_nullable":true,"is_variadic":false}],"has_default_value":false,"is_nullable":true,"is_variadic":false,"default_value":null}],"return_type":[{"name":"MatchCondition","namespace":"Flow\\ETL\\Function\\MatchCases","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"CORE","type":"SCALAR_FUNCTION"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/ETL\/DSL\/functions.php","start_line_in_file":2054,"slug":"clock","name":"clock","namespace":"Flow\\ETL\\DSL","parameters":[{"name":"time_zone","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"'UTC'"}],"return_type":[{"name":"ClockInterface","namespace":"Psr\\Clock","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"CORE","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/Floe\/DSL\/functions.php","start_line_in_file":31,"slug":"from-floe","name":"from_floe","namespace":"Flow\\Floe\\DSL","parameters":[{"name":"path","type":[{"name":"Path","namespace":"Flow\\Filesystem","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"codec","type":[{"name":"Codec","namespace":"Flow\\Floe","is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"Flow\\Floe\\Codec\\NoopCodec::..."},{"name":"chunk_size","type":[{"name":"int","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"65536"},{"name":"filesystem","type":[{"name":"Filesystem","namespace":"Flow\\Filesystem","is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"Flow\\Filesystem\\Local\\NativeLocalFilesystem::..."}],"return_type":[{"name":"FloeExtractor","namespace":"Flow\\Floe","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"FLOE","type":"EXTRACTOR"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIEBwYXJhbSBQYXRofHN0cmluZyAkcGF0aAogKi8="},{"repository_path":"src\/core\/etl\/src\/Flow\/Floe\/DSL\/functions.php","start_line_in_file":44,"slug":"to-floe","name":"to_floe","namespace":"Flow\\Floe\\DSL","parameters":[{"name":"path","type":[{"name":"Path","namespace":"Flow\\Filesystem","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"metadata","type":[{"name":"Metadata","namespace":"Flow\\ETL\\Schema","is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"null"},{"name":"options","type":[{"name":"Options","namespace":"Flow\\Floe","is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"Flow\\Floe\\Options::..."},{"name":"filesystem","type":[{"name":"Filesystem","namespace":"Flow\\Filesystem","is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"Flow\\Filesystem\\Local\\NativeLocalFilesystem::..."}],"return_type":[{"name":"FloeLoader","namespace":"Flow\\Floe","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"FLOE","type":"LOADER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIEBwYXJhbSBQYXRofHN0cmluZyAkcGF0aAogKi8="},{"repository_path":"src\/core\/etl\/src\/Flow\/Floe\/DSL\/functions.php","start_line_in_file":54,"slug":"floe-options","name":"floe_options","namespace":"Flow\\Floe\\DSL","parameters":[{"name":"buffer_size","type":[{"name":"int","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"65536"},{"name":"codec","type":[{"name":"Codec","namespace":"Flow\\Floe","is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"Flow\\Floe\\Codec\\NoopCodec::..."}],"return_type":[{"name":"Options","namespace":"Flow\\Floe","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"FLOE","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/Floe\/DSL\/functions.php","start_line_in_file":66,"slug":"merge-floe","name":"merge_floe","namespace":"Flow\\Floe\\DSL","parameters":[{"name":"sources","type":[{"name":"array","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"dest","type":[{"name":"Path","namespace":"Flow\\Filesystem","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"backend","type":[{"name":"Backend","namespace":"Flow\\ETL\\Column","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"compact","type":[{"name":"bool","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"false"},{"name":"metadata","type":[{"name":"Metadata","namespace":"Flow\\ETL\\Schema","is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"null"},{"name":"filesystem","type":[{"name":"Filesystem","namespace":"Flow\\Filesystem","is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"Flow\\Filesystem\\Local\\NativeLocalFilesystem::..."}],"return_type":[{"name":"void","namespace":null,"is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"FLOE","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIE1lcmdlcyBzZXZlcmFsIEZsb2UgZmlsZXMgKHNhbWUgb3IgYXBwZW5kLWNvbXBhdGlibGUgZXZvbHZpbmcgc2NoZW1hKSBpbnRvIG9uZS4gQnl0ZS1zcGxpY2VzIGZyYW1lCiAqIHJlZ2lvbnMgYnkgZGVmYXVsdCAoTyhieXRlcyksIG5vIHJlLWVuY29kZSk7IGNvbXBhY3QgcmUtZW5jb2RlcyBhbGwgcm93cyBpbnRvIGZld2VyIHNlY3Rpb25zLgogKgogKiBAcGFyYW0gYXJyYXk8aW50LCBQYXRofHN0cmluZz4gJHNvdXJjZXMKICov"},{"repository_path":"src\/core\/etl\/src\/Flow\/Serializer\/DSL\/functions.php","start_line_in_file":18,"slug":"serialize-to-string","name":"serialize_to_string","namespace":"Flow\\Serializer\\DSL","parameters":[{"name":"serializer","type":[{"name":"Serializer","namespace":"Flow\\Serializer","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"rows","type":[{"name":"Rows","namespace":"Flow\\ETL","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"CORE","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/core\/etl\/src\/Flow\/Serializer\/DSL\/functions.php","start_line_in_file":27,"slug":"unserialize-from-string","name":"unserialize_from_string","namespace":"Flow\\Serializer\\DSL","parameters":[{"name":"serializer","type":[{"name":"Serializer","namespace":"Flow\\Serializer","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"payload","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"Rows","namespace":"Flow\\ETL","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"CORE","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/adapter\/etl-adapter-avro\/src\/Flow\/ETL\/Adapter\/Avro\/functions.php","start_line_in_file":21,"slug":"from-avro","name":"from_avro","namespace":"Flow\\ETL\\DSL\\Adapter\\Avro","parameters":[{"name":"path","type":[{"name":"Path","namespace":"Flow\\Filesystem","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"filesystem","type":[{"name":"Filesystem","namespace":"Flow\\Filesystem","is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"Flow\\Filesystem\\Local\\NativeLocalFilesystem::..."}],"return_type":[{"name":"AvroExtractor","namespace":"Flow\\ETL\\Adapter\\Avro\\FlixTech","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"AVRO","type":"EXTRACTOR"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/adapter\/etl-adapter-avro\/src\/Flow\/ETL\/Adapter\/Avro\/functions.php","start_line_in_file":27,"slug":"to-avro","name":"to_avro","namespace":"Flow\\ETL\\DSL\\Adapter\\Avro","parameters":[{"name":"path","type":[{"name":"Path","namespace":"Flow\\Filesystem","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"schema","type":[{"name":"Schema","namespace":"Flow\\ETL","is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"null"},{"name":"filesystem","type":[{"name":"Filesystem","namespace":"Flow\\Filesystem","is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"Flow\\Filesystem\\Local\\NativeLocalFilesystem::..."}],"return_type":[{"name":"AvroLoader","namespace":"Flow\\ETL\\Adapter\\Avro\\FlixTech","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"AVRO","type":"LOADER"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/adapter\/etl-adapter-chartjs\/src\/Flow\/ETL\/Adapter\/ChartJS\/functions.php","start_line_in_file":23,"slug":"bar-chart","name":"bar_chart","namespace":"Flow\\ETL\\Adapter\\ChartJS","parameters":[{"name":"label","type":[{"name":"Reference","namespace":"Flow\\ETL\\Row","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"datasets","type":[{"name":"References","namespace":"Flow\\ETL\\Row","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"BarChart","namespace":"Flow\\ETL\\Adapter\\ChartJS\\Chart","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"CHART_JS","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/adapter\/etl-adapter-chartjs\/src\/Flow\/ETL\/Adapter\/ChartJS\/functions.php","start_line_in_file":29,"slug":"line-chart","name":"line_chart","namespace":"Flow\\ETL\\Adapter\\ChartJS","parameters":[{"name":"label","type":[{"name":"Reference","namespace":"Flow\\ETL\\Row","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"datasets","type":[{"name":"References","namespace":"Flow\\ETL\\Row","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"LineChart","namespace":"Flow\\ETL\\Adapter\\ChartJS\\Chart","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"CHART_JS","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/adapter\/etl-adapter-chartjs\/src\/Flow\/ETL\/Adapter\/ChartJS\/functions.php","start_line_in_file":35,"slug":"pie-chart","name":"pie_chart","namespace":"Flow\\ETL\\Adapter\\ChartJS","parameters":[{"name":"label","type":[{"name":"Reference","namespace":"Flow\\ETL\\Row","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"datasets","type":[{"name":"References","namespace":"Flow\\ETL\\Row","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"PieChart","namespace":"Flow\\ETL\\Adapter\\ChartJS\\Chart","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"CHART_JS","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/adapter\/etl-adapter-chartjs\/src\/Flow\/ETL\/Adapter\/ChartJS\/functions.php","start_line_in_file":41,"slug":"to-chartjs","name":"to_chartjs","namespace":"Flow\\ETL\\Adapter\\ChartJS","parameters":[{"name":"type","type":[{"name":"Chart","namespace":"Flow\\ETL\\Adapter\\ChartJS","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"ChartJSLoader","namespace":"Flow\\ETL\\Adapter\\ChartJS","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"CHART_JS","type":"LOADER"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/adapter\/etl-adapter-chartjs\/src\/Flow\/ETL\/Adapter\/ChartJS\/functions.php","start_line_in_file":52,"slug":"to-chartjs-file","name":"to_chartjs_file","namespace":"Flow\\ETL\\Adapter\\ChartJS","parameters":[{"name":"type","type":[{"name":"Chart","namespace":"Flow\\ETL\\Adapter\\ChartJS","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"output","type":[{"name":"Path","namespace":"Flow\\Filesystem","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false},{"name":"null","namespace":null,"is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"null"},{"name":"template","type":[{"name":"Path","namespace":"Flow\\Filesystem","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false},{"name":"null","namespace":null,"is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"null"},{"name":"filesystem","type":[{"name":"Filesystem","namespace":"Flow\\Filesystem","is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"Flow\\Filesystem\\Local\\NativeLocalFilesystem::..."}],"return_type":[{"name":"ChartJSLoader","namespace":"Flow\\ETL\\Adapter\\ChartJS","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"CHART_JS","type":"LOADER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIEBwYXJhbSBDaGFydCAkdHlwZQogKiBAcGFyYW0gbnVsbHxQYXRofHN0cmluZyAkb3V0cHV0IC0gQGRlcHJlY2F0ZWQgdXNlICRsb2FkZXItPndpdGhPdXRwdXRQYXRoKCkgaW5zdGVhZAogKiBAcGFyYW0gbnVsbHxQYXRofHN0cmluZyAkdGVtcGxhdGUgLSBAZGVwcmVjYXRlZCB1c2UgJGxvYWRlci0+d2l0aFRlbXBsYXRlKCkgaW5zdGVhZAogKi8="},{"repository_path":"src\/adapter\/etl-adapter-chartjs\/src\/Flow\/ETL\/Adapter\/ChartJS\/functions.php","start_line_in_file":84,"slug":"to-chartjs-var","name":"to_chartjs_var","namespace":"Flow\\ETL\\Adapter\\ChartJS","parameters":[{"name":"type","type":[{"name":"Chart","namespace":"Flow\\ETL\\Adapter\\ChartJS","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"output","type":[{"name":"array","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"ChartJSLoader","namespace":"Flow\\ETL\\Adapter\\ChartJS","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"CHART_JS","type":"LOADER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIEBwYXJhbSBDaGFydCAkdHlwZQogKiBAcGFyYW0gYXJyYXk8YXJyYXkta2V5LCBtaXhlZD4gJG91dHB1dCAtIEBkZXByZWNhdGVkIHVzZSAkbG9hZGVyLT53aXRoT3V0cHV0VmFyKCkgaW5zdGVhZAogKi8="},{"repository_path":"src\/adapter\/etl-adapter-csv\/src\/Flow\/ETL\/Adapter\/CSV\/functions.php","start_line_in_file":35,"slug":"from-csv","name":"from_csv","namespace":"Flow\\ETL\\Adapter\\CSV","parameters":[{"name":"path","type":[{"name":"Path","namespace":"Flow\\Filesystem","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"with_header","type":[{"name":"bool","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"true"},{"name":"empty_to_null","type":[{"name":"bool","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"true"},{"name":"separator","type":[{"name":"string","namespace":null,"is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"null"},{"name":"enclosure","type":[{"name":"string","namespace":null,"is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"null"},{"name":"escape","type":[{"name":"string","namespace":null,"is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"null"},{"name":"characters_read_in_line","type":[{"name":"int","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"10485760"},{"name":"schema","type":[{"name":"Schema","namespace":"Flow\\ETL","is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"null"},{"name":"filesystem","type":[{"name":"Filesystem","namespace":"Flow\\Filesystem","is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"Flow\\Filesystem\\Local\\NativeLocalFilesystem::..."}],"return_type":[{"name":"CSVExtractor","namespace":"Flow\\ETL\\Adapter\\CSV","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"CSV","type":"EXTRACTOR"}},{"name":"DocumentationExample","namespace":"Flow\\Documentation\\Attribute","arguments":{"topic":"reading","example":"csv"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIEBwYXJhbSBQYXRofHN0cmluZyAkcGF0aAogKiBAcGFyYW0gYm9vbCAkZW1wdHlfdG9fbnVsbCAtIEBkZXByZWNhdGVkIHVzZSAkbG9hZGVyLT53aXRoRW1wdHlUb051bGwoKSBpbnN0ZWFkCiAqIEBwYXJhbSBib29sICR3aXRoX2hlYWRlciAtIEBkZXByZWNhdGVkIHVzZSAkbG9hZGVyLT53aXRoSGVhZGVyKCkgaW5zdGVhZAogKiBAcGFyYW0gbnVsbHxzdHJpbmcgJHNlcGFyYXRvciAtIEBkZXByZWNhdGVkIHVzZSAkbG9hZGVyLT53aXRoU2VwYXJhdG9yKCkgaW5zdGVhZAogKiBAcGFyYW0gbnVsbHxzdHJpbmcgJGVuY2xvc3VyZSAtIEBkZXByZWNhdGVkIHVzZSAkbG9hZGVyLT53aXRoRW5jbG9zdXJlKCkgaW5zdGVhZAogKiBAcGFyYW0gbnVsbHxzdHJpbmcgJGVzY2FwZSAtIEBkZXByZWNhdGVkIHVzZSAkbG9hZGVyLT53aXRoRXNjYXBlKCkgaW5zdGVhZAogKiBAcGFyYW0gaW50PDEsIG1heD4gJGNoYXJhY3RlcnNfcmVhZF9pbl9saW5lIC0gQGRlcHJlY2F0ZWQgdXNlICRsb2FkZXItPndpdGhDaGFyYWN0ZXJzUmVhZEluTGluZSgpIGluc3RlYWQKICogQHBhcmFtIG51bGx8U2NoZW1hICRzY2hlbWEgLSBAZGVwcmVjYXRlZCB1c2UgJGxvYWRlci0+d2l0aFNjaGVtYSgpIGluc3RlYWQKICov"},{"repository_path":"src\/adapter\/etl-adapter-csv\/src\/Flow\/ETL\/Adapter\/CSV\/functions.php","start_line_in_file":80,"slug":"to-csv","name":"to_csv","namespace":"Flow\\ETL\\Adapter\\CSV","parameters":[{"name":"uri","type":[{"name":"Path","namespace":"Flow\\Filesystem","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"with_header","type":[{"name":"bool","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"true"},{"name":"separator","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"','"},{"name":"enclosure","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"'\\\"'"},{"name":"escape","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"'\\\\'"},{"name":"new_line_separator","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"'\\n'"},{"name":"datetime_format","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"'Y-m-d\\\\TH:i:sP'"},{"name":"filesystem","type":[{"name":"Filesystem","namespace":"Flow\\Filesystem","is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"Flow\\Filesystem\\Local\\NativeLocalFilesystem::..."}],"return_type":[{"name":"CSVLoader","namespace":"Flow\\ETL\\Adapter\\CSV","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"CSV","type":"LOADER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIEBwYXJhbSBQYXRofHN0cmluZyAkdXJpCiAqIEBwYXJhbSBib29sICR3aXRoX2hlYWRlciAtIEBkZXByZWNhdGVkIHVzZSAkbG9hZGVyLT53aXRoSGVhZGVyKCkgaW5zdGVhZAogKiBAcGFyYW0gc3RyaW5nICRzZXBhcmF0b3IgLSBAZGVwcmVjYXRlZCB1c2UgJGxvYWRlci0+d2l0aFNlcGFyYXRvcigpIGluc3RlYWQKICogQHBhcmFtIHN0cmluZyAkZW5jbG9zdXJlIC0gQGRlcHJlY2F0ZWQgdXNlICRsb2FkZXItPndpdGhFbmNsb3N1cmUoKSBpbnN0ZWFkCiAqIEBwYXJhbSBzdHJpbmcgJGVzY2FwZSAtIEBkZXByZWNhdGVkIHVzZSAkbG9hZGVyLT53aXRoRXNjYXBlKCkgaW5zdGVhZAogKiBAcGFyYW0gc3RyaW5nICRuZXdfbGluZV9zZXBhcmF0b3IgLSBAZGVwcmVjYXRlZCB1c2UgJGxvYWRlci0+d2l0aE5ld0xpbmVTZXBhcmF0b3IoKSBpbnN0ZWFkCiAqIEBwYXJhbSBzdHJpbmcgJGRhdGV0aW1lX2Zvcm1hdCAtIEBkZXByZWNhdGVkIHVzZSAkbG9hZGVyLT53aXRoRGF0ZVRpbWVGb3JtYXQoKSBpbnN0ZWFkCiAqLw=="},{"repository_path":"src\/adapter\/etl-adapter-csv\/src\/Flow\/ETL\/Adapter\/CSV\/functions.php","start_line_in_file":106,"slug":"csv-detect-separator","name":"csv_detect_separator","namespace":"Flow\\ETL\\Adapter\\CSV","parameters":[{"name":"stream","type":[{"name":"SourceStream","namespace":"Flow\\Filesystem","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"lines","type":[{"name":"int","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"5"},{"name":"fallback","type":[{"name":"Option","namespace":"Flow\\ETL\\Adapter\\CSV\\Detector","is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"Flow\\ETL\\Adapter\\CSV\\Detector\\Option::..."},{"name":"options","type":[{"name":"Options","namespace":"Flow\\ETL\\Adapter\\CSV\\Detector","is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"null"}],"return_type":[{"name":"Option","namespace":"Flow\\ETL\\Adapter\\CSV\\Detector","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"CSV","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIEBwYXJhbSBTb3VyY2VTdHJlYW0gJHN0cmVhbSAtIHZhbGlkIHJlc291cmNlIHRvIENTViBmaWxlCiAqIEBwYXJhbSBpbnQ8MSwgbWF4PiAkbGluZXMgLSBudW1iZXIgb2YgbGluZXMgdG8gcmVhZCBmcm9tIENTViBmaWxlLCBkZWZhdWx0IDUsIG1vcmUgbGluZXMgbWVhbnMgbW9yZSBhY2N1cmF0ZSBkZXRlY3Rpb24gYnV0IHNsb3dlciBkZXRlY3Rpb24KICogQHBhcmFtIG51bGx8T3B0aW9uICRmYWxsYmFjayAtIGZhbGxiYWNrIG9wdGlvbiB0byB1c2Ugd2hlbiBubyBiZXN0IG9wdGlvbiBjYW4gYmUgZGV0ZWN0ZWQsIGRlZmF1bHQgaXMgT3B0aW9uKCcsJywgJyInLCAnXFwnKQogKiBAcGFyYW0gbnVsbHxPcHRpb25zICRvcHRpb25zIC0gb3B0aW9ucyB0byB1c2UgZm9yIGRldGVjdGlvbiwgZGVmYXVsdCBpcyBPcHRpb25zOjphbGwoKQogKi8="},{"repository_path":"src\/adapter\/etl-adapter-doctrine\/src\/Flow\/ETL\/Adapter\/Doctrine\/functions.php","start_line_in_file":41,"slug":"dbal-dataframe-factory","name":"dbal_dataframe_factory","namespace":"Flow\\ETL\\Adapter\\Doctrine","parameters":[{"name":"connection","type":[{"name":"Connection","namespace":"Doctrine\\DBAL","is_nullable":false,"is_variadic":false},{"name":"array","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"query","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"parameters","type":[{"name":"QueryParameter","namespace":"Flow\\ETL\\Adapter\\Doctrine","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":true,"default_value":null}],"return_type":[{"name":"DbalDataFrameFactory","namespace":"Flow\\ETL\\Adapter\\Doctrine","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"DOCTRINE","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIEBwYXJhbSBhcnJheTxzdHJpbmcsIG1peGVkPnxDb25uZWN0aW9uICRjb25uZWN0aW9uCiAqIEBwYXJhbSBzdHJpbmcgJHF1ZXJ5CiAqIEBwYXJhbSBRdWVyeVBhcmFtZXRlciAuLi4kcGFyYW1ldGVycwogKi8="},{"repository_path":"src\/adapter\/etl-adapter-doctrine\/src\/Flow\/ETL\/Adapter\/Doctrine\/functions.php","start_line_in_file":61,"slug":"from-dbal-limit-offset","name":"from_dbal_limit_offset","namespace":"Flow\\ETL\\Adapter\\Doctrine","parameters":[{"name":"connection","type":[{"name":"Connection","namespace":"Doctrine\\DBAL","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"table","type":[{"name":"Table","namespace":"Flow\\ETL\\Adapter\\Doctrine","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"order_by","type":[{"name":"OrderBy","namespace":"Flow\\ETL\\Adapter\\Doctrine","is_nullable":false,"is_variadic":false},{"name":"array","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"page_size","type":[{"name":"int","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"1000"},{"name":"maximum","type":[{"name":"int","namespace":null,"is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"null"}],"return_type":[{"name":"DbalLimitOffsetExtractor","namespace":"Flow\\ETL\\Adapter\\Doctrine","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"DOCTRINE","type":"EXTRACTOR"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIEBwYXJhbSBDb25uZWN0aW9uICRjb25uZWN0aW9uCiAqIEBwYXJhbSBzdHJpbmd8VGFibGUgJHRhYmxlCiAqIEBwYXJhbSBhcnJheTxPcmRlckJ5PnxPcmRlckJ5ICRvcmRlcl9ieQogKiBAcGFyYW0gaW50ICRwYWdlX3NpemUgLSBiZWNvbWVzIHRoZSBleHRyYWN0b3IncyBiYXRjaCBzaXplOiByb3dzIHBlciBwYWdlCiAqIEBwYXJhbSBudWxsfGludCAkbWF4aW11bQogKgogKiBAdGhyb3dzIEludmFsaWRBcmd1bWVudEV4Y2VwdGlvbgogKi8="},{"repository_path":"src\/adapter\/etl-adapter-doctrine\/src\/Flow\/ETL\/Adapter\/Doctrine\/functions.php","start_line_in_file":88,"slug":"from-dbal-limit-offset-qb","name":"from_dbal_limit_offset_qb","namespace":"Flow\\ETL\\Adapter\\Doctrine","parameters":[{"name":"connection","type":[{"name":"Connection","namespace":"Doctrine\\DBAL","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"queryBuilder","type":[{"name":"QueryBuilder","namespace":"Doctrine\\DBAL\\Query","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"page_size","type":[{"name":"int","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"1000"},{"name":"maximum","type":[{"name":"int","namespace":null,"is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"null"},{"name":"offset","type":[{"name":"int","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"0"}],"return_type":[{"name":"DbalLimitOffsetExtractor","namespace":"Flow\\ETL\\Adapter\\Doctrine","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"DOCTRINE","type":"EXTRACTOR"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIEBwYXJhbSBDb25uZWN0aW9uICRjb25uZWN0aW9uCiAqIEBwYXJhbSBpbnQgJHBhZ2Vfc2l6ZSAtIGJlY29tZXMgdGhlIGV4dHJhY3RvcidzIGJhdGNoIHNpemU6IHJvd3MgcGVyIHBhZ2UKICogQHBhcmFtIG51bGx8aW50ICRtYXhpbXVtIC0gbWF4aW11bSBjYW4gYWxzbyBiZSB0YWtlbiBmcm9tIGEgcXVlcnkgYnVpbGRlciwgJG1heGltdW0gaG93ZXZlciBpcyB1c2VkIHJlZ2FyZGxlc3Mgb2YgdGhlIHF1ZXJ5IGJ1aWxkZXIgaWYgaXQncyBzZXQKICogQHBhcmFtIGludCAkb2Zmc2V0IC0gb2Zmc2V0IGNhbiBhbHNvIGJlIHRha2VuIGZyb20gYSBxdWVyeSBidWlsZGVyLCAkb2Zmc2V0IGhvd2V2ZXIgaXMgdXNlZCByZWdhcmRsZXNzIG9mIHRoZSBxdWVyeSBidWlsZGVyIGlmIGl0J3Mgc2V0IHRvIG5vbiAwIHZhbHVlCiAqLw=="},{"repository_path":"src\/adapter\/etl-adapter-doctrine\/src\/Flow\/ETL\/Adapter\/Doctrine\/functions.php","start_line_in_file":107,"slug":"from-dbal-key-set-qb","name":"from_dbal_key_set_qb","namespace":"Flow\\ETL\\Adapter\\Doctrine","parameters":[{"name":"connection","type":[{"name":"Connection","namespace":"Doctrine\\DBAL","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"queryBuilder","type":[{"name":"QueryBuilder","namespace":"Doctrine\\DBAL\\Query","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"key_set","type":[{"name":"KeySet","namespace":"Flow\\ETL\\Adapter\\Doctrine\\Pagination","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"DbalKeySetExtractor","namespace":"Flow\\ETL\\Adapter\\Doctrine","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"DOCTRINE","type":"EXTRACTOR"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/adapter\/etl-adapter-doctrine\/src\/Flow\/ETL\/Adapter\/Doctrine\/functions.php","start_line_in_file":117,"slug":"from-dbal-queries","name":"from_dbal_queries","namespace":"Flow\\ETL\\Adapter\\Doctrine","parameters":[{"name":"connection","type":[{"name":"Connection","namespace":"Doctrine\\DBAL","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"query","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"parameters_set","type":[{"name":"ParametersSet","namespace":"Flow\\ETL\\Adapter\\Doctrine","is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"null"},{"name":"types","type":[{"name":"array","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"[]"}],"return_type":[{"name":"DbalQueryExtractor","namespace":"Flow\\ETL\\Adapter\\Doctrine","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"DOCTRINE","type":"EXTRACTOR"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIEBwYXJhbSBudWxsfFBhcmFtZXRlcnNTZXQgJHBhcmFtZXRlcnNfc2V0IC0gZWFjaCBvbmUgcGFyYW1ldGVycyBhcnJheSB3aWxsIGJlIGV2YWx1YXRlZCBhcyBuZXcgcXVlcnkKICogQHBhcmFtIGFycmF5PGludDwwLCBtYXg+fHN0cmluZywgRGJhbEFycmF5VHlwZXxEYmFsUGFyYW1ldGVyVHlwZXxEYmFsVHlwZXxzdHJpbmc+ICR0eXBlcwogKi8="},{"repository_path":"src\/adapter\/etl-adapter-doctrine\/src\/Flow\/ETL\/Adapter\/Doctrine\/functions.php","start_line_in_file":143,"slug":"dbal-from-queries","name":"dbal_from_queries","namespace":"Flow\\ETL\\Adapter\\Doctrine","parameters":[{"name":"connection","type":[{"name":"Connection","namespace":"Doctrine\\DBAL","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"query","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"parameters_set","type":[{"name":"ParametersSet","namespace":"Flow\\ETL\\Adapter\\Doctrine","is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"null"},{"name":"types","type":[{"name":"array","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"[]"}],"return_type":[{"name":"DbalQueryExtractor","namespace":"Flow\\ETL\\Adapter\\Doctrine","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"DOCTRINE","type":"EXTRACTOR"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIEBkZXByZWNhdGVkIHVzZSBmcm9tX2RiYWxfcXVlcmllcygpIGluc3RlYWQKICoKICogQHBhcmFtIG51bGx8UGFyYW1ldGVyc1NldCAkcGFyYW1ldGVyc19zZXQgLSBlYWNoIG9uZSBwYXJhbWV0ZXJzIGFycmF5IHdpbGwgYmUgZXZhbHVhdGVkIGFzIG5ldyBxdWVyeQogKiBAcGFyYW0gYXJyYXk8aW50PDAsIG1heD58c3RyaW5nLCBEYmFsQXJyYXlUeXBlfERiYWxQYXJhbWV0ZXJUeXBlfERiYWxUeXBlfHN0cmluZz4gJHR5cGVzCiAqLw=="},{"repository_path":"src\/adapter\/etl-adapter-doctrine\/src\/Flow\/ETL\/Adapter\/Doctrine\/functions.php","start_line_in_file":157,"slug":"from-dbal-query","name":"from_dbal_query","namespace":"Flow\\ETL\\Adapter\\Doctrine","parameters":[{"name":"connection","type":[{"name":"Connection","namespace":"Doctrine\\DBAL","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"query","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"parameters","type":[{"name":"array","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"[]"},{"name":"types","type":[{"name":"array","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"[]"}],"return_type":[{"name":"DbalQueryExtractor","namespace":"Flow\\ETL\\Adapter\\Doctrine","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"DOCTRINE","type":"EXTRACTOR"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIEBwYXJhbSBhcnJheTxzdHJpbmcsIG1peGVkPnxsaXN0PG1peGVkPiAkcGFyYW1ldGVycyAtIEBkZXByZWNhdGVkIHVzZSBEYmFsUXVlcnlFeHRyYWN0b3I6OndpdGhQYXJhbWV0ZXJzKCkgaW5zdGVhZAogKiBAcGFyYW0gYXJyYXk8aW50PDAsIG1heD58c3RyaW5nLCBEYmFsQXJyYXlUeXBlfERiYWxQYXJhbWV0ZXJUeXBlfERiYWxUeXBlfHN0cmluZz4gJHR5cGVzIC0gQGRlcHJlY2F0ZWQgdXNlIERiYWxRdWVyeUV4dHJhY3Rvcjo6d2l0aFR5cGVzKCkgaW5zdGVhZAogKi8="},{"repository_path":"src\/adapter\/etl-adapter-doctrine\/src\/Flow\/ETL\/Adapter\/Doctrine\/functions.php","start_line_in_file":173,"slug":"dbal-from-query","name":"dbal_from_query","namespace":"Flow\\ETL\\Adapter\\Doctrine","parameters":[{"name":"connection","type":[{"name":"Connection","namespace":"Doctrine\\DBAL","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"query","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"parameters","type":[{"name":"array","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"[]"},{"name":"types","type":[{"name":"array","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"[]"}],"return_type":[{"name":"DbalQueryExtractor","namespace":"Flow\\ETL\\Adapter\\Doctrine","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"DOCTRINE","type":"EXTRACTOR"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIEBkZXByZWNhdGVkIHVzZSBmcm9tX2RiYWxfcXVlcnkoKSBpbnN0ZWFkCiAqCiAqIEBwYXJhbSBhcnJheTxzdHJpbmcsIG1peGVkPnxsaXN0PG1peGVkPiAkcGFyYW1ldGVycyAtIEBkZXByZWNhdGVkIHVzZSBEYmFsUXVlcnlFeHRyYWN0b3I6OndpdGhQYXJhbWV0ZXJzKCkgaW5zdGVhZAogKiBAcGFyYW0gYXJyYXk8aW50PDAsIG1heD58c3RyaW5nLCBEYmFsQXJyYXlUeXBlfERiYWxQYXJhbWV0ZXJUeXBlfERiYWxUeXBlfHN0cmluZz4gJHR5cGVzIC0gQGRlcHJlY2F0ZWQgdXNlIERiYWxRdWVyeUV4dHJhY3Rvcjo6d2l0aFR5cGVzKCkgaW5zdGVhZAogKi8="},{"repository_path":"src\/adapter\/etl-adapter-doctrine\/src\/Flow\/ETL\/Adapter\/Doctrine\/functions.php","start_line_in_file":199,"slug":"to-dbal-table-insert","name":"to_dbal_table_insert","namespace":"Flow\\ETL\\Adapter\\Doctrine","parameters":[{"name":"connection","type":[{"name":"Connection","namespace":"Doctrine\\DBAL","is_nullable":false,"is_variadic":false},{"name":"array","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"table","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"options","type":[{"name":"InsertOptions","namespace":"Flow\\Doctrine\\Bulk","is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"null"}],"return_type":[{"name":"DbalLoader","namespace":"Flow\\ETL\\Adapter\\Doctrine","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"DOCTRINE","type":"LOADER"}},{"name":"DocumentationExample","namespace":"Flow\\Documentation\\Attribute","arguments":{"topic":"writing","example":"dbal","option":"upsert"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIEluc2VydCBuZXcgcm93cyBpbnRvIGEgZGF0YWJhc2UgdGFibGUuCiAqIEluc2VydCBjYW4gYWxzbyBiZSB1c2VkIGFzIGFuIHVwc2VydCB3aXRoIHRoZSBoZWxwIG9mIEluc2VydE9wdGlvbnMuCiAqIEluc2VydE9wdGlvbnMgYXJlIHBsYXRmb3JtIHNwZWNpZmljLCBzbyBwbGVhc2UgY2hvb3NlIHRoZSByaWdodCBvbmUgZm9yIHlvdXIgZGF0YWJhc2UuCiAqCiAqICAtIE15U1FMSW5zZXJ0T3B0aW9ucwogKiAgLSBQb3N0Z3JlU1FMSW5zZXJ0T3B0aW9ucwogKiAgLSBTcWxpdGVJbnNlcnRPcHRpb25zCiAqCiAqIEluIG9yZGVyIHRvIGNvbnRyb2wgdGhlIHNpemUgb2YgdGhlIHNpbmdsZSBpbnNlcnQsIHVzZSBEYXRhRnJhbWU6OmNodW5rU2l6ZSgpIG1ldGhvZCBqdXN0IGJlZm9yZSBjYWxsaW5nIERhdGFGcmFtZTo6bG9hZCgpLgogKgogKiBAcGFyYW0gYXJyYXk8c3RyaW5nLCBtaXhlZD58Q29ubmVjdGlvbiAkY29ubmVjdGlvbgogKgogKiBAdGhyb3dzIEludmFsaWRBcmd1bWVudEV4Y2VwdGlvbgogKi8="},{"repository_path":"src\/adapter\/etl-adapter-doctrine\/src\/Flow\/ETL\/Adapter\/Doctrine\/functions.php","start_line_in_file":216,"slug":"to-dbal-table-update","name":"to_dbal_table_update","namespace":"Flow\\ETL\\Adapter\\Doctrine","parameters":[{"name":"connection","type":[{"name":"Connection","namespace":"Doctrine\\DBAL","is_nullable":false,"is_variadic":false},{"name":"array","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"table","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"options","type":[{"name":"UpdateOptions","namespace":"Flow\\Doctrine\\Bulk","is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"null"}],"return_type":[{"name":"DbalLoader","namespace":"Flow\\ETL\\Adapter\\Doctrine","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"DOCTRINE","type":"LOADER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqICBVcGRhdGUgZXhpc3Rpbmcgcm93cyBpbiBkYXRhYmFzZS4KICoKICogIEluIG9yZGVyIHRvIGNvbnRyb2wgdGhlIHNpemUgb2YgdGhlIHNpbmdsZSByZXF1ZXN0LCB1c2UgRGF0YUZyYW1lOjpjaHVua1NpemUoKSBtZXRob2QganVzdCBiZWZvcmUgY2FsbGluZyBEYXRhRnJhbWU6OmxvYWQoKS4KICoKICogQHBhcmFtIGFycmF5PHN0cmluZywgbWl4ZWQ+fENvbm5lY3Rpb24gJGNvbm5lY3Rpb24KICoKICogQHRocm93cyBJbnZhbGlkQXJndW1lbnRFeGNlcHRpb24KICov"},{"repository_path":"src\/adapter\/etl-adapter-doctrine\/src\/Flow\/ETL\/Adapter\/Doctrine\/functions.php","start_line_in_file":235,"slug":"to-dbal-table-delete","name":"to_dbal_table_delete","namespace":"Flow\\ETL\\Adapter\\Doctrine","parameters":[{"name":"connection","type":[{"name":"Connection","namespace":"Doctrine\\DBAL","is_nullable":false,"is_variadic":false},{"name":"array","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"table","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"DbalLoader","namespace":"Flow\\ETL\\Adapter\\Doctrine","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"DOCTRINE","type":"LOADER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIERlbGV0ZSByb3dzIGZyb20gZGF0YWJhc2UgdGFibGUgYmFzZWQgb24gdGhlIHByb3ZpZGVkIGRhdGEuCiAqCiAqIEluIG9yZGVyIHRvIGNvbnRyb2wgdGhlIHNpemUgb2YgdGhlIHNpbmdsZSByZXF1ZXN0LCB1c2UgRGF0YUZyYW1lOjpjaHVua1NpemUoKSBtZXRob2QganVzdCBiZWZvcmUgY2FsbGluZyBEYXRhRnJhbWU6OmxvYWQoKS4KICoKICogQHBhcmFtIGFycmF5PHN0cmluZywgbWl4ZWQ+fENvbm5lY3Rpb24gJGNvbm5lY3Rpb24KICoKICogQHRocm93cyBJbnZhbGlkQXJndW1lbnRFeGNlcHRpb24KICov"},{"repository_path":"src\/adapter\/etl-adapter-doctrine\/src\/Flow\/ETL\/Adapter\/Doctrine\/functions.php","start_line_in_file":250,"slug":"to-dbal-schema-table","name":"to_dbal_schema_table","namespace":"Flow\\ETL\\Adapter\\Doctrine","parameters":[{"name":"schema","type":[{"name":"Schema","namespace":"Flow\\ETL","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"table_name","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"table_options","type":[{"name":"array","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"[]"},{"name":"types_map","type":[{"name":"array","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"[]"}],"return_type":[{"name":"Table","namespace":"Doctrine\\DBAL\\Schema","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"DOCTRINE","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENvbnZlcnRzIGEgRmxvd1xFVExcU2NoZW1hIHRvIGEgRG9jdHJpbmVcREJBTFxTY2hlbWFcVGFibGUuCiAqCiAqIEBwYXJhbSBTY2hlbWEgJHNjaGVtYQogKiBAcGFyYW0gYXJyYXk8c3RyaW5nLCBtaXhlZD4gJHRhYmxlX29wdGlvbnMKICogQHBhcmFtIGFycmF5PGNsYXNzLXN0cmluZzxcRmxvd1xUeXBlc1xUeXBlPG1peGVkPj4sIGNsYXNzLXN0cmluZzxcRG9jdHJpbmVcREJBTFxUeXBlc1xUeXBlPj4gJHR5cGVzX21hcAogKi8="},{"repository_path":"src\/adapter\/etl-adapter-doctrine\/src\/Flow\/ETL\/Adapter\/Doctrine\/functions.php","start_line_in_file":267,"slug":"table-schema-to-flow-schema","name":"table_schema_to_flow_schema","namespace":"Flow\\ETL\\Adapter\\Doctrine","parameters":[{"name":"table","type":[{"name":"Table","namespace":"Doctrine\\DBAL\\Schema","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"types_map","type":[{"name":"array","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"[]"}],"return_type":[{"name":"Schema","namespace":"Flow\\ETL","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"DOCTRINE","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENvbnZlcnRzIGEgRG9jdHJpbmVcREJBTFxTY2hlbWFcVGFibGUgdG8gYSBGbG93XEVUTFxTY2hlbWEuCiAqCiAqIEBwYXJhbSBhcnJheTxjbGFzcy1zdHJpbmc8XEZsb3dcVHlwZXNcVHlwZTxtaXhlZD4+LCBjbGFzcy1zdHJpbmc8XERvY3RyaW5lXERCQUxcVHlwZXNcVHlwZT4+ICR0eXBlc19tYXAKICoKICogQHJldHVybiBTY2hlbWEKICov"},{"repository_path":"src\/adapter\/etl-adapter-doctrine\/src\/Flow\/ETL\/Adapter\/Doctrine\/functions.php","start_line_in_file":278,"slug":"postgresql-insert-options","name":"postgresql_insert_options","namespace":"Flow\\ETL\\Adapter\\Doctrine","parameters":[{"name":"skip_conflicts","type":[{"name":"bool","namespace":null,"is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"null"},{"name":"constraint","type":[{"name":"string","namespace":null,"is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"null"},{"name":"conflict_columns","type":[{"name":"array","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"[]"},{"name":"update_columns","type":[{"name":"array","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"[]"}],"return_type":[{"name":"PostgreSQLInsertOptions","namespace":"Flow\\Doctrine\\Bulk\\Dialect","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"DOCTRINE","type":"HELPER"}},{"name":"DocumentationExample","namespace":"Flow\\Documentation\\Attribute","arguments":{"topic":"writing","example":"dbal","option":"upsert"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIEBwYXJhbSBhcnJheTxzdHJpbmc+ICRjb25mbGljdF9jb2x1bW5zCiAqIEBwYXJhbSBhcnJheTxzdHJpbmc+ICR1cGRhdGVfY29sdW1ucwogKi8="},{"repository_path":"src\/adapter\/etl-adapter-doctrine\/src\/Flow\/ETL\/Adapter\/Doctrine\/functions.php","start_line_in_file":291,"slug":"mysql-insert-options","name":"mysql_insert_options","namespace":"Flow\\ETL\\Adapter\\Doctrine","parameters":[{"name":"skip_conflicts","type":[{"name":"bool","namespace":null,"is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"null"},{"name":"upsert","type":[{"name":"bool","namespace":null,"is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"null"},{"name":"update_columns","type":[{"name":"array","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"[]"}],"return_type":[{"name":"MySQLInsertOptions","namespace":"Flow\\Doctrine\\Bulk\\Dialect","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"DOCTRINE","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIEBwYXJhbSBhcnJheTxzdHJpbmc+ICR1cGRhdGVfY29sdW1ucwogKi8="},{"repository_path":"src\/adapter\/etl-adapter-doctrine\/src\/Flow\/ETL\/Adapter\/Doctrine\/functions.php","start_line_in_file":304,"slug":"sqlite-insert-options","name":"sqlite_insert_options","namespace":"Flow\\ETL\\Adapter\\Doctrine","parameters":[{"name":"skip_conflicts","type":[{"name":"bool","namespace":null,"is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"null"},{"name":"conflict_columns","type":[{"name":"array","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"[]"},{"name":"update_columns","type":[{"name":"array","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"[]"}],"return_type":[{"name":"SqliteInsertOptions","namespace":"Flow\\Doctrine\\Bulk\\Dialect","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"DOCTRINE","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIEBwYXJhbSBhcnJheTxzdHJpbmc+ICRjb25mbGljdF9jb2x1bW5zCiAqIEBwYXJhbSBhcnJheTxzdHJpbmc+ICR1cGRhdGVfY29sdW1ucwogKi8="},{"repository_path":"src\/adapter\/etl-adapter-doctrine\/src\/Flow\/ETL\/Adapter\/Doctrine\/functions.php","start_line_in_file":317,"slug":"postgresql-update-options","name":"postgresql_update_options","namespace":"Flow\\ETL\\Adapter\\Doctrine","parameters":[{"name":"primary_key_columns","type":[{"name":"array","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"[]"},{"name":"update_columns","type":[{"name":"array","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"[]"}],"return_type":[{"name":"PostgreSQLUpdateOptions","namespace":"Flow\\Doctrine\\Bulk\\Dialect","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"DOCTRINE","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIEBwYXJhbSBhcnJheTxzdHJpbmc+ICRwcmltYXJ5X2tleV9jb2x1bW5zCiAqIEBwYXJhbSBhcnJheTxzdHJpbmc+ICR1cGRhdGVfY29sdW1ucwogKi8="},{"repository_path":"src\/adapter\/etl-adapter-doctrine\/src\/Flow\/ETL\/Adapter\/Doctrine\/functions.php","start_line_in_file":338,"slug":"to-dbal-transaction","name":"to_dbal_transaction","namespace":"Flow\\ETL\\Adapter\\Doctrine","parameters":[{"name":"connection","type":[{"name":"Connection","namespace":"Doctrine\\DBAL","is_nullable":false,"is_variadic":false},{"name":"array","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"sinks","type":[{"name":"Loader","namespace":"Flow\\ETL","is_nullable":false,"is_variadic":false},{"name":"Sink","namespace":"Flow\\ETL","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":true,"default_value":null}],"return_type":[{"name":"Transactional","namespace":"Flow\\ETL\\Sink","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"DOCTRINE","type":"LOADER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIFdyaXRlIGV2ZXJ5IHNpbmsgd2l0aGluIGRhdGFiYXNlIHRyYW5zYWN0aW9ucy4KICogRWFjaCBiYXRjaCBvZiByb3dzIGlzIHdyaXR0ZW4gaW4gaXRzIG93biB0cmFuc2FjdGlvbjsgcm93cyBhIHNpbmsncyBUcmFuc2Zvcm1hdGlvbiBkZWxpdmVycyB3aGVuCiAqIHRoZSBydW4gZW5kcyAoYmxvY2tpbmcgb3BlcmF0aW9ucyBkcmFpbiB0aGVyZSkgYXJlIGNvbW1pdHRlZCBpbiBvbmUgZmluYWwgdHJhbnNhY3Rpb24uCiAqIElmIGFueSBzaW5rIGZhaWxzLCB0aGUgb3BlbiB0cmFuc2FjdGlvbiBpcyByb2xsZWQgYmFjay4KICogQSBwbGFpbiBMb2FkZXIgY2hpbGQgaXMgYSBiYXJlIHNpbmsgcm9vdDsgYSB0b190cmFuc2Zvcm1hdGlvbiguLi4pIGNoaWxkIGRlbGl2ZXJzIGluc2lkZSB0aGUgc2FtZQogKiB0cmFuc2FjdGlvbi4gRXZlcnkgY2hpbGQncyBsb2FkZXIgbXVzdCB1c2UgdGhlIHNhbWUgY29ubmVjdGlvbiBhcyB0aGUgdHJhbnNhY3Rpb246IHBhc3Mgb25lIGxpdmUKICogQ29ubmVjdGlvbiB0byBib3RoIC0gYSBsb2FkZXIgYnVpbHQgZnJvbSBhcnJheSBwYXJhbXMgb3BlbnMgaXRzIG93biBjb25uZWN0aW9uIGFuZCBlc2NhcGVzIHRoZQogKiB0cmFuc2FjdGlvbi4KICoKICogQHBhcmFtIGFycmF5PHN0cmluZywgbWl4ZWQ+fENvbm5lY3Rpb24gJGNvbm5lY3Rpb24KICogQHBhcmFtIExvYWRlcnxTaW5rIC4uLiRzaW5rcyAtIHNpbmtzIHdyaXR0ZW4gd2l0aGluIHRoZSB0cmFuc2FjdGlvbgogKgogKiBAdGhyb3dzIEludmFsaWRBcmd1bWVudEV4Y2VwdGlvbgogKi8="},{"repository_path":"src\/adapter\/etl-adapter-doctrine\/src\/Flow\/ETL\/Adapter\/Doctrine\/functions.php","start_line_in_file":347,"slug":"pagination-key-asc","name":"pagination_key_asc","namespace":"Flow\\ETL\\Adapter\\Doctrine","parameters":[{"name":"column","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"type","type":[{"name":"ParameterType","namespace":"Doctrine\\DBAL","is_nullable":false,"is_variadic":false},{"name":"Type","namespace":"Doctrine\\DBAL\\Types","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"Doctrine\\DBAL\\ParameterType::..."}],"return_type":[{"name":"Key","namespace":"Flow\\ETL\\Adapter\\Doctrine\\Pagination","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"DOCTRINE","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/adapter\/etl-adapter-doctrine\/src\/Flow\/ETL\/Adapter\/Doctrine\/functions.php","start_line_in_file":353,"slug":"pagination-key-desc","name":"pagination_key_desc","namespace":"Flow\\ETL\\Adapter\\Doctrine","parameters":[{"name":"column","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"type","type":[{"name":"ParameterType","namespace":"Doctrine\\DBAL","is_nullable":false,"is_variadic":false},{"name":"Type","namespace":"Doctrine\\DBAL\\Types","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"Doctrine\\DBAL\\ParameterType::..."}],"return_type":[{"name":"Key","namespace":"Flow\\ETL\\Adapter\\Doctrine\\Pagination","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"DOCTRINE","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/adapter\/etl-adapter-doctrine\/src\/Flow\/ETL\/Adapter\/Doctrine\/functions.php","start_line_in_file":359,"slug":"pagination-key-set","name":"pagination_key_set","namespace":"Flow\\ETL\\Adapter\\Doctrine","parameters":[{"name":"keys","type":[{"name":"Key","namespace":"Flow\\ETL\\Adapter\\Doctrine\\Pagination","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":true,"default_value":null}],"return_type":[{"name":"KeySet","namespace":"Flow\\ETL\\Adapter\\Doctrine\\Pagination","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"DOCTRINE","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/adapter\/etl-adapter-excel\/src\/Flow\/ETL\/Adapter\/Excel\/DSL\/functions.php","start_line_in_file":22,"slug":"from-excel","name":"from_excel","namespace":"Flow\\ETL\\Adapter\\Excel\\DSL","parameters":[{"name":"path","type":[{"name":"Path","namespace":"Flow\\Filesystem","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"filesystem","type":[{"name":"Filesystem","namespace":"Flow\\Filesystem","is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"Flow\\Filesystem\\Local\\NativeLocalFilesystem::..."}],"return_type":[{"name":"ExcelExtractor","namespace":"Flow\\ETL\\Adapter\\Excel","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"EXCEL","type":"EXTRACTOR"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/adapter\/etl-adapter-excel\/src\/Flow\/ETL\/Adapter\/Excel\/DSL\/functions.php","start_line_in_file":28,"slug":"to-excel","name":"to_excel","namespace":"Flow\\ETL\\Adapter\\Excel\\DSL","parameters":[{"name":"path","type":[{"name":"Path","namespace":"Flow\\Filesystem","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"filesystem","type":[{"name":"Filesystem","namespace":"Flow\\Filesystem","is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"Flow\\Filesystem\\Local\\NativeLocalFilesystem::..."}],"return_type":[{"name":"ExcelLoader","namespace":"Flow\\ETL\\Adapter\\Excel","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"EXCEL","type":"LOADER"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/adapter\/etl-adapter-excel\/src\/Flow\/ETL\/Adapter\/Excel\/DSL\/functions.php","start_line_in_file":34,"slug":"is-valid-excel-sheet-name","name":"is_valid_excel_sheet_name","namespace":"Flow\\ETL\\Adapter\\Excel\\DSL","parameters":[{"name":"sheet_name","type":[{"name":"ScalarFunction","namespace":"Flow\\ETL\\Function","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"IsValidExcelSheetName","namespace":"Flow\\ETL\\Adapter\\Excel\\Function","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"EXCEL","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/adapter\/etl-adapter-google-sheet\/src\/Flow\/ETL\/Adapter\/GoogleSheet\/functions.php","start_line_in_file":22,"slug":"from-google-sheet","name":"from_google_sheet","namespace":"Flow\\ETL\\Adapter\\GoogleSheet","parameters":[{"name":"auth_config","type":[{"name":"Sheets","namespace":"Google\\Service","is_nullable":false,"is_variadic":false},{"name":"array","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"spreadsheet_id","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"sheet_name","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"with_header","type":[{"name":"bool","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"true"},{"name":"rows_per_page","type":[{"name":"int","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"1000"},{"name":"options","type":[{"name":"array","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"[]"}],"return_type":[{"name":"GoogleSheetExtractor","namespace":"Flow\\ETL\\Adapter\\GoogleSheet","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"GOOGLE_SHEET","type":"EXTRACTOR"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIEBwYXJhbSBhcnJheXt0eXBlOiBzdHJpbmcsIHByb2plY3RfaWQ6IHN0cmluZywgcHJpdmF0ZV9rZXlfaWQ6IHN0cmluZywgcHJpdmF0ZV9rZXk6IHN0cmluZywgY2xpZW50X2VtYWlsOiBzdHJpbmcsIGNsaWVudF9pZDogc3RyaW5nLCBhdXRoX3VyaTogc3RyaW5nLCB0b2tlbl91cmk6IHN0cmluZywgYXV0aF9wcm92aWRlcl94NTA5X2NlcnRfdXJsOiBzdHJpbmcsIGNsaWVudF94NTA5X2NlcnRfdXJsOiBzdHJpbmd9fFNoZWV0cyAkYXV0aF9jb25maWcKICogQHBhcmFtIHN0cmluZyAkc3ByZWFkc2hlZXRfaWQKICogQHBhcmFtIHN0cmluZyAkc2hlZXRfbmFtZQogKiBAcGFyYW0gYm9vbCAkd2l0aF9oZWFkZXIgLSBAZGVwcmVjYXRlZCB1c2Ugd2l0aEhlYWRlciBtZXRob2QgaW5zdGVhZAogKiBAcGFyYW0gaW50ICRyb3dzX3Blcl9wYWdlIC0gaG93IG1hbnkgcm93cyBwZXIgcGFnZSB0byBmZXRjaCBmcm9tIEdvb2dsZSBTaGVldHMgQVBJIC0gQGRlcHJlY2F0ZWQgdXNlIHdpdGhSb3dzUGVyUGFnZSBtZXRob2QgaW5zdGVhZAogKiBAcGFyYW0gYXJyYXl7ZGF0ZVRpbWVSZW5kZXJPcHRpb24\/OiBzdHJpbmcsIG1ham9yRGltZW5zaW9uPzogc3RyaW5nLCB2YWx1ZVJlbmRlck9wdGlvbj86IHN0cmluZ30gJG9wdGlvbnMgLSBAZGVwcmVjYXRlZCB1c2Ugd2l0aE9wdGlvbnMgbWV0aG9kIGluc3RlYWQKICov"},{"repository_path":"src\/adapter\/etl-adapter-google-sheet\/src\/Flow\/ETL\/Adapter\/GoogleSheet\/functions.php","start_line_in_file":56,"slug":"from-google-sheet-columns","name":"from_google_sheet_columns","namespace":"Flow\\ETL\\Adapter\\GoogleSheet","parameters":[{"name":"auth_config","type":[{"name":"Sheets","namespace":"Google\\Service","is_nullable":false,"is_variadic":false},{"name":"array","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"spreadsheet_id","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"sheet_name","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"start_range_column","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"end_range_column","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"with_header","type":[{"name":"bool","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"true"},{"name":"rows_per_page","type":[{"name":"int","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"1000"},{"name":"options","type":[{"name":"array","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"[]"}],"return_type":[{"name":"GoogleSheetExtractor","namespace":"Flow\\ETL\\Adapter\\GoogleSheet","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"GOOGLE_SHEET","type":"EXTRACTOR"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIEBwYXJhbSBhcnJheXt0eXBlOiBzdHJpbmcsIHByb2plY3RfaWQ6IHN0cmluZywgcHJpdmF0ZV9rZXlfaWQ6IHN0cmluZywgcHJpdmF0ZV9rZXk6IHN0cmluZywgY2xpZW50X2VtYWlsOiBzdHJpbmcsIGNsaWVudF9pZDogc3RyaW5nLCBhdXRoX3VyaTogc3RyaW5nLCB0b2tlbl91cmk6IHN0cmluZywgYXV0aF9wcm92aWRlcl94NTA5X2NlcnRfdXJsOiBzdHJpbmcsIGNsaWVudF94NTA5X2NlcnRfdXJsOiBzdHJpbmd9fFNoZWV0cyAkYXV0aF9jb25maWcKICogQHBhcmFtIHN0cmluZyAkc3ByZWFkc2hlZXRfaWQKICogQHBhcmFtIHN0cmluZyAkc2hlZXRfbmFtZQogKiBAcGFyYW0gc3RyaW5nICRzdGFydF9yYW5nZV9jb2x1bW4KICogQHBhcmFtIHN0cmluZyAkZW5kX3JhbmdlX2NvbHVtbgogKiBAcGFyYW0gYm9vbCAkd2l0aF9oZWFkZXIgLSBAZGVwcmVjYXRlZCB1c2Ugd2l0aEhlYWRlciBtZXRob2QgaW5zdGVhZAogKiBAcGFyYW0gaW50ICRyb3dzX3Blcl9wYWdlIC0gaG93IG1hbnkgcm93cyBwZXIgcGFnZSB0byBmZXRjaCBmcm9tIEdvb2dsZSBTaGVldHMgQVBJLCBkZWZhdWx0IDEwMDAgLSBAZGVwcmVjYXRlZCB1c2Ugd2l0aFJvd3NQZXJQYWdlIG1ldGhvZCBpbnN0ZWFkCiAqIEBwYXJhbSBhcnJheXtkYXRlVGltZVJlbmRlck9wdGlvbj86IHN0cmluZywgbWFqb3JEaW1lbnNpb24\/OiBzdHJpbmcsIHZhbHVlUmVuZGVyT3B0aW9uPzogc3RyaW5nfSAkb3B0aW9ucyAtIEBkZXByZWNhdGVkIHVzZSB3aXRoT3B0aW9ucyBtZXRob2QgaW5zdGVhZAogKi8="},{"repository_path":"src\/adapter\/etl-adapter-http\/src\/Flow\/ETL\/Adapter\/Http\/DSL\/functions.php","start_line_in_file":35,"slug":"from-dynamic-http-requests","name":"from_dynamic_http_requests","namespace":"Flow\\ETL\\Adapter\\Http","parameters":[{"name":"client","type":[{"name":"ClientInterface","namespace":"Psr\\Http\\Client","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"requestFactory","type":[{"name":"NextRequestFactory","namespace":"Flow\\ETL\\Adapter\\Http\\DynamicExtractor","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"schema","type":[{"name":"Schema","namespace":"Flow\\ETL","is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"null"}],"return_type":[{"name":"PsrHttpClientDynamicExtractor","namespace":"Flow\\ETL\\Adapter\\Http","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"HTTP","type":"EXTRACTOR"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/adapter\/etl-adapter-http\/src\/Flow\/ETL\/Adapter\/Http\/DSL\/functions.php","start_line_in_file":53,"slug":"from-static-http-requests","name":"from_static_http_requests","namespace":"Flow\\ETL\\Adapter\\Http","parameters":[{"name":"client","type":[{"name":"ClientInterface","namespace":"Psr\\Http\\Client","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"requests","type":[{"name":"iterable","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"schema","type":[{"name":"Schema","namespace":"Flow\\ETL","is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"null"}],"return_type":[{"name":"PsrHttpClientStaticExtractor","namespace":"Flow\\ETL\\Adapter\\Http","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"HTTP","type":"EXTRACTOR"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIEBwYXJhbSBpdGVyYWJsZTxSZXF1ZXN0SW50ZXJmYWNlPiAkcmVxdWVzdHMKICov"},{"repository_path":"src\/adapter\/etl-adapter-http\/src\/Flow\/ETL\/Adapter\/Http\/DSL\/functions.php","start_line_in_file":68,"slug":"from-http-paginated","name":"from_http_paginated","namespace":"Flow\\ETL\\Adapter\\Http","parameters":[{"name":"client","type":[{"name":"ClientInterface","namespace":"Psr\\Http\\Client","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"request","type":[{"name":"RequestInterface","namespace":"Psr\\Http\\Message","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"paginator","type":[{"name":"Paginator","namespace":"Flow\\ETL\\Adapter\\Http\\Pagination","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"schema","type":[{"name":"Schema","namespace":"Flow\\ETL","is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"null"}],"return_type":[{"name":"PsrHttpClientPaginatedExtractor","namespace":"Flow\\ETL\\Adapter\\Http","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"HTTP","type":"EXTRACTOR"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/adapter\/etl-adapter-http\/src\/Flow\/ETL\/Adapter\/Http\/DSL\/functions.php","start_line_in_file":84,"slug":"http-pagination-page-number","name":"http_pagination_page_number","namespace":"Flow\\ETL\\Adapter\\Http","parameters":[{"name":"inject","type":[{"name":"RequestOption","namespace":"Flow\\ETL\\Adapter\\Http\\Pagination","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"start_page","type":[{"name":"int","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"1"},{"name":"page_size","type":[{"name":"int","namespace":null,"is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"null"},{"name":"size_option","type":[{"name":"RequestOption","namespace":"Flow\\ETL\\Adapter\\Http\\Pagination","is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"null"},{"name":"inject_on_first_request","type":[{"name":"bool","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"true"},{"name":"records_path","type":[{"name":"string","namespace":null,"is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"null"},{"name":"stop_on_client_error","type":[{"name":"bool","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"true"},{"name":"stop_when","type":[{"name":"StopWhen","namespace":"Flow\\ETL\\Adapter\\Http\\Pagination","is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"null"}],"return_type":[{"name":"Paginator","namespace":"Flow\\ETL\\Adapter\\Http\\Pagination","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"HTTP","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/adapter\/etl-adapter-http\/src\/Flow\/ETL\/Adapter\/Http\/DSL\/functions.php","start_line_in_file":104,"slug":"http-pagination-offset","name":"http_pagination_offset","namespace":"Flow\\ETL\\Adapter\\Http","parameters":[{"name":"offset_option","type":[{"name":"RequestOption","namespace":"Flow\\ETL\\Adapter\\Http\\Pagination","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"limit_option","type":[{"name":"RequestOption","namespace":"Flow\\ETL\\Adapter\\Http\\Pagination","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"limit","type":[{"name":"int","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"start_offset","type":[{"name":"int","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"0"},{"name":"total_path","type":[{"name":"string","namespace":null,"is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"null"},{"name":"inject_on_first_request","type":[{"name":"bool","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"true"},{"name":"stop_on_client_error","type":[{"name":"bool","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"true"},{"name":"stop_when","type":[{"name":"StopWhen","namespace":"Flow\\ETL\\Adapter\\Http\\Pagination","is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"null"}],"return_type":[{"name":"Paginator","namespace":"Flow\\ETL\\Adapter\\Http\\Pagination","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"HTTP","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/adapter\/etl-adapter-http\/src\/Flow\/ETL\/Adapter\/Http\/DSL\/functions.php","start_line_in_file":124,"slug":"http-pagination-cursor","name":"http_pagination_cursor","namespace":"Flow\\ETL\\Adapter\\Http","parameters":[{"name":"cursor_path","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"inject","type":[{"name":"RequestOption","namespace":"Flow\\ETL\\Adapter\\Http\\Pagination","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"stop_on_client_error","type":[{"name":"bool","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"true"},{"name":"stop_when","type":[{"name":"StopWhen","namespace":"Flow\\ETL\\Adapter\\Http\\Pagination","is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"null"}],"return_type":[{"name":"Paginator","namespace":"Flow\\ETL\\Adapter\\Http\\Pagination","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"HTTP","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/adapter\/etl-adapter-http\/src\/Flow\/ETL\/Adapter\/Http\/DSL\/functions.php","start_line_in_file":140,"slug":"http-pagination-next-url","name":"http_pagination_next_url","namespace":"Flow\\ETL\\Adapter\\Http","parameters":[{"name":"path","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"stop_on_client_error","type":[{"name":"bool","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"true"},{"name":"stop_when","type":[{"name":"StopWhen","namespace":"Flow\\ETL\\Adapter\\Http\\Pagination","is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"null"}],"return_type":[{"name":"Paginator","namespace":"Flow\\ETL\\Adapter\\Http\\Pagination","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"HTTP","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/adapter\/etl-adapter-http\/src\/Flow\/ETL\/Adapter\/Http\/DSL\/functions.php","start_line_in_file":155,"slug":"http-pagination-link-header","name":"http_pagination_link_header","namespace":"Flow\\ETL\\Adapter\\Http","parameters":[{"name":"rel","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"'next'"},{"name":"stop_on_client_error","type":[{"name":"bool","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"true"},{"name":"stop_when","type":[{"name":"StopWhen","namespace":"Flow\\ETL\\Adapter\\Http\\Pagination","is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"null"}],"return_type":[{"name":"Paginator","namespace":"Flow\\ETL\\Adapter\\Http\\Pagination","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"HTTP","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/adapter\/etl-adapter-http\/src\/Flow\/ETL\/Adapter\/Http\/DSL\/functions.php","start_line_in_file":164,"slug":"http-pagination-last-record-cursor","name":"http_pagination_last_record_cursor","namespace":"Flow\\ETL\\Adapter\\Http","parameters":[{"name":"record_path","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"inject","type":[{"name":"RequestOption","namespace":"Flow\\ETL\\Adapter\\Http\\Pagination","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"stop_on_client_error","type":[{"name":"bool","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"true"},{"name":"stop_when","type":[{"name":"StopWhen","namespace":"Flow\\ETL\\Adapter\\Http\\Pagination","is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"null"}],"return_type":[{"name":"Paginator","namespace":"Flow\\ETL\\Adapter\\Http\\Pagination","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"HTTP","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/adapter\/etl-adapter-http\/src\/Flow\/ETL\/Adapter\/Http\/DSL\/functions.php","start_line_in_file":180,"slug":"http-request-option-query","name":"http_request_option_query","namespace":"Flow\\ETL\\Adapter\\Http","parameters":[{"name":"name","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"RequestOption","namespace":"Flow\\ETL\\Adapter\\Http\\Pagination","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"HTTP","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/adapter\/etl-adapter-http\/src\/Flow\/ETL\/Adapter\/Http\/DSL\/functions.php","start_line_in_file":186,"slug":"http-request-option-header","name":"http_request_option_header","namespace":"Flow\\ETL\\Adapter\\Http","parameters":[{"name":"name","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"RequestOption","namespace":"Flow\\ETL\\Adapter\\Http\\Pagination","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"HTTP","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/adapter\/etl-adapter-http\/src\/Flow\/ETL\/Adapter\/Http\/DSL\/functions.php","start_line_in_file":192,"slug":"http-request-option-body","name":"http_request_option_body","namespace":"Flow\\ETL\\Adapter\\Http","parameters":[{"name":"path","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"stream_factory","type":[{"name":"StreamFactoryInterface","namespace":"Psr\\Http\\Message","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"RequestOption","namespace":"Flow\\ETL\\Adapter\\Http\\Pagination","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"HTTP","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/adapter\/etl-adapter-http\/src\/Flow\/ETL\/Adapter\/Http\/DSL\/functions.php","start_line_in_file":198,"slug":"http-request-option-uri","name":"http_request_option_uri","namespace":"Flow\\ETL\\Adapter\\Http","parameters":[],"return_type":[{"name":"RequestOption","namespace":"Flow\\ETL\\Adapter\\Http\\Pagination","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"HTTP","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/adapter\/etl-adapter-http\/src\/Flow\/ETL\/Adapter\/Http\/DSL\/functions.php","start_line_in_file":204,"slug":"http-stop-when-path-missing","name":"http_stop_when_path_missing","namespace":"Flow\\ETL\\Adapter\\Http","parameters":[{"name":"path","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"StopWhen","namespace":"Flow\\ETL\\Adapter\\Http\\Pagination","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"HTTP","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/adapter\/etl-adapter-http\/src\/Flow\/ETL\/Adapter\/Http\/DSL\/functions.php","start_line_in_file":210,"slug":"http-stop-when-empty-path","name":"http_stop_when_empty_path","namespace":"Flow\\ETL\\Adapter\\Http","parameters":[{"name":"path","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"StopWhen","namespace":"Flow\\ETL\\Adapter\\Http\\Pagination","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"HTTP","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/adapter\/etl-adapter-http\/src\/Flow\/ETL\/Adapter\/Http\/DSL\/functions.php","start_line_in_file":216,"slug":"http-stop-when-flag-false","name":"http_stop_when_flag_false","namespace":"Flow\\ETL\\Adapter\\Http","parameters":[{"name":"path","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"StopWhen","namespace":"Flow\\ETL\\Adapter\\Http\\Pagination","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"HTTP","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/adapter\/etl-adapter-http\/src\/Flow\/ETL\/Adapter\/Http\/DSL\/functions.php","start_line_in_file":222,"slug":"http-stop-when-flag-true","name":"http_stop_when_flag_true","namespace":"Flow\\ETL\\Adapter\\Http","parameters":[{"name":"path","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"StopWhen","namespace":"Flow\\ETL\\Adapter\\Http\\Pagination","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"HTTP","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/adapter\/etl-adapter-http\/src\/Flow\/ETL\/Adapter\/Http\/DSL\/functions.php","start_line_in_file":228,"slug":"http-stop-when-total-reached","name":"http_stop_when_total_reached","namespace":"Flow\\ETL\\Adapter\\Http","parameters":[{"name":"total_path","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"StopWhen","namespace":"Flow\\ETL\\Adapter\\Http\\Pagination","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"HTTP","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/adapter\/etl-adapter-http\/src\/Flow\/ETL\/Adapter\/Http\/DSL\/functions.php","start_line_in_file":234,"slug":"http-stop-when-max-pages","name":"http_stop_when_max_pages","namespace":"Flow\\ETL\\Adapter\\Http","parameters":[{"name":"pages","type":[{"name":"int","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"StopWhen","namespace":"Flow\\ETL\\Adapter\\Http\\Pagination","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"HTTP","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/adapter\/etl-adapter-http\/src\/Flow\/ETL\/Adapter\/Http\/DSL\/functions.php","start_line_in_file":240,"slug":"http-stop-when-max-results","name":"http_stop_when_max_results","namespace":"Flow\\ETL\\Adapter\\Http","parameters":[{"name":"count","type":[{"name":"int","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"StopWhen","namespace":"Flow\\ETL\\Adapter\\Http\\Pagination","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"HTTP","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/adapter\/etl-adapter-json\/src\/Flow\/ETL\/Adapter\/JSON\/functions.php","start_line_in_file":32,"slug":"from-json","name":"from_json","namespace":"Flow\\ETL\\Adapter\\JSON","parameters":[{"name":"path","type":[{"name":"Path","namespace":"Flow\\Filesystem","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"pointer","type":[{"name":"string","namespace":null,"is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"null"},{"name":"schema","type":[{"name":"Schema","namespace":"Flow\\ETL","is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"null"},{"name":"filesystem","type":[{"name":"Filesystem","namespace":"Flow\\Filesystem","is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"Flow\\Filesystem\\Local\\NativeLocalFilesystem::..."}],"return_type":[{"name":"JsonExtractor","namespace":"Flow\\ETL\\Adapter\\JSON\\JSONMachine","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"JSON","type":"EXTRACTOR"}},{"name":"DocumentationExample","namespace":"Flow\\Documentation\\Attribute","arguments":{"topic":"reading","example":"json"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIEBwYXJhbSBQYXRofHN0cmluZyAkcGF0aCAtIHN0cmluZyBpcyBpbnRlcm5hbGx5IHR1cm5lZCBpbnRvIHN0cmVhbQogKiBAcGFyYW0gP3N0cmluZyAkcG9pbnRlciAtIGlmIHlvdSB3YW50IHRvIGl0ZXJhdGUgb25seSByZXN1bHRzIG9mIGEgc3VidHJlZSwgdXNlIGEgcG9pbnRlciwgcmVhZCBtb3JlIGF0IGh0dHBzOi8vZ2l0aHViLmNvbS9oYWxheGEvanNvbi1tYWNoaW5lI3BhcnNpbmctYS1zdWJ0cmVlIC0gQGRlcHJlY2F0ZSB1c2Ugd2l0aFBvaW50ZXIgbWV0aG9kIGluc3RlYWQKICogQHBhcmFtIG51bGx8U2NoZW1hICRzY2hlbWEgLSBlbmZvcmNlIHNjaGVtYSBvbiB0aGUgZXh0cmFjdGVkIGRhdGEgLSBAZGVwcmVjYXRlIHVzZSB3aXRoU2NoZW1hIG1ldGhvZCBpbnN0ZWFkCiAqLw=="},{"repository_path":"src\/adapter\/etl-adapter-json\/src\/Flow\/ETL\/Adapter\/JSON\/functions.php","start_line_in_file":58,"slug":"from-json-lines","name":"from_json_lines","namespace":"Flow\\ETL\\Adapter\\JSON","parameters":[{"name":"path","type":[{"name":"Path","namespace":"Flow\\Filesystem","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"filesystem","type":[{"name":"Filesystem","namespace":"Flow\\Filesystem","is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"Flow\\Filesystem\\Local\\NativeLocalFilesystem::..."}],"return_type":[{"name":"JsonExtractor","namespace":"Flow\\ETL\\Adapter\\JSON\\JSONMachine","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"JSON","type":"EXTRACTOR"}},{"name":"DocumentationExample","namespace":"Flow\\Documentation\\Attribute","arguments":{"topic":"reading","example":"jsonl"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIFVzZWQgdG8gcmVhZCBmcm9tIGEgSlNPTiBsaW5lcyBodHRwczovL2pzb25saW5lcy5vcmcvIGZvcm1hdHRlZCBmaWxlLgogKgogKiBAcGFyYW0gUGF0aHxzdHJpbmcgJHBhdGggLSBzdHJpbmcgaXMgaW50ZXJuYWxseSB0dXJuZWQgaW50byBzdHJlYW0KICov"},{"repository_path":"src\/adapter\/etl-adapter-json\/src\/Flow\/ETL\/Adapter\/JSON\/functions.php","start_line_in_file":72,"slug":"to-json","name":"to_json","namespace":"Flow\\ETL\\Adapter\\JSON","parameters":[{"name":"path","type":[{"name":"Path","namespace":"Flow\\Filesystem","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"flags","type":[{"name":"int","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"4194304"},{"name":"date_time_format","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"'Y-m-d\\\\TH:i:sP'"},{"name":"put_rows_in_new_lines","type":[{"name":"bool","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"false"},{"name":"filesystem","type":[{"name":"Filesystem","namespace":"Flow\\Filesystem","is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"Flow\\Filesystem\\Local\\NativeLocalFilesystem::..."}],"return_type":[{"name":"JsonLoader","namespace":"Flow\\ETL\\Adapter\\JSON","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"JSON","type":"LOADER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIEBwYXJhbSBQYXRofHN0cmluZyAkcGF0aAogKiBAcGFyYW0gaW50ICRmbGFncyAtIFBIUCBKU09OIEZsYWdzIC0gQGRlcHJlY2F0ZSB1c2Ugd2l0aEZsYWdzIG1ldGhvZCBpbnN0ZWFkCiAqIEBwYXJhbSBzdHJpbmcgJGRhdGVfdGltZV9mb3JtYXQgLSBmb3JtYXQgZm9yIERhdGVUaW1lSW50ZXJmYWNlOjpmb3JtYXQoKSAtIEBkZXByZWNhdGUgdXNlIHdpdGhEYXRlVGltZUZvcm1hdCBtZXRob2QgaW5zdGVhZAogKiBAcGFyYW0gYm9vbCAkcHV0X3Jvd3NfaW5fbmV3X2xpbmVzIC0gaWYgeW91IHdhbnQgdG8gcHV0IGVhY2ggcm93IGluIGEgbmV3IGxpbmUgLSBAZGVwcmVjYXRlIHVzZSB3aXRoUm93c0luTmV3TGluZXMgbWV0aG9kIGluc3RlYWQKICoKICogQHJldHVybiBKc29uTG9hZGVyCiAqLw=="},{"repository_path":"src\/adapter\/etl-adapter-json\/src\/Flow\/ETL\/Adapter\/JSON\/functions.php","start_line_in_file":93,"slug":"to-json-lines","name":"to_json_lines","namespace":"Flow\\ETL\\Adapter\\JSON","parameters":[{"name":"path","type":[{"name":"Path","namespace":"Flow\\Filesystem","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"filesystem","type":[{"name":"Filesystem","namespace":"Flow\\Filesystem","is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"Flow\\Filesystem\\Local\\NativeLocalFilesystem::..."}],"return_type":[{"name":"JsonLoader","namespace":"Flow\\ETL\\Adapter\\JSON","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"JSON","type":"LOADER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIFVzZWQgdG8gd3JpdGUgdG8gYSBKU09OIGxpbmVzIGh0dHBzOi8vanNvbmxpbmVzLm9yZy8gZm9ybWF0dGVkIGZpbGUuCiAqCiAqIEBwYXJhbSBQYXRofHN0cmluZyAkcGF0aAogKgogKiBAcmV0dXJuIEpzb25Mb2FkZXIKICov"},{"repository_path":"src\/adapter\/etl-adapter-json\/src\/Flow\/ETL\/Adapter\/JSON\/functions.php","start_line_in_file":107,"slug":"schema-from-json-schema","name":"schema_from_json_schema","namespace":"Flow\\ETL\\Adapter\\JSON","parameters":[{"name":"json_schema","type":[{"name":"Path","namespace":"Flow\\Filesystem","is_nullable":false,"is_variadic":false},{"name":"array","namespace":null,"is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"client","type":[{"name":"ClientInterface","namespace":"Psr\\Http\\Client","is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"null"},{"name":"request_factory","type":[{"name":"RequestFactoryInterface","namespace":"Psr\\Http\\Message","is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"null"},{"name":"filesystem","type":[{"name":"Filesystem","namespace":"Flow\\Filesystem","is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"Flow\\Filesystem\\Local\\NativeLocalFilesystem::..."}],"return_type":[{"name":"Schema","namespace":"Flow\\ETL","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"JSON","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENvbnZlcnQgYSBKU09OIFNjaGVtYSAoaHR0cHM6Ly9qc29uLXNjaGVtYS5vcmcpIGRvY3VtZW50IGludG8gYSBGbG93IFNjaGVtYS4KICoKICogQHBhcmFtIGFycmF5PHN0cmluZywgbWl4ZWQ+fFBhdGh8c3RyaW5nICRqc29uX3NjaGVtYSAtIGRlY29kZWQgZG9jdW1lbnQsIHJhdyBKU09OIGRvY3VtZW50IG9yIGEgcGF0aCB0byBhIHNjaGVtYSBmaWxlCiAqIEBwYXJhbSBudWxsfENsaWVudEludGVyZmFjZSAkY2xpZW50IC0gUFNSLTE4IGh0dHAgY2xpZW50LCByZXF1aXJlZCB0byByZXNvbHZlIHJlbW90ZSBodHRwKHMpIHJlZmVyZW5jZXMKICogQHBhcmFtIG51bGx8UmVxdWVzdEZhY3RvcnlJbnRlcmZhY2UgJHJlcXVlc3RfZmFjdG9yeSAtIFBTUi0xNyByZXF1ZXN0IGZhY3RvcnksIHJlcXVpcmVkIHRvIHJlc29sdmUgcmVtb3RlIGh0dHAocykgcmVmZXJlbmNlcwogKiBAcGFyYW0gRmlsZXN5c3RlbSAkZmlsZXN5c3RlbSAtIGZpbGVzeXN0ZW0gdXNlZCB0byByZWFkIGxvY2FsIHNjaGVtYSByZWZlcmVuY2VzCiAqLw=="},{"repository_path":"src\/adapter\/etl-adapter-json\/src\/Flow\/ETL\/Adapter\/JSON\/functions.php","start_line_in_file":122,"slug":"schema-to-json-schema","name":"schema_to_json_schema","namespace":"Flow\\ETL\\Adapter\\JSON","parameters":[{"name":"schema","type":[{"name":"Schema","namespace":"Flow\\ETL","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"array","namespace":null,"is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"JSON","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENvbnZlcnQgYSBGbG93IFNjaGVtYSBpbnRvIGEgSlNPTiBTY2hlbWEgKGh0dHBzOi8vanNvbi1zY2hlbWEub3JnLCBkcmFmdCAyMDIwLTEyKSBkb2N1bWVudC4KICoKICogQHJldHVybiBhcnJheTxzdHJpbmcsIG1peGVkPgogKi8="},{"repository_path":"src\/adapter\/etl-adapter-parquet\/src\/Flow\/ETL\/Adapter\/Parquet\/functions.php","start_line_in_file":36,"slug":"from-parquet","name":"from_parquet","namespace":"Flow\\ETL\\Adapter\\Parquet","parameters":[{"name":"path","type":[{"name":"Path","namespace":"Flow\\Filesystem","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"columns","type":[{"name":"array","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"[]"},{"name":"options","type":[{"name":"Options","namespace":"Flow\\Parquet","is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"Flow\\Parquet\\Options::..."},{"name":"byte_order","type":[{"name":"ByteOrder","namespace":"Flow\\Parquet\\Binary","is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"Flow\\Parquet\\Binary\\ByteOrder::..."},{"name":"offset","type":[{"name":"int","namespace":null,"is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"null"},{"name":"engine","type":[{"name":"ParquetEngine","namespace":"Flow\\Parquet","is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"null"},{"name":"filesystem","type":[{"name":"Filesystem","namespace":"Flow\\Filesystem","is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"Flow\\Filesystem\\Local\\NativeLocalFilesystem::..."}],"return_type":[{"name":"ParquetExtractor","namespace":"Flow\\ETL\\Adapter\\Parquet","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PARQUET","type":"EXTRACTOR"}},{"name":"DocumentationExample","namespace":"Flow\\Documentation\\Attribute","arguments":{"topic":"reading","example":"parquet"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIEBwYXJhbSBQYXRofHN0cmluZyAkcGF0aAogKiBAcGFyYW0gYXJyYXk8c3RyaW5nPiAkY29sdW1ucyAtIGxpc3Qgb2YgY29sdW1ucyB0byByZWFkIGZyb20gcGFycXVldCBmaWxlIC0gQGRlcHJlY2F0ZWQgdXNlIGB3aXRoQ29sdW1uc2AgbWV0aG9kIGluc3RlYWQKICogQHBhcmFtIE9wdGlvbnMgJG9wdGlvbnMgLSBAZGVwcmVjYXRlZCB1c2UgYHdpdGhPcHRpb25zYCBtZXRob2QgaW5zdGVhZAogKiBAcGFyYW0gQnl0ZU9yZGVyICRieXRlX29yZGVyIC0gQGRlcHJlY2F0ZWQgdXNlIGB3aXRoQnl0ZU9yZGVyYCBtZXRob2QgaW5zdGVhZAogKiBAcGFyYW0gbnVsbHxpbnQgJG9mZnNldCAtIEBkZXByZWNhdGVkIHVzZSBgd2l0aE9mZnNldGAgbWV0aG9kIGluc3RlYWQKICov"},{"repository_path":"src\/adapter\/etl-adapter-parquet\/src\/Flow\/ETL\/Adapter\/Parquet\/functions.php","start_line_in_file":69,"slug":"to-parquet","name":"to_parquet","namespace":"Flow\\ETL\\Adapter\\Parquet","parameters":[{"name":"path","type":[{"name":"Path","namespace":"Flow\\Filesystem","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"options","type":[{"name":"Options","namespace":"Flow\\Parquet","is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"null"},{"name":"compressions","type":[{"name":"Compressions","namespace":"Flow\\Parquet\\ParquetFile","is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"Flow\\Parquet\\ParquetFile\\Compressions::..."},{"name":"schema","type":[{"name":"Schema","namespace":"Flow\\ETL","is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"null"},{"name":"engine","type":[{"name":"ParquetEngine","namespace":"Flow\\Parquet","is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"null"},{"name":"filesystem","type":[{"name":"Filesystem","namespace":"Flow\\Filesystem","is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"Flow\\Filesystem\\Local\\NativeLocalFilesystem::..."}],"return_type":[{"name":"ParquetLoader","namespace":"Flow\\ETL\\Adapter\\Parquet","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PARQUET","type":"LOADER"}},{"name":"DocumentationExample","namespace":"Flow\\Documentation\\Attribute","arguments":{"topic":"writing","example":"parquet"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIEBwYXJhbSBQYXRofHN0cmluZyAkcGF0aAogKiBAcGFyYW0gbnVsbHxPcHRpb25zICRvcHRpb25zIC0gQGRlcHJlY2F0ZWQgdXNlIGB3aXRoT3B0aW9uc2AgbWV0aG9kIGluc3RlYWQKICogQHBhcmFtIENvbXByZXNzaW9ucyAkY29tcHJlc3Npb25zIC0gQGRlcHJlY2F0ZWQgdXNlIGB3aXRoQ29tcHJlc3Npb25zYCBtZXRob2QgaW5zdGVhZAogKiBAcGFyYW0gbnVsbHxTY2hlbWEgJHNjaGVtYSAtIEBkZXByZWNhdGVkIHVzZSBgd2l0aFNjaGVtYWAgbWV0aG9kIGluc3RlYWQKICov"},{"repository_path":"src\/adapter\/etl-adapter-parquet\/src\/Flow\/ETL\/Adapter\/Parquet\/functions.php","start_line_in_file":100,"slug":"array-to-generator","name":"array_to_generator","namespace":"Flow\\ETL\\Adapter\\Parquet","parameters":[{"name":"data","type":[{"name":"array","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"Generator","namespace":"","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PARQUET","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIEB0ZW1wbGF0ZSBUCiAqCiAqIEBwYXJhbSBhcnJheTxUPiAkZGF0YQogKgogKiBAcmV0dXJuIFxHZW5lcmF0b3I8VD4KICov"},{"repository_path":"src\/adapter\/etl-adapter-parquet\/src\/Flow\/ETL\/Adapter\/Parquet\/functions.php","start_line_in_file":111,"slug":"empty-generator","name":"empty_generator","namespace":"Flow\\ETL\\Adapter\\Parquet","parameters":[],"return_type":[{"name":"Generator","namespace":"","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PARQUET","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIEBkZXByZWNhdGVkIHVzZSBGbG93XFBhcnF1ZXRcZW1wdHlfZ2VuZXJhdG9yKCkgaW5zdGVhZAogKi8="},{"repository_path":"src\/adapter\/etl-adapter-parquet\/src\/Flow\/ETL\/Adapter\/Parquet\/functions.php","start_line_in_file":117,"slug":"schema-to-parquet","name":"schema_to_parquet","namespace":"Flow\\ETL\\Adapter\\Parquet","parameters":[{"name":"schema","type":[{"name":"Schema","namespace":"Flow\\ETL","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"Schema","namespace":"Flow\\Parquet\\ParquetFile","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PARQUET","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/adapter\/etl-adapter-parquet\/src\/Flow\/ETL\/Adapter\/Parquet\/functions.php","start_line_in_file":123,"slug":"schema-from-parquet","name":"schema_from_parquet","namespace":"Flow\\ETL\\Adapter\\Parquet","parameters":[{"name":"schema","type":[{"name":"Schema","namespace":"Flow\\Parquet\\ParquetFile","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"Schema","namespace":"Flow\\ETL","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PARQUET","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/adapter\/etl-adapter-postgresql\/src\/Flow\/ETL\/Adapter\/PostgreSql\/functions.php","start_line_in_file":39,"slug":"from-pgsql-cursor","name":"from_pgsql_cursor","namespace":"Flow\\ETL\\Adapter\\PostgreSql","parameters":[{"name":"client","type":[{"name":"Client","namespace":"Flow\\PostgreSql\\Client","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"query","type":[{"name":"Sql","namespace":"Flow\\PostgreSql\\QueryBuilder","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"parameters","type":[{"name":"array","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"[]"}],"return_type":[{"name":"PostgreSqlCursorExtractor","namespace":"Flow\\ETL\\Adapter\\PostgreSql","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"POSTGRESQL","type":"EXTRACTOR"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBhIFBvc3RncmVTUUwgY3Vyc29yIGV4dHJhY3RvciB1c2luZyBzZXJ2ZXItc2lkZSBjdXJzb3JzIGZvciBtZW1vcnktZWZmaWNpZW50IGV4dHJhY3Rpb24uCiAqCiAqIFVzZXMgREVDTEFSRSBDVVJTT1IgKyBGRVRDSCB0byBzdHJlYW0gZGF0YSB3aXRob3V0IGxvYWRpbmcgZW50aXJlIHJlc3VsdCBzZXQgaW50byBtZW1vcnkuCiAqIFRoaXMgaXMgdGhlIG9ubHkgd2F5IHRvIGFjaGlldmUgdHJ1ZSBsb3cgbWVtb3J5IGV4dHJhY3Rpb24gd2l0aCBQSFAncyBleHQtcGdzcWwuCiAqCiAqIE5vdGU6IFJlcXVpcmVzIGEgdHJhbnNhY3Rpb24gY29udGV4dCAoYXV0by1zdGFydGVkIGlmIG5vdCBpbiBvbmUpLgogKgogKiBAcGFyYW0gQ2xpZW50ICRjbGllbnQgUG9zdGdyZVNRTCBjbGllbnQKICogQHBhcmFtIFNxbHxzdHJpbmcgJHF1ZXJ5IFNRTCBxdWVyeSB0byBleGVjdXRlICh3cmFwcGVkIGluIERFQ0xBUkUgQ1VSU09SKQogKiBAcGFyYW0gbGlzdDxtaXhlZD4gJHBhcmFtZXRlcnMgVmFsdWVzIGJvdW5kIGJ5IHBvc2l0aW9uIHRvICQxLCAkMiwgLi4uIHBsYWNlaG9sZGVyczsgd3JhcCB3aXRoIHtAc2VlIFxGbG93XFBvc3RncmVTcWxcRFNMXHR5cGVkKCl9IHRvIGZvcmNlIGEgc3BlY2lmaWMgUG9zdGdyZVNRTCB0eXBlCiAqLw=="},{"repository_path":"src\/adapter\/etl-adapter-postgresql\/src\/Flow\/ETL\/Adapter\/PostgreSql\/functions.php","start_line_in_file":55,"slug":"from-pgsql-limit-offset","name":"from_pgsql_limit_offset","namespace":"Flow\\ETL\\Adapter\\PostgreSql","parameters":[{"name":"client","type":[{"name":"Client","namespace":"Flow\\PostgreSql\\Client","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"query","type":[{"name":"Sql","namespace":"Flow\\PostgreSql\\QueryBuilder","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"parameters","type":[{"name":"array","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"[]"}],"return_type":[{"name":"PostgreSqlLimitOffsetExtractor","namespace":"Flow\\ETL\\Adapter\\PostgreSql","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"POSTGRESQL","type":"EXTRACTOR"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBhIFBvc3RncmVTUUwgZXh0cmFjdG9yIHVzaW5nIExJTUlUL09GRlNFVCBwYWdpbmF0aW9uLgogKgogKiBTdWl0YWJsZSBmb3Igc21hbGxlciBkYXRhc2V0cy4gRm9yIGxhcmdlIGRhdGFzZXRzLCBjb25zaWRlciB1c2luZyBrZXlzZXQgcGFnaW5hdGlvbgogKiAoZnJvbV9wZ3NxbF9rZXlfc2V0KSB3aGljaCBpcyBtb3JlIGVmZmljaWVudC4KICoKICogQHBhcmFtIENsaWVudCAkY2xpZW50IFBvc3RncmVTUUwgY2xpZW50CiAqIEBwYXJhbSBTcWx8c3RyaW5nICRxdWVyeSBTUUwgcXVlcnkgdG8gZXhlY3V0ZSAobXVzdCBoYXZlIE9SREVSIEJZIGNsYXVzZSkKICogQHBhcmFtIGxpc3Q8bWl4ZWQ+ICRwYXJhbWV0ZXJzIFZhbHVlcyBib3VuZCBieSBwb3NpdGlvbiB0byAkMSwgJDIsIC4uLiBwbGFjZWhvbGRlcnM7IHdyYXAgd2l0aCB7QHNlZSBcRmxvd1xQb3N0Z3JlU3FsXERTTFx0eXBlZCgpfSB0byBmb3JjZSBhIHNwZWNpZmljIFBvc3RncmVTUUwgdHlwZQogKi8="},{"repository_path":"src\/adapter\/etl-adapter-postgresql\/src\/Flow\/ETL\/Adapter\/PostgreSql\/functions.php","start_line_in_file":75,"slug":"from-pgsql-key-set","name":"from_pgsql_key_set","namespace":"Flow\\ETL\\Adapter\\PostgreSql","parameters":[{"name":"client","type":[{"name":"Client","namespace":"Flow\\PostgreSql\\Client","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"query","type":[{"name":"Sql","namespace":"Flow\\PostgreSql\\QueryBuilder","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"keySet","type":[{"name":"KeySet","namespace":"Flow\\ETL\\Adapter\\PostgreSql\\Pagination","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"parameters","type":[{"name":"array","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"[]"}],"return_type":[{"name":"PostgreSqlKeySetExtractor","namespace":"Flow\\ETL\\Adapter\\PostgreSql","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"POSTGRESQL","type":"EXTRACTOR"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBhIFBvc3RncmVTUUwgZXh0cmFjdG9yIHVzaW5nIGtleXNldCAoY3Vyc29yLWJhc2VkKSBwYWdpbmF0aW9uLgogKgogKiBNb3JlIGVmZmljaWVudCB0aGFuIExJTUlUL09GRlNFVCBmb3IgbGFyZ2UgZGF0YXNldHMgLSB1c2VzIGluZGV4ZWQgV0hFUkUgY29uZGl0aW9ucwogKiBpbnN0ZWFkIG9mIHNraXBwaW5nIHJvd3MuCiAqCiAqIEBwYXJhbSBDbGllbnQgJGNsaWVudCBQb3N0Z3JlU1FMIGNsaWVudAogKiBAcGFyYW0gU3FsfHN0cmluZyAkcXVlcnkgU1FMIHF1ZXJ5IHRvIGV4ZWN1dGUgKG11c3QgaGF2ZSBPUkRFUiBCWSBtYXRjaGluZyBrZXlzZXQgY29sdW1ucykKICogQHBhcmFtIEtleVNldCAka2V5U2V0IENvbHVtbnMgdG8gdXNlIGZvciBrZXlzZXQgcGFnaW5hdGlvbgogKiBAcGFyYW0gbGlzdDxtaXhlZD4gJHBhcmFtZXRlcnMgVmFsdWVzIGJvdW5kIGJ5IHBvc2l0aW9uIHRvICQxLCAkMiwgLi4uIHBsYWNlaG9sZGVyczsgd3JhcCB3aXRoIHtAc2VlIFxGbG93XFBvc3RncmVTcWxcRFNMXHR5cGVkKCl9IHRvIGZvcmNlIGEgc3BlY2lmaWMgUG9zdGdyZVNRTCB0eXBlCiAqLw=="},{"repository_path":"src\/adapter\/etl-adapter-postgresql\/src\/Flow\/ETL\/Adapter\/PostgreSql\/functions.php","start_line_in_file":85,"slug":"pgsql-pagination-key-asc","name":"pgsql_pagination_key_asc","namespace":"Flow\\ETL\\Adapter\\PostgreSql","parameters":[{"name":"column","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"Key","namespace":"Flow\\ETL\\Adapter\\PostgreSql\\Pagination","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"POSTGRESQL","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/adapter\/etl-adapter-postgresql\/src\/Flow\/ETL\/Adapter\/PostgreSql\/functions.php","start_line_in_file":91,"slug":"pgsql-pagination-key-desc","name":"pgsql_pagination_key_desc","namespace":"Flow\\ETL\\Adapter\\PostgreSql","parameters":[{"name":"column","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"Key","namespace":"Flow\\ETL\\Adapter\\PostgreSql\\Pagination","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"POSTGRESQL","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/adapter\/etl-adapter-postgresql\/src\/Flow\/ETL\/Adapter\/PostgreSql\/functions.php","start_line_in_file":97,"slug":"pgsql-pagination-key-set","name":"pgsql_pagination_key_set","namespace":"Flow\\ETL\\Adapter\\PostgreSql","parameters":[{"name":"keys","type":[{"name":"Key","namespace":"Flow\\ETL\\Adapter\\PostgreSql\\Pagination","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":true,"default_value":null}],"return_type":[{"name":"KeySet","namespace":"Flow\\ETL\\Adapter\\PostgreSql\\Pagination","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"POSTGRESQL","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/adapter\/etl-adapter-postgresql\/src\/Flow\/ETL\/Adapter\/PostgreSql\/functions.php","start_line_in_file":103,"slug":"to-pgsql-table","name":"to_pgsql_table","namespace":"Flow\\ETL\\Adapter\\PostgreSql","parameters":[{"name":"client","type":[{"name":"Client","namespace":"Flow\\PostgreSql\\Client","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"table","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"PostgreSqlLoader","namespace":"Flow\\ETL\\Adapter\\PostgreSql","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"POSTGRESQL","type":"LOADER"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/adapter\/etl-adapter-postgresql\/src\/Flow\/ETL\/Adapter\/PostgreSql\/functions.php","start_line_in_file":112,"slug":"to-pgsql-transaction","name":"to_pgsql_transaction","namespace":"Flow\\ETL\\Adapter\\PostgreSql","parameters":[{"name":"client","type":[{"name":"Client","namespace":"Flow\\PostgreSql\\Client","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"sinks","type":[{"name":"Loader","namespace":"Flow\\ETL","is_nullable":false,"is_variadic":false},{"name":"Sink","namespace":"Flow\\ETL","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":true,"default_value":null}],"return_type":[{"name":"Transactional","namespace":"Flow\\ETL\\Sink","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"POSTGRESQL","type":"LOADER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIFdyaXRlIGV2ZXJ5IHNpbmsgd2l0aGluIFBvc3RncmVTUUwgdHJhbnNhY3Rpb25zLgogKi8="},{"repository_path":"src\/adapter\/etl-adapter-postgresql\/src\/Flow\/ETL\/Adapter\/PostgreSql\/functions.php","start_line_in_file":126,"slug":"pgsql-insert-options","name":"pgsql_insert_options","namespace":"Flow\\ETL\\Adapter\\PostgreSql","parameters":[{"name":"skipConflicts","type":[{"name":"bool","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"false"},{"name":"conflictColumns","type":[{"name":"array","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"[]"},{"name":"conflictConstraint","type":[{"name":"string","namespace":null,"is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"null"},{"name":"updateColumns","type":[{"name":"array","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"[]"}],"return_type":[{"name":"InsertOptions","namespace":"Flow\\ETL\\Adapter\\PostgreSql\\LoaderOptions","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"POSTGRESQL","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBpbnNlcnQgb3B0aW9ucyBmb3IgUG9zdGdyZVNRTCBsb2FkZXIuCiAqCiAqIEBwYXJhbSBib29sICRza2lwQ29uZmxpY3RzIElmIHRydWUsIHVzZSBPTiBDT05GTElDVCBETyBOT1RISU5HCiAqIEBwYXJhbSBsaXN0PHN0cmluZz4gJGNvbmZsaWN0Q29sdW1ucyBDb2x1bW4gbmFtZXMgZm9yIE9OIENPTkZMSUNUIChjb2x1bW5zKQogKiBAcGFyYW0gbnVsbHxzdHJpbmcgJGNvbmZsaWN0Q29uc3RyYWludCBDb25zdHJhaW50IG5hbWUgZm9yIE9OIENPTkZMSUNUIE9OIENPTlNUUkFJTlQKICogQHBhcmFtIGxpc3Q8c3RyaW5nPiAkdXBkYXRlQ29sdW1ucyBDb2x1bW5zIHRvIHVwZGF0ZSBvbiBjb25mbGljdCAoZW1wdHkgPSBhbGwgbm9uLWtleSBjb2x1bW5zKQogKi8="},{"repository_path":"src\/adapter\/etl-adapter-postgresql\/src\/Flow\/ETL\/Adapter\/PostgreSql\/functions.php","start_line_in_file":141,"slug":"pgsql-update-options","name":"pgsql_update_options","namespace":"Flow\\ETL\\Adapter\\PostgreSql","parameters":[{"name":"primaryKeys","type":[{"name":"array","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"UpdateOptions","namespace":"Flow\\ETL\\Adapter\\PostgreSql\\LoaderOptions","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"POSTGRESQL","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSB1cGRhdGUgb3B0aW9ucyBmb3IgUG9zdGdyZVNRTCBsb2FkZXIuCiAqCiAqIEBwYXJhbSBsaXN0PHN0cmluZz4gJHByaW1hcnlLZXlzIENvbHVtbnMgdG8gdXNlIGluIFdIRVJFIGNsYXVzZSBmb3IgbWF0Y2hpbmcgcm93cwogKi8="},{"repository_path":"src\/adapter\/etl-adapter-postgresql\/src\/Flow\/ETL\/Adapter\/PostgreSql\/functions.php","start_line_in_file":152,"slug":"pgsql-delete-options","name":"pgsql_delete_options","namespace":"Flow\\ETL\\Adapter\\PostgreSql","parameters":[{"name":"primaryKeys","type":[{"name":"array","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"DeleteOptions","namespace":"Flow\\ETL\\Adapter\\PostgreSql\\LoaderOptions","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"POSTGRESQL","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBkZWxldGUgb3B0aW9ucyBmb3IgUG9zdGdyZVNRTCBsb2FkZXIuCiAqCiAqIEBwYXJhbSBsaXN0PHN0cmluZz4gJHByaW1hcnlLZXlzIENvbHVtbnMgdG8gdXNlIGluIFdIRVJFIGNsYXVzZSBmb3IgbWF0Y2hpbmcgcm93cwogKi8="},{"repository_path":"src\/adapter\/etl-adapter-postgresql\/src\/Flow\/ETL\/Adapter\/PostgreSql\/functions.php","start_line_in_file":164,"slug":"to-pgsql-schema-table","name":"to_pgsql_schema_table","namespace":"Flow\\ETL\\Adapter\\PostgreSql","parameters":[{"name":"schema","type":[{"name":"Schema","namespace":"Flow\\ETL","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"tableName","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"databaseSchema","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"'public'"},{"name":"typesMap","type":[{"name":"EntryTypesMap","namespace":"Flow\\ETL\\Adapter\\PostgreSql","is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"null"},{"name":"options","type":[{"name":"TableOptions","namespace":"Flow\\PostgreSql\\Schema","is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"null"}],"return_type":[{"name":"Table","namespace":"Flow\\PostgreSql\\Schema","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"POSTGRESQL","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENvbnZlcnQgYSBGbG93IFNjaGVtYSBpbnRvIGEgUG9zdGdyZVNRTCB0YWJsZSBkZWZpbml0aW9uLgogKgogKiBAcGFyYW0gc3RyaW5nICRkYXRhYmFzZVNjaGVtYSBQb3N0Z3JlU1FMIHNjaGVtYSAobmFtZXNwYWNlKSB0aGUgdGFibGUgYmVsb25ncyB0bwogKiBAcGFyYW0gP1RhYmxlT3B0aW9ucyAkb3B0aW9ucyB0YWJsZS1sZXZlbCBvcHRpb25zIHRoZSBGbG93IFNjaGVtYSBjYW5ub3QgZXhwcmVzcyAoZS5nLiBVTkxPR0dFRCkKICov"},{"repository_path":"src\/adapter\/etl-adapter-postgresql\/src\/Flow\/ETL\/Adapter\/PostgreSql\/functions.php","start_line_in_file":178,"slug":"pgsql-table-to-flow-schema","name":"pgsql_table_to_flow_schema","namespace":"Flow\\ETL\\Adapter\\PostgreSql","parameters":[{"name":"table","type":[{"name":"Table","namespace":"Flow\\PostgreSql\\Schema","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"typesMap","type":[{"name":"EntryTypesMap","namespace":"Flow\\ETL\\Adapter\\PostgreSql","is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"null"}],"return_type":[{"name":"Schema","namespace":"Flow\\ETL","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"POSTGRESQL","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENvbnZlcnQgYSBQb3N0Z3JlU1FMIHRhYmxlIGRlZmluaXRpb24gaW50byBhIEZsb3cgU2NoZW1hLgogKi8="},{"repository_path":"src\/adapter\/etl-adapter-postgresql\/src\/Flow\/ETL\/Adapter\/PostgreSql\/functions.php","start_line_in_file":184,"slug":"pgsql-schema-sort-by-type","name":"pgsql_schema_sort_by_type","namespace":"Flow\\ETL\\Adapter\\PostgreSql","parameters":[{"name":"typesMap","type":[{"name":"EntryTypesMap","namespace":"Flow\\ETL\\Adapter\\PostgreSql","is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"null"}],"return_type":[{"name":"TypeStrategy","namespace":"Flow\\ETL\\Adapter\\PostgreSql\\Schema\\SortingStrategy","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"POSTGRESQL","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/adapter\/etl-adapter-seal\/src\/Flow\/ETL\/Adapter\/Seal\/functions.php","start_line_in_file":15,"slug":"to-seal-upsert","name":"to_seal_upsert","namespace":"Flow\\ETL\\Adapter\\Seal","parameters":[{"name":"engine","type":[{"name":"EngineInterface","namespace":"CmsIg\\Seal","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"index","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"SealLoader","namespace":"Flow\\ETL\\Adapter\\Seal","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"SEAL","type":"LOADER"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/adapter\/etl-adapter-seal\/src\/Flow\/ETL\/Adapter\/Seal\/functions.php","start_line_in_file":21,"slug":"to-seal-delete","name":"to_seal_delete","namespace":"Flow\\ETL\\Adapter\\Seal","parameters":[{"name":"engine","type":[{"name":"EngineInterface","namespace":"CmsIg\\Seal","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"index","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"SealLoader","namespace":"Flow\\ETL\\Adapter\\Seal","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"SEAL","type":"LOADER"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/adapter\/etl-adapter-seal\/src\/Flow\/ETL\/Adapter\/Seal\/functions.php","start_line_in_file":27,"slug":"to-seal-schema","name":"to_seal_schema","namespace":"Flow\\ETL\\Adapter\\Seal","parameters":[{"name":"schema","type":[{"name":"Schema","namespace":"Flow\\ETL","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"index_name","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"identifier","type":[{"name":"string","namespace":null,"is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"null"}],"return_type":[{"name":"Schema","namespace":"CmsIg\\Seal\\Schema","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"SEAL","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/adapter\/etl-adapter-seal\/src\/Flow\/ETL\/Adapter\/Seal\/functions.php","start_line_in_file":33,"slug":"seal-schema-to-flow","name":"seal_schema_to_flow","namespace":"Flow\\ETL\\Adapter\\Seal","parameters":[{"name":"schema","type":[{"name":"Schema","namespace":"CmsIg\\Seal\\Schema","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"Schema","namespace":"Flow\\ETL","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"SEAL","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/adapter\/etl-adapter-text\/src\/Flow\/ETL\/Adapter\/Text\/functions.php","start_line_in_file":21,"slug":"from-text","name":"from_text","namespace":"Flow\\ETL\\Adapter\\Text","parameters":[{"name":"path","type":[{"name":"Path","namespace":"Flow\\Filesystem","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"filesystem","type":[{"name":"Filesystem","namespace":"Flow\\Filesystem","is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"Flow\\Filesystem\\Local\\NativeLocalFilesystem::..."}],"return_type":[{"name":"TextExtractor","namespace":"Flow\\ETL\\Adapter\\Text","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"TEXT","type":"EXTRACTOR"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIEBwYXJhbSBQYXRofHN0cmluZyAkcGF0aAogKi8="},{"repository_path":"src\/adapter\/etl-adapter-text\/src\/Flow\/ETL\/Adapter\/Text\/functions.php","start_line_in_file":31,"slug":"to-text","name":"to_text","namespace":"Flow\\ETL\\Adapter\\Text","parameters":[{"name":"path","type":[{"name":"Path","namespace":"Flow\\Filesystem","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"new_line_separator","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"'\\n'"},{"name":"filesystem","type":[{"name":"Filesystem","namespace":"Flow\\Filesystem","is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"Flow\\Filesystem\\Local\\NativeLocalFilesystem::..."}],"return_type":[{"name":"TextLoader","namespace":"Flow\\ETL\\Adapter\\Text","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"TEXT","type":"LOADER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIEBwYXJhbSBQYXRofHN0cmluZyAkcGF0aAogKiBAcGFyYW0gc3RyaW5nICRuZXdfbGluZV9zZXBhcmF0b3IgLSBkZWZhdWx0IFBIUF9FT0wgLSBAZGVwcmVjYXRlZCB1c2Ugd2l0aE5ld0xpbmVTZXBhcmF0b3IgbWV0aG9kIGluc3RlYWQKICov"},{"repository_path":"src\/adapter\/etl-adapter-xml\/src\/Flow\/ETL\/Adapter\/XML\/functions.php","start_line_in_file":38,"slug":"from-xml","name":"from_xml","namespace":"Flow\\ETL\\Adapter\\XML","parameters":[{"name":"path","type":[{"name":"Path","namespace":"Flow\\Filesystem","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"xml_node_path","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"''"},{"name":"filesystem","type":[{"name":"Filesystem","namespace":"Flow\\Filesystem","is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"Flow\\Filesystem\\Local\\NativeLocalFilesystem::..."}],"return_type":[{"name":"XMLParserExtractor","namespace":"Flow\\ETL\\Adapter\\XML","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"XML","type":"EXTRACTOR"}},{"name":"DocumentationExample","namespace":"Flow\\Documentation\\Attribute","arguments":{"topic":"reading","example":"xml"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqICBJbiBvcmRlciB0byBpdGVyYXRlIG9ubHkgb3ZlciA8ZWxlbWVudD4gbm9kZXMgdXNlIGBmcm9tX3htbCgkZmlsZSktPndpdGhYTUxOb2RlUGF0aCgncm9vdC9lbGVtZW50cy9lbGVtZW50JylgLgogKgogKiAgPHJvb3Q+CiAqICAgIDxlbGVtZW50cz4KICogICAgICA8ZWxlbWVudD48L2VsZW1lbnQ+CiAqICAgICAgPGVsZW1lbnQ+PC9lbGVtZW50PgogKiAgICA8ZWxlbWVudHM+CiAqICA8L3Jvb3Q+CiAqCiAqICBYTUwgTm9kZSBQYXRoIGRvZXMgbm90IHN1cHBvcnQgYXR0cmlidXRlcyBhbmQgaXQncyBub3QgeHBhdGgsIGl0IGlzIGp1c3QgYSBzZXF1ZW5jZQogKiAgb2Ygbm9kZSBuYW1lcyBzZXBhcmF0ZWQgd2l0aCBzbGFzaC4KICoKICogQHBhcmFtIFBhdGh8c3RyaW5nICRwYXRoCiAqIEBwYXJhbSBzdHJpbmcgJHhtbF9ub2RlX3BhdGggLSBAZGVwcmVjYXRlZCB1c2UgYGZyb21feG1sKCRmaWxlKS0+d2l0aFhNTE5vZGVQYXRoKCR4bWxOb2RlUGF0aClgIG1ldGhvZCBpbnN0ZWFkCiAqLw=="},{"repository_path":"src\/adapter\/etl-adapter-xml\/src\/Flow\/ETL\/Adapter\/XML\/functions.php","start_line_in_file":57,"slug":"to-xml","name":"to_xml","namespace":"Flow\\ETL\\Adapter\\XML","parameters":[{"name":"path","type":[{"name":"Path","namespace":"Flow\\Filesystem","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"root_element_name","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"'rows'"},{"name":"row_element_name","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"'row'"},{"name":"attribute_prefix","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"'_'"},{"name":"date_time_format","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"'Y-m-d\\\\TH:i:s.uP'"},{"name":"xml_writer","type":[{"name":"XMLWriter","namespace":"Flow\\ETL\\Adapter\\XML","is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"Flow\\ETL\\Adapter\\XML\\XMLWriter\\StringXMLWriter::..."},{"name":"filesystem","type":[{"name":"Filesystem","namespace":"Flow\\Filesystem","is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"Flow\\Filesystem\\Local\\NativeLocalFilesystem::..."}],"return_type":[{"name":"XMLLoader","namespace":"Flow\\ETL\\Adapter\\XML\\Loader","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"XML","type":"LOADER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIEBwYXJhbSBQYXRofHN0cmluZyAkcGF0aAogKiBAcGFyYW0gc3RyaW5nICRyb290X2VsZW1lbnRfbmFtZSAtIEBkZXByZWNhdGVkIHVzZSBgd2l0aFJvb3RFbGVtZW50TmFtZSgpYCBtZXRob2QgaW5zdGVhZAogKiBAcGFyYW0gc3RyaW5nICRyb3dfZWxlbWVudF9uYW1lIC0gQGRlcHJlY2F0ZWQgdXNlIGB3aXRoUm93RWxlbWVudE5hbWUoKWAgbWV0aG9kIGluc3RlYWQKICogQHBhcmFtIHN0cmluZyAkYXR0cmlidXRlX3ByZWZpeCAtIEBkZXByZWNhdGVkIHVzZSBgd2l0aEF0dHJpYnV0ZVByZWZpeCgpYCBtZXRob2QgaW5zdGVhZAogKiBAcGFyYW0gc3RyaW5nICRkYXRlX3RpbWVfZm9ybWF0IC0gQGRlcHJlY2F0ZWQgdXNlIGB3aXRoRGF0ZVRpbWVGb3JtYXQoKWAgbWV0aG9kIGluc3RlYWQKICogQHBhcmFtIFhNTFdyaXRlciAkeG1sX3dyaXRlcgogKi8="},{"repository_path":"src\/lib\/filesystem\/src\/Flow\/Filesystem\/DSL\/functions.php","start_line_in_file":32,"slug":"mount","name":"mount","namespace":"Flow\\Filesystem\\DSL","parameters":[{"name":"protocol","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"Mount","namespace":"Flow\\Filesystem","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"FILESYSTEM","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/lib\/filesystem\/src\/Flow\/Filesystem\/DSL\/functions.php","start_line_in_file":38,"slug":"partition","name":"partition","namespace":"Flow\\Filesystem\\DSL","parameters":[{"name":"name","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"value","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"Partition","namespace":"Flow\\Filesystem","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"FILESYSTEM","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/lib\/filesystem\/src\/Flow\/Filesystem\/DSL\/functions.php","start_line_in_file":44,"slug":"partitions","name":"partitions","namespace":"Flow\\Filesystem\\DSL","parameters":[{"name":"partition","type":[{"name":"Partition","namespace":"Flow\\Filesystem","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":true,"default_value":null}],"return_type":[{"name":"Partitions","namespace":"Flow\\Filesystem","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"FILESYSTEM","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/lib\/filesystem\/src\/Flow\/Filesystem\/DSL\/functions.php","start_line_in_file":63,"slug":"path","name":"path","namespace":"Flow\\Filesystem\\DSL","parameters":[{"name":"path","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"options","type":[{"name":"Options","namespace":"Flow\\Filesystem\\Path","is_nullable":false,"is_variadic":false},{"name":"array","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"[]"}],"return_type":[{"name":"Path","namespace":"Flow\\Filesystem","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"FILESYSTEM","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIFBhdGggc3VwcG9ydHMgZ2xvYiBwYXR0ZXJucy4KICogRXhhbXBsZXM6CiAqICAtIHBhdGgoJyouY3N2JykgLSBhbnkgY3N2IGZpbGUgaW4gY3VycmVudCBkaXJlY3RvcnkKICogIC0gcGF0aCgnLyoqIC8gKi5jc3YnKSAtIGFueSBjc3YgZmlsZSBpbiBhbnkgc3ViZGlyZWN0b3J5IChyZW1vdmUgZW1wdHkgc3BhY2VzKQogKiAgLSBwYXRoKCcvZGlyL3BhcnRpdGlvbj0qIC8qLnBhcnF1ZXQnKSAtIGFueSBwYXJxdWV0IGZpbGUgaW4gZ2l2ZW4gcGFydGl0aW9uIGRpcmVjdG9yeS4KICoKICogR2xvYiBwYXR0ZXJuIGlzIGFsc28gc3VwcG9ydGVkIGJ5IHJlbW90ZSBmaWxlc3lzdGVtcyBsaWtlIEF6dXJlCiAqCiAqICAtIHBhdGgoJ2F6dXJlLWJsb2I6Ly9kaXJlY3RvcnkvKi5jc3YnKSAtIGFueSBjc3YgZmlsZSBpbiBnaXZlbiBkaXJlY3RvcnkKICoKICogQHBhcmFtIGFycmF5PHN0cmluZywgbnVsbHxib29sfGZsb2F0fGludHxzdHJpbmd8XFVuaXRFbnVtPnxQYXRoXE9wdGlvbnMgJG9wdGlvbnMKICov"},{"repository_path":"src\/lib\/filesystem\/src\/Flow\/Filesystem\/DSL\/functions.php","start_line_in_file":74,"slug":"path-real","name":"path_real","namespace":"Flow\\Filesystem\\DSL","parameters":[{"name":"path","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"options","type":[{"name":"array","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"[]"}],"return_type":[{"name":"Path","namespace":"Flow\\Filesystem","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"FILESYSTEM","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIFJlc29sdmUgcmVhbCBwYXRoIGZyb20gZ2l2ZW4gcGF0aC4KICoKICogQHBhcmFtIGFycmF5PHN0cmluZywgbnVsbHxib29sfGZsb2F0fGludHxzdHJpbmd8XFVuaXRFbnVtPiAkb3B0aW9ucwogKi8="},{"repository_path":"src\/lib\/filesystem\/src\/Flow\/Filesystem\/DSL\/functions.php","start_line_in_file":80,"slug":"native-local-filesystem","name":"native_local_filesystem","namespace":"Flow\\Filesystem\\DSL","parameters":[{"name":"protocol","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"'file'"}],"return_type":[{"name":"NativeLocalFilesystem","namespace":"Flow\\Filesystem\\Local","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"FILESYSTEM","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/lib\/filesystem\/src\/Flow\/Filesystem\/DSL\/functions.php","start_line_in_file":90,"slug":"stdout-filesystem","name":"stdout_filesystem","namespace":"Flow\\Filesystem\\DSL","parameters":[{"name":"protocol","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"'stdout'"}],"return_type":[{"name":"StdOutFilesystem","namespace":"Flow\\Filesystem\\Local","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"FILESYSTEM","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIFdyaXRlLW9ubHkgZmlsZXN5c3RlbSB1c2VmdWwgd2hlbiB3ZSBqdXN0IHdhbnQgdG8gd3JpdGUgdGhlIG91dHB1dCB0byBzdGRvdXQuCiAqIFRoZSBtYWluIHVzZSBjYXNlIGlzIGZvciBzdHJlYW1pbmcgZGF0YXNldHMgb3ZlciBodHRwLgogKi8="},{"repository_path":"src\/lib\/filesystem\/src\/Flow\/Filesystem\/DSL\/functions.php","start_line_in_file":99,"slug":"memory-filesystem","name":"memory_filesystem","namespace":"Flow\\Filesystem\\DSL","parameters":[{"name":"protocol","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"'memory'"}],"return_type":[{"name":"MemoryFilesystem","namespace":"Flow\\Filesystem\\Local","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"FILESYSTEM","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBhIG5ldyBtZW1vcnkgZmlsZXN5c3RlbSBhbmQgd3JpdGVzIGRhdGEgdG8gaXQgaW4gbWVtb3J5LgogKi8="},{"repository_path":"src\/lib\/filesystem\/src\/Flow\/Filesystem\/DSL\/functions.php","start_line_in_file":110,"slug":"fstab","name":"fstab","namespace":"Flow\\Filesystem\\DSL","parameters":[{"name":"filesystems","type":[{"name":"Filesystem","namespace":"Flow\\Filesystem","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":true,"default_value":null}],"return_type":[{"name":"FilesystemTable","namespace":"Flow\\Filesystem","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"FILESYSTEM","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBhIG5ldyBmaWxlc3lzdGVtIHRhYmxlIHdpdGggZ2l2ZW4gZmlsZXN5c3RlbXMuCiAqIEZpbGVzeXN0ZW1zIGNhbiBiZSBhbHNvIG1vdW50ZWQgbGF0ZXIuCiAqIElmIG5vIGZpbGVzeXN0ZW1zIGFyZSBwcm92aWRlZCwgbG9jYWwgZmlsZXN5c3RlbSBpcyBtb3VudGVkLgogKi8="},{"repository_path":"src\/lib\/filesystem\/src\/Flow\/Filesystem\/DSL\/functions.php","start_line_in_file":126,"slug":"traceable-filesystem","name":"traceable_filesystem","namespace":"Flow\\Filesystem\\DSL","parameters":[{"name":"filesystem","type":[{"name":"Filesystem","namespace":"Flow\\Filesystem","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"telemetryConfig","type":[{"name":"FilesystemTelemetryConfig","namespace":"Flow\\Filesystem\\Telemetry","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"TraceableFilesystem","namespace":"Flow\\Filesystem\\Telemetry","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"FILESYSTEM","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIFdyYXAgYSBmaWxlc3lzdGVtIHdpdGggdGVsZW1ldHJ5IHRyYWNpbmcgc3VwcG9ydC4KICogQWxsIGZpbGVzeXN0ZW0gYW5kIHN0cmVhbSBvcGVyYXRpb25zIHdpbGwgYmUgdHJhY2VkIGFjY29yZGluZyB0byB0aGUgY29uZmlndXJhdGlvbi4KICov"},{"repository_path":"src\/lib\/filesystem\/src\/Flow\/Filesystem\/DSL\/functions.php","start_line_in_file":135,"slug":"filesystem-telemetry-config","name":"filesystem_telemetry_config","namespace":"Flow\\Filesystem\\DSL","parameters":[{"name":"telemetry","type":[{"name":"Telemetry","namespace":"Flow\\Telemetry","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"clock","type":[{"name":"ClockInterface","namespace":"Psr\\Clock","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"options","type":[{"name":"FilesystemTelemetryOptions","namespace":"Flow\\Filesystem\\Telemetry","is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"null"}],"return_type":[{"name":"FilesystemTelemetryConfig","namespace":"Flow\\Filesystem\\Telemetry","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"FILESYSTEM","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBhIHRlbGVtZXRyeSBjb25maWd1cmF0aW9uIGZvciB0aGUgZmlsZXN5c3RlbS4KICov"},{"repository_path":"src\/lib\/filesystem\/src\/Flow\/Filesystem\/DSL\/functions.php","start_line_in_file":150,"slug":"filesystem-telemetry-options","name":"filesystem_telemetry_options","namespace":"Flow\\Filesystem\\DSL","parameters":[{"name":"trace_streams","type":[{"name":"bool","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"true"},{"name":"collect_metrics","type":[{"name":"bool","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"true"}],"return_type":[{"name":"FilesystemTelemetryOptions","namespace":"Flow\\Filesystem\\Telemetry","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"FILESYSTEM","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBvcHRpb25zIGZvciBmaWxlc3lzdGVtIHRlbGVtZXRyeS4KICoKICogQHBhcmFtIGJvb2wgJHRyYWNlX3N0cmVhbXMgQ3JlYXRlIGEgc2luZ2xlIHNwYW4gcGVyIHN0cmVhbSBsaWZlY3ljbGUgKGRlZmF1bHQ6IE9OKQogKiBAcGFyYW0gYm9vbCAkY29sbGVjdF9tZXRyaWNzIENvbGxlY3QgbWV0cmljcyBmb3IgYnl0ZXMvb3BlcmF0aW9uIGNvdW50cyAoZGVmYXVsdDogT04pCiAqLw=="},{"repository_path":"src\/lib\/filesystem\/src\/Flow\/Filesystem\/DSL\/functions.php","start_line_in_file":163,"slug":"file-copy","name":"file_copy","namespace":"Flow\\Filesystem\\DSL","parameters":[{"name":"table","type":[{"name":"FilesystemTable","namespace":"Flow\\Filesystem","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"options","type":[{"name":"OperationOptions","namespace":"Flow\\Filesystem\\Operations","is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"null"}],"return_type":[{"name":"Copy","namespace":"Flow\\Filesystem\\Operations","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"FILESYSTEM","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENvcHkgYSBmaWxlIGZyb20gb25lIHBhdGggdG8gYW5vdGhlciwgYWNyb3NzIGFueSBmaWxlc3lzdGVtcyBtb3VudGVkIGluIHRoZSB0YWJsZS4KICogQWx3YXlzIHN0cmVhbXMgYnl0ZXM7IHNhbWUtZmlsZXN5c3RlbSBjb3BpZXMgZG8gbm90IHVzZSBzZXJ2ZXItc2lkZSBvcHRpbWl6YXRpb25zCiAqIGJlY2F1c2UgYEZpbGVzeXN0ZW06Om12YCBpcyBhIG1vdmUsIG5vdCBhIGNvcHkuCiAqLw=="},{"repository_path":"src\/lib\/filesystem\/src\/Flow\/Filesystem\/DSL\/functions.php","start_line_in_file":174,"slug":"file-move","name":"file_move","namespace":"Flow\\Filesystem\\DSL","parameters":[{"name":"table","type":[{"name":"FilesystemTable","namespace":"Flow\\Filesystem","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"options","type":[{"name":"OperationOptions","namespace":"Flow\\Filesystem\\Operations","is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"null"}],"return_type":[{"name":"Move","namespace":"Flow\\Filesystem\\Operations","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"FILESYSTEM","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIE1vdmUgYSBmaWxlIGZyb20gb25lIHBhdGggdG8gYW5vdGhlciwgYWNyb3NzIGFueSBmaWxlc3lzdGVtcyBtb3VudGVkIGluIHRoZSB0YWJsZS4KICogSW50cmEtZmlsZXN5c3RlbSBtb3ZlcyBkZWxlZ2F0ZSB0byBgRmlsZXN5c3RlbTo6bXZgIGZvciBzZXJ2ZXItc2lkZSBvcHRpbWl6YXRpb25zOwogKiBjcm9zcy1maWxlc3lzdGVtIG1vdmVzIHN0cmVhbS1jb3B5IHRoZW4gcmVtb3ZlIHRoZSBzb3VyY2UgKG5vbi1hdG9taWMpLgogKi8="},{"repository_path":"src\/lib\/filesystem\/src\/Flow\/Filesystem\/DSL\/functions.php","start_line_in_file":185,"slug":"operation-options","name":"operation_options","namespace":"Flow\\Filesystem\\DSL","parameters":[{"name":"chunkSize","type":[{"name":"int","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"8192"}],"return_type":[{"name":"OperationOptions","namespace":"Flow\\Filesystem\\Operations","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"FILESYSTEM","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIE9wdGlvbnMgc2hhcmVkIGJ5IGZpbGVzeXN0ZW0gb3BlcmF0aW9ucy4KICoKICogQHBhcmFtIGludCAkY2h1bmtTaXplIE51bWJlciBvZiBieXRlcyByZWFkL3dyaXR0ZW4gcGVyIGl0ZXJhdGlvbiB3aGVuIHN0cmVhbWluZyBhY3Jvc3MgZmlsZXN5c3RlbXMgKGRlZmF1bHQ6IDgxOTIpCiAqLw=="},{"repository_path":"src\/lib\/types\/src\/Flow\/Types\/DSL\/functions.php","start_line_in_file":71,"slug":"type-structure","name":"type_structure","namespace":"Flow\\Types\\DSL","parameters":[{"name":"elements","type":[{"name":"array","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"[]"}],"return_type":[{"name":"StructureType","namespace":"Flow\\Types\\Type\\Logical","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"TYPES","type":"TYPE"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIEB0ZW1wbGF0ZSBUCiAqCiAqIEBwYXJhbSBhcnJheTxhcnJheS1rZXksIFN0cnVjdHVyZUVsZW1lbnQ8VD58VHlwZTxUPj4gJGVsZW1lbnRzCiAqCiAqIEByZXR1cm4gU3RydWN0dXJlVHlwZTxhcnJheTxhcnJheS1rZXksIFQ+PgogKi8="},{"repository_path":"src\/lib\/types\/src\/Flow\/Types\/DSL\/functions.php","start_line_in_file":86,"slug":"structure-element","name":"structure_element","namespace":"Flow\\Types\\DSL","parameters":[{"name":"name","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false},{"name":"int","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"type","type":[{"name":"Type","namespace":"Flow\\Types","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"optional","type":[{"name":"bool","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"false"}],"return_type":[{"name":"StructureElement","namespace":"Flow\\Types\\Type\\Logical","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"TYPES","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIEB0ZW1wbGF0ZSBUCiAqIEB0ZW1wbGF0ZSBUT3B0aW9uYWwgb2YgYm9vbAogKgogKiBAcGFyYW0gVHlwZTxUPiAkdHlwZQogKiBAcGFyYW0gVE9wdGlvbmFsICRvcHRpb25hbAogKgogKiBAcmV0dXJuIFN0cnVjdHVyZUVsZW1lbnQ8VCwgVE9wdGlvbmFsPgogKi8="},{"repository_path":"src\/lib\/types\/src\/Flow\/Types\/DSL\/functions.php","start_line_in_file":101,"slug":"type-union","name":"type_union","namespace":"Flow\\Types\\DSL","parameters":[{"name":"first","type":[{"name":"Type","namespace":"Flow\\Types","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"second","type":[{"name":"Type","namespace":"Flow\\Types","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"types","type":[{"name":"Type","namespace":"Flow\\Types","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":true,"default_value":null}],"return_type":[{"name":"UnionType","namespace":"Flow\\Types\\Type\\Native","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"TYPES","type":"TYPE"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIEB0ZW1wbGF0ZSBUCiAqCiAqIEBwYXJhbSBUeXBlPFQ+ICRmaXJzdAogKiBAcGFyYW0gVHlwZTxUPiAkc2Vjb25kCiAqIEBwYXJhbSBUeXBlPFQ+IC4uLiR0eXBlcwogKgogKiBAcmV0dXJuIFVuaW9uVHlwZTxULCBUPgogKi8="},{"repository_path":"src\/lib\/types\/src\/Flow\/Types\/DSL\/functions.php","start_line_in_file":122,"slug":"type-intersection","name":"type_intersection","namespace":"Flow\\Types\\DSL","parameters":[{"name":"first","type":[{"name":"Type","namespace":"Flow\\Types","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"second","type":[{"name":"Type","namespace":"Flow\\Types","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"types","type":[{"name":"Type","namespace":"Flow\\Types","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":true,"default_value":null}],"return_type":[{"name":"IntersectionType","namespace":"Flow\\Types\\Type\\Native","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"TYPES","type":"TYPE"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIEB0ZW1wbGF0ZSBUCiAqCiAqIEBwYXJhbSBUeXBlPFQ+ICRmaXJzdAogKiBAcGFyYW0gVHlwZTxUPiAkc2Vjb25kCiAqIEBwYXJhbSBUeXBlPFQ+IC4uLiR0eXBlcwogKgogKiBAcmV0dXJuIEludGVyc2VjdGlvblR5cGU8VCwgVD4KICov"},{"repository_path":"src\/lib\/types\/src\/Flow\/Types\/DSL\/functions.php","start_line_in_file":137,"slug":"type-numeric-string","name":"type_numeric_string","namespace":"Flow\\Types\\DSL","parameters":[],"return_type":[{"name":"NumericStringType","namespace":"Flow\\Types\\Type\\Logical","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"TYPES","type":"TYPE"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIEByZXR1cm4gTnVtZXJpY1N0cmluZ1R5cGU8bnVtZXJpYy1zdHJpbmc+CiAqLw=="},{"repository_path":"src\/lib\/types\/src\/Flow\/Types\/DSL\/functions.php","start_line_in_file":150,"slug":"type-optional","name":"type_optional","namespace":"Flow\\Types\\DSL","parameters":[{"name":"type","type":[{"name":"Type","namespace":"Flow\\Types","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"OptionalType","namespace":"Flow\\Types\\Type\\Logical","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"TYPES","type":"TYPE"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIEB0ZW1wbGF0ZSBUCiAqCiAqIEBwYXJhbSBUeXBlPFQ+ICR0eXBlCiAqCiAqIEByZXR1cm4gT3B0aW9uYWxUeXBlPFQ+CiAqLw=="},{"repository_path":"src\/lib\/types\/src\/Flow\/Types\/DSL\/functions.php","start_line_in_file":161,"slug":"type-from-array","name":"type_from_array","namespace":"Flow\\Types\\DSL","parameters":[{"name":"data","type":[{"name":"array","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"Type","namespace":"Flow\\Types","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"TYPES","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIEBwYXJhbSBhcnJheTxzdHJpbmcsIG1peGVkPiAkZGF0YQogKgogKiBAcmV0dXJuIFR5cGU8bWl4ZWQ+CiAqLw=="},{"repository_path":"src\/lib\/types\/src\/Flow\/Types\/DSL\/functions.php","start_line_in_file":172,"slug":"type-is-nullable","name":"type_is_nullable","namespace":"Flow\\Types\\DSL","parameters":[{"name":"type","type":[{"name":"Type","namespace":"Flow\\Types","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"bool","namespace":null,"is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"TYPES","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIEB0ZW1wbGF0ZSBUCiAqCiAqIEBwYXJhbSBUeXBlPFQ+ICR0eXBlCiAqLw=="},{"repository_path":"src\/lib\/types\/src\/Flow\/Types\/DSL\/functions.php","start_line_in_file":188,"slug":"type-bare","name":"type_bare","namespace":"Flow\\Types\\DSL","parameters":[{"name":"type","type":[{"name":"Type","namespace":"Flow\\Types","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"Type","namespace":"Flow\\Types","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"TYPES","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIFN0cmlwIGV4YWN0bHkgb25lIGxldmVsIG9mIG51bGxhYmlsaXR5LCB3aGljaGV2ZXIgb2YgdGhlIHR3byBzcGVsbGluZ3MgY2FycmllcyBpdAogKiAoT3B0aW9uYWxUeXBlLCBvciBhIFVuaW9uVHlwZSBjb250YWluaW5nIE51bGxUeXBlKS4gVG90YWw6IGEgTk9UIE5VTEwgdHlwZSBpcyByZXR1cm5lZCB1bmNoYW5nZWQuCiAqCiAqIEB0ZW1wbGF0ZSBUCiAqCiAqIEBwYXJhbSBUeXBlPFQ+ICR0eXBlCiAqCiAqIEByZXR1cm4gVHlwZTxtaXhlZD4KICov"},{"repository_path":"src\/lib\/types\/src\/Flow\/Types\/DSL\/functions.php","start_line_in_file":198,"slug":"type-equals","name":"type_equals","namespace":"Flow\\Types\\DSL","parameters":[{"name":"left","type":[{"name":"Type","namespace":"Flow\\Types","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"right","type":[{"name":"Type","namespace":"Flow\\Types","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"bool","namespace":null,"is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"TYPES","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIEBwYXJhbSBUeXBlPG1peGVkPiAkbGVmdAogKiBAcGFyYW0gVHlwZTxtaXhlZD4gJHJpZ2h0CiAqLw=="},{"repository_path":"src\/lib\/types\/src\/Flow\/Types\/DSL\/functions.php","start_line_in_file":211,"slug":"types","name":"types","namespace":"Flow\\Types\\DSL","parameters":[{"name":"types","type":[{"name":"Type","namespace":"Flow\\Types","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":true,"default_value":null}],"return_type":[{"name":"Types","namespace":"Flow\\Types\\Type","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"TYPES","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIEB0ZW1wbGF0ZSBUCiAqCiAqIEBwYXJhbSBUeXBlPFQ+IC4uLiR0eXBlcwogKgogKiBAcmV0dXJuIFR5cGVzPFQ+CiAqLw=="},{"repository_path":"src\/lib\/types\/src\/Flow\/Types\/DSL\/functions.php","start_line_in_file":224,"slug":"type-list","name":"type_list","namespace":"Flow\\Types\\DSL","parameters":[{"name":"element","type":[{"name":"Type","namespace":"Flow\\Types","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"ListType","namespace":"Flow\\Types\\Type\\Logical","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"TYPES","type":"TYPE"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIEB0ZW1wbGF0ZSBUCiAqCiAqIEBwYXJhbSBUeXBlPFQ+ICRlbGVtZW50CiAqCiAqIEByZXR1cm4gTGlzdFR5cGU8bGlzdDxUPj4KICov"},{"repository_path":"src\/lib\/types\/src\/Flow\/Types\/DSL\/functions.php","start_line_in_file":239,"slug":"type-map","name":"type_map","namespace":"Flow\\Types\\DSL","parameters":[{"name":"key_type","type":[{"name":"Type","namespace":"Flow\\Types","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"value_type","type":[{"name":"Type","namespace":"Flow\\Types","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"MapType","namespace":"Flow\\Types\\Type\\Logical","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"TYPES","type":"TYPE"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIEB0ZW1wbGF0ZSBUS2V5IG9mIGFycmF5LWtleQogKiBAdGVtcGxhdGUgVFZhbHVlCiAqCiAqIEBwYXJhbSBUeXBlPFRLZXk+ICRrZXlfdHlwZQogKiBAcGFyYW0gVHlwZTxUVmFsdWU+ICR2YWx1ZV90eXBlCiAqCiAqIEByZXR1cm4gTWFwVHlwZTxhcnJheTxUS2V5LCBUVmFsdWU+PgogKi8="},{"repository_path":"src\/lib\/types\/src\/Flow\/Types\/DSL\/functions.php","start_line_in_file":248,"slug":"type-json","name":"type_json","namespace":"Flow\\Types\\DSL","parameters":[],"return_type":[{"name":"JsonType","namespace":"Flow\\Types\\Type\\Logical","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"TYPES","type":"TYPE"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIEByZXR1cm4gSnNvblR5cGU8SnNvbj4KICov"},{"repository_path":"src\/lib\/types\/src\/Flow\/Types\/DSL\/functions.php","start_line_in_file":257,"slug":"type-datetime","name":"type_datetime","namespace":"Flow\\Types\\DSL","parameters":[{"name":"zone","type":[{"name":"DateTimeZone","namespace":"","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"'UTC'"}],"return_type":[{"name":"DateTimeType","namespace":"Flow\\Types\\Type\\Logical","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"TYPES","type":"TYPE"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIEByZXR1cm4gRGF0ZVRpbWVUeXBlPFxEYXRlVGltZUludGVyZmFjZT4KICov"},{"repository_path":"src\/lib\/types\/src\/Flow\/Types\/DSL\/functions.php","start_line_in_file":266,"slug":"type-date","name":"type_date","namespace":"Flow\\Types\\DSL","parameters":[],"return_type":[{"name":"DateType","namespace":"Flow\\Types\\Type\\Logical","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"TYPES","type":"TYPE"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIEByZXR1cm4gRGF0ZVR5cGU8XERhdGVUaW1lSW50ZXJmYWNlPgogKi8="},{"repository_path":"src\/lib\/types\/src\/Flow\/Types\/DSL\/functions.php","start_line_in_file":275,"slug":"type-time","name":"type_time","namespace":"Flow\\Types\\DSL","parameters":[],"return_type":[{"name":"TimeType","namespace":"Flow\\Types\\Type\\Logical","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"TYPES","type":"TYPE"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIEByZXR1cm4gVGltZVR5cGU8XERhdGVJbnRlcnZhbD4KICov"},{"repository_path":"src\/lib\/types\/src\/Flow\/Types\/DSL\/functions.php","start_line_in_file":284,"slug":"type-time-zone","name":"type_time_zone","namespace":"Flow\\Types\\DSL","parameters":[],"return_type":[{"name":"TimeZoneType","namespace":"Flow\\Types\\Type\\Logical","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"TYPES","type":"TYPE"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIEByZXR1cm4gVGltZVpvbmVUeXBlPFxEYXRlVGltZVpvbmU+CiAqLw=="},{"repository_path":"src\/lib\/types\/src\/Flow\/Types\/DSL\/functions.php","start_line_in_file":293,"slug":"type-xml","name":"type_xml","namespace":"Flow\\Types\\DSL","parameters":[],"return_type":[{"name":"XMLType","namespace":"Flow\\Types\\Type\\Logical","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"TYPES","type":"TYPE"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIEByZXR1cm4gWE1MVHlwZTxcRE9NRG9jdW1lbnR8WE1MRG9jdW1lbnQ+CiAqLw=="},{"repository_path":"src\/lib\/types\/src\/Flow\/Types\/DSL\/functions.php","start_line_in_file":302,"slug":"type-xml-element","name":"type_xml_element","namespace":"Flow\\Types\\DSL","parameters":[],"return_type":[{"name":"XMLElementType","namespace":"Flow\\Types\\Type\\Logical","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"TYPES","type":"TYPE"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIEByZXR1cm4gWE1MRWxlbWVudFR5cGU8XERPTUVsZW1lbnR8RWxlbWVudD4KICov"},{"repository_path":"src\/lib\/types\/src\/Flow\/Types\/DSL\/functions.php","start_line_in_file":311,"slug":"type-uuid","name":"type_uuid","namespace":"Flow\\Types\\DSL","parameters":[],"return_type":[{"name":"UuidType","namespace":"Flow\\Types\\Type\\Logical","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"TYPES","type":"TYPE"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIEByZXR1cm4gVXVpZFR5cGU8VXVpZD4KICov"},{"repository_path":"src\/lib\/types\/src\/Flow\/Types\/DSL\/functions.php","start_line_in_file":320,"slug":"type-integer","name":"type_integer","namespace":"Flow\\Types\\DSL","parameters":[],"return_type":[{"name":"IntegerType","namespace":"Flow\\Types\\Type\\Native","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"TYPES","type":"TYPE"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIEByZXR1cm4gSW50ZWdlclR5cGU8aW50PgogKi8="},{"repository_path":"src\/lib\/types\/src\/Flow\/Types\/DSL\/functions.php","start_line_in_file":329,"slug":"type-string","name":"type_string","namespace":"Flow\\Types\\DSL","parameters":[],"return_type":[{"name":"StringType","namespace":"Flow\\Types\\Type\\Native","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"TYPES","type":"TYPE"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIEByZXR1cm4gU3RyaW5nVHlwZTxzdHJpbmc+CiAqLw=="},{"repository_path":"src\/lib\/types\/src\/Flow\/Types\/DSL\/functions.php","start_line_in_file":338,"slug":"type-float","name":"type_float","namespace":"Flow\\Types\\DSL","parameters":[],"return_type":[{"name":"FloatType","namespace":"Flow\\Types\\Type\\Native","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"TYPES","type":"TYPE"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIEByZXR1cm4gRmxvYXRUeXBlPGZsb2F0PgogKi8="},{"repository_path":"src\/lib\/types\/src\/Flow\/Types\/DSL\/functions.php","start_line_in_file":347,"slug":"type-boolean","name":"type_boolean","namespace":"Flow\\Types\\DSL","parameters":[],"return_type":[{"name":"BooleanType","namespace":"Flow\\Types\\Type\\Native","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"TYPES","type":"TYPE"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIEByZXR1cm4gQm9vbGVhblR5cGU8Ym9vbD4KICov"},{"repository_path":"src\/lib\/types\/src\/Flow\/Types\/DSL\/functions.php","start_line_in_file":360,"slug":"type-instance-of","name":"type_instance_of","namespace":"Flow\\Types\\DSL","parameters":[{"name":"class","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"InstanceOfType","namespace":"Flow\\Types\\Type\\Logical","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"TYPES","type":"TYPE"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIEB0ZW1wbGF0ZSBUIG9mIG9iamVjdAogKgogKiBAcGFyYW0gY2xhc3Mtc3RyaW5nPFQ+ICRjbGFzcwogKgogKiBAcmV0dXJuIEluc3RhbmNlT2ZUeXBlPFQ+CiAqLw=="},{"repository_path":"src\/lib\/types\/src\/Flow\/Types\/DSL\/functions.php","start_line_in_file":369,"slug":"type-object","name":"type_object","namespace":"Flow\\Types\\DSL","parameters":[],"return_type":[{"name":"ObjectType","namespace":"Flow\\Types\\Type\\Native","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"TYPES","type":"TYPE"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIEByZXR1cm4gT2JqZWN0VHlwZTxvYmplY3Q+CiAqLw=="},{"repository_path":"src\/lib\/types\/src\/Flow\/Types\/DSL\/functions.php","start_line_in_file":378,"slug":"type-scalar","name":"type_scalar","namespace":"Flow\\Types\\DSL","parameters":[],"return_type":[{"name":"ScalarType","namespace":"Flow\\Types\\Type\\Logical","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"TYPES","type":"TYPE"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIEByZXR1cm4gU2NhbGFyVHlwZTxib29sfGZsb2F0fGludHxzdHJpbmc+CiAqLw=="},{"repository_path":"src\/lib\/types\/src\/Flow\/Types\/DSL\/functions.php","start_line_in_file":387,"slug":"type-resource","name":"type_resource","namespace":"Flow\\Types\\DSL","parameters":[],"return_type":[{"name":"ResourceType","namespace":"Flow\\Types\\Type\\Native","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"TYPES","type":"TYPE"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIEByZXR1cm4gUmVzb3VyY2VUeXBlPHJlc291cmNlPgogKi8="},{"repository_path":"src\/lib\/types\/src\/Flow\/Types\/DSL\/functions.php","start_line_in_file":396,"slug":"type-array","name":"type_array","namespace":"Flow\\Types\\DSL","parameters":[],"return_type":[{"name":"ArrayType","namespace":"Flow\\Types\\Type\\Native","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"TYPES","type":"TYPE"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIEByZXR1cm4gQXJyYXlUeXBlPGFycmF5PG1peGVkPj4KICov"},{"repository_path":"src\/lib\/types\/src\/Flow\/Types\/DSL\/functions.php","start_line_in_file":405,"slug":"type-callable","name":"type_callable","namespace":"Flow\\Types\\DSL","parameters":[],"return_type":[{"name":"CallableType","namespace":"Flow\\Types\\Type\\Native","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"TYPES","type":"TYPE"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIEByZXR1cm4gQ2FsbGFibGVUeXBlPGNhbGxhYmxlPgogKi8="},{"repository_path":"src\/lib\/types\/src\/Flow\/Types\/DSL\/functions.php","start_line_in_file":414,"slug":"type-null","name":"type_null","namespace":"Flow\\Types\\DSL","parameters":[],"return_type":[{"name":"NullType","namespace":"Flow\\Types\\Type\\Native","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"TYPES","type":"TYPE"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIEByZXR1cm4gTnVsbFR5cGU8bnVsbD4KICov"},{"repository_path":"src\/lib\/types\/src\/Flow\/Types\/DSL\/functions.php","start_line_in_file":423,"slug":"type-mixed","name":"type_mixed","namespace":"Flow\\Types\\DSL","parameters":[],"return_type":[{"name":"MixedType","namespace":"Flow\\Types\\Type\\Native","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"TYPES","type":"TYPE"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIEByZXR1cm4gTWl4ZWRUeXBlPG1peGVkPgogKi8="},{"repository_path":"src\/lib\/types\/src\/Flow\/Types\/DSL\/functions.php","start_line_in_file":432,"slug":"type-positive-integer","name":"type_positive_integer","namespace":"Flow\\Types\\DSL","parameters":[],"return_type":[{"name":"PositiveIntegerType","namespace":"Flow\\Types\\Type\\Logical","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"TYPES","type":"TYPE"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIEByZXR1cm4gUG9zaXRpdmVJbnRlZ2VyVHlwZTxpbnQ8MCwgbWF4Pj4KICov"},{"repository_path":"src\/lib\/types\/src\/Flow\/Types\/DSL\/functions.php","start_line_in_file":441,"slug":"type-non-empty-string","name":"type_non_empty_string","namespace":"Flow\\Types\\DSL","parameters":[],"return_type":[{"name":"NonEmptyStringType","namespace":"Flow\\Types\\Type\\Logical","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"TYPES","type":"TYPE"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIEByZXR1cm4gTm9uRW1wdHlTdHJpbmdUeXBlPG5vbi1lbXB0eS1zdHJpbmc+CiAqLw=="},{"repository_path":"src\/lib\/types\/src\/Flow\/Types\/DSL\/functions.php","start_line_in_file":450,"slug":"type-empty-array","name":"type_empty_array","namespace":"Flow\\Types\\DSL","parameters":[],"return_type":[{"name":"EmptyArrayType","namespace":"Flow\\Types\\Type\\Native","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"TYPES","type":"TYPE"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIEByZXR1cm4gRW1wdHlBcnJheVR5cGU8YXJyYXl7fT4KICov"},{"repository_path":"src\/lib\/types\/src\/Flow\/Types\/DSL\/functions.php","start_line_in_file":463,"slug":"type-enum","name":"type_enum","namespace":"Flow\\Types\\DSL","parameters":[{"name":"class","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"EnumType","namespace":"Flow\\Types\\Type\\Native","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"TYPES","type":"TYPE"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIEB0ZW1wbGF0ZSBUIG9mIFVuaXRFbnVtCiAqCiAqIEBwYXJhbSBjbGFzcy1zdHJpbmc8VD4gJGNsYXNzCiAqCiAqIEByZXR1cm4gRW51bVR5cGU8VD4KICov"},{"repository_path":"src\/lib\/types\/src\/Flow\/Types\/DSL\/functions.php","start_line_in_file":476,"slug":"type-literal","name":"type_literal","namespace":"Flow\\Types\\DSL","parameters":[{"name":"value","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false},{"name":"int","namespace":null,"is_nullable":false,"is_variadic":false},{"name":"float","namespace":null,"is_nullable":false,"is_variadic":false},{"name":"bool","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"LiteralType","namespace":"Flow\\Types\\Type\\Logical","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"TYPES","type":"TYPE"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIEB0ZW1wbGF0ZSBUIG9mIGJvb2x8ZmxvYXR8aW50fHN0cmluZwogKgogKiBAcGFyYW0gVCAkdmFsdWUKICoKICogQHJldHVybiBMaXRlcmFsVHlwZTxUPgogKi8="},{"repository_path":"src\/lib\/types\/src\/Flow\/Types\/DSL\/functions.php","start_line_in_file":485,"slug":"type-html","name":"type_html","namespace":"Flow\\Types\\DSL","parameters":[],"return_type":[{"name":"HTMLType","namespace":"Flow\\Types\\Type\\Logical","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"TYPES","type":"TYPE"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIEByZXR1cm4gSFRNTFR5cGU8SFRNTERvY3VtZW50PgogKi8="},{"repository_path":"src\/lib\/types\/src\/Flow\/Types\/DSL\/functions.php","start_line_in_file":494,"slug":"type-html-element","name":"type_html_element","namespace":"Flow\\Types\\DSL","parameters":[],"return_type":[{"name":"HTMLElementType","namespace":"Flow\\Types\\Type\\Logical","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"TYPES","type":"TYPE"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIEByZXR1cm4gSFRNTEVsZW1lbnRUeXBlPEhUTUxFbGVtZW50PgogKi8="},{"repository_path":"src\/lib\/types\/src\/Flow\/Types\/DSL\/functions.php","start_line_in_file":506,"slug":"type-is","name":"type_is","namespace":"Flow\\Types\\DSL","parameters":[{"name":"type","type":[{"name":"Type","namespace":"Flow\\Types","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"typeClass","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"bool","namespace":null,"is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"TYPES","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIEB0ZW1wbGF0ZSBUCiAqCiAqIEBwYXJhbSBUeXBlPFQ+ICR0eXBlCiAqIEBwYXJhbSBjbGFzcy1zdHJpbmc8VHlwZTxtaXhlZD4+ICR0eXBlQ2xhc3MKICov"},{"repository_path":"src\/lib\/types\/src\/Flow\/Types\/DSL\/functions.php","start_line_in_file":519,"slug":"type-is-any","name":"type_is_any","namespace":"Flow\\Types\\DSL","parameters":[{"name":"type","type":[{"name":"Type","namespace":"Flow\\Types","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"typeClass","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"typeClasses","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":true,"default_value":null}],"return_type":[{"name":"bool","namespace":null,"is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"TYPES","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIEB0ZW1wbGF0ZSBUCiAqCiAqIEBwYXJhbSBUeXBlPFQ+ICR0eXBlCiAqIEBwYXJhbSBjbGFzcy1zdHJpbmc8VHlwZTxtaXhlZD4+ICR0eXBlQ2xhc3MKICogQHBhcmFtIGNsYXNzLXN0cmluZzxUeXBlPG1peGVkPj4gLi4uJHR5cGVDbGFzc2VzCiAqLw=="},{"repository_path":"src\/lib\/types\/src\/Flow\/Types\/DSL\/functions.php","start_line_in_file":528,"slug":"get-type","name":"get_type","namespace":"Flow\\Types\\DSL","parameters":[{"name":"value","type":[{"name":"mixed","namespace":null,"is_nullable":true,"is_variadic":false}],"has_default_value":false,"is_nullable":true,"is_variadic":false,"default_value":null}],"return_type":[{"name":"Type","namespace":"Flow\\Types","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"TYPES","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIEByZXR1cm4gVHlwZTxtaXhlZD4KICov"},{"repository_path":"src\/lib\/types\/src\/Flow\/Types\/DSL\/functions.php","start_line_in_file":541,"slug":"type-class-string","name":"type_class_string","namespace":"Flow\\Types\\DSL","parameters":[{"name":"class","type":[{"name":"string","namespace":null,"is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"null"}],"return_type":[{"name":"ClassStringType","namespace":"Flow\\Types\\Type\\Logical","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"TYPES","type":"TYPE"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIEB0ZW1wbGF0ZSBUIG9mIG9iamVjdAogKgogKiBAcGFyYW0gbnVsbHxjbGFzcy1zdHJpbmc8VD4gJGNsYXNzCiAqCiAqIEByZXR1cm4gKCRjbGFzcyBpcyBudWxsID8gQ2xhc3NTdHJpbmdUeXBlPGNsYXNzLXN0cmluZz4gOiBDbGFzc1N0cmluZ1R5cGU8Y2xhc3Mtc3RyaW5nPFQ+PikKICov"},{"repository_path":"src\/lib\/types\/src\/Flow\/Types\/DSL\/functions.php","start_line_in_file":547,"slug":"dom-element-to-string","name":"dom_element_to_string","namespace":"Flow\\Types\\DSL","parameters":[{"name":"element","type":[{"name":"DOMElement","namespace":"","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"format_output","type":[{"name":"bool","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"false"},{"name":"preserver_white_space","type":[{"name":"bool","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"false"}],"return_type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false},{"name":"false","namespace":null,"is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"TYPES","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/schema.php","start_line_in_file":134,"slug":"column","name":"column","namespace":"Flow\\PostgreSql\\DSL","parameters":[{"name":"name","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"type","type":[{"name":"ColumnType","namespace":"Flow\\PostgreSql\\QueryBuilder\\Schema","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"ColumnDefinition","namespace":"Flow\\PostgreSql\\QueryBuilder\\Schema","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBhIGNvbHVtbiBkZWZpbml0aW9uIGZvciBDUkVBVEUgVEFCTEUuCiAqCiAqIEBwYXJhbSBzdHJpbmcgJG5hbWUgQ29sdW1uIG5hbWUKICogQHBhcmFtIENvbHVtblR5cGUgJHR5cGUgQ29sdW1uIGRhdGEgdHlwZQogKi8="},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/schema.php","start_line_in_file":143,"slug":"catalog","name":"catalog","namespace":"Flow\\PostgreSql\\DSL","parameters":[{"name":"schemas","type":[{"name":"array","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"Catalog","namespace":"Flow\\PostgreSql\\Schema","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIEBwYXJhbSBsaXN0PFNjaGVtYT4gJHNjaGVtYXMKICov"},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/schema.php","start_line_in_file":154,"slug":"primary-key","name":"primary_key","namespace":"Flow\\PostgreSql\\DSL","parameters":[{"name":"columns","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":true,"default_value":null}],"return_type":[{"name":"PrimaryKeyConstraint","namespace":"Flow\\PostgreSql\\QueryBuilder\\Schema\\Constraint","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBhIFBSSU1BUlkgS0VZIGNvbnN0cmFpbnQuCiAqCiAqIEBwYXJhbSBzdHJpbmcgLi4uJGNvbHVtbnMgQ29sdW1ucyB0aGF0IGZvcm0gdGhlIHByaW1hcnkga2V5CiAqLw=="},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/schema.php","start_line_in_file":165,"slug":"unique-constraint","name":"unique_constraint","namespace":"Flow\\PostgreSql\\DSL","parameters":[{"name":"columns","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":true,"default_value":null}],"return_type":[{"name":"UniqueConstraint","namespace":"Flow\\PostgreSql\\QueryBuilder\\Schema\\Constraint","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBhIFVOSVFVRSBjb25zdHJhaW50LgogKgogKiBAcGFyYW0gc3RyaW5nIC4uLiRjb2x1bW5zIENvbHVtbnMgdGhhdCBtdXN0IGJlIHVuaXF1ZSB0b2dldGhlcgogKi8="},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/schema.php","start_line_in_file":178,"slug":"foreign-key","name":"foreign_key","namespace":"Flow\\PostgreSql\\DSL","parameters":[{"name":"columns","type":[{"name":"array","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"referenceTable","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"referenceColumns","type":[{"name":"array","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"[]"}],"return_type":[{"name":"ForeignKeyConstraint","namespace":"Flow\\PostgreSql\\QueryBuilder\\Schema\\Constraint","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBhIEZPUkVJR04gS0VZIGNvbnN0cmFpbnQuCiAqCiAqIEBwYXJhbSBsaXN0PHN0cmluZz4gJGNvbHVtbnMgTG9jYWwgY29sdW1ucwogKiBAcGFyYW0gc3RyaW5nICRyZWZlcmVuY2VUYWJsZSBSZWZlcmVuY2VkIHRhYmxlCiAqIEBwYXJhbSBsaXN0PHN0cmluZz4gJHJlZmVyZW5jZUNvbHVtbnMgUmVmZXJlbmNlZCBjb2x1bW5zIChkZWZhdWx0cyB0byBzYW1lIGFzICRjb2x1bW5zIGlmIGVtcHR5KQogKi8="},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/schema.php","start_line_in_file":187,"slug":"check-constraint","name":"check_constraint","namespace":"Flow\\PostgreSql\\DSL","parameters":[{"name":"condition","type":[{"name":"Condition","namespace":"Flow\\PostgreSql\\QueryBuilder\\Condition","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"CheckConstraint","namespace":"Flow\\PostgreSql\\QueryBuilder\\Schema\\Constraint","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBhIENIRUNLIGNvbnN0cmFpbnQuCiAqLw=="},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/schema.php","start_line_in_file":218,"slug":"create","name":"create","namespace":"Flow\\PostgreSql\\DSL","parameters":[],"return_type":[{"name":"CreateFactory","namespace":"Flow\\PostgreSql\\QueryBuilder\\Factory","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"SCHEMA"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBhIGZhY3RvcnkgZm9yIGJ1aWxkaW5nIENSRUFURSBzdGF0ZW1lbnRzLgogKgogKiBQcm92aWRlcyBhIHVuaWZpZWQgZW50cnkgcG9pbnQgZm9yIGFsbCBDUkVBVEUgb3BlcmF0aW9uczoKICogLSBjcmVhdGUoKS0+dGFibGUoKSAtIENSRUFURSBUQUJMRQogKiAtIGNyZWF0ZSgpLT50YWJsZUFzKCkgLSBDUkVBVEUgVEFCTEUgQVMKICogLSBjcmVhdGUoKS0+aW5kZXgoKSAtIENSRUFURSBJTkRFWAogKiAtIGNyZWF0ZSgpLT52aWV3KCkgLSBDUkVBVEUgVklFVwogKiAtIGNyZWF0ZSgpLT5tYXRlcmlhbGl6ZWRWaWV3KCkgLSBDUkVBVEUgTUFURVJJQUxJWkVEIFZJRVcKICogLSBjcmVhdGUoKS0+c2VxdWVuY2UoKSAtIENSRUFURSBTRVFVRU5DRQogKiAtIGNyZWF0ZSgpLT5zY2hlbWEoKSAtIENSRUFURSBTQ0hFTUEKICogLSBjcmVhdGUoKS0+cm9sZSgpIC0gQ1JFQVRFIFJPTEUKICogLSBjcmVhdGUoKS0+ZnVuY3Rpb24oKSAtIENSRUFURSBGVU5DVElPTgogKiAtIGNyZWF0ZSgpLT5wcm9jZWR1cmUoKSAtIENSRUFURSBQUk9DRURVUkUKICogLSBjcmVhdGUoKS0+dHJpZ2dlcigpIC0gQ1JFQVRFIFRSSUdHRVIKICogLSBjcmVhdGUoKS0+cnVsZSgpIC0gQ1JFQVRFIFJVTEUKICogLSBjcmVhdGUoKS0+ZXh0ZW5zaW9uKCkgLSBDUkVBVEUgRVhURU5TSU9OCiAqIC0gY3JlYXRlKCktPmNvbXBvc2l0ZVR5cGUoKSAtIENSRUFURSBUWVBFIChjb21wb3NpdGUpCiAqIC0gY3JlYXRlKCktPmVudW1UeXBlKCkgLSBDUkVBVEUgVFlQRSAoZW51bSkKICogLSBjcmVhdGUoKS0+cmFuZ2VUeXBlKCkgLSBDUkVBVEUgVFlQRSAocmFuZ2UpCiAqIC0gY3JlYXRlKCktPmRvbWFpbigpIC0gQ1JFQVRFIERPTUFJTgogKgogKiBFeGFtcGxlOiBjcmVhdGUoKS0+dGFibGUoJ3VzZXJzJyktPmNvbHVtbnMoY29sX2RlZignaWQnLCBjb2x1bW5fdHlwZV9zZXJpYWwoKSkpCiAqIEV4YW1wbGU6IGNyZWF0ZSgpLT5pbmRleCgnaWR4X2VtYWlsJyktPm9uKCd1c2VycycpLT5jb2x1bW5zKCdlbWFpbCcpCiAqLw=="},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/schema.php","start_line_in_file":247,"slug":"drop","name":"drop","namespace":"Flow\\PostgreSql\\DSL","parameters":[],"return_type":[{"name":"DropFactory","namespace":"Flow\\PostgreSql\\QueryBuilder\\Factory","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"SCHEMA"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBhIGZhY3RvcnkgZm9yIGJ1aWxkaW5nIERST1Agc3RhdGVtZW50cy4KICoKICogUHJvdmlkZXMgYSB1bmlmaWVkIGVudHJ5IHBvaW50IGZvciBhbGwgRFJPUCBvcGVyYXRpb25zOgogKiAtIGRyb3AoKS0+dGFibGUoKSAtIERST1AgVEFCTEUKICogLSBkcm9wKCktPmluZGV4KCkgLSBEUk9QIElOREVYCiAqIC0gZHJvcCgpLT52aWV3KCkgLSBEUk9QIFZJRVcKICogLSBkcm9wKCktPm1hdGVyaWFsaXplZFZpZXcoKSAtIERST1AgTUFURVJJQUxJWkVEIFZJRVcKICogLSBkcm9wKCktPnNlcXVlbmNlKCkgLSBEUk9QIFNFUVVFTkNFCiAqIC0gZHJvcCgpLT5zY2hlbWEoKSAtIERST1AgU0NIRU1BCiAqIC0gZHJvcCgpLT5yb2xlKCkgLSBEUk9QIFJPTEUKICogLSBkcm9wKCktPmZ1bmN0aW9uKCkgLSBEUk9QIEZVTkNUSU9OCiAqIC0gZHJvcCgpLT5wcm9jZWR1cmUoKSAtIERST1AgUFJPQ0VEVVJFCiAqIC0gZHJvcCgpLT50cmlnZ2VyKCkgLSBEUk9QIFRSSUdHRVIKICogLSBkcm9wKCktPnJ1bGUoKSAtIERST1AgUlVMRQogKiAtIGRyb3AoKS0+ZXh0ZW5zaW9uKCkgLSBEUk9QIEVYVEVOU0lPTgogKiAtIGRyb3AoKS0+dHlwZSgpIC0gRFJPUCBUWVBFCiAqIC0gZHJvcCgpLT5kb21haW4oKSAtIERST1AgRE9NQUlOCiAqIC0gZHJvcCgpLT5vd25lZCgpIC0gRFJPUCBPV05FRAogKgogKiBFeGFtcGxlOiBkcm9wKCktPnRhYmxlKCd1c2VycycsICdvcmRlcnMnKS0+aWZFeGlzdHMoKS0+Y2FzY2FkZSgpCiAqIEV4YW1wbGU6IGRyb3AoKS0+aW5kZXgoJ2lkeF9lbWFpbCcpLT5pZkV4aXN0cygpCiAqLw=="},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/schema.php","start_line_in_file":281,"slug":"alter","name":"alter","namespace":"Flow\\PostgreSql\\DSL","parameters":[],"return_type":[{"name":"AlterFactory","namespace":"Flow\\PostgreSql\\QueryBuilder\\Factory","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"SCHEMA"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBhIGZhY3RvcnkgZm9yIGJ1aWxkaW5nIEFMVEVSIHN0YXRlbWVudHMuCiAqCiAqIFByb3ZpZGVzIGEgdW5pZmllZCBlbnRyeSBwb2ludCBmb3IgYWxsIEFMVEVSIG9wZXJhdGlvbnM6CiAqIC0gYWx0ZXIoKS0+dGFibGUoKSAtIEFMVEVSIFRBQkxFCiAqIC0gYWx0ZXIoKS0+aW5kZXgoKSAtIEFMVEVSIElOREVYCiAqIC0gYWx0ZXIoKS0+dmlldygpIC0gQUxURVIgVklFVwogKiAtIGFsdGVyKCktPm1hdGVyaWFsaXplZFZpZXcoKSAtIEFMVEVSIE1BVEVSSUFMSVpFRCBWSUVXCiAqIC0gYWx0ZXIoKS0+c2VxdWVuY2UoKSAtIEFMVEVSIFNFUVVFTkNFCiAqIC0gYWx0ZXIoKS0+c2NoZW1hKCkgLSBBTFRFUiBTQ0hFTUEKICogLSBhbHRlcigpLT5yb2xlKCkgLSBBTFRFUiBST0xFCiAqIC0gYWx0ZXIoKS0+ZnVuY3Rpb24oKSAtIEFMVEVSIEZVTkNUSU9OCiAqIC0gYWx0ZXIoKS0+cHJvY2VkdXJlKCkgLSBBTFRFUiBQUk9DRURVUkUKICogLSBhbHRlcigpLT50cmlnZ2VyKCkgLSBBTFRFUiBUUklHR0VSCiAqIC0gYWx0ZXIoKS0+ZXh0ZW5zaW9uKCkgLSBBTFRFUiBFWFRFTlNJT04KICogLSBhbHRlcigpLT5lbnVtVHlwZSgpIC0gQUxURVIgVFlQRSAoZW51bSkKICogLSBhbHRlcigpLT5kb21haW4oKSAtIEFMVEVSIERPTUFJTgogKgogKiBSZW5hbWUgb3BlcmF0aW9ucyBhcmUgYWxzbyB1bmRlciBhbHRlcigpOgogKiAtIGFsdGVyKCktPmluZGV4KCdvbGQnKS0+cmVuYW1lVG8oJ25ldycpCiAqIC0gYWx0ZXIoKS0+dmlldygnb2xkJyktPnJlbmFtZVRvKCduZXcnKQogKiAtIGFsdGVyKCktPnNjaGVtYSgnb2xkJyktPnJlbmFtZVRvKCduZXcnKQogKiAtIGFsdGVyKCktPnJvbGUoJ29sZCcpLT5yZW5hbWVUbygnbmV3JykKICogLSBhbHRlcigpLT50cmlnZ2VyKCdvbGQnKS0+b24oJ3RhYmxlJyktPnJlbmFtZVRvKCduZXcnKQogKgogKiBFeGFtcGxlOiBhbHRlcigpLT50YWJsZSgndXNlcnMnKS0+YWRkQ29sdW1uKGNvbF9kZWYoJ2VtYWlsJywgY29sdW1uX3R5cGVfdGV4dCgpKSkKICogRXhhbXBsZTogYWx0ZXIoKS0+c2VxdWVuY2UoJ3VzZXJfaWRfc2VxJyktPnJlc3RhcnQoMTAwMCkKICov"},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/schema.php","start_line_in_file":292,"slug":"truncate-table","name":"truncate_table","namespace":"Flow\\PostgreSql\\DSL","parameters":[{"name":"tables","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":true,"default_value":null}],"return_type":[{"name":"TruncateFinalStep","namespace":"Flow\\PostgreSql\\QueryBuilder\\Schema\\Truncate","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBhIFRSVU5DQVRFIFRBQkxFIGJ1aWxkZXIuCiAqCiAqIEBwYXJhbSBzdHJpbmcgLi4uJHRhYmxlcyBUYWJsZSBuYW1lcyB0byB0cnVuY2F0ZQogKi8="},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/schema.php","start_line_in_file":310,"slug":"refresh-materialized-view","name":"refresh_materialized_view","namespace":"Flow\\PostgreSql\\DSL","parameters":[{"name":"name","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"schema","type":[{"name":"string","namespace":null,"is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"null"}],"return_type":[{"name":"RefreshMatViewOptionsStep","namespace":"Flow\\PostgreSql\\QueryBuilder\\Schema\\View\\RefreshMaterializedView","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"SCHEMA"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBhIFJFRlJFU0ggTUFURVJJQUxJWkVEIFZJRVcgYnVpbGRlci4KICoKICogRXhhbXBsZTogcmVmcmVzaF9tYXRlcmlhbGl6ZWRfdmlldygndXNlcl9zdGF0cycpCiAqIFByb2R1Y2VzOiBSRUZSRVNIIE1BVEVSSUFMSVpFRCBWSUVXIHVzZXJfc3RhdHMKICoKICogRXhhbXBsZTogcmVmcmVzaF9tYXRlcmlhbGl6ZWRfdmlldygndXNlcl9zdGF0cycpLT5jb25jdXJyZW50bHkoKS0+d2l0aERhdGEoKQogKiBQcm9kdWNlczogUkVGUkVTSCBNQVRFUklBTElaRUQgVklFVyBDT05DVVJSRU5UTFkgdXNlcl9zdGF0cyBXSVRIIERBVEEKICoKICogQHBhcmFtIHN0cmluZyAkbmFtZSBWaWV3IG5hbWUgKG1heSBpbmNsdWRlIHNjaGVtYSBhcyAic2NoZW1hLnZpZXciKQogKiBAcGFyYW0gbnVsbHxzdHJpbmcgJHNjaGVtYSBTY2hlbWEgbmFtZSAob3B0aW9uYWwsIG92ZXJyaWRlcyBwYXJzZWQgc2NoZW1hKQogKi8="},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/schema.php","start_line_in_file":319,"slug":"ref-action-cascade","name":"ref_action_cascade","namespace":"Flow\\PostgreSql\\DSL","parameters":[],"return_type":[{"name":"ReferentialAction","namespace":"Flow\\PostgreSql\\QueryBuilder\\Schema","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIEdldCBhIENBU0NBREUgcmVmZXJlbnRpYWwgYWN0aW9uLgogKi8="},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/schema.php","start_line_in_file":328,"slug":"ref-action-restrict","name":"ref_action_restrict","namespace":"Flow\\PostgreSql\\DSL","parameters":[],"return_type":[{"name":"ReferentialAction","namespace":"Flow\\PostgreSql\\QueryBuilder\\Schema","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIEdldCBhIFJFU1RSSUNUIHJlZmVyZW50aWFsIGFjdGlvbi4KICov"},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/schema.php","start_line_in_file":337,"slug":"ref-action-set-null","name":"ref_action_set_null","namespace":"Flow\\PostgreSql\\DSL","parameters":[],"return_type":[{"name":"ReferentialAction","namespace":"Flow\\PostgreSql\\QueryBuilder\\Schema","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIEdldCBhIFNFVCBOVUxMIHJlZmVyZW50aWFsIGFjdGlvbi4KICov"},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/schema.php","start_line_in_file":346,"slug":"ref-action-set-default","name":"ref_action_set_default","namespace":"Flow\\PostgreSql\\DSL","parameters":[],"return_type":[{"name":"ReferentialAction","namespace":"Flow\\PostgreSql\\QueryBuilder\\Schema","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIEdldCBhIFNFVCBERUZBVUxUIHJlZmVyZW50aWFsIGFjdGlvbi4KICov"},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/schema.php","start_line_in_file":355,"slug":"ref-action-no-action","name":"ref_action_no_action","namespace":"Flow\\PostgreSql\\DSL","parameters":[],"return_type":[{"name":"ReferentialAction","namespace":"Flow\\PostgreSql\\QueryBuilder\\Schema","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIEdldCBhIE5PIEFDVElPTiByZWZlcmVudGlhbCBhY3Rpb24uCiAqLw=="},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/schema.php","start_line_in_file":370,"slug":"reindex-index","name":"reindex_index","namespace":"Flow\\PostgreSql\\DSL","parameters":[{"name":"name","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"ReindexFinalStep","namespace":"Flow\\PostgreSql\\QueryBuilder\\Schema\\Index\\Reindex","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"SCHEMA"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIFN0YXJ0IGJ1aWxkaW5nIGEgUkVJTkRFWCBJTkRFWCBzdGF0ZW1lbnQuCiAqCiAqIFVzZSBjaGFpbmFibGUgbWV0aG9kczogLT5jb25jdXJyZW50bHkoKSwgLT52ZXJib3NlKCksIC0+dGFibGVzcGFjZSgpCiAqCiAqIEV4YW1wbGU6IHJlaW5kZXhfaW5kZXgoJ2lkeF91c2Vyc19lbWFpbCcpLT5jb25jdXJyZW50bHkoKQogKgogKiBAcGFyYW0gc3RyaW5nICRuYW1lIFRoZSBpbmRleCBuYW1lIChtYXkgaW5jbHVkZSBzY2hlbWE6IHNjaGVtYS5pbmRleCkKICov"},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/schema.php","start_line_in_file":385,"slug":"reindex-table","name":"reindex_table","namespace":"Flow\\PostgreSql\\DSL","parameters":[{"name":"name","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"ReindexFinalStep","namespace":"Flow\\PostgreSql\\QueryBuilder\\Schema\\Index\\Reindex","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"SCHEMA"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIFN0YXJ0IGJ1aWxkaW5nIGEgUkVJTkRFWCBUQUJMRSBzdGF0ZW1lbnQuCiAqCiAqIFVzZSBjaGFpbmFibGUgbWV0aG9kczogLT5jb25jdXJyZW50bHkoKSwgLT52ZXJib3NlKCksIC0+dGFibGVzcGFjZSgpCiAqCiAqIEV4YW1wbGU6IHJlaW5kZXhfdGFibGUoJ3VzZXJzJyktPmNvbmN1cnJlbnRseSgpCiAqCiAqIEBwYXJhbSBzdHJpbmcgJG5hbWUgVGhlIHRhYmxlIG5hbWUgKG1heSBpbmNsdWRlIHNjaGVtYTogc2NoZW1hLnRhYmxlKQogKi8="},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/schema.php","start_line_in_file":400,"slug":"reindex-schema","name":"reindex_schema","namespace":"Flow\\PostgreSql\\DSL","parameters":[{"name":"name","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"ReindexFinalStep","namespace":"Flow\\PostgreSql\\QueryBuilder\\Schema\\Index\\Reindex","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"SCHEMA"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIFN0YXJ0IGJ1aWxkaW5nIGEgUkVJTkRFWCBTQ0hFTUEgc3RhdGVtZW50LgogKgogKiBVc2UgY2hhaW5hYmxlIG1ldGhvZHM6IC0+Y29uY3VycmVudGx5KCksIC0+dmVyYm9zZSgpLCAtPnRhYmxlc3BhY2UoKQogKgogKiBFeGFtcGxlOiByZWluZGV4X3NjaGVtYSgncHVibGljJyktPmNvbmN1cnJlbnRseSgpCiAqCiAqIEBwYXJhbSBzdHJpbmcgJG5hbWUgVGhlIHNjaGVtYSBuYW1lCiAqLw=="},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/schema.php","start_line_in_file":415,"slug":"reindex-database","name":"reindex_database","namespace":"Flow\\PostgreSql\\DSL","parameters":[{"name":"name","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"ReindexFinalStep","namespace":"Flow\\PostgreSql\\QueryBuilder\\Schema\\Index\\Reindex","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"SCHEMA"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIFN0YXJ0IGJ1aWxkaW5nIGEgUkVJTkRFWCBEQVRBQkFTRSBzdGF0ZW1lbnQuCiAqCiAqIFVzZSBjaGFpbmFibGUgbWV0aG9kczogLT5jb25jdXJyZW50bHkoKSwgLT52ZXJib3NlKCksIC0+dGFibGVzcGFjZSgpCiAqCiAqIEV4YW1wbGU6IHJlaW5kZXhfZGF0YWJhc2UoJ215ZGInKS0+Y29uY3VycmVudGx5KCkKICoKICogQHBhcmFtIHN0cmluZyAkbmFtZSBUaGUgZGF0YWJhc2UgbmFtZQogKi8="},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/schema.php","start_line_in_file":430,"slug":"index-col","name":"index_col","namespace":"Flow\\PostgreSql\\DSL","parameters":[{"name":"name","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"IndexColumn","namespace":"Flow\\PostgreSql\\QueryBuilder\\Schema\\Index","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"SCHEMA"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBhbiBpbmRleCBjb2x1bW4gc3BlY2lmaWNhdGlvbi4KICoKICogVXNlIGNoYWluYWJsZSBtZXRob2RzOiAtPmFzYygpLCAtPmRlc2MoKSwgLT5udWxsc0ZpcnN0KCksIC0+bnVsbHNMYXN0KCksIC0+b3BjbGFzcygpLCAtPmNvbGxhdGUoKQogKgogKiBFeGFtcGxlOiBpbmRleF9jb2woJ2VtYWlsJyktPmRlc2MoKS0+bnVsbHNMYXN0KCkKICoKICogQHBhcmFtIHN0cmluZyAkbmFtZSBUaGUgY29sdW1uIG5hbWUKICov"},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/schema.php","start_line_in_file":445,"slug":"index-expr","name":"index_expr","namespace":"Flow\\PostgreSql\\DSL","parameters":[{"name":"expression","type":[{"name":"Expression","namespace":"Flow\\PostgreSql\\QueryBuilder\\Expression","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"IndexColumn","namespace":"Flow\\PostgreSql\\QueryBuilder\\Schema\\Index","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"SCHEMA"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBhbiBpbmRleCBjb2x1bW4gc3BlY2lmaWNhdGlvbiBmcm9tIGFuIGV4cHJlc3Npb24uCiAqCiAqIFVzZSBjaGFpbmFibGUgbWV0aG9kczogLT5hc2MoKSwgLT5kZXNjKCksIC0+bnVsbHNGaXJzdCgpLCAtPm51bGxzTGFzdCgpLCAtPm9wY2xhc3MoKSwgLT5jb2xsYXRlKCkKICoKICogRXhhbXBsZTogaW5kZXhfZXhwcihmbl9jYWxsKCdsb3dlcicsIGNvbCgnZW1haWwnKSkpLT5kZXNjKCkKICoKICogQHBhcmFtIEV4cHJlc3Npb24gJGV4cHJlc3Npb24gVGhlIGV4cHJlc3Npb24gdG8gaW5kZXgKICov"},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/schema.php","start_line_in_file":454,"slug":"index-method-btree","name":"index_method_btree","namespace":"Flow\\PostgreSql\\DSL","parameters":[],"return_type":[{"name":"IndexMethod","namespace":"Flow\\PostgreSql\\QueryBuilder\\Schema\\Index","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIEdldCB0aGUgQlRSRUUgaW5kZXggbWV0aG9kLgogKi8="},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/schema.php","start_line_in_file":463,"slug":"index-method-hash","name":"index_method_hash","namespace":"Flow\\PostgreSql\\DSL","parameters":[],"return_type":[{"name":"IndexMethod","namespace":"Flow\\PostgreSql\\QueryBuilder\\Schema\\Index","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIEdldCB0aGUgSEFTSCBpbmRleCBtZXRob2QuCiAqLw=="},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/schema.php","start_line_in_file":472,"slug":"index-method-gist","name":"index_method_gist","namespace":"Flow\\PostgreSql\\DSL","parameters":[],"return_type":[{"name":"IndexMethod","namespace":"Flow\\PostgreSql\\QueryBuilder\\Schema\\Index","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIEdldCB0aGUgR0lTVCBpbmRleCBtZXRob2QuCiAqLw=="},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/schema.php","start_line_in_file":481,"slug":"index-method-spgist","name":"index_method_spgist","namespace":"Flow\\PostgreSql\\DSL","parameters":[],"return_type":[{"name":"IndexMethod","namespace":"Flow\\PostgreSql\\QueryBuilder\\Schema\\Index","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIEdldCB0aGUgU1BHSVNUIGluZGV4IG1ldGhvZC4KICov"},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/schema.php","start_line_in_file":490,"slug":"index-method-gin","name":"index_method_gin","namespace":"Flow\\PostgreSql\\DSL","parameters":[],"return_type":[{"name":"IndexMethod","namespace":"Flow\\PostgreSql\\QueryBuilder\\Schema\\Index","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIEdldCB0aGUgR0lOIGluZGV4IG1ldGhvZC4KICov"},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/schema.php","start_line_in_file":499,"slug":"index-method-brin","name":"index_method_brin","namespace":"Flow\\PostgreSql\\DSL","parameters":[],"return_type":[{"name":"IndexMethod","namespace":"Flow\\PostgreSql\\QueryBuilder\\Schema\\Index","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIEdldCB0aGUgQlJJTiBpbmRleCBtZXRob2QuCiAqLw=="},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/schema.php","start_line_in_file":511,"slug":"vacuum","name":"vacuum","namespace":"Flow\\PostgreSql\\DSL","parameters":[],"return_type":[{"name":"VacuumFinalStep","namespace":"Flow\\PostgreSql\\QueryBuilder\\Utility","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBhIFZBQ1VVTSBidWlsZGVyLgogKgogKiBFeGFtcGxlOiB2YWN1dW0oKS0+dGFibGUoJ3VzZXJzJykKICogUHJvZHVjZXM6IFZBQ1VVTSB1c2VycwogKi8="},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/schema.php","start_line_in_file":523,"slug":"analyze","name":"analyze","namespace":"Flow\\PostgreSql\\DSL","parameters":[],"return_type":[{"name":"AnalyzeFinalStep","namespace":"Flow\\PostgreSql\\QueryBuilder\\Utility","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBhbiBBTkFMWVpFIGJ1aWxkZXIuCiAqCiAqIEV4YW1wbGU6IGFuYWx5emUoKS0+dGFibGUoJ3VzZXJzJykKICogUHJvZHVjZXM6IEFOQUxZWkUgdXNlcnMKICov"},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/schema.php","start_line_in_file":537,"slug":"explain","name":"explain","namespace":"Flow\\PostgreSql\\DSL","parameters":[{"name":"query","type":[{"name":"SelectFinalStep","namespace":"Flow\\PostgreSql\\QueryBuilder\\Select","is_nullable":false,"is_variadic":false},{"name":"InsertBuilder","namespace":"Flow\\PostgreSql\\QueryBuilder\\Insert","is_nullable":false,"is_variadic":false},{"name":"UpdateBuilder","namespace":"Flow\\PostgreSql\\QueryBuilder\\Update","is_nullable":false,"is_variadic":false},{"name":"DeleteBuilder","namespace":"Flow\\PostgreSql\\QueryBuilder\\Delete","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"ExplainFinalStep","namespace":"Flow\\PostgreSql\\QueryBuilder\\Utility","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBhbiBFWFBMQUlOIGJ1aWxkZXIgZm9yIGEgcXVlcnkuCiAqCiAqIEV4YW1wbGU6IGV4cGxhaW4oc2VsZWN0KCktPmZyb20oJ3VzZXJzJykpCiAqIFByb2R1Y2VzOiBFWFBMQUlOIFNFTEVDVCAqIEZST00gdXNlcnMKICoKICogQHBhcmFtIERlbGV0ZUJ1aWxkZXJ8SW5zZXJ0QnVpbGRlcnxTZWxlY3RGaW5hbFN0ZXB8VXBkYXRlQnVpbGRlciAkcXVlcnkgUXVlcnkgdG8gZXhwbGFpbgogKi8="},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/schema.php","start_line_in_file":549,"slug":"lock-table","name":"lock_table","namespace":"Flow\\PostgreSql\\DSL","parameters":[{"name":"tables","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":true,"default_value":null}],"return_type":[{"name":"LockFinalStep","namespace":"Flow\\PostgreSql\\QueryBuilder\\Utility","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBhIExPQ0sgVEFCTEUgYnVpbGRlci4KICoKICogRXhhbXBsZTogbG9ja190YWJsZSgndXNlcnMnLCAnb3JkZXJzJyktPmFjY2Vzc0V4Y2x1c2l2ZSgpCiAqIFByb2R1Y2VzOiBMT0NLIFRBQkxFIHVzZXJzLCBvcmRlcnMgSU4gQUNDRVNTIEVYQ0xVU0lWRSBNT0RFCiAqLw=="},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/schema.php","start_line_in_file":564,"slug":"comment","name":"comment","namespace":"Flow\\PostgreSql\\DSL","parameters":[{"name":"target","type":[{"name":"CommentTarget","namespace":"Flow\\PostgreSql\\QueryBuilder\\Utility","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"name","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"CommentFinalStep","namespace":"Flow\\PostgreSql\\QueryBuilder\\Utility","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBhIENPTU1FTlQgT04gYnVpbGRlci4KICoKICogRXhhbXBsZTogY29tbWVudChDb21tZW50VGFyZ2V0OjpUQUJMRSwgJ3VzZXJzJyktPmlzKCdVc2VyIGFjY291bnRzIHRhYmxlJykKICogUHJvZHVjZXM6IENPTU1FTlQgT04gVEFCTEUgdXNlcnMgSVMgJ1VzZXIgYWNjb3VudHMgdGFibGUnCiAqCiAqIEBwYXJhbSBDb21tZW50VGFyZ2V0ICR0YXJnZXQgVGFyZ2V0IHR5cGUgKFRBQkxFLCBDT0xVTU4sIElOREVYLCBldGMuKQogKiBAcGFyYW0gc3RyaW5nICRuYW1lIFRhcmdldCBuYW1lICh1c2UgJ3RhYmxlLmNvbHVtbicgZm9yIENPTFVNTiB0YXJnZXRzKQogKi8="},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/schema.php","start_line_in_file":576,"slug":"cluster","name":"cluster","namespace":"Flow\\PostgreSql\\DSL","parameters":[],"return_type":[{"name":"ClusterFinalStep","namespace":"Flow\\PostgreSql\\QueryBuilder\\Utility","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBhIENMVVNURVIgYnVpbGRlci4KICoKICogRXhhbXBsZTogY2x1c3RlcigpLT50YWJsZSgndXNlcnMnKS0+dXNpbmcoJ2lkeF91c2Vyc19wa2V5JykKICogUHJvZHVjZXM6IENMVVNURVIgdXNlcnMgVVNJTkcgaWR4X3VzZXJzX3BrZXkKICov"},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/schema.php","start_line_in_file":590,"slug":"discard","name":"discard","namespace":"Flow\\PostgreSql\\DSL","parameters":[{"name":"type","type":[{"name":"DiscardType","namespace":"Flow\\PostgreSql\\QueryBuilder\\Utility","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"DiscardFinalStep","namespace":"Flow\\PostgreSql\\QueryBuilder\\Utility","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBhIERJU0NBUkQgYnVpbGRlci4KICoKICogRXhhbXBsZTogZGlzY2FyZChEaXNjYXJkVHlwZTo6QUxMKQogKiBQcm9kdWNlczogRElTQ0FSRCBBTEwKICoKICogQHBhcmFtIERpc2NhcmRUeXBlICR0eXBlIFR5cGUgb2YgcmVzb3VyY2VzIHRvIGRpc2NhcmQgKEFMTCwgUExBTlMsIFNFUVVFTkNFUywgVEVNUCkKICov"},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/schema.php","start_line_in_file":609,"slug":"grant","name":"grant","namespace":"Flow\\PostgreSql\\DSL","parameters":[{"name":"privileges","type":[{"name":"TablePrivilege","namespace":"Flow\\PostgreSql\\QueryBuilder\\Schema\\Grant","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":true,"default_value":null}],"return_type":[{"name":"GrantOnStep","namespace":"Flow\\PostgreSql\\QueryBuilder\\Schema\\Grant","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBhIEdSQU5UIHByaXZpbGVnZXMgYnVpbGRlci4KICoKICogRXhhbXBsZTogZ3JhbnQoVGFibGVQcml2aWxlZ2U6OlNFTEVDVCktPm9uVGFibGUoJ3VzZXJzJyktPnRvKCdhcHBfdXNlcicpCiAqIFByb2R1Y2VzOiBHUkFOVCBTRUxFQ1QgT04gdXNlcnMgVE8gYXBwX3VzZXIKICoKICogRXhhbXBsZTogZ3JhbnQoVGFibGVQcml2aWxlZ2U6OkFMTCktPm9uQWxsVGFibGVzSW5TY2hlbWEoJ3B1YmxpYycpLT50bygnYWRtaW4nKQogKiBQcm9kdWNlczogR1JBTlQgQUxMIE9OIEFMTCBUQUJMRVMgSU4gU0NIRU1BIHB1YmxpYyBUTyBhZG1pbgogKgogKiBAcGFyYW0gc3RyaW5nfFRhYmxlUHJpdmlsZWdlIC4uLiRwcml2aWxlZ2VzIFRoZSBwcml2aWxlZ2VzIHRvIGdyYW50CiAqCiAqIEByZXR1cm4gR3JhbnRPblN0ZXAgQnVpbGRlciBmb3IgZ3JhbnQgb3B0aW9ucwogKi8="},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/schema.php","start_line_in_file":628,"slug":"grant-role","name":"grant_role","namespace":"Flow\\PostgreSql\\DSL","parameters":[{"name":"roles","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":true,"default_value":null}],"return_type":[{"name":"GrantRoleToStep","namespace":"Flow\\PostgreSql\\QueryBuilder\\Schema\\Grant","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBhIEdSQU5UIHJvbGUgYnVpbGRlci4KICoKICogRXhhbXBsZTogZ3JhbnRfcm9sZSgnYWRtaW4nKS0+dG8oJ3VzZXIxJykKICogUHJvZHVjZXM6IEdSQU5UIGFkbWluIFRPIHVzZXIxCiAqCiAqIEV4YW1wbGU6IGdyYW50X3JvbGUoJ2FkbWluJywgJ2RldmVsb3BlcicpLT50bygndXNlcjEnKS0+d2l0aEFkbWluT3B0aW9uKCkKICogUHJvZHVjZXM6IEdSQU5UIGFkbWluLCBkZXZlbG9wZXIgVE8gdXNlcjEgV0lUSCBBRE1JTiBPUFRJT04KICoKICogQHBhcmFtIHN0cmluZyAuLi4kcm9sZXMgVGhlIHJvbGVzIHRvIGdyYW50CiAqCiAqIEByZXR1cm4gR3JhbnRSb2xlVG9TdGVwIEJ1aWxkZXIgZm9yIGdyYW50IHJvbGUgb3B0aW9ucwogKi8="},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/schema.php","start_line_in_file":647,"slug":"revoke","name":"revoke","namespace":"Flow\\PostgreSql\\DSL","parameters":[{"name":"privileges","type":[{"name":"TablePrivilege","namespace":"Flow\\PostgreSql\\QueryBuilder\\Schema\\Grant","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":true,"default_value":null}],"return_type":[{"name":"RevokeOnStep","namespace":"Flow\\PostgreSql\\QueryBuilder\\Schema\\Grant","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBhIFJFVk9LRSBwcml2aWxlZ2VzIGJ1aWxkZXIuCiAqCiAqIEV4YW1wbGU6IHJldm9rZShUYWJsZVByaXZpbGVnZTo6U0VMRUNUKS0+b25UYWJsZSgndXNlcnMnKS0+ZnJvbSgnYXBwX3VzZXInKQogKiBQcm9kdWNlczogUkVWT0tFIFNFTEVDVCBPTiB1c2VycyBGUk9NIGFwcF91c2VyCiAqCiAqIEV4YW1wbGU6IHJldm9rZShUYWJsZVByaXZpbGVnZTo6QUxMKS0+b25UYWJsZSgndXNlcnMnKS0+ZnJvbSgnYXBwX3VzZXInKS0+Y2FzY2FkZSgpCiAqIFByb2R1Y2VzOiBSRVZPS0UgQUxMIE9OIHVzZXJzIEZST00gYXBwX3VzZXIgQ0FTQ0FERQogKgogKiBAcGFyYW0gc3RyaW5nfFRhYmxlUHJpdmlsZWdlIC4uLiRwcml2aWxlZ2VzIFRoZSBwcml2aWxlZ2VzIHRvIHJldm9rZQogKgogKiBAcmV0dXJuIFJldm9rZU9uU3RlcCBCdWlsZGVyIGZvciByZXZva2Ugb3B0aW9ucwogKi8="},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/schema.php","start_line_in_file":666,"slug":"revoke-role","name":"revoke_role","namespace":"Flow\\PostgreSql\\DSL","parameters":[{"name":"roles","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":true,"default_value":null}],"return_type":[{"name":"RevokeRoleFromStep","namespace":"Flow\\PostgreSql\\QueryBuilder\\Schema\\Grant","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBhIFJFVk9LRSByb2xlIGJ1aWxkZXIuCiAqCiAqIEV4YW1wbGU6IHJldm9rZV9yb2xlKCdhZG1pbicpLT5mcm9tKCd1c2VyMScpCiAqIFByb2R1Y2VzOiBSRVZPS0UgYWRtaW4gRlJPTSB1c2VyMQogKgogKiBFeGFtcGxlOiByZXZva2Vfcm9sZSgnYWRtaW4nKS0+ZnJvbSgndXNlcjEnKS0+Y2FzY2FkZSgpCiAqIFByb2R1Y2VzOiBSRVZPS0UgYWRtaW4gRlJPTSB1c2VyMSBDQVNDQURFCiAqCiAqIEBwYXJhbSBzdHJpbmcgLi4uJHJvbGVzIFRoZSByb2xlcyB0byByZXZva2UKICoKICogQHJldHVybiBSZXZva2VSb2xlRnJvbVN0ZXAgQnVpbGRlciBmb3IgcmV2b2tlIHJvbGUgb3B0aW9ucwogKi8="},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/schema.php","start_line_in_file":682,"slug":"set-role","name":"set_role","namespace":"Flow\\PostgreSql\\DSL","parameters":[{"name":"role","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"SetRoleFinalStep","namespace":"Flow\\PostgreSql\\QueryBuilder\\Schema\\Session","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBhIFNFVCBST0xFIGJ1aWxkZXIuCiAqCiAqIEV4YW1wbGU6IHNldF9yb2xlKCdhZG1pbicpCiAqIFByb2R1Y2VzOiBTRVQgUk9MRSBhZG1pbgogKgogKiBAcGFyYW0gc3RyaW5nICRyb2xlIFRoZSByb2xlIHRvIHNldAogKgogKiBAcmV0dXJuIFNldFJvbGVGaW5hbFN0ZXAgQnVpbGRlciBmb3Igc2V0IHJvbGUKICov"},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/schema.php","start_line_in_file":696,"slug":"reset-role","name":"reset_role","namespace":"Flow\\PostgreSql\\DSL","parameters":[],"return_type":[{"name":"ResetRoleFinalStep","namespace":"Flow\\PostgreSql\\QueryBuilder\\Schema\\Session","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBhIFJFU0VUIFJPTEUgYnVpbGRlci4KICoKICogRXhhbXBsZTogcmVzZXRfcm9sZSgpCiAqIFByb2R1Y2VzOiBSRVNFVCBST0xFCiAqCiAqIEByZXR1cm4gUmVzZXRSb2xlRmluYWxTdGVwIEJ1aWxkZXIgZm9yIHJlc2V0IHJvbGUKICov"},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/schema.php","start_line_in_file":712,"slug":"reassign-owned","name":"reassign_owned","namespace":"Flow\\PostgreSql\\DSL","parameters":[{"name":"roles","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":true,"default_value":null}],"return_type":[{"name":"ReassignOwnedToStep","namespace":"Flow\\PostgreSql\\QueryBuilder\\Schema\\Ownership","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBhIFJFQVNTSUdOIE9XTkVEIGJ1aWxkZXIuCiAqCiAqIEV4YW1wbGU6IHJlYXNzaWduX293bmVkKCdvbGRfcm9sZScpLT50bygnbmV3X3JvbGUnKQogKiBQcm9kdWNlczogUkVBU1NJR04gT1dORUQgQlkgb2xkX3JvbGUgVE8gbmV3X3JvbGUKICoKICogQHBhcmFtIHN0cmluZyAuLi4kcm9sZXMgVGhlIHJvbGVzIHdob3NlIG93bmVkIG9iamVjdHMgc2hvdWxkIGJlIHJlYXNzaWduZWQKICoKICogQHJldHVybiBSZWFzc2lnbk93bmVkVG9TdGVwIEJ1aWxkZXIgZm9yIHJlYXNzaWduIG93bmVkIG9wdGlvbnMKICov"},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/schema.php","start_line_in_file":731,"slug":"drop-owned","name":"drop_owned","namespace":"Flow\\PostgreSql\\DSL","parameters":[{"name":"roles","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":true,"default_value":null}],"return_type":[{"name":"DropOwnedFinalStep","namespace":"Flow\\PostgreSql\\QueryBuilder\\Schema\\Ownership","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBhIERST1AgT1dORUQgYnVpbGRlci4KICoKICogRXhhbXBsZTogZHJvcF9vd25lZCgncm9sZTEnKQogKiBQcm9kdWNlczogRFJPUCBPV05FRCBCWSByb2xlMQogKgogKiBFeGFtcGxlOiBkcm9wX293bmVkKCdyb2xlMScsICdyb2xlMicpLT5jYXNjYWRlKCkKICogUHJvZHVjZXM6IERST1AgT1dORUQgQlkgcm9sZTEsIHJvbGUyIENBU0NBREUKICoKICogQHBhcmFtIHN0cmluZyAuLi4kcm9sZXMgVGhlIHJvbGVzIHdob3NlIG93bmVkIG9iamVjdHMgc2hvdWxkIGJlIGRyb3BwZWQKICoKICogQHJldHVybiBEcm9wT3duZWRGaW5hbFN0ZXAgQnVpbGRlciBmb3IgZHJvcCBvd25lZCBvcHRpb25zCiAqLw=="},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/schema.php","start_line_in_file":749,"slug":"func-arg","name":"func_arg","namespace":"Flow\\PostgreSql\\DSL","parameters":[{"name":"type","type":[{"name":"ColumnType","namespace":"Flow\\PostgreSql\\QueryBuilder\\Schema","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"FunctionArgument","namespace":"Flow\\PostgreSql\\QueryBuilder\\Schema\\Function","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZXMgYSBuZXcgZnVuY3Rpb24gYXJndW1lbnQgZm9yIHVzZSBpbiBmdW5jdGlvbi9wcm9jZWR1cmUgZGVmaW5pdGlvbnMuCiAqCiAqIEV4YW1wbGU6IGZ1bmNfYXJnKGNvbHVtbl90eXBlX2ludGVnZXIoKSkKICogRXhhbXBsZTogZnVuY19hcmcoY29sdW1uX3R5cGVfdGV4dCgpKS0+bmFtZWQoJ3VzZXJuYW1lJykKICogRXhhbXBsZTogZnVuY19hcmcoY29sdW1uX3R5cGVfaW50ZWdlcigpKS0+bmFtZWQoJ2NvdW50JyktPmRlZmF1bHQoJzAnKQogKiBFeGFtcGxlOiBmdW5jX2FyZyhjb2x1bW5fdHlwZV90ZXh0KCkpLT5vdXQoKQogKgogKiBAcGFyYW0gQ29sdW1uVHlwZSAkdHlwZSBUaGUgUG9zdGdyZVNRTCBkYXRhIHR5cGUgZm9yIHRoZSBhcmd1bWVudAogKgogKiBAcmV0dXJuIEZ1bmN0aW9uQXJndW1lbnQgQnVpbGRlciBmb3IgZnVuY3Rpb24gYXJndW1lbnQgb3B0aW9ucwogKi8="},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/schema.php","start_line_in_file":768,"slug":"call","name":"call","namespace":"Flow\\PostgreSql\\DSL","parameters":[{"name":"procedure","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"CallFinalStep","namespace":"Flow\\PostgreSql\\QueryBuilder\\Schema\\Function","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZXMgYSBDQUxMIHN0YXRlbWVudCBidWlsZGVyIGZvciBpbnZva2luZyBhIHByb2NlZHVyZS4KICoKICogRXhhbXBsZTogY2FsbCgndXBkYXRlX3N0YXRzJyktPndpdGgoMTIzKQogKiBQcm9kdWNlczogQ0FMTCB1cGRhdGVfc3RhdHMoMTIzKQogKgogKiBFeGFtcGxlOiBjYWxsKCdwcm9jZXNzX2RhdGEnKS0+d2l0aCgndGVzdCcsIDQyLCB0cnVlKQogKiBQcm9kdWNlczogQ0FMTCBwcm9jZXNzX2RhdGEoJ3Rlc3QnLCA0MiwgdHJ1ZSkKICoKICogQHBhcmFtIHN0cmluZyAkcHJvY2VkdXJlIFRoZSBuYW1lIG9mIHRoZSBwcm9jZWR1cmUgdG8gY2FsbAogKgogKiBAcmV0dXJuIENhbGxGaW5hbFN0ZXAgQnVpbGRlciBmb3IgY2FsbCBzdGF0ZW1lbnQgb3B0aW9ucwogKi8="},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/schema.php","start_line_in_file":787,"slug":"do-block","name":"do_block","namespace":"Flow\\PostgreSql\\DSL","parameters":[{"name":"code","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"DoFinalStep","namespace":"Flow\\PostgreSql\\QueryBuilder\\Schema\\Function","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZXMgYSBETyBzdGF0ZW1lbnQgYnVpbGRlciBmb3IgZXhlY3V0aW5nIGFuIGFub255bW91cyBjb2RlIGJsb2NrLgogKgogKiBFeGFtcGxlOiBkb19ibG9jaygnQkVHSU4gUkFJU0UgTk9USUNFICQkSGVsbG8gV29ybGQkJDsgRU5EOycpCiAqIFByb2R1Y2VzOiBETyAkJCBCRUdJTiBSQUlTRSBOT1RJQ0UgJCRIZWxsbyBXb3JsZCQkOyBFTkQ7ICQkIExBTkdVQUdFIHBscGdzcWwKICoKICogRXhhbXBsZTogZG9fYmxvY2soJ1NFTEVDVCAxJyktPmxhbmd1YWdlKCdzcWwnKQogKiBQcm9kdWNlczogRE8gJCQgU0VMRUNUIDEgJCQgTEFOR1VBR0Ugc3FsCiAqCiAqIEBwYXJhbSBzdHJpbmcgJGNvZGUgVGhlIGFub255bW91cyBjb2RlIGJsb2NrIHRvIGV4ZWN1dGUKICoKICogQHJldHVybiBEb0ZpbmFsU3RlcCBCdWlsZGVyIGZvciBETyBzdGF0ZW1lbnQgb3B0aW9ucwogKi8="},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/schema.php","start_line_in_file":807,"slug":"type-attr","name":"type_attr","namespace":"Flow\\PostgreSql\\DSL","parameters":[{"name":"name","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"type","type":[{"name":"ColumnType","namespace":"Flow\\PostgreSql\\QueryBuilder\\Schema","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"TypeAttribute","namespace":"Flow\\PostgreSql\\QueryBuilder\\Schema\\Type","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZXMgYSB0eXBlIGF0dHJpYnV0ZSBmb3IgY29tcG9zaXRlIHR5cGVzLgogKgogKiBFeGFtcGxlOiB0eXBlX2F0dHIoJ25hbWUnLCBjb2x1bW5fdHlwZV90ZXh0KCkpCiAqIFByb2R1Y2VzOiBuYW1lIHRleHQKICoKICogRXhhbXBsZTogdHlwZV9hdHRyKCdkZXNjcmlwdGlvbicsIGNvbHVtbl90eXBlX3RleHQoKSktPmNvbGxhdGUoJ2VuX1VTJykKICogUHJvZHVjZXM6IGRlc2NyaXB0aW9uIHRleHQgQ09MTEFURSAiZW5fVVMiCiAqCiAqIEBwYXJhbSBzdHJpbmcgJG5hbWUgVGhlIGF0dHJpYnV0ZSBuYW1lCiAqIEBwYXJhbSBDb2x1bW5UeXBlICR0eXBlIFRoZSBhdHRyaWJ1dGUgdHlwZQogKgogKiBAcmV0dXJuIFR5cGVBdHRyaWJ1dGUgVHlwZSBhdHRyaWJ1dGUgdmFsdWUgb2JqZWN0CiAqLw=="},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/schema.php","start_line_in_file":816,"slug":"column-type-integer","name":"column_type_integer","namespace":"Flow\\PostgreSql\\DSL","parameters":[],"return_type":[{"name":"ColumnType","namespace":"Flow\\PostgreSql\\QueryBuilder\\Schema","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBhbiBpbnRlZ2VyIGRhdGEgdHlwZSAoUG9zdGdyZVNRTCBpbnQ0KS4KICov"},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/schema.php","start_line_in_file":825,"slug":"column-type-smallint","name":"column_type_smallint","namespace":"Flow\\PostgreSql\\DSL","parameters":[],"return_type":[{"name":"ColumnType","namespace":"Flow\\PostgreSql\\QueryBuilder\\Schema","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBhIHNtYWxsaW50IGRhdGEgdHlwZSAoUG9zdGdyZVNRTCBpbnQyKS4KICov"},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/schema.php","start_line_in_file":834,"slug":"column-type-bigint","name":"column_type_bigint","namespace":"Flow\\PostgreSql\\DSL","parameters":[],"return_type":[{"name":"ColumnType","namespace":"Flow\\PostgreSql\\QueryBuilder\\Schema","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBhIGJpZ2ludCBkYXRhIHR5cGUgKFBvc3RncmVTUUwgaW50OCkuCiAqLw=="},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/schema.php","start_line_in_file":843,"slug":"column-type-boolean","name":"column_type_boolean","namespace":"Flow\\PostgreSql\\DSL","parameters":[],"return_type":[{"name":"ColumnType","namespace":"Flow\\PostgreSql\\QueryBuilder\\Schema","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBhIGJvb2xlYW4gZGF0YSB0eXBlLgogKi8="},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/schema.php","start_line_in_file":852,"slug":"column-type-text","name":"column_type_text","namespace":"Flow\\PostgreSql\\DSL","parameters":[],"return_type":[{"name":"ColumnType","namespace":"Flow\\PostgreSql\\QueryBuilder\\Schema","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBhIHRleHQgZGF0YSB0eXBlLgogKi8="},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/schema.php","start_line_in_file":861,"slug":"column-type-varchar","name":"column_type_varchar","namespace":"Flow\\PostgreSql\\DSL","parameters":[{"name":"length","type":[{"name":"int","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"ColumnType","namespace":"Flow\\PostgreSql\\QueryBuilder\\Schema","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBhIHZhcmNoYXIgZGF0YSB0eXBlIHdpdGggbGVuZ3RoIGNvbnN0cmFpbnQuCiAqLw=="},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/schema.php","start_line_in_file":870,"slug":"column-type-char","name":"column_type_char","namespace":"Flow\\PostgreSql\\DSL","parameters":[{"name":"length","type":[{"name":"int","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"ColumnType","namespace":"Flow\\PostgreSql\\QueryBuilder\\Schema","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBhIGNoYXIgZGF0YSB0eXBlIHdpdGggbGVuZ3RoIGNvbnN0cmFpbnQuCiAqLw=="},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/schema.php","start_line_in_file":879,"slug":"column-type-numeric","name":"column_type_numeric","namespace":"Flow\\PostgreSql\\DSL","parameters":[{"name":"precision","type":[{"name":"int","namespace":null,"is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"null"},{"name":"scale","type":[{"name":"int","namespace":null,"is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"null"}],"return_type":[{"name":"ColumnType","namespace":"Flow\\PostgreSql\\QueryBuilder\\Schema","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBhIG51bWVyaWMgZGF0YSB0eXBlIHdpdGggb3B0aW9uYWwgcHJlY2lzaW9uIGFuZCBzY2FsZS4KICov"},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/schema.php","start_line_in_file":888,"slug":"column-type-decimal","name":"column_type_decimal","namespace":"Flow\\PostgreSql\\DSL","parameters":[{"name":"precision","type":[{"name":"int","namespace":null,"is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"null"},{"name":"scale","type":[{"name":"int","namespace":null,"is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"null"}],"return_type":[{"name":"ColumnType","namespace":"Flow\\PostgreSql\\QueryBuilder\\Schema","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBhIGRlY2ltYWwgZGF0YSB0eXBlIHdpdGggb3B0aW9uYWwgcHJlY2lzaW9uIGFuZCBzY2FsZS4KICov"},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/schema.php","start_line_in_file":897,"slug":"column-type-real","name":"column_type_real","namespace":"Flow\\PostgreSql\\DSL","parameters":[],"return_type":[{"name":"ColumnType","namespace":"Flow\\PostgreSql\\QueryBuilder\\Schema","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBhIHJlYWwgZGF0YSB0eXBlIChQb3N0Z3JlU1FMIGZsb2F0NCkuCiAqLw=="},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/schema.php","start_line_in_file":906,"slug":"column-type-double-precision","name":"column_type_double_precision","namespace":"Flow\\PostgreSql\\DSL","parameters":[],"return_type":[{"name":"ColumnType","namespace":"Flow\\PostgreSql\\QueryBuilder\\Schema","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBhIGRvdWJsZSBwcmVjaXNpb24gZGF0YSB0eXBlIChQb3N0Z3JlU1FMIGZsb2F0OCkuCiAqLw=="},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/schema.php","start_line_in_file":915,"slug":"column-type-date","name":"column_type_date","namespace":"Flow\\PostgreSql\\DSL","parameters":[],"return_type":[{"name":"ColumnType","namespace":"Flow\\PostgreSql\\QueryBuilder\\Schema","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBhIGRhdGUgZGF0YSB0eXBlLgogKi8="},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/schema.php","start_line_in_file":924,"slug":"column-type-time","name":"column_type_time","namespace":"Flow\\PostgreSql\\DSL","parameters":[{"name":"precision","type":[{"name":"int","namespace":null,"is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"null"}],"return_type":[{"name":"ColumnType","namespace":"Flow\\PostgreSql\\QueryBuilder\\Schema","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBhIHRpbWUgZGF0YSB0eXBlIHdpdGggb3B0aW9uYWwgcHJlY2lzaW9uLgogKi8="},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/schema.php","start_line_in_file":933,"slug":"column-type-timestamp","name":"column_type_timestamp","namespace":"Flow\\PostgreSql\\DSL","parameters":[{"name":"precision","type":[{"name":"int","namespace":null,"is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"null"}],"return_type":[{"name":"ColumnType","namespace":"Flow\\PostgreSql\\QueryBuilder\\Schema","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBhIHRpbWVzdGFtcCBkYXRhIHR5cGUgd2l0aCBvcHRpb25hbCBwcmVjaXNpb24uCiAqLw=="},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/schema.php","start_line_in_file":942,"slug":"column-type-timestamptz","name":"column_type_timestamptz","namespace":"Flow\\PostgreSql\\DSL","parameters":[{"name":"precision","type":[{"name":"int","namespace":null,"is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"null"}],"return_type":[{"name":"ColumnType","namespace":"Flow\\PostgreSql\\QueryBuilder\\Schema","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBhIHRpbWVzdGFtcCB3aXRoIHRpbWUgem9uZSBkYXRhIHR5cGUgd2l0aCBvcHRpb25hbCBwcmVjaXNpb24uCiAqLw=="},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/schema.php","start_line_in_file":951,"slug":"column-type-interval","name":"column_type_interval","namespace":"Flow\\PostgreSql\\DSL","parameters":[],"return_type":[{"name":"ColumnType","namespace":"Flow\\PostgreSql\\QueryBuilder\\Schema","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBhbiBpbnRlcnZhbCBkYXRhIHR5cGUuCiAqLw=="},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/schema.php","start_line_in_file":960,"slug":"column-type-uuid","name":"column_type_uuid","namespace":"Flow\\PostgreSql\\DSL","parameters":[],"return_type":[{"name":"ColumnType","namespace":"Flow\\PostgreSql\\QueryBuilder\\Schema","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBhIFVVSUQgZGF0YSB0eXBlLgogKi8="},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/schema.php","start_line_in_file":969,"slug":"column-type-json","name":"column_type_json","namespace":"Flow\\PostgreSql\\DSL","parameters":[],"return_type":[{"name":"ColumnType","namespace":"Flow\\PostgreSql\\QueryBuilder\\Schema","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBhIEpTT04gZGF0YSB0eXBlLgogKi8="},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/schema.php","start_line_in_file":978,"slug":"column-type-jsonb","name":"column_type_jsonb","namespace":"Flow\\PostgreSql\\DSL","parameters":[],"return_type":[{"name":"ColumnType","namespace":"Flow\\PostgreSql\\QueryBuilder\\Schema","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBhIEpTT05CIGRhdGEgdHlwZS4KICov"},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/schema.php","start_line_in_file":987,"slug":"column-type-bytea","name":"column_type_bytea","namespace":"Flow\\PostgreSql\\DSL","parameters":[],"return_type":[{"name":"ColumnType","namespace":"Flow\\PostgreSql\\QueryBuilder\\Schema","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBhIGJ5dGVhIGRhdGEgdHlwZS4KICov"},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/schema.php","start_line_in_file":996,"slug":"column-type-xml","name":"column_type_xml","namespace":"Flow\\PostgreSql\\DSL","parameters":[],"return_type":[{"name":"ColumnType","namespace":"Flow\\PostgreSql\\QueryBuilder\\Schema","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBhbiBYTUwgZGF0YSB0eXBlLgogKi8="},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/schema.php","start_line_in_file":1005,"slug":"column-type-inet","name":"column_type_inet","namespace":"Flow\\PostgreSql\\DSL","parameters":[],"return_type":[{"name":"ColumnType","namespace":"Flow\\PostgreSql\\QueryBuilder\\Schema","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBhbiBpbmV0IGRhdGEgdHlwZS4KICov"},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/schema.php","start_line_in_file":1014,"slug":"column-type-cidr","name":"column_type_cidr","namespace":"Flow\\PostgreSql\\DSL","parameters":[],"return_type":[{"name":"ColumnType","namespace":"Flow\\PostgreSql\\QueryBuilder\\Schema","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBhIGNpZHIgZGF0YSB0eXBlLgogKi8="},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/schema.php","start_line_in_file":1023,"slug":"column-type-macaddr","name":"column_type_macaddr","namespace":"Flow\\PostgreSql\\DSL","parameters":[],"return_type":[{"name":"ColumnType","namespace":"Flow\\PostgreSql\\QueryBuilder\\Schema","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBhIG1hY2FkZHIgZGF0YSB0eXBlLgogKi8="},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/schema.php","start_line_in_file":1032,"slug":"column-type-serial","name":"column_type_serial","namespace":"Flow\\PostgreSql\\DSL","parameters":[],"return_type":[{"name":"ColumnType","namespace":"Flow\\PostgreSql\\QueryBuilder\\Schema","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBhIHNlcmlhbCBkYXRhIHR5cGUuCiAqLw=="},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/schema.php","start_line_in_file":1041,"slug":"column-type-smallserial","name":"column_type_smallserial","namespace":"Flow\\PostgreSql\\DSL","parameters":[],"return_type":[{"name":"ColumnType","namespace":"Flow\\PostgreSql\\QueryBuilder\\Schema","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBhIHNtYWxsc2VyaWFsIGRhdGEgdHlwZS4KICov"},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/schema.php","start_line_in_file":1050,"slug":"column-type-bigserial","name":"column_type_bigserial","namespace":"Flow\\PostgreSql\\DSL","parameters":[],"return_type":[{"name":"ColumnType","namespace":"Flow\\PostgreSql\\QueryBuilder\\Schema","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBhIGJpZ3NlcmlhbCBkYXRhIHR5cGUuCiAqLw=="},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/schema.php","start_line_in_file":1059,"slug":"column-type-array","name":"column_type_array","namespace":"Flow\\PostgreSql\\DSL","parameters":[{"name":"elementType","type":[{"name":"ColumnType","namespace":"Flow\\PostgreSql\\QueryBuilder\\Schema","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"ColumnType","namespace":"Flow\\PostgreSql\\QueryBuilder\\Schema","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBhbiBhcnJheSBkYXRhIHR5cGUgZnJvbSBhbiBlbGVtZW50IHR5cGUuCiAqLw=="},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/schema.php","start_line_in_file":1071,"slug":"column-type-custom","name":"column_type_custom","namespace":"Flow\\PostgreSql\\DSL","parameters":[{"name":"typeName","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"schema","type":[{"name":"string","namespace":null,"is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"null"}],"return_type":[{"name":"ColumnType","namespace":"Flow\\PostgreSql\\QueryBuilder\\Schema","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBhIGN1c3RvbSBkYXRhIHR5cGUuCiAqCiAqIEBwYXJhbSBzdHJpbmcgJHR5cGVOYW1lIFR5cGUgbmFtZQogKiBAcGFyYW0gbnVsbHxzdHJpbmcgJHNjaGVtYSBPcHRpb25hbCBzY2hlbWEgbmFtZQogKi8="},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/schema.php","start_line_in_file":1084,"slug":"column-type-from-string","name":"column_type_from_string","namespace":"Flow\\PostgreSql\\DSL","parameters":[{"name":"typeName","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"ColumnType","namespace":"Flow\\PostgreSql\\QueryBuilder\\Schema","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIFBhcnNlIGEgUG9zdGdyZVNRTCB0eXBlIHN0cmluZyBpbnRvIGEgQ29sdW1uVHlwZS4KICoKICogSGFuZGxlcyBhbGwgUG9zdGdyZVNRTCB0eXBlIHN5bnRheCBpbmNsdWRpbmcgcHJlY2lzaW9uLCBhcnJheXMsIGFuZCBzY2hlbWEtcXVhbGlmaWVkIHR5cGVzLgogKgogKiBAcGFyYW0gc3RyaW5nICR0eXBlTmFtZSBQb3N0Z3JlU1FMIHR5cGUgc3RyaW5nIChlLmcuLCAnaW50ZWdlcicsICdjaGFyYWN0ZXIgdmFyeWluZygyNTUpJywgJ3RleHRbXScpCiAqLw=="},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/schema.php","start_line_in_file":1092,"slug":"value-type-text","name":"value_type_text","namespace":"Flow\\PostgreSql\\DSL","parameters":[],"return_type":[{"name":"ValueType","namespace":"Flow\\PostgreSql\\Client\\Types","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/schema.php","start_line_in_file":1098,"slug":"value-type-varchar","name":"value_type_varchar","namespace":"Flow\\PostgreSql\\DSL","parameters":[],"return_type":[{"name":"ValueType","namespace":"Flow\\PostgreSql\\Client\\Types","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/schema.php","start_line_in_file":1104,"slug":"value-type-char","name":"value_type_char","namespace":"Flow\\PostgreSql\\DSL","parameters":[],"return_type":[{"name":"ValueType","namespace":"Flow\\PostgreSql\\Client\\Types","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/schema.php","start_line_in_file":1110,"slug":"value-type-bpchar","name":"value_type_bpchar","namespace":"Flow\\PostgreSql\\DSL","parameters":[],"return_type":[{"name":"ValueType","namespace":"Flow\\PostgreSql\\Client\\Types","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/schema.php","start_line_in_file":1116,"slug":"value-type-int2","name":"value_type_int2","namespace":"Flow\\PostgreSql\\DSL","parameters":[],"return_type":[{"name":"ValueType","namespace":"Flow\\PostgreSql\\Client\\Types","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/schema.php","start_line_in_file":1122,"slug":"value-type-smallint","name":"value_type_smallint","namespace":"Flow\\PostgreSql\\DSL","parameters":[],"return_type":[{"name":"ValueType","namespace":"Flow\\PostgreSql\\Client\\Types","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/schema.php","start_line_in_file":1128,"slug":"value-type-int4","name":"value_type_int4","namespace":"Flow\\PostgreSql\\DSL","parameters":[],"return_type":[{"name":"ValueType","namespace":"Flow\\PostgreSql\\Client\\Types","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/schema.php","start_line_in_file":1134,"slug":"value-type-integer","name":"value_type_integer","namespace":"Flow\\PostgreSql\\DSL","parameters":[],"return_type":[{"name":"ValueType","namespace":"Flow\\PostgreSql\\Client\\Types","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/schema.php","start_line_in_file":1140,"slug":"value-type-int8","name":"value_type_int8","namespace":"Flow\\PostgreSql\\DSL","parameters":[],"return_type":[{"name":"ValueType","namespace":"Flow\\PostgreSql\\Client\\Types","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/schema.php","start_line_in_file":1146,"slug":"value-type-bigint","name":"value_type_bigint","namespace":"Flow\\PostgreSql\\DSL","parameters":[],"return_type":[{"name":"ValueType","namespace":"Flow\\PostgreSql\\Client\\Types","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/schema.php","start_line_in_file":1152,"slug":"value-type-float4","name":"value_type_float4","namespace":"Flow\\PostgreSql\\DSL","parameters":[],"return_type":[{"name":"ValueType","namespace":"Flow\\PostgreSql\\Client\\Types","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/schema.php","start_line_in_file":1158,"slug":"value-type-real","name":"value_type_real","namespace":"Flow\\PostgreSql\\DSL","parameters":[],"return_type":[{"name":"ValueType","namespace":"Flow\\PostgreSql\\Client\\Types","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/schema.php","start_line_in_file":1164,"slug":"value-type-float8","name":"value_type_float8","namespace":"Flow\\PostgreSql\\DSL","parameters":[],"return_type":[{"name":"ValueType","namespace":"Flow\\PostgreSql\\Client\\Types","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/schema.php","start_line_in_file":1170,"slug":"value-type-double","name":"value_type_double","namespace":"Flow\\PostgreSql\\DSL","parameters":[],"return_type":[{"name":"ValueType","namespace":"Flow\\PostgreSql\\Client\\Types","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/schema.php","start_line_in_file":1176,"slug":"value-type-numeric","name":"value_type_numeric","namespace":"Flow\\PostgreSql\\DSL","parameters":[],"return_type":[{"name":"ValueType","namespace":"Flow\\PostgreSql\\Client\\Types","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/schema.php","start_line_in_file":1182,"slug":"value-type-money","name":"value_type_money","namespace":"Flow\\PostgreSql\\DSL","parameters":[],"return_type":[{"name":"ValueType","namespace":"Flow\\PostgreSql\\Client\\Types","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/schema.php","start_line_in_file":1188,"slug":"value-type-bool","name":"value_type_bool","namespace":"Flow\\PostgreSql\\DSL","parameters":[],"return_type":[{"name":"ValueType","namespace":"Flow\\PostgreSql\\Client\\Types","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/schema.php","start_line_in_file":1194,"slug":"value-type-boolean","name":"value_type_boolean","namespace":"Flow\\PostgreSql\\DSL","parameters":[],"return_type":[{"name":"ValueType","namespace":"Flow\\PostgreSql\\Client\\Types","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/schema.php","start_line_in_file":1200,"slug":"value-type-bytea","name":"value_type_bytea","namespace":"Flow\\PostgreSql\\DSL","parameters":[],"return_type":[{"name":"ValueType","namespace":"Flow\\PostgreSql\\Client\\Types","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/schema.php","start_line_in_file":1206,"slug":"value-type-bit","name":"value_type_bit","namespace":"Flow\\PostgreSql\\DSL","parameters":[],"return_type":[{"name":"ValueType","namespace":"Flow\\PostgreSql\\Client\\Types","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/schema.php","start_line_in_file":1212,"slug":"value-type-varbit","name":"value_type_varbit","namespace":"Flow\\PostgreSql\\DSL","parameters":[],"return_type":[{"name":"ValueType","namespace":"Flow\\PostgreSql\\Client\\Types","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/schema.php","start_line_in_file":1218,"slug":"value-type-date","name":"value_type_date","namespace":"Flow\\PostgreSql\\DSL","parameters":[],"return_type":[{"name":"ValueType","namespace":"Flow\\PostgreSql\\Client\\Types","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/schema.php","start_line_in_file":1224,"slug":"value-type-time","name":"value_type_time","namespace":"Flow\\PostgreSql\\DSL","parameters":[],"return_type":[{"name":"ValueType","namespace":"Flow\\PostgreSql\\Client\\Types","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/schema.php","start_line_in_file":1230,"slug":"value-type-timetz","name":"value_type_timetz","namespace":"Flow\\PostgreSql\\DSL","parameters":[],"return_type":[{"name":"ValueType","namespace":"Flow\\PostgreSql\\Client\\Types","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/schema.php","start_line_in_file":1236,"slug":"value-type-timestamp","name":"value_type_timestamp","namespace":"Flow\\PostgreSql\\DSL","parameters":[],"return_type":[{"name":"ValueType","namespace":"Flow\\PostgreSql\\Client\\Types","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/schema.php","start_line_in_file":1242,"slug":"value-type-timestamptz","name":"value_type_timestamptz","namespace":"Flow\\PostgreSql\\DSL","parameters":[],"return_type":[{"name":"ValueType","namespace":"Flow\\PostgreSql\\Client\\Types","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/schema.php","start_line_in_file":1248,"slug":"value-type-interval","name":"value_type_interval","namespace":"Flow\\PostgreSql\\DSL","parameters":[],"return_type":[{"name":"ValueType","namespace":"Flow\\PostgreSql\\Client\\Types","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/schema.php","start_line_in_file":1254,"slug":"value-type-json","name":"value_type_json","namespace":"Flow\\PostgreSql\\DSL","parameters":[],"return_type":[{"name":"ValueType","namespace":"Flow\\PostgreSql\\Client\\Types","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/schema.php","start_line_in_file":1260,"slug":"value-type-jsonb","name":"value_type_jsonb","namespace":"Flow\\PostgreSql\\DSL","parameters":[],"return_type":[{"name":"ValueType","namespace":"Flow\\PostgreSql\\Client\\Types","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/schema.php","start_line_in_file":1266,"slug":"value-type-uuid","name":"value_type_uuid","namespace":"Flow\\PostgreSql\\DSL","parameters":[],"return_type":[{"name":"ValueType","namespace":"Flow\\PostgreSql\\Client\\Types","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/schema.php","start_line_in_file":1272,"slug":"value-type-inet","name":"value_type_inet","namespace":"Flow\\PostgreSql\\DSL","parameters":[],"return_type":[{"name":"ValueType","namespace":"Flow\\PostgreSql\\Client\\Types","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/schema.php","start_line_in_file":1278,"slug":"value-type-cidr","name":"value_type_cidr","namespace":"Flow\\PostgreSql\\DSL","parameters":[],"return_type":[{"name":"ValueType","namespace":"Flow\\PostgreSql\\Client\\Types","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/schema.php","start_line_in_file":1284,"slug":"value-type-macaddr","name":"value_type_macaddr","namespace":"Flow\\PostgreSql\\DSL","parameters":[],"return_type":[{"name":"ValueType","namespace":"Flow\\PostgreSql\\Client\\Types","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/schema.php","start_line_in_file":1290,"slug":"value-type-macaddr8","name":"value_type_macaddr8","namespace":"Flow\\PostgreSql\\DSL","parameters":[],"return_type":[{"name":"ValueType","namespace":"Flow\\PostgreSql\\Client\\Types","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/schema.php","start_line_in_file":1296,"slug":"value-type-xml","name":"value_type_xml","namespace":"Flow\\PostgreSql\\DSL","parameters":[],"return_type":[{"name":"ValueType","namespace":"Flow\\PostgreSql\\Client\\Types","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/schema.php","start_line_in_file":1302,"slug":"value-type-oid","name":"value_type_oid","namespace":"Flow\\PostgreSql\\DSL","parameters":[],"return_type":[{"name":"ValueType","namespace":"Flow\\PostgreSql\\Client\\Types","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/schema.php","start_line_in_file":1308,"slug":"value-type-text-array","name":"value_type_text_array","namespace":"Flow\\PostgreSql\\DSL","parameters":[],"return_type":[{"name":"ValueType","namespace":"Flow\\PostgreSql\\Client\\Types","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/schema.php","start_line_in_file":1314,"slug":"value-type-varchar-array","name":"value_type_varchar_array","namespace":"Flow\\PostgreSql\\DSL","parameters":[],"return_type":[{"name":"ValueType","namespace":"Flow\\PostgreSql\\Client\\Types","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/schema.php","start_line_in_file":1320,"slug":"value-type-int2-array","name":"value_type_int2_array","namespace":"Flow\\PostgreSql\\DSL","parameters":[],"return_type":[{"name":"ValueType","namespace":"Flow\\PostgreSql\\Client\\Types","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/schema.php","start_line_in_file":1326,"slug":"value-type-int4-array","name":"value_type_int4_array","namespace":"Flow\\PostgreSql\\DSL","parameters":[],"return_type":[{"name":"ValueType","namespace":"Flow\\PostgreSql\\Client\\Types","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/schema.php","start_line_in_file":1332,"slug":"value-type-int8-array","name":"value_type_int8_array","namespace":"Flow\\PostgreSql\\DSL","parameters":[],"return_type":[{"name":"ValueType","namespace":"Flow\\PostgreSql\\Client\\Types","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/schema.php","start_line_in_file":1338,"slug":"value-type-float4-array","name":"value_type_float4_array","namespace":"Flow\\PostgreSql\\DSL","parameters":[],"return_type":[{"name":"ValueType","namespace":"Flow\\PostgreSql\\Client\\Types","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/schema.php","start_line_in_file":1344,"slug":"value-type-float8-array","name":"value_type_float8_array","namespace":"Flow\\PostgreSql\\DSL","parameters":[],"return_type":[{"name":"ValueType","namespace":"Flow\\PostgreSql\\Client\\Types","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/schema.php","start_line_in_file":1350,"slug":"value-type-bool-array","name":"value_type_bool_array","namespace":"Flow\\PostgreSql\\DSL","parameters":[],"return_type":[{"name":"ValueType","namespace":"Flow\\PostgreSql\\Client\\Types","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/schema.php","start_line_in_file":1356,"slug":"value-type-uuid-array","name":"value_type_uuid_array","namespace":"Flow\\PostgreSql\\DSL","parameters":[],"return_type":[{"name":"ValueType","namespace":"Flow\\PostgreSql\\Client\\Types","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/schema.php","start_line_in_file":1362,"slug":"value-type-json-array","name":"value_type_json_array","namespace":"Flow\\PostgreSql\\DSL","parameters":[],"return_type":[{"name":"ValueType","namespace":"Flow\\PostgreSql\\Client\\Types","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/schema.php","start_line_in_file":1368,"slug":"value-type-jsonb-array","name":"value_type_jsonb_array","namespace":"Flow\\PostgreSql\\DSL","parameters":[],"return_type":[{"name":"ValueType","namespace":"Flow\\PostgreSql\\Client\\Types","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/schema.php","start_line_in_file":1384,"slug":"schema","name":"schema","namespace":"Flow\\PostgreSql\\DSL","parameters":[{"name":"name","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"tables","type":[{"name":"array","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"[]"},{"name":"sequences","type":[{"name":"array","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"[]"},{"name":"views","type":[{"name":"array","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"[]"},{"name":"materializedViews","type":[{"name":"array","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"[]"},{"name":"functions","type":[{"name":"array","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"[]"},{"name":"procedures","type":[{"name":"array","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"[]"},{"name":"domains","type":[{"name":"array","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"[]"},{"name":"extensions","type":[{"name":"array","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"[]"}],"return_type":[{"name":"Schema","namespace":"Flow\\PostgreSql\\Schema","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIEBwYXJhbSBsaXN0PFNjaGVtYVRhYmxlPiAkdGFibGVzCiAqIEBwYXJhbSBsaXN0PFNjaGVtYVNlcXVlbmNlPiAkc2VxdWVuY2VzCiAqIEBwYXJhbSBsaXN0PFNjaGVtYVZpZXc+ICR2aWV3cwogKiBAcGFyYW0gbGlzdDxTY2hlbWFNYXRlcmlhbGl6ZWRWaWV3PiAkbWF0ZXJpYWxpemVkVmlld3MKICogQHBhcmFtIGxpc3Q8U2NoZW1hRnVuY3Rpb24+ICRmdW5jdGlvbnMKICogQHBhcmFtIGxpc3Q8U2NoZW1hUHJvY2VkdXJlPiAkcHJvY2VkdXJlcwogKiBAcGFyYW0gbGlzdDxTY2hlbWFEb21haW4+ICRkb21haW5zCiAqIEBwYXJhbSBsaXN0PFNjaGVtYUV4dGVuc2lvbj4gJGV4dGVuc2lvbnMKICov"},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/schema.php","start_line_in_file":1420,"slug":"schema-table","name":"schema_table","namespace":"Flow\\PostgreSql\\DSL","parameters":[{"name":"name","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"columns","type":[{"name":"array","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"primaryKey","type":[{"name":"PrimaryKey","namespace":"Flow\\PostgreSql\\Schema\\Constraint","is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"null"},{"name":"indexes","type":[{"name":"array","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"[]"},{"name":"foreignKeys","type":[{"name":"array","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"[]"},{"name":"uniqueConstraints","type":[{"name":"array","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"[]"},{"name":"checkConstraints","type":[{"name":"array","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"[]"},{"name":"excludeConstraints","type":[{"name":"array","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"[]"},{"name":"triggers","type":[{"name":"array","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"[]"},{"name":"schema","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"'public'"},{"name":"unlogged","type":[{"name":"bool","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"false"},{"name":"partitionStrategy","type":[{"name":"PartitionStrategy","namespace":"Flow\\PostgreSql\\Schema","is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"null"},{"name":"partitionColumns","type":[{"name":"array","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"[]"},{"name":"inherits","type":[{"name":"array","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"[]"},{"name":"tablespace","type":[{"name":"string","namespace":null,"is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"null"}],"return_type":[{"name":"Table","namespace":"Flow\\PostgreSql\\Schema","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIEBwYXJhbSBub24tZW1wdHktbGlzdDxTY2hlbWFDb2x1bW4+ICRjb2x1bW5zCiAqIEBwYXJhbSBsaXN0PFNjaGVtYUluZGV4PiAkaW5kZXhlcwogKiBAcGFyYW0gbGlzdDxTY2hlbWFGb3JlaWduS2V5PiAkZm9yZWlnbktleXMKICogQHBhcmFtIGxpc3Q8U2NoZW1hVW5pcXVlQ29uc3RyYWludD4gJHVuaXF1ZUNvbnN0cmFpbnRzCiAqIEBwYXJhbSBsaXN0PFNjaGVtYUNoZWNrQ29uc3RyYWludD4gJGNoZWNrQ29uc3RyYWludHMKICogQHBhcmFtIGxpc3Q8U2NoZW1hRXhjbHVkZUNvbnN0cmFpbnQ+ICRleGNsdWRlQ29uc3RyYWludHMKICogQHBhcmFtIGxpc3Q8U2NoZW1hVHJpZ2dlcj4gJHRyaWdnZXJzCiAqIEBwYXJhbSBsaXN0PHN0cmluZz4gJHBhcnRpdGlvbkNvbHVtbnMKICogQHBhcmFtIGxpc3Q8c3RyaW5nPiAkaW5oZXJpdHMKICov"},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/schema.php","start_line_in_file":1465,"slug":"schema-table-options","name":"schema_table_options","namespace":"Flow\\PostgreSql\\DSL","parameters":[{"name":"foreignKeys","type":[{"name":"array","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"[]"},{"name":"checkConstraints","type":[{"name":"array","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"[]"},{"name":"excludeConstraints","type":[{"name":"array","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"[]"},{"name":"triggers","type":[{"name":"array","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"[]"},{"name":"unlogged","type":[{"name":"bool","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"false"},{"name":"partitionStrategy","type":[{"name":"PartitionStrategy","namespace":"Flow\\PostgreSql\\Schema","is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"null"},{"name":"partitionColumns","type":[{"name":"array","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"[]"},{"name":"inherits","type":[{"name":"array","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"[]"},{"name":"tablespace","type":[{"name":"string","namespace":null,"is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"null"}],"return_type":[{"name":"TableOptions","namespace":"Flow\\PostgreSql\\Schema","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIEBwYXJhbSBsaXN0PFNjaGVtYUZvcmVpZ25LZXk+ICRmb3JlaWduS2V5cwogKiBAcGFyYW0gbGlzdDxTY2hlbWFDaGVja0NvbnN0cmFpbnQ+ICRjaGVja0NvbnN0cmFpbnRzCiAqIEBwYXJhbSBsaXN0PFNjaGVtYUV4Y2x1ZGVDb25zdHJhaW50PiAkZXhjbHVkZUNvbnN0cmFpbnRzCiAqIEBwYXJhbSBsaXN0PFNjaGVtYVRyaWdnZXI+ICR0cmlnZ2VycwogKiBAcGFyYW0gbGlzdDxzdHJpbmc+ICRwYXJ0aXRpb25Db2x1bW5zCiAqIEBwYXJhbSBsaXN0PHN0cmluZz4gJGluaGVyaXRzCiAqLw=="},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/schema.php","start_line_in_file":1490,"slug":"schema-column","name":"schema_column","namespace":"Flow\\PostgreSql\\DSL","parameters":[{"name":"name","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"type","type":[{"name":"ColumnType","namespace":"Flow\\PostgreSql\\QueryBuilder\\Schema","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"nullable","type":[{"name":"bool","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"true"},{"name":"default","type":[{"name":"Expression","namespace":"Flow\\PostgreSql\\QueryBuilder\\Expression","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false},{"name":"int","namespace":null,"is_nullable":false,"is_variadic":false},{"name":"float","namespace":null,"is_nullable":false,"is_variadic":false},{"name":"bool","namespace":null,"is_nullable":false,"is_variadic":false},{"name":"null","namespace":null,"is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"null"},{"name":"isIdentity","type":[{"name":"bool","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"false"},{"name":"identityGeneration","type":[{"name":"IdentityGeneration","namespace":"Flow\\PostgreSql\\Schema","is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"null"},{"name":"isGenerated","type":[{"name":"bool","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"false"},{"name":"generationExpression","type":[{"name":"string","namespace":null,"is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"null"},{"name":"ordinalPosition","type":[{"name":"int","namespace":null,"is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"null"}],"return_type":[{"name":"Column","namespace":"Flow\\PostgreSql\\Schema","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/schema.php","start_line_in_file":1515,"slug":"schema-column-integer","name":"schema_column_integer","namespace":"Flow\\PostgreSql\\DSL","parameters":[{"name":"name","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"nullable","type":[{"name":"bool","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"true"},{"name":"default","type":[{"name":"Expression","namespace":"Flow\\PostgreSql\\QueryBuilder\\Expression","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false},{"name":"int","namespace":null,"is_nullable":false,"is_variadic":false},{"name":"float","namespace":null,"is_nullable":false,"is_variadic":false},{"name":"bool","namespace":null,"is_nullable":false,"is_variadic":false},{"name":"null","namespace":null,"is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"null"}],"return_type":[{"name":"Column","namespace":"Flow\\PostgreSql\\Schema","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/schema.php","start_line_in_file":1524,"slug":"schema-column-smallint","name":"schema_column_smallint","namespace":"Flow\\PostgreSql\\DSL","parameters":[{"name":"name","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"nullable","type":[{"name":"bool","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"true"},{"name":"default","type":[{"name":"Expression","namespace":"Flow\\PostgreSql\\QueryBuilder\\Expression","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false},{"name":"int","namespace":null,"is_nullable":false,"is_variadic":false},{"name":"float","namespace":null,"is_nullable":false,"is_variadic":false},{"name":"bool","namespace":null,"is_nullable":false,"is_variadic":false},{"name":"null","namespace":null,"is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"null"}],"return_type":[{"name":"Column","namespace":"Flow\\PostgreSql\\Schema","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/schema.php","start_line_in_file":1533,"slug":"schema-column-bigint","name":"schema_column_bigint","namespace":"Flow\\PostgreSql\\DSL","parameters":[{"name":"name","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"nullable","type":[{"name":"bool","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"true"},{"name":"default","type":[{"name":"Expression","namespace":"Flow\\PostgreSql\\QueryBuilder\\Expression","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false},{"name":"int","namespace":null,"is_nullable":false,"is_variadic":false},{"name":"float","namespace":null,"is_nullable":false,"is_variadic":false},{"name":"bool","namespace":null,"is_nullable":false,"is_variadic":false},{"name":"null","namespace":null,"is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"null"}],"return_type":[{"name":"Column","namespace":"Flow\\PostgreSql\\Schema","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/schema.php","start_line_in_file":1542,"slug":"schema-column-serial","name":"schema_column_serial","namespace":"Flow\\PostgreSql\\DSL","parameters":[{"name":"name","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"Column","namespace":"Flow\\PostgreSql\\Schema","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/schema.php","start_line_in_file":1548,"slug":"schema-column-small-serial","name":"schema_column_small_serial","namespace":"Flow\\PostgreSql\\DSL","parameters":[{"name":"name","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"Column","namespace":"Flow\\PostgreSql\\Schema","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/schema.php","start_line_in_file":1554,"slug":"schema-column-big-serial","name":"schema_column_big_serial","namespace":"Flow\\PostgreSql\\DSL","parameters":[{"name":"name","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"Column","namespace":"Flow\\PostgreSql\\Schema","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/schema.php","start_line_in_file":1560,"slug":"schema-column-boolean","name":"schema_column_boolean","namespace":"Flow\\PostgreSql\\DSL","parameters":[{"name":"name","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"nullable","type":[{"name":"bool","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"true"},{"name":"default","type":[{"name":"Expression","namespace":"Flow\\PostgreSql\\QueryBuilder\\Expression","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false},{"name":"int","namespace":null,"is_nullable":false,"is_variadic":false},{"name":"float","namespace":null,"is_nullable":false,"is_variadic":false},{"name":"bool","namespace":null,"is_nullable":false,"is_variadic":false},{"name":"null","namespace":null,"is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"null"}],"return_type":[{"name":"Column","namespace":"Flow\\PostgreSql\\Schema","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/schema.php","start_line_in_file":1569,"slug":"schema-column-text","name":"schema_column_text","namespace":"Flow\\PostgreSql\\DSL","parameters":[{"name":"name","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"nullable","type":[{"name":"bool","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"true"},{"name":"default","type":[{"name":"Expression","namespace":"Flow\\PostgreSql\\QueryBuilder\\Expression","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false},{"name":"int","namespace":null,"is_nullable":false,"is_variadic":false},{"name":"float","namespace":null,"is_nullable":false,"is_variadic":false},{"name":"bool","namespace":null,"is_nullable":false,"is_variadic":false},{"name":"null","namespace":null,"is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"null"}],"return_type":[{"name":"Column","namespace":"Flow\\PostgreSql\\Schema","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/schema.php","start_line_in_file":1578,"slug":"schema-column-varchar","name":"schema_column_varchar","namespace":"Flow\\PostgreSql\\DSL","parameters":[{"name":"name","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"length","type":[{"name":"int","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"nullable","type":[{"name":"bool","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"true"},{"name":"default","type":[{"name":"Expression","namespace":"Flow\\PostgreSql\\QueryBuilder\\Expression","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false},{"name":"int","namespace":null,"is_nullable":false,"is_variadic":false},{"name":"float","namespace":null,"is_nullable":false,"is_variadic":false},{"name":"bool","namespace":null,"is_nullable":false,"is_variadic":false},{"name":"null","namespace":null,"is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"null"}],"return_type":[{"name":"Column","namespace":"Flow\\PostgreSql\\Schema","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/schema.php","start_line_in_file":1588,"slug":"schema-column-char","name":"schema_column_char","namespace":"Flow\\PostgreSql\\DSL","parameters":[{"name":"name","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"length","type":[{"name":"int","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"nullable","type":[{"name":"bool","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"true"},{"name":"default","type":[{"name":"Expression","namespace":"Flow\\PostgreSql\\QueryBuilder\\Expression","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false},{"name":"int","namespace":null,"is_nullable":false,"is_variadic":false},{"name":"float","namespace":null,"is_nullable":false,"is_variadic":false},{"name":"bool","namespace":null,"is_nullable":false,"is_variadic":false},{"name":"null","namespace":null,"is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"null"}],"return_type":[{"name":"Column","namespace":"Flow\\PostgreSql\\Schema","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/schema.php","start_line_in_file":1598,"slug":"schema-column-numeric","name":"schema_column_numeric","namespace":"Flow\\PostgreSql\\DSL","parameters":[{"name":"name","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"precision","type":[{"name":"int","namespace":null,"is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"null"},{"name":"scale","type":[{"name":"int","namespace":null,"is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"null"},{"name":"nullable","type":[{"name":"bool","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"true"},{"name":"default","type":[{"name":"Expression","namespace":"Flow\\PostgreSql\\QueryBuilder\\Expression","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false},{"name":"int","namespace":null,"is_nullable":false,"is_variadic":false},{"name":"float","namespace":null,"is_nullable":false,"is_variadic":false},{"name":"bool","namespace":null,"is_nullable":false,"is_variadic":false},{"name":"null","namespace":null,"is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"null"}],"return_type":[{"name":"Column","namespace":"Flow\\PostgreSql\\Schema","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/schema.php","start_line_in_file":1609,"slug":"schema-column-real","name":"schema_column_real","namespace":"Flow\\PostgreSql\\DSL","parameters":[{"name":"name","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"nullable","type":[{"name":"bool","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"true"},{"name":"default","type":[{"name":"Expression","namespace":"Flow\\PostgreSql\\QueryBuilder\\Expression","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false},{"name":"int","namespace":null,"is_nullable":false,"is_variadic":false},{"name":"float","namespace":null,"is_nullable":false,"is_variadic":false},{"name":"bool","namespace":null,"is_nullable":false,"is_variadic":false},{"name":"null","namespace":null,"is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"null"}],"return_type":[{"name":"Column","namespace":"Flow\\PostgreSql\\Schema","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/schema.php","start_line_in_file":1618,"slug":"schema-column-double-precision","name":"schema_column_double_precision","namespace":"Flow\\PostgreSql\\DSL","parameters":[{"name":"name","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"nullable","type":[{"name":"bool","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"true"},{"name":"default","type":[{"name":"Expression","namespace":"Flow\\PostgreSql\\QueryBuilder\\Expression","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false},{"name":"int","namespace":null,"is_nullable":false,"is_variadic":false},{"name":"float","namespace":null,"is_nullable":false,"is_variadic":false},{"name":"bool","namespace":null,"is_nullable":false,"is_variadic":false},{"name":"null","namespace":null,"is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"null"}],"return_type":[{"name":"Column","namespace":"Flow\\PostgreSql\\Schema","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/schema.php","start_line_in_file":1627,"slug":"schema-column-date","name":"schema_column_date","namespace":"Flow\\PostgreSql\\DSL","parameters":[{"name":"name","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"nullable","type":[{"name":"bool","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"true"},{"name":"default","type":[{"name":"Expression","namespace":"Flow\\PostgreSql\\QueryBuilder\\Expression","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false},{"name":"int","namespace":null,"is_nullable":false,"is_variadic":false},{"name":"float","namespace":null,"is_nullable":false,"is_variadic":false},{"name":"bool","namespace":null,"is_nullable":false,"is_variadic":false},{"name":"null","namespace":null,"is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"null"}],"return_type":[{"name":"Column","namespace":"Flow\\PostgreSql\\Schema","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/schema.php","start_line_in_file":1636,"slug":"schema-column-time","name":"schema_column_time","namespace":"Flow\\PostgreSql\\DSL","parameters":[{"name":"name","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"precision","type":[{"name":"int","namespace":null,"is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"null"},{"name":"nullable","type":[{"name":"bool","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"true"},{"name":"default","type":[{"name":"Expression","namespace":"Flow\\PostgreSql\\QueryBuilder\\Expression","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false},{"name":"int","namespace":null,"is_nullable":false,"is_variadic":false},{"name":"float","namespace":null,"is_nullable":false,"is_variadic":false},{"name":"bool","namespace":null,"is_nullable":false,"is_variadic":false},{"name":"null","namespace":null,"is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"null"}],"return_type":[{"name":"Column","namespace":"Flow\\PostgreSql\\Schema","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/schema.php","start_line_in_file":1646,"slug":"schema-column-timestamp","name":"schema_column_timestamp","namespace":"Flow\\PostgreSql\\DSL","parameters":[{"name":"name","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"precision","type":[{"name":"int","namespace":null,"is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"null"},{"name":"nullable","type":[{"name":"bool","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"true"},{"name":"default","type":[{"name":"Expression","namespace":"Flow\\PostgreSql\\QueryBuilder\\Expression","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false},{"name":"int","namespace":null,"is_nullable":false,"is_variadic":false},{"name":"float","namespace":null,"is_nullable":false,"is_variadic":false},{"name":"bool","namespace":null,"is_nullable":false,"is_variadic":false},{"name":"null","namespace":null,"is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"null"}],"return_type":[{"name":"Column","namespace":"Flow\\PostgreSql\\Schema","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/schema.php","start_line_in_file":1656,"slug":"schema-column-timestamp-tz","name":"schema_column_timestamp_tz","namespace":"Flow\\PostgreSql\\DSL","parameters":[{"name":"name","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"precision","type":[{"name":"int","namespace":null,"is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"null"},{"name":"nullable","type":[{"name":"bool","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"true"},{"name":"default","type":[{"name":"Expression","namespace":"Flow\\PostgreSql\\QueryBuilder\\Expression","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false},{"name":"int","namespace":null,"is_nullable":false,"is_variadic":false},{"name":"float","namespace":null,"is_nullable":false,"is_variadic":false},{"name":"bool","namespace":null,"is_nullable":false,"is_variadic":false},{"name":"null","namespace":null,"is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"null"}],"return_type":[{"name":"Column","namespace":"Flow\\PostgreSql\\Schema","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/schema.php","start_line_in_file":1666,"slug":"schema-column-interval","name":"schema_column_interval","namespace":"Flow\\PostgreSql\\DSL","parameters":[{"name":"name","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"nullable","type":[{"name":"bool","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"true"},{"name":"default","type":[{"name":"Expression","namespace":"Flow\\PostgreSql\\QueryBuilder\\Expression","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false},{"name":"int","namespace":null,"is_nullable":false,"is_variadic":false},{"name":"float","namespace":null,"is_nullable":false,"is_variadic":false},{"name":"bool","namespace":null,"is_nullable":false,"is_variadic":false},{"name":"null","namespace":null,"is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"null"}],"return_type":[{"name":"Column","namespace":"Flow\\PostgreSql\\Schema","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/schema.php","start_line_in_file":1675,"slug":"schema-column-uuid","name":"schema_column_uuid","namespace":"Flow\\PostgreSql\\DSL","parameters":[{"name":"name","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"nullable","type":[{"name":"bool","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"true"},{"name":"default","type":[{"name":"Expression","namespace":"Flow\\PostgreSql\\QueryBuilder\\Expression","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false},{"name":"int","namespace":null,"is_nullable":false,"is_variadic":false},{"name":"float","namespace":null,"is_nullable":false,"is_variadic":false},{"name":"bool","namespace":null,"is_nullable":false,"is_variadic":false},{"name":"null","namespace":null,"is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"null"}],"return_type":[{"name":"Column","namespace":"Flow\\PostgreSql\\Schema","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/schema.php","start_line_in_file":1684,"slug":"schema-column-json","name":"schema_column_json","namespace":"Flow\\PostgreSql\\DSL","parameters":[{"name":"name","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"nullable","type":[{"name":"bool","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"true"},{"name":"default","type":[{"name":"Expression","namespace":"Flow\\PostgreSql\\QueryBuilder\\Expression","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false},{"name":"int","namespace":null,"is_nullable":false,"is_variadic":false},{"name":"float","namespace":null,"is_nullable":false,"is_variadic":false},{"name":"bool","namespace":null,"is_nullable":false,"is_variadic":false},{"name":"null","namespace":null,"is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"null"}],"return_type":[{"name":"Column","namespace":"Flow\\PostgreSql\\Schema","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/schema.php","start_line_in_file":1693,"slug":"schema-column-jsonb","name":"schema_column_jsonb","namespace":"Flow\\PostgreSql\\DSL","parameters":[{"name":"name","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"nullable","type":[{"name":"bool","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"true"},{"name":"default","type":[{"name":"Expression","namespace":"Flow\\PostgreSql\\QueryBuilder\\Expression","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false},{"name":"int","namespace":null,"is_nullable":false,"is_variadic":false},{"name":"float","namespace":null,"is_nullable":false,"is_variadic":false},{"name":"bool","namespace":null,"is_nullable":false,"is_variadic":false},{"name":"null","namespace":null,"is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"null"}],"return_type":[{"name":"Column","namespace":"Flow\\PostgreSql\\Schema","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/schema.php","start_line_in_file":1702,"slug":"schema-column-bytea","name":"schema_column_bytea","namespace":"Flow\\PostgreSql\\DSL","parameters":[{"name":"name","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"nullable","type":[{"name":"bool","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"true"},{"name":"default","type":[{"name":"Expression","namespace":"Flow\\PostgreSql\\QueryBuilder\\Expression","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false},{"name":"int","namespace":null,"is_nullable":false,"is_variadic":false},{"name":"float","namespace":null,"is_nullable":false,"is_variadic":false},{"name":"bool","namespace":null,"is_nullable":false,"is_variadic":false},{"name":"null","namespace":null,"is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"null"}],"return_type":[{"name":"Column","namespace":"Flow\\PostgreSql\\Schema","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/schema.php","start_line_in_file":1711,"slug":"schema-column-inet","name":"schema_column_inet","namespace":"Flow\\PostgreSql\\DSL","parameters":[{"name":"name","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"nullable","type":[{"name":"bool","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"true"},{"name":"default","type":[{"name":"Expression","namespace":"Flow\\PostgreSql\\QueryBuilder\\Expression","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false},{"name":"int","namespace":null,"is_nullable":false,"is_variadic":false},{"name":"float","namespace":null,"is_nullable":false,"is_variadic":false},{"name":"bool","namespace":null,"is_nullable":false,"is_variadic":false},{"name":"null","namespace":null,"is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"null"}],"return_type":[{"name":"Column","namespace":"Flow\\PostgreSql\\Schema","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/schema.php","start_line_in_file":1720,"slug":"schema-column-cidr","name":"schema_column_cidr","namespace":"Flow\\PostgreSql\\DSL","parameters":[{"name":"name","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"nullable","type":[{"name":"bool","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"true"},{"name":"default","type":[{"name":"Expression","namespace":"Flow\\PostgreSql\\QueryBuilder\\Expression","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false},{"name":"int","namespace":null,"is_nullable":false,"is_variadic":false},{"name":"float","namespace":null,"is_nullable":false,"is_variadic":false},{"name":"bool","namespace":null,"is_nullable":false,"is_variadic":false},{"name":"null","namespace":null,"is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"null"}],"return_type":[{"name":"Column","namespace":"Flow\\PostgreSql\\Schema","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/schema.php","start_line_in_file":1729,"slug":"schema-column-macaddr","name":"schema_column_macaddr","namespace":"Flow\\PostgreSql\\DSL","parameters":[{"name":"name","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"nullable","type":[{"name":"bool","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"true"},{"name":"default","type":[{"name":"Expression","namespace":"Flow\\PostgreSql\\QueryBuilder\\Expression","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false},{"name":"int","namespace":null,"is_nullable":false,"is_variadic":false},{"name":"float","namespace":null,"is_nullable":false,"is_variadic":false},{"name":"bool","namespace":null,"is_nullable":false,"is_variadic":false},{"name":"null","namespace":null,"is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"null"}],"return_type":[{"name":"Column","namespace":"Flow\\PostgreSql\\Schema","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/schema.php","start_line_in_file":1741,"slug":"schema-primary-key","name":"schema_primary_key","namespace":"Flow\\PostgreSql\\DSL","parameters":[{"name":"columns","type":[{"name":"array","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"name","type":[{"name":"string","namespace":null,"is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"null"}],"return_type":[{"name":"PrimaryKey","namespace":"Flow\\PostgreSql\\Schema\\Constraint","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIEBwYXJhbSBub24tZW1wdHktbGlzdDxzdHJpbmc+ICRjb2x1bW5zCiAqLw=="},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/schema.php","start_line_in_file":1751,"slug":"schema-foreign-key","name":"schema_foreign_key","namespace":"Flow\\PostgreSql\\DSL","parameters":[{"name":"columns","type":[{"name":"array","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"referenceTable","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"referenceColumns","type":[{"name":"array","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"name","type":[{"name":"string","namespace":null,"is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"null"},{"name":"referenceSchema","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"'public'"},{"name":"onUpdate","type":[{"name":"ReferentialAction","namespace":"Flow\\PostgreSql\\QueryBuilder\\Schema","is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"Flow\\PostgreSql\\QueryBuilder\\Schema\\ReferentialAction::..."},{"name":"onDelete","type":[{"name":"ReferentialAction","namespace":"Flow\\PostgreSql\\QueryBuilder\\Schema","is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"Flow\\PostgreSql\\QueryBuilder\\Schema\\ReferentialAction::..."},{"name":"deferrable","type":[{"name":"bool","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"false"},{"name":"initiallyDeferred","type":[{"name":"bool","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"false"}],"return_type":[{"name":"ForeignKey","namespace":"Flow\\PostgreSql\\Schema\\Constraint","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIEBwYXJhbSBub24tZW1wdHktbGlzdDxzdHJpbmc+ICRjb2x1bW5zCiAqIEBwYXJhbSBub24tZW1wdHktbGlzdDxzdHJpbmc+ICRyZWZlcmVuY2VDb2x1bW5zCiAqLw=="},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/schema.php","start_line_in_file":1779,"slug":"schema-unique","name":"schema_unique","namespace":"Flow\\PostgreSql\\DSL","parameters":[{"name":"columns","type":[{"name":"array","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"name","type":[{"name":"string","namespace":null,"is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"null"},{"name":"nullsNotDistinct","type":[{"name":"bool","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"false"}],"return_type":[{"name":"UniqueConstraint","namespace":"Flow\\PostgreSql\\Schema\\Constraint","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIEBwYXJhbSBub24tZW1wdHktbGlzdDxzdHJpbmc+ICRjb2x1bW5zCiAqLw=="},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/schema.php","start_line_in_file":1785,"slug":"schema-check","name":"schema_check","namespace":"Flow\\PostgreSql\\DSL","parameters":[{"name":"expression","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"name","type":[{"name":"string","namespace":null,"is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"null"},{"name":"noInherit","type":[{"name":"bool","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"false"}],"return_type":[{"name":"CheckConstraint","namespace":"Flow\\PostgreSql\\Schema\\Constraint","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/schema.php","start_line_in_file":1791,"slug":"schema-exclude","name":"schema_exclude","namespace":"Flow\\PostgreSql\\DSL","parameters":[{"name":"definition","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"name","type":[{"name":"string","namespace":null,"is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"null"}],"return_type":[{"name":"ExcludeConstraint","namespace":"Flow\\PostgreSql\\Schema\\Constraint","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/schema.php","start_line_in_file":1800,"slug":"schema-index","name":"schema_index","namespace":"Flow\\PostgreSql\\DSL","parameters":[{"name":"name","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"columns","type":[{"name":"array","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"unique","type":[{"name":"bool","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"false"},{"name":"method","type":[{"name":"IndexMethod","namespace":"Flow\\PostgreSql\\Schema","is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"Flow\\PostgreSql\\Schema\\IndexMethod::..."},{"name":"primary","type":[{"name":"bool","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"false"},{"name":"predicate","type":[{"name":"string","namespace":null,"is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"null"}],"return_type":[{"name":"Index","namespace":"Flow\\PostgreSql\\Schema","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIEBwYXJhbSBub24tZW1wdHktbGlzdDxzdHJpbmc+ICRjb2x1bW5zCiAqLw=="},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/schema.php","start_line_in_file":1812,"slug":"schema-sequence","name":"schema_sequence","namespace":"Flow\\PostgreSql\\DSL","parameters":[{"name":"name","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"dataType","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"'bigint'"},{"name":"startValue","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false},{"name":"int","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"1"},{"name":"minValue","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false},{"name":"int","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"1"},{"name":"maxValue","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false},{"name":"int","namespace":null,"is_nullable":false,"is_variadic":false},{"name":"null","namespace":null,"is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"null"},{"name":"incrementBy","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false},{"name":"int","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"1"},{"name":"cycle","type":[{"name":"bool","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"false"},{"name":"cacheValue","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false},{"name":"int","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"1"},{"name":"ownedByTable","type":[{"name":"string","namespace":null,"is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"null"},{"name":"ownedByColumn","type":[{"name":"string","namespace":null,"is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"null"}],"return_type":[{"name":"Sequence","namespace":"Flow\\PostgreSql\\Schema","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/schema.php","start_line_in_file":1839,"slug":"schema-view","name":"schema_view","namespace":"Flow\\PostgreSql\\DSL","parameters":[{"name":"name","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"definition","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"isUpdatable","type":[{"name":"bool","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"false"}],"return_type":[{"name":"View","namespace":"Flow\\PostgreSql\\Schema","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/schema.php","start_line_in_file":1848,"slug":"schema-materialized-view","name":"schema_materialized_view","namespace":"Flow\\PostgreSql\\DSL","parameters":[{"name":"name","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"definition","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"indexes","type":[{"name":"array","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"[]"}],"return_type":[{"name":"MaterializedView","namespace":"Flow\\PostgreSql\\Schema","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIEBwYXJhbSBsaXN0PFNjaGVtYUluZGV4PiAkaW5kZXhlcwogKi8="},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/schema.php","start_line_in_file":1857,"slug":"schema-function","name":"schema_function","namespace":"Flow\\PostgreSql\\DSL","parameters":[{"name":"name","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"returnType","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"argumentTypes","type":[{"name":"array","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"[]"},{"name":"language","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"'sql'"},{"name":"definition","type":[{"name":"string","namespace":null,"is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"null"},{"name":"isStrict","type":[{"name":"bool","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"false"},{"name":"volatility","type":[{"name":"FunctionVolatility","namespace":"Flow\\PostgreSql\\Schema","is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"null"}],"return_type":[{"name":"Func","namespace":"Flow\\PostgreSql\\Schema","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIEBwYXJhbSBsaXN0PHN0cmluZz4gJGFyZ3VtZW50VHlwZXMKICov"},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/schema.php","start_line_in_file":1873,"slug":"schema-procedure","name":"schema_procedure","namespace":"Flow\\PostgreSql\\DSL","parameters":[{"name":"name","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"argumentTypes","type":[{"name":"array","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"[]"},{"name":"language","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"'sql'"},{"name":"definition","type":[{"name":"string","namespace":null,"is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"null"}],"return_type":[{"name":"Procedure","namespace":"Flow\\PostgreSql\\Schema","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIEBwYXJhbSBsaXN0PHN0cmluZz4gJGFyZ3VtZW50VHlwZXMKICov"},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/schema.php","start_line_in_file":1886,"slug":"schema-trigger","name":"schema_trigger","namespace":"Flow\\PostgreSql\\DSL","parameters":[{"name":"name","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"tableName","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"timing","type":[{"name":"TriggerTiming","namespace":"Flow\\PostgreSql\\Schema","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"events","type":[{"name":"array","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"functionName","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"forEachRow","type":[{"name":"bool","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"false"},{"name":"whenCondition","type":[{"name":"string","namespace":null,"is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"null"}],"return_type":[{"name":"Trigger","namespace":"Flow\\PostgreSql\\Schema","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIEBwYXJhbSBub24tZW1wdHktbGlzdDxUcmlnZ2VyRXZlbnQ+ICRldmVudHMKICov"},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/schema.php","start_line_in_file":1902,"slug":"schema-domain","name":"schema_domain","namespace":"Flow\\PostgreSql\\DSL","parameters":[{"name":"name","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"baseType","type":[{"name":"ColumnType","namespace":"Flow\\PostgreSql\\QueryBuilder\\Schema","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"nullable","type":[{"name":"bool","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"true"},{"name":"default","type":[{"name":"Expression","namespace":"Flow\\PostgreSql\\QueryBuilder\\Expression","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false},{"name":"int","namespace":null,"is_nullable":false,"is_variadic":false},{"name":"float","namespace":null,"is_nullable":false,"is_variadic":false},{"name":"bool","namespace":null,"is_nullable":false,"is_variadic":false},{"name":"null","namespace":null,"is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"null"},{"name":"checkConstraints","type":[{"name":"array","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"[]"}],"return_type":[{"name":"Domain","namespace":"Flow\\PostgreSql\\Schema","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIEBwYXJhbSBsaXN0PFNjaGVtYUNoZWNrQ29uc3RyYWludD4gJGNoZWNrQ29uc3RyYWludHMKICov"},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/schema.php","start_line_in_file":1913,"slug":"schema-extension","name":"schema_extension","namespace":"Flow\\PostgreSql\\DSL","parameters":[{"name":"name","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"version","type":[{"name":"string","namespace":null,"is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"null"}],"return_type":[{"name":"Extension","namespace":"Flow\\PostgreSql\\Schema","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/schema.php","start_line_in_file":1922,"slug":"client-catalog-provider","name":"client_catalog_provider","namespace":"Flow\\PostgreSql\\DSL","parameters":[{"name":"client","type":[{"name":"Client","namespace":"Flow\\PostgreSql\\Client","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"schemaNames","type":[{"name":"array","namespace":null,"is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"null"},{"name":"exclusionPolicy","type":[{"name":"ExclusionPolicy","namespace":"Flow\\PostgreSql\\Schema\\Exclusion","is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"null"}],"return_type":[{"name":"CatalogProvider","namespace":"Flow\\PostgreSql\\Schema","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIEBwYXJhbSA\/bGlzdDxzdHJpbmc+ICRzY2hlbWFOYW1lcwogKi8="},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/schema.php","start_line_in_file":1931,"slug":"exclude-any","name":"exclude_any","namespace":"Flow\\PostgreSql\\DSL","parameters":[{"name":"policies","type":[{"name":"ExclusionPolicy","namespace":"Flow\\PostgreSql\\Schema\\Exclusion","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":true,"default_value":null}],"return_type":[{"name":"ExclusionPolicy","namespace":"Flow\\PostgreSql\\Schema\\Exclusion","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/schema.php","start_line_in_file":1937,"slug":"exclude-exact","name":"exclude_exact","namespace":"Flow\\PostgreSql\\DSL","parameters":[{"name":"name","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"ExclusionPolicy","namespace":"Flow\\PostgreSql\\Schema\\Exclusion","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/schema.php","start_line_in_file":1943,"slug":"exclude-starts-with","name":"exclude_starts_with","namespace":"Flow\\PostgreSql\\DSL","parameters":[{"name":"prefix","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"ExclusionPolicy","namespace":"Flow\\PostgreSql\\Schema\\Exclusion","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/schema.php","start_line_in_file":1949,"slug":"exclude-ends-with","name":"exclude_ends_with","namespace":"Flow\\PostgreSql\\DSL","parameters":[{"name":"suffix","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"ExclusionPolicy","namespace":"Flow\\PostgreSql\\Schema\\Exclusion","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/schema.php","start_line_in_file":1955,"slug":"exclude-pattern","name":"exclude_pattern","namespace":"Flow\\PostgreSql\\DSL","parameters":[{"name":"pattern","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"ExclusionPolicy","namespace":"Flow\\PostgreSql\\Schema\\Exclusion","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/schema.php","start_line_in_file":1961,"slug":"exclude-schema","name":"exclude_schema","namespace":"Flow\\PostgreSql\\DSL","parameters":[{"name":"schema","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"ExclusionPolicy","namespace":"Flow\\PostgreSql\\Schema\\Exclusion","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/schema.php","start_line_in_file":1967,"slug":"exclude-scoped","name":"exclude_scoped","namespace":"Flow\\PostgreSql\\DSL","parameters":[{"name":"policy","type":[{"name":"ExclusionPolicy","namespace":"Flow\\PostgreSql\\Schema\\Exclusion","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"type","type":[{"name":"SchemaObjectType","namespace":"Flow\\PostgreSql\\Schema\\Exclusion","is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"null"},{"name":"schema","type":[{"name":"string","namespace":null,"is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"null"}],"return_type":[{"name":"ExclusionPolicy","namespace":"Flow\\PostgreSql\\Schema\\Exclusion","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/schema.php","start_line_in_file":1976,"slug":"manual-catalog-provider","name":"manual_catalog_provider","namespace":"Flow\\PostgreSql\\DSL","parameters":[{"name":"catalog","type":[{"name":"Catalog","namespace":"Flow\\PostgreSql\\Schema","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"CatalogProvider","namespace":"Flow\\PostgreSql\\Schema","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/schema.php","start_line_in_file":1982,"slug":"chain-catalog-provider","name":"chain_catalog_provider","namespace":"Flow\\PostgreSql\\DSL","parameters":[{"name":"providers","type":[{"name":"CatalogProvider","namespace":"Flow\\PostgreSql\\Schema","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":true,"default_value":null}],"return_type":[{"name":"ChainCatalogProvider","namespace":"Flow\\PostgreSql\\Schema","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/schema.php","start_line_in_file":1991,"slug":"catalog-comparator","name":"catalog_comparator","namespace":"Flow\\PostgreSql\\DSL","parameters":[{"name":"renameStrategy","type":[{"name":"RenameStrategy","namespace":"Flow\\PostgreSql\\Schema\\Diff","is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"null"},{"name":"viewDependencyResolver","type":[{"name":"ViewDependencyResolver","namespace":"Flow\\PostgreSql\\Schema\\Diff","is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"null"},{"name":"tableOrderStrategy","type":[{"name":"ExecutionOrderStrategy","namespace":"Flow\\PostgreSql\\Schema","is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"null"},{"name":"dropIfExists","type":[{"name":"bool","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"false"}],"return_type":[{"name":"CatalogComparator","namespace":"Flow\\PostgreSql\\Schema\\Diff","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIEBwYXJhbSBudWxsfEV4ZWN1dGlvbk9yZGVyU3RyYXRlZ3k8XEZsb3dcUG9zdGdyZVNxbFxTY2hlbWFcVGFibGU+ICR0YWJsZU9yZGVyU3RyYXRlZ3kKICov"},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/schema.php","start_line_in_file":2006,"slug":"ast-view-dependency-resolver","name":"ast_view_dependency_resolver","namespace":"Flow\\PostgreSql\\DSL","parameters":[],"return_type":[{"name":"AstViewDependencyResolver","namespace":"Flow\\PostgreSql\\Schema\\Diff","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/schema.php","start_line_in_file":2012,"slug":"noop-view-dependency-resolver","name":"noop_view_dependency_resolver","namespace":"Flow\\PostgreSql\\DSL","parameters":[],"return_type":[{"name":"NoopViewDependencyResolver","namespace":"Flow\\PostgreSql\\Schema\\Diff","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/schema.php","start_line_in_file":2018,"slug":"foreign-key-dependency-order","name":"foreign_key_dependency_order","namespace":"Flow\\PostgreSql\\DSL","parameters":[],"return_type":[{"name":"ForeignKeyDependencyOrder","namespace":"Flow\\PostgreSql\\Schema","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/schema.php","start_line_in_file":2027,"slug":"no-execution-order","name":"no_execution_order","namespace":"Flow\\PostgreSql\\DSL","parameters":[],"return_type":[{"name":"NoExecutionOrder","namespace":"Flow\\PostgreSql\\Schema","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIEByZXR1cm4gTm9FeGVjdXRpb25PcmRlcjxtaXhlZD4KICov"},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/schema.php","start_line_in_file":2033,"slug":"view-dependency-order","name":"view_dependency_order","namespace":"Flow\\PostgreSql\\DSL","parameters":[],"return_type":[{"name":"ViewDependencyOrder","namespace":"Flow\\PostgreSql\\Schema","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/schema.php","start_line_in_file":2039,"slug":"materialized-view-dependency-order","name":"materialized_view_dependency_order","namespace":"Flow\\PostgreSql\\DSL","parameters":[],"return_type":[{"name":"MaterializedViewDependencyOrder","namespace":"Flow\\PostgreSql\\Schema","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/query.php","start_line_in_file":117,"slug":"select","name":"select","namespace":"Flow\\PostgreSql\\DSL","parameters":[{"name":"expressions","type":[{"name":"Expression","namespace":"Flow\\PostgreSql\\QueryBuilder\\Expression","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":true,"default_value":null}],"return_type":[{"name":"SelectBuilder","namespace":"Flow\\PostgreSql\\QueryBuilder\\Select","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBhIG5ldyBTRUxFQ1QgcXVlcnkgYnVpbGRlci4KICoKICogQHBhcmFtIEV4cHJlc3Npb258c3RyaW5nIC4uLiRleHByZXNzaW9ucyBDb2x1bW5zIHRvIHNlbGVjdC4gSWYgZW1wdHksIHJldHVybnMgU2VsZWN0U2VsZWN0U3RlcC4KICov"},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/query.php","start_line_in_file":134,"slug":"parsed-select","name":"parsed_select","namespace":"Flow\\PostgreSql\\DSL","parameters":[{"name":"sql","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"ParsedSelect","namespace":"Flow\\PostgreSql\\QueryBuilder\\Select","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBhIFNlbGVjdEZpbmFsU3RlcCBmcm9tIGEgcmF3IFNRTCBTRUxFQ1Qgc3RyaW5nLgogKi8="},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/query.php","start_line_in_file":146,"slug":"with","name":"with","namespace":"Flow\\PostgreSql\\DSL","parameters":[{"name":"ctes","type":[{"name":"CTE","namespace":"Flow\\PostgreSql\\QueryBuilder\\Clause","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":true,"default_value":null}],"return_type":[{"name":"WithBuilder","namespace":"Flow\\PostgreSql\\QueryBuilder\\With","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBhIFdJVEggY2xhdXNlIGJ1aWxkZXIgZm9yIENURXMuCiAqCiAqIEV4YW1wbGU6IHdpdGgoY3RlKCd1c2VycycsICRzdWJxdWVyeSkpLT5zZWxlY3Qoc3RhcigpKS0+ZnJvbSh0YWJsZSgndXNlcnMnKSkKICogRXhhbXBsZTogd2l0aChjdGUoJ2EnLCAkcTEpLCBjdGUoJ2InLCAkcTIpKS0+cmVjdXJzaXZlKCktPnNlbGVjdCguLi4pLT5mcm9tKHRhYmxlKCdhJykpCiAqLw=="},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/query.php","start_line_in_file":159,"slug":"insert","name":"insert","namespace":"Flow\\PostgreSql\\DSL","parameters":[],"return_type":[{"name":"InsertIntoStep","namespace":"Flow\\PostgreSql\\QueryBuilder\\Insert","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBhIG5ldyBJTlNFUlQgcXVlcnkgYnVpbGRlci4KICov"},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/query.php","start_line_in_file":175,"slug":"bulk-insert","name":"bulk_insert","namespace":"Flow\\PostgreSql\\DSL","parameters":[{"name":"table","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"columns","type":[{"name":"array","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"rowCount","type":[{"name":"int","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"BulkInsert","namespace":"Flow\\PostgreSql\\QueryBuilder\\Insert","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBhbiBvcHRpbWl6ZWQgYnVsayBJTlNFUlQgcXVlcnkgZm9yIGhpZ2gtcGVyZm9ybWFuY2UgbXVsdGktcm93IGluc2VydHMuCiAqCiAqIFVubGlrZSBpbnNlcnQoKSB3aGljaCB1c2VzIGltbXV0YWJsZSBidWlsZGVyIHBhdHRlcm5zIChPKG7CsikgZm9yIG4gcm93cyksCiAqIHRoaXMgZnVuY3Rpb24gZ2VuZXJhdGVzIFNRTCBkaXJlY3RseSB1c2luZyBzdHJpbmcgb3BlcmF0aW9ucyAoTyhuKSBjb21wbGV4aXR5KS4KICoKICogQHBhcmFtIHN0cmluZyAkdGFibGUgVGFibGUgbmFtZQogKiBAcGFyYW0gbGlzdDxzdHJpbmc+ICRjb2x1bW5zIENvbHVtbiBuYW1lcwogKiBAcGFyYW0gaW50ICRyb3dDb3VudCBOdW1iZXIgb2Ygcm93cyB0byBpbnNlcnQKICov"},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/query.php","start_line_in_file":184,"slug":"update","name":"update","namespace":"Flow\\PostgreSql\\DSL","parameters":[],"return_type":[{"name":"UpdateTableStep","namespace":"Flow\\PostgreSql\\QueryBuilder\\Update","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBhIG5ldyBVUERBVEUgcXVlcnkgYnVpbGRlci4KICov"},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/query.php","start_line_in_file":193,"slug":"delete","name":"delete","namespace":"Flow\\PostgreSql\\DSL","parameters":[],"return_type":[{"name":"DeleteFromStep","namespace":"Flow\\PostgreSql\\QueryBuilder\\Delete","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBhIG5ldyBERUxFVEUgcXVlcnkgYnVpbGRlci4KICov"},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/query.php","start_line_in_file":205,"slug":"merge","name":"merge","namespace":"Flow\\PostgreSql\\DSL","parameters":[{"name":"table","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"alias","type":[{"name":"string","namespace":null,"is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"null"}],"return_type":[{"name":"MergeUsingStep","namespace":"Flow\\PostgreSql\\QueryBuilder\\Merge","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBhIG5ldyBNRVJHRSBxdWVyeSBidWlsZGVyLgogKgogKiBAcGFyYW0gc3RyaW5nICR0YWJsZSBUYXJnZXQgdGFibGUgbmFtZQogKiBAcGFyYW0gbnVsbHxzdHJpbmcgJGFsaWFzIE9wdGlvbmFsIHRhYmxlIGFsaWFzCiAqLw=="},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/query.php","start_line_in_file":219,"slug":"copy","name":"copy","namespace":"Flow\\PostgreSql\\DSL","parameters":[],"return_type":[{"name":"CopyFactory","namespace":"Flow\\PostgreSql\\QueryBuilder\\Factory","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBhIG5ldyBDT1BZIHF1ZXJ5IGJ1aWxkZXIgZm9yIGRhdGEgaW1wb3J0L2V4cG9ydC4KICoKICogVXNhZ2U6CiAqICAgY29weSgpLT5mcm9tKCd1c2VycycpLT5maWxlKCcvdG1wL3VzZXJzLmNzdicpLT5mb3JtYXQoQ29weUZvcm1hdDo6Q1NWKQogKiAgIGNvcHkoKS0+dG8oJ3VzZXJzJyktPmZpbGUoJy90bXAvdXNlcnMuY3N2JyktPmZvcm1hdChDb3B5Rm9ybWF0OjpDU1YpCiAqICAgY29weSgpLT50b1F1ZXJ5KHNlbGVjdCguLi4pKS0+ZmlsZSgnL3RtcC9kYXRhLmNzdicpCiAqLw=="},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/query.php","start_line_in_file":231,"slug":"listen","name":"listen","namespace":"Flow\\PostgreSql\\DSL","parameters":[{"name":"channel","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"ListenFinalStep","namespace":"Flow\\PostgreSql\\QueryBuilder\\Listen","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBhIExJU1RFTiBzdGF0ZW1lbnQgdG8gc3Vic2NyaWJlIHRoZSBjdXJyZW50IHNlc3Npb24gdG8gYSBub3RpZmljYXRpb24gY2hhbm5lbC4KICoKICogVXNhZ2U6CiAqICAgbGlzdGVuKCdteV9jaGFubmVsJyktPnRvU3FsKCkgIC8vIExJU1RFTiBteV9jaGFubmVsCiAqLw=="},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/query.php","start_line_in_file":243,"slug":"unlisten","name":"unlisten","namespace":"Flow\\PostgreSql\\DSL","parameters":[{"name":"channel","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"UnlistenFinalStep","namespace":"Flow\\PostgreSql\\QueryBuilder\\Unlisten","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBhbiBVTkxJU1RFTiBzdGF0ZW1lbnQgdG8gdW5zdWJzY3JpYmUgdGhlIGN1cnJlbnQgc2Vzc2lvbiBmcm9tIGEgbm90aWZpY2F0aW9uIGNoYW5uZWwuCiAqCiAqIFVzYWdlOgogKiAgIHVubGlzdGVuKCdteV9jaGFubmVsJyktPnRvU3FsKCkgIC8vIFVOTElTVEVOIG15X2NoYW5uZWwKICov"},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/query.php","start_line_in_file":256,"slug":"notify","name":"notify","namespace":"Flow\\PostgreSql\\DSL","parameters":[{"name":"channel","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"NotifyFinalStep","namespace":"Flow\\PostgreSql\\QueryBuilder\\Notify","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBhIE5PVElGWSBzdGF0ZW1lbnQgdG8gc2VuZCBhIG5vdGlmaWNhdGlvbiBvbiBhIGNoYW5uZWwsIG9wdGlvbmFsbHkgd2l0aCBhIHBheWxvYWQuCiAqCiAqIFVzYWdlOgogKiAgIG5vdGlmeSgnbXlfY2hhbm5lbCcpLT50b1NxbCgpICAgICAgICAgICAgICAgICAgICAgICAgICAgLy8gTk9USUZZIG15X2NoYW5uZWwKICogICBub3RpZnkoJ215X2NoYW5uZWwnKS0+d2l0aFBheWxvYWQoJ2hlbGxvJyktPnRvU3FsKCkgICAgIC8vIE5PVElGWSBteV9jaGFubmVsLCAnaGVsbG8nCiAqLw=="},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/query.php","start_line_in_file":277,"slug":"col","name":"col","namespace":"Flow\\PostgreSql\\DSL","parameters":[{"name":"column","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"table","type":[{"name":"string","namespace":null,"is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"null"},{"name":"schema","type":[{"name":"string","namespace":null,"is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"null"}],"return_type":[{"name":"Column","namespace":"Flow\\PostgreSql\\QueryBuilder\\Expression","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBhIGNvbHVtbiByZWZlcmVuY2UgZXhwcmVzc2lvbi4KICoKICogQ2FuIGJlIHVzZWQgaW4gdHdvIG1vZGVzOgogKiAtIFBhcnNlIG1vZGU6IGNvbCgndXNlcnMuaWQnKSBvciBjb2woJ3NjaGVtYS50YWJsZS5jb2x1bW4nKSAtIHBhcnNlcyBkb3Qtc2VwYXJhdGVkIHN0cmluZwogKiAtIEV4cGxpY2l0IG1vZGU6IGNvbCgnaWQnLCAndXNlcnMnKSBvciBjb2woJ2lkJywgJ3VzZXJzJywgJ3NjaGVtYScpIC0gc2VwYXJhdGUgYXJndW1lbnRzCiAqCiAqIFdoZW4gJHRhYmxlIG9yICRzY2hlbWEgaXMgcHJvdmlkZWQsICRjb2x1bW4gbXVzdCBiZSBhIHBsYWluIGNvbHVtbiBuYW1lIChubyBkb3RzKS4KICoKICogQHBhcmFtIHN0cmluZyAkY29sdW1uIENvbHVtbiBuYW1lLCBvciBkb3Qtc2VwYXJhdGVkIHBhdGggbGlrZSAidGFibGUuY29sdW1uIiBvciAic2NoZW1hLnRhYmxlLmNvbHVtbiIKICogQHBhcmFtIG51bGx8c3RyaW5nICR0YWJsZSBUYWJsZSBuYW1lIChvcHRpb25hbCwgdHJpZ2dlcnMgZXhwbGljaXQgbW9kZSkKICogQHBhcmFtIG51bGx8c3RyaW5nICRzY2hlbWEgU2NoZW1hIG5hbWUgKG9wdGlvbmFsLCByZXF1aXJlcyAkdGFibGUpCiAqCiAqIEB0aHJvd3MgSW52YWxpZEV4cHJlc3Npb25FeGNlcHRpb24gd2hlbiAkc2NoZW1hIGlzIHByb3ZpZGVkIHdpdGhvdXQgJHRhYmxlLCBvciB3aGVuICRjb2x1bW4gY29udGFpbnMgZG90cyBpbiBleHBsaWNpdCBtb2RlCiAqLw=="},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/query.php","start_line_in_file":304,"slug":"star","name":"star","namespace":"Flow\\PostgreSql\\DSL","parameters":[{"name":"table","type":[{"name":"string","namespace":null,"is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"null"}],"return_type":[{"name":"Star","namespace":"Flow\\PostgreSql\\QueryBuilder\\Expression","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBhIFNFTEVDVCAqIGV4cHJlc3Npb24uCiAqLw=="},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/query.php","start_line_in_file":320,"slug":"literal","name":"literal","namespace":"Flow\\PostgreSql\\DSL","parameters":[{"name":"value","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false},{"name":"int","namespace":null,"is_nullable":false,"is_variadic":false},{"name":"float","namespace":null,"is_nullable":false,"is_variadic":false},{"name":"bool","namespace":null,"is_nullable":false,"is_variadic":false},{"name":"null","namespace":null,"is_nullable":true,"is_variadic":false}],"has_default_value":false,"is_nullable":true,"is_variadic":false,"default_value":null}],"return_type":[{"name":"Literal","namespace":"Flow\\PostgreSql\\QueryBuilder\\Expression","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBhIGxpdGVyYWwgdmFsdWUgZm9yIHVzZSBpbiBxdWVyaWVzLgogKgogKiBBdXRvbWF0aWNhbGx5IGRldGVjdHMgdGhlIHR5cGUgYW5kIGNyZWF0ZXMgdGhlIGFwcHJvcHJpYXRlIGxpdGVyYWw6CiAqIC0gbGl0ZXJhbCgnaGVsbG8nKSBjcmVhdGVzIGEgc3RyaW5nIGxpdGVyYWwKICogLSBsaXRlcmFsKDQyKSBjcmVhdGVzIGFuIGludGVnZXIgbGl0ZXJhbAogKiAtIGxpdGVyYWwoMy4xNCkgY3JlYXRlcyBhIGZsb2F0IGxpdGVyYWwKICogLSBsaXRlcmFsKHRydWUpIGNyZWF0ZXMgYSBib29sZWFuIGxpdGVyYWwKICogLSBsaXRlcmFsKG51bGwpIGNyZWF0ZXMgYSBOVUxMIGxpdGVyYWwKICov"},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/query.php","start_line_in_file":335,"slug":"param","name":"param","namespace":"Flow\\PostgreSql\\DSL","parameters":[{"name":"position","type":[{"name":"int","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"Parameter","namespace":"Flow\\PostgreSql\\QueryBuilder\\Expression","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBhIHBvc2l0aW9uYWwgcGFyYW1ldGVyICgkMSwgJDIsIGV0Yy4pLgogKi8="},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/query.php","start_line_in_file":344,"slug":"parameters","name":"parameters","namespace":"Flow\\PostgreSql\\DSL","parameters":[{"name":"count","type":[{"name":"int","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"startAt","type":[{"name":"int","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"1"}],"return_type":[{"name":"array","namespace":null,"is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIEByZXR1cm4gbGlzdDxQYXJhbWV0ZXI+CiAqLw=="},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/query.php","start_line_in_file":370,"slug":"func","name":"func","namespace":"Flow\\PostgreSql\\DSL","parameters":[{"name":"name","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"args","type":[{"name":"array","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"[]"}],"return_type":[{"name":"FunctionCall","namespace":"Flow\\PostgreSql\\QueryBuilder\\Expression","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBhIGZ1bmN0aW9uIGNhbGwgZXhwcmVzc2lvbi4KICoKICogQHBhcmFtIHN0cmluZyAkbmFtZSBGdW5jdGlvbiBuYW1lIChjYW4gaW5jbHVkZSBzY2hlbWEgbGlrZSAicGdfY2F0YWxvZy5ub3ciKQogKiBAcGFyYW0gbGlzdDxFeHByZXNzaW9ufHN0cmluZz4gJGFyZ3MgRnVuY3Rpb24gYXJndW1lbnRzCiAqLw=="},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/query.php","start_line_in_file":404,"slug":"agg","name":"agg","namespace":"Flow\\PostgreSql\\DSL","parameters":[{"name":"name","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"args","type":[{"name":"array","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"[]"},{"name":"distinct","type":[{"name":"bool","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"false"}],"return_type":[{"name":"AggregateCall","namespace":"Flow\\PostgreSql\\QueryBuilder\\Expression","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBhbiBhZ2dyZWdhdGUgZnVuY3Rpb24gY2FsbCAoQ09VTlQsIFNVTSwgQVZHLCBldGMuKS4KICoKICogQHBhcmFtIHN0cmluZyAkbmFtZSBBZ2dyZWdhdGUgZnVuY3Rpb24gbmFtZQogKiBAcGFyYW0gbGlzdDxFeHByZXNzaW9ufHN0cmluZz4gJGFyZ3MgRnVuY3Rpb24gYXJndW1lbnRzCiAqIEBwYXJhbSBib29sICRkaXN0aW5jdCBVc2UgRElTVElOQ1QgbW9kaWZpZXIKICov"},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/query.php","start_line_in_file":418,"slug":"agg-count","name":"agg_count","namespace":"Flow\\PostgreSql\\DSL","parameters":[{"name":"expr","type":[{"name":"Expression","namespace":"Flow\\PostgreSql\\QueryBuilder\\Expression","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false},{"name":"null","namespace":null,"is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"null"},{"name":"distinct","type":[{"name":"bool","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"false"}],"return_type":[{"name":"AggregateCall","namespace":"Flow\\PostgreSql\\QueryBuilder\\Expression","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBDT1VOVCgqKSBhZ2dyZWdhdGUuCiAqLw=="},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/query.php","start_line_in_file":431,"slug":"count-all","name":"count_all","namespace":"Flow\\PostgreSql\\DSL","parameters":[],"return_type":[{"name":"AggregateCall","namespace":"Flow\\PostgreSql\\QueryBuilder\\Expression","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBDT1VOVCgqKSBhZ2dyZWdhdGUuCiAqLw=="},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/query.php","start_line_in_file":440,"slug":"agg-sum","name":"agg_sum","namespace":"Flow\\PostgreSql\\DSL","parameters":[{"name":"expr","type":[{"name":"Expression","namespace":"Flow\\PostgreSql\\QueryBuilder\\Expression","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"distinct","type":[{"name":"bool","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"false"}],"return_type":[{"name":"AggregateCall","namespace":"Flow\\PostgreSql\\QueryBuilder\\Expression","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBTVU0gYWdncmVnYXRlLgogKi8="},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/query.php","start_line_in_file":449,"slug":"agg-avg","name":"agg_avg","namespace":"Flow\\PostgreSql\\DSL","parameters":[{"name":"expr","type":[{"name":"Expression","namespace":"Flow\\PostgreSql\\QueryBuilder\\Expression","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"distinct","type":[{"name":"bool","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"false"}],"return_type":[{"name":"AggregateCall","namespace":"Flow\\PostgreSql\\QueryBuilder\\Expression","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBBVkcgYWdncmVnYXRlLgogKi8="},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/query.php","start_line_in_file":458,"slug":"agg-min","name":"agg_min","namespace":"Flow\\PostgreSql\\DSL","parameters":[{"name":"expr","type":[{"name":"Expression","namespace":"Flow\\PostgreSql\\QueryBuilder\\Expression","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"AggregateCall","namespace":"Flow\\PostgreSql\\QueryBuilder\\Expression","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBNSU4gYWdncmVnYXRlLgogKi8="},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/query.php","start_line_in_file":467,"slug":"agg-max","name":"agg_max","namespace":"Flow\\PostgreSql\\DSL","parameters":[{"name":"expr","type":[{"name":"Expression","namespace":"Flow\\PostgreSql\\QueryBuilder\\Expression","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"AggregateCall","namespace":"Flow\\PostgreSql\\QueryBuilder\\Expression","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBNQVggYWdncmVnYXRlLgogKi8="},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/query.php","start_line_in_file":478,"slug":"coalesce","name":"coalesce","namespace":"Flow\\PostgreSql\\DSL","parameters":[{"name":"expressions","type":[{"name":"Expression","namespace":"Flow\\PostgreSql\\QueryBuilder\\Expression","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":true,"default_value":null}],"return_type":[{"name":"Coalesce","namespace":"Flow\\PostgreSql\\QueryBuilder\\Expression","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBhIENPQUxFU0NFIGV4cHJlc3Npb24uCiAqCiAqIEBwYXJhbSBFeHByZXNzaW9ufHN0cmluZyAuLi4kZXhwcmVzc2lvbnMgRXhwcmVzc2lvbnMgdG8gY29hbGVzY2UKICov"},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/query.php","start_line_in_file":489,"slug":"nullif","name":"nullif","namespace":"Flow\\PostgreSql\\DSL","parameters":[{"name":"expr1","type":[{"name":"Expression","namespace":"Flow\\PostgreSql\\QueryBuilder\\Expression","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"expr2","type":[{"name":"Expression","namespace":"Flow\\PostgreSql\\QueryBuilder\\Expression","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"NullIf","namespace":"Flow\\PostgreSql\\QueryBuilder\\Expression","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBhIE5VTExJRiBleHByZXNzaW9uLgogKi8="},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/query.php","start_line_in_file":503,"slug":"greatest","name":"greatest","namespace":"Flow\\PostgreSql\\DSL","parameters":[{"name":"expressions","type":[{"name":"Expression","namespace":"Flow\\PostgreSql\\QueryBuilder\\Expression","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":true,"default_value":null}],"return_type":[{"name":"Greatest","namespace":"Flow\\PostgreSql\\QueryBuilder\\Expression","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBhIEdSRUFURVNUIGV4cHJlc3Npb24uCiAqCiAqIEBwYXJhbSBFeHByZXNzaW9ufHN0cmluZyAuLi4kZXhwcmVzc2lvbnMgRXhwcmVzc2lvbnMgdG8gY29tcGFyZQogKi8="},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/query.php","start_line_in_file":516,"slug":"least","name":"least","namespace":"Flow\\PostgreSql\\DSL","parameters":[{"name":"expressions","type":[{"name":"Expression","namespace":"Flow\\PostgreSql\\QueryBuilder\\Expression","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":true,"default_value":null}],"return_type":[{"name":"Least","namespace":"Flow\\PostgreSql\\QueryBuilder\\Expression","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBhIExFQVNUIGV4cHJlc3Npb24uCiAqCiAqIEBwYXJhbSBFeHByZXNzaW9ufHN0cmluZyAuLi4kZXhwcmVzc2lvbnMgRXhwcmVzc2lvbnMgdG8gY29tcGFyZQogKi8="},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/query.php","start_line_in_file":530,"slug":"cast","name":"cast","namespace":"Flow\\PostgreSql\\DSL","parameters":[{"name":"expr","type":[{"name":"Expression","namespace":"Flow\\PostgreSql\\QueryBuilder\\Expression","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"dataType","type":[{"name":"ColumnType","namespace":"Flow\\PostgreSql\\QueryBuilder\\Schema","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"TypeCast","namespace":"Flow\\PostgreSql\\QueryBuilder\\Expression","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBhIHR5cGUgY2FzdCBleHByZXNzaW9uLgogKgogKiBAcGFyYW0gRXhwcmVzc2lvbnxzdHJpbmcgJGV4cHIgRXhwcmVzc2lvbiB0byBjYXN0CiAqIEBwYXJhbSBDb2x1bW5UeXBlICRkYXRhVHlwZSBUYXJnZXQgZGF0YSB0eXBlICh1c2UgY29sdW1uX3R5cGVfKiBmdW5jdGlvbnMpCiAqLw=="},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/query.php","start_line_in_file":542,"slug":"collate","name":"collate","namespace":"Flow\\PostgreSql\\DSL","parameters":[{"name":"expr","type":[{"name":"Expression","namespace":"Flow\\PostgreSql\\QueryBuilder\\Expression","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"collation","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"Collate","namespace":"Flow\\PostgreSql\\QueryBuilder\\Expression","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBhIENPTExBVEUgZXhwcmVzc2lvbi4KICoKICogQHBhcmFtIEV4cHJlc3Npb258c3RyaW5nICRleHByIEV4cHJlc3Npb24gdG8gY29sbGF0ZQogKiBAcGFyYW0gc3RyaW5nICRjb2xsYXRpb24gQ29sbGF0aW9uIG5hbWUgYXMgd3JpdHRlbiBpbiBTUUw6ICdDJywgJyJkZV9ERSInLCAncGdfY2F0YWxvZy4iQyInCiAqLw=="},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/query.php","start_line_in_file":557,"slug":"current-timestamp","name":"current_timestamp","namespace":"Flow\\PostgreSql\\DSL","parameters":[],"return_type":[{"name":"SQLValueFunctionExpression","namespace":"Flow\\PostgreSql\\QueryBuilder\\Expression","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIFNRTCBzdGFuZGFyZCBDVVJSRU5UX1RJTUVTVEFNUCBmdW5jdGlvbi4KICoKICogUmV0dXJucyB0aGUgY3VycmVudCBkYXRlIGFuZCB0aW1lIChhdCB0aGUgc3RhcnQgb2YgdGhlIHRyYW5zYWN0aW9uKS4KICogVXNlZnVsIGFzIGEgY29sdW1uIGRlZmF1bHQgdmFsdWUgb3IgaW4gU0VMRUNUIHF1ZXJpZXMuCiAqCiAqIEV4YW1wbGU6IGNvbHVtbignY3JlYXRlZF9hdCcsIGNvbHVtbl90eXBlX3RpbWVzdGFtcCgpKS0+ZGVmYXVsdChjdXJyZW50X3RpbWVzdGFtcCgpKQogKiBFeGFtcGxlOiBzZWxlY3QoKS0+c2VsZWN0KGN1cnJlbnRfdGltZXN0YW1wKCktPmFzKCdub3cnKSkKICov"},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/query.php","start_line_in_file":572,"slug":"current-date","name":"current_date","namespace":"Flow\\PostgreSql\\DSL","parameters":[],"return_type":[{"name":"SQLValueFunctionExpression","namespace":"Flow\\PostgreSql\\QueryBuilder\\Expression","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIFNRTCBzdGFuZGFyZCBDVVJSRU5UX0RBVEUgZnVuY3Rpb24uCiAqCiAqIFJldHVybnMgdGhlIGN1cnJlbnQgZGF0ZSAoYXQgdGhlIHN0YXJ0IG9mIHRoZSB0cmFuc2FjdGlvbikuCiAqIFVzZWZ1bCBhcyBhIGNvbHVtbiBkZWZhdWx0IHZhbHVlIG9yIGluIFNFTEVDVCBxdWVyaWVzLgogKgogKiBFeGFtcGxlOiBjb2x1bW4oJ2JpcnRoX2RhdGUnLCBjb2x1bW5fdHlwZV9kYXRlKCkpLT5kZWZhdWx0KGN1cnJlbnRfZGF0ZSgpKQogKiBFeGFtcGxlOiBzZWxlY3QoKS0+c2VsZWN0KGN1cnJlbnRfZGF0ZSgpLT5hcygndG9kYXknKSkKICov"},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/query.php","start_line_in_file":587,"slug":"current-time","name":"current_time","namespace":"Flow\\PostgreSql\\DSL","parameters":[],"return_type":[{"name":"SQLValueFunctionExpression","namespace":"Flow\\PostgreSql\\QueryBuilder\\Expression","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIFNRTCBzdGFuZGFyZCBDVVJSRU5UX1RJTUUgZnVuY3Rpb24uCiAqCiAqIFJldHVybnMgdGhlIGN1cnJlbnQgdGltZSAoYXQgdGhlIHN0YXJ0IG9mIHRoZSB0cmFuc2FjdGlvbikuCiAqIFVzZWZ1bCBhcyBhIGNvbHVtbiBkZWZhdWx0IHZhbHVlIG9yIGluIFNFTEVDVCBxdWVyaWVzLgogKgogKiBFeGFtcGxlOiBjb2x1bW4oJ3N0YXJ0X3RpbWUnLCBjb2x1bW5fdHlwZV90aW1lKCkpLT5kZWZhdWx0KGN1cnJlbnRfdGltZSgpKQogKiBFeGFtcGxlOiBzZWxlY3QoKS0+c2VsZWN0KGN1cnJlbnRfdGltZSgpLT5hcygnbm93X3RpbWUnKSkKICov"},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/query.php","start_line_in_file":600,"slug":"case-when","name":"case_when","namespace":"Flow\\PostgreSql\\DSL","parameters":[{"name":"whenClauses","type":[{"name":"array","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"elseResult","type":[{"name":"Expression","namespace":"Flow\\PostgreSql\\QueryBuilder\\Expression","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false},{"name":"null","namespace":null,"is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"null"},{"name":"operand","type":[{"name":"Expression","namespace":"Flow\\PostgreSql\\QueryBuilder\\Expression","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false},{"name":"null","namespace":null,"is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"null"}],"return_type":[{"name":"CaseExpression","namespace":"Flow\\PostgreSql\\QueryBuilder\\Expression","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBhIENBU0UgZXhwcmVzc2lvbi4KICoKICogQHBhcmFtIG5vbi1lbXB0eS1saXN0PFdoZW5DbGF1c2U+ICR3aGVuQ2xhdXNlcyBXSEVOIGNsYXVzZXMKICogQHBhcmFtIG51bGx8RXhwcmVzc2lvbnxzdHJpbmcgJGVsc2VSZXN1bHQgRUxTRSByZXN1bHQgKG9wdGlvbmFsKQogKiBAcGFyYW0gbnVsbHxFeHByZXNzaW9ufHN0cmluZyAkb3BlcmFuZCBDQVNFIG9wZXJhbmQgZm9yIHNpbXBsZSBDQVNFIChvcHRpb25hbCkKICov"},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/query.php","start_line_in_file":616,"slug":"when","name":"when","namespace":"Flow\\PostgreSql\\DSL","parameters":[{"name":"condition","type":[{"name":"Expression","namespace":"Flow\\PostgreSql\\QueryBuilder\\Expression","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"result","type":[{"name":"Expression","namespace":"Flow\\PostgreSql\\QueryBuilder\\Expression","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"WhenClause","namespace":"Flow\\PostgreSql\\QueryBuilder\\Expression","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBhIFdIRU4gY2xhdXNlIGZvciBDQVNFIGV4cHJlc3Npb24uCiAqLw=="},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/query.php","start_line_in_file":628,"slug":"sub-select","name":"sub_select","namespace":"Flow\\PostgreSql\\DSL","parameters":[{"name":"query","type":[{"name":"SelectFinalStep","namespace":"Flow\\PostgreSql\\QueryBuilder\\Select","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"Subquery","namespace":"Flow\\PostgreSql\\QueryBuilder\\Expression","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBhIHN1YnF1ZXJ5IGV4cHJlc3Npb24uCiAqLw=="},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/query.php","start_line_in_file":642,"slug":"array-expr","name":"array_expr","namespace":"Flow\\PostgreSql\\DSL","parameters":[{"name":"elements","type":[{"name":"array","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"ArrayExpression","namespace":"Flow\\PostgreSql\\QueryBuilder\\Expression","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBhbiBhcnJheSBleHByZXNzaW9uLgogKgogKiBAcGFyYW0gbGlzdDxFeHByZXNzaW9ufHN0cmluZz4gJGVsZW1lbnRzIEFycmF5IGVsZW1lbnRzCiAqLw=="},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/query.php","start_line_in_file":655,"slug":"row-expr","name":"row_expr","namespace":"Flow\\PostgreSql\\DSL","parameters":[{"name":"elements","type":[{"name":"array","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"RowExpression","namespace":"Flow\\PostgreSql\\QueryBuilder\\Expression","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBhIHJvdyBleHByZXNzaW9uLgogKgogKiBAcGFyYW0gbGlzdDxFeHByZXNzaW9ufHN0cmluZz4gJGVsZW1lbnRzIFJvdyBlbGVtZW50cwogKi8="},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/query.php","start_line_in_file":666,"slug":"binary-expr","name":"binary_expr","namespace":"Flow\\PostgreSql\\DSL","parameters":[{"name":"left","type":[{"name":"Expression","namespace":"Flow\\PostgreSql\\QueryBuilder\\Expression","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"operator","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"right","type":[{"name":"Expression","namespace":"Flow\\PostgreSql\\QueryBuilder\\Expression","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"BinaryExpression","namespace":"Flow\\PostgreSql\\QueryBuilder\\Expression","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBhIGJpbmFyeSBleHByZXNzaW9uIChsZWZ0IG9wIHJpZ2h0KS4KICov"},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/query.php","start_line_in_file":684,"slug":"window-func","name":"window_func","namespace":"Flow\\PostgreSql\\DSL","parameters":[{"name":"name","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"args","type":[{"name":"array","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"[]"},{"name":"partitionBy","type":[{"name":"array","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"[]"},{"name":"orderBy","type":[{"name":"array","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"[]"}],"return_type":[{"name":"WindowFunction","namespace":"Flow\\PostgreSql\\QueryBuilder\\Expression","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBhIHdpbmRvdyBmdW5jdGlvbi4KICoKICogQHBhcmFtIHN0cmluZyAkbmFtZSBGdW5jdGlvbiBuYW1lCiAqIEBwYXJhbSBsaXN0PEV4cHJlc3Npb258c3RyaW5nPiAkYXJncyBGdW5jdGlvbiBhcmd1bWVudHMKICogQHBhcmFtIGxpc3Q8RXhwcmVzc2lvbnxzdHJpbmc+ICRwYXJ0aXRpb25CeSBQQVJUSVRJT04gQlkgZXhwcmVzc2lvbnMKICogQHBhcmFtIGxpc3Q8T3JkZXJCeT4gJG9yZGVyQnkgT1JERVIgQlkgaXRlbXMKICov"},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/query.php","start_line_in_file":705,"slug":"concat","name":"concat","namespace":"Flow\\PostgreSql\\DSL","parameters":[{"name":"expressions","type":[{"name":"Expression","namespace":"Flow\\PostgreSql\\QueryBuilder\\Expression","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":true,"default_value":null}],"return_type":[{"name":"BinaryExpression","namespace":"Flow\\PostgreSql\\QueryBuilder\\Expression","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENvbmNhdGVuYXRlIGV4cHJlc3Npb25zIHdpdGggdGhlIHx8IG9wZXJhdG9yLgogKgogKiBFeGFtcGxlOiBjb25jYXQoY29sKCdzY2hlbWEnKSwgbGl0ZXJhbCgnLicpLCBjb2woJ3RhYmxlJykpCiAqIFByb2R1Y2VzOiBzY2hlbWEgfHwgJy4nIHx8IHRhYmxlCiAqCiAqIEBwYXJhbSBFeHByZXNzaW9ufHN0cmluZyAuLi4kZXhwcmVzc2lvbnMgQXQgbGVhc3QgMiBleHByZXNzaW9ucyB0byBjb25jYXRlbmF0ZQogKi8="},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/query.php","start_line_in_file":735,"slug":"table","name":"table","namespace":"Flow\\PostgreSql\\DSL","parameters":[{"name":"name","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"schema","type":[{"name":"string","namespace":null,"is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"null"}],"return_type":[{"name":"Table","namespace":"Flow\\PostgreSql\\QueryBuilder\\Table","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBhIHRhYmxlIHJlZmVyZW5jZS4KICoKICogU3VwcG9ydHMgZG90IG5vdGF0aW9uIGZvciBzY2hlbWEtcXVhbGlmaWVkIG5hbWVzOiAicHVibGljLnVzZXJzIiBvciBleHBsaWNpdCBzY2hlbWEgcGFyYW1ldGVyLgogKiBEb3VibGUtcXVvdGVkIGlkZW50aWZpZXJzIHByZXNlcnZlIGRvdHM6ICcibXkudGFibGUiJyBjcmVhdGVzIGEgc2luZ2xlIGlkZW50aWZpZXIuCiAqCiAqIEBwYXJhbSBzdHJpbmcgJG5hbWUgVGFibGUgbmFtZSAobWF5IGluY2x1ZGUgc2NoZW1hIGFzICJzY2hlbWEudGFibGUiKQogKiBAcGFyYW0gbnVsbHxzdHJpbmcgJHNjaGVtYSBTY2hlbWEgbmFtZSAob3B0aW9uYWwsIG92ZXJyaWRlcyBwYXJzZWQgc2NoZW1hKQogKi8="},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/query.php","start_line_in_file":750,"slug":"derived","name":"derived","namespace":"Flow\\PostgreSql\\DSL","parameters":[{"name":"query","type":[{"name":"SelectFinalStep","namespace":"Flow\\PostgreSql\\QueryBuilder\\Select","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"alias","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"DerivedTable","namespace":"Flow\\PostgreSql\\QueryBuilder\\Table","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBhIGRlcml2ZWQgdGFibGUgKHN1YnF1ZXJ5IGluIEZST00gY2xhdXNlKS4KICov"},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/query.php","start_line_in_file":764,"slug":"lateral","name":"lateral","namespace":"Flow\\PostgreSql\\DSL","parameters":[{"name":"reference","type":[{"name":"TableReference","namespace":"Flow\\PostgreSql\\QueryBuilder\\Table","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"Lateral","namespace":"Flow\\PostgreSql\\QueryBuilder\\Table","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBhIExBVEVSQUwgc3VicXVlcnkuCiAqCiAqIEBwYXJhbSBUYWJsZVJlZmVyZW5jZSAkcmVmZXJlbmNlIFRoZSBzdWJxdWVyeSBvciB0YWJsZSBmdW5jdGlvbiByZWZlcmVuY2UKICov"},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/query.php","start_line_in_file":776,"slug":"table-func","name":"table_func","namespace":"Flow\\PostgreSql\\DSL","parameters":[{"name":"function","type":[{"name":"FunctionCall","namespace":"Flow\\PostgreSql\\QueryBuilder\\Expression","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"withOrdinality","type":[{"name":"bool","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"false"}],"return_type":[{"name":"TableFunction","namespace":"Flow\\PostgreSql\\QueryBuilder\\Table","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBhIHRhYmxlIGZ1bmN0aW9uIHJlZmVyZW5jZS4KICoKICogQHBhcmFtIEZ1bmN0aW9uQ2FsbCAkZnVuY3Rpb24gVGhlIHRhYmxlLXZhbHVlZCBmdW5jdGlvbgogKiBAcGFyYW0gYm9vbCAkd2l0aE9yZGluYWxpdHkgV2hldGhlciB0byBhZGQgV0lUSCBPUkRJTkFMSVRZCiAqLw=="},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/query.php","start_line_in_file":795,"slug":"values-table","name":"values_table","namespace":"Flow\\PostgreSql\\DSL","parameters":[{"name":"rows","type":[{"name":"RowExpression","namespace":"Flow\\PostgreSql\\QueryBuilder\\Expression","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":true,"default_value":null}],"return_type":[{"name":"ValuesTable","namespace":"Flow\\PostgreSql\\QueryBuilder\\Table","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBhIFZBTFVFUyBjbGF1c2UgYXMgYSB0YWJsZSByZWZlcmVuY2UuCiAqCiAqIFVzYWdlOgogKiAgIHNlbGVjdCgpLT5mcm9tKAogKiAgICAgICB2YWx1ZXNfdGFibGUoCiAqICAgICAgICAgICByb3dfZXhwcihbbGl0ZXJhbCgxKSwgbGl0ZXJhbCgnQWxpY2UnKV0pLAogKiAgICAgICAgICAgcm93X2V4cHIoW2xpdGVyYWwoMiksIGxpdGVyYWwoJ0JvYicpXSkKICogICAgICAgKS0+YXMoJ3QnLCBbJ2lkJywgJ25hbWUnXSkKICogICApCiAqCiAqIEdlbmVyYXRlczogU0VMRUNUICogRlJPTSAoVkFMVUVTICgxLCAnQWxpY2UnKSwgKDIsICdCb2InKSkgQVMgdChpZCwgbmFtZSkKICov"},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/query.php","start_line_in_file":804,"slug":"order-by","name":"order_by","namespace":"Flow\\PostgreSql\\DSL","parameters":[{"name":"expr","type":[{"name":"Expression","namespace":"Flow\\PostgreSql\\QueryBuilder\\Expression","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"direction","type":[{"name":"SortDirection","namespace":"Flow\\PostgreSql\\QueryBuilder\\Clause","is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"Flow\\PostgreSql\\QueryBuilder\\Clause\\SortDirection::..."},{"name":"nulls","type":[{"name":"NullsPosition","namespace":"Flow\\PostgreSql\\QueryBuilder\\Clause","is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"Flow\\PostgreSql\\QueryBuilder\\Clause\\NullsPosition::..."}],"return_type":[{"name":"OrderBy","namespace":"Flow\\PostgreSql\\QueryBuilder\\Clause","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBhbiBPUkRFUiBCWSBpdGVtLgogKi8="},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/query.php","start_line_in_file":816,"slug":"asc","name":"asc","namespace":"Flow\\PostgreSql\\DSL","parameters":[{"name":"expr","type":[{"name":"Expression","namespace":"Flow\\PostgreSql\\QueryBuilder\\Expression","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"nulls","type":[{"name":"NullsPosition","namespace":"Flow\\PostgreSql\\QueryBuilder\\Clause","is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"Flow\\PostgreSql\\QueryBuilder\\Clause\\NullsPosition::..."}],"return_type":[{"name":"OrderBy","namespace":"Flow\\PostgreSql\\QueryBuilder\\Clause","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBhbiBPUkRFUiBCWSBpdGVtIHdpdGggQVNDIGRpcmVjdGlvbi4KICov"},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/query.php","start_line_in_file":825,"slug":"desc","name":"desc","namespace":"Flow\\PostgreSql\\DSL","parameters":[{"name":"expr","type":[{"name":"Expression","namespace":"Flow\\PostgreSql\\QueryBuilder\\Expression","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"nulls","type":[{"name":"NullsPosition","namespace":"Flow\\PostgreSql\\QueryBuilder\\Clause","is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"Flow\\PostgreSql\\QueryBuilder\\Clause\\NullsPosition::..."}],"return_type":[{"name":"OrderBy","namespace":"Flow\\PostgreSql\\QueryBuilder\\Clause","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBhbiBPUkRFUiBCWSBpdGVtIHdpdGggREVTQyBkaXJlY3Rpb24uCiAqLw=="},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/query.php","start_line_in_file":839,"slug":"cte","name":"cte","namespace":"Flow\\PostgreSql\\DSL","parameters":[{"name":"name","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"query","type":[{"name":"SelectFinalStep","namespace":"Flow\\PostgreSql\\QueryBuilder\\Select","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"columnNames","type":[{"name":"array","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"[]"},{"name":"materialization","type":[{"name":"CTEMaterialization","namespace":"Flow\\PostgreSql\\QueryBuilder\\Clause","is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"Flow\\PostgreSql\\QueryBuilder\\Clause\\CTEMaterialization::..."},{"name":"recursive","type":[{"name":"bool","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"false"}],"return_type":[{"name":"CTE","namespace":"Flow\\PostgreSql\\QueryBuilder\\Clause","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBhIENURSAoQ29tbW9uIFRhYmxlIEV4cHJlc3Npb24pLgogKgogKiBAcGFyYW0gc3RyaW5nICRuYW1lIENURSBuYW1lCiAqIEBwYXJhbSBTZWxlY3RGaW5hbFN0ZXAgJHF1ZXJ5IENURSBxdWVyeQogKiBAcGFyYW0gYXJyYXk8c3RyaW5nPiAkY29sdW1uTmFtZXMgQ29sdW1uIGFsaWFzZXMgKG9wdGlvbmFsKQogKiBAcGFyYW0gQ1RFTWF0ZXJpYWxpemF0aW9uICRtYXRlcmlhbGl6YXRpb24gTWF0ZXJpYWxpemF0aW9uIGhpbnQKICov"},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/query.php","start_line_in_file":861,"slug":"window-def","name":"window_def","namespace":"Flow\\PostgreSql\\DSL","parameters":[{"name":"name","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"partitionBy","type":[{"name":"array","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"[]"},{"name":"orderBy","type":[{"name":"array","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"[]"},{"name":"frame","type":[{"name":"WindowFrame","namespace":"Flow\\PostgreSql\\QueryBuilder\\Clause","is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"null"}],"return_type":[{"name":"WindowDefinition","namespace":"Flow\\PostgreSql\\QueryBuilder\\Clause","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBhIHdpbmRvdyBkZWZpbml0aW9uIGZvciBXSU5ET1cgY2xhdXNlLgogKgogKiBAcGFyYW0gc3RyaW5nICRuYW1lIFdpbmRvdyBuYW1lCiAqIEBwYXJhbSBsaXN0PEV4cHJlc3Npb258c3RyaW5nPiAkcGFydGl0aW9uQnkgUEFSVElUSU9OIEJZIGV4cHJlc3Npb25zCiAqIEBwYXJhbSBsaXN0PE9yZGVyQnk+ICRvcmRlckJ5IE9SREVSIEJZIGl0ZW1zCiAqIEBwYXJhbSBudWxsfFdpbmRvd0ZyYW1lICRmcmFtZSBXaW5kb3cgZnJhbWUgc3BlY2lmaWNhdGlvbgogKi8="},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/query.php","start_line_in_file":881,"slug":"window-frame","name":"window_frame","namespace":"Flow\\PostgreSql\\DSL","parameters":[{"name":"mode","type":[{"name":"FrameMode","namespace":"Flow\\PostgreSql\\QueryBuilder\\Clause","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"start","type":[{"name":"FrameBound","namespace":"Flow\\PostgreSql\\QueryBuilder\\Clause","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"end","type":[{"name":"FrameBound","namespace":"Flow\\PostgreSql\\QueryBuilder\\Clause","is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"null"},{"name":"exclusion","type":[{"name":"FrameExclusion","namespace":"Flow\\PostgreSql\\QueryBuilder\\Clause","is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"Flow\\PostgreSql\\QueryBuilder\\Clause\\FrameExclusion::..."}],"return_type":[{"name":"WindowFrame","namespace":"Flow\\PostgreSql\\QueryBuilder\\Clause","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBhIHdpbmRvdyBmcmFtZSBzcGVjaWZpY2F0aW9uLgogKi8="},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/query.php","start_line_in_file":894,"slug":"frame-current-row","name":"frame_current_row","namespace":"Flow\\PostgreSql\\DSL","parameters":[],"return_type":[{"name":"FrameBound","namespace":"Flow\\PostgreSql\\QueryBuilder\\Clause","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBhIGZyYW1lIGJvdW5kIGZvciBDVVJSRU5UIFJPVy4KICov"},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/query.php","start_line_in_file":903,"slug":"frame-unbounded-preceding","name":"frame_unbounded_preceding","namespace":"Flow\\PostgreSql\\DSL","parameters":[],"return_type":[{"name":"FrameBound","namespace":"Flow\\PostgreSql\\QueryBuilder\\Clause","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBhIGZyYW1lIGJvdW5kIGZvciBVTkJPVU5ERUQgUFJFQ0VESU5HLgogKi8="},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/query.php","start_line_in_file":912,"slug":"frame-unbounded-following","name":"frame_unbounded_following","namespace":"Flow\\PostgreSql\\DSL","parameters":[],"return_type":[{"name":"FrameBound","namespace":"Flow\\PostgreSql\\QueryBuilder\\Clause","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBhIGZyYW1lIGJvdW5kIGZvciBVTkJPVU5ERUQgRk9MTE9XSU5HLgogKi8="},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/query.php","start_line_in_file":921,"slug":"frame-preceding","name":"frame_preceding","namespace":"Flow\\PostgreSql\\DSL","parameters":[{"name":"offset","type":[{"name":"Expression","namespace":"Flow\\PostgreSql\\QueryBuilder\\Expression","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"FrameBound","namespace":"Flow\\PostgreSql\\QueryBuilder\\Clause","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBhIGZyYW1lIGJvdW5kIGZvciBOIFBSRUNFRElORy4KICov"},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/query.php","start_line_in_file":930,"slug":"frame-following","name":"frame_following","namespace":"Flow\\PostgreSql\\DSL","parameters":[{"name":"offset","type":[{"name":"Expression","namespace":"Flow\\PostgreSql\\QueryBuilder\\Expression","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"FrameBound","namespace":"Flow\\PostgreSql\\QueryBuilder\\Clause","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBhIGZyYW1lIGJvdW5kIGZvciBOIEZPTExPV0lORy4KICov"},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/query.php","start_line_in_file":943,"slug":"lock-for","name":"lock_for","namespace":"Flow\\PostgreSql\\DSL","parameters":[{"name":"strength","type":[{"name":"LockStrength","namespace":"Flow\\PostgreSql\\QueryBuilder\\Clause","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"tables","type":[{"name":"array","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"[]"},{"name":"waitPolicy","type":[{"name":"LockWaitPolicy","namespace":"Flow\\PostgreSql\\QueryBuilder\\Clause","is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"Flow\\PostgreSql\\QueryBuilder\\Clause\\LockWaitPolicy::..."}],"return_type":[{"name":"LockingClause","namespace":"Flow\\PostgreSql\\QueryBuilder\\Clause","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBhIGxvY2tpbmcgY2xhdXNlIChGT1IgVVBEQVRFLCBGT1IgU0hBUkUsIGV0Yy4pLgogKgogKiBAcGFyYW0gTG9ja1N0cmVuZ3RoICRzdHJlbmd0aCBMb2NrIHN0cmVuZ3RoCiAqIEBwYXJhbSBsaXN0PHN0cmluZz4gJHRhYmxlcyBUYWJsZXMgdG8gbG9jayAoZW1wdHkgZm9yIGFsbCkKICogQHBhcmFtIExvY2tXYWl0UG9saWN5ICR3YWl0UG9saWN5IFdhaXQgcG9saWN5CiAqLw=="},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/query.php","start_line_in_file":957,"slug":"for-update","name":"for_update","namespace":"Flow\\PostgreSql\\DSL","parameters":[{"name":"tables","type":[{"name":"array","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"[]"}],"return_type":[{"name":"LockingClause","namespace":"Flow\\PostgreSql\\QueryBuilder\\Clause","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBhIEZPUiBVUERBVEUgbG9ja2luZyBjbGF1c2UuCiAqCiAqIEBwYXJhbSBsaXN0PHN0cmluZz4gJHRhYmxlcyBUYWJsZXMgdG8gbG9jayAoZW1wdHkgZm9yIGFsbCkKICov"},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/query.php","start_line_in_file":968,"slug":"for-share","name":"for_share","namespace":"Flow\\PostgreSql\\DSL","parameters":[{"name":"tables","type":[{"name":"array","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"[]"}],"return_type":[{"name":"LockingClause","namespace":"Flow\\PostgreSql\\QueryBuilder\\Clause","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBhIEZPUiBTSEFSRSBsb2NraW5nIGNsYXVzZS4KICoKICogQHBhcmFtIGxpc3Q8c3RyaW5nPiAkdGFibGVzIFRhYmxlcyB0byBsb2NrIChlbXB0eSBmb3IgYWxsKQogKi8="},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/query.php","start_line_in_file":977,"slug":"on-conflict-nothing","name":"on_conflict_nothing","namespace":"Flow\\PostgreSql\\DSL","parameters":[{"name":"target","type":[{"name":"ConflictTarget","namespace":"Flow\\PostgreSql\\QueryBuilder\\Clause","is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"null"}],"return_type":[{"name":"OnConflictClause","namespace":"Flow\\PostgreSql\\QueryBuilder\\Clause","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBhbiBPTiBDT05GTElDVCBETyBOT1RISU5HIGNsYXVzZS4KICov"},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/query.php","start_line_in_file":989,"slug":"on-conflict-update","name":"on_conflict_update","namespace":"Flow\\PostgreSql\\DSL","parameters":[{"name":"target","type":[{"name":"ConflictTarget","namespace":"Flow\\PostgreSql\\QueryBuilder\\Clause","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"updates","type":[{"name":"array","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"OnConflictClause","namespace":"Flow\\PostgreSql\\QueryBuilder\\Clause","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBhbiBPTiBDT05GTElDVCBETyBVUERBVEUgY2xhdXNlLgogKgogKiBAcGFyYW0gQ29uZmxpY3RUYXJnZXQgJHRhcmdldCBDb25mbGljdCB0YXJnZXQgKGNvbHVtbnMgb3IgY29uc3RyYWludCkKICogQHBhcmFtIGFycmF5PHN0cmluZywgRXhwcmVzc2lvbnxzdHJpbmc+ICR1cGRhdGVzIENvbHVtbiB1cGRhdGVzCiAqLw=="},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/query.php","start_line_in_file":1003,"slug":"conflict-columns","name":"conflict_columns","namespace":"Flow\\PostgreSql\\DSL","parameters":[{"name":"columns","type":[{"name":"array","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"ConflictTarget","namespace":"Flow\\PostgreSql\\QueryBuilder\\Clause","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBhIGNvbmZsaWN0IHRhcmdldCBmb3IgT04gQ09ORkxJQ1QgKGNvbHVtbnMpLgogKgogKiBAcGFyYW0gbGlzdDxzdHJpbmc+ICRjb2x1bW5zIENvbHVtbnMgdGhhdCBkZWZpbmUgdW5pcXVlbmVzcwogKi8="},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/query.php","start_line_in_file":1012,"slug":"conflict-constraint","name":"conflict_constraint","namespace":"Flow\\PostgreSql\\DSL","parameters":[{"name":"name","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"ConflictTarget","namespace":"Flow\\PostgreSql\\QueryBuilder\\Clause","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBhIGNvbmZsaWN0IHRhcmdldCBmb3IgT04gQ09ORkxJQ1QgT04gQ09OU1RSQUlOVC4KICov"},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/query.php","start_line_in_file":1023,"slug":"returning","name":"returning","namespace":"Flow\\PostgreSql\\DSL","parameters":[{"name":"expressions","type":[{"name":"Expression","namespace":"Flow\\PostgreSql\\QueryBuilder\\Expression","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":true,"default_value":null}],"return_type":[{"name":"ReturningClause","namespace":"Flow\\PostgreSql\\QueryBuilder\\Clause","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBhIFJFVFVSTklORyBjbGF1c2UuCiAqCiAqIEBwYXJhbSBFeHByZXNzaW9ufHN0cmluZyAuLi4kZXhwcmVzc2lvbnMgRXhwcmVzc2lvbnMgdG8gcmV0dXJuCiAqLw=="},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/query.php","start_line_in_file":1034,"slug":"returning-all","name":"returning_all","namespace":"Flow\\PostgreSql\\DSL","parameters":[],"return_type":[{"name":"ReturningClause","namespace":"Flow\\PostgreSql\\QueryBuilder\\Clause","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBhIFJFVFVSTklORyAqIGNsYXVzZS4KICov"},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/query.php","start_line_in_file":1046,"slug":"begin","name":"begin","namespace":"Flow\\PostgreSql\\DSL","parameters":[],"return_type":[{"name":"BeginOptionsStep","namespace":"Flow\\PostgreSql\\QueryBuilder\\Transaction","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBhIEJFR0lOIHRyYW5zYWN0aW9uIGJ1aWxkZXIuCiAqCiAqIEV4YW1wbGU6IGJlZ2luKCktPmlzb2xhdGlvbkxldmVsKElzb2xhdGlvbkxldmVsOjpTRVJJQUxJWkFCTEUpLT5yZWFkT25seSgpCiAqIFByb2R1Y2VzOiBCRUdJTiBJU09MQVRJT04gTEVWRUwgU0VSSUFMSVpBQkxFIFJFQUQgT05MWQogKi8="},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/query.php","start_line_in_file":1058,"slug":"commit","name":"commit","namespace":"Flow\\PostgreSql\\DSL","parameters":[],"return_type":[{"name":"CommitOptionsStep","namespace":"Flow\\PostgreSql\\QueryBuilder\\Transaction","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBhIENPTU1JVCB0cmFuc2FjdGlvbiBidWlsZGVyLgogKgogKiBFeGFtcGxlOiBjb21taXQoKS0+YW5kQ2hhaW4oKQogKiBQcm9kdWNlczogQ09NTUlUIEFORCBDSEFJTgogKi8="},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/query.php","start_line_in_file":1070,"slug":"rollback","name":"rollback","namespace":"Flow\\PostgreSql\\DSL","parameters":[],"return_type":[{"name":"RollbackOptionsStep","namespace":"Flow\\PostgreSql\\QueryBuilder\\Transaction","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBhIFJPTExCQUNLIHRyYW5zYWN0aW9uIGJ1aWxkZXIuCiAqCiAqIEV4YW1wbGU6IHJvbGxiYWNrKCktPnRvU2F2ZXBvaW50KCdteV9zYXZlcG9pbnQnKQogKiBQcm9kdWNlczogUk9MTEJBQ0sgVE8gU0FWRVBPSU5UIG15X3NhdmVwb2ludAogKi8="},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/query.php","start_line_in_file":1082,"slug":"savepoint","name":"savepoint","namespace":"Flow\\PostgreSql\\DSL","parameters":[{"name":"name","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"SavepointFinalStep","namespace":"Flow\\PostgreSql\\QueryBuilder\\Transaction","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBhIFNBVkVQT0lOVC4KICoKICogRXhhbXBsZTogc2F2ZXBvaW50KCdteV9zYXZlcG9pbnQnKQogKiBQcm9kdWNlczogU0FWRVBPSU5UIG15X3NhdmVwb2ludAogKi8="},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/query.php","start_line_in_file":1094,"slug":"release-savepoint","name":"release_savepoint","namespace":"Flow\\PostgreSql\\DSL","parameters":[{"name":"name","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"SavepointFinalStep","namespace":"Flow\\PostgreSql\\QueryBuilder\\Transaction","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIFJlbGVhc2UgYSBTQVZFUE9JTlQuCiAqCiAqIEV4YW1wbGU6IHJlbGVhc2Vfc2F2ZXBvaW50KCdteV9zYXZlcG9pbnQnKQogKiBQcm9kdWNlczogUkVMRUFTRSBteV9zYXZlcG9pbnQKICov"},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/query.php","start_line_in_file":1106,"slug":"set-transaction","name":"set_transaction","namespace":"Flow\\PostgreSql\\DSL","parameters":[],"return_type":[{"name":"SetTransactionOptionsStep","namespace":"Flow\\PostgreSql\\QueryBuilder\\Transaction","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBhIFNFVCBUUkFOU0FDVElPTiBidWlsZGVyLgogKgogKiBFeGFtcGxlOiBzZXRfdHJhbnNhY3Rpb24oKS0+aXNvbGF0aW9uTGV2ZWwoSXNvbGF0aW9uTGV2ZWw6OlNFUklBTElaQUJMRSktPnJlYWRPbmx5KCkKICogUHJvZHVjZXM6IFNFVCBUUkFOU0FDVElPTiBJU09MQVRJT04gTEVWRUwgU0VSSUFMSVpBQkxFLCBSRUFEIE9OTFkKICov"},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/query.php","start_line_in_file":1118,"slug":"set-session-transaction","name":"set_session_transaction","namespace":"Flow\\PostgreSql\\DSL","parameters":[],"return_type":[{"name":"SetTransactionOptionsStep","namespace":"Flow\\PostgreSql\\QueryBuilder\\Transaction","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBhIFNFVCBTRVNTSU9OIENIQVJBQ1RFUklTVElDUyBBUyBUUkFOU0FDVElPTiBidWlsZGVyLgogKgogKiBFeGFtcGxlOiBzZXRfc2Vzc2lvbl90cmFuc2FjdGlvbigpLT5pc29sYXRpb25MZXZlbChJc29sYXRpb25MZXZlbDo6U0VSSUFMSVpBQkxFKQogKiBQcm9kdWNlczogU0VUIFNFU1NJT04gQ0hBUkFDVEVSSVNUSUNTIEFTIFRSQU5TQUNUSU9OIElTT0xBVElPTiBMRVZFTCBTRVJJQUxJWkFCTEUKICov"},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/query.php","start_line_in_file":1130,"slug":"transaction-snapshot","name":"transaction_snapshot","namespace":"Flow\\PostgreSql\\DSL","parameters":[{"name":"snapshotId","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"SetTransactionFinalStep","namespace":"Flow\\PostgreSql\\QueryBuilder\\Transaction","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBhIFNFVCBUUkFOU0FDVElPTiBTTkFQU0hPVCBidWlsZGVyLgogKgogKiBFeGFtcGxlOiB0cmFuc2FjdGlvbl9zbmFwc2hvdCgnMDAwMDAwMDMtMDAwMDAwMUEtMScpCiAqIFByb2R1Y2VzOiBTRVQgVFJBTlNBQ1RJT04gU05BUFNIT1QgJzAwMDAwMDAzLTAwMDAwMDFBLTEnCiAqLw=="},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/query.php","start_line_in_file":1142,"slug":"prepare-transaction","name":"prepare_transaction","namespace":"Flow\\PostgreSql\\DSL","parameters":[{"name":"transactionId","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"PreparedTransactionFinalStep","namespace":"Flow\\PostgreSql\\QueryBuilder\\Transaction","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBhIFBSRVBBUkUgVFJBTlNBQ1RJT04gYnVpbGRlci4KICoKICogRXhhbXBsZTogcHJlcGFyZV90cmFuc2FjdGlvbignbXlfdHJhbnNhY3Rpb24nKQogKiBQcm9kdWNlczogUFJFUEFSRSBUUkFOU0FDVElPTiAnbXlfdHJhbnNhY3Rpb24nCiAqLw=="},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/query.php","start_line_in_file":1154,"slug":"commit-prepared","name":"commit_prepared","namespace":"Flow\\PostgreSql\\DSL","parameters":[{"name":"transactionId","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"PreparedTransactionFinalStep","namespace":"Flow\\PostgreSql\\QueryBuilder\\Transaction","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBhIENPTU1JVCBQUkVQQVJFRCBidWlsZGVyLgogKgogKiBFeGFtcGxlOiBjb21taXRfcHJlcGFyZWQoJ215X3RyYW5zYWN0aW9uJykKICogUHJvZHVjZXM6IENPTU1JVCBQUkVQQVJFRCAnbXlfdHJhbnNhY3Rpb24nCiAqLw=="},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/query.php","start_line_in_file":1166,"slug":"rollback-prepared","name":"rollback_prepared","namespace":"Flow\\PostgreSql\\DSL","parameters":[{"name":"transactionId","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"PreparedTransactionFinalStep","namespace":"Flow\\PostgreSql\\QueryBuilder\\Transaction","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBhIFJPTExCQUNLIFBSRVBBUkVEIGJ1aWxkZXIuCiAqCiAqIEV4YW1wbGU6IHJvbGxiYWNrX3ByZXBhcmVkKCdteV90cmFuc2FjdGlvbicpCiAqIFByb2R1Y2VzOiBST0xMQkFDSyBQUkVQQVJFRCAnbXlfdHJhbnNhY3Rpb24nCiAqLw=="},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/query.php","start_line_in_file":1189,"slug":"declare-cursor","name":"declare_cursor","namespace":"Flow\\PostgreSql\\DSL","parameters":[{"name":"cursorName","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"query","type":[{"name":"SelectFinalStep","namespace":"Flow\\PostgreSql\\QueryBuilder\\Select","is_nullable":false,"is_variadic":false},{"name":"Sql","namespace":"Flow\\PostgreSql\\QueryBuilder","is_nullable":false,"is_variadic":false},{"name":"ParsedQuery","namespace":"Flow\\PostgreSql","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"DeclareCursorOptionsStep","namespace":"Flow\\PostgreSql\\QueryBuilder\\Cursor","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIERlY2xhcmUgYSBzZXJ2ZXItc2lkZSBjdXJzb3IgZm9yIGEgcXVlcnkuCiAqCiAqIEN1cnNvcnMgbXVzdCBiZSBkZWNsYXJlZCB3aXRoaW4gYSB0cmFuc2FjdGlvbiBhbmQgcHJvdmlkZSBtZW1vcnktZWZmaWNpZW50CiAqIGl0ZXJhdGlvbiBvdmVyIGxhcmdlIHJlc3VsdCBzZXRzIHZpYSBGRVRDSCBjb21tYW5kcy4KICoKICogRXhhbXBsZSB3aXRoIHF1ZXJ5IGJ1aWxkZXI6CiAqICAgZGVjbGFyZV9jdXJzb3IoJ215X2N1cnNvcicsIHNlbGVjdChzdGFyKCkpLT5mcm9tKHRhYmxlKCd1c2VycycpKSktPm5vU2Nyb2xsKCkKICogICBQcm9kdWNlczogREVDTEFSRSBteV9jdXJzb3IgTk8gU0NST0xMIENVUlNPUiBGT1IgU0VMRUNUICogRlJPTSB1c2VycwogKgogKiBFeGFtcGxlIHdpdGggcmF3IFNRTDoKICogICBkZWNsYXJlX2N1cnNvcignbXlfY3Vyc29yJywgJ1NFTEVDVCAqIEZST00gdXNlcnMgV0hFUkUgYWN0aXZlID0gdHJ1ZScpLT53aXRoSG9sZCgpCiAqICAgUHJvZHVjZXM6IERFQ0xBUkUgbXlfY3Vyc29yIE5PIFNDUk9MTCBDVVJTT1IgV0lUSCBIT0xEIEZPUiBTRUxFQ1QgKiBGUk9NIHVzZXJzIFdIRVJFIGFjdGl2ZSA9IHRydWUKICoKICogQHBhcmFtIHN0cmluZyAkY3Vyc29yTmFtZSBVbmlxdWUgY3Vyc29yIG5hbWUKICogQHBhcmFtIFBhcnNlZFF1ZXJ5fFNlbGVjdEZpbmFsU3RlcHxTcWx8c3RyaW5nICRxdWVyeSBRdWVyeSB0byBpdGVyYXRlIG92ZXIKICov"},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/query.php","start_line_in_file":1214,"slug":"fetch","name":"fetch","namespace":"Flow\\PostgreSql\\DSL","parameters":[{"name":"cursorName","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"FetchCursorBuilder","namespace":"Flow\\PostgreSql\\QueryBuilder\\Cursor","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIEZldGNoIHJvd3MgZnJvbSBhIGN1cnNvci4KICoKICogRXhhbXBsZTogZmV0Y2goJ215X2N1cnNvcicpLT5mb3J3YXJkKDEwMCkKICogUHJvZHVjZXM6IEZFVENIIEZPUldBUkQgMTAwIG15X2N1cnNvcgogKgogKiBFeGFtcGxlOiBmZXRjaCgnbXlfY3Vyc29yJyktPmFsbCgpCiAqIFByb2R1Y2VzOiBGRVRDSCBBTEwgbXlfY3Vyc29yCiAqCiAqIEBwYXJhbSBzdHJpbmcgJGN1cnNvck5hbWUgQ3Vyc29yIHRvIGZldGNoIGZyb20KICov"},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/query.php","start_line_in_file":1231,"slug":"close-cursor","name":"close_cursor","namespace":"Flow\\PostgreSql\\DSL","parameters":[{"name":"cursorName","type":[{"name":"string","namespace":null,"is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"null"}],"return_type":[{"name":"CloseCursorFinalStep","namespace":"Flow\\PostgreSql\\QueryBuilder\\Cursor","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENsb3NlIGEgY3Vyc29yLgogKgogKiBFeGFtcGxlOiBjbG9zZV9jdXJzb3IoJ215X2N1cnNvcicpCiAqIFByb2R1Y2VzOiBDTE9TRSBteV9jdXJzb3IKICoKICogRXhhbXBsZTogY2xvc2VfY3Vyc29yKCkgLSBjbG9zZXMgYWxsIGN1cnNvcnMKICogUHJvZHVjZXM6IENMT1NFIEFMTAogKgogKiBAcGFyYW0gbnVsbHxzdHJpbmcgJGN1cnNvck5hbWUgQ3Vyc29yIHRvIGNsb3NlLCBvciBudWxsIHRvIGNsb3NlIGFsbAogKi8="},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/condition.php","start_line_in_file":42,"slug":"eq","name":"eq","namespace":"Flow\\PostgreSql\\DSL","parameters":[{"name":"left","type":[{"name":"Expression","namespace":"Flow\\PostgreSql\\QueryBuilder\\Expression","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"right","type":[{"name":"Expression","namespace":"Flow\\PostgreSql\\QueryBuilder\\Expression","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"Comparison","namespace":"Flow\\PostgreSql\\QueryBuilder\\Condition","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBhbiBlcXVhbGl0eSBjb21wYXJpc29uIChjb2x1bW4gPSB2YWx1ZSkuCiAqLw=="},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/condition.php","start_line_in_file":55,"slug":"ne","name":"ne","namespace":"Flow\\PostgreSql\\DSL","parameters":[{"name":"left","type":[{"name":"Expression","namespace":"Flow\\PostgreSql\\QueryBuilder\\Expression","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"right","type":[{"name":"Expression","namespace":"Flow\\PostgreSql\\QueryBuilder\\Expression","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"Comparison","namespace":"Flow\\PostgreSql\\QueryBuilder\\Condition","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBhIG5vdC1lcXVhbCBjb21wYXJpc29uIChjb2x1bW4gIT0gdmFsdWUpLgogKi8="},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/condition.php","start_line_in_file":68,"slug":"lt","name":"lt","namespace":"Flow\\PostgreSql\\DSL","parameters":[{"name":"left","type":[{"name":"Expression","namespace":"Flow\\PostgreSql\\QueryBuilder\\Expression","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"right","type":[{"name":"Expression","namespace":"Flow\\PostgreSql\\QueryBuilder\\Expression","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"Comparison","namespace":"Flow\\PostgreSql\\QueryBuilder\\Condition","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBhIGxlc3MtdGhhbiBjb21wYXJpc29uIChjb2x1bW4gPCB2YWx1ZSkuCiAqLw=="},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/condition.php","start_line_in_file":81,"slug":"le","name":"le","namespace":"Flow\\PostgreSql\\DSL","parameters":[{"name":"left","type":[{"name":"Expression","namespace":"Flow\\PostgreSql\\QueryBuilder\\Expression","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"right","type":[{"name":"Expression","namespace":"Flow\\PostgreSql\\QueryBuilder\\Expression","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"Comparison","namespace":"Flow\\PostgreSql\\QueryBuilder\\Condition","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBhIGxlc3MtdGhhbi1vci1lcXVhbCBjb21wYXJpc29uIChjb2x1bW4gPD0gdmFsdWUpLgogKi8="},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/condition.php","start_line_in_file":94,"slug":"gt","name":"gt","namespace":"Flow\\PostgreSql\\DSL","parameters":[{"name":"left","type":[{"name":"Expression","namespace":"Flow\\PostgreSql\\QueryBuilder\\Expression","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"right","type":[{"name":"Expression","namespace":"Flow\\PostgreSql\\QueryBuilder\\Expression","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"Comparison","namespace":"Flow\\PostgreSql\\QueryBuilder\\Condition","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBhIGdyZWF0ZXItdGhhbiBjb21wYXJpc29uIChjb2x1bW4gPiB2YWx1ZSkuCiAqLw=="},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/condition.php","start_line_in_file":107,"slug":"ge","name":"ge","namespace":"Flow\\PostgreSql\\DSL","parameters":[{"name":"left","type":[{"name":"Expression","namespace":"Flow\\PostgreSql\\QueryBuilder\\Expression","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"right","type":[{"name":"Expression","namespace":"Flow\\PostgreSql\\QueryBuilder\\Expression","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"Comparison","namespace":"Flow\\PostgreSql\\QueryBuilder\\Condition","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBhIGdyZWF0ZXItdGhhbi1vci1lcXVhbCBjb21wYXJpc29uIChjb2x1bW4gPj0gdmFsdWUpLgogKi8="},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/condition.php","start_line_in_file":120,"slug":"between","name":"between","namespace":"Flow\\PostgreSql\\DSL","parameters":[{"name":"expr","type":[{"name":"Expression","namespace":"Flow\\PostgreSql\\QueryBuilder\\Expression","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"low","type":[{"name":"Expression","namespace":"Flow\\PostgreSql\\QueryBuilder\\Expression","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"high","type":[{"name":"Expression","namespace":"Flow\\PostgreSql\\QueryBuilder\\Expression","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"not","type":[{"name":"bool","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"false"}],"return_type":[{"name":"Between","namespace":"Flow\\PostgreSql\\QueryBuilder\\Condition","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBhIEJFVFdFRU4gY29uZGl0aW9uLgogKi8="},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/condition.php","start_line_in_file":139,"slug":"in","name":"in_","namespace":"Flow\\PostgreSql\\DSL","parameters":[{"name":"expr","type":[{"name":"Expression","namespace":"Flow\\PostgreSql\\QueryBuilder\\Expression","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"values","type":[{"name":"array","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"In","namespace":"Flow\\PostgreSql\\QueryBuilder\\Condition","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBhbiBJTiBjb25kaXRpb24uCiAqCiAqIEBwYXJhbSBFeHByZXNzaW9ufHN0cmluZyAkZXhwciBFeHByZXNzaW9uIHRvIGNoZWNrCiAqIEBwYXJhbSBsaXN0PEV4cHJlc3Npb24+ICR2YWx1ZXMgTGlzdCBvZiB2YWx1ZXMgKG11c3QgYmUgbm9uLWVtcHR5KQogKgogKiBAdGhyb3dzIFxJbnZhbGlkQXJndW1lbnRFeGNlcHRpb24gd2hlbiB2YWx1ZXMgYXJyYXkgaXMgZW1wdHkKICov"},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/condition.php","start_line_in_file":156,"slug":"is-null","name":"is_null","namespace":"Flow\\PostgreSql\\DSL","parameters":[{"name":"expr","type":[{"name":"Expression","namespace":"Flow\\PostgreSql\\QueryBuilder\\Expression","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"not","type":[{"name":"bool","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"false"}],"return_type":[{"name":"IsNull","namespace":"Flow\\PostgreSql\\QueryBuilder\\Condition","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBhbiBJUyBOVUxMIGNvbmRpdGlvbi4KICov"},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/condition.php","start_line_in_file":165,"slug":"like","name":"like","namespace":"Flow\\PostgreSql\\DSL","parameters":[{"name":"expr","type":[{"name":"Expression","namespace":"Flow\\PostgreSql\\QueryBuilder\\Expression","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"pattern","type":[{"name":"Expression","namespace":"Flow\\PostgreSql\\QueryBuilder\\Expression","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"caseInsensitive","type":[{"name":"bool","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"false"},{"name":"negated","type":[{"name":"bool","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"false"}],"return_type":[{"name":"Like","namespace":"Flow\\PostgreSql\\QueryBuilder\\Condition","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBhIExJS0UgY29uZGl0aW9uLgogKi8="},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/condition.php","start_line_in_file":183,"slug":"similar-to","name":"similar_to","namespace":"Flow\\PostgreSql\\DSL","parameters":[{"name":"expr","type":[{"name":"Expression","namespace":"Flow\\PostgreSql\\QueryBuilder\\Expression","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"pattern","type":[{"name":"Expression","namespace":"Flow\\PostgreSql\\QueryBuilder\\Expression","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"SimilarTo","namespace":"Flow\\PostgreSql\\QueryBuilder\\Condition","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBhIFNJTUlMQVIgVE8gY29uZGl0aW9uLgogKi8="},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/condition.php","start_line_in_file":195,"slug":"distinct-from","name":"distinct_from","namespace":"Flow\\PostgreSql\\DSL","parameters":[{"name":"left","type":[{"name":"Expression","namespace":"Flow\\PostgreSql\\QueryBuilder\\Expression","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"right","type":[{"name":"Expression","namespace":"Flow\\PostgreSql\\QueryBuilder\\Expression","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"not","type":[{"name":"bool","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"false"}],"return_type":[{"name":"IsDistinctFrom","namespace":"Flow\\PostgreSql\\QueryBuilder\\Condition","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBhbiBJUyBESVNUSU5DVCBGUk9NIGNvbmRpdGlvbi4KICov"},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/condition.php","start_line_in_file":208,"slug":"exists","name":"exists","namespace":"Flow\\PostgreSql\\DSL","parameters":[{"name":"subquery","type":[{"name":"SelectFinalStep","namespace":"Flow\\PostgreSql\\QueryBuilder\\Select","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"Exists","namespace":"Flow\\PostgreSql\\QueryBuilder\\Condition","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBhbiBFWElTVFMgY29uZGl0aW9uLgogKi8="},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/condition.php","start_line_in_file":223,"slug":"any","name":"any_","namespace":"Flow\\PostgreSql\\DSL","parameters":[{"name":"left","type":[{"name":"Expression","namespace":"Flow\\PostgreSql\\QueryBuilder\\Expression","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"operator","type":[{"name":"ComparisonOperator","namespace":"Flow\\PostgreSql\\QueryBuilder\\Condition","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"arrayOrSubquery","type":[{"name":"Expression","namespace":"Flow\\PostgreSql\\QueryBuilder\\Expression","is_nullable":false,"is_variadic":false},{"name":"SelectFinalStep","namespace":"Flow\\PostgreSql\\QueryBuilder\\Select","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"Any","namespace":"Flow\\PostgreSql\\QueryBuilder\\Condition","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBhbiBBTlkgY29uZGl0aW9uIHdpdGggYSBzdWJxdWVyeSBvciBhcnJheSBleHByZXNzaW9uLgogKgogKiBFeGFtcGxlOiBhbnlfKGNvbCgnaWQnKSwgQ29tcGFyaXNvbk9wZXJhdG9yOjpFUSwgc2VsZWN0KGNvbCgndXNlcl9pZCcpKS0+ZnJvbSh0YWJsZSgnb3JkZXJzJykpKQogKiBFeGFtcGxlOiBhbnlfKGNvbCgnYXR0bnVtJywgJ2EnKSwgQ29tcGFyaXNvbk9wZXJhdG9yOjpFUSwgY29sKCdjb25rZXknLCAnY29uJykpCiAqLw=="},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/condition.php","start_line_in_file":243,"slug":"all","name":"all_","namespace":"Flow\\PostgreSql\\DSL","parameters":[{"name":"left","type":[{"name":"Expression","namespace":"Flow\\PostgreSql\\QueryBuilder\\Expression","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"operator","type":[{"name":"ComparisonOperator","namespace":"Flow\\PostgreSql\\QueryBuilder\\Condition","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"arrayOrSubquery","type":[{"name":"Expression","namespace":"Flow\\PostgreSql\\QueryBuilder\\Expression","is_nullable":false,"is_variadic":false},{"name":"SelectFinalStep","namespace":"Flow\\PostgreSql\\QueryBuilder\\Select","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"All","namespace":"Flow\\PostgreSql\\QueryBuilder\\Condition","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBhbiBBTEwgY29uZGl0aW9uIHdpdGggYSBzdWJxdWVyeSBvciBhcnJheSBleHByZXNzaW9uLgogKgogKiBFeGFtcGxlOiBhbGxfKGNvbCgnaWQnKSwgQ29tcGFyaXNvbk9wZXJhdG9yOjpFUSwgc2VsZWN0KGNvbCgndXNlcl9pZCcpKS0+ZnJvbSh0YWJsZSgnb3JkZXJzJykpKQogKiBFeGFtcGxlOiBhbGxfKGNvbCgndmFsdWUnKSwgQ29tcGFyaXNvbk9wZXJhdG9yOjpHVCwgY29sKCd0aHJlc2hvbGRzJykpCiAqLw=="},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/condition.php","start_line_in_file":262,"slug":"is-true","name":"is_true","namespace":"Flow\\PostgreSql\\DSL","parameters":[{"name":"expr","type":[{"name":"Expression","namespace":"Flow\\PostgreSql\\QueryBuilder\\Expression","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"BooleanCondition","namespace":"Flow\\PostgreSql\\QueryBuilder\\Condition","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIFdyYXAgYW4gZXhwcmVzc2lvbiBhcyBhIGJvb2xlYW4gY29uZGl0aW9uIGZvciB1c2UgaW4gV0hFUkUvSEFWSU5HL0pPSU4gT04uCiAqCiAqIEV4YW1wbGU6IGlzX3RydWUoY29sKCdpc19hY3RpdmUnKSkg4oCUIHVzZXMgYSBib29sZWFuIGNvbHVtbiBpbiBXSEVSRSBjbGF1c2UuCiAqLw=="},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/condition.php","start_line_in_file":274,"slug":"not-like","name":"not_like","namespace":"Flow\\PostgreSql\\DSL","parameters":[{"name":"expr","type":[{"name":"Expression","namespace":"Flow\\PostgreSql\\QueryBuilder\\Expression","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"pattern","type":[{"name":"Expression","namespace":"Flow\\PostgreSql\\QueryBuilder\\Expression","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"caseInsensitive","type":[{"name":"bool","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"false"}],"return_type":[{"name":"Like","namespace":"Flow\\PostgreSql\\QueryBuilder\\Condition","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBhIE5PVCBMSUtFIGNvbmRpdGlvbi4KICoKICogRXhhbXBsZTogbm90X2xpa2UoY29sKCduYW1lJyksIGxpdGVyYWwoJ3BnXyUnKSkKICogUHJvZHVjZXM6IG5hbWUgTk9UIExJS0UgJ3BnXyUnCiAqLw=="},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/condition.php","start_line_in_file":302,"slug":"conditions","name":"conditions","namespace":"Flow\\PostgreSql\\DSL","parameters":[],"return_type":[{"name":"ConditionBuilder","namespace":"Flow\\PostgreSql\\QueryBuilder\\Condition","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBhIGNvbmRpdGlvbiBidWlsZGVyIGZvciBmbHVlbnQgY29uZGl0aW9uIGNvbXBvc2l0aW9uLgogKgogKiBUaGlzIGJ1aWxkZXIgYWxsb3dzIGluY3JlbWVudGFsIGNvbmRpdGlvbiBidWlsZGluZyB3aXRoIGEgZmx1ZW50IEFQSToKICoKICogYGBgcGhwCiAqICRidWlsZGVyID0gY29uZGl0aW9ucygpOwogKgogKiBpZiAoJGhhc0ZpbHRlcikgewogKiAgICAgJGJ1aWxkZXIgPSAkYnVpbGRlci0+YW5kKGVxKGNvbCgnc3RhdHVzJyksIGxpdGVyYWwoJ2FjdGl2ZScpKSk7CiAqIH0KICoKICogaWYgKCEkYnVpbGRlci0+aXNFbXB0eSgpKSB7CiAqICAgICAkcXVlcnkgPSBzZWxlY3QoKS0+ZnJvbSh0YWJsZSgndXNlcnMnKSktPndoZXJlKCRidWlsZGVyKTsKICogfQogKiBgYGAKICov"},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/condition.php","start_line_in_file":313,"slug":"and","name":"and_","namespace":"Flow\\PostgreSql\\DSL","parameters":[{"name":"conditions","type":[{"name":"Condition","namespace":"Flow\\PostgreSql\\QueryBuilder\\Condition","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":true,"default_value":null}],"return_type":[{"name":"AndCondition","namespace":"Flow\\PostgreSql\\QueryBuilder\\Condition","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENvbWJpbmUgY29uZGl0aW9ucyB3aXRoIEFORC4KICoKICogQHBhcmFtIENvbmRpdGlvbiAuLi4kY29uZGl0aW9ucyBDb25kaXRpb25zIHRvIGNvbWJpbmUKICov"},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/condition.php","start_line_in_file":324,"slug":"or","name":"or_","namespace":"Flow\\PostgreSql\\DSL","parameters":[{"name":"conditions","type":[{"name":"Condition","namespace":"Flow\\PostgreSql\\QueryBuilder\\Condition","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":true,"default_value":null}],"return_type":[{"name":"OrCondition","namespace":"Flow\\PostgreSql\\QueryBuilder\\Condition","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENvbWJpbmUgY29uZGl0aW9ucyB3aXRoIE9SLgogKgogKiBAcGFyYW0gQ29uZGl0aW9uIC4uLiRjb25kaXRpb25zIENvbmRpdGlvbnMgdG8gY29tYmluZQogKi8="},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/condition.php","start_line_in_file":336,"slug":"not","name":"not_","namespace":"Flow\\PostgreSql\\DSL","parameters":[{"name":"expression","type":[{"name":"Expression","namespace":"Flow\\PostgreSql\\QueryBuilder\\Expression","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"NotCondition","namespace":"Flow\\PostgreSql\\QueryBuilder\\Condition","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIE5lZ2F0ZSBhIGNvbmRpdGlvbiBvciBleHByZXNzaW9uIHdpdGggTk9ULgogKgogKiBBY2NlcHRzIGJvdGggQ29uZGl0aW9uIGFuZCBFeHByZXNzaW9uIOKAlCBOT1QgYWx3YXlzIHByb2R1Y2VzIGEgYm9vbGVhbiByZXN1bHQuCiAqIENhbiBiZSB1c2VkIGluIFdIRVJFIGNsYXVzZXMgYW5kIFNFTEVDVCBsaXN0cyAodmlhIC0+YXMoJ2FsaWFzJykpLgogKi8="},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/condition.php","start_line_in_file":348,"slug":"json-contains","name":"json_contains","namespace":"Flow\\PostgreSql\\DSL","parameters":[{"name":"left","type":[{"name":"Expression","namespace":"Flow\\PostgreSql\\QueryBuilder\\Expression","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"right","type":[{"name":"Expression","namespace":"Flow\\PostgreSql\\QueryBuilder\\Expression","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"OperatorCondition","namespace":"Flow\\PostgreSql\\QueryBuilder\\Condition","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBhIEpTT05CIGNvbnRhaW5zIGNvbmRpdGlvbiAoQD4pLgogKgogKiBFeGFtcGxlOiBqc29uX2NvbnRhaW5zKGNvbCgnbWV0YWRhdGEnKSwgbGl0ZXJhbF9qc29uKCd7ImNhdGVnb3J5IjogImVsZWN0cm9uaWNzIn0nKSkKICogUHJvZHVjZXM6IG1ldGFkYXRhIEA+ICd7ImNhdGVnb3J5IjogImVsZWN0cm9uaWNzIn0nCiAqLw=="},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/condition.php","start_line_in_file":364,"slug":"json-contained-by","name":"json_contained_by","namespace":"Flow\\PostgreSql\\DSL","parameters":[{"name":"left","type":[{"name":"Expression","namespace":"Flow\\PostgreSql\\QueryBuilder\\Expression","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"right","type":[{"name":"Expression","namespace":"Flow\\PostgreSql\\QueryBuilder\\Expression","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"OperatorCondition","namespace":"Flow\\PostgreSql\\QueryBuilder\\Condition","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBhIEpTT05CIGlzIGNvbnRhaW5lZCBieSBjb25kaXRpb24gKDxAKS4KICoKICogRXhhbXBsZToganNvbl9jb250YWluZWRfYnkoY29sKCdtZXRhZGF0YScpLCBsaXRlcmFsX2pzb24oJ3siY2F0ZWdvcnkiOiAiZWxlY3Ryb25pY3MiLCAicHJpY2UiOiAxMDB9JykpCiAqIFByb2R1Y2VzOiBtZXRhZGF0YSA8QCAneyJjYXRlZ29yeSI6ICJlbGVjdHJvbmljcyIsICJwcmljZSI6IDEwMH0nCiAqLw=="},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/condition.php","start_line_in_file":381,"slug":"json-get","name":"json_get","namespace":"Flow\\PostgreSql\\DSL","parameters":[{"name":"expr","type":[{"name":"Expression","namespace":"Flow\\PostgreSql\\QueryBuilder\\Expression","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"key","type":[{"name":"Expression","namespace":"Flow\\PostgreSql\\QueryBuilder\\Expression","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"BinaryExpression","namespace":"Flow\\PostgreSql\\QueryBuilder\\Expression","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBhIEpTT04gZmllbGQgYWNjZXNzIGV4cHJlc3Npb24gKC0+KS4KICogUmV0dXJucyBKU09OLgogKgogKiBFeGFtcGxlOiBqc29uX2dldChjb2woJ21ldGFkYXRhJyksIGxpdGVyYWxfc3RyaW5nKCdjYXRlZ29yeScpKQogKiBQcm9kdWNlczogbWV0YWRhdGEgLT4gJ2NhdGVnb3J5JwogKi8="},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/condition.php","start_line_in_file":398,"slug":"json-get-text","name":"json_get_text","namespace":"Flow\\PostgreSql\\DSL","parameters":[{"name":"expr","type":[{"name":"Expression","namespace":"Flow\\PostgreSql\\QueryBuilder\\Expression","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"key","type":[{"name":"Expression","namespace":"Flow\\PostgreSql\\QueryBuilder\\Expression","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"BinaryExpression","namespace":"Flow\\PostgreSql\\QueryBuilder\\Expression","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBhIEpTT04gZmllbGQgYWNjZXNzIGV4cHJlc3Npb24gKC0+PikuCiAqIFJldHVybnMgdGV4dC4KICoKICogRXhhbXBsZToganNvbl9nZXRfdGV4dChjb2woJ21ldGFkYXRhJyksIGxpdGVyYWxfc3RyaW5nKCduYW1lJykpCiAqIFByb2R1Y2VzOiBtZXRhZGF0YSAtPj4gJ25hbWUnCiAqLw=="},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/condition.php","start_line_in_file":415,"slug":"json-path","name":"json_path","namespace":"Flow\\PostgreSql\\DSL","parameters":[{"name":"expr","type":[{"name":"Expression","namespace":"Flow\\PostgreSql\\QueryBuilder\\Expression","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"path","type":[{"name":"Expression","namespace":"Flow\\PostgreSql\\QueryBuilder\\Expression","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"BinaryExpression","namespace":"Flow\\PostgreSql\\QueryBuilder\\Expression","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBhIEpTT04gcGF0aCBhY2Nlc3MgZXhwcmVzc2lvbiAoIz4pLgogKiBSZXR1cm5zIEpTT04uCiAqCiAqIEV4YW1wbGU6IGpzb25fcGF0aChjb2woJ21ldGFkYXRhJyksIGxpdGVyYWxfc3RyaW5nKCd7Y2F0ZWdvcnksbmFtZX0nKSkKICogUHJvZHVjZXM6IG1ldGFkYXRhICM+ICd7Y2F0ZWdvcnksbmFtZX0nCiAqLw=="},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/condition.php","start_line_in_file":432,"slug":"json-path-text","name":"json_path_text","namespace":"Flow\\PostgreSql\\DSL","parameters":[{"name":"expr","type":[{"name":"Expression","namespace":"Flow\\PostgreSql\\QueryBuilder\\Expression","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"path","type":[{"name":"Expression","namespace":"Flow\\PostgreSql\\QueryBuilder\\Expression","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"BinaryExpression","namespace":"Flow\\PostgreSql\\QueryBuilder\\Expression","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBhIEpTT04gcGF0aCBhY2Nlc3MgZXhwcmVzc2lvbiAoIz4+KS4KICogUmV0dXJucyB0ZXh0LgogKgogKiBFeGFtcGxlOiBqc29uX3BhdGhfdGV4dChjb2woJ21ldGFkYXRhJyksIGxpdGVyYWxfc3RyaW5nKCd7Y2F0ZWdvcnksbmFtZX0nKSkKICogUHJvZHVjZXM6IG1ldGFkYXRhICM+PiAne2NhdGVnb3J5LG5hbWV9JwogKi8="},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/condition.php","start_line_in_file":448,"slug":"json-exists","name":"json_exists","namespace":"Flow\\PostgreSql\\DSL","parameters":[{"name":"expr","type":[{"name":"Expression","namespace":"Flow\\PostgreSql\\QueryBuilder\\Expression","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"key","type":[{"name":"Expression","namespace":"Flow\\PostgreSql\\QueryBuilder\\Expression","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"OperatorCondition","namespace":"Flow\\PostgreSql\\QueryBuilder\\Condition","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBhIEpTT05CIGtleSBleGlzdHMgY29uZGl0aW9uICg\/KS4KICoKICogRXhhbXBsZToganNvbl9leGlzdHMoY29sKCdtZXRhZGF0YScpLCBsaXRlcmFsX3N0cmluZygnY2F0ZWdvcnknKSkKICogUHJvZHVjZXM6IG1ldGFkYXRhID8gJ2NhdGVnb3J5JwogKi8="},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/condition.php","start_line_in_file":464,"slug":"json-exists-any","name":"json_exists_any","namespace":"Flow\\PostgreSql\\DSL","parameters":[{"name":"expr","type":[{"name":"Expression","namespace":"Flow\\PostgreSql\\QueryBuilder\\Expression","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"keys","type":[{"name":"Expression","namespace":"Flow\\PostgreSql\\QueryBuilder\\Expression","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"OperatorCondition","namespace":"Flow\\PostgreSql\\QueryBuilder\\Condition","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBhIEpTT05CIGFueSBrZXkgZXhpc3RzIGNvbmRpdGlvbiAoP3wpLgogKgogKiBFeGFtcGxlOiBqc29uX2V4aXN0c19hbnkoY29sKCdtZXRhZGF0YScpLCBhcnJheV9leHByKFtsaXRlcmFsKCdjYXRlZ29yeScpLCBsaXRlcmFsKCduYW1lJyldKSkKICogUHJvZHVjZXM6IG1ldGFkYXRhID98IGFycmF5WydjYXRlZ29yeScsICduYW1lJ10KICov"},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/condition.php","start_line_in_file":480,"slug":"json-exists-all","name":"json_exists_all","namespace":"Flow\\PostgreSql\\DSL","parameters":[{"name":"expr","type":[{"name":"Expression","namespace":"Flow\\PostgreSql\\QueryBuilder\\Expression","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"keys","type":[{"name":"Expression","namespace":"Flow\\PostgreSql\\QueryBuilder\\Expression","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"OperatorCondition","namespace":"Flow\\PostgreSql\\QueryBuilder\\Condition","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBhIEpTT05CIGFsbCBrZXlzIGV4aXN0IGNvbmRpdGlvbiAoPyYpLgogKgogKiBFeGFtcGxlOiBqc29uX2V4aXN0c19hbGwoY29sKCdtZXRhZGF0YScpLCBhcnJheV9leHByKFtsaXRlcmFsKCdjYXRlZ29yeScpLCBsaXRlcmFsKCduYW1lJyldKSkKICogUHJvZHVjZXM6IG1ldGFkYXRhID8mIGFycmF5WydjYXRlZ29yeScsICduYW1lJ10KICov"},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/condition.php","start_line_in_file":496,"slug":"array-contains","name":"array_contains","namespace":"Flow\\PostgreSql\\DSL","parameters":[{"name":"left","type":[{"name":"Expression","namespace":"Flow\\PostgreSql\\QueryBuilder\\Expression","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"right","type":[{"name":"Expression","namespace":"Flow\\PostgreSql\\QueryBuilder\\Expression","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"OperatorCondition","namespace":"Flow\\PostgreSql\\QueryBuilder\\Condition","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBhbiBhcnJheSBjb250YWlucyBjb25kaXRpb24gKEA+KS4KICoKICogRXhhbXBsZTogYXJyYXlfY29udGFpbnMoY29sKCd0YWdzJyksIGFycmF5X2V4cHIoW2xpdGVyYWwoJ3NhbGUnKV0pKQogKiBQcm9kdWNlczogdGFncyBAPiBBUlJBWVsnc2FsZSddCiAqLw=="},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/condition.php","start_line_in_file":512,"slug":"array-contained-by","name":"array_contained_by","namespace":"Flow\\PostgreSql\\DSL","parameters":[{"name":"left","type":[{"name":"Expression","namespace":"Flow\\PostgreSql\\QueryBuilder\\Expression","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"right","type":[{"name":"Expression","namespace":"Flow\\PostgreSql\\QueryBuilder\\Expression","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"OperatorCondition","namespace":"Flow\\PostgreSql\\QueryBuilder\\Condition","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBhbiBhcnJheSBpcyBjb250YWluZWQgYnkgY29uZGl0aW9uICg8QCkuCiAqCiAqIEV4YW1wbGU6IGFycmF5X2NvbnRhaW5lZF9ieShjb2woJ3RhZ3MnKSwgYXJyYXlfZXhwcihbbGl0ZXJhbCgnc2FsZScpLCBsaXRlcmFsKCdmZWF0dXJlZCcpLCBsaXRlcmFsKCduZXcnKV0pKQogKiBQcm9kdWNlczogdGFncyA8QCBBUlJBWVsnc2FsZScsICdmZWF0dXJlZCcsICduZXcnXQogKi8="},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/condition.php","start_line_in_file":528,"slug":"array-overlap","name":"array_overlap","namespace":"Flow\\PostgreSql\\DSL","parameters":[{"name":"left","type":[{"name":"Expression","namespace":"Flow\\PostgreSql\\QueryBuilder\\Expression","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"right","type":[{"name":"Expression","namespace":"Flow\\PostgreSql\\QueryBuilder\\Expression","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"OperatorCondition","namespace":"Flow\\PostgreSql\\QueryBuilder\\Condition","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBhbiBhcnJheSBvdmVybGFwIGNvbmRpdGlvbiAoJiYpLgogKgogKiBFeGFtcGxlOiBhcnJheV9vdmVybGFwKGNvbCgndGFncycpLCBhcnJheV9leHByKFtsaXRlcmFsKCdzYWxlJyksIGxpdGVyYWwoJ2ZlYXR1cmVkJyldKSkKICogUHJvZHVjZXM6IHRhZ3MgJiYgQVJSQVlbJ3NhbGUnLCAnZmVhdHVyZWQnXQogKi8="},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/condition.php","start_line_in_file":546,"slug":"regex-match","name":"regex_match","namespace":"Flow\\PostgreSql\\DSL","parameters":[{"name":"expr","type":[{"name":"Expression","namespace":"Flow\\PostgreSql\\QueryBuilder\\Expression","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"pattern","type":[{"name":"Expression","namespace":"Flow\\PostgreSql\\QueryBuilder\\Expression","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"OperatorCondition","namespace":"Flow\\PostgreSql\\QueryBuilder\\Condition","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBhIFBPU0lYIHJlZ2V4IG1hdGNoIGNvbmRpdGlvbiAofikuCiAqIENhc2Utc2Vuc2l0aXZlLgogKgogKiBFeGFtcGxlOiByZWdleF9tYXRjaChjb2woJ2VtYWlsJyksIGxpdGVyYWxfc3RyaW5nKCcuKkBnbWFpbFxcLmNvbScpKQogKgogKiBQcm9kdWNlczogZW1haWwgfiAnLipAZ21haWxcLmNvbScKICov"},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/condition.php","start_line_in_file":564,"slug":"regex-imatch","name":"regex_imatch","namespace":"Flow\\PostgreSql\\DSL","parameters":[{"name":"expr","type":[{"name":"Expression","namespace":"Flow\\PostgreSql\\QueryBuilder\\Expression","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"pattern","type":[{"name":"Expression","namespace":"Flow\\PostgreSql\\QueryBuilder\\Expression","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"OperatorCondition","namespace":"Flow\\PostgreSql\\QueryBuilder\\Condition","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBhIFBPU0lYIHJlZ2V4IG1hdGNoIGNvbmRpdGlvbiAofiopLgogKiBDYXNlLWluc2Vuc2l0aXZlLgogKgogKiBFeGFtcGxlOiByZWdleF9pbWF0Y2goY29sKCdlbWFpbCcpLCBsaXRlcmFsX3N0cmluZygnLipAZ21haWxcXC5jb20nKSkKICoKICogUHJvZHVjZXM6IGVtYWlsIH4qICcuKkBnbWFpbFwuY29tJwogKi8="},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/condition.php","start_line_in_file":582,"slug":"not-regex-match","name":"not_regex_match","namespace":"Flow\\PostgreSql\\DSL","parameters":[{"name":"expr","type":[{"name":"Expression","namespace":"Flow\\PostgreSql\\QueryBuilder\\Expression","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"pattern","type":[{"name":"Expression","namespace":"Flow\\PostgreSql\\QueryBuilder\\Expression","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"OperatorCondition","namespace":"Flow\\PostgreSql\\QueryBuilder\\Condition","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBhIFBPU0lYIHJlZ2V4IG5vdCBtYXRjaCBjb25kaXRpb24gKCF+KS4KICogQ2FzZS1zZW5zaXRpdmUuCiAqCiAqIEV4YW1wbGU6IG5vdF9yZWdleF9tYXRjaChjb2woJ2VtYWlsJyksIGxpdGVyYWxfc3RyaW5nKCcuKkBzcGFtXFwuY29tJykpCiAqCiAqIFByb2R1Y2VzOiBlbWFpbCAhfiAnLipAc3BhbVwuY29tJwogKi8="},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/condition.php","start_line_in_file":600,"slug":"not-regex-imatch","name":"not_regex_imatch","namespace":"Flow\\PostgreSql\\DSL","parameters":[{"name":"expr","type":[{"name":"Expression","namespace":"Flow\\PostgreSql\\QueryBuilder\\Expression","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"pattern","type":[{"name":"Expression","namespace":"Flow\\PostgreSql\\QueryBuilder\\Expression","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"OperatorCondition","namespace":"Flow\\PostgreSql\\QueryBuilder\\Condition","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBhIFBPU0lYIHJlZ2V4IG5vdCBtYXRjaCBjb25kaXRpb24gKCF+KikuCiAqIENhc2UtaW5zZW5zaXRpdmUuCiAqCiAqIEV4YW1wbGU6IG5vdF9yZWdleF9pbWF0Y2goY29sKCdlbWFpbCcpLCBsaXRlcmFsX3N0cmluZygnLipAc3BhbVxcLmNvbScpKQogKgogKiBQcm9kdWNlczogZW1haWwgIX4qICcuKkBzcGFtXC5jb20nCiAqLw=="},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/condition.php","start_line_in_file":616,"slug":"text-search-match","name":"text_search_match","namespace":"Flow\\PostgreSql\\DSL","parameters":[{"name":"document","type":[{"name":"Expression","namespace":"Flow\\PostgreSql\\QueryBuilder\\Expression","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"query","type":[{"name":"Expression","namespace":"Flow\\PostgreSql\\QueryBuilder\\Expression","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"OperatorCondition","namespace":"Flow\\PostgreSql\\QueryBuilder\\Condition","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBhIGZ1bGwtdGV4dCBzZWFyY2ggbWF0Y2ggY29uZGl0aW9uIChAQCkuCiAqCiAqIEV4YW1wbGU6IHRleHRfc2VhcmNoX21hdGNoKGNvbCgnZG9jdW1lbnQnKSwgZnVuYygndG9fdHNxdWVyeScsIFtsaXRlcmFsKCdlbmdsaXNoJyksIGxpdGVyYWwoJ2hlbGxvICYgd29ybGQnKV0pKQogKiBQcm9kdWNlczogZG9jdW1lbnQgQEAgdG9fdHNxdWVyeSgnZW5nbGlzaCcsICdoZWxsbyAmIHdvcmxkJykKICov"},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/parser.php","start_line_in_file":33,"slug":"sql-parser","name":"sql_parser","namespace":"Flow\\PostgreSql\\DSL","parameters":[],"return_type":[{"name":"Parser","namespace":"Flow\\PostgreSql","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/parser.php","start_line_in_file":39,"slug":"sql-parse","name":"sql_parse","namespace":"Flow\\PostgreSql\\DSL","parameters":[{"name":"sql","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"ParsedQuery","namespace":"Flow\\PostgreSql","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/parser.php","start_line_in_file":49,"slug":"sql-fingerprint","name":"sql_fingerprint","namespace":"Flow\\PostgreSql\\DSL","parameters":[{"name":"sql","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"string","namespace":null,"is_nullable":true,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIFJldHVybnMgYSBmaW5nZXJwcmludCBvZiB0aGUgZ2l2ZW4gU1FMIHF1ZXJ5LgogKiBMaXRlcmFsIHZhbHVlcyBhcmUgbm9ybWFsaXplZCBzbyB0aGV5IHdvbid0IGFmZmVjdCB0aGUgZmluZ2VycHJpbnQuCiAqLw=="},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/parser.php","start_line_in_file":60,"slug":"sql-normalize","name":"sql_normalize","namespace":"Flow\\PostgreSql\\DSL","parameters":[{"name":"sql","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"string","namespace":null,"is_nullable":true,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIE5vcm1hbGl6ZSBTUUwgcXVlcnkgYnkgcmVwbGFjaW5nIGxpdGVyYWwgdmFsdWVzIGFuZCBuYW1lZCBwYXJhbWV0ZXJzIHdpdGggcG9zaXRpb25hbCBwYXJhbWV0ZXJzLgogKiBXSEVSRSBpZCA9IDppZCB3aWxsIGJlIGNoYW5nZWQgaW50byBXSEVSRSBpZCA9ICQxCiAqIFdIRVJFIGlkID0gMSB3aWxsIGJlIGNoYW5nZWQgaW50byBXSEVSRSBpZCA9ICQxLgogKi8="},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/parser.php","start_line_in_file":70,"slug":"sql-normalize-utility","name":"sql_normalize_utility","namespace":"Flow\\PostgreSql\\DSL","parameters":[{"name":"sql","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"string","namespace":null,"is_nullable":true,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIE5vcm1hbGl6ZSB1dGlsaXR5IFNRTCBzdGF0ZW1lbnRzIChEREwgbGlrZSBDUkVBVEUsIEFMVEVSLCBEUk9QKS4KICogVGhpcyBoYW5kbGVzIERETCBzdGF0ZW1lbnRzIGRpZmZlcmVudGx5IGZyb20gcGdfbm9ybWFsaXplKCkgd2hpY2ggaXMgb3B0aW1pemVkIGZvciBETUwuCiAqLw=="},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/parser.php","start_line_in_file":81,"slug":"sql-split","name":"sql_split","namespace":"Flow\\PostgreSql\\DSL","parameters":[{"name":"sql","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"array","namespace":null,"is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIFNwbGl0IHN0cmluZyB3aXRoIG11bHRpcGxlIFNRTCBzdGF0ZW1lbnRzIGludG8gYXJyYXkgb2YgaW5kaXZpZHVhbCBzdGF0ZW1lbnRzLgogKgogKiBAcmV0dXJuIGFycmF5PHN0cmluZz4KICov"},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/parser.php","start_line_in_file":90,"slug":"sql-deparse-options","name":"sql_deparse_options","namespace":"Flow\\PostgreSql\\DSL","parameters":[],"return_type":[{"name":"DeparseOptions","namespace":"Flow\\PostgreSql","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBEZXBhcnNlT3B0aW9ucyBmb3IgY29uZmlndXJpbmcgU1FMIGZvcm1hdHRpbmcuCiAqLw=="},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/parser.php","start_line_in_file":104,"slug":"sql-deparse","name":"sql_deparse","namespace":"Flow\\PostgreSql\\DSL","parameters":[{"name":"query","type":[{"name":"ParsedQuery","namespace":"Flow\\PostgreSql","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"options","type":[{"name":"DeparseOptions","namespace":"Flow\\PostgreSql","is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"null"}],"return_type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENvbnZlcnQgYSBQYXJzZWRRdWVyeSBBU1QgYmFjayB0byBTUUwgc3RyaW5nLgogKgogKiBXaGVuIGNhbGxlZCB3aXRob3V0IG9wdGlvbnMsIHJldHVybnMgdGhlIFNRTCBhcyBhIHNpbXBsZSBzdHJpbmcuCiAqIFdoZW4gY2FsbGVkIHdpdGggRGVwYXJzZU9wdGlvbnMsIGFwcGxpZXMgZm9ybWF0dGluZyAocHJldHR5LXByaW50aW5nLCBpbmRlbnRhdGlvbiwgZXRjLikuCiAqCiAqIEB0aHJvd3MgXFJ1bnRpbWVFeGNlcHRpb24gaWYgZGVwYXJzaW5nIGZhaWxzCiAqLw=="},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/parser.php","start_line_in_file":120,"slug":"sql-format","name":"sql_format","namespace":"Flow\\PostgreSql\\DSL","parameters":[{"name":"sql","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"options","type":[{"name":"DeparseOptions","namespace":"Flow\\PostgreSql","is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"null"}],"return_type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIFBhcnNlIGFuZCBmb3JtYXQgU1FMIHF1ZXJ5IHdpdGggcHJldHR5IHByaW50aW5nLgogKgogKiBUaGlzIGlzIGEgY29udmVuaWVuY2UgZnVuY3Rpb24gdGhhdCBwYXJzZXMgU1FMIGFuZCByZXR1cm5zIGl0IGZvcm1hdHRlZC4KICoKICogQHBhcmFtIHN0cmluZyAkc3FsIFRoZSBTUUwgcXVlcnkgdG8gZm9ybWF0CiAqIEBwYXJhbSBudWxsfERlcGFyc2VPcHRpb25zICRvcHRpb25zIEZvcm1hdHRpbmcgb3B0aW9ucyAoZGVmYXVsdHMgdG8gcHJldHR5LXByaW50IGVuYWJsZWQpCiAqCiAqIEB0aHJvd3MgXFJ1bnRpbWVFeGNlcHRpb24gaWYgcGFyc2luZyBvciBkZXBhcnNpbmcgZmFpbHMKICov"},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/parser.php","start_line_in_file":132,"slug":"sql-summary","name":"sql_summary","namespace":"Flow\\PostgreSql\\DSL","parameters":[{"name":"sql","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"options","type":[{"name":"int","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"0"},{"name":"truncateLimit","type":[{"name":"int","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"0"}],"return_type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIEdlbmVyYXRlIGEgc3VtbWFyeSBvZiBwYXJzZWQgcXVlcmllcyBpbiBwcm90b2J1ZiBmb3JtYXQuCiAqIFVzZWZ1bCBmb3IgcXVlcnkgbW9uaXRvcmluZyBhbmQgbG9nZ2luZyB3aXRob3V0IGZ1bGwgQVNUIG92ZXJoZWFkLgogKi8="},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/parser.php","start_line_in_file":147,"slug":"sql-to-paginated-query","name":"sql_to_paginated_query","namespace":"Flow\\PostgreSql\\DSL","parameters":[{"name":"sql","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"limit","type":[{"name":"int","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"offset","type":[{"name":"int","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"0"}],"return_type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIFRyYW5zZm9ybSBhIFNRTCBxdWVyeSBpbnRvIGEgcGFnaW5hdGVkIHF1ZXJ5IHdpdGggTElNSVQgYW5kIE9GRlNFVC4KICoKICogQHBhcmFtIHN0cmluZyAkc3FsIFRoZSBTUUwgcXVlcnkgdG8gcGFnaW5hdGUKICogQHBhcmFtIGludCAkbGltaXQgTWF4aW11bSBudW1iZXIgb2Ygcm93cyB0byByZXR1cm4KICogQHBhcmFtIGludCAkb2Zmc2V0IE51bWJlciBvZiByb3dzIHRvIHNraXAgKHJlcXVpcmVzIE9SREVSIEJZIGluIHF1ZXJ5KQogKgogKiBAcmV0dXJuIHN0cmluZyBUaGUgcGFnaW5hdGVkIFNRTCBxdWVyeQogKi8="},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/parser.php","start_line_in_file":164,"slug":"sql-to-limited-query","name":"sql_to_limited_query","namespace":"Flow\\PostgreSql\\DSL","parameters":[{"name":"sql","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"limit","type":[{"name":"int","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIFRyYW5zZm9ybSBhIFNRTCBxdWVyeSB0byBsaW1pdCByZXN1bHRzIHRvIGEgc3BlY2lmaWMgbnVtYmVyIG9mIHJvd3MuCiAqCiAqIEBwYXJhbSBzdHJpbmcgJHNxbCBUaGUgU1FMIHF1ZXJ5IHRvIGxpbWl0CiAqIEBwYXJhbSBpbnQgJGxpbWl0IE1heGltdW0gbnVtYmVyIG9mIHJvd3MgdG8gcmV0dXJuCiAqCiAqIEByZXR1cm4gc3RyaW5nIFRoZSBsaW1pdGVkIFNRTCBxdWVyeQogKi8="},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/parser.php","start_line_in_file":183,"slug":"sql-to-count-query","name":"sql_to_count_query","namespace":"Flow\\PostgreSql\\DSL","parameters":[{"name":"sql","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIFRyYW5zZm9ybSBhIFNRTCBxdWVyeSBpbnRvIGEgQ09VTlQgcXVlcnkgZm9yIHBhZ2luYXRpb24uCiAqCiAqIFdyYXBzIHRoZSBxdWVyeSBpbjogU0VMRUNUIENPVU5UKCopIEZST00gKC4uLikgQVMgX2NvdW50X3N1YnEKICogUmVtb3ZlcyBPUkRFUiBCWSBhbmQgTElNSVQvT0ZGU0VUIGZyb20gdGhlIGlubmVyIHF1ZXJ5LgogKgogKiBAcGFyYW0gc3RyaW5nICRzcWwgVGhlIFNRTCBxdWVyeSB0byB0cmFuc2Zvcm0KICoKICogQHJldHVybiBzdHJpbmcgVGhlIENPVU5UIHF1ZXJ5CiAqLw=="},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/parser.php","start_line_in_file":205,"slug":"sql-to-keyset-query","name":"sql_to_keyset_query","namespace":"Flow\\PostgreSql\\DSL","parameters":[{"name":"sql","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"limit","type":[{"name":"int","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"columns","type":[{"name":"array","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"cursor","type":[{"name":"array","namespace":null,"is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"null"}],"return_type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIFRyYW5zZm9ybSBhIFNRTCBxdWVyeSBpbnRvIGEga2V5c2V0IChjdXJzb3ItYmFzZWQpIHBhZ2luYXRlZCBxdWVyeS4KICoKICogTW9yZSBlZmZpY2llbnQgdGhhbiBPRkZTRVQgZm9yIGxhcmdlIGRhdGFzZXRzIC0gdXNlcyBpbmRleGVkIFdIRVJFIGNvbmRpdGlvbnMuCiAqIEF1dG9tYXRpY2FsbHkgZGV0ZWN0cyBleGlzdGluZyBxdWVyeSBwYXJhbWV0ZXJzIGFuZCBhcHBlbmRzIGtleXNldCBwbGFjZWhvbGRlcnMgYXQgdGhlIGVuZC4KICoKICogQHBhcmFtIHN0cmluZyAkc3FsIFRoZSBTUUwgcXVlcnkgdG8gcGFnaW5hdGUgKG11c3QgaGF2ZSBPUkRFUiBCWSkKICogQHBhcmFtIGludCAkbGltaXQgTWF4aW11bSBudW1iZXIgb2Ygcm93cyB0byByZXR1cm4KICogQHBhcmFtIGxpc3Q8S2V5c2V0Q29sdW1uPiAkY29sdW1ucyBDb2x1bW5zIGZvciBrZXlzZXQgcGFnaW5hdGlvbiAobXVzdCBtYXRjaCBPUkRFUiBCWSkKICogQHBhcmFtIG51bGx8bGlzdDxudWxsfGJvb2x8ZmxvYXR8aW50fHN0cmluZz4gJGN1cnNvciBWYWx1ZXMgZnJvbSBsYXN0IHJvdyBvZiBwcmV2aW91cyBwYWdlIChudWxsIGZvciBmaXJzdCBwYWdlKQogKgogKiBAcmV0dXJuIHN0cmluZyBUaGUgcGFnaW5hdGVkIFNRTCBxdWVyeQogKi8="},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/parser.php","start_line_in_file":220,"slug":"sql-keyset-column","name":"sql_keyset_column","namespace":"Flow\\PostgreSql\\DSL","parameters":[{"name":"column","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"order","type":[{"name":"SortOrder","namespace":"Flow\\PostgreSql\\AST\\Transformers","is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"Flow\\PostgreSql\\AST\\Transformers\\SortOrder::..."}],"return_type":[{"name":"KeysetColumn","namespace":"Flow\\PostgreSql\\AST\\Transformers","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBhIEtleXNldENvbHVtbiBmb3Iga2V5c2V0IHBhZ2luYXRpb24uCiAqCiAqIEBwYXJhbSBzdHJpbmcgJGNvbHVtbiBDb2x1bW4gbmFtZSAoY2FuIGluY2x1ZGUgdGFibGUgYWxpYXMgbGlrZSAidS5pZCIpCiAqIEBwYXJhbSBTb3J0T3JkZXIgJG9yZGVyIFNvcnQgb3JkZXIgKEFTQyBvciBERVNDKQogKi8="},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/parser.php","start_line_in_file":229,"slug":"sql-query-columns","name":"sql_query_columns","namespace":"Flow\\PostgreSql\\DSL","parameters":[{"name":"query","type":[{"name":"ParsedQuery","namespace":"Flow\\PostgreSql","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"Columns","namespace":"Flow\\PostgreSql\\Extractors","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIEV4dHJhY3QgY29sdW1ucyBmcm9tIGEgcGFyc2VkIFNRTCBxdWVyeS4KICov"},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/parser.php","start_line_in_file":238,"slug":"sql-query-tables","name":"sql_query_tables","namespace":"Flow\\PostgreSql\\DSL","parameters":[{"name":"query","type":[{"name":"ParsedQuery","namespace":"Flow\\PostgreSql","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"Tables","namespace":"Flow\\PostgreSql\\Extractors","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIEV4dHJhY3QgdGFibGVzIGZyb20gYSBwYXJzZWQgU1FMIHF1ZXJ5LgogKi8="},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/parser.php","start_line_in_file":247,"slug":"sql-query-functions","name":"sql_query_functions","namespace":"Flow\\PostgreSql\\DSL","parameters":[{"name":"query","type":[{"name":"ParsedQuery","namespace":"Flow\\PostgreSql","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"Functions","namespace":"Flow\\PostgreSql\\Extractors","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIEV4dHJhY3QgZnVuY3Rpb25zIGZyb20gYSBwYXJzZWQgU1FMIHF1ZXJ5LgogKi8="},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/parser.php","start_line_in_file":256,"slug":"sql-query-order-by","name":"sql_query_order_by","namespace":"Flow\\PostgreSql\\DSL","parameters":[{"name":"query","type":[{"name":"ParsedQuery","namespace":"Flow\\PostgreSql","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"OrderBy","namespace":"Flow\\PostgreSql\\Extractors","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIEV4dHJhY3QgT1JERVIgQlkgY2xhdXNlcyBmcm9tIGEgcGFyc2VkIFNRTCBxdWVyeS4KICov"},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/parser.php","start_line_in_file":270,"slug":"sql-query-depth","name":"sql_query_depth","namespace":"Flow\\PostgreSql\\DSL","parameters":[{"name":"sql","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"int","namespace":null,"is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIEdldCB0aGUgbWF4aW11bSBuZXN0aW5nIGRlcHRoIG9mIGEgU1FMIHF1ZXJ5LgogKgogKiBFeGFtcGxlOgogKiAtICJTRUxFQ1QgKiBGUk9NIHQiID0+IDEKICogLSAiU0VMRUNUICogRlJPTSAoU0VMRUNUICogRlJPTSB0KSIgPT4gMgogKiAtICJTRUxFQ1QgKiBGUk9NIChTRUxFQ1QgKiBGUk9NIChTRUxFQ1QgKiBGUk9NIHQpKSIgPT4gMwogKi8="},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/parser.php","start_line_in_file":287,"slug":"sql-to-explain","name":"sql_to_explain","namespace":"Flow\\PostgreSql\\DSL","parameters":[{"name":"sql","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"config","type":[{"name":"ExplainConfig","namespace":"Flow\\PostgreSql\\AST\\Transformers","is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"null"}],"return_type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIFRyYW5zZm9ybSBhIFNRTCBxdWVyeSBpbnRvIGFuIEVYUExBSU4gcXVlcnkuCiAqCiAqIFJldHVybnMgdGhlIG1vZGlmaWVkIFNRTCB3aXRoIEVYUExBSU4gd3JhcHBlZCBhcm91bmQgaXQuCiAqIERlZmF1bHRzIHRvIEVYUExBSU4gQU5BTFlaRSB3aXRoIEpTT04gZm9ybWF0IGZvciBlYXN5IHBhcnNpbmcuCiAqCiAqIEBwYXJhbSBzdHJpbmcgJHNxbCBUaGUgU1FMIHF1ZXJ5IHRvIGV4cGxhaW4KICogQHBhcmFtIG51bGx8RXhwbGFpbkNvbmZpZyAkY29uZmlnIEVYUExBSU4gY29uZmlndXJhdGlvbiAoZGVmYXVsdHMgdG8gZm9yQW5hbHlzaXMoKSkKICoKICogQHJldHVybiBzdHJpbmcgVGhlIEVYUExBSU4gcXVlcnkKICov"},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/parser.php","start_line_in_file":307,"slug":"sql-explain-config","name":"sql_explain_config","namespace":"Flow\\PostgreSql\\DSL","parameters":[{"name":"analyze","type":[{"name":"bool","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"true"},{"name":"verbose","type":[{"name":"bool","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"false"},{"name":"costs","type":[{"name":"bool","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"true"},{"name":"buffers","type":[{"name":"bool","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"true"},{"name":"timing","type":[{"name":"bool","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"true"},{"name":"format","type":[{"name":"ExplainFormat","namespace":"Flow\\PostgreSql\\QueryBuilder\\Utility","is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"Flow\\PostgreSql\\QueryBuilder\\Utility\\ExplainFormat::..."}],"return_type":[{"name":"ExplainConfig","namespace":"Flow\\PostgreSql\\AST\\Transformers","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBhbiBFeHBsYWluQ29uZmlnIGZvciBjdXN0b21pemluZyBFWFBMQUlOIG9wdGlvbnMuCiAqCiAqIEBwYXJhbSBib29sICRhbmFseXplIFdoZXRoZXIgdG8gYWN0dWFsbHkgZXhlY3V0ZSB0aGUgcXVlcnkgKEFOQUxZWkUpCiAqIEBwYXJhbSBib29sICR2ZXJib3NlIEluY2x1ZGUgdmVyYm9zZSBvdXRwdXQKICogQHBhcmFtIGJvb2wgJGNvc3RzIEluY2x1ZGUgY29zdCBlc3RpbWF0ZXMgKGRlZmF1bHQgdHJ1ZSkKICogQHBhcmFtIGJvb2wgJGJ1ZmZlcnMgSW5jbHVkZSBidWZmZXIgdXNhZ2Ugc3RhdGlzdGljcyAocmVxdWlyZXMgYW5hbHl6ZSkKICogQHBhcmFtIGJvb2wgJHRpbWluZyBJbmNsdWRlIHRpbWluZyBpbmZvcm1hdGlvbiAocmVxdWlyZXMgYW5hbHl6ZSkKICogQHBhcmFtIEV4cGxhaW5Gb3JtYXQgJGZvcm1hdCBPdXRwdXQgZm9ybWF0IChKU09OIHJlY29tbWVuZGVkIGZvciBwYXJzaW5nKQogKi8="},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/parser.php","start_line_in_file":329,"slug":"sql-explain-modifier","name":"sql_explain_modifier","namespace":"Flow\\PostgreSql\\DSL","parameters":[{"name":"config","type":[{"name":"ExplainConfig","namespace":"Flow\\PostgreSql\\AST\\Transformers","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"ExplainModifier","namespace":"Flow\\PostgreSql\\AST\\Transformers","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBhbiBFeHBsYWluTW9kaWZpZXIgZm9yIHRyYW5zZm9ybWluZyBxdWVyaWVzIGludG8gRVhQTEFJTiBxdWVyaWVzLgogKi8="},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/parser.php","start_line_in_file":342,"slug":"sql-explain-parse","name":"sql_explain_parse","namespace":"Flow\\PostgreSql\\DSL","parameters":[{"name":"jsonOutput","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"Plan","namespace":"Flow\\PostgreSql\\Explain\\Plan","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIFBhcnNlIEVYUExBSU4gSlNPTiBvdXRwdXQgaW50byBhIFBsYW4gb2JqZWN0LgogKgogKiBAcGFyYW0gc3RyaW5nICRqc29uT3V0cHV0IFRoZSBKU09OIG91dHB1dCBmcm9tIEVYUExBSU4gKEZPUk1BVCBKU09OKQogKgogKiBAcmV0dXJuIFBsYW4gVGhlIHBhcnNlZCBleGVjdXRpb24gcGxhbgogKi8="},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/parser.php","start_line_in_file":355,"slug":"sql-analyze","name":"sql_analyze","namespace":"Flow\\PostgreSql\\DSL","parameters":[{"name":"plan","type":[{"name":"Plan","namespace":"Flow\\PostgreSql\\Explain\\Plan","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"PlanAnalyzer","namespace":"Flow\\PostgreSql\\Explain\\Analyzer","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBhIHBsYW4gYW5hbHl6ZXIgZm9yIGFuYWx5emluZyBFWFBMQUlOIHBsYW5zLgogKgogKiBAcGFyYW0gUGxhbiAkcGxhbiBUaGUgZXhlY3V0aW9uIHBsYW4gdG8gYW5hbHl6ZQogKgogKiBAcmV0dXJuIFBsYW5BbmFseXplciBUaGUgYW5hbHl6ZXIgZm9yIGV4dHJhY3RpbmcgaW5zaWdodHMKICov"},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/client.php","start_line_in_file":46,"slug":"pgsql-connection","name":"pgsql_connection","namespace":"Flow\\PostgreSql\\DSL","parameters":[{"name":"connectionString","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"ConnectionParameters","namespace":"Flow\\PostgreSql\\Client","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBjb25uZWN0aW9uIHBhcmFtZXRlcnMgZnJvbSBhIGNvbm5lY3Rpb24gc3RyaW5nLgogKgogKiBBY2NlcHRzIGxpYnBxLXN0eWxlIGNvbm5lY3Rpb24gc3RyaW5nczoKICogLSBLZXktdmFsdWUgZm9ybWF0OiAiaG9zdD1sb2NhbGhvc3QgcG9ydD01NDMyIGRibmFtZT1teWRiIHVzZXI9bXl1c2VyIHBhc3N3b3JkPXNlY3JldCIKICogLSBVUkkgZm9ybWF0OiAicG9zdGdyZXNxbDovL3VzZXI6cGFzc3dvcmRAbG9jYWxob3N0OjU0MzIvZGJuYW1lIgogKgogKiBAZXhhbXBsZQogKiAkcGFyYW1zID0gcGdzcWxfY29ubmVjdGlvbignaG9zdD1sb2NhbGhvc3QgZGJuYW1lPW15ZGInKTsKICogJHBhcmFtcyA9IHBnc3FsX2Nvbm5lY3Rpb24oJ3Bvc3RncmVzcWw6Ly91c2VyOnBhc3NAbG9jYWxob3N0L215ZGInKTsKICov"},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/client.php","start_line_in_file":68,"slug":"pgsql-connection-dsn","name":"pgsql_connection_dsn","namespace":"Flow\\PostgreSql\\DSL","parameters":[{"name":"dsn","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"ConnectionParameters","namespace":"Flow\\PostgreSql\\Client","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBjb25uZWN0aW9uIHBhcmFtZXRlcnMgZnJvbSBhIERTTiBzdHJpbmcuCiAqCiAqIFBhcnNlcyBzdGFuZGFyZCBQb3N0Z3JlU1FMIERTTiBmb3JtYXQgY29tbW9ubHkgdXNlZCBpbiBlbnZpcm9ubWVudCB2YXJpYWJsZXMKICogKGUuZy4sIERBVEFCQVNFX1VSTCkuIFN1cHBvcnRzIHBvc3RncmVzOi8vLCBwb3N0Z3Jlc3FsOi8vLCBhbmQgcGdzcWw6Ly8gc2NoZW1lcy4KICoKICogQHBhcmFtIHN0cmluZyAkZHNuIERTTiBzdHJpbmcgaW4gZm9ybWF0OiBwb3N0Z3JlczovL3VzZXI6cGFzc3dvcmRAaG9zdDpwb3J0L2RhdGFiYXNlP29wdGlvbnMKICoKICogQHRocm93cyBDbGllbnRcRHNuUGFyc2VyRXhjZXB0aW9uIElmIHRoZSBEU04gY2Fubm90IGJlIHBhcnNlZAogKgogKiBAZXhhbXBsZQogKiAkcGFyYW1zID0gcGdzcWxfY29ubmVjdGlvbl9kc24oJ3Bvc3RncmVzOi8vbXl1c2VyOnNlY3JldEBsb2NhbGhvc3Q6NTQzMi9teWRiJyk7CiAqICRwYXJhbXMgPSBwZ3NxbF9jb25uZWN0aW9uX2RzbigncG9zdGdyZXNxbDovL3VzZXI6cGFzc0BkYi5leGFtcGxlLmNvbS9hcHA\/c3NsbW9kZT1yZXF1aXJlJyk7CiAqICRwYXJhbXMgPSBwZ3NxbF9jb25uZWN0aW9uX2RzbigncGdzcWw6Ly91c2VyOnBhc3NAbG9jYWxob3N0L215ZGInKTsgLy8gU3ltZm9ueS9Eb2N0cmluZSBmb3JtYXQKICogJHBhcmFtcyA9IHBnc3FsX2Nvbm5lY3Rpb25fZHNuKGdldGVudignREFUQUJBU0VfVVJMJykpOwogKi8="},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/client.php","start_line_in_file":95,"slug":"pgsql-connection-params","name":"pgsql_connection_params","namespace":"Flow\\PostgreSql\\DSL","parameters":[{"name":"database","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"host","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"'localhost'"},{"name":"port","type":[{"name":"int","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"5432"},{"name":"user","type":[{"name":"string","namespace":null,"is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"null"},{"name":"password","type":[{"name":"string","namespace":null,"is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"null"},{"name":"options","type":[{"name":"array","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"[]"}],"return_type":[{"name":"ConnectionParameters","namespace":"Flow\\PostgreSql\\Client","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBjb25uZWN0aW9uIHBhcmFtZXRlcnMgZnJvbSBpbmRpdmlkdWFsIHZhbHVlcy4KICoKICogQWxsb3dzIHNwZWNpZnlpbmcgY29ubmVjdGlvbiBwYXJhbWV0ZXJzIGluZGl2aWR1YWxseSBmb3IgYmV0dGVyIHR5cGUgc2FmZXR5CiAqIGFuZCBJREUgc3VwcG9ydC4KICoKICogQHBhcmFtIHN0cmluZyAkZGF0YWJhc2UgRGF0YWJhc2UgbmFtZSAocmVxdWlyZWQpCiAqIEBwYXJhbSBzdHJpbmcgJGhvc3QgSG9zdG5hbWUgKGRlZmF1bHQ6IGxvY2FsaG9zdCkKICogQHBhcmFtIGludCAkcG9ydCBQb3J0IG51bWJlciAoZGVmYXVsdDogNTQzMikKICogQHBhcmFtIG51bGx8c3RyaW5nICR1c2VyIFVzZXJuYW1lIChvcHRpb25hbCkKICogQHBhcmFtIG51bGx8c3RyaW5nICRwYXNzd29yZCBQYXNzd29yZCAob3B0aW9uYWwpCiAqIEBwYXJhbSBhcnJheTxzdHJpbmcsIHN0cmluZz4gJG9wdGlvbnMgQWRkaXRpb25hbCBsaWJwcSBvcHRpb25zCiAqCiAqIEBleGFtcGxlCiAqICRwYXJhbXMgPSBwZ3NxbF9jb25uZWN0aW9uX3BhcmFtcygKICogICAgIGRhdGFiYXNlOiAnbXlkYicsCiAqICAgICBob3N0OiAnbG9jYWxob3N0JywKICogICAgIHVzZXI6ICdteXVzZXInLAogKiAgICAgcGFzc3dvcmQ6ICdzZWNyZXQnLAogKiApOwogKi8="},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/client.php","start_line_in_file":126,"slug":"pgsql-client","name":"pgsql_client","namespace":"Flow\\PostgreSql\\DSL","parameters":[{"name":"params","type":[{"name":"ConnectionParameters","namespace":"Flow\\PostgreSql\\Client","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"valueConverters","type":[{"name":"ValueConverters","namespace":"Flow\\PostgreSql\\Client\\Types","is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"null"},{"name":"context","type":[{"name":"Context","namespace":"Flow\\PostgreSql\\Client","is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"null"}],"return_type":[{"name":"Client","namespace":"Flow\\PostgreSql\\Client","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBhIFBvc3RncmVTUUwgY2xpZW50IHVzaW5nIGV4dC1wZ3NxbC4KICoKICogVGhlIGNsaWVudCBjb25uZWN0cyBpbW1lZGlhdGVseSBhbmQgaXMgcmVhZHkgdG8gZXhlY3V0ZSBxdWVyaWVzLgogKgogKiBAcGFyYW0gQ2xpZW50XENvbm5lY3Rpb25QYXJhbWV0ZXJzICRwYXJhbXMgQ29ubmVjdGlvbiBwYXJhbWV0ZXJzCiAqIEBwYXJhbSBudWxsfFZhbHVlQ29udmVydGVycyAkdmFsdWVDb252ZXJ0ZXJzIEN1c3RvbSB0eXBlIGNvbnZlcnRlcnMgKG9wdGlvbmFsKQogKiBAcGFyYW0gbnVsbHxDb250ZXh0ICRjb250ZXh0IEJhc2UgbWFwcGVyIENvbnRleHQg4oCUIHRoZSBDbGllbnQgZW5yaWNoZXMgaXQgd2l0aCBzcWwvcGFyYW1ldGVycy9zZWxmIHBlciBxdWVyeSBiZWZvcmUgaGFuZGluZyBpdCB0byBSb3dNYXBwZXI6Om1hcCgpCiAqCiAqIEB0aHJvd3MgQ29ubmVjdGlvbkV4Y2VwdGlvbiBJZiBjb25uZWN0aW9uIGZhaWxzCiAqLw=="},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/client.php","start_line_in_file":143,"slug":"postgresql-context","name":"postgresql_context","namespace":"Flow\\PostgreSql\\DSL","parameters":[{"name":"data","type":[{"name":"array","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"[]"},{"name":"catalog","type":[{"name":"Catalog","namespace":"Flow\\PostgreSql\\Schema","is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"null"}],"return_type":[{"name":"Context","namespace":"Flow\\PostgreSql\\Client","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBhIFJvd01hcHBlciBDb250ZXh0IHNlZWRlZCB3aXRoIHVzZXItc3VwcGxpZWQga2V5L3ZhbHVlIGRhdGEgYW5kIGFuIG9wdGlvbmFsIENhdGFsb2cuCiAqCiAqIFRoZSBDb250ZXh0IGlzIGxhdGVyIGVucmljaGVkIHdpdGggYSBRdWVyeSAoc3FsICsgcGFyYW1ldGVycykgYW5kIHRoZSBleGVjdXRpbmcgQ2xpZW50IGJ5IHRoZQogKiBQb3N0Z3JlU1FMIENsaWVudCBiZWZvcmUgYmVpbmcgaGFuZGVkIHRvIFJvd01hcHBlcjo6bWFwKCkuCiAqCiAqIEBwYXJhbSBhcnJheTxzdHJpbmcsIG1peGVkPiAkZGF0YSBVc2VyLXN1cHBsaWVkIGtleS92YWx1ZSBwYWlycwogKi8="},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/client.php","start_line_in_file":176,"slug":"postgresql-telemetry-options","name":"postgresql_telemetry_options","namespace":"Flow\\PostgreSql\\DSL","parameters":[{"name":"traceQueries","type":[{"name":"bool","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"true"},{"name":"transactionSpans","type":[{"name":"TransactionSpanMode","namespace":"Flow\\PostgreSql\\Client\\Telemetry","is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"Flow\\PostgreSql\\Client\\Telemetry\\TransactionSpanMode::..."},{"name":"collectMetrics","type":[{"name":"bool","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"true"},{"name":"logQueries","type":[{"name":"bool","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"false"},{"name":"maxQueryLength","type":[{"name":"int","namespace":null,"is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"1000"},{"name":"includeParameters","type":[{"name":"bool","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"false"},{"name":"maxParameters","type":[{"name":"int","namespace":null,"is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"10"},{"name":"maxParameterLength","type":[{"name":"int","namespace":null,"is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"100"}],"return_type":[{"name":"PostgreSqlTelemetryOptions","namespace":"Flow\\PostgreSql\\Client\\Telemetry","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSB0ZWxlbWV0cnkgb3B0aW9ucyBmb3IgUG9zdGdyZVNRTCBjbGllbnQgaW5zdHJ1bWVudGF0aW9uLgogKgogKiBDb250cm9scyB3aGljaCB0ZWxlbWV0cnkgc2lnbmFscyAodHJhY2VzLCBtZXRyaWNzLCBsb2dzKSBhcmUgZW5hYmxlZAogKiBhbmQgaG93IHF1ZXJ5IGluZm9ybWF0aW9uIGlzIGNhcHR1cmVkLgogKgogKiBAcGFyYW0gYm9vbCAkdHJhY2VRdWVyaWVzIENyZWF0ZSBzcGFucyBmb3IgcXVlcnkgZXhlY3V0aW9uIChkZWZhdWx0OiB0cnVlKQogKiBAcGFyYW0gVHJhbnNhY3Rpb25TcGFuTW9kZSAkdHJhbnNhY3Rpb25TcGFucyBIb3cgdHJhbnNhY3Rpb25zIGFyZSB0cmFjZWQ6IEdST1VQRUQgKGRlZmF1bHQpLCBQRVJfT1BFUkFUSU9OIG9yIE9GRgogKiBAcGFyYW0gYm9vbCAkY29sbGVjdE1ldHJpY3MgQ29sbGVjdCBkdXJhdGlvbiBhbmQgcm93IGNvdW50IG1ldHJpY3MgKGRlZmF1bHQ6IHRydWUpCiAqIEBwYXJhbSBib29sICRsb2dRdWVyaWVzIExvZyBleGVjdXRlZCBxdWVyaWVzIChkZWZhdWx0OiBmYWxzZSkKICogQHBhcmFtIG51bGx8aW50ICRtYXhRdWVyeUxlbmd0aCBNYXhpbXVtIHF1ZXJ5IHRleHQgbGVuZ3RoIGluIHRlbGVtZXRyeSAoZGVmYXVsdDogMTAwMCwgbnVsbCA9IHVubGltaXRlZCkKICogQHBhcmFtIGJvb2wgJGluY2x1ZGVQYXJhbWV0ZXJzIEluY2x1ZGUgcXVlcnkgcGFyYW1ldGVycyBpbiB0ZWxlbWV0cnkgKGRlZmF1bHQ6IGZhbHNlLCBzZWN1cml0eSBjb25zaWRlcmF0aW9uKQogKgogKiBAZXhhbXBsZQogKiAvLyBEZWZhdWx0IG9wdGlvbnMgKHRyYWNlcyBhbmQgbWV0cmljcyBlbmFibGVkKQogKiAkb3B0aW9ucyA9IHBvc3RncmVzcWxfdGVsZW1ldHJ5X29wdGlvbnMoKTsKICoKICogLy8gRW5hYmxlIHF1ZXJ5IGxvZ2dpbmcKICogJG9wdGlvbnMgPSBwb3N0Z3Jlc3FsX3RlbGVtZXRyeV9vcHRpb25zKGxvZ1F1ZXJpZXM6IHRydWUpOwogKgogKiAvLyBNZXRyaWNzIG9ubHksIG5vIHNwYW5zCiAqICRvcHRpb25zID0gcG9zdGdyZXNxbF90ZWxlbWV0cnlfb3B0aW9ucygKICogICAgIHRyYWNlUXVlcmllczogZmFsc2UsCiAqICAgICB0cmFuc2FjdGlvblNwYW5zOiBUcmFuc2FjdGlvblNwYW5Nb2RlOjpPRkYsCiAqICAgICBjb2xsZWN0TWV0cmljczogdHJ1ZSwKICogKTsKICov"},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/client.php","start_line_in_file":214,"slug":"postgresql-telemetry-config","name":"postgresql_telemetry_config","namespace":"Flow\\PostgreSql\\DSL","parameters":[{"name":"telemetry","type":[{"name":"Telemetry","namespace":"Flow\\Telemetry","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"clock","type":[{"name":"ClockInterface","namespace":"Psr\\Clock","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"options","type":[{"name":"PostgreSqlTelemetryOptions","namespace":"Flow\\PostgreSql\\Client\\Telemetry","is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"null"}],"return_type":[{"name":"PostgreSqlTelemetryConfig","namespace":"Flow\\PostgreSql\\Client\\Telemetry","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSB0ZWxlbWV0cnkgY29uZmlndXJhdGlvbiBmb3IgUG9zdGdyZVNRTCBjbGllbnQuCiAqCiAqIEJ1bmRsZXMgdGVsZW1ldHJ5IGluc3RhbmNlLCBjbG9jaywgYW5kIG9wdGlvbnMgbmVlZGVkIHRvIGluc3RydW1lbnQgYSBQb3N0Z3JlU1FMIGNsaWVudC4KICoKICogQHBhcmFtIFRlbGVtZXRyeSAkdGVsZW1ldHJ5IFRoZSB0ZWxlbWV0cnkgaW5zdGFuY2UKICogQHBhcmFtIENsb2NrSW50ZXJmYWNlICRjbG9jayBDbG9jayBmb3IgdGltZXN0YW1wcwogKiBAcGFyYW0gbnVsbHxQb3N0Z3JlU3FsVGVsZW1ldHJ5T3B0aW9ucyAkb3B0aW9ucyBUZWxlbWV0cnkgb3B0aW9ucyAoZGVmYXVsdDogYWxsIGVuYWJsZWQpCiAqCiAqIEBleGFtcGxlCiAqICRjb25maWcgPSBwb3N0Z3Jlc3FsX3RlbGVtZXRyeV9jb25maWcoCiAqICAgICB0ZWxlbWV0cnkocmVzb3VyY2UoWydzZXJ2aWNlLm5hbWUnID0+ICdteS1hcHAnXSkpLAogKiAgICAgbmV3IFN5c3RlbUNsb2NrKCksCiAqICk7CiAqLw=="},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/client.php","start_line_in_file":260,"slug":"traceable-postgresql-client","name":"traceable_postgresql_client","namespace":"Flow\\PostgreSql\\DSL","parameters":[{"name":"client","type":[{"name":"Client","namespace":"Flow\\PostgreSql\\Client","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"telemetryConfig","type":[{"name":"PostgreSqlTelemetryConfig","namespace":"Flow\\PostgreSql\\Client\\Telemetry","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"TraceableClient","namespace":"Flow\\PostgreSql\\Client\\Telemetry","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIFdyYXAgYSBQb3N0Z3JlU1FMIGNsaWVudCB3aXRoIHRlbGVtZXRyeSBpbnN0cnVtZW50YXRpb24uCiAqCiAqIFJldHVybnMgYSBkZWNvcmF0b3IgdGhhdCBhZGRzIHNwYW5zLCBtZXRyaWNzLCBhbmQgbG9ncyB0byBhbGwKICogcXVlcnkgYW5kIHRyYW5zYWN0aW9uIG9wZXJhdGlvbnMgZm9sbG93aW5nIE9wZW5UZWxlbWV0cnkgY29udmVudGlvbnMuCiAqCiAqIEBwYXJhbSBDbGllbnRcQ2xpZW50ICRjbGllbnQgVGhlIFBvc3RncmVTUUwgY2xpZW50IHRvIGluc3RydW1lbnQKICogQHBhcmFtIFBvc3RncmVTcWxUZWxlbWV0cnlDb25maWcgJHRlbGVtZXRyeUNvbmZpZyBUZWxlbWV0cnkgY29uZmlndXJhdGlvbgogKgogKiBAZXhhbXBsZQogKiAkY2xpZW50ID0gcGdzcWxfY2xpZW50KHBnc3FsX2Nvbm5lY3Rpb24oJ2hvc3Q9bG9jYWxob3N0IGRibmFtZT1teWRiJykpOwogKgogKiAkdHJhY2VhYmxlQ2xpZW50ID0gdHJhY2VhYmxlX3Bvc3RncmVzcWxfY2xpZW50KAogKiAgICAgJGNsaWVudCwKICogICAgIHBvc3RncmVzcWxfdGVsZW1ldHJ5X2NvbmZpZygKICogICAgICAgICB0ZWxlbWV0cnkocmVzb3VyY2UoWydzZXJ2aWNlLm5hbWUnID0+ICdteS1hcHAnXSkpLAogKiAgICAgICAgIG5ldyBTeXN0ZW1DbG9jaygpLAogKiAgICAgICAgIHBvc3RncmVzcWxfdGVsZW1ldHJ5X29wdGlvbnMoCiAqICAgICAgICAgICAgIHRyYWNlUXVlcmllczogdHJ1ZSwKICogICAgICAgICAgICAgdHJhbnNhY3Rpb25TcGFuczogVHJhbnNhY3Rpb25TcGFuTW9kZTo6R1JPVVBFRCwKICogICAgICAgICAgICAgY29sbGVjdE1ldHJpY3M6IHRydWUsCiAqICAgICAgICAgICAgIGxvZ1F1ZXJpZXM6IHRydWUsCiAqICAgICAgICAgICAgIG1heFF1ZXJ5TGVuZ3RoOiA1MDAsCiAqICAgICAgICAgKSwKICogICAgICksCiAqICk7CiAqCiAqIC8vIEFsbCBvcGVyYXRpb25zIG5vdyB0cmFjZWQKICogJHRyYWNlYWJsZUNsaWVudC0+dHJhbnNhY3Rpb24oZnVuY3Rpb24gKENsaWVudCAkY2xpZW50KSB7CiAqICAgICAkdXNlciA9ICRjbGllbnQtPmZldGNoU2luZ2xlKCdTRUxFQ1QgKiBGUk9NIHVzZXJzIFdIRVJFIGlkID0gJDEnLCBbMTIzXSk7CiAqICAgICAkY2xpZW50LT5leGVjdXRlKCdVUERBVEUgdXNlcnMgU0VUIGxhc3RfbG9naW4gPSBOT1coKSBXSEVSRSBpZCA9ICQxJywgWzEyM10pOwogKiB9KTsKICov"},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/client.php","start_line_in_file":300,"slug":"constructor-mapper","name":"constructor_mapper","namespace":"Flow\\PostgreSql\\DSL","parameters":[{"name":"class","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"ConstructorMapper","namespace":"Flow\\PostgreSql\\Client\\RowMapper","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIEB0ZW1wbGF0ZSBUIG9mIG9iamVjdAogKgogKiBAcGFyYW0gY2xhc3Mtc3RyaW5nPFQ+ICRjbGFzcwogKgogKiBAcmV0dXJuIENvbnN0cnVjdG9yTWFwcGVyPFQ+CiAqLw=="},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/client.php","start_line_in_file":315,"slug":"type-mapper","name":"type_mapper","namespace":"Flow\\PostgreSql\\DSL","parameters":[{"name":"type","type":[{"name":"Type","namespace":"Flow\\Types","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"next","type":[{"name":"RowMapper","namespace":"Flow\\PostgreSql\\Client","is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"null"}],"return_type":[{"name":"TypeMapper","namespace":"Flow\\PostgreSql\\Client\\RowMapper","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIEB0ZW1wbGF0ZSBUVHlwZQogKiBAdGVtcGxhdGUgVE91dAogKgogKiBAcGFyYW0gRmxvd1R5cGU8VFR5cGU+ICR0eXBlCiAqIEBwYXJhbSBudWxsfFJvd01hcHBlcjxUT3V0PiAkbmV4dAogKgogKiBAcmV0dXJuICgkbmV4dCBpcyBudWxsID8gVHlwZU1hcHBlcjxUVHlwZSwgVFR5cGU+IDogVHlwZU1hcHBlcjxUVHlwZSwgVE91dD4pCiAqLw=="},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/client.php","start_line_in_file":343,"slug":"static-factory-mapper","name":"static_factory_mapper","namespace":"Flow\\PostgreSql\\DSL","parameters":[{"name":"class","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"method","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"StaticFactoryMapper","namespace":"Flow\\PostgreSql\\Client\\RowMapper","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBhIHJvdyBtYXBwZXIgYmFja2VkIGJ5IGEgcHVibGljIHN0YXRpYyBmYWN0b3J5IG1ldGhvZC4KICoKICogVGhlIGZhY3RvcnkgbWV0aG9kIG11c3QgYWNjZXB0IGEgc2luZ2xlIGFycmF5PHN0cmluZywgbWl4ZWQ+ICRyb3cgYW5kIHJldHVybgogKiBhbiBpbnN0YW5jZSBvZiB0aGUgdGFyZ2V0IGNsYXNzLiBJZiB5b3VyIGZhY3RvcnkgbmVlZHMgYWNjZXNzIHRvIHRoZSBtYXBwaW5nCiAqIENvbnRleHQgKHNxbC9wYXJhbWV0ZXJzL2NsaWVudC9jYXRhbG9nL3VzZXItZGF0YSksIGltcGxlbWVudCBSb3dNYXBwZXIgZGlyZWN0bHkuCiAqCiAqIEB0ZW1wbGF0ZSBUIG9mIG9iamVjdAogKgogKiBAcGFyYW0gY2xhc3Mtc3RyaW5nPFQ+ICRjbGFzcwogKiBAcGFyYW0gbm9uLWVtcHR5LXN0cmluZyAkbWV0aG9kCiAqCiAqIEByZXR1cm4gU3RhdGljRmFjdG9yeU1hcHBlcjxUPgogKi8="},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/client.php","start_line_in_file":372,"slug":"typed","name":"typed","namespace":"Flow\\PostgreSql\\DSL","parameters":[{"name":"value","type":[{"name":"mixed","namespace":null,"is_nullable":true,"is_variadic":false}],"has_default_value":false,"is_nullable":true,"is_variadic":false,"default_value":null},{"name":"targetType","type":[{"name":"ValueType","namespace":"Flow\\PostgreSql\\Client\\Types","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"TypedValue","namespace":"Flow\\PostgreSql\\Client","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIFdyYXAgYSB2YWx1ZSB3aXRoIGV4cGxpY2l0IFBvc3RncmVTUUwgdHlwZSBpbmZvcm1hdGlvbiBmb3IgcGFyYW1ldGVyIGJpbmRpbmcuCiAqCiAqIFVzZSB3aGVuIGF1dG8tZGV0ZWN0aW9uIGlzbid0IHN1ZmZpY2llbnQgb3Igd2hlbiB5b3UgbmVlZCB0byBzcGVjaWZ5CiAqIHRoZSBleGFjdCBQb3N0Z3JlU1FMIHR5cGUgKHNpbmNlIG9uZSBQSFAgdHlwZSBjYW4gbWFwIHRvIG11bHRpcGxlIFBvc3RncmVTUUwgdHlwZXMpOgogKiAtIGludCBjb3VsZCBiZSBJTlQyLCBJTlQ0LCBvciBJTlQ4CiAqIC0gc3RyaW5nIGNvdWxkIGJlIFRFWFQsIFZBUkNIQVIsIG9yIENIQVIKICogLSBhcnJheSBtdXN0IGFsd2F5cyB1c2UgdHlwZWQoKSBzaW5jZSBhdXRvLWRldGVjdGlvbiBjYW5ub3QgZGV0ZXJtaW5lIGVsZW1lbnQgdHlwZQogKiAtIERhdGVUaW1lSW50ZXJmYWNlIGNvdWxkIGJlIFRJTUVTVEFNUCBvciBUSU1FU1RBTVBUWgogKiAtIEpzb24gY291bGQgYmUgSlNPTiBvciBKU09OQgogKgogKiBAcGFyYW0gbWl4ZWQgJHZhbHVlIFRoZSB2YWx1ZSB0byBiaW5kCiAqIEBwYXJhbSBWYWx1ZVR5cGUgJHRhcmdldFR5cGUgVGhlIFBvc3RncmVTUUwgdHlwZSB0byBjb252ZXJ0IHRoZSB2YWx1ZSB0bwogKgogKiBAZXhhbXBsZQogKiAkY2xpZW50LT5mZXRjaCgKICogICAgICdTRUxFQ1QgKiBGUk9NIHVzZXJzIFdIRVJFIGlkID0gJDEgQU5EIHRhZ3MgPSAkMicsCiAqICAgICBbCiAqICAgICAgICAgdHlwZWQoJzU1MGU4NDAwLWUyOWItNDFkNC1hNzE2LTQ0NjY1NTQ0MDAwMCcsIFZhbHVlVHlwZTo6VVVJRCksCiAqICAgICAgICAgdHlwZWQoWyd0YWcxJywgJ3RhZzInXSwgVmFsdWVUeXBlOjpURVhUX0FSUkFZKSwKICogICAgIF0KICogKTsKICov"},{"repository_path":"src\/lib\/postgresql\/src\/Flow\/PostgreSql\/DSL\/client.php","start_line_in_file":386,"slug":"converted-parameters","name":"converted_parameters","namespace":"Flow\\PostgreSql\\DSL","parameters":[{"name":"values","type":[{"name":"array","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"ConvertedParameters","namespace":"Flow\\PostgreSql\\Client","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PG_QUERY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIFBhcmFtZXRlcnMgYWxyZWFkeSBpbiBQb3N0Z3JlU1FMJ3MgdGV4dCBmb3JtLCB3aGljaCBDbGllbnQ6OmV4ZWN1dGUoKSBzZW5kcyB3aXRob3V0IHJ1bm5pbmcgYSBjb252ZXJ0ZXIuCiAqCiAqIEBwYXJhbSBsaXN0PG51bGx8c3RyaW5nPiAkdmFsdWVzCiAqCiAqIEBleGFtcGxlCiAqICRjbGllbnQtPmV4ZWN1dGUoJ1VQREFURSB1c2VycyBTRVQgYWN0aXZlID0gJDEgV0hFUkUgaWQgPSAkMicsIGNvbnZlcnRlZF9wYXJhbWV0ZXJzKFsnZicsICcxJ10pKTsKICov"},{"repository_path":"src\/lib\/telemetry\/src\/Flow\/Telemetry\/DSL\/functions.php","start_line_in_file":126,"slug":"trace-id","name":"trace_id","namespace":"Flow\\Telemetry\\DSL","parameters":[{"name":"hex","type":[{"name":"string","namespace":null,"is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"null"}],"return_type":[{"name":"TraceId","namespace":"Flow\\Telemetry\\Context","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"TELEMETRY","type":"TYPE"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBhIFRyYWNlSWQuCiAqCiAqIElmIGEgaGV4IHN0cmluZyBpcyBwcm92aWRlZCwgY3JlYXRlcyBhIFRyYWNlSWQgZnJvbSBpdC4KICogT3RoZXJ3aXNlLCBnZW5lcmF0ZXMgYSBuZXcgcmFuZG9tIFRyYWNlSWQuCiAqCiAqIEBwYXJhbSBudWxsfHN0cmluZyAkaGV4IE9wdGlvbmFsIDMyLWNoYXJhY3RlciBoZXhhZGVjaW1hbCBzdHJpbmcKICoKICogQHRocm93cyBcSW52YWxpZEFyZ3VtZW50RXhjZXB0aW9uIGlmIHRoZSBoZXggc3RyaW5nIGlzIGludmFsaWQKICov"},{"repository_path":"src\/lib\/telemetry\/src\/Flow\/Telemetry\/DSL\/functions.php","start_line_in_file":146,"slug":"span-id","name":"span_id","namespace":"Flow\\Telemetry\\DSL","parameters":[{"name":"hex","type":[{"name":"string","namespace":null,"is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"null"}],"return_type":[{"name":"SpanId","namespace":"Flow\\Telemetry\\Context","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"TELEMETRY","type":"TYPE"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBhIFNwYW5JZC4KICoKICogSWYgYSBoZXggc3RyaW5nIGlzIHByb3ZpZGVkLCBjcmVhdGVzIGEgU3BhbklkIGZyb20gaXQuCiAqIE90aGVyd2lzZSwgZ2VuZXJhdGVzIGEgbmV3IHJhbmRvbSBTcGFuSWQuCiAqCiAqIEBwYXJhbSBudWxsfHN0cmluZyAkaGV4IE9wdGlvbmFsIDE2LWNoYXJhY3RlciBoZXhhZGVjaW1hbCBzdHJpbmcKICoKICogQHRocm93cyBcSW52YWxpZEFyZ3VtZW50RXhjZXB0aW9uIGlmIHRoZSBoZXggc3RyaW5nIGlzIGludmFsaWQKICov"},{"repository_path":"src\/lib\/telemetry\/src\/Flow\/Telemetry\/DSL\/functions.php","start_line_in_file":161,"slug":"baggage","name":"baggage","namespace":"Flow\\Telemetry\\DSL","parameters":[{"name":"entries","type":[{"name":"array","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"[]"}],"return_type":[{"name":"Baggage","namespace":"Flow\\Telemetry\\Context","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"TELEMETRY","type":"TYPE"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBhIEJhZ2dhZ2UuCiAqCiAqIEBwYXJhbSBhcnJheTxzdHJpbmcsIHN0cmluZz4gJGVudHJpZXMgSW5pdGlhbCBrZXktdmFsdWUgZW50cmllcwogKi8="},{"repository_path":"src\/lib\/telemetry\/src\/Flow\/Telemetry\/DSL\/functions.php","start_line_in_file":175,"slug":"context","name":"context","namespace":"Flow\\Telemetry\\DSL","parameters":[{"name":"baggage","type":[{"name":"Baggage","namespace":"Flow\\Telemetry\\Context","is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"null"}],"return_type":[{"name":"Context","namespace":"Flow\\Telemetry\\Context","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"TELEMETRY","type":"TYPE"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBhIHJvb3QgQ29udGV4dCAobm8gYWN0aXZlIHNwYW4pLgogKgogKiBBIHNwYW4gY3JlYXRlZCBpbiB0aGlzIGNvbnRleHQgYmVjb21lcyBhIG5ldyB0cmFjZSByb290LiBBdHRhY2ggYW4gYWN0aXZlIHNwYW4gd2l0aAogKiBDb250ZXh0Ojp3aXRoQWN0aXZlU3BhbigpIHRvIG1ha2Ugc3Vic2VxdWVudCBzcGFucyBpdHMgY2hpbGRyZW4uCiAqCiAqIEBwYXJhbSBudWxsfEJhZ2dhZ2UgJGJhZ2dhZ2UgT3B0aW9uYWwgQmFnZ2FnZSB0byB1c2UKICov"},{"repository_path":"src\/lib\/telemetry\/src\/Flow\/Telemetry\/DSL\/functions.php","start_line_in_file":189,"slug":"memory-context-storage","name":"memory_context_storage","namespace":"Flow\\Telemetry\\DSL","parameters":[{"name":"context","type":[{"name":"Context","namespace":"Flow\\Telemetry\\Context","is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"null"}],"return_type":[{"name":"MemoryContextStorage","namespace":"Flow\\Telemetry\\Context","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"TELEMETRY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBhIE1lbW9yeUNvbnRleHRTdG9yYWdlLgogKgogKiBJbi1tZW1vcnkgY29udGV4dCBzdG9yYWdlIGZvciBzdG9yaW5nIGFuZCByZXRyaWV2aW5nIHRoZSBjdXJyZW50IGNvbnRleHQuCiAqIEEgc2luZ2xlIGluc3RhbmNlIHNob3VsZCBiZSBzaGFyZWQgYWNyb3NzIGFsbCBwcm92aWRlcnMgd2l0aGluIGEgcmVxdWVzdCBsaWZlY3ljbGUuCiAqCiAqIEBwYXJhbSBudWxsfENvbnRleHQgJGNvbnRleHQgT3B0aW9uYWwgaW5pdGlhbCBjb250ZXh0CiAqLw=="},{"repository_path":"src\/lib\/telemetry\/src\/Flow\/Telemetry\/DSL\/functions.php","start_line_in_file":200,"slug":"resource","name":"resource","namespace":"Flow\\Telemetry\\DSL","parameters":[{"name":"attributes","type":[{"name":"Attributes","namespace":"Flow\\Telemetry","is_nullable":false,"is_variadic":false},{"name":"array","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"[]"}],"return_type":[{"name":"Resource","namespace":"Flow\\Telemetry","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"TELEMETRY","type":"TYPE"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBhIFJlc291cmNlLgogKgogKiBAcGFyYW0gYXJyYXk8c3RyaW5nLCBhcnJheTxhcnJheS1rZXksIG1peGVkPnxib29sfFxEYXRlVGltZUludGVyZmFjZXxmbG9hdHxpbnR8c3RyaW5nfFxUaHJvd2FibGU+fEF0dHJpYnV0ZXMgJGF0dHJpYnV0ZXMgUmVzb3VyY2UgYXR0cmlidXRlcwogKi8="},{"repository_path":"src\/lib\/telemetry\/src\/Flow\/Telemetry\/DSL\/functions.php","start_line_in_file":213,"slug":"span-context","name":"span_context","namespace":"Flow\\Telemetry\\DSL","parameters":[{"name":"traceId","type":[{"name":"TraceId","namespace":"Flow\\Telemetry\\Context","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"spanId","type":[{"name":"SpanId","namespace":"Flow\\Telemetry\\Context","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"parentSpanId","type":[{"name":"SpanId","namespace":"Flow\\Telemetry\\Context","is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"null"}],"return_type":[{"name":"SpanContext","namespace":"Flow\\Telemetry\\Tracer","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"TELEMETRY","type":"TYPE"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBhIFNwYW5Db250ZXh0LgogKgogKiBAcGFyYW0gVHJhY2VJZCAkdHJhY2VJZCBUaGUgdHJhY2UgSUQKICogQHBhcmFtIFNwYW5JZCAkc3BhbklkIFRoZSBzcGFuIElECiAqIEBwYXJhbSBudWxsfFNwYW5JZCAkcGFyZW50U3BhbklkIE9wdGlvbmFsIHBhcmVudCBzcGFuIElECiAqLw=="},{"repository_path":"src\/lib\/telemetry\/src\/Flow\/Telemetry\/DSL\/functions.php","start_line_in_file":226,"slug":"span-event","name":"span_event","namespace":"Flow\\Telemetry\\DSL","parameters":[{"name":"name","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"timestamp","type":[{"name":"DateTimeImmutable","namespace":"","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"attributes","type":[{"name":"Attributes","namespace":"Flow\\Telemetry","is_nullable":false,"is_variadic":false},{"name":"array","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"[]"}],"return_type":[{"name":"GenericEvent","namespace":"Flow\\Telemetry\\Tracer","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"TELEMETRY","type":"TYPE"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBhIFNwYW5FdmVudCAoR2VuZXJpY0V2ZW50KSB3aXRoIGFuIGV4cGxpY2l0IHRpbWVzdGFtcC4KICoKICogQHBhcmFtIHN0cmluZyAkbmFtZSBFdmVudCBuYW1lCiAqIEBwYXJhbSBcRGF0ZVRpbWVJbW11dGFibGUgJHRpbWVzdGFtcCBFdmVudCB0aW1lc3RhbXAKICogQHBhcmFtIGFycmF5PHN0cmluZywgYXJyYXk8YXJyYXkta2V5LCBtaXhlZD58Ym9vbHxcRGF0ZVRpbWVJbnRlcmZhY2V8ZmxvYXR8aW50fHN0cmluZ3xcVGhyb3dhYmxlPnxBdHRyaWJ1dGVzICRhdHRyaWJ1dGVzIEV2ZW50IGF0dHJpYnV0ZXMKICov"},{"repository_path":"src\/lib\/telemetry\/src\/Flow\/Telemetry\/DSL\/functions.php","start_line_in_file":238,"slug":"span-link","name":"span_link","namespace":"Flow\\Telemetry\\DSL","parameters":[{"name":"context","type":[{"name":"SpanContext","namespace":"Flow\\Telemetry\\Tracer","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"attributes","type":[{"name":"Attributes","namespace":"Flow\\Telemetry","is_nullable":false,"is_variadic":false},{"name":"array","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"[]"}],"return_type":[{"name":"SpanLink","namespace":"Flow\\Telemetry\\Tracer","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"TELEMETRY","type":"TYPE"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBhIFNwYW5MaW5rLgogKgogKiBAcGFyYW0gU3BhbkNvbnRleHQgJGNvbnRleHQgVGhlIGxpbmtlZCBzcGFuIGNvbnRleHQKICogQHBhcmFtIGFycmF5PHN0cmluZywgYXJyYXk8YXJyYXkta2V5LCBtaXhlZD58Ym9vbHxcRGF0ZVRpbWVJbnRlcmZhY2V8ZmxvYXR8aW50fHN0cmluZ3xcVGhyb3dhYmxlPnxBdHRyaWJ1dGVzICRhdHRyaWJ1dGVzIExpbmsgYXR0cmlidXRlcwogKi8="},{"repository_path":"src\/lib\/telemetry\/src\/Flow\/Telemetry\/DSL\/functions.php","start_line_in_file":254,"slug":"span-limits","name":"span_limits","namespace":"Flow\\Telemetry\\DSL","parameters":[{"name":"attributeCountLimit","type":[{"name":"int","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"128"},{"name":"eventCountLimit","type":[{"name":"int","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"128"},{"name":"linkCountLimit","type":[{"name":"int","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"128"},{"name":"attributePerEventCountLimit","type":[{"name":"int","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"128"},{"name":"attributePerLinkCountLimit","type":[{"name":"int","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"128"},{"name":"attributeValueLengthLimit","type":[{"name":"int","namespace":null,"is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"null"}],"return_type":[{"name":"SpanLimits","namespace":"Flow\\Telemetry\\Tracer","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"TELEMETRY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBTcGFuTGltaXRzIGNvbmZpZ3VyYXRpb24uCiAqCiAqIEBwYXJhbSBpbnQgJGF0dHJpYnV0ZUNvdW50TGltaXQgTWF4aW11bSBudW1iZXIgb2YgYXR0cmlidXRlcyBwZXIgc3BhbgogKiBAcGFyYW0gaW50ICRldmVudENvdW50TGltaXQgTWF4aW11bSBudW1iZXIgb2YgZXZlbnRzIHBlciBzcGFuCiAqIEBwYXJhbSBpbnQgJGxpbmtDb3VudExpbWl0IE1heGltdW0gbnVtYmVyIG9mIGxpbmtzIHBlciBzcGFuCiAqIEBwYXJhbSBpbnQgJGF0dHJpYnV0ZVBlckV2ZW50Q291bnRMaW1pdCBNYXhpbXVtIG51bWJlciBvZiBhdHRyaWJ1dGVzIHBlciBldmVudAogKiBAcGFyYW0gaW50ICRhdHRyaWJ1dGVQZXJMaW5rQ291bnRMaW1pdCBNYXhpbXVtIG51bWJlciBvZiBhdHRyaWJ1dGVzIHBlciBsaW5rCiAqIEBwYXJhbSBudWxsfGludCAkYXR0cmlidXRlVmFsdWVMZW5ndGhMaW1pdCBNYXhpbXVtIGxlbmd0aCBmb3Igc3RyaW5nIGF0dHJpYnV0ZSB2YWx1ZXMgKG51bGwgPSB1bmxpbWl0ZWQpCiAqLw=="},{"repository_path":"src\/lib\/telemetry\/src\/Flow\/Telemetry\/DSL\/functions.php","start_line_in_file":279,"slug":"log-record-limits","name":"log_record_limits","namespace":"Flow\\Telemetry\\DSL","parameters":[{"name":"attributeCountLimit","type":[{"name":"int","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"128"},{"name":"attributeValueLengthLimit","type":[{"name":"int","namespace":null,"is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"null"}],"return_type":[{"name":"LogRecordLimits","namespace":"Flow\\Telemetry\\Logger","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"TELEMETRY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBMb2dSZWNvcmRMaW1pdHMgY29uZmlndXJhdGlvbi4KICoKICogQHBhcmFtIGludCAkYXR0cmlidXRlQ291bnRMaW1pdCBNYXhpbXVtIG51bWJlciBvZiBhdHRyaWJ1dGVzIHBlciBsb2cgcmVjb3JkCiAqIEBwYXJhbSBudWxsfGludCAkYXR0cmlidXRlVmFsdWVMZW5ndGhMaW1pdCBNYXhpbXVtIGxlbmd0aCBmb3Igc3RyaW5nIGF0dHJpYnV0ZSB2YWx1ZXMgKG51bGwgPSB1bmxpbWl0ZWQpCiAqLw=="},{"repository_path":"src\/lib\/telemetry\/src\/Flow\/Telemetry\/DSL\/functions.php","start_line_in_file":290,"slug":"metric-limits","name":"metric_limits","namespace":"Flow\\Telemetry\\DSL","parameters":[{"name":"cardinalityLimit","type":[{"name":"int","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"2000"}],"return_type":[{"name":"MetricLimits","namespace":"Flow\\Telemetry\\Meter","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"TELEMETRY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBNZXRyaWNMaW1pdHMgY29uZmlndXJhdGlvbi4KICoKICogQHBhcmFtIGludCAkY2FyZGluYWxpdHlMaW1pdCBNYXhpbXVtIG51bWJlciBvZiB1bmlxdWUgYXR0cmlidXRlIGNvbWJpbmF0aW9ucyBwZXIgaW5zdHJ1bWVudAogKi8="},{"repository_path":"src\/lib\/telemetry\/src\/Flow\/Telemetry\/DSL\/functions.php","start_line_in_file":301,"slug":"void-span-processor","name":"void_span_processor","namespace":"Flow\\Telemetry\\DSL","parameters":[],"return_type":[{"name":"VoidSpanProcessor","namespace":"Flow\\Telemetry\\Provider\\Void","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"TELEMETRY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBhIFZvaWRTcGFuUHJvY2Vzc29yLgogKgogKiBOby1vcCBzcGFuIHByb2Nlc3NvciB0aGF0IGRpc2NhcmRzIGFsbCBkYXRhLgogKi8="},{"repository_path":"src\/lib\/telemetry\/src\/Flow\/Telemetry\/DSL\/functions.php","start_line_in_file":312,"slug":"void-metric-processor","name":"void_metric_processor","namespace":"Flow\\Telemetry\\DSL","parameters":[],"return_type":[{"name":"VoidMetricProcessor","namespace":"Flow\\Telemetry\\Provider\\Void","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"TELEMETRY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBhIFZvaWRNZXRyaWNQcm9jZXNzb3IuCiAqCiAqIE5vLW9wIG1ldHJpYyBwcm9jZXNzb3IgdGhhdCBkaXNjYXJkcyBhbGwgZGF0YS4KICov"},{"repository_path":"src\/lib\/telemetry\/src\/Flow\/Telemetry\/DSL\/functions.php","start_line_in_file":323,"slug":"void-log-processor","name":"void_log_processor","namespace":"Flow\\Telemetry\\DSL","parameters":[],"return_type":[{"name":"VoidLogProcessor","namespace":"Flow\\Telemetry\\Provider\\Void","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"TELEMETRY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBhIFZvaWRMb2dQcm9jZXNzb3IuCiAqCiAqIE5vLW9wIGxvZyBwcm9jZXNzb3IgdGhhdCBkaXNjYXJkcyBhbGwgZGF0YS4KICov"},{"repository_path":"src\/lib\/telemetry\/src\/Flow\/Telemetry\/DSL\/functions.php","start_line_in_file":334,"slug":"void-exporter","name":"void_exporter","namespace":"Flow\\Telemetry\\DSL","parameters":[],"return_type":[{"name":"VoidExporter","namespace":"Flow\\Telemetry\\Provider\\Void","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"TELEMETRY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBhIFZvaWRFeHBvcnRlci4KICoKICogTm8tb3AgdW5pZmllZCBleHBvcnRlciB0aGF0IGRpc2NhcmRzIGxvZ3MsIG1ldHJpY3MsIGFuZCBzcGFucy4KICov"},{"repository_path":"src\/lib\/telemetry\/src\/Flow\/Telemetry\/DSL\/functions.php","start_line_in_file":348,"slug":"memory-exporter","name":"memory_exporter","namespace":"Flow\\Telemetry\\DSL","parameters":[{"name":"maxEntriesPerSignal","type":[{"name":"int","namespace":null,"is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"null"}],"return_type":[{"name":"MemoryExporter","namespace":"Flow\\Telemetry\\Provider\\Memory","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"TELEMETRY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBhIE1lbW9yeUV4cG9ydGVyLgogKgogKiBVbmlmaWVkIGV4cG9ydGVyIHRoYXQgc3RvcmVzIGxvZ3MsIG1ldHJpY3MsIGFuZCBzcGFucyBpbiBtZW1vcnkgZm9yIGRpcmVjdCBhY2Nlc3MuCiAqIFVzZWZ1bCBmb3IgdGVzdGluZyBhbmQgaW5zcGVjdGlvbiB3aXRob3V0IHNlcmlhbGl6YXRpb24uCiAqCiAqIEBwYXJhbSBudWxsfGludCAkbWF4RW50cmllc1BlclNpZ25hbCBtYXhpbXVtIGVudHJpZXMgcmV0YWluZWQgcGVyIHNpZ25hbCB0eXBlOyBudWxsIGtlZXBzIGV2ZXJ5dGhpbmcKICov"},{"repository_path":"src\/lib\/telemetry\/src\/Flow\/Telemetry\/DSL\/functions.php","start_line_in_file":360,"slug":"memory-span-processor","name":"memory_span_processor","namespace":"Flow\\Telemetry\\DSL","parameters":[{"name":"exporter","type":[{"name":"Exporter","namespace":"Flow\\Telemetry\\Exporter","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"errorHandler","type":[{"name":"ErrorHandler","namespace":"Flow\\Telemetry\\ErrorHandler","is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"Flow\\Telemetry\\ErrorHandler\\ErrorLogHandler::..."}],"return_type":[{"name":"MemorySpanProcessor","namespace":"Flow\\Telemetry\\Provider\\Memory","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"TELEMETRY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBhIE1lbW9yeVNwYW5Qcm9jZXNzb3IuCiAqCiAqIEBwYXJhbSBFeHBvcnRlciAkZXhwb3J0ZXIgVGhlIGV4cG9ydGVyIHRvIHNlbmQgc3BhbnMgdG8KICogQHBhcmFtIEVycm9ySGFuZGxlciAkZXJyb3JIYW5kbGVyIEhhbmRsZXIgZm9yIFRocm93YWJsZXMgcmFpc2VkIGJ5IHRoZSBleHBvcnRlcgogKi8="},{"repository_path":"src\/lib\/telemetry\/src\/Flow\/Telemetry\/DSL\/functions.php","start_line_in_file":374,"slug":"memory-metric-processor","name":"memory_metric_processor","namespace":"Flow\\Telemetry\\DSL","parameters":[{"name":"exporter","type":[{"name":"Exporter","namespace":"Flow\\Telemetry\\Exporter","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"errorHandler","type":[{"name":"ErrorHandler","namespace":"Flow\\Telemetry\\ErrorHandler","is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"Flow\\Telemetry\\ErrorHandler\\ErrorLogHandler::..."}],"return_type":[{"name":"MemoryMetricProcessor","namespace":"Flow\\Telemetry\\Provider\\Memory","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"TELEMETRY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBhIE1lbW9yeU1ldHJpY1Byb2Nlc3Nvci4KICoKICogQHBhcmFtIEV4cG9ydGVyICRleHBvcnRlciBUaGUgZXhwb3J0ZXIgdG8gc2VuZCBtZXRyaWNzIHRvCiAqIEBwYXJhbSBFcnJvckhhbmRsZXIgJGVycm9ySGFuZGxlciBIYW5kbGVyIGZvciBUaHJvd2FibGVzIHJhaXNlZCBieSB0aGUgZXhwb3J0ZXIKICov"},{"repository_path":"src\/lib\/telemetry\/src\/Flow\/Telemetry\/DSL\/functions.php","start_line_in_file":388,"slug":"memory-log-processor","name":"memory_log_processor","namespace":"Flow\\Telemetry\\DSL","parameters":[{"name":"exporter","type":[{"name":"Exporter","namespace":"Flow\\Telemetry\\Exporter","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"errorHandler","type":[{"name":"ErrorHandler","namespace":"Flow\\Telemetry\\ErrorHandler","is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"Flow\\Telemetry\\ErrorHandler\\ErrorLogHandler::..."}],"return_type":[{"name":"MemoryLogProcessor","namespace":"Flow\\Telemetry\\Provider\\Memory","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"TELEMETRY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBhIE1lbW9yeUxvZ1Byb2Nlc3Nvci4KICoKICogQHBhcmFtIEV4cG9ydGVyICRleHBvcnRlciBUaGUgZXhwb3J0ZXIgdG8gc2VuZCBsb2dzIHRvCiAqIEBwYXJhbSBFcnJvckhhbmRsZXIgJGVycm9ySGFuZGxlciBIYW5kbGVyIGZvciBUaHJvd2FibGVzIHJhaXNlZCBieSB0aGUgZXhwb3J0ZXIKICov"},{"repository_path":"src\/lib\/telemetry\/src\/Flow\/Telemetry\/DSL\/functions.php","start_line_in_file":406,"slug":"tracer-provider","name":"tracer_provider","namespace":"Flow\\Telemetry\\DSL","parameters":[{"name":"processor","type":[{"name":"SpanProcessor","namespace":"Flow\\Telemetry\\Tracer","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"clock","type":[{"name":"ClockInterface","namespace":"Psr\\Clock","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"contextStorage","type":[{"name":"ContextStorage","namespace":"Flow\\Telemetry\\Context","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"sampler","type":[{"name":"Sampler","namespace":"Flow\\Telemetry\\Tracer\\Sampler","is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"Flow\\Telemetry\\Tracer\\Sampler\\ParentBasedSampler::..."},{"name":"limits","type":[{"name":"SpanLimits","namespace":"Flow\\Telemetry\\Tracer","is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"Flow\\Telemetry\\Tracer\\SpanLimits::..."},{"name":"errorHandler","type":[{"name":"ErrorHandler","namespace":"Flow\\Telemetry\\ErrorHandler","is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"Flow\\Telemetry\\ErrorHandler\\ErrorLogHandler::..."}],"return_type":[{"name":"TracerProvider","namespace":"Flow\\Telemetry\\Tracer","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"TELEMETRY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBhIFRyYWNlclByb3ZpZGVyLgogKgogKiBAcGFyYW0gU3BhblByb2Nlc3NvciAkcHJvY2Vzc29yIFRoZSBwcm9jZXNzb3IgZm9yIHNwYW5zCiAqIEBwYXJhbSBDbG9ja0ludGVyZmFjZSAkY2xvY2sgVGhlIGNsb2NrIGZvciB0aW1lc3RhbXBzCiAqIEBwYXJhbSBDb250ZXh0U3RvcmFnZSAkY29udGV4dFN0b3JhZ2UgU3RvcmFnZSBmb3IgY29udGV4dCBwcm9wYWdhdGlvbgogKiBAcGFyYW0gU2FtcGxlciAkc2FtcGxlciBTYW1wbGluZyBzdHJhdGVneSBmb3Igc3BhbnMKICogQHBhcmFtIFNwYW5MaW1pdHMgJGxpbWl0cyBMaW1pdHMgZm9yIHNwYW4gYXR0cmlidXRlcywgZXZlbnRzLCBhbmQgbGlua3MKICogQHBhcmFtIEVycm9ySGFuZGxlciAkZXJyb3JIYW5kbGVyIEhhbmRsZXIgZm9yIHJ1bnRpbWUgVGhyb3dhYmxlcyByYWlzZWQgYnkgdGhlIHByb2Nlc3NvcgogKi8="},{"repository_path":"src\/lib\/telemetry\/src\/Flow\/Telemetry\/DSL\/functions.php","start_line_in_file":427,"slug":"logger-provider","name":"logger_provider","namespace":"Flow\\Telemetry\\DSL","parameters":[{"name":"processor","type":[{"name":"LogProcessor","namespace":"Flow\\Telemetry\\Logger","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"clock","type":[{"name":"ClockInterface","namespace":"Psr\\Clock","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"contextStorage","type":[{"name":"ContextStorage","namespace":"Flow\\Telemetry\\Context","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"limits","type":[{"name":"LogRecordLimits","namespace":"Flow\\Telemetry\\Logger","is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"Flow\\Telemetry\\Logger\\LogRecordLimits::..."},{"name":"errorHandler","type":[{"name":"ErrorHandler","namespace":"Flow\\Telemetry\\ErrorHandler","is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"Flow\\Telemetry\\ErrorHandler\\ErrorLogHandler::..."}],"return_type":[{"name":"LoggerProvider","namespace":"Flow\\Telemetry\\Logger","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"TELEMETRY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBhIExvZ2dlclByb3ZpZGVyLgogKgogKiBAcGFyYW0gTG9nUHJvY2Vzc29yICRwcm9jZXNzb3IgVGhlIHByb2Nlc3NvciBmb3IgbG9ncwogKiBAcGFyYW0gQ2xvY2tJbnRlcmZhY2UgJGNsb2NrIFRoZSBjbG9jayBmb3IgdGltZXN0YW1wcwogKiBAcGFyYW0gQ29udGV4dFN0b3JhZ2UgJGNvbnRleHRTdG9yYWdlIFN0b3JhZ2UgZm9yIHNwYW4gY29ycmVsYXRpb24KICogQHBhcmFtIExvZ1JlY29yZExpbWl0cyAkbGltaXRzIExpbWl0cyBmb3IgbG9nIHJlY29yZCBhdHRyaWJ1dGVzCiAqIEBwYXJhbSBFcnJvckhhbmRsZXIgJGVycm9ySGFuZGxlciBIYW5kbGVyIGZvciBydW50aW1lIFRocm93YWJsZXMgcmFpc2VkIGJ5IHRoZSBwcm9jZXNzb3IKICov"},{"repository_path":"src\/lib\/telemetry\/src\/Flow\/Telemetry\/DSL\/functions.php","start_line_in_file":448,"slug":"meter-provider","name":"meter_provider","namespace":"Flow\\Telemetry\\DSL","parameters":[{"name":"processor","type":[{"name":"MetricProcessor","namespace":"Flow\\Telemetry\\Meter","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"clock","type":[{"name":"ClockInterface","namespace":"Psr\\Clock","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"temporality","type":[{"name":"AggregationTemporality","namespace":"Flow\\Telemetry\\Meter","is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"Flow\\Telemetry\\Meter\\AggregationTemporality::..."},{"name":"exemplarFilter","type":[{"name":"ExemplarFilter","namespace":"Flow\\Telemetry\\Meter\\Exemplar","is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"Flow\\Telemetry\\Meter\\Exemplar\\TraceBasedExemplarFilter::..."},{"name":"limits","type":[{"name":"MetricLimits","namespace":"Flow\\Telemetry\\Meter","is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"Flow\\Telemetry\\Meter\\MetricLimits::..."},{"name":"errorHandler","type":[{"name":"ErrorHandler","namespace":"Flow\\Telemetry\\ErrorHandler","is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"Flow\\Telemetry\\ErrorHandler\\ErrorLogHandler::..."}],"return_type":[{"name":"MeterProvider","namespace":"Flow\\Telemetry\\Meter","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"TELEMETRY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBhIE1ldGVyUHJvdmlkZXIuCiAqCiAqIEBwYXJhbSBNZXRyaWNQcm9jZXNzb3IgJHByb2Nlc3NvciBUaGUgcHJvY2Vzc29yIGZvciBtZXRyaWNzCiAqIEBwYXJhbSBDbG9ja0ludGVyZmFjZSAkY2xvY2sgVGhlIGNsb2NrIGZvciB0aW1lc3RhbXBzCiAqIEBwYXJhbSBBZ2dyZWdhdGlvblRlbXBvcmFsaXR5ICR0ZW1wb3JhbGl0eSBBZ2dyZWdhdGlvbiB0ZW1wb3JhbGl0eSBmb3IgbWV0cmljcwogKiBAcGFyYW0gRXhlbXBsYXJGaWx0ZXIgJGV4ZW1wbGFyRmlsdGVyIEZpbHRlciBmb3IgZXhlbXBsYXIgc2FtcGxpbmcgKGRlZmF1bHQ6IFRyYWNlQmFzZWRFeGVtcGxhckZpbHRlcikKICogQHBhcmFtIE1ldHJpY0xpbWl0cyAkbGltaXRzIENhcmRpbmFsaXR5IGxpbWl0cyBmb3IgbWV0cmljIGluc3RydW1lbnRzCiAqIEBwYXJhbSBFcnJvckhhbmRsZXIgJGVycm9ySGFuZGxlciBIYW5kbGVyIGZvciBydW50aW1lIFRocm93YWJsZXMgcmFpc2VkIGJ5IHRoZSBwcm9jZXNzb3IKICov"},{"repository_path":"src\/lib\/telemetry\/src\/Flow\/Telemetry\/DSL\/functions.php","start_line_in_file":471,"slug":"telemetry","name":"telemetry","namespace":"Flow\\Telemetry\\DSL","parameters":[{"name":"resource","type":[{"name":"Resource","namespace":"Flow\\Telemetry","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"tracerProvider","type":[{"name":"TracerProvider","namespace":"Flow\\Telemetry\\Tracer","is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"null"},{"name":"meterProvider","type":[{"name":"MeterProvider","namespace":"Flow\\Telemetry\\Meter","is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"null"},{"name":"loggerProvider","type":[{"name":"LoggerProvider","namespace":"Flow\\Telemetry\\Logger","is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"null"},{"name":"errorHandler","type":[{"name":"ErrorHandler","namespace":"Flow\\Telemetry\\ErrorHandler","is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"Flow\\Telemetry\\ErrorHandler\\ErrorLogHandler::..."}],"return_type":[{"name":"Telemetry","namespace":"Flow\\Telemetry","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"TELEMETRY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBhIG5ldyBUZWxlbWV0cnkgaW5zdGFuY2Ugd2l0aCB0aGUgZ2l2ZW4gcHJvdmlkZXJzLgogKgogKiBJZiBwcm92aWRlcnMgYXJlIG5vdCBzcGVjaWZpZWQsIHZvaWQgcHJvdmlkZXJzIChuby1vcCkgYXJlIHVzZWQuCiAqCiAqIEBwYXJhbSBcRmxvd1xUZWxlbWV0cnlcUmVzb3VyY2UgJHJlc291cmNlIFRoZSByZXNvdXJjZSBkZXNjcmliaW5nIHRoZSBlbnRpdHkgcHJvZHVjaW5nIHRlbGVtZXRyeQogKiBAcGFyYW0gbnVsbHxUcmFjZXJQcm92aWRlciAkdHJhY2VyUHJvdmlkZXIgVGhlIHRyYWNlciBwcm92aWRlciAobnVsbCBmb3Igdm9pZC9kaXNhYmxlZCkKICogQHBhcmFtIG51bGx8TWV0ZXJQcm92aWRlciAkbWV0ZXJQcm92aWRlciBUaGUgbWV0ZXIgcHJvdmlkZXIgKG51bGwgZm9yIHZvaWQvZGlzYWJsZWQpCiAqIEBwYXJhbSBudWxsfExvZ2dlclByb3ZpZGVyICRsb2dnZXJQcm92aWRlciBUaGUgbG9nZ2VyIHByb3ZpZGVyIChudWxsIGZvciB2b2lkL2Rpc2FibGVkKQogKiBAcGFyYW0gRXJyb3JIYW5kbGVyICRlcnJvckhhbmRsZXIgSGFuZGxlciBwcm9wYWdhdGVkIHRvIGRlZmF1bHQgdm9pZCBwcm92aWRlcnMgd2hlbiBleHBsaWNpdCBvbmVzIGFyZSBub3Qgc3VwcGxpZWQKICov"},{"repository_path":"src\/lib\/telemetry\/src\/Flow\/Telemetry\/DSL\/functions.php","start_line_in_file":507,"slug":"instrumentation-scope","name":"instrumentation_scope","namespace":"Flow\\Telemetry\\DSL","parameters":[{"name":"name","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"version","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"'unknown'"},{"name":"schemaUrl","type":[{"name":"string","namespace":null,"is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"null"},{"name":"attributes","type":[{"name":"Attributes","namespace":"Flow\\Telemetry","is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"Flow\\Telemetry\\Attributes::..."}],"return_type":[{"name":"InstrumentationScope","namespace":"Flow\\Telemetry","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"TELEMETRY","type":"TYPE"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBhbiBJbnN0cnVtZW50YXRpb25TY29wZS4KICoKICogQHBhcmFtIHN0cmluZyAkbmFtZSBUaGUgaW5zdHJ1bWVudGF0aW9uIHNjb3BlIG5hbWUKICogQHBhcmFtIHN0cmluZyAkdmVyc2lvbiBUaGUgaW5zdHJ1bWVudGF0aW9uIHNjb3BlIHZlcnNpb24KICogQHBhcmFtIG51bGx8c3RyaW5nICRzY2hlbWFVcmwgT3B0aW9uYWwgc2NoZW1hIFVSTAogKi8="},{"repository_path":"src\/lib\/telemetry\/src\/Flow\/Telemetry\/DSL\/functions.php","start_line_in_file":524,"slug":"batching-span-processor","name":"batching_span_processor","namespace":"Flow\\Telemetry\\DSL","parameters":[{"name":"exporter","type":[{"name":"Exporter","namespace":"Flow\\Telemetry\\Exporter","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"batchSize","type":[{"name":"int","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"512"},{"name":"errorHandler","type":[{"name":"ErrorHandler","namespace":"Flow\\Telemetry\\ErrorHandler","is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"Flow\\Telemetry\\ErrorHandler\\ErrorLogHandler::..."}],"return_type":[{"name":"BatchingSpanProcessor","namespace":"Flow\\Telemetry\\Tracer\\Processor","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"TELEMETRY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBhIEJhdGNoaW5nU3BhblByb2Nlc3Nvci4KICoKICogQHBhcmFtIEV4cG9ydGVyICRleHBvcnRlciBUaGUgZXhwb3J0ZXIgdG8gc2VuZCBzcGFucyB0bwogKiBAcGFyYW0gaW50ICRiYXRjaFNpemUgTnVtYmVyIG9mIHNwYW5zIHRvIGNvbGxlY3QgYmVmb3JlIGV4cG9ydGluZyAoZGVmYXVsdCA1MTIpCiAqIEBwYXJhbSBFcnJvckhhbmRsZXIgJGVycm9ySGFuZGxlciBIYW5kbGVyIGZvciBUaHJvd2FibGVzIHJhaXNlZCBieSB0aGUgZXhwb3J0ZXIKICov"},{"repository_path":"src\/lib\/telemetry\/src\/Flow\/Telemetry\/DSL\/functions.php","start_line_in_file":539,"slug":"pass-through-span-processor","name":"pass_through_span_processor","namespace":"Flow\\Telemetry\\DSL","parameters":[{"name":"exporter","type":[{"name":"Exporter","namespace":"Flow\\Telemetry\\Exporter","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"errorHandler","type":[{"name":"ErrorHandler","namespace":"Flow\\Telemetry\\ErrorHandler","is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"Flow\\Telemetry\\ErrorHandler\\ErrorLogHandler::..."}],"return_type":[{"name":"PassThroughSpanProcessor","namespace":"Flow\\Telemetry\\Tracer\\Processor","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"TELEMETRY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBhIFBhc3NUaHJvdWdoU3BhblByb2Nlc3Nvci4KICoKICogQHBhcmFtIEV4cG9ydGVyICRleHBvcnRlciBUaGUgZXhwb3J0ZXIgdG8gc2VuZCBzcGFucyB0bwogKiBAcGFyYW0gRXJyb3JIYW5kbGVyICRlcnJvckhhbmRsZXIgSGFuZGxlciBmb3IgVGhyb3dhYmxlcyByYWlzZWQgYnkgdGhlIGV4cG9ydGVyCiAqLw=="},{"repository_path":"src\/lib\/telemetry\/src\/Flow\/Telemetry\/DSL\/functions.php","start_line_in_file":554,"slug":"batching-metric-processor","name":"batching_metric_processor","namespace":"Flow\\Telemetry\\DSL","parameters":[{"name":"exporter","type":[{"name":"Exporter","namespace":"Flow\\Telemetry\\Exporter","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"batchSize","type":[{"name":"int","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"512"},{"name":"errorHandler","type":[{"name":"ErrorHandler","namespace":"Flow\\Telemetry\\ErrorHandler","is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"Flow\\Telemetry\\ErrorHandler\\ErrorLogHandler::..."}],"return_type":[{"name":"BatchingMetricProcessor","namespace":"Flow\\Telemetry\\Meter\\Processor","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"TELEMETRY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBhIEJhdGNoaW5nTWV0cmljUHJvY2Vzc29yLgogKgogKiBAcGFyYW0gRXhwb3J0ZXIgJGV4cG9ydGVyIFRoZSBleHBvcnRlciB0byBzZW5kIG1ldHJpY3MgdG8KICogQHBhcmFtIGludCAkYmF0Y2hTaXplIE51bWJlciBvZiBtZXRyaWNzIHRvIGNvbGxlY3QgYmVmb3JlIGV4cG9ydGluZyAoZGVmYXVsdCA1MTIpCiAqIEBwYXJhbSBFcnJvckhhbmRsZXIgJGVycm9ySGFuZGxlciBIYW5kbGVyIGZvciBUaHJvd2FibGVzIHJhaXNlZCBieSB0aGUgZXhwb3J0ZXIKICov"},{"repository_path":"src\/lib\/telemetry\/src\/Flow\/Telemetry\/DSL\/functions.php","start_line_in_file":569,"slug":"pass-through-metric-processor","name":"pass_through_metric_processor","namespace":"Flow\\Telemetry\\DSL","parameters":[{"name":"exporter","type":[{"name":"Exporter","namespace":"Flow\\Telemetry\\Exporter","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"errorHandler","type":[{"name":"ErrorHandler","namespace":"Flow\\Telemetry\\ErrorHandler","is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"Flow\\Telemetry\\ErrorHandler\\ErrorLogHandler::..."}],"return_type":[{"name":"PassThroughMetricProcessor","namespace":"Flow\\Telemetry\\Meter\\Processor","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"TELEMETRY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBhIFBhc3NUaHJvdWdoTWV0cmljUHJvY2Vzc29yLgogKgogKiBAcGFyYW0gRXhwb3J0ZXIgJGV4cG9ydGVyIFRoZSBleHBvcnRlciB0byBzZW5kIG1ldHJpY3MgdG8KICogQHBhcmFtIEVycm9ySGFuZGxlciAkZXJyb3JIYW5kbGVyIEhhbmRsZXIgZm9yIFRocm93YWJsZXMgcmFpc2VkIGJ5IHRoZSBleHBvcnRlcgogKi8="},{"repository_path":"src\/lib\/telemetry\/src\/Flow\/Telemetry\/DSL\/functions.php","start_line_in_file":584,"slug":"batching-log-processor","name":"batching_log_processor","namespace":"Flow\\Telemetry\\DSL","parameters":[{"name":"exporter","type":[{"name":"Exporter","namespace":"Flow\\Telemetry\\Exporter","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"batchSize","type":[{"name":"int","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"512"},{"name":"errorHandler","type":[{"name":"ErrorHandler","namespace":"Flow\\Telemetry\\ErrorHandler","is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"Flow\\Telemetry\\ErrorHandler\\ErrorLogHandler::..."}],"return_type":[{"name":"BatchingLogProcessor","namespace":"Flow\\Telemetry\\Logger\\Processor","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"TELEMETRY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBhIEJhdGNoaW5nTG9nUHJvY2Vzc29yLgogKgogKiBAcGFyYW0gRXhwb3J0ZXIgJGV4cG9ydGVyIFRoZSBleHBvcnRlciB0byBzZW5kIGxvZ3MgdG8KICogQHBhcmFtIGludCAkYmF0Y2hTaXplIE51bWJlciBvZiBsb2dzIHRvIGNvbGxlY3QgYmVmb3JlIGV4cG9ydGluZyAoZGVmYXVsdCA1MTIpCiAqIEBwYXJhbSBFcnJvckhhbmRsZXIgJGVycm9ySGFuZGxlciBIYW5kbGVyIGZvciBUaHJvd2FibGVzIHJhaXNlZCBieSB0aGUgZXhwb3J0ZXIKICov"},{"repository_path":"src\/lib\/telemetry\/src\/Flow\/Telemetry\/DSL\/functions.php","start_line_in_file":599,"slug":"pass-through-log-processor","name":"pass_through_log_processor","namespace":"Flow\\Telemetry\\DSL","parameters":[{"name":"exporter","type":[{"name":"Exporter","namespace":"Flow\\Telemetry\\Exporter","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"errorHandler","type":[{"name":"ErrorHandler","namespace":"Flow\\Telemetry\\ErrorHandler","is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"Flow\\Telemetry\\ErrorHandler\\ErrorLogHandler::..."}],"return_type":[{"name":"PassThroughLogProcessor","namespace":"Flow\\Telemetry\\Logger\\Processor","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"TELEMETRY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBhIFBhc3NUaHJvdWdoTG9nUHJvY2Vzc29yLgogKgogKiBAcGFyYW0gRXhwb3J0ZXIgJGV4cG9ydGVyIFRoZSBleHBvcnRlciB0byBzZW5kIGxvZ3MgdG8KICogQHBhcmFtIEVycm9ySGFuZGxlciAkZXJyb3JIYW5kbGVyIEhhbmRsZXIgZm9yIFRocm93YWJsZXMgcmFpc2VkIGJ5IHRoZSBleHBvcnRlcgogKi8="},{"repository_path":"src\/lib\/telemetry\/src\/Flow\/Telemetry\/DSL\/functions.php","start_line_in_file":614,"slug":"pipeline-log-processor","name":"pipeline_log_processor","namespace":"Flow\\Telemetry\\DSL","parameters":[{"name":"middleware","type":[{"name":"array","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"sink","type":[{"name":"LogSink","namespace":"Flow\\Telemetry\\Logger","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"PipelineLogProcessor","namespace":"Flow\\Telemetry\\Logger\\Processor","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"TELEMETRY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBhIFBpcGVsaW5lTG9nUHJvY2Vzc29yOiBydW4gZWFjaCBsb2cgZW50cnkgdGhyb3VnaCBhbiBvcmRlcmVkIGNoYWluIG9mCiAqIG1pZGRsZXdhcmUsIHRoZW4gZm9yd2FyZCB0aGUgc3Vydml2b3JzIHRvIGEgc2luZ2xlIHNpbmsuCiAqCiAqIEBwYXJhbSBsaXN0PExvZ01pZGRsZXdhcmU+ICRtaWRkbGV3YXJlIHJ1biBpbiBvcmRlcjsgdGhlIGZpcnN0IHRvIGRyb3AgYW4gZW50cnkgc2hvcnQtY2lyY3VpdHMgdGhlIHJlc3QKICogQHBhcmFtIExvZ1NpbmsgJHNpbmsgdGhlIHRlcm1pbmFsIHByb2Nlc3NvciB0aGF0IGV4cG9ydHMgc3Vydml2aW5nIGVudHJpZXMKICov"},{"repository_path":"src\/lib\/telemetry\/src\/Flow\/Telemetry\/DSL\/functions.php","start_line_in_file":626,"slug":"enriching-log-middleware","name":"enriching_log_middleware","namespace":"Flow\\Telemetry\\DSL","parameters":[{"name":"attributes","type":[{"name":"Attributes","namespace":"Flow\\Telemetry","is_nullable":false,"is_variadic":false},{"name":"array","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"EnrichingLogMiddleware","namespace":"Flow\\Telemetry\\Logger\\Middleware","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"TELEMETRY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBhbiBFbnJpY2hpbmdMb2dNaWRkbGV3YXJlIHRoYXQgbWVyZ2VzIGRlZmF1bHQgYXR0cmlidXRlcyBpbnRvIGV2ZXJ5IGxvZwogKiBlbnRyeS4gQXR0cmlidXRlcyBzZXQgYXQgdGhlIGNhbGwgc2l0ZSB3aW4gb3ZlciB0aGVzZSBkZWZhdWx0cy4KICoKICogQHBhcmFtIGFycmF5PHN0cmluZywgYXJyYXk8YXJyYXkta2V5LCBtaXhlZD58Ym9vbHxcRGF0ZVRpbWVJbnRlcmZhY2V8ZmxvYXR8aW50fHN0cmluZ3xcVGhyb3dhYmxlPnxBdHRyaWJ1dGVzICRhdHRyaWJ1dGVzCiAqLw=="},{"repository_path":"src\/lib\/telemetry\/src\/Flow\/Telemetry\/DSL\/functions.php","start_line_in_file":637,"slug":"attribute-filtering-log-middleware","name":"attribute_filtering_log_middleware","namespace":"Flow\\Telemetry\\DSL","parameters":[{"name":"filter","type":[{"name":"AttributeFilter","namespace":"Flow\\Telemetry\\Filter","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"AttributeFilteringLogMiddleware","namespace":"Flow\\Telemetry\\Logger\\Middleware","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"TELEMETRY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBhbiBBdHRyaWJ1dGVGaWx0ZXJpbmdMb2dNaWRkbGV3YXJlIHRoYXQgZHJvcHMgbG9nIGVudHJpZXMgbWF0Y2hpbmcgdGhlIGZpbHRlci4KICoKICogQHBhcmFtIEF0dHJpYnV0ZUZpbHRlciAkZmlsdGVyIFRoZSBhdHRyaWJ1dGUgZmlsdGVyIHRvIGFwcGx5CiAqLw=="},{"repository_path":"src\/lib\/telemetry\/src\/Flow\/Telemetry\/DSL\/functions.php","start_line_in_file":648,"slug":"severity-filtering-log-middleware","name":"severity_filtering_log_middleware","namespace":"Flow\\Telemetry\\DSL","parameters":[{"name":"minimumSeverity","type":[{"name":"Severity","namespace":"Flow\\Telemetry\\Logger","is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"Flow\\Telemetry\\Logger\\Severity::..."}],"return_type":[{"name":"SeverityFilteringLogMiddleware","namespace":"Flow\\Telemetry\\Logger\\Middleware","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"TELEMETRY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBhIFNldmVyaXR5RmlsdGVyaW5nTG9nTWlkZGxld2FyZSB0aGF0IGRyb3BzIGxvZyBlbnRyaWVzIGJlbG93IGEgbWluaW11bSBzZXZlcml0eS4KICoKICogQHBhcmFtIFNldmVyaXR5ICRtaW5pbXVtU2V2ZXJpdHkgTWluaW11bSBzZXZlcml0eSBsZXZlbCAoZGVmYXVsdDogSU5GTykKICov"},{"repository_path":"src\/lib\/telemetry\/src\/Flow\/Telemetry\/DSL\/functions.php","start_line_in_file":662,"slug":"attribute-rule","name":"attribute_rule","namespace":"Flow\\Telemetry\\DSL","parameters":[{"name":"path","type":[{"name":"array","namespace":null,"is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"mode","type":[{"name":"MatchMode","namespace":"Flow\\Telemetry\\Filter","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"expected","type":[{"name":"DateTimeInterface","namespace":"","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false},{"name":"int","namespace":null,"is_nullable":false,"is_variadic":false},{"name":"float","namespace":null,"is_nullable":false,"is_variadic":false},{"name":"bool","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"caseSensitive","type":[{"name":"bool","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"true"}],"return_type":[{"name":"AttributeRule","namespace":"Flow\\Telemetry\\Filter","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"TELEMETRY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBhIHNpbmdsZSBhdHRyaWJ1dGUtbWF0Y2hpbmcgcnVsZSBmb3IgYW4gQXR0cmlidXRlRmlsdGVyLgogKgogKiBAcGFyYW0gYXJyYXk8c3RyaW5nPnxzdHJpbmcgJHBhdGggYXR0cmlidXRlIHBhdGg6IGEgdG9wLWxldmVsIGtleSwgb3Igc2VnbWVudHMgZGVzY2VuZGluZyBpbnRvIG5lc3RlZCBhcnJheSB2YWx1ZXMKICogQHBhcmFtIE1hdGNoTW9kZSAkbW9kZSBjb21wYXJpc29uIGFwcGxpZWQgYmV0d2VlbiB0aGUgdmFsdWUgYXQgdGhlIHBhdGggYW5kIHRoZSBleHBlY3RlZCB2YWx1ZQogKiBAcGFyYW0gYm9vbHxEYXRlVGltZUludGVyZmFjZXxmbG9hdHxpbnR8c3RyaW5nICRleHBlY3RlZCBleHBlY3RlZCB2YWx1ZSAobXVzdCBiZSBhIHN0cmluZyBmb3IgdGhlIHBhdHRlcm4gbW9kZXMpCiAqIEBwYXJhbSBib29sICRjYXNlU2Vuc2l0aXZlIGFwcGxpZXMgdG8gdGhlIHN1YnN0cmluZyBtb2RlcyBvbmx5IChTVEFSVFNfV0lUSCwgRU5EU19XSVRILCBDT05UQUlOUykKICov"},{"repository_path":"src\/lib\/telemetry\/src\/Flow\/Telemetry\/DSL\/functions.php","start_line_in_file":675,"slug":"all","name":"all","namespace":"Flow\\Telemetry\\DSL","parameters":[{"name":"matchers","type":[{"name":"Matcher","namespace":"Flow\\Telemetry\\Filter","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":true,"default_value":null}],"return_type":[{"name":"All","namespace":"Flow\\Telemetry\\Filter","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"TELEMETRY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENvbWJpbmUgbWF0Y2hlcnMgc28gdGhhdCBldmVyeSBvbmUgbXVzdCBtYXRjaCAobG9naWNhbCBBTkQpLgogKi8="},{"repository_path":"src\/lib\/telemetry\/src\/Flow\/Telemetry\/DSL\/functions.php","start_line_in_file":684,"slug":"any","name":"any","namespace":"Flow\\Telemetry\\DSL","parameters":[{"name":"matchers","type":[{"name":"Matcher","namespace":"Flow\\Telemetry\\Filter","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":true,"default_value":null}],"return_type":[{"name":"Any","namespace":"Flow\\Telemetry\\Filter","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"TELEMETRY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENvbWJpbmUgbWF0Y2hlcnMgc28gdGhhdCBhdCBsZWFzdCBvbmUgbXVzdCBtYXRjaCAobG9naWNhbCBPUikuCiAqLw=="},{"repository_path":"src\/lib\/telemetry\/src\/Flow\/Telemetry\/DSL\/functions.php","start_line_in_file":693,"slug":"not","name":"not","namespace":"Flow\\Telemetry\\DSL","parameters":[{"name":"matcher","type":[{"name":"Matcher","namespace":"Flow\\Telemetry\\Filter","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"Not","namespace":"Flow\\Telemetry\\Filter","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"TELEMETRY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIE5lZ2F0ZSBhIG1hdGNoZXIgKGxvZ2ljYWwgTk9UKS4KICov"},{"repository_path":"src\/lib\/telemetry\/src\/Flow\/Telemetry\/DSL\/functions.php","start_line_in_file":712,"slug":"attribute-filter","name":"attribute_filter","namespace":"Flow\\Telemetry\\DSL","parameters":[{"name":"matcher","type":[{"name":"Matcher","namespace":"Flow\\Telemetry\\Filter","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"exclude","type":[{"name":"bool","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"true"},{"name":"sources","type":[{"name":"array","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"[...]"},{"name":"cacheDir","type":[{"name":"string","namespace":null,"is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"null"},{"name":"cacheDirPermissions","type":[{"name":"int","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"448"}],"return_type":[{"name":"AttributeFilter","namespace":"Flow\\Telemetry\\Filter","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"TELEMETRY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBhbiBBdHRyaWJ1dGVGaWx0ZXIgZnJvbSBhIG1hdGNoZXIuCiAqCiAqIEBwYXJhbSBNYXRjaGVyICRtYXRjaGVyIHRoZSBtYXRjaGVyIHRvIGV2YWx1YXRlIGFnYWluc3QgYSBzaWduYWwncyBhdHRyaWJ1dGVzIChjb21wb3NlIHdpdGggYWxsKCksIGFueSgpLCBub3QoKSkKICogQHBhcmFtIGJvb2wgJGV4Y2x1ZGUgd2hlbiB0cnVlIChkZWZhdWx0KSBhIG1hdGNoIGRyb3BzIHRoZSBzaWduYWw7IHdoZW4gZmFsc2Ugb25seSBtYXRjaGluZyBzaWduYWxzIGFyZSBrZXB0CiAqIEBwYXJhbSBsaXN0PEF0dHJpYnV0ZVNvdXJjZT4gJHNvdXJjZXMgd2hpY2ggYXR0cmlidXRlIHNldHMgdG8gaW5zcGVjdCAoc2lnbmFsLCByZXNvdXJjZSBhbmQvb3Igc2NvcGUpOyB0aGUgbWF0Y2hlciBpcwogKiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIE9SLWNvbWJpbmVkIGFjcm9zcyB0aGVtLCBkZWZhdWx0aW5nIHRvIHRoZSBzaWduYWwncyBvd24gYXR0cmlidXRlcwogKiBAcGFyYW0gbnVsbHxzdHJpbmcgJGNhY2hlRGlyIGRpcmVjdG9yeSBmb3IgdGhlIGdlbmVyYXRlZCBtYXRjaGVyIGZpbGUgKGRlZmF1bHRzIHRvIHRoZSBzeXN0ZW0gdGVtcCBkaXJlY3RvcnkpLiBJdCBpcwogKiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIGByZXF1aXJlYGQsIHNvIGl0IE1VU1QgYmUgdHJ1c3RlZCAtIG5vdCB3cml0YWJsZSBieSB1bnRydXN0ZWQgdXNlcnMuIFByZWZlciBhbgogKiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIGFwcGxpY2F0aW9uLXByaXZhdGUgZGlyZWN0b3J5IG92ZXIgdGhlIHNoYXJlZCBzeXN0ZW0gdGVtcCBpbiBtdWx0aS10ZW5hbnQgZW52aXJvbm1lbnRzLgogKiBAcGFyYW0gaW50ICRjYWNoZURpclBlcm1pc3Npb25zIG1vZGUgYXBwbGllZCB3aGVuIHRoZSBjYWNoZSBkaXJlY3RvcnkgaXMgY3JlYXRlZCAob2N0YWwsIHN1YmplY3QgdG8gdW1hc2s7IGRlZmF1bHRzCiAqICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgdG8gMDcwMCAtIG93bmVyIG9ubHksIHNpbmNlIHRoZSBkaXJlY3RvcnkgaG9sZHMgYHJlcXVpcmVgZCBQSFApCiAqLw=="},{"repository_path":"src\/lib\/telemetry\/src\/Flow\/Telemetry\/DSL\/functions.php","start_line_in_file":729,"slug":"attribute-filtering-metric-processor","name":"attribute_filtering_metric_processor","namespace":"Flow\\Telemetry\\DSL","parameters":[{"name":"processor","type":[{"name":"MetricProcessor","namespace":"Flow\\Telemetry\\Meter","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"filter","type":[{"name":"AttributeFilter","namespace":"Flow\\Telemetry\\Filter","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"AttributeFilteringMetricProcessor","namespace":"Flow\\Telemetry\\Meter\\Processor","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"TELEMETRY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBhbiBBdHRyaWJ1dGVGaWx0ZXJpbmdNZXRyaWNQcm9jZXNzb3IuCiAqCiAqIEBwYXJhbSBNZXRyaWNQcm9jZXNzb3IgJHByb2Nlc3NvciBUaGUgcHJvY2Vzc29yIHRvIHdyYXAKICogQHBhcmFtIEF0dHJpYnV0ZUZpbHRlciAkZmlsdGVyIFRoZSBhdHRyaWJ1dGUgZmlsdGVyIHRvIGFwcGx5CiAqLw=="},{"repository_path":"src\/lib\/telemetry\/src\/Flow\/Telemetry\/DSL\/functions.php","start_line_in_file":743,"slug":"attribute-filtering-span-processor","name":"attribute_filtering_span_processor","namespace":"Flow\\Telemetry\\DSL","parameters":[{"name":"processor","type":[{"name":"SpanProcessor","namespace":"Flow\\Telemetry\\Tracer","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"filter","type":[{"name":"AttributeFilter","namespace":"Flow\\Telemetry\\Filter","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"AttributeFilteringSpanProcessor","namespace":"Flow\\Telemetry\\Tracer\\Processor","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"TELEMETRY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBhbiBBdHRyaWJ1dGVGaWx0ZXJpbmdTcGFuUHJvY2Vzc29yLgogKgogKiBAcGFyYW0gU3BhblByb2Nlc3NvciAkcHJvY2Vzc29yIFRoZSBwcm9jZXNzb3IgdG8gd3JhcAogKiBAcGFyYW0gQXR0cmlidXRlRmlsdGVyICRmaWx0ZXIgVGhlIGF0dHJpYnV0ZSBmaWx0ZXIgdG8gYXBwbHkKICov"},{"repository_path":"src\/lib\/telemetry\/src\/Flow\/Telemetry\/DSL\/functions.php","start_line_in_file":762,"slug":"console-exporter","name":"console_exporter","namespace":"Flow\\Telemetry\\DSL","parameters":[{"name":"colors","type":[{"name":"bool","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"true"},{"name":"maxLogBodyLength","type":[{"name":"int","namespace":null,"is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"100"},{"name":"logOptions","type":[{"name":"ConsoleLogOptions","namespace":"Flow\\Telemetry\\Provider\\Console","is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"Flow\\Telemetry\\Provider\\Console\\ConsoleLogOptions::..."},{"name":"metricOptions","type":[{"name":"ConsoleMetricOptions","namespace":"Flow\\Telemetry\\Provider\\Console","is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"Flow\\Telemetry\\Provider\\Console\\ConsoleMetricOptions::..."},{"name":"spanOptions","type":[{"name":"ConsoleSpanOptions","namespace":"Flow\\Telemetry\\Provider\\Console","is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"Flow\\Telemetry\\Provider\\Console\\ConsoleSpanOptions::..."}],"return_type":[{"name":"ConsoleExporter","namespace":"Flow\\Telemetry\\Provider\\Console","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"TELEMETRY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBhIHVuaWZpZWQgQ29uc29sZUV4cG9ydGVyIGZvciBsb2dzLCBtZXRyaWNzLCBhbmQgc3BhbnMuCiAqCiAqIE91dHB1dHMgdGVsZW1ldHJ5IHRvIHRoZSBjb25zb2xlIHdpdGggQVNDSUkgdGFibGUgZm9ybWF0dGluZyBhbmQgb3B0aW9uYWwgQU5TSSBjb2xvcnMuCiAqCiAqIEBwYXJhbSBib29sICRjb2xvcnMgV2hldGhlciB0byB1c2UgQU5TSSBjb2xvcnMgKGRlZmF1bHQ6IHRydWUpCiAqIEBwYXJhbSBudWxsfGludCAkbWF4TG9nQm9keUxlbmd0aCBNYXhpbXVtIGxlbmd0aCBmb3IgbG9nIGJvZHkrYXR0cmlidXRlcyBjb2x1bW4gKG51bGwgPSBubyBsaW1pdCkKICogQHBhcmFtIENvbnNvbGVMb2dPcHRpb25zICRsb2dPcHRpb25zIERpc3BsYXkgb3B0aW9ucyBmb3IgbG9nIHJlY29yZHMKICogQHBhcmFtIENvbnNvbGVNZXRyaWNPcHRpb25zICRtZXRyaWNPcHRpb25zIERpc3BsYXkgb3B0aW9ucyBmb3IgbWV0cmljcwogKiBAcGFyYW0gQ29uc29sZVNwYW5PcHRpb25zICRzcGFuT3B0aW9ucyBEaXNwbGF5IG9wdGlvbnMgZm9yIHNwYW5zCiAqLw=="},{"repository_path":"src\/lib\/telemetry\/src\/Flow\/Telemetry\/DSL\/functions.php","start_line_in_file":776,"slug":"console-span-options","name":"console_span_options","namespace":"Flow\\Telemetry\\DSL","parameters":[],"return_type":[{"name":"ConsoleSpanOptions","namespace":"Flow\\Telemetry\\Provider\\Console","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"TELEMETRY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBDb25zb2xlU3Bhbk9wdGlvbnMgd2l0aCBhbGwgZGlzcGxheSBvcHRpb25zIGVuYWJsZWQgKGRlZmF1bHQgYmVoYXZpb3IpLgogKi8="},{"repository_path":"src\/lib\/telemetry\/src\/Flow\/Telemetry\/DSL\/functions.php","start_line_in_file":785,"slug":"console-span-options-minimal","name":"console_span_options_minimal","namespace":"Flow\\Telemetry\\DSL","parameters":[],"return_type":[{"name":"ConsoleSpanOptions","namespace":"Flow\\Telemetry\\Provider\\Console","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"TELEMETRY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBDb25zb2xlU3Bhbk9wdGlvbnMgd2l0aCBtaW5pbWFsIGRpc3BsYXkgKGxlZ2FjeSBjb21wYWN0IGZvcm1hdCkuCiAqLw=="},{"repository_path":"src\/lib\/telemetry\/src\/Flow\/Telemetry\/DSL\/functions.php","start_line_in_file":794,"slug":"console-log-options","name":"console_log_options","namespace":"Flow\\Telemetry\\DSL","parameters":[],"return_type":[{"name":"ConsoleLogOptions","namespace":"Flow\\Telemetry\\Provider\\Console","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"TELEMETRY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBDb25zb2xlTG9nT3B0aW9ucyB3aXRoIGFsbCBkaXNwbGF5IG9wdGlvbnMgZW5hYmxlZCAoZGVmYXVsdCBiZWhhdmlvcikuCiAqLw=="},{"repository_path":"src\/lib\/telemetry\/src\/Flow\/Telemetry\/DSL\/functions.php","start_line_in_file":803,"slug":"console-log-options-minimal","name":"console_log_options_minimal","namespace":"Flow\\Telemetry\\DSL","parameters":[],"return_type":[{"name":"ConsoleLogOptions","namespace":"Flow\\Telemetry\\Provider\\Console","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"TELEMETRY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBDb25zb2xlTG9nT3B0aW9ucyB3aXRoIG1pbmltYWwgZGlzcGxheSAobGVnYWN5IGNvbXBhY3QgZm9ybWF0KS4KICov"},{"repository_path":"src\/lib\/telemetry\/src\/Flow\/Telemetry\/DSL\/functions.php","start_line_in_file":812,"slug":"console-metric-options","name":"console_metric_options","namespace":"Flow\\Telemetry\\DSL","parameters":[],"return_type":[{"name":"ConsoleMetricOptions","namespace":"Flow\\Telemetry\\Provider\\Console","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"TELEMETRY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBDb25zb2xlTWV0cmljT3B0aW9ucyB3aXRoIGFsbCBkaXNwbGF5IG9wdGlvbnMgZW5hYmxlZCAoZGVmYXVsdCBiZWhhdmlvcikuCiAqLw=="},{"repository_path":"src\/lib\/telemetry\/src\/Flow\/Telemetry\/DSL\/functions.php","start_line_in_file":821,"slug":"console-metric-options-minimal","name":"console_metric_options_minimal","namespace":"Flow\\Telemetry\\DSL","parameters":[],"return_type":[{"name":"ConsoleMetricOptions","namespace":"Flow\\Telemetry\\Provider\\Console","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"TELEMETRY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBDb25zb2xlTWV0cmljT3B0aW9ucyB3aXRoIG1pbmltYWwgZGlzcGxheSAobGVnYWN5IGNvbXBhY3QgZm9ybWF0KS4KICov"},{"repository_path":"src\/lib\/telemetry\/src\/Flow\/Telemetry\/DSL\/functions.php","start_line_in_file":830,"slug":"always-on-exemplar-filter","name":"always_on_exemplar_filter","namespace":"Flow\\Telemetry\\DSL","parameters":[],"return_type":[{"name":"AlwaysOnExemplarFilter","namespace":"Flow\\Telemetry\\Meter\\Exemplar","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"TELEMETRY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBhbiBBbHdheXNPbkV4ZW1wbGFyRmlsdGVyLgogKi8="},{"repository_path":"src\/lib\/telemetry\/src\/Flow\/Telemetry\/DSL\/functions.php","start_line_in_file":839,"slug":"always-off-exemplar-filter","name":"always_off_exemplar_filter","namespace":"Flow\\Telemetry\\DSL","parameters":[],"return_type":[{"name":"AlwaysOffExemplarFilter","namespace":"Flow\\Telemetry\\Meter\\Exemplar","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"TELEMETRY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBhbiBBbHdheXNPZmZFeGVtcGxhckZpbHRlci4KICov"},{"repository_path":"src\/lib\/telemetry\/src\/Flow\/Telemetry\/DSL\/functions.php","start_line_in_file":848,"slug":"trace-based-exemplar-filter","name":"trace_based_exemplar_filter","namespace":"Flow\\Telemetry\\DSL","parameters":[],"return_type":[{"name":"TraceBasedExemplarFilter","namespace":"Flow\\Telemetry\\Meter\\Exemplar","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"TELEMETRY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBhIFRyYWNlQmFzZWRFeGVtcGxhckZpbHRlci4KICov"},{"repository_path":"src\/lib\/telemetry\/src\/Flow\/Telemetry\/DSL\/functions.php","start_line_in_file":857,"slug":"always-on-sampler","name":"always_on_sampler","namespace":"Flow\\Telemetry\\DSL","parameters":[],"return_type":[{"name":"AlwaysOnSampler","namespace":"Flow\\Telemetry\\Tracer\\Sampler","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"TELEMETRY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBhbiBBbHdheXNPblNhbXBsZXIuIFJlY29yZHMgYW5kIHNhbXBsZXMgZXZlcnkgc3Bhbi4KICov"},{"repository_path":"src\/lib\/telemetry\/src\/Flow\/Telemetry\/DSL\/functions.php","start_line_in_file":866,"slug":"always-off-sampler","name":"always_off_sampler","namespace":"Flow\\Telemetry\\DSL","parameters":[],"return_type":[{"name":"AlwaysOffSampler","namespace":"Flow\\Telemetry\\Tracer\\Sampler","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"TELEMETRY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBhbiBBbHdheXNPZmZTYW1wbGVyLiBEcm9wcyBldmVyeSBzcGFuLgogKi8="},{"repository_path":"src\/lib\/telemetry\/src\/Flow\/Telemetry\/DSL\/functions.php","start_line_in_file":877,"slug":"trace-id-ratio-based-sampler","name":"trace_id_ratio_based_sampler","namespace":"Flow\\Telemetry\\DSL","parameters":[{"name":"ratio","type":[{"name":"float","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"TraceIdRatioBasedSampler","namespace":"Flow\\Telemetry\\Tracer\\Sampler","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"TELEMETRY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBhIFRyYWNlSWRSYXRpb0Jhc2VkU2FtcGxlci4gU2FtcGxlcyBhIGRldGVybWluaXN0aWMgZnJhY3Rpb24gb2YgdHJhY2VzLgogKgogKiBAcGFyYW0gZmxvYXQgJHJhdGlvIFNhbXBsaW5nIHByb2JhYmlsaXR5IGJldHdlZW4gMC4wIGFuZCAxLjAKICov"},{"repository_path":"src\/lib\/telemetry\/src\/Flow\/Telemetry\/DSL\/functions.php","start_line_in_file":889,"slug":"parent-based-sampler","name":"parent_based_sampler","namespace":"Flow\\Telemetry\\DSL","parameters":[{"name":"root","type":[{"name":"Sampler","namespace":"Flow\\Telemetry\\Tracer\\Sampler","is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"Flow\\Telemetry\\Tracer\\Sampler\\AlwaysOnSampler::..."}],"return_type":[{"name":"ParentBasedSampler","namespace":"Flow\\Telemetry\\Tracer\\Sampler","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"TELEMETRY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBhIFBhcmVudEJhc2VkU2FtcGxlci4gSG9ub3JzIHRoZSBwYXJlbnQgc3BhbidzIHNhbXBsaW5nIGRlY2lzaW9uLCBmYWxsaW5nCiAqIGJhY2sgdG8gdGhlIHJvb3Qgc2FtcGxlciBmb3Igc3BhbnMgd2l0aG91dCBhIHBhcmVudC4KICoKICogQHBhcmFtIFNhbXBsZXIgJHJvb3QgU2FtcGxlciB1c2VkIGZvciByb290IHNwYW5zIChubyBwYXJlbnQpCiAqLw=="},{"repository_path":"src\/lib\/telemetry\/src\/Flow\/Telemetry\/DSL\/functions.php","start_line_in_file":905,"slug":"attribute-matching-sampler","name":"attribute_matching_sampler","namespace":"Flow\\Telemetry\\DSL","parameters":[{"name":"filter","type":[{"name":"AttributeFilter","namespace":"Flow\\Telemetry\\Filter","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"delegate","type":[{"name":"Sampler","namespace":"Flow\\Telemetry\\Tracer\\Sampler","is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"Flow\\Telemetry\\Tracer\\Sampler\\AlwaysOnSampler::..."}],"return_type":[{"name":"AttributeMatchingSampler","namespace":"Flow\\Telemetry\\Tracer\\Sampler","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"TELEMETRY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBhbiBBdHRyaWJ1dGVNYXRjaGluZ1NhbXBsZXIuIERyb3BzIHNwYW5zIHdob3NlIHN0YXJ0LXRpbWUgYXR0cmlidXRlcyBtYXRjaAogKiB0aGUgZmlsdGVyIChvciBrZWVwcyBPTkxZIG1hdGNoaW5nIHNwYW5zIHdoZW4gdGhlIGZpbHRlcidzIGV4Y2x1ZGUgaXMgZmFsc2UpLCBhbmQKICogZGVmZXJzIGFsbCBvdGhlciBzcGFucyB0byB0aGUgZGVsZWdhdGUgc2FtcGxlci4KICoKICogT25seSBhdHRyaWJ1dGVzIGF2YWlsYWJsZSBhdCBzcGFuIHN0YXJ0IGFyZSB2aXNpYmxlOyBhdHRyaWJ1dGVzIGFkZGVkIGxhdGVyIGFyZSBub3QuCiAqCiAqIEBwYXJhbSBBdHRyaWJ1dGVGaWx0ZXIgJGZpbHRlciBUaGUgYXR0cmlidXRlIGZpbHRlciBldmFsdWF0ZWQgYWdhaW5zdCB0aGUgc3BhbidzIHN0YXJ0IGF0dHJpYnV0ZXMKICogQHBhcmFtIFNhbXBsZXIgJGRlbGVnYXRlIFNhbXBsZXIgdGhhdCBkZWNpZGVzIHNwYW5zIHdoaWNoIGRvIG5vdCBtYXRjaCAoZGVmYXVsdDogQWx3YXlzT25TYW1wbGVyKQogKi8="},{"repository_path":"src\/lib\/telemetry\/src\/Flow\/Telemetry\/DSL\/functions.php","start_line_in_file":919,"slug":"propagation-context","name":"propagation_context","namespace":"Flow\\Telemetry\\DSL","parameters":[{"name":"spanContext","type":[{"name":"SpanContext","namespace":"Flow\\Telemetry\\Tracer","is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"null"},{"name":"baggage","type":[{"name":"Baggage","namespace":"Flow\\Telemetry\\Context","is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"null"}],"return_type":[{"name":"PropagationContext","namespace":"Flow\\Telemetry\\Propagation","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"TELEMETRY","type":"TYPE"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBhIFByb3BhZ2F0aW9uQ29udGV4dC4KICoKICogQHBhcmFtIG51bGx8U3BhbkNvbnRleHQgJHNwYW5Db250ZXh0IE9wdGlvbmFsIHNwYW4gY29udGV4dAogKiBAcGFyYW0gbnVsbHxCYWdnYWdlICRiYWdnYWdlIE9wdGlvbmFsIGJhZ2dhZ2UKICov"},{"repository_path":"src\/lib\/telemetry\/src\/Flow\/Telemetry\/DSL\/functions.php","start_line_in_file":930,"slug":"array-carrier","name":"array_carrier","namespace":"Flow\\Telemetry\\DSL","parameters":[{"name":"data","type":[{"name":"array","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"[]"}],"return_type":[{"name":"ArrayCarrier","namespace":"Flow\\Telemetry\\Propagation","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"TELEMETRY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBhbiBBcnJheUNhcnJpZXIuCiAqCiAqIEBwYXJhbSBhcnJheTxzdHJpbmcsIHN0cmluZz4gJGRhdGEgSW5pdGlhbCBjYXJyaWVyIGRhdGEKICov"},{"repository_path":"src\/lib\/telemetry\/src\/Flow\/Telemetry\/DSL\/functions.php","start_line_in_file":939,"slug":"superglobal-carrier","name":"superglobal_carrier","namespace":"Flow\\Telemetry\\DSL","parameters":[],"return_type":[{"name":"SuperglobalCarrier","namespace":"Flow\\Telemetry\\Propagation","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"TELEMETRY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBhIFN1cGVyZ2xvYmFsQ2Fycmllci4KICov"},{"repository_path":"src\/lib\/telemetry\/src\/Flow\/Telemetry\/DSL\/functions.php","start_line_in_file":948,"slug":"w3c-trace-context","name":"w3c_trace_context","namespace":"Flow\\Telemetry\\DSL","parameters":[],"return_type":[{"name":"W3CTraceContext","namespace":"Flow\\Telemetry\\Propagation","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"TELEMETRY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBhIFczQ1RyYWNlQ29udGV4dCBwcm9wYWdhdG9yLgogKi8="},{"repository_path":"src\/lib\/telemetry\/src\/Flow\/Telemetry\/DSL\/functions.php","start_line_in_file":957,"slug":"w3c-baggage","name":"w3c_baggage","namespace":"Flow\\Telemetry\\DSL","parameters":[],"return_type":[{"name":"W3CBaggage","namespace":"Flow\\Telemetry\\Propagation","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"TELEMETRY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBhIFczQ0JhZ2dhZ2UgcHJvcGFnYXRvci4KICov"},{"repository_path":"src\/lib\/telemetry\/src\/Flow\/Telemetry\/DSL\/functions.php","start_line_in_file":968,"slug":"composite-propagator","name":"composite_propagator","namespace":"Flow\\Telemetry\\DSL","parameters":[{"name":"propagators","type":[{"name":"Propagator","namespace":"Flow\\Telemetry\\Propagation","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":true,"default_value":null}],"return_type":[{"name":"CompositePropagator","namespace":"Flow\\Telemetry\\Propagation","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"TELEMETRY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBhIENvbXBvc2l0ZVByb3BhZ2F0b3IuCiAqCiAqIEBwYXJhbSBQcm9wYWdhdG9yIC4uLiRwcm9wYWdhdG9ycyBUaGUgcHJvcGFnYXRvcnMgdG8gY29tYmluZQogKi8="},{"repository_path":"src\/lib\/telemetry\/src\/Flow\/Telemetry\/DSL\/functions.php","start_line_in_file":979,"slug":"chain-detector","name":"chain_detector","namespace":"Flow\\Telemetry\\DSL","parameters":[{"name":"detectors","type":[{"name":"ResourceDetector","namespace":"Flow\\Telemetry\\Resource","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":true,"default_value":null}],"return_type":[{"name":"ChainDetector","namespace":"Flow\\Telemetry\\Resource\\Detector","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"TELEMETRY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBhIENoYWluRGV0ZWN0b3IuCiAqCiAqIEBwYXJhbSBSZXNvdXJjZURldGVjdG9yIC4uLiRkZXRlY3RvcnMgVGhlIGRldGVjdG9ycyB0byBjaGFpbgogKi8="},{"repository_path":"src\/lib\/telemetry\/src\/Flow\/Telemetry\/DSL\/functions.php","start_line_in_file":988,"slug":"os-detector","name":"os_detector","namespace":"Flow\\Telemetry\\DSL","parameters":[],"return_type":[{"name":"OsDetector","namespace":"Flow\\Telemetry\\Resource\\Detector","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"TELEMETRY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBhbiBPc0RldGVjdG9yLgogKi8="},{"repository_path":"src\/lib\/telemetry\/src\/Flow\/Telemetry\/DSL\/functions.php","start_line_in_file":997,"slug":"host-detector","name":"host_detector","namespace":"Flow\\Telemetry\\DSL","parameters":[],"return_type":[{"name":"HostDetector","namespace":"Flow\\Telemetry\\Resource\\Detector","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"TELEMETRY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBhIEhvc3REZXRlY3Rvci4KICov"},{"repository_path":"src\/lib\/telemetry\/src\/Flow\/Telemetry\/DSL\/functions.php","start_line_in_file":1006,"slug":"process-detector","name":"process_detector","namespace":"Flow\\Telemetry\\DSL","parameters":[],"return_type":[{"name":"ProcessDetector","namespace":"Flow\\Telemetry\\Resource\\Detector","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"TELEMETRY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBhIFByb2Nlc3NEZXRlY3Rvci4KICov"},{"repository_path":"src\/lib\/telemetry\/src\/Flow\/Telemetry\/DSL\/functions.php","start_line_in_file":1015,"slug":"environment-detector","name":"environment_detector","namespace":"Flow\\Telemetry\\DSL","parameters":[],"return_type":[{"name":"EnvironmentDetector","namespace":"Flow\\Telemetry\\Resource\\Detector","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"TELEMETRY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBhbiBFbnZpcm9ubWVudERldGVjdG9yLgogKi8="},{"repository_path":"src\/lib\/telemetry\/src\/Flow\/Telemetry\/DSL\/functions.php","start_line_in_file":1024,"slug":"composer-detector","name":"composer_detector","namespace":"Flow\\Telemetry\\DSL","parameters":[],"return_type":[{"name":"ComposerDetector","namespace":"Flow\\Telemetry\\Resource\\Detector","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"TELEMETRY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBhIENvbXBvc2VyRGV0ZWN0b3IuCiAqLw=="},{"repository_path":"src\/lib\/telemetry\/src\/Flow\/Telemetry\/DSL\/functions.php","start_line_in_file":1036,"slug":"git-detector","name":"git_detector","namespace":"Flow\\Telemetry\\DSL","parameters":[{"name":"workingDirectory","type":[{"name":"string","namespace":null,"is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"null"},{"name":"gitBinary","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"'git'"}],"return_type":[{"name":"GitDetector","namespace":"Flow\\Telemetry\\Resource\\Detector","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"TELEMETRY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBhIEdpdERldGVjdG9yLgogKgogKiBAcGFyYW0gbnVsbHxzdHJpbmcgJHdvcmtpbmdEaXJlY3RvcnkgRGlyZWN0b3J5IHRvIHJ1biBnaXQgaW4gKGRlZmF1bHQ6IGN1cnJlbnQgd29ya2luZyBkaXJlY3RvcnkpCiAqIEBwYXJhbSBzdHJpbmcgJGdpdEJpbmFyeSBQYXRoIHRvIHRoZSBnaXQgYmluYXJ5IChkZWZhdWx0OiAiZ2l0IiwgcmVzb2x2ZWQgZnJvbSAkUEFUSCkKICov"},{"repository_path":"src\/lib\/telemetry\/src\/Flow\/Telemetry\/DSL\/functions.php","start_line_in_file":1047,"slug":"manual-detector","name":"manual_detector","namespace":"Flow\\Telemetry\\DSL","parameters":[{"name":"attributes","type":[{"name":"array","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"ManualDetector","namespace":"Flow\\Telemetry\\Resource\\Detector","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"TELEMETRY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBhIE1hbnVhbERldGVjdG9yLgogKgogKiBAcGFyYW0gYXJyYXk8c3RyaW5nLCBhcnJheTxhcnJheS1rZXksIG1peGVkPnxib29sfFxEYXRlVGltZUludGVyZmFjZXxmbG9hdHxpbnR8c3RyaW5nfFxUaHJvd2FibGU+ICRhdHRyaWJ1dGVzIFJlc291cmNlIGF0dHJpYnV0ZXMKICov"},{"repository_path":"src\/lib\/telemetry\/src\/Flow\/Telemetry\/DSL\/functions.php","start_line_in_file":1059,"slug":"caching-detector","name":"caching_detector","namespace":"Flow\\Telemetry\\DSL","parameters":[{"name":"detector","type":[{"name":"ResourceDetector","namespace":"Flow\\Telemetry\\Resource","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"cachePath","type":[{"name":"string","namespace":null,"is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"null"}],"return_type":[{"name":"CachingDetector","namespace":"Flow\\Telemetry\\Resource\\Detector","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"TELEMETRY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBhIENhY2hpbmdEZXRlY3Rvci4KICoKICogQHBhcmFtIFJlc291cmNlRGV0ZWN0b3IgJGRldGVjdG9yIFRoZSBkZXRlY3RvciB0byB3cmFwCiAqIEBwYXJhbSBudWxsfHN0cmluZyAkY2FjaGVQYXRoIENhY2hlIGZpbGUgcGF0aCAoZGVmYXVsdDogc3lzX2dldF90ZW1wX2RpcigpL2Zsb3dfdGVsZW1ldHJ5X3Jlc291cmNlLmNhY2hlKQogKi8="},{"repository_path":"src\/lib\/telemetry\/src\/Flow\/Telemetry\/DSL\/functions.php","start_line_in_file":1070,"slug":"resource-detector","name":"resource_detector","namespace":"Flow\\Telemetry\\DSL","parameters":[{"name":"detectors","type":[{"name":"array","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"[]"}],"return_type":[{"name":"ChainDetector","namespace":"Flow\\Telemetry\\Resource\\Detector","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"TELEMETRY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBhIHJlc291cmNlIGRldGVjdG9yIGNoYWluLgogKgogKiBAcGFyYW0gYXJyYXk8UmVzb3VyY2VEZXRlY3Rvcj4gJGRldGVjdG9ycyBPcHRpb25hbCBjdXN0b20gZGV0ZWN0b3JzIChlbXB0eSA9IHVzZSBkZWZhdWx0cykKICov"},{"repository_path":"src\/lib\/telemetry\/src\/Flow\/Telemetry\/DSL\/functions.php","start_line_in_file":1089,"slug":"error-log-handler","name":"error_log_handler","namespace":"Flow\\Telemetry\\DSL","parameters":[{"name":"messageType","type":[{"name":"ErrorLogMessageType","namespace":"Flow\\Telemetry\\ErrorHandler","is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"Flow\\Telemetry\\ErrorHandler\\ErrorLogMessageType::..."},{"name":"expandNewlines","type":[{"name":"bool","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"false"},{"name":"messagePrefix","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"'[flow-telemetry]'"}],"return_type":[{"name":"ErrorLogHandler","namespace":"Flow\\Telemetry\\ErrorHandler","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"TELEMETRY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSB0aGUgZGVmYXVsdCBFcnJvckxvZ0hhbmRsZXIuIFdyaXRlcyB2aWEgUEhQJ3MgZXJyb3JfbG9nKCkuCiAqLw=="},{"repository_path":"src\/lib\/telemetry\/src\/Flow\/Telemetry\/DSL\/functions.php","start_line_in_file":1102,"slug":"stream-error-handler","name":"stream_error_handler","namespace":"Flow\\Telemetry\\DSL","parameters":[{"name":"destination","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"filePermissions","type":[{"name":"int","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"420"},{"name":"createDirectories","type":[{"name":"bool","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"true"},{"name":"messagePrefix","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"'[flow-telemetry]'"}],"return_type":[{"name":"StreamHandler","namespace":"Flow\\Telemetry\\ErrorHandler","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"TELEMETRY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBhIFN0cmVhbUhhbmRsZXIuIEFwcGVuZHMgZm9ybWF0dGVkIFRocm93YWJsZXMgKG9uZSBwZXIgbGluZSkgdG8gYSBmaWxlCiAqIHBhdGggb3IgcGhwOi8vIHN0cmVhbSB3cmFwcGVyLgogKi8="},{"repository_path":"src\/lib\/telemetry\/src\/Flow\/Telemetry\/DSL\/functions.php","start_line_in_file":1115,"slug":"syslog-error-handler","name":"syslog_error_handler","namespace":"Flow\\Telemetry\\DSL","parameters":[{"name":"ident","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"'flow-telemetry'"},{"name":"facility","type":[{"name":"SyslogFacility","namespace":"Flow\\Telemetry\\ErrorHandler","is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"Flow\\Telemetry\\ErrorHandler\\SyslogFacility::..."},{"name":"logOpts","type":[{"name":"int","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"1"},{"name":"severity","type":[{"name":"SyslogSeverity","namespace":"Flow\\Telemetry\\ErrorHandler","is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"Flow\\Telemetry\\ErrorHandler\\SyslogSeverity::..."}],"return_type":[{"name":"SyslogHandler","namespace":"Flow\\Telemetry\\ErrorHandler","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"TELEMETRY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBhIFN5c2xvZ0hhbmRsZXIuIFdyaXRlcyB2aWEgb3BlbmxvZy9zeXNsb2cvY2xvc2Vsb2cuCiAqLw=="},{"repository_path":"src\/lib\/telemetry\/src\/Flow\/Telemetry\/DSL\/functions.php","start_line_in_file":1128,"slug":"udp-syslog-error-handler","name":"udp_syslog_error_handler","namespace":"Flow\\Telemetry\\DSL","parameters":[{"name":"host","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"port","type":[{"name":"int","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"514"},{"name":"ident","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"'flow-telemetry'"},{"name":"facility","type":[{"name":"SyslogFacility","namespace":"Flow\\Telemetry\\ErrorHandler","is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"Flow\\Telemetry\\ErrorHandler\\SyslogFacility::..."},{"name":"severity","type":[{"name":"SyslogSeverity","namespace":"Flow\\Telemetry\\ErrorHandler","is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"Flow\\Telemetry\\ErrorHandler\\SyslogSeverity::..."}],"return_type":[{"name":"UdpSyslogHandler","namespace":"Flow\\Telemetry\\ErrorHandler","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"TELEMETRY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBhIFVkcFN5c2xvZ0hhbmRsZXIuIFNlbmRzIFJGQyA1NDI0LXN0eWxlIHN5c2xvZyBmcmFtZXMgb3ZlciBVRFAuCiAqLw=="},{"repository_path":"src\/lib\/telemetry\/src\/Flow\/Telemetry\/DSL\/functions.php","start_line_in_file":1142,"slug":"composite-error-handler","name":"composite_error_handler","namespace":"Flow\\Telemetry\\DSL","parameters":[{"name":"handlers","type":[{"name":"ErrorHandler","namespace":"Flow\\Telemetry\\ErrorHandler","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":true,"default_value":null}],"return_type":[{"name":"CompositeErrorHandler","namespace":"Flow\\Telemetry\\ErrorHandler","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"TELEMETRY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIEZhbiBlcnJvcnMgb3V0IHRvIG11bHRpcGxlIGhhbmRsZXJzLgogKi8="},{"repository_path":"src\/lib\/telemetry\/src\/Flow\/Telemetry\/DSL\/functions.php","start_line_in_file":1151,"slug":"null-error-handler","name":"null_error_handler","namespace":"Flow\\Telemetry\\DSL","parameters":[],"return_type":[{"name":"NullErrorHandler","namespace":"Flow\\Telemetry\\ErrorHandler","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"TELEMETRY","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIERpc2NhcmQgZXZlcnkgZXJyb3IuIFVzZSBvbmx5IGluIHRlc3RzIG9yIGZvciBleHBsaWNpdCBzaWxlbmNlLgogKi8="},{"repository_path":"src\/lib\/azure-sdk\/src\/Flow\/Azure\/SDK\/DSL\/functions.php","start_line_in_file":29,"slug":"azurite-url-factory","name":"azurite_url_factory","namespace":"Flow\\Azure\\SDK\\DSL","parameters":[{"name":"host","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"'localhost'"},{"name":"port","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"'10000'"},{"name":"secure","type":[{"name":"bool","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"false"}],"return_type":[{"name":"AzuriteURLFactory","namespace":"Flow\\Azure\\SDK\\BlobService\\URLFactory","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"AZURE_SDK","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/lib\/azure-sdk\/src\/Flow\/Azure\/SDK\/DSL\/functions.php","start_line_in_file":38,"slug":"azure-shared-key-authorization-factory","name":"azure_shared_key_authorization_factory","namespace":"Flow\\Azure\\SDK\\DSL","parameters":[{"name":"account","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"key","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"SharedKeyFactory","namespace":"Flow\\Azure\\SDK\\AuthorizationFactory","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"AZURE_SDK","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/lib\/azure-sdk\/src\/Flow\/Azure\/SDK\/DSL\/functions.php","start_line_in_file":48,"slug":"azure-blob-service-config","name":"azure_blob_service_config","namespace":"Flow\\Azure\\SDK\\DSL","parameters":[{"name":"account","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"container","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"Configuration","namespace":"Flow\\Azure\\SDK\\BlobService","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"AZURE_SDK","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/lib\/azure-sdk\/src\/Flow\/Azure\/SDK\/DSL\/functions.php","start_line_in_file":54,"slug":"azure-url-factory","name":"azure_url_factory","namespace":"Flow\\Azure\\SDK\\DSL","parameters":[{"name":"host","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"'blob.core.windows.net'"}],"return_type":[{"name":"AzureURLFactory","namespace":"Flow\\Azure\\SDK\\BlobService\\URLFactory","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"AZURE_SDK","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/lib\/azure-sdk\/src\/Flow\/Azure\/SDK\/DSL\/functions.php","start_line_in_file":60,"slug":"azure-http-factory","name":"azure_http_factory","namespace":"Flow\\Azure\\SDK\\DSL","parameters":[{"name":"request_factory","type":[{"name":"RequestFactoryInterface","namespace":"Psr\\Http\\Message","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"stream_factory","type":[{"name":"StreamFactoryInterface","namespace":"Psr\\Http\\Message","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"HttpFactory","namespace":"Flow\\Azure\\SDK","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"AZURE_SDK","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/lib\/azure-sdk\/src\/Flow\/Azure\/SDK\/DSL\/functions.php","start_line_in_file":68,"slug":"azure-blob-service","name":"azure_blob_service","namespace":"Flow\\Azure\\SDK\\DSL","parameters":[{"name":"configuration","type":[{"name":"Configuration","namespace":"Flow\\Azure\\SDK\\BlobService","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"azure_authorization_factory","type":[{"name":"AuthorizationFactory","namespace":"Flow\\Azure\\SDK","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"client","type":[{"name":"ClientInterface","namespace":"Psr\\Http\\Client","is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"null"},{"name":"azure_http_factory","type":[{"name":"HttpFactory","namespace":"Flow\\Azure\\SDK","is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"null"},{"name":"azure_url_factory","type":[{"name":"URLFactory","namespace":"Flow\\Azure\\SDK","is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"null"},{"name":"logger","type":[{"name":"LoggerInterface","namespace":"Psr\\Log","is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"null"}],"return_type":[{"name":"BlobServiceInterface","namespace":"Flow\\Azure\\SDK","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"AZURE_SDK","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/bridge\/filesystem\/azure\/src\/Flow\/Filesystem\/Bridge\/Azure\/DSL\/functions.php","start_line_in_file":16,"slug":"azure-filesystem-options","name":"azure_filesystem_options","namespace":"Flow\\Filesystem\\Bridge\\Azure\\DSL","parameters":[],"return_type":[{"name":"Options","namespace":"Flow\\Filesystem\\Bridge\\Azure","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"AZURE_FILESYSTEM","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/bridge\/filesystem\/azure\/src\/Flow\/Filesystem\/Bridge\/Azure\/DSL\/functions.php","start_line_in_file":22,"slug":"azure-filesystem","name":"azure_filesystem","namespace":"Flow\\Filesystem\\Bridge\\Azure\\DSL","parameters":[{"name":"blob_service","type":[{"name":"BlobServiceInterface","namespace":"Flow\\Azure\\SDK","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"options","type":[{"name":"Options","namespace":"Flow\\Filesystem\\Bridge\\Azure","is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"Flow\\Filesystem\\Bridge\\Azure\\Options::..."},{"name":"protocol","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"'azure-blob'"}],"return_type":[{"name":"AzureBlobFilesystem","namespace":"Flow\\Filesystem\\Bridge\\Azure","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"AZURE_FILESYSTEM","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/bridge\/filesystem\/async-aws\/src\/Flow\/Filesystem\/Bridge\/AsyncAWS\/DSL\/functions.php","start_line_in_file":20,"slug":"aws-s3-client","name":"aws_s3_client","namespace":"Flow\\Filesystem\\Bridge\\AsyncAWS\\DSL","parameters":[{"name":"configuration","type":[{"name":"array","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"S3Client","namespace":"AsyncAws\\S3","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"S3_FILESYSTEM","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIEBwYXJhbSBhcnJheTxDb25maWd1cmF0aW9uOjpPUFRJT05fKiwgbnVsbHxzdHJpbmc+ICRjb25maWd1cmF0aW9uIC0gZm9yIGRldGFpbHMgcGxlYXNlIHNlZSBodHRwczovL2FzeW5jLWF3cy5jb20vY2xpZW50cy9zMy5odG1sCiAqLw=="},{"repository_path":"src\/bridge\/filesystem\/async-aws\/src\/Flow\/Filesystem\/Bridge\/AsyncAWS\/DSL\/functions.php","start_line_in_file":26,"slug":"aws-s3-filesystem","name":"aws_s3_filesystem","namespace":"Flow\\Filesystem\\Bridge\\AsyncAWS\\DSL","parameters":[{"name":"bucket","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"s3Client","type":[{"name":"S3Client","namespace":"AsyncAws\\S3","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"options","type":[{"name":"Options","namespace":"Flow\\Filesystem\\Bridge\\AsyncAWS","is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"Flow\\Filesystem\\Bridge\\AsyncAWS\\Options::..."},{"name":"protocol","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"'aws-s3'"}],"return_type":[{"name":"AsyncAWSS3Filesystem","namespace":"Flow\\Filesystem\\Bridge\\AsyncAWS","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"S3_FILESYSTEM","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/bridge\/filesystem\/sftp\/src\/Flow\/Filesystem\/Bridge\/SFTP\/DSL\/functions.php","start_line_in_file":24,"slug":"sftp-client","name":"sftp_client","namespace":"Flow\\Filesystem\\Bridge\\SFTP\\DSL","parameters":[{"name":"host","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"user","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"credential","type":[{"name":"PrivateKey","namespace":"phpseclib4\\Crypt\\Common","is_nullable":false,"is_variadic":false},{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"port","type":[{"name":"int","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"22"}],"return_type":[{"name":"SFTP","namespace":"phpseclib4\\Net","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"SFTP_FILESYSTEM","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIEB0aHJvd3MgUnVudGltZUV4Y2VwdGlvbgogKi8="},{"repository_path":"src\/bridge\/filesystem\/sftp\/src\/Flow\/Filesystem\/Bridge\/SFTP\/DSL\/functions.php","start_line_in_file":46,"slug":"sftp-filesystem","name":"sftp_filesystem","namespace":"Flow\\Filesystem\\Bridge\\SFTP\\DSL","parameters":[{"name":"sftp","type":[{"name":"SFTP","namespace":"phpseclib4\\Net","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"options","type":[{"name":"Options","namespace":"Flow\\Filesystem\\Bridge\\SFTP","is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"Flow\\Filesystem\\Bridge\\SFTP\\Options::..."},{"name":"protocol","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"'sftp'"}],"return_type":[{"name":"SFTPFilesystem","namespace":"Flow\\Filesystem\\Bridge\\SFTP","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"SFTP_FILESYSTEM","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/bridge\/filesystem\/sftp\/src\/Flow\/Filesystem\/Bridge\/SFTP\/DSL\/functions.php","start_line_in_file":52,"slug":"sftp-filesystem-options","name":"sftp_filesystem_options","namespace":"Flow\\Filesystem\\Bridge\\SFTP\\DSL","parameters":[],"return_type":[{"name":"Options","namespace":"Flow\\Filesystem\\Bridge\\SFTP","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"SFTP_FILESYSTEM","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/bridge\/monolog\/telemetry\/src\/Flow\/Bridge\/Monolog\/Telemetry\/DSL\/functions.php","start_line_in_file":40,"slug":"value-normalizer","name":"value_normalizer","namespace":"Flow\\Bridge\\Monolog\\Telemetry\\DSL","parameters":[],"return_type":[{"name":"ValueNormalizer","namespace":"Flow\\Bridge\\Monolog\\Telemetry","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"MONOLOG_TELEMETRY_BRIDGE","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBhIFZhbHVlTm9ybWFsaXplciBmb3IgY29udmVydGluZyBhcmJpdHJhcnkgUEhQIHZhbHVlcyB0byBUZWxlbWV0cnkgYXR0cmlidXRlIHR5cGVzLgogKgogKiBUaGUgbm9ybWFsaXplciBoYW5kbGVzOgogKiAtIG51bGwg4oaSICdudWxsJyBzdHJpbmcKICogLSBzY2FsYXJzIChzdHJpbmcsIGludCwgZmxvYXQsIGJvb2wpIOKGkiB1bmNoYW5nZWQKICogLSBEYXRlVGltZUludGVyZmFjZSDihpIgdW5jaGFuZ2VkCiAqIC0gVGhyb3dhYmxlIOKGkiB1bmNoYW5nZWQKICogLSBhcnJheXMg4oaSIHJlY3Vyc2l2ZWx5IG5vcm1hbGl6ZWQKICogLSBvYmplY3RzIHdpdGggX190b1N0cmluZygpIOKGkiBzdHJpbmcgY2FzdAogKiAtIG9iamVjdHMgd2l0aG91dCBfX3RvU3RyaW5nKCkg4oaSIGNsYXNzIG5hbWUKICogLSBvdGhlciB0eXBlcyDihpIgZ2V0X2RlYnVnX3R5cGUoKSByZXN1bHQKICoKICogRXhhbXBsZSB1c2FnZToKICogYGBgcGhwCiAqICRub3JtYWxpemVyID0gdmFsdWVfbm9ybWFsaXplcigpOwogKiAkbm9ybWFsaXplZCA9ICRub3JtYWxpemVyLT5ub3JtYWxpemUoJHZhbHVlKTsKICogYGBgCiAqLw=="},{"repository_path":"src\/bridge\/monolog\/telemetry\/src\/Flow\/Bridge\/Monolog\/Telemetry\/DSL\/functions.php","start_line_in_file":73,"slug":"severity-mapper","name":"severity_mapper","namespace":"Flow\\Bridge\\Monolog\\Telemetry\\DSL","parameters":[{"name":"customMapping","type":[{"name":"array","namespace":null,"is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"null"}],"return_type":[{"name":"SeverityMapper","namespace":"Flow\\Bridge\\Monolog\\Telemetry","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"MONOLOG_TELEMETRY_BRIDGE","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBhIFNldmVyaXR5TWFwcGVyIGZvciBtYXBwaW5nIE1vbm9sb2cgbGV2ZWxzIHRvIFRlbGVtZXRyeSBzZXZlcml0aWVzLgogKgogKiBAcGFyYW0gbnVsbHxhcnJheTxpbnQsIFNldmVyaXR5PiAkY3VzdG9tTWFwcGluZyBPcHRpb25hbCBjdXN0b20gbWFwcGluZyAoTW9ub2xvZyBMZXZlbCB2YWx1ZSA9PiBUZWxlbWV0cnkgU2V2ZXJpdHkpCiAqCiAqIEV4YW1wbGUgd2l0aCBkZWZhdWx0IG1hcHBpbmc6CiAqIGBgYHBocAogKiAkbWFwcGVyID0gc2V2ZXJpdHlfbWFwcGVyKCk7CiAqIGBgYAogKgogKiBFeGFtcGxlIHdpdGggY3VzdG9tIG1hcHBpbmc6CiAqIGBgYHBocAogKiB1c2UgTW9ub2xvZ1xMZXZlbDsKICogdXNlIEZsb3dcVGVsZW1ldHJ5XExvZ2dlclxTZXZlcml0eTsKICoKICogJG1hcHBlciA9IHNldmVyaXR5X21hcHBlcihbCiAqICAgICBMZXZlbDo6RGVidWctPnZhbHVlID0+IFNldmVyaXR5OjpERUJVRywKICogICAgIExldmVsOjpJbmZvLT52YWx1ZSA9PiBTZXZlcml0eTo6SU5GTywKICogICAgIExldmVsOjpOb3RpY2UtPnZhbHVlID0+IFNldmVyaXR5OjpXQVJOLCAgLy8gQ3VzdG9tOiBOT1RJQ0Ug4oaSIFdBUk4gaW5zdGVhZCBvZiBJTkZPCiAqICAgICBMZXZlbDo6V2FybmluZy0+dmFsdWUgPT4gU2V2ZXJpdHk6OldBUk4sCiAqICAgICBMZXZlbDo6RXJyb3ItPnZhbHVlID0+IFNldmVyaXR5OjpFUlJPUiwKICogICAgIExldmVsOjpDcml0aWNhbC0+dmFsdWUgPT4gU2V2ZXJpdHk6OkZBVEFMLAogKiAgICAgTGV2ZWw6OkFsZXJ0LT52YWx1ZSA9PiBTZXZlcml0eTo6RkFUQUwsCiAqICAgICBMZXZlbDo6RW1lcmdlbmN5LT52YWx1ZSA9PiBTZXZlcml0eTo6RkFUQUwsCiAqIF0pOwogKiBgYGAKICov"},{"repository_path":"src\/bridge\/monolog\/telemetry\/src\/Flow\/Bridge\/Monolog\/Telemetry\/DSL\/functions.php","start_line_in_file":107,"slug":"log-record-converter","name":"log_record_converter","namespace":"Flow\\Bridge\\Monolog\\Telemetry\\DSL","parameters":[{"name":"severityMapper","type":[{"name":"SeverityMapper","namespace":"Flow\\Bridge\\Monolog\\Telemetry","is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"null"},{"name":"valueNormalizer","type":[{"name":"ValueNormalizer","namespace":"Flow\\Bridge\\Monolog\\Telemetry","is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"null"}],"return_type":[{"name":"LogRecordConverter","namespace":"Flow\\Bridge\\Monolog\\Telemetry","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"MONOLOG_TELEMETRY_BRIDGE","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBhIExvZ1JlY29yZENvbnZlcnRlciBmb3IgY29udmVydGluZyBNb25vbG9nIExvZ1JlY29yZCB0byBUZWxlbWV0cnkgTG9nUmVjb3JkLgogKgogKiBUaGUgY29udmVydGVyIGhhbmRsZXM6CiAqIC0gU2V2ZXJpdHkgbWFwcGluZyBmcm9tIE1vbm9sb2cgTGV2ZWwgdG8gVGVsZW1ldHJ5IFNldmVyaXR5CiAqIC0gTWVzc2FnZSBib2R5IGNvbnZlcnNpb24KICogLSBDaGFubmVsIGFuZCBsZXZlbCBuYW1lIGFzIG1vbm9sb2cuKiBhdHRyaWJ1dGVzCiAqIC0gQ29udGV4dCB2YWx1ZXMgYXMgY29udGV4dC4qIGF0dHJpYnV0ZXMgKFRocm93YWJsZXMgdXNlIHNldEV4Y2VwdGlvbigpKQogKiAtIEV4dHJhIHZhbHVlcyBhcyBleHRyYS4qIGF0dHJpYnV0ZXMKICoKICogQHBhcmFtIG51bGx8U2V2ZXJpdHlNYXBwZXIgJHNldmVyaXR5TWFwcGVyIEN1c3RvbSBzZXZlcml0eSBtYXBwZXIgKGRlZmF1bHRzIHRvIHN0YW5kYXJkIG1hcHBpbmcpCiAqIEBwYXJhbSBudWxsfFZhbHVlTm9ybWFsaXplciAkdmFsdWVOb3JtYWxpemVyIEN1c3RvbSB2YWx1ZSBub3JtYWxpemVyIChkZWZhdWx0cyB0byBzdGFuZGFyZCBub3JtYWxpemVyKQogKgogKiBFeGFtcGxlIHVzYWdlOgogKiBgYGBwaHAKICogJGNvbnZlcnRlciA9IGxvZ19yZWNvcmRfY29udmVydGVyKCk7CiAqICR0ZWxlbWV0cnlSZWNvcmQgPSAkY29udmVydGVyLT5jb252ZXJ0KCRtb25vbG9nUmVjb3JkKTsKICogYGBgCiAqCiAqIEV4YW1wbGUgd2l0aCBjdXN0b20gbWFwcGVyOgogKiBgYGBwaHAKICogJGNvbnZlcnRlciA9IGxvZ19yZWNvcmRfY29udmVydGVyKAogKiAgICAgc2V2ZXJpdHlNYXBwZXI6IHNldmVyaXR5X21hcHBlcihbCiAqICAgICAgICAgTGV2ZWw6OkRlYnVnLT52YWx1ZSA9PiBTZXZlcml0eTo6VFJBQ0UsCiAqICAgICBdKQogKiApOwogKiBgYGAKICov"},{"repository_path":"src\/bridge\/monolog\/telemetry\/src\/Flow\/Bridge\/Monolog\/Telemetry\/DSL\/functions.php","start_line_in_file":149,"slug":"telemetry-handler","name":"telemetry_handler","namespace":"Flow\\Bridge\\Monolog\\Telemetry\\DSL","parameters":[{"name":"logger","type":[{"name":"Logger","namespace":"Flow\\Telemetry\\Logger","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"converter","type":[{"name":"LogRecordConverter","namespace":"Flow\\Bridge\\Monolog\\Telemetry","is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"Flow\\Bridge\\Monolog\\Telemetry\\LogRecordConverter::..."},{"name":"level","type":[{"name":"Level","namespace":"Monolog","is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"Monolog\\Level::..."},{"name":"bubble","type":[{"name":"bool","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"true"},{"name":"errorHandler","type":[{"name":"ErrorHandler","namespace":"Flow\\Telemetry\\ErrorHandler","is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"Flow\\Telemetry\\ErrorHandler\\ErrorLogHandler::..."}],"return_type":[{"name":"TelemetryHandler","namespace":"Flow\\Bridge\\Monolog\\Telemetry","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"MONOLOG_TELEMETRY_BRIDGE","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBhIFRlbGVtZXRyeUhhbmRsZXIgZm9yIGZvcndhcmRpbmcgTW9ub2xvZyBsb2dzIHRvIEZsb3cgVGVsZW1ldHJ5LgogKgogKiBAcGFyYW0gTG9nZ2VyICRsb2dnZXIgVGhlIEZsb3cgVGVsZW1ldHJ5IGxvZ2dlciB0byBmb3J3YXJkIGxvZ3MgdG8KICogQHBhcmFtIExvZ1JlY29yZENvbnZlcnRlciAkY29udmVydGVyIENvbnZlcnRlciB0byB0cmFuc2Zvcm0gTW9ub2xvZyBMb2dSZWNvcmQgdG8gVGVsZW1ldHJ5IExvZ1JlY29yZAogKiBAcGFyYW0gTGV2ZWwgJGxldmVsIFRoZSBtaW5pbXVtIGxvZ2dpbmcgbGV2ZWwgYXQgd2hpY2ggdGhpcyBoYW5kbGVyIHdpbGwgYmUgdHJpZ2dlcmVkCiAqIEBwYXJhbSBib29sICRidWJibGUgV2hldGhlciBtZXNzYWdlcyBoYW5kbGVkIGJ5IHRoaXMgaGFuZGxlciBzaG91bGQgYnViYmxlIHVwIHRvIG90aGVyIGhhbmRsZXJzCiAqCiAqIEV4YW1wbGUgdXNhZ2U6CiAqIGBgYHBocAogKiB1c2UgTW9ub2xvZ1xMb2dnZXIgYXMgTW9ub2xvZ0xvZ2dlcjsKICogdXNlIGZ1bmN0aW9uIEZsb3dcQnJpZGdlXE1vbm9sb2dcVGVsZW1ldHJ5XERTTFx0ZWxlbWV0cnlfaGFuZGxlcjsKICogdXNlIGZ1bmN0aW9uIEZsb3dcVGVsZW1ldHJ5XERTTFx0ZWxlbWV0cnk7CiAqCiAqICR0ZWxlbWV0cnkgPSB0ZWxlbWV0cnkoKTsKICogJGxvZ2dlciA9ICR0ZWxlbWV0cnktPmxvZ2dlcignbXktYXBwJyk7CiAqCiAqICRtb25vbG9nID0gbmV3IE1vbm9sb2dMb2dnZXIoJ2NoYW5uZWwnKTsKICogJG1vbm9sb2ctPnB1c2hIYW5kbGVyKHRlbGVtZXRyeV9oYW5kbGVyKCRsb2dnZXIpKTsKICoKICogJG1vbm9sb2ctPmluZm8oJ1VzZXIgbG9nZ2VkIGluJywgWyd1c2VyX2lkJyA9PiAxMjNdKTsKICogLy8g4oaSIEZvcndhcmRlZCB0byBGbG93IFRlbGVtZXRyeSB3aXRoIElORk8gc2V2ZXJpdHkKICogYGBgCiAqCiAqIEV4YW1wbGUgd2l0aCBjdXN0b20gY29udmVydGVyOgogKiBgYGBwaHAKICogJGNvbnZlcnRlciA9IGxvZ19yZWNvcmRfY29udmVydGVyKAogKiAgICAgc2V2ZXJpdHlNYXBwZXI6IHNldmVyaXR5X21hcHBlcihbCiAqICAgICAgICAgTGV2ZWw6OkRlYnVnLT52YWx1ZSA9PiBTZXZlcml0eTo6VFJBQ0UsCiAqICAgICBdKQogKiApOwogKiAkbW9ub2xvZy0+cHVzaEhhbmRsZXIodGVsZW1ldHJ5X2hhbmRsZXIoJGxvZ2dlciwgJGNvbnZlcnRlcikpOwogKiBgYGAKICov"},{"repository_path":"src\/bridge\/symfony\/http-foundation-telemetry\/src\/Flow\/Bridge\/Symfony\/HttpFoundationTelemetry\/DSL\/functions.php","start_line_in_file":16,"slug":"symfony-request-carrier","name":"symfony_request_carrier","namespace":"Flow\\Bridge\\Symfony\\HttpFoundationTelemetry\\DSL","parameters":[{"name":"request","type":[{"name":"Request","namespace":"Symfony\\Component\\HttpFoundation","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"RequestCarrier","namespace":"Flow\\Bridge\\Symfony\\HttpFoundationTelemetry","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"SYMFONY_HTTP_FOUNDATION_TELEMETRY_BRIDGE","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/bridge\/symfony\/http-foundation-telemetry\/src\/Flow\/Bridge\/Symfony\/HttpFoundationTelemetry\/DSL\/functions.php","start_line_in_file":22,"slug":"symfony-response-carrier","name":"symfony_response_carrier","namespace":"Flow\\Bridge\\Symfony\\HttpFoundationTelemetry\\DSL","parameters":[{"name":"response","type":[{"name":"Response","namespace":"Symfony\\Component\\HttpFoundation","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"ResponseCarrier","namespace":"Flow\\Bridge\\Symfony\\HttpFoundationTelemetry","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"SYMFONY_HTTP_FOUNDATION_TELEMETRY_BRIDGE","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/bridge\/psr7\/telemetry\/src\/Flow\/Bridge\/Psr7\/Telemetry\/DSL\/functions.php","start_line_in_file":16,"slug":"psr7-request-carrier","name":"psr7_request_carrier","namespace":"Flow\\Bridge\\Psr7\\Telemetry\\DSL","parameters":[{"name":"request","type":[{"name":"ServerRequestInterface","namespace":"Psr\\Http\\Message","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"RequestCarrier","namespace":"Flow\\Bridge\\Psr7\\Telemetry","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PSR7_TELEMETRY_BRIDGE","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/bridge\/psr7\/telemetry\/src\/Flow\/Bridge\/Psr7\/Telemetry\/DSL\/functions.php","start_line_in_file":22,"slug":"psr7-response-carrier","name":"psr7_response_carrier","namespace":"Flow\\Bridge\\Psr7\\Telemetry\\DSL","parameters":[{"name":"response","type":[{"name":"ResponseInterface","namespace":"Psr\\Http\\Message","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"ResponseCarrier","namespace":"Flow\\Bridge\\Psr7\\Telemetry","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PSR7_TELEMETRY_BRIDGE","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/bridge\/psr18\/telemetry\/src\/Flow\/Bridge\/Psr18\/Telemetry\/DSL\/functions.php","start_line_in_file":15,"slug":"psr18-traceable-client","name":"psr18_traceable_client","namespace":"Flow\\Bridge\\Psr18\\Telemetry\\DSL","parameters":[{"name":"client","type":[{"name":"ClientInterface","namespace":"Psr\\Http\\Client","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"telemetry","type":[{"name":"Telemetry","namespace":"Flow\\Telemetry","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null}],"return_type":[{"name":"PSR18TraceableClient","namespace":"Flow\\Bridge\\Psr18\\Telemetry","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"PSR18_TELEMETRY_BRIDGE","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":null},{"repository_path":"src\/bridge\/telemetry\/otlp\/src\/Flow\/Bridge\/Telemetry\/OTLP\/DSL\/functions.php","start_line_in_file":49,"slug":"otlp-json-serializer","name":"otlp_json_serializer","namespace":"Flow\\Bridge\\Telemetry\\OTLP\\DSL","parameters":[],"return_type":[{"name":"JsonSerializer","namespace":"Flow\\Bridge\\Telemetry\\OTLP\\Serializer","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"TELEMETRY_OTLP","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBhIEpTT04gc2VyaWFsaXplciBmb3IgT1RMUC4KICoKICogUmV0dXJucyBhIEpzb25TZXJpYWxpemVyIHRoYXQgY29udmVydHMgdGVsZW1ldHJ5IGRhdGEgdG8gT1RMUCBKU09OIHdpcmUgZm9ybWF0LgogKiBVc2UgdGhpcyB3aXRoIEN1cmxUcmFuc3BvcnQgZm9yIEpTT04gb3ZlciBIVFRQLgogKgogKiBFeGFtcGxlIHVzYWdlOgogKiBgYGBwaHAKICogJHNlcmlhbGl6ZXIgPSBvdGxwX2pzb25fc2VyaWFsaXplcigpOwogKiAkdHJhbnNwb3J0ID0gb3RscF9jdXJsX3RyYW5zcG9ydCgkZW5kcG9pbnQsICRzZXJpYWxpemVyKTsKICogYGBgCiAqLw=="},{"repository_path":"src\/bridge\/telemetry\/otlp\/src\/Flow\/Bridge\/Telemetry\/OTLP\/DSL\/functions.php","start_line_in_file":70,"slug":"otlp-protobuf-serializer","name":"otlp_protobuf_serializer","namespace":"Flow\\Bridge\\Telemetry\\OTLP\\DSL","parameters":[],"return_type":[{"name":"ProtobufSerializer","namespace":"Flow\\Bridge\\Telemetry\\OTLP\\Serializer","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"TELEMETRY_OTLP","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBhIFByb3RvYnVmIHNlcmlhbGl6ZXIgZm9yIE9UTFAuCiAqCiAqIFJldHVybnMgYSBQcm90b2J1ZlNlcmlhbGl6ZXIgdGhhdCBjb252ZXJ0cyB0ZWxlbWV0cnkgZGF0YSB0byBPVExQIFByb3RvYnVmIGJpbmFyeSBmb3JtYXQuCiAqIFVzZSB0aGlzIHdpdGggQ3VybFRyYW5zcG9ydCBmb3IgUHJvdG9idWYgb3ZlciBIVFRQLCBvciB3aXRoIEdycGNUcmFuc3BvcnQuCiAqCiAqIFJlcXVpcmVzOgogKiAtIGdvb2dsZS9wcm90b2J1ZiBwYWNrYWdlCiAqCiAqIEV4YW1wbGUgdXNhZ2U6CiAqIGBgYHBocAogKiAkc2VyaWFsaXplciA9IG90bHBfcHJvdG9idWZfc2VyaWFsaXplcigpOwogKiAkdHJhbnNwb3J0ID0gb3RscF9jdXJsX3RyYW5zcG9ydCgkZW5kcG9pbnQsICRzZXJpYWxpemVyKTsKICogYGBgCiAqLw=="},{"repository_path":"src\/bridge\/telemetry\/otlp\/src\/Flow\/Bridge\/Telemetry\/OTLP\/DSL\/functions.php","start_line_in_file":94,"slug":"otlp-grpc-transport","name":"otlp_grpc_transport","namespace":"Flow\\Bridge\\Telemetry\\OTLP\\DSL","parameters":[{"name":"endpoint","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"headers","type":[{"name":"array","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"[]"},{"name":"insecure","type":[{"name":"bool","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"true"},{"name":"timeoutMs","type":[{"name":"int","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"250"},{"name":"shutdownTimeoutMs","type":[{"name":"int","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"5000"},{"name":"failover","type":[{"name":"Transport","namespace":"Flow\\Bridge\\Telemetry\\OTLP\\Transport","is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"null"}],"return_type":[{"name":"Transport","namespace":"Flow\\Bridge\\Telemetry\\OTLP\\Transport","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"TELEMETRY_OTLP","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBhIGdSUEMgdHJhbnNwb3J0IGZvciBPVExQIGVuZHBvaW50cy4KICoKICogQ3JlYXRlcyBhIEdycGNUcmFuc3BvcnQgY29uZmlndXJlZCB0byBzZW5kIHRlbGVtZXRyeSBkYXRhIHRvIGFuIE9UTFAtY29tcGF0aWJsZQogKiBlbmRwb2ludCB1c2luZyBnUlBDIHByb3RvY29sIHdpdGggUHJvdG9idWYgc2VyaWFsaXphdGlvbi4gT1RMUC9nUlBDIG1hbmRhdGVzCiAqIFByb3RvYnVmLCBzbyB0aGUgc2VyaWFsaXplciBpcyBidWlsdCBpbnRlcm5hbGx5IGFuZCBub3QgY29uZmlndXJhYmxlLgogKgogKiBSZXF1aXJlczoKICogLSBleHQtZ3JwYyBQSFAgZXh0ZW5zaW9uCiAqIC0gZ29vZ2xlL3Byb3RvYnVmIHBhY2thZ2UKICoKICogQHBhcmFtIHN0cmluZyAkZW5kcG9pbnQgZ1JQQyBlbmRwb2ludCAoZS5nLiwgJ2xvY2FsaG9zdDo0MzE3JykKICogQHBhcmFtIGFycmF5PHN0cmluZywgc3RyaW5nPiAkaGVhZGVycyBBZGRpdGlvbmFsIGhlYWRlcnMgKG1ldGFkYXRhKSB0byBpbmNsdWRlIGluIHJlcXVlc3RzCiAqIEBwYXJhbSBib29sICRpbnNlY3VyZSBXaGV0aGVyIHRvIHVzZSBpbnNlY3VyZSBjaGFubmVsIGNyZWRlbnRpYWxzIChkZWZhdWx0IHRydWUgZm9yIGxvY2FsIGRldikKICogQHBhcmFtIGludCAkdGltZW91dE1zIFBlci1jYWxsIGRlYWRsaW5lIGluIG1pbGxpc2Vjb25kcyAoY292ZXJzIGNvbm5lY3QgKyBzZW5kICsgcmVjZWl2ZSkKICogQHBhcmFtIGludCAkc2h1dGRvd25UaW1lb3V0TXMgV2FsbC1jbG9jayBidWRnZXQgZm9yIGRyYWluaW5nIHBlbmRpbmcgY2FsbHMgYXQgc2h1dGRvd24KICogQHBhcmFtID9UcmFuc3BvcnQgJGZhaWxvdmVyIE9wdGlvbmFsIGZhaWxvdmVyIHRyYW5zcG9ydCByZWNlaXZpbmcgcHJpb3IgYmF0Y2hlcyB3aGVuIHByaW1hcnkgZmFpbHMKICov"},{"repository_path":"src\/bridge\/telemetry\/otlp\/src\/Flow\/Bridge\/Telemetry\/OTLP\/DSL\/functions.php","start_line_in_file":109,"slug":"otlp-curl-options","name":"otlp_curl_options","namespace":"Flow\\Bridge\\Telemetry\\OTLP\\DSL","parameters":[],"return_type":[{"name":"CurlTransportOptions","namespace":"Flow\\Bridge\\Telemetry\\OTLP\\Transport","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"TELEMETRY_OTLP","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBjdXJsIHRyYW5zcG9ydCBvcHRpb25zIGZvciBPVExQLgogKi8="},{"repository_path":"src\/bridge\/telemetry\/otlp\/src\/Flow\/Bridge\/Telemetry\/OTLP\/DSL\/functions.php","start_line_in_file":130,"slug":"otlp-curl-transport","name":"otlp_curl_transport","namespace":"Flow\\Bridge\\Telemetry\\OTLP\\DSL","parameters":[{"name":"endpoint","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"serializer","type":[{"name":"JsonSerializer","namespace":"Flow\\Bridge\\Telemetry\\OTLP\\Serializer","is_nullable":false,"is_variadic":false},{"name":"ProtobufSerializer","namespace":"Flow\\Bridge\\Telemetry\\OTLP\\Serializer","is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"Flow\\Bridge\\Telemetry\\OTLP\\Serializer\\JsonSerializer::..."},{"name":"options","type":[{"name":"CurlTransportOptions","namespace":"Flow\\Bridge\\Telemetry\\OTLP\\Transport","is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"Flow\\Bridge\\Telemetry\\OTLP\\Transport\\CurlTransportOptions::..."},{"name":"failover","type":[{"name":"Transport","namespace":"Flow\\Bridge\\Telemetry\\OTLP\\Transport","is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"null"}],"return_type":[{"name":"Transport","namespace":"Flow\\Bridge\\Telemetry\\OTLP\\Transport","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"TELEMETRY_OTLP","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBhIHN5bmNocm9ub3VzIGN1cmwgdHJhbnNwb3J0IGZvciBPVExQIGVuZHBvaW50cy4KICoKICogQ3JlYXRlcyBhIEN1cmxUcmFuc3BvcnQgdGhhdCBkcml2ZXMgZWFjaCByZXF1ZXN0IHRvIGNvbXBsZXRpb24gYW5kIHJlcG9ydHMgdGhlCiAqIG91dGNvbWUgaW1tZWRpYXRlbHkgKHJldHVybnMgb24gc3VjY2VzcywgdGhyb3dzIG9uIGZhaWx1cmUpLiBPVExQL0hUVFAgYWxsb3dzCiAqIEpTT04gb3IgUHJvdG9idWYgZW5jb2Rpbmc7IGRlZmF1bHRzIHRvIEpTT04uIEtlZXBpbmcgZXhwb3J0IG9mZiB0aGUgYXBwbGljYXRpb24KICogaG90IHBhdGggaXMgdGhlIGpvYiBvZiB0aGUgYmF0Y2hpbmcgcHJvY2Vzc29yIGluIGZyb250IG9mIHRoZSBleHBvcnRlci4KICoKICogUmVxdWlyZXM6IGV4dC1jdXJsIFBIUCBleHRlbnNpb24KICoKICogQHBhcmFtIHN0cmluZyAkZW5kcG9pbnQgT1RMUCBlbmRwb2ludCBVUkwgKGUuZy4sICdodHRwOi8vbG9jYWxob3N0OjQzMTgnKQogKiBAcGFyYW0gSnNvblNlcmlhbGl6ZXJ8UHJvdG9idWZTZXJpYWxpemVyICRzZXJpYWxpemVyIFNlcmlhbGl6ZXIgZm9yIGVuY29kaW5nIHRlbGVtZXRyeSBkYXRhIChKU09OIG9yIFByb3RvYnVmKQogKiBAcGFyYW0gQ3VybFRyYW5zcG9ydE9wdGlvbnMgJG9wdGlvbnMgVHJhbnNwb3J0IGNvbmZpZ3VyYXRpb24gb3B0aW9ucwogKiBAcGFyYW0gP1RyYW5zcG9ydCAkZmFpbG92ZXIgT3B0aW9uYWwgZmFpbG92ZXIgdHJhbnNwb3J0IHJlY2VpdmluZyB0aGUgYmF0Y2ggd2hlbiB0aGUgcHJpbWFyeSBmYWlscwogKi8="},{"repository_path":"src\/bridge\/telemetry\/otlp\/src\/Flow\/Bridge\/Telemetry\/OTLP\/DSL\/functions.php","start_line_in_file":143,"slug":"otlp-async-curl-options","name":"otlp_async_curl_options","namespace":"Flow\\Bridge\\Telemetry\\OTLP\\DSL","parameters":[],"return_type":[{"name":"AsyncCurlTransportOptions","namespace":"Flow\\Bridge\\Telemetry\\OTLP\\Transport","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"TELEMETRY_OTLP","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBhc3luYyBjdXJsIHRyYW5zcG9ydCBvcHRpb25zIGZvciBPVExQLgogKi8="},{"repository_path":"src\/bridge\/telemetry\/otlp\/src\/Flow\/Bridge\/Telemetry\/OTLP\/DSL\/functions.php","start_line_in_file":158,"slug":"otlp-async-curl-transport","name":"otlp_async_curl_transport","namespace":"Flow\\Bridge\\Telemetry\\OTLP\\DSL","parameters":[{"name":"endpoint","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"serializer","type":[{"name":"JsonSerializer","namespace":"Flow\\Bridge\\Telemetry\\OTLP\\Serializer","is_nullable":false,"is_variadic":false},{"name":"ProtobufSerializer","namespace":"Flow\\Bridge\\Telemetry\\OTLP\\Serializer","is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"Flow\\Bridge\\Telemetry\\OTLP\\Serializer\\JsonSerializer::..."},{"name":"options","type":[{"name":"AsyncCurlTransportOptions","namespace":"Flow\\Bridge\\Telemetry\\OTLP\\Transport","is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"Flow\\Bridge\\Telemetry\\OTLP\\Transport\\AsyncCurlTransportOptions::..."},{"name":"failover","type":[{"name":"Transport","namespace":"Flow\\Bridge\\Telemetry\\OTLP\\Transport","is_nullable":true,"is_variadic":false}],"has_default_value":true,"is_nullable":true,"is_variadic":false,"default_value":"null"},{"name":"error_handler","type":[{"name":"ErrorHandler","namespace":"Flow\\Telemetry\\ErrorHandler","is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"Flow\\Telemetry\\ErrorHandler\\ErrorLogHandler::..."}],"return_type":[{"name":"AsyncCurlTransport","namespace":"Flow\\Bridge\\Telemetry\\OTLP\\Transport","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"TELEMETRY_OTLP","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBhbiBhc3luY2hyb25vdXMgY3VybCB0cmFuc3BvcnQgZm9yIE9UTFAgZW5kcG9pbnRzLgogKgogKiBAcGFyYW0gc3RyaW5nICRlbmRwb2ludCBPVExQIGVuZHBvaW50IFVSTCAoZS5nLiwgJ2h0dHA6Ly9sb2NhbGhvc3Q6NDMxOCcpCiAqIEBwYXJhbSBKc29uU2VyaWFsaXplcnxQcm90b2J1ZlNlcmlhbGl6ZXIgJHNlcmlhbGl6ZXIgU2VyaWFsaXplciBmb3IgZW5jb2RpbmcgdGVsZW1ldHJ5IGRhdGEgKEpTT04gb3IgUHJvdG9idWYpCiAqIEBwYXJhbSBBc3luY0N1cmxUcmFuc3BvcnRPcHRpb25zICRvcHRpb25zIFRyYW5zcG9ydCBjb25maWd1cmF0aW9uIG9wdGlvbnMKICogQHBhcmFtID9UcmFuc3BvcnQgJGZhaWxvdmVyIE9wdGlvbmFsIGZhaWxvdmVyIHRyYW5zcG9ydCByZWNlaXZpbmcgcHJpb3IgYmF0Y2hlcyB3aGVuIHByaW1hcnkgZmFpbHMKICogQHBhcmFtIEVycm9ySGFuZGxlciAkZXJyb3JfaGFuZGxlciBIYW5kbGVyIGZvciBmYWlsdXJlcyByZWFwZWQgb24gc2VuZCgpL3RpY2soKS9zaHV0ZG93bigpIChubyBmYWlsb3ZlcikKICov"},{"repository_path":"src\/bridge\/telemetry\/otlp\/src\/Flow\/Bridge\/Telemetry\/OTLP\/DSL\/functions.php","start_line_in_file":180,"slug":"otlp-stream-transport","name":"otlp_stream_transport","namespace":"Flow\\Bridge\\Telemetry\\OTLP\\DSL","parameters":[{"name":"destination","type":[{"name":"string","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"filePermissions","type":[{"name":"int","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"420"},{"name":"createDirectories","type":[{"name":"bool","namespace":null,"is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"true"}],"return_type":[{"name":"Transport","namespace":"Flow\\Bridge\\Telemetry\\OTLP\\Transport","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"TELEMETRY_OTLP","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBhIHN0cmVhbSB0cmFuc3BvcnQgZm9yIE9UTFAgdGhhdCB3cml0ZXMgSlNPTkwgdG8gYSBzaW5nbGUgZGVzdGluYXRpb24uCiAqCiAqIEFjY2VwdHMgYW4gYWJzb2x1dGUgZmlsZSBwYXRoIG9yIGEgcGhwOi8vIHN0cmVhbSB3cmFwcGVyIHN1Y2ggYXMKICogJ3BocDovL3N0ZG91dCcsICdwaHA6Ly9zdGRlcnInLCAncGhwOi8vbWVtb3J5Jywgb3IgJ3BocDovL3RlbXAnLiBFYWNoCiAqIGV4cG9ydCgpIGNhbGwgYXBwZW5kcyBvbmUgSlNPTiBMaW5lIHVuZGVyIExPQ0tfRVggc28gY29uY3VycmVudCB3cml0ZXJzCiAqIGludGVybGVhdmUgYXQgbGluZSBib3VuZGFyaWVzLiBUaGUgJGZpbGVQZXJtaXNzaW9ucyBhbmQgJGNyZWF0ZURpcmVjdG9yaWVzCiAqIHBhcmFtZXRlcnMgYXBwbHkgb25seSB3aGVuIHRoZSBkZXN0aW5hdGlvbiBpcyBhIGZpbGUgcGF0aC4KICoKICogUGVyIHRoZSBPVExQIEZpbGUgRXhwb3J0ZXIgc3BlYyBvbmx5IEpTT04gZW5jb2RpbmcgaXMgc3VwcG9ydGVkLgogKi8="},{"repository_path":"src\/bridge\/telemetry\/otlp\/src\/Flow\/Bridge\/Telemetry\/OTLP\/DSL\/functions.php","start_line_in_file":203,"slug":"otlp-exporter","name":"otlp_exporter","namespace":"Flow\\Bridge\\Telemetry\\OTLP\\DSL","parameters":[{"name":"transport","type":[{"name":"Transport","namespace":"Flow\\Bridge\\Telemetry\\OTLP\\Transport","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"errorHandler","type":[{"name":"ErrorHandler","namespace":"Flow\\Telemetry\\ErrorHandler","is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"Flow\\Telemetry\\ErrorHandler\\ErrorLogHandler::..."}],"return_type":[{"name":"OTLPExporter","namespace":"Flow\\Bridge\\Telemetry\\OTLP\\Exporter","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"TELEMETRY_OTLP","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBhbiBPVExQIGV4cG9ydGVyIHRoYXQgZGlzcGF0Y2hlcyBsb2dzLCBtZXRyaWNzLCBhbmQgc3BhbnMgdGhyb3VnaCBhIHNpbmdsZSB0cmFuc3BvcnQuCiAqCiAqIEV4YW1wbGUgdXNhZ2U6CiAqIGBgYHBocAogKiAkZXhwb3J0ZXIgPSBvdGxwX2V4cG9ydGVyKCR0cmFuc3BvcnQpOwogKiAkc3BhblByb2Nlc3NvciA9IGJhdGNoaW5nX3NwYW5fcHJvY2Vzc29yKCRleHBvcnRlcik7CiAqICRtZXRyaWNQcm9jZXNzb3IgPSBiYXRjaGluZ19tZXRyaWNfcHJvY2Vzc29yKCRleHBvcnRlcik7CiAqICRsb2dQcm9jZXNzb3IgPSBiYXRjaGluZ19sb2dfcHJvY2Vzc29yKCRleHBvcnRlcik7CiAqIGBgYAogKgogKiBAcGFyYW0gVHJhbnNwb3J0ICR0cmFuc3BvcnQgVGhlIHRyYW5zcG9ydCBmb3Igc2VuZGluZyB0ZWxlbWV0cnkgZGF0YQogKiBAcGFyYW0gRXJyb3JIYW5kbGVyICRlcnJvckhhbmRsZXIgSGFuZGxlciBmb3IgVGhyb3dhYmxlcyByYWlzZWQgYnkgdGhlIHRyYW5zcG9ydAogKi8="},{"repository_path":"src\/bridge\/telemetry\/otlp\/src\/Flow\/Bridge\/Telemetry\/OTLP\/DSL\/functions.php","start_line_in_file":217,"slug":"otlp-tracer-provider","name":"otlp_tracer_provider","namespace":"Flow\\Bridge\\Telemetry\\OTLP\\DSL","parameters":[{"name":"processor","type":[{"name":"SpanProcessor","namespace":"Flow\\Telemetry\\Tracer","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"clock","type":[{"name":"ClockInterface","namespace":"Psr\\Clock","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"sampler","type":[{"name":"Sampler","namespace":"Flow\\Telemetry\\Tracer\\Sampler","is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"Flow\\Telemetry\\Tracer\\Sampler\\ParentBasedSampler::..."},{"name":"contextStorage","type":[{"name":"ContextStorage","namespace":"Flow\\Telemetry\\Context","is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"Flow\\Telemetry\\Context\\MemoryContextStorage::..."}],"return_type":[{"name":"TracerProvider","namespace":"Flow\\Telemetry\\Tracer","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"TELEMETRY_OTLP","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBhIHRyYWNlciBwcm92aWRlciBjb25maWd1cmVkIGZvciBPVExQIGV4cG9ydC4KICoKICogQHBhcmFtIFNwYW5Qcm9jZXNzb3IgJHByb2Nlc3NvciBUaGUgcHJvY2Vzc29yIGZvciBoYW5kbGluZyBzcGFucwogKiBAcGFyYW0gQ2xvY2tJbnRlcmZhY2UgJGNsb2NrIFRoZSBjbG9jayBmb3IgdGltZXN0YW1wcwogKiBAcGFyYW0gU2FtcGxlciAkc2FtcGxlciBUaGUgc2FtcGxlciBmb3IgZGVjaWRpbmcgd2hldGhlciB0byByZWNvcmQgc3BhbnMKICogQHBhcmFtIENvbnRleHRTdG9yYWdlICRjb250ZXh0U3RvcmFnZSBUaGUgY29udGV4dCBzdG9yYWdlIGZvciBwcm9wYWdhdGluZyB0cmFjZSBjb250ZXh0CiAqLw=="},{"repository_path":"src\/bridge\/telemetry\/otlp\/src\/Flow\/Bridge\/Telemetry\/OTLP\/DSL\/functions.php","start_line_in_file":234,"slug":"otlp-meter-provider","name":"otlp_meter_provider","namespace":"Flow\\Bridge\\Telemetry\\OTLP\\DSL","parameters":[{"name":"processor","type":[{"name":"MetricProcessor","namespace":"Flow\\Telemetry\\Meter","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"clock","type":[{"name":"ClockInterface","namespace":"Psr\\Clock","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"temporality","type":[{"name":"AggregationTemporality","namespace":"Flow\\Telemetry\\Meter","is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"Flow\\Telemetry\\Meter\\AggregationTemporality::..."}],"return_type":[{"name":"MeterProvider","namespace":"Flow\\Telemetry\\Meter","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"TELEMETRY_OTLP","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBhIG1ldGVyIHByb3ZpZGVyIGNvbmZpZ3VyZWQgZm9yIE9UTFAgZXhwb3J0LgogKgogKiBAcGFyYW0gTWV0cmljUHJvY2Vzc29yICRwcm9jZXNzb3IgVGhlIHByb2Nlc3NvciBmb3IgaGFuZGxpbmcgbWV0cmljcwogKiBAcGFyYW0gQ2xvY2tJbnRlcmZhY2UgJGNsb2NrIFRoZSBjbG9jayBmb3IgdGltZXN0YW1wcwogKiBAcGFyYW0gQWdncmVnYXRpb25UZW1wb3JhbGl0eSAkdGVtcG9yYWxpdHkgVGhlIGFnZ3JlZ2F0aW9uIHRlbXBvcmFsaXR5IGZvciBtZXRyaWNzCiAqLw=="},{"repository_path":"src\/bridge\/telemetry\/otlp\/src\/Flow\/Bridge\/Telemetry\/OTLP\/DSL\/functions.php","start_line_in_file":250,"slug":"otlp-logger-provider","name":"otlp_logger_provider","namespace":"Flow\\Bridge\\Telemetry\\OTLP\\DSL","parameters":[{"name":"processor","type":[{"name":"LogProcessor","namespace":"Flow\\Telemetry\\Logger","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"clock","type":[{"name":"ClockInterface","namespace":"Psr\\Clock","is_nullable":false,"is_variadic":false}],"has_default_value":false,"is_nullable":false,"is_variadic":false,"default_value":null},{"name":"contextStorage","type":[{"name":"ContextStorage","namespace":"Flow\\Telemetry\\Context","is_nullable":false,"is_variadic":false}],"has_default_value":true,"is_nullable":false,"is_variadic":false,"default_value":"Flow\\Telemetry\\Context\\MemoryContextStorage::..."}],"return_type":[{"name":"LoggerProvider","namespace":"Flow\\Telemetry\\Logger","is_nullable":false,"is_variadic":false}],"attributes":[{"name":"DocumentationDSL","namespace":"Flow\\Documentation\\Attribute","arguments":{"module":"TELEMETRY_OTLP","type":"HELPER"}}],"scalar_function_chain":false,"doc_comment":"LyoqCiAqIENyZWF0ZSBhIGxvZ2dlciBwcm92aWRlciBjb25maWd1cmVkIGZvciBPVExQIGV4cG9ydC4KICoKICogQHBhcmFtIExvZ1Byb2Nlc3NvciAkcHJvY2Vzc29yIFRoZSBwcm9jZXNzb3IgZm9yIGhhbmRsaW5nIGxvZyByZWNvcmRzCiAqIEBwYXJhbSBDbG9ja0ludGVyZmFjZSAkY2xvY2sgVGhlIGNsb2NrIGZvciB0aW1lc3RhbXBzCiAqIEBwYXJhbSBDb250ZXh0U3RvcmFnZSAkY29udGV4dFN0b3JhZ2UgVGhlIGNvbnRleHQgc3RvcmFnZSBmb3IgcHJvcGFnYXRpbmcgY29udGV4dAogKi8="}] \ No newline at end of file diff --git a/web/landing/src/Flow/Website/Service/FlowConfigFactory.php b/web/landing/src/Flow/Website/Service/FlowConfigFactory.php index 6fc3545e7a..142a2a6222 100644 --- a/web/landing/src/Flow/Website/Service/FlowConfigFactory.php +++ b/web/landing/src/Flow/Website/Service/FlowConfigFactory.php @@ -5,8 +5,10 @@ namespace Flow\Website\Service; use Flow\ETL\Cache\Implementation\PSRSimpleCache; +use Flow\ETL\Column\AdaptiveBackend; use Flow\ETL\Config\ConfigBuilder; use Flow\Filesystem\Filesystem; +use Flow\Floe\FloeSerializer; use Flow\Telemetry\Provider\Clock\SystemClock; use Flow\Telemetry\Telemetry; use Symfony\Component\Cache\Adapter\FilesystemAdapter; @@ -68,13 +70,16 @@ public function telemetry(): Telemetry private function cache(string $directoryName, int $ttl): PSRSimpleCache { - return new PSRSimpleCache(new Psr16Cache( - new FilesystemAdapter( - 'flow-contributors', - $ttl, - directory: type_string()->assert($this->parameters->get('kernel.cache_dir')) . '/' - . ltrim($directoryName, '/'), + return new PSRSimpleCache( + new Psr16Cache( + new FilesystemAdapter( + 'flow-contributors', + $ttl, + directory: type_string()->assert($this->parameters->get('kernel.cache_dir')) . '/' + . ltrim($directoryName, '/'), + ), ), - )); + new FloeSerializer(new AdaptiveBackend()), + ); } } diff --git a/web/landing/templates/documentation/navigation_left.html.twig b/web/landing/templates/documentation/navigation_left.html.twig index 2287846322..6d22d87ea4 100644 --- a/web/landing/templates/documentation/navigation_left.html.twig +++ b/web/landing/templates/documentation/navigation_left.html.twig @@ -44,6 +44,7 @@
  • {{ nav.nav_link('/documentation/components/core/building-blocks', 'Building Blocks', currentPath) }}
  • +
  • {{ nav.nav_link('/documentation/components/core/column-backend', 'Column Backend', currentPath) }}
  • {{ nav.nav_link('/documentation/components/core/data-retrieval', 'Data Retrieval', currentPath) }}
  • {{ nav.nav_link('/documentation/components/core/data-manipulation', 'Data Manipulation', currentPath) }}
  • {{ nav.nav_link('/documentation/components/core/select-drop', 'Select / Drop', currentPath) }}
  • diff --git a/web/landing/tests/Flow/Website/Tests/Functional/PlaygroundBranchSinkTest.php b/web/landing/tests/Flow/Website/Tests/Functional/PlaygroundBranchSinkTest.php new file mode 100644 index 0000000000..5f0849ea88 --- /dev/null +++ b/web/landing/tests/Flow/Website/Tests/Functional/PlaygroundBranchSinkTest.php @@ -0,0 +1,46 @@ +openPlayground('/playground'); + $page = $this->pageOf($browser); + + $this->setPlaygroundCode($page, <<<'PHP' + read(from_sequence_number('id', 1, 9)) + ->batchSize(3) + ->write(to_branch(ref('id')->isEven(), to_array($even))) + ->write(to_branch(ref('id')->isOdd(), to_array($odd))) + ->run(); + echo 'even=' . implode(',', array_column($even, 'id')) . ';odd=' . implode(',', array_column($odd, 'id')) . ';'; + PHP); + + $page->evaluate('() => document.getElementById("action-run").click()'); + + $browser->waitUntilVisible('[data-playground-output-target="container"] .output-message'); + + static::assertSame( + '', + type_string()->assert($page->evaluate( + '() => Array.from(document.querySelectorAll(\'[data-playground-output-target="container"] .output-error\')).map(e => e.textContent).join("\n")', + )), + ); + $browser->assertSeeIn('[data-playground-output-target="container"]', 'even=2,4,6,8;odd=1,3,5,7,9;'); + } +}